From dba97b845d3c789ca10f5a9b5492377016f66be8 Mon Sep 17 00:00:00 2001 From: akablockchain1 Date: Mon, 17 Mar 2025 15:26:47 +0300 Subject: [PATCH 1/3] Template for scheduled parse and prove feed service is implemeted --- .../diffuse-parse-proove-feed.service | 10 ++++++++++ .../diffuse-parse-proove-feed.timer | 11 +++++++++++ .../parse_and_prove_feed.sh | 5 +++++ 3 files changed, 26 insertions(+) create mode 100644 services/scheduled_parse_prove_feed/diffuse-parse-proove-feed.service create mode 100644 services/scheduled_parse_prove_feed/diffuse-parse-proove-feed.timer create mode 100644 services/scheduled_parse_prove_feed/parse_and_prove_feed.sh diff --git a/services/scheduled_parse_prove_feed/diffuse-parse-proove-feed.service b/services/scheduled_parse_prove_feed/diffuse-parse-proove-feed.service new file mode 100644 index 0000000..d4e1d10 --- /dev/null +++ b/services/scheduled_parse_prove_feed/diffuse-parse-proove-feed.service @@ -0,0 +1,10 @@ +[Unit] +Description=Diffuse Datafeed periodic job + +[Service] +Type=oneshot +User=ubuntu ; <-- пользователь, от которого запускается скрипт +Group=ubuntu ; <-- группа, если нужно +WorkingDirectory=/home/ubuntu/our_org/cron/diffuse-datafeed ; <-- путь до репозитория +EnvironmentFile=/home/ubuntu/our_org/cron/diffuse-datafeed/.env ; <-- подключаем .env +ExecStart=/bin/bash /home/ubuntu/our_org/cron/diffuse-datafeed/cron_feeder.sh \ No newline at end of file diff --git a/services/scheduled_parse_prove_feed/diffuse-parse-proove-feed.timer b/services/scheduled_parse_prove_feed/diffuse-parse-proove-feed.timer new file mode 100644 index 0000000..0da9a90 --- /dev/null +++ b/services/scheduled_parse_prove_feed/diffuse-parse-proove-feed.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Runs Diffuse Datafeed job every 5 seconds, sequentially + +[Timer] +OnBootSec=5 +OnUnitActiveSec=5 +AccuracySec=1s +Persistent=false + +[Install] +WantedBy=timers.target diff --git a/services/scheduled_parse_prove_feed/parse_and_prove_feed.sh b/services/scheduled_parse_prove_feed/parse_and_prove_feed.sh new file mode 100644 index 0000000..1ae770d --- /dev/null +++ b/services/scheduled_parse_prove_feed/parse_and_prove_feed.sh @@ -0,0 +1,5 @@ +echo "start feeding: $(date)" >> /var/logs/diffuse/parse_and_prove_feed.log + +echo "execution" >> parse_and_prove_feed.log 2>&1 + +echo "binance data request and proving finished: $(date)" >> /var/logs/diffuse/parse_and_prove_feed.log \ No newline at end of file From 04fffdf81efc111a8ca277ac91ec1ff50dd76db5 Mon Sep 17 00:00:00 2001 From: akablockchain1 Date: Mon, 17 Mar 2025 15:31:58 +0300 Subject: [PATCH 2/3] Template for scheduled parse and prove feed service is implemeted --- .../diffuse-parse-proove-feed.service | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/services/scheduled_parse_prove_feed/diffuse-parse-proove-feed.service b/services/scheduled_parse_prove_feed/diffuse-parse-proove-feed.service index d4e1d10..f6ac951 100644 --- a/services/scheduled_parse_prove_feed/diffuse-parse-proove-feed.service +++ b/services/scheduled_parse_prove_feed/diffuse-parse-proove-feed.service @@ -3,8 +3,8 @@ Description=Diffuse Datafeed periodic job [Service] Type=oneshot -User=ubuntu ; <-- пользователь, от которого запускается скрипт -Group=ubuntu ; <-- группа, если нужно -WorkingDirectory=/home/ubuntu/our_org/cron/diffuse-datafeed ; <-- путь до репозитория -EnvironmentFile=/home/ubuntu/our_org/cron/diffuse-datafeed/.env ; <-- подключаем .env -ExecStart=/bin/bash /home/ubuntu/our_org/cron/diffuse-datafeed/cron_feeder.sh \ No newline at end of file +User=ubuntu +Group=ubuntu +WorkingDirectory=PUT_YOUR_ABSOLUT_PATH_TO_DATAFEED_CLI_HERE ; +EnvironmentFile=PUT_YOUR_ABSOLUT_PATH_TO_DATAFEED_CLI_HERE/.env ; +ExecStart=PUT_YOUR_ABSOLUT_PATH_TO_DATAFEED_CLI_HERE/services/scheduled_parse_prove_feed/parse_and_prove_feed.sh \ No newline at end of file From cb5c9ed0eb40a6bb9009c12243b75f64a48a6b54 Mon Sep 17 00:00:00 2001 From: akablockchain1 Date: Tue, 18 Mar 2025 13:00:23 +0300 Subject: [PATCH 3/3] scheduled parse and proove service configuration --- .../scheduled_parse_prove_feed/.env-example | 2 + services/scheduled_parse_prove_feed/README.md | 55 +++++++++++++++++++ .../diffuse-parse-proove-feed.service | 5 +- .../parse_and_prove_feed.sh | 10 +++- services/scheduled_parse_prove_feed/run.sh | 17 ++++++ services/scheduled_parse_prove_feed/stop.sh | 10 ++++ 6 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 services/scheduled_parse_prove_feed/.env-example create mode 100644 services/scheduled_parse_prove_feed/README.md create mode 100644 services/scheduled_parse_prove_feed/run.sh create mode 100644 services/scheduled_parse_prove_feed/stop.sh diff --git a/services/scheduled_parse_prove_feed/.env-example b/services/scheduled_parse_prove_feed/.env-example new file mode 100644 index 0000000..021d39f --- /dev/null +++ b/services/scheduled_parse_prove_feed/.env-example @@ -0,0 +1,2 @@ +ETH_WALLET_PRIVATE_KEY=PUT_YOUR_ETH_WALLET_PRIVATE_KEY_HERE +ALCHEMY_API_KEY=PUT_YOUR_ETH_WALLET_PRIVATE_KEY_HERE diff --git a/services/scheduled_parse_prove_feed/README.md b/services/scheduled_parse_prove_feed/README.md new file mode 100644 index 0000000..2821722 --- /dev/null +++ b/services/scheduled_parse_prove_feed/README.md @@ -0,0 +1,55 @@ +# Diffuse Datafeed (systemd service + timer) + +This project allows you to schedule the `parse_and_prove_feed.sh` script using `systemd`. The script, in turn, runs the Python program `parse_prove_feed.py` with parameters from the `.env` file and writes logs to `/var/log/diffuse/parse_and_prove_feed.log`. + +## Installation + +1. Place the project files in the desired directory, for example: `~/datafeed-cli/`. +2. Create a `.env` file based on `.env-example` and fill in the required variables. +3. In the files `diffuse-parse-proove-feed.service` and `diffuse-parse-proove-feed.timer`, specify: + - The absolute paths to the directory containing the scripts and `.env`. + - If needed, adjust the timer frequency (by default, every 5 seconds). +4. In `parse_and_prove_feed.sh`, specify any parameters needed for `parse_prove_feed.py`. +5. Run the setup script: + +```bash +chmod +x run.sh +./run.sh +``` + +This script copies the relevant files to the system directories, reloads `systemd`, and enables the timer. + +## Checking Status + +- To check the timer status: + +```bash +sudo systemctl status diffuse-parse-proove-feed.timer +``` + +- To view logs: + +```bash +tail -f /var/log/diffuse/parse_and_prove_feed.log +``` + +## Stopping + +To stop the service and timer: + +```bash +./stop.sh +``` + +This script will stop and disable both the service and the timer (check with `systemctl status diffuse-parse-proove-feed.timer`). + +## Changing the Schedule + +To change the run frequency, edit `OnUnitActiveSec` and other settings in `diffuse-parse-proove-feed.timer`, then run: + +```bash +sudo systemctl daemon-reload +sudo systemctl restart diffuse-parse-proove-feed.timer +``` + +All logs will continue to be written to `/var/log/diffuse/parse_and_prove_feed.log`. diff --git a/services/scheduled_parse_prove_feed/diffuse-parse-proove-feed.service b/services/scheduled_parse_prove_feed/diffuse-parse-proove-feed.service index f6ac951..9684e4c 100644 --- a/services/scheduled_parse_prove_feed/diffuse-parse-proove-feed.service +++ b/services/scheduled_parse_prove_feed/diffuse-parse-proove-feed.service @@ -3,8 +3,7 @@ Description=Diffuse Datafeed periodic job [Service] Type=oneshot -User=ubuntu -Group=ubuntu +User=PUT_YOUR_USER_HERE WorkingDirectory=PUT_YOUR_ABSOLUT_PATH_TO_DATAFEED_CLI_HERE ; -EnvironmentFile=PUT_YOUR_ABSOLUT_PATH_TO_DATAFEED_CLI_HERE/.env ; +EnvironmentFile=PUT_YOUR_ABSOLUT_PATH_TO_DATAFEED_CLI_HERE/services/scheduled_parse_prove_feed/.env ; ExecStart=PUT_YOUR_ABSOLUT_PATH_TO_DATAFEED_CLI_HERE/services/scheduled_parse_prove_feed/parse_and_prove_feed.sh \ No newline at end of file diff --git a/services/scheduled_parse_prove_feed/parse_and_prove_feed.sh b/services/scheduled_parse_prove_feed/parse_and_prove_feed.sh index 1ae770d..374f9ed 100644 --- a/services/scheduled_parse_prove_feed/parse_and_prove_feed.sh +++ b/services/scheduled_parse_prove_feed/parse_and_prove_feed.sh @@ -1,5 +1,9 @@ -echo "start feeding: $(date)" >> /var/logs/diffuse/parse_and_prove_feed.log +#!/usr/bin/env bash -echo "execution" >> parse_and_prove_feed.log 2>&1 +mkdir -p /var/log/diffuse -echo "binance data request and proving finished: $(date)" >> /var/logs/diffuse/parse_and_prove_feed.log \ No newline at end of file +echo "$(date) start feeding" >> /var/log/diffuse/parse_and_prove_feed.log + +python3 cli/parse_prove_feed.py PUT_YOUR_CLI_PARAMETERS_HERE >> /var/log/diffuse/parse_and_prove_feed.log 2>&1 + +echo "$(date) data request and proving finished: " >> /var/log/diffuse/parse_and_prove_feed.log \ No newline at end of file diff --git a/services/scheduled_parse_prove_feed/run.sh b/services/scheduled_parse_prove_feed/run.sh new file mode 100644 index 0000000..f7f2f70 --- /dev/null +++ b/services/scheduled_parse_prove_feed/run.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +sudo mkdir -p /var/log/diffuse + +sudo cp diffuse-parse-proove-feed.service /etc/systemd/system/ +sudo cp diffuse-parse-proove-feed.timer /etc/systemd/system/ + +sudo chmod +x PUT_YOUR_ABSOLUT_PATH_TO_DATAFEED_CLI_HERE/services/scheduled_parse_prove_feed/parse_and_prove_feed.sh + +sudo systemctl daemon-reload + +sudo systemctl enable diffuse-parse-proove-feed.timer +sudo systemctl start diffuse-parse-proove-feed.timer + +echo "System-wide service and timer have been installed and started (running under root)." +echo "Check status via: sudo systemctl status diffuse-parse-proove-feed.timer" +echo "Logs will be written to: /var/log/diffuse/parse_and_prove_feed.log" diff --git a/services/scheduled_parse_prove_feed/stop.sh b/services/scheduled_parse_prove_feed/stop.sh new file mode 100644 index 0000000..6486591 --- /dev/null +++ b/services/scheduled_parse_prove_feed/stop.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +systemctl stop diffuse-parse-proove-feed.timer +systemctl disable diffuse-parse-proove-feed.timer + +systemctl stop diffuse-parse-proove-feed.service +systemctl disable diffuse-parse-proove-feed.service + +echo "Timer and service 'diffuse-parse-proove-feed' have been stopped and disabled." +echo "You can verify via: systemctl status diffuse-parse-proove-feed.timer"