From ea92326c920ff292b2a18712a7d248d1f4a3b464 Mon Sep 17 00:00:00 2001 From: Matthias Huber Date: Wed, 4 Dec 2024 15:08:33 +0100 Subject: [PATCH 01/13] update dependiencies, use now pushover package #57 --- Makefile | 2 +- requirements.txt | 11 ++++++----- src/notification/pushovernotifier.py | 9 ++++++--- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 3fa3229..24fb60c 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ venv: ## bootstrap python3 venv install: venv ## install kubot dependencies @source venv/bin/activate; \ - pip install --upgrade pip setuptools==57.5.0; \ + pip install --upgrade pip setuptools==75.6.0; \ LIBRARY_PATH=$LIBRARY_PATH:/opt/homebrew/opt/openssl/lib pip install -r requirements.txt; test: venv ## run pytest suite diff --git a/requirements.txt b/requirements.txt index f67c291..d64b289 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,22 +1,23 @@ attrs==20.3.0 certifi==2020.6.20 chardet==3.0.4 +charset-normalizer==3.4.0 contextlib2==0.6.0.post1 idna==2.10 iniconfig==1.1.1 -kucoin-python==1.0.6 +kucoin-python==1.0.25 markdown2==2.3.10 packaging==20.7 peewee==3.14.0 pluggy==0.13.1 -psycopg2-binary==2.8.6 +psycopg2-binary==2.9.10 +pushover @ git+https://github.com/Wyattjoh/pushover@5852545c5b9cf2717e1eafc4c8b134a08b0994da py==1.9.0 pyarmor==6.6.0 pyparsing==2.4.7 pytest==6.1.2 -python-pushover==0.4 -requests==2.24.0 +requests==2.32.3 schema==0.7.2 toml==0.10.2 -urllib3==1.25.11 +urllib3==2.2.3 websockets==8.1 diff --git a/src/notification/pushovernotifier.py b/src/notification/pushovernotifier.py index c155d7d..7ce0d2b 100644 --- a/src/notification/pushovernotifier.py +++ b/src/notification/pushovernotifier.py @@ -1,7 +1,7 @@ import re from notification.notify import Notifier -from pushover import Client +from pushover import Pushover from logger import Logger REGEX_PUSHOVER_KEYS = r'^\w{30,30}$' @@ -11,7 +11,8 @@ class PushoverNotifier(Notifier): def __init__(self, config): self.user_key = config.user_key self.api_token = config.api_token - self.client = Client(self.user_key, api_token=self.api_token) + self.client = Pushover(self.api_token) + self.client.user(self.user_key) @staticmethod def is_valid_config(config): @@ -27,7 +28,9 @@ def api(self): def send_message(self, message, title=None): try: - self.api.send_message(message, title=title) + msg = self.api.msg(message) + msg.set("title", title) + self.api.send(msg) except Exception as e: Logger().logger.error("Pushover send message error: %s", e) From eda032b8be278957c1053e72da44345fbc834fe8 Mon Sep 17 00:00:00 2001 From: Desy Date: Wed, 8 Jan 2025 19:24:23 +0100 Subject: [PATCH 02/13] implement ledger check #57 --- src/kubot.py | 34 ++++++++++++++++++++++------------ src/schemas/config.py | 2 +- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/kubot.py b/src/kubot.py index a618f6a..8df04d0 100644 --- a/src/kubot.py +++ b/src/kubot.py @@ -45,16 +45,17 @@ def schedule_checks(self, interval): self.cleanup_database() for currency in self.__currencies: try: - min_int_rate = self.get_min_daily_interest_rate(currency) - min_int_rate_charge = float(format(min_int_rate + config.charge, '.5f')) - if min_int_rate_charge <= config.minimum_rate: - self.__minimum_rate = config.minimum_rate - elif self.__minimum_rate == const.DEFAULT_MIN_RATE or abs(min_int_rate_charge - self.__minimum_rate) >= config.correction: - self.__minimum_rate = min_int_rate_charge - self.get_lending_assets(currency) - self.check_active_loans(min_int_rate, currency) - self.lend_loans(min_int_rate, currency) - self.check_active_lendings(currency) + # min_int_rate = self.get_min_daily_interest_rate(currency) + # min_int_rate_charge = float(format(min_int_rate + config.charge, '.5f')) + # if min_int_rate_charge <= config.minimum_rate: + # self.__minimum_rate = config.minimum_rate + # elif self.__minimum_rate == const.DEFAULT_MIN_RATE or abs(min_int_rate_charge - self.__minimum_rate) >= config.correction: + # self.__minimum_rate = min_int_rate_charge + # self.get_lending_assets(currency) + # self.check_active_loans(min_int_rate, currency) + # self.lend_loans(min_int_rate, currency) + # self.check_active_lendings(currency) + self.check_ledger(currency) except (socket.timeout, requests.exceptions.Timeout) as e: Logger().logger.error("Currency: %s, Transport Exception occurred: %s", currency.name, e) except Exception as e: @@ -117,6 +118,15 @@ def get_min_daily_interest_rate(self, currency): else: return config.default_interest + def check_ledger(self, currency): + current_page = 1 + page_size = 50 + active_list = self.__user.get_account_ledger(pageSize=page_size, currency=currency.name, currentPage=current_page) + for page in range(current_page + 1, active_list['totalPage'] + 1): + result = self.__user.get_account_ledger(pageSize=page_size, currency=currency.name, currentPage=page) + active_list['items'].extend(result['items']) + print(active_list) + def check_active_lendings(self, currency): current_page = 1 page_size = 50 @@ -163,8 +173,8 @@ def main(): convert_float_to_percentage(config.charge))) # initialize database - with db: - db.create_tables([FundingMarket, ActiveLendOrder, LendingAssets]) + # with db: + # db.create_tables([FundingMarket, ActiveLendOrder, LendingAssets]) # initialize notifier systems notifiers = [] diff --git a/src/schemas/config.py b/src/schemas/config.py index 09b6682..02c2e4b 100644 --- a/src/schemas/config.py +++ b/src/schemas/config.py @@ -2,7 +2,7 @@ currencies = Schema([ { - 'currency': Or("USDT"), + 'currency': Or("USDT", "USDC"), 'term': Or(7, 14, 28), 'reserved_amount': And(int, lambda a: a >= 0) } From c8556861bc57e398e52c36323fecddfd5bbaa0a4 Mon Sep 17 00:00:00 2001 From: Desy Date: Fri, 6 Jun 2025 16:40:09 +0200 Subject: [PATCH 03/13] add first draft of dual investment dashboard, add dual invesetment handling #57 --- README.md | 4 + .../kubot-dual-investment-dashboard.json | 196 ++++++++++++++++++ requirements.txt | 2 +- src/config/config.py | 16 +- src/database/models/base.py | 4 +- src/database/models/ledger.py | 7 + src/database/models/symbols.py | 7 + src/kubot.py | 77 ++++--- src/schemas/config.py | 9 + src/schemas/static.py | 5 + 10 files changed, 293 insertions(+), 34 deletions(-) create mode 100644 provisioning/dashboards/Kubot/kubot-dual-investment-dashboard.json create mode 100644 src/database/models/ledger.py create mode 100644 src/database/models/symbols.py create mode 100644 src/schemas/static.py diff --git a/README.md b/README.md index 004c8e5..742fda7 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,10 @@ Lendbot for Kucoin with Grafana Dashboard, Pushover and Slack Support - kubot run interval in seconds - **currencies** - currency specific settings +- **mode**: + - bot running mode either dual investment (DUAL) or lending (LENDING) +- **symbols**: + - dual investment symbols to track - *optional*: minimum_rate - unset: minimum lending limit would be disabled - *optional*: pushover user_key, api_token diff --git a/provisioning/dashboards/Kubot/kubot-dual-investment-dashboard.json b/provisioning/dashboards/Kubot/kubot-dual-investment-dashboard.json new file mode 100644 index 0000000..38d44cb --- /dev/null +++ b/provisioning/dashboards/Kubot/kubot-dual-investment-dashboard.json @@ -0,0 +1,196 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "postgres", + "uid": "P0689FA73A582FCBA" + }, + "description": "ETH-USDT Last Traded Price", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "lasttradedprice" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 14, + "w": 10, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.2", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "P0689FA73A582FCBA" + }, + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n time AS \"time\",\n ((symbol->>'lastTradedPrice')::float) as lastTradedPrice\nFROM symbolassets\nWHERE (symbol->>'symbol') = 'ETH-USDT'\nORDER BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ], + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "table": "symbolassets", + "timeColumn": "time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Last Traded Price", + "type": "timeseries" + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Dual Investment", + "uid": "heg0onYNk", + "version": 4, + "weekStart": "" +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index d64b289..f8ed1f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ charset-normalizer==3.4.0 contextlib2==0.6.0.post1 idna==2.10 iniconfig==1.1.1 -kucoin-python==1.0.25 +kucoin-python @ git+https://github.com/desytech/kucoin-python-sdk.git@master markdown2==2.3.10 packaging==20.7 peewee==3.14.0 diff --git a/src/config/config.py b/src/config/config.py index 0e61ab6..71eda6b 100644 --- a/src/config/config.py +++ b/src/config/config.py @@ -4,8 +4,8 @@ from configparser import ConfigParser, ExtendedInterpolation import const -from schemas.config import currencies as currencies_schema - +from schemas.config import currencies as currencies_schema, symbols as symbols_schema, modes as modes_schema +from schemas.static import Modes def property_wrapper(default=None): def inner_decorator(f): @@ -96,5 +96,17 @@ def slack_api_token(self): def slack_channel(self): return self.__config['slack'].get('channel') + @property + @property_wrapper(default=Modes.LENDING) + def mode(self): + modes = Modes(self.__config['bot'].get('mode')) + return modes_schema.validate(modes) + + @property + @property_wrapper(default=[]) + def symbols(self): + symbols = json.loads(self.__config['bot'].get('symbols')) + return symbols_schema.validate(symbols) + config = Config() diff --git a/src/database/models/base.py b/src/database/models/base.py index 996be9f..a909f49 100644 --- a/src/database/models/base.py +++ b/src/database/models/base.py @@ -2,8 +2,8 @@ from playhouse.pool import PooledPostgresqlExtDatabase db = PooledPostgresqlExtDatabase('kubot', user='kubot', - password='kubot', host='postgres', - port=5432, max_connections=8, stale_timeout=300) + password='kubot', host='localhost', + port=5433, max_connections=8, stale_timeout=300) class BaseModel(Model): class Meta: diff --git a/src/database/models/ledger.py b/src/database/models/ledger.py new file mode 100644 index 0000000..ef3a15f --- /dev/null +++ b/src/database/models/ledger.py @@ -0,0 +1,7 @@ +from playhouse.postgres_ext import * +from datetime import datetime +from database.models.base import BaseModel + +class LedgerAssets(BaseModel): + time = DateTimeField(default=datetime.utcnow) + ledger = JSONField() \ No newline at end of file diff --git a/src/database/models/symbols.py b/src/database/models/symbols.py new file mode 100644 index 0000000..67571b2 --- /dev/null +++ b/src/database/models/symbols.py @@ -0,0 +1,7 @@ +from playhouse.postgres_ext import * +from datetime import datetime +from database.models.base import BaseModel + +class SymbolAssets(BaseModel): + time = DateTimeField(default=datetime.utcnow) + symbol = JSONField() \ No newline at end of file diff --git a/src/kubot.py b/src/kubot.py index 8df04d0..1b02f68 100644 --- a/src/kubot.py +++ b/src/kubot.py @@ -7,8 +7,11 @@ from database.models.market import FundingMarket from database.models.activeorder import ActiveLendOrder from database.models.assets import LendingAssets +from database.models.ledger import LedgerAssets +from database.models.symbols import SymbolAssets from kucoin.client import Margin, User from config.config import config +from schemas.static import Modes from logger import Logger from datetime import datetime, timedelta from notification.pushovernotifier import PushoverNotifier @@ -20,9 +23,10 @@ class Scheduler(object): - def __init__(self, notifiers, currencies): + def __init__(self, notifiers, currencies, mode): self.__client = Margin(config.api_key, config.api_secret, config.api_passphrase) self.__user = User(config.api_key, config.api_secret, config.api_passphrase) + self.__user_public = User(config.api_key, config.api_secret, config.api_passphrase, "https://www.kucoin.com") self.__notifiers = notifiers self.__currencies = currencies self.__scheduler = sched.scheduler(time.time, time.sleep) @@ -39,27 +43,33 @@ def cleanup_database(self): FundingMarket.delete().where(FundingMarket.time < time_delta).execute() ActiveLendOrder.delete().where(ActiveLendOrder.time < time_delta).execute() LendingAssets.delete().where(LendingAssets.time < time_delta).execute() + LedgerAssets.delete().where(LedgerAssets.time < time_delta).execute() + SymbolAssets.delete().where(SymbolAssets.time < time_delta).execute() def schedule_checks(self, interval): self.__scheduler.enter(interval, 1, self.schedule_checks, argument=(interval,)) self.cleanup_database() - for currency in self.__currencies: - try: - # min_int_rate = self.get_min_daily_interest_rate(currency) - # min_int_rate_charge = float(format(min_int_rate + config.charge, '.5f')) - # if min_int_rate_charge <= config.minimum_rate: - # self.__minimum_rate = config.minimum_rate - # elif self.__minimum_rate == const.DEFAULT_MIN_RATE or abs(min_int_rate_charge - self.__minimum_rate) >= config.correction: - # self.__minimum_rate = min_int_rate_charge - # self.get_lending_assets(currency) - # self.check_active_loans(min_int_rate, currency) - # self.lend_loans(min_int_rate, currency) - # self.check_active_lendings(currency) - self.check_ledger(currency) - except (socket.timeout, requests.exceptions.Timeout) as e: - Logger().logger.error("Currency: %s, Transport Exception occurred: %s", currency.name, e) - except Exception as e: - Logger().logger.error("Currency: %s, Generic Error occurred: %s", currency.name, e) + try: + match config.mode: + case Modes.LENDING: + for currency in self.__currencies: + min_int_rate = self.get_min_daily_interest_rate(currency) + min_int_rate_charge = float(format(min_int_rate + config.charge, '.5f')) + if min_int_rate_charge <= config.minimum_rate: + self.__minimum_rate = config.minimum_rate + elif self.__minimum_rate == const.DEFAULT_MIN_RATE or abs(min_int_rate_charge - self.__minimum_rate) >= config.correction: + self.__minimum_rate = min_int_rate_charge + self.get_lending_assets(currency) + self.check_active_loans(min_int_rate, currency) + self.lend_loans(min_int_rate, currency) + self.check_active_lendings(currency) + case Modes.DUAL: + self.check_ledger() + self.check_symbol_trigger() + except (socket.timeout, requests.exceptions.Timeout) as e: + Logger().logger.error("Transport Exception occurred: %s", e) + except Exception as e: + Logger().logger.error("Generic Error occurred: %s", e) def get_lending_assets(self, currency): asset = self.__client.get_lend_record(currency=currency.name) @@ -118,14 +128,21 @@ def get_min_daily_interest_rate(self, currency): else: return config.default_interest - def check_ledger(self, currency): + def check_ledger(self): current_page = 1 page_size = 50 - active_list = self.__user.get_account_ledger(pageSize=page_size, currency=currency.name, currentPage=current_page) - for page in range(current_page + 1, active_list['totalPage'] + 1): - result = self.__user.get_account_ledger(pageSize=page_size, currency=currency.name, currentPage=page) - active_list['items'].extend(result['items']) - print(active_list) + ledger = self.__user.get_account_ledger(pageSize=page_size, currentPage=current_page) + for page in range(current_page + 1, ledger['totalPage'] + 1): + result = self.__user.get_account_ledger(pageSize=page_size, currentPage=page) + ledger['items'].extend(result['items']) + ledger_asset = LedgerAssets(ledger=ledger) + Logger().logger.info('%s rows saved into the ledger table', ledger_asset.save()) + + def check_symbol_trigger(self): + for symbol in config.symbols: + result = self.__user_public.get_symbol_ticks(symbols=symbol)[0] + symbol_asset = SymbolAssets(symbol=result) + Logger().logger.info('%s rows saved into the %s symbols table', symbol_asset.save(), symbol) def check_active_lendings(self, currency): current_page = 1 @@ -165,16 +182,18 @@ def try_add_notifier(notifier, current_notifiers): def main(): Logger().logger.info("Starting Kubot Version {} - " - "Config: Correction: {}, Default Interest: {}, Minimum Rate: {}, Charge: {}" + "Config: Mode: {}, Correction: {}, Default Interest: {}, Minimum Rate: {}, Charge: {}, Symbols: {}" .format(get_version(), + config.mode.value, convert_float_to_percentage(config.correction), convert_float_to_percentage(config.default_interest), 'disabled' if config.minimum_rate == const.DEFAULT_MIN_RATE else convert_float_to_percentage(config.minimum_rate), - convert_float_to_percentage(config.charge))) + convert_float_to_percentage(config.charge), + config.symbols)) # initialize database - # with db: - # db.create_tables([FundingMarket, ActiveLendOrder, LendingAssets]) + with db: + db.create_tables([FundingMarket, ActiveLendOrder, LendingAssets, LedgerAssets, SymbolAssets]) # initialize notifier systems notifiers = [] @@ -187,7 +206,7 @@ def main(): currencies = [Currency(currency) for currency in config.currencies] # start main scheduler process - Scheduler(notifiers=notifiers, currencies=currencies) + Scheduler(notifiers=notifiers, currencies=currencies, mode=config.mode) if __name__ == "__main__": diff --git a/src/schemas/config.py b/src/schemas/config.py index 02c2e4b..2c0d10e 100644 --- a/src/schemas/config.py +++ b/src/schemas/config.py @@ -1,4 +1,5 @@ from schema import Schema, Or, And +from .static import Modes currencies = Schema([ { @@ -7,3 +8,11 @@ 'reserved_amount': And(int, lambda a: a >= 0) } ]) + +symbols = Schema([ + Or("ETH-USDT", "ETH-BTC") +]) + +modes = Schema( + Or(Modes.DUAL, Modes.LENDING) +) diff --git a/src/schemas/static.py b/src/schemas/static.py new file mode 100644 index 0000000..510305a --- /dev/null +++ b/src/schemas/static.py @@ -0,0 +1,5 @@ +from enum import Enum + +class Modes(Enum): + LENDING="LENDING" + DUAL="DUAL" \ No newline at end of file From 39fd46094c856c43bbc3b93793f4d02c387211af Mon Sep 17 00:00:00 2001 From: Desy Date: Fri, 6 Jun 2025 17:09:35 +0200 Subject: [PATCH 04/13] revert database setting #57 --- Dockerfile | 4 ++-- config/config.demo | 2 ++ src/database/models/base.py | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7f560c3..a5b6fcc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ -FROM python:3.8-alpine as base +FROM python:3.13-alpine as base RUN pip install --upgrade pip -RUN apk --update add gcc musl-dev postgresql-dev +RUN apk --update add gcc musl-dev postgresql-dev git COPY . /app WORKDIR /app diff --git a/config/config.demo b/config/config.demo index 474399c..43ba16a 100644 --- a/config/config.demo +++ b/config/config.demo @@ -10,6 +10,8 @@ minimum_rate = 0.0005 charge = 0.00005 interval = 300 currencies = [{"currency": "USDT", "term": 28, "reserved_amount": 10}] +symbols = ["ETH-USDT", "ETH-BTC"] +mode = DUAL [pushover] user_key = user_key diff --git a/src/database/models/base.py b/src/database/models/base.py index a909f49..996be9f 100644 --- a/src/database/models/base.py +++ b/src/database/models/base.py @@ -2,8 +2,8 @@ from playhouse.pool import PooledPostgresqlExtDatabase db = PooledPostgresqlExtDatabase('kubot', user='kubot', - password='kubot', host='localhost', - port=5433, max_connections=8, stale_timeout=300) + password='kubot', host='postgres', + port=5432, max_connections=8, stale_timeout=300) class BaseModel(Model): class Meta: From 1381ab457a2ec0bd74aee4e6dd437a337f0e98b0 Mon Sep 17 00:00:00 2001 From: Matthias Huber Date: Wed, 11 Jun 2025 14:55:42 +0200 Subject: [PATCH 05/13] add tls support for grafana #57 --- .gitignore | 3 ++- Makefile | 8 ++++++-- docker-compose.yml | 5 +++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index a384f3c..d398542 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ config/config venv +certs .idea .pytest_cache __pycache__ .vscode/settings.json kubot_* -*.html \ No newline at end of file +*.html diff --git a/Makefile b/Makefile index 24fb60c..d9d84d7 100644 --- a/Makefile +++ b/Makefile @@ -27,10 +27,10 @@ build-dev: ## build kubot development docker image docker build --target development --tag ${image}:${version} . docker image prune -f -compose-dev: ## compose and start kubot suite in development mode +compose-dev: gen-certs ## compose and start kubot suite in development mode KUBOT_IMAGE=${image} KUBOT_VERSION=${version} docker-compose up -d -development: ## start kubot database and gui +development: gen-certs ## start kubot database and gui KUBOT_IMAGE=${image} KUBOT_VERSION=${version} docker-compose up -d postgres grafana run-d: ## run kubot docker image detached @@ -44,6 +44,10 @@ run: ## run kubot docker image attached venv: ## bootstrap python3 venv @test -d "venv" || python3 -m venv venv +gen-certs: ## generate grafana certs + @mkdir -p certs + @openssl req -x509 -newkey rsa:2048 -keyout certs/grafana.key -out certs/grafana.crt -days 365 -nodes -subj "/CN=localhost" + install: venv ## install kubot dependencies @source venv/bin/activate; \ pip install --upgrade pip setuptools==75.6.0; \ diff --git a/docker-compose.yml b/docker-compose.yml index 046c31c..bd3c918 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,8 +33,13 @@ services: restart: on-failure:10 depends_on: - postgres + environment: + - GF_SERVER_PROTOCOL=https + - GF_SERVER_CERT_FILE=/etc/grafana/certs/grafana.crt + - GF_SERVER_CERT_KEY=/etc/grafana/certs/grafana.key volumes: - grafana-storage:/var/lib/grafana + - ./certs:/etc/grafana/certs:ro - ./provisioning:/etc/grafana/provisioning ports: - 3000:3000 From 3fe41265a1f9fe85de07e3493821e372ebe2c7dc Mon Sep 17 00:00:00 2001 From: Matthias Huber Date: Thu, 12 Jun 2025 16:06:36 +0200 Subject: [PATCH 06/13] add category currency #57 --- .../kubot-dual-investment-dashboard.json | 171 ++++++++++++++++-- src/config/config.py | 11 +- src/database/models/category.py | 8 + src/kubot.py | 10 +- src/schemas/config.py | 6 +- src/schemas/static.py | 3 + 6 files changed, 194 insertions(+), 15 deletions(-) create mode 100644 src/database/models/category.py diff --git a/provisioning/dashboards/Kubot/kubot-dual-investment-dashboard.json b/provisioning/dashboards/Kubot/kubot-dual-investment-dashboard.json index 38d44cb..8c58498 100644 --- a/provisioning/dashboards/Kubot/kubot-dual-investment-dashboard.json +++ b/provisioning/dashboards/Kubot/kubot-dual-investment-dashboard.json @@ -26,8 +26,154 @@ "graphTooltip": 0, "id": 3, "links": [], - "liveNow": false, "panels": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "P0689FA73A582FCBA" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 64, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "annualrate" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "strikeprice" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "left" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "annualrate" + }, + "properties": [] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 3, + "options": { + "barRadius": 0, + "barWidth": 1, + "colorByField": "Time", + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "text": {}, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xField": "Time", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 200 + }, + "pluginVersion": "12.0.1", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "P0689FA73A582FCBA" + }, + "editorMode": "code", + "format": "time_series", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT\n time as \"time\",\n ((product ->> 'strikePrice')::numeric) AS strikePrice,\n ((product ->> 'annualRate')::numeric * 100) AS annualRate\nFROM\n categorycurrency,\n json_array_elements(items) AS outer_entry,\n json_array_elements(outer_entry -> 'classicSeriesInfo') AS series,\n json_array_elements(series -> 'productList') AS product\nWHERE\n outer_entry ->> 'currency' = 'ETH' and ((product ->> 'duration')::numeric) = 1 and ((product ->> 'strikePrice')::numeric) > 1 and ((product ->> 'seriesNames')) = 'ETH-U' order by 1;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "ETH Dual Investment", + "type": "barchart" + }, { "datasource": { "type": "postgres", @@ -40,11 +186,13 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -53,6 +201,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -74,8 +223,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -112,10 +260,10 @@ ] }, "gridPos": { - "h": 14, - "w": 10, + "h": 10, + "w": 12, "x": 0, - "y": 0 + "y": 8 }, "id": 2, "options": { @@ -126,11 +274,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, - "pluginVersion": "9.1.2", + "pluginVersion": "12.0.1", "targets": [ { "datasource": { @@ -177,8 +326,9 @@ "type": "timeseries" } ], - "schemaVersion": 37, - "style": "dark", + "preload": false, + "refresh": "", + "schemaVersion": 41, "tags": [], "templating": { "list": [] @@ -191,6 +341,5 @@ "timezone": "", "title": "Dual Investment", "uid": "heg0onYNk", - "version": 4, - "weekStart": "" + "version": 9 } \ No newline at end of file diff --git a/src/config/config.py b/src/config/config.py index 71eda6b..c7196fe 100644 --- a/src/config/config.py +++ b/src/config/config.py @@ -4,9 +4,11 @@ from configparser import ConfigParser, ExtendedInterpolation import const -from schemas.config import currencies as currencies_schema, symbols as symbols_schema, modes as modes_schema +from schemas.config import currencies as currencies_schema, symbols as symbols_schema, modes as modes_schema, \ + category_currency as category_currency_schema from schemas.static import Modes + def property_wrapper(default=None): def inner_decorator(f): def wrapped(*args, **kwargs): @@ -108,5 +110,10 @@ def symbols(self): symbols = json.loads(self.__config['bot'].get('symbols')) return symbols_schema.validate(symbols) - + @property + @property_wrapper(default=[]) + def category_currency(self): + category_currency = json.loads(self.__config['bot'].get('category_currency')) + return category_currency_schema.validate(category_currency) +'' config = Config() diff --git a/src/database/models/category.py b/src/database/models/category.py new file mode 100644 index 0000000..4bb2a07 --- /dev/null +++ b/src/database/models/category.py @@ -0,0 +1,8 @@ +from playhouse.postgres_ext import * +from datetime import datetime +from database.models.base import BaseModel + +class CategoryCurrency(BaseModel): + time = DateTimeField(default=datetime.utcnow) + category = CharField() + items = JSONField() \ No newline at end of file diff --git a/src/kubot.py b/src/kubot.py index 1b02f68..b0cfab4 100644 --- a/src/kubot.py +++ b/src/kubot.py @@ -4,6 +4,7 @@ import const from database.models.base import db +from database.models.category import CategoryCurrency from database.models.market import FundingMarket from database.models.activeorder import ActiveLendOrder from database.models.assets import LendingAssets @@ -66,6 +67,7 @@ def schedule_checks(self, interval): case Modes.DUAL: self.check_ledger() self.check_symbol_trigger() + self.check_dual_investments() except (socket.timeout, requests.exceptions.Timeout) as e: Logger().logger.error("Transport Exception occurred: %s", e) except Exception as e: @@ -144,6 +146,12 @@ def check_symbol_trigger(self): symbol_asset = SymbolAssets(symbol=result) Logger().logger.info('%s rows saved into the %s symbols table', symbol_asset.save(), symbol) + def check_dual_investments(self): + for category in config.category_currency: + result = self.__user_public.get_category_currency(category=category) + category_currency = CategoryCurrency(category=category, items=result) + Logger().logger.info('%s rows saved into %s category currency table', category_currency.save(), category) + def check_active_lendings(self, currency): current_page = 1 page_size = 50 @@ -193,7 +201,7 @@ def main(): # initialize database with db: - db.create_tables([FundingMarket, ActiveLendOrder, LendingAssets, LedgerAssets, SymbolAssets]) + db.create_tables([FundingMarket, ActiveLendOrder, LendingAssets, LedgerAssets, SymbolAssets, CategoryCurrency]) # initialize notifier systems notifiers = [] diff --git a/src/schemas/config.py b/src/schemas/config.py index 2c0d10e..e80059f 100644 --- a/src/schemas/config.py +++ b/src/schemas/config.py @@ -1,5 +1,5 @@ from schema import Schema, Or, And -from .static import Modes +from .static import Modes, CategoryCurrencyEnum currencies = Schema([ { @@ -16,3 +16,7 @@ modes = Schema( Or(Modes.DUAL, Modes.LENDING) ) + +category_currency = Schema([ + Or(CategoryCurrencyEnum.DUAL.value) +]) diff --git a/src/schemas/static.py b/src/schemas/static.py index 510305a..4b5e78d 100644 --- a/src/schemas/static.py +++ b/src/schemas/static.py @@ -2,4 +2,7 @@ class Modes(Enum): LENDING="LENDING" + DUAL="DUAL" + +class CategoryCurrencyEnum(Enum): DUAL="DUAL" \ No newline at end of file From 050a5d734f2b4b0b76c7538cc69bf3ce675fb2a3 Mon Sep 17 00:00:00 2001 From: Matthias Huber Date: Thu, 12 Jun 2025 16:11:31 +0200 Subject: [PATCH 07/13] add category currency to config demo --- config/config.demo | 1 + 1 file changed, 1 insertion(+) diff --git a/config/config.demo b/config/config.demo index 43ba16a..a572d8e 100644 --- a/config/config.demo +++ b/config/config.demo @@ -11,6 +11,7 @@ charge = 0.00005 interval = 300 currencies = [{"currency": "USDT", "term": 28, "reserved_amount": 10}] symbols = ["ETH-USDT", "ETH-BTC"] +category_currency = ["DUAL"] mode = DUAL [pushover] From 8c34a9627bbc05db270b6ee2b5cf2dad9bba6ca4 Mon Sep 17 00:00:00 2001 From: Matthias Huber Date: Fri, 13 Jun 2025 11:02:08 +0200 Subject: [PATCH 08/13] add housekeeping #57 --- src/kubot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/kubot.py b/src/kubot.py index b0cfab4..6a297bf 100644 --- a/src/kubot.py +++ b/src/kubot.py @@ -46,6 +46,7 @@ def cleanup_database(self): LendingAssets.delete().where(LendingAssets.time < time_delta).execute() LedgerAssets.delete().where(LedgerAssets.time < time_delta).execute() SymbolAssets.delete().where(SymbolAssets.time < time_delta).execute() + CategoryCurrency.delete().where(CategoryCurrency.time < time_delta).execute() def schedule_checks(self, interval): self.__scheduler.enter(interval, 1, self.schedule_checks, argument=(interval,)) From 291cf66d51e49c9f3c784f7a33b757e6e3466d17 Mon Sep 17 00:00:00 2001 From: Matthias Huber Date: Mon, 21 Jul 2025 17:01:47 +0200 Subject: [PATCH 09/13] update dashboard #57 --- .../kubot-dual-investment-dashboard.json | 326 +++++++++++------- 1 file changed, 204 insertions(+), 122 deletions(-) diff --git a/provisioning/dashboards/Kubot/kubot-dual-investment-dashboard.json b/provisioning/dashboards/Kubot/kubot-dual-investment-dashboard.json index 8c58498..6e8adef 100644 --- a/provisioning/dashboards/Kubot/kubot-dual-investment-dashboard.json +++ b/provisioning/dashboards/Kubot/kubot-dual-investment-dashboard.json @@ -26,190 +26,261 @@ "graphTooltip": 0, "id": 3, "links": [], + "liveNow": false, "panels": [ { - "datasource": { - "type": "grafana-postgresql-datasource", - "uid": "P0689FA73A582FCBA" + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true }, + "pluginVersion": "8.3.1", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "P0689FA73A582FCBA" + }, + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n time,\n (item ->> 'amount') AS amount,\n COUNT(*) AS occurrences\nFROM\n ledgerassets t,\n LATERAL json_array_elements(t.ledger -> 'items') AS item\nWHERE\n item ->> 'direction' = 'in'\nGROUP BY\n time, amount\norder by 1;", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "table": "ledgerassets", + "timeColumn": "time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Panel Title", + "type": "gauge" + }, + { + "description": "ETH-USDT Last Traded Price", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "fillOpacity": 64, - "gradientMode": "hue", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, "thresholdsStyle": { "mode": "off" } }, - "fieldMinMax": false, + "decimals": 2, + "displayName": "ETH-USDT", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", "value": 80 } ] - } + }, + "unit": "none" }, "overrides": [ { + "__systemRef": "hideSeriesFrom", "matcher": { - "id": "byName", - "options": "annualrate" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "ETH-USDT" + ], + "prefix": "All except:", + "readOnly": true } - ] - }, - { - "matcher": { - "id": "byName", - "options": "strikeprice" }, "properties": [ { - "id": "custom.axisPlacement", - "value": "left" + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } } ] - }, - { - "matcher": { - "id": "byName", - "options": "annualrate" - }, - "properties": [] } ] }, "gridPos": { - "h": 8, - "w": 24, - "x": 0, + "h": 18, + "w": 12, + "x": 12, "y": 0 }, - "id": 3, + "id": 2, "options": { - "barRadius": 0, - "barWidth": 1, - "colorByField": "Time", - "fullHighlight": false, - "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "orientation": "auto", - "showValue": "auto", - "stacking": "normal", - "text": {}, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" - }, - "xField": "Time", - "xTickLabelRotation": 0, - "xTickLabelSpacing": 200 + } }, "pluginVersion": "12.0.1", "targets": [ { "datasource": { - "type": "grafana-postgresql-datasource", + "type": "postgres", "uid": "P0689FA73A582FCBA" }, - "editorMode": "code", "format": "time_series", - "hide": false, + "group": [], + "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n time as \"time\",\n ((product ->> 'strikePrice')::numeric) AS strikePrice,\n ((product ->> 'annualRate')::numeric * 100) AS annualRate\nFROM\n categorycurrency,\n json_array_elements(items) AS outer_entry,\n json_array_elements(outer_entry -> 'classicSeriesInfo') AS series,\n json_array_elements(series -> 'productList') AS product\nWHERE\n outer_entry ->> 'currency' = 'ETH' and ((product ->> 'duration')::numeric) = 1 and ((product ->> 'strikePrice')::numeric) > 1 and ((product ->> 'seriesNames')) = 'ETH-U' order by 1;", + "rawSql": "SELECT\n time AS \"time\",\n ((symbol->>'lastTradedPrice')::float) as lastTradedPrice\nFROM symbolassets\nWHERE (symbol->>'symbol') = 'ETH-USDT'\nORDER BY 1", "refId": "A", - "sql": { - "columns": [ + "select": [ + [ { - "parameters": [], - "type": "function" + "params": [ + "value" + ], + "type": "column" } ], - "groupBy": [ + [ { - "property": { - "type": "string" - }, - "type": "groupBy" + "params": [ + "value" + ], + "type": "column" } - ], - "limit": 50 - } + ] + ], + "table": "symbolassets", + "timeColumn": "time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] } ], - "title": "ETH Dual Investment", - "type": "barchart" + "title": "Last Traded Price ETH - USDT", + "type": "timeseries" }, { - "datasource": { - "type": "postgres", - "uid": "P0689FA73A582FCBA" - }, - "description": "ETH-USDT Last Traded Price", + "description": "Interest over ETH and USDT", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", "axisLabel": "", - "axisPlacement": "auto", + "axisPlacement": "right", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": false, + "showPoints": "never", + "spanNulls": true, "stacking": { "group": "A", "mode": "none" @@ -218,42 +289,65 @@ "mode": "off" } }, + "decimals": 2, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", "value": 80 } ] - } + }, + "unit": "currencyUSD" }, "overrides": [ { - "__systemRef": "hideSeriesFrom", "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "lasttradedprice" - ], - "prefix": "All except:", - "readOnly": true + "id": "byName", + "options": "amount ETH" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "left" + }, + { + "id": "displayName", + "value": "Amount ETH" + }, + { + "id": "custom.axisLabel", + "value": "ETH" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 5 } + ] + }, + { + "matcher": { + "id": "byName", + "options": "amount USDT" }, "properties": [ { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } + "id": "displayName", + "value": "Amount USDT" + }, + { + "id": "custom.axisLabel", + "value": "USDT" } ] } @@ -265,21 +359,18 @@ "x": 0, "y": 8 }, - "id": 2, + "id": 4, "options": { "legend": { "calcs": [], "displayMode": "list", - "placement": "bottom", - "showLegend": true + "placement": "bottom" }, "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" + "mode": "single" } }, - "pluginVersion": "12.0.1", + "pluginVersion": "8.3.1", "targets": [ { "datasource": { @@ -290,17 +381,9 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n time AS \"time\",\n ((symbol->>'lastTradedPrice')::float) as lastTradedPrice\nFROM symbolassets\nWHERE (symbol->>'symbol') = 'ETH-USDT'\nORDER BY 1", + "rawSql": "SELECT\n time as \"time\",\n ((item->>'amount')::float) as amount,\n ((item->>'currency')) as currency\nFROM\n ledgerassets t,\n LATERAL json_array_elements(t.ledger -> 'items') AS item\nWHERE\n (item ->> 'direction' = 'in')\nGROUP BY\n item ->> 'id', amount, time, currency\nORDER BY 1;\n", "refId": "A", "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ], [ { "params": [ @@ -310,9 +393,7 @@ } ] ], - "table": "symbolassets", "timeColumn": "time", - "timeColumnType": "timestamp", "where": [ { "name": "$__timeFilter", @@ -322,24 +403,25 @@ ] } ], - "title": "Last Traded Price", + "title": "Profit USDT / ETH", "type": "timeseries" } ], - "preload": false, "refresh": "", - "schemaVersion": 41, + "schemaVersion": 33, + "style": "dark", "tags": [], "templating": { "list": [] }, "time": { - "from": "now-6h", + "from": "now-30d", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Dual Investment", "uid": "heg0onYNk", - "version": 9 + "version": 16, + "weekStart": "" } \ No newline at end of file From e11271786eac08885d9d4f605528abe76809081c Mon Sep 17 00:00:00 2001 From: Matthias Huber Date: Mon, 21 Jul 2025 18:36:39 +0200 Subject: [PATCH 10/13] add profit table #57 --- .../kubot-dual-investment-dashboard.json | 333 ++++++++++-------- 1 file changed, 192 insertions(+), 141 deletions(-) diff --git a/provisioning/dashboards/Kubot/kubot-dual-investment-dashboard.json b/provisioning/dashboards/Kubot/kubot-dual-investment-dashboard.json index 6e8adef..50795b4 100644 --- a/provisioning/dashboards/Kubot/kubot-dual-investment-dashboard.json +++ b/provisioning/dashboards/Kubot/kubot-dual-investment-dashboard.json @@ -26,115 +26,139 @@ "graphTooltip": 0, "id": 3, "links": [], - "liveNow": false, "panels": [ { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Profit USDT\n", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "continuous-RdYlGr" }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 1, + "spanNulls": false + }, + "decimals": 2, + "displayName": "USDT", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "color": "green" } ] - } + }, + "unit": "currencyUSD" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, + "h": 5, + "w": 24, "x": 0, "y": 0 }, "id": 6, "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "alignValue": "center", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "showThresholdLabels": false, - "showThresholdMarkers": true + "mergeValues": true, + "rowHeight": 0.86, + "showValue": "auto", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "8.3.1", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { - "type": "postgres", + "type": "grafana-postgresql-datasource", "uid": "P0689FA73A582FCBA" }, - "format": "time_series", - "group": [], - "metricColumn": "none", + "editorMode": "code", + "format": "table", + "hide": false, "rawQuery": true, - "rawSql": "SELECT\n time,\n (item ->> 'amount') AS amount,\n COUNT(*) AS occurrences\nFROM\n ledgerassets t,\n LATERAL json_array_elements(t.ledger -> 'items') AS item\nWHERE\n item ->> 'direction' = 'in'\nGROUP BY\n time, amount\norder by 1;", + "rawSql": "WITH daily_sum AS (\n SELECT\n ((item ->> 'createdAt')::numeric) as createdAt,\n ((item ->> 'amount')::numeric) AS amount\n FROM\n ledgerassets t,\n LATERAL json_array_elements(t.ledger -> 'items') AS item\n WHERE\n item ->> 'direction' = 'in'\n AND item ->> 'currency' = 'USDT'\n GROUP BY\n createdAt, amount\n)\n\nSELECT\n createdAt as \"time\",\n\n amount - LEAD(amount) OVER (ORDER BY createdAt DESC) AS diff_with_next_day\nFROM\n daily_sum\nORDER BY\n time DESC;", "refId": "A", - "select": [ - [ + "sql": { + "columns": [ { - "params": [ - "value" - ], - "type": "column" + "parameters": [], + "type": "function" } - ] - ], - "table": "ledgerassets", - "timeColumn": "time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } } ], - "title": "Panel Title", - "type": "gauge" + "title": "Profit USDT", + "type": "state-timeline" }, { - "description": "ETH-USDT Last Traded Price", + "datasource": { + "type": "postgres", + "uid": "P0689FA73A582FCBA" + }, + "description": "Interest over ETH and USDT", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", - "axisPlacement": "auto", + "axisPlacement": "right", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "spanNulls": false, + "showPoints": "never", + "spanNulls": true, "stacking": { "group": "A", "mode": "none" @@ -144,14 +168,12 @@ } }, "decimals": 2, - "displayName": "ETH-USDT", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -159,42 +181,62 @@ } ] }, - "unit": "none" + "unit": "currencyUSD" }, "overrides": [ { - "__systemRef": "hideSeriesFrom", "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "ETH-USDT" - ], - "prefix": "All except:", - "readOnly": true + "id": "byName", + "options": "amount ETH" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "left" + }, + { + "id": "displayName", + "value": "Amount ETH" + }, + { + "id": "custom.axisLabel", + "value": "ETH" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 5 } + ] + }, + { + "matcher": { + "id": "byName", + "options": "amount USDT" }, "properties": [ { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } + "id": "displayName", + "value": "Amount USDT" + }, + { + "id": "custom.axisLabel", + "value": "USDT" } ] } ] }, "gridPos": { - "h": 18, + "h": 10, "w": 12, - "x": 12, - "y": 0 + "x": 0, + "y": 5 }, - "id": 2, + "id": 4, "options": { "legend": { "calcs": [], @@ -208,18 +250,19 @@ "sort": "none" } }, - "pluginVersion": "12.0.1", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "postgres", "uid": "P0689FA73A582FCBA" }, + "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n time AS \"time\",\n ((symbol->>'lastTradedPrice')::float) as lastTradedPrice\nFROM symbolassets\nWHERE (symbol->>'symbol') = 'ETH-USDT'\nORDER BY 1", + "rawSql": "SELECT\n time as \"time\",\n ((item->>'amount')::float) as amount,\n ((item->>'currency')) as currency\nFROM\n ledgerassets t,\n LATERAL json_array_elements(t.ledger -> 'items') AS item\nWHERE\n (item ->> 'direction' = 'in')\nGROUP BY\n item ->> 'id', amount, time, currency\nORDER BY 1;\n", "refId": "A", "select": [ [ @@ -229,19 +272,26 @@ ], "type": "column" } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } ], - [ + "groupBy": [ { - "params": [ - "value" - ], - "type": "column" + "property": { + "type": "string" + }, + "type": "groupBy" } - ] - ], - "table": "symbolassets", + ], + "limit": 50 + }, "timeColumn": "time", - "timeColumnType": "timestamp", "where": [ { "name": "$__timeFilter", @@ -251,36 +301,45 @@ ] } ], - "title": "Last Traded Price ETH - USDT", + "title": "Profit USDT / ETH", "type": "timeseries" }, { - "description": "Interest over ETH and USDT", + "datasource": { + "type": "postgres", + "uid": "P0689FA73A582FCBA" + }, + "description": "ETH-USDT Last Traded Price", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", - "axisPlacement": "right", + "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 10, + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "never", - "spanNulls": true, + "showPoints": "auto", + "spanNulls": false, "stacking": { "group": "A", "mode": "none" @@ -290,13 +349,13 @@ } }, "decimals": 2, + "displayName": "ETH-USDT", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -304,50 +363,30 @@ } ] }, - "unit": "currencyUSD" + "unit": "none" }, "overrides": [ { + "__systemRef": "hideSeriesFrom", "matcher": { - "id": "byName", - "options": "amount ETH" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "left" - }, - { - "id": "displayName", - "value": "Amount ETH" - }, - { - "id": "custom.axisLabel", - "value": "ETH" - }, - { - "id": "unit", - "value": "short" - }, - { - "id": "decimals", - "value": 5 + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "ETH-USDT" + ], + "prefix": "All except:", + "readOnly": true } - ] - }, - { - "matcher": { - "id": "byName", - "options": "amount USDT" }, "properties": [ { - "id": "displayName", - "value": "Amount USDT" - }, - { - "id": "custom.axisLabel", - "value": "USDT" + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } } ] } @@ -356,21 +395,24 @@ "gridPos": { "h": 10, "w": 12, - "x": 0, - "y": 8 + "x": 12, + "y": 5 }, - "id": 4, + "id": 2, "options": { "legend": { "calcs": [], "displayMode": "list", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { - "mode": "single" + "hideZeros": false, + "mode": "single", + "sort": "none" } }, - "pluginVersion": "8.3.1", + "pluginVersion": "12.0.2", "targets": [ { "datasource": { @@ -381,9 +423,17 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n time as \"time\",\n ((item->>'amount')::float) as amount,\n ((item->>'currency')) as currency\nFROM\n ledgerassets t,\n LATERAL json_array_elements(t.ledger -> 'items') AS item\nWHERE\n (item ->> 'direction' = 'in')\nGROUP BY\n item ->> 'id', amount, time, currency\nORDER BY 1;\n", + "rawSql": "SELECT\n time AS \"time\",\n ((symbol->>'lastTradedPrice')::float) as lastTradedPrice\nFROM symbolassets\nWHERE (symbol->>'symbol') = 'ETH-USDT'\nORDER BY 1", "refId": "A", "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ], [ { "params": [ @@ -393,7 +443,9 @@ } ] ], + "table": "symbolassets", "timeColumn": "time", + "timeColumnType": "timestamp", "where": [ { "name": "$__timeFilter", @@ -403,13 +455,13 @@ ] } ], - "title": "Profit USDT / ETH", + "title": "Last Traded Price ETH - USDT", "type": "timeseries" } ], + "preload": false, "refresh": "", - "schemaVersion": 33, - "style": "dark", + "schemaVersion": 41, "tags": [], "templating": { "list": [] @@ -422,6 +474,5 @@ "timezone": "", "title": "Dual Investment", "uid": "heg0onYNk", - "version": 16, - "weekStart": "" + "version": 24 } \ No newline at end of file From 62f4b6cd1acdfe1b605cb437e2de5e36b59348bc Mon Sep 17 00:00:00 2001 From: Matthias Huber Date: Tue, 22 Jul 2025 14:37:29 +0200 Subject: [PATCH 11/13] add total interest #57 --- README.md | 1 + .../kubot-dual-investment-dashboard.json | 276 +++++++++++++++++- 2 files changed, 270 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 742fda7..8429aab 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ Lendbot for Kucoin with Grafana Dashboard, Pushover and Slack Support ![Kubot](https://github.com/desytech/kubot/workflows/Kubot/badge.svg)

+

diff --git a/provisioning/dashboards/Kubot/kubot-dual-investment-dashboard.json b/provisioning/dashboards/Kubot/kubot-dual-investment-dashboard.json index 50795b4..db53a90 100644 --- a/provisioning/dashboards/Kubot/kubot-dual-investment-dashboard.json +++ b/provisioning/dashboards/Kubot/kubot-dual-investment-dashboard.json @@ -32,12 +32,160 @@ "type": "datasource", "uid": "-- Mixed --" }, - "description": "Profit USDT\n", + "description": "Total Interest\n", "fieldConfig": { "defaults": { "color": { "mode": "continuous-RdYlGr" }, + "decimals": 2, + "displayName": "USDT", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 8, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "P0689FA73A582FCBA" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "WITH daily_sum_usdt AS (\n SELECT\n ((item ->> 'createdAt')::numeric) as createdAt,\n ((item ->> 'amount')::numeric) AS amount\n FROM\n ledgerassets t,\n LATERAL json_array_elements(t.ledger -> 'items') AS item\n WHERE\n item ->> 'direction' = 'in'\n AND item ->> 'currency' = 'USDT'\n GROUP BY\n createdAt, amount\n),\nsummarized_amounts AS (\n SELECT\n createdAt as \"time\",\n\n amount - LEAD(amount) OVER (ORDER BY createdAt DESC) AS diff_with_next_day\n FROM\n daily_sum_usdt\n ORDER BY\n time DESC\n)\n\nselect sum(diff_with_next_day) as sum_usdt from summarized_amounts;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "P0689FA73A582FCBA" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "WITH daily_sum AS (\n SELECT\n to_timestamp(((item ->> 'createdAt')::numeric) / 1000) as createdAt,\n ((item ->> 'amount')::numeric) AS amount\n FROM\n ledgerassets t,\n LATERAL json_array_elements(t.ledger -> 'items') AS item\n WHERE\n item ->> 'direction' = 'in'\n AND item ->> 'currency' = 'ETH'\n GROUP BY\n createdAt, amount\n),\nsummarized_amounts_eth AS (\n SELECT\n createdAt as \"time\",\n ((amount - LEAD(amount) OVER (ORDER BY createdAt DESC)) * l2.lastTradedPrice) AS diff_with_next_day\n FROM\n daily_sum\n JOIN LATERAL (select ((symbol->>'lastTradedPrice')::float) as lastTradedPrice from symbolassets as l2 where symbol->>'symbolCode' = 'ETH-USDT' order by ABS(EXTRACT(EPOCH FROM (daily_sum.createdAt - l2.time))) asc limit 1) as l2 on true\n GROUP BY daily_sum.createdAt, daily_sum.amount, l2.lastTradedPrice\n order by\n daily_sum.createdAt DESC\n)\nselect sum(diff_with_next_day) as sum_eth from summarized_amounts_eth;", + "refId": "B", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Total Interest", + "transformations": [ + { + "id": "concatenate", + "options": {} + }, + { + "id": "calculateField", + "options": { + "alias": "interest", + "binary": { + "left": { + "matcher": { + "id": "byName", + "options": "sum_usdt" + } + }, + "operator": "+", + "right": { + "matcher": { + "id": "byName", + "options": "sum_eth" + } + } + }, + "cumulative": { + "field": "sum_usdt", + "reducer": "sum" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "type": "gauge" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Profit ETH\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { "axisPlacement": "auto", "fillOpacity": 70, @@ -67,10 +215,106 @@ }, "gridPos": { "h": 5, - "w": 24, - "x": 0, + "w": 20, + "x": 4, "y": 0 }, + "id": 7, + "options": { + "alignValue": "center", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.86, + "showValue": "auto", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.2", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "P0689FA73A582FCBA" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "WITH daily_sum AS (\n SELECT\n to_timestamp(((item ->> 'createdAt')::numeric) / 1000) as createdAt,\n ((item ->> 'amount')::numeric) AS amount\n FROM\n ledgerassets t,\n LATERAL json_array_elements(t.ledger -> 'items') AS item\n WHERE\n item ->> 'direction' = 'in'\n AND item ->> 'currency' = 'ETH'\n GROUP BY\n createdAt, amount\n)\n\nSELECT\n createdAt as \"time\",\n ((amount - LEAD(amount) OVER (ORDER BY createdAt DESC)) * l2.lastTradedPrice) AS diff_with_next_day\nFROM\n daily_sum\nJOIN LATERAL (select ((symbol->>'lastTradedPrice')::float) as lastTradedPrice from symbolassets as l2 where symbol->>'symbolCode' = 'ETH-USDT' order by ABS(EXTRACT(EPOCH FROM (daily_sum.createdAt - l2.time))) asc limit 1) as l2 on true\nGROUP BY daily_sum.createdAt, daily_sum.amount, l2.lastTradedPrice\norder by\n daily_sum.createdAt DESC;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Profit ETH", + "type": "state-timeline" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Profit USDT\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 1, + "spanNulls": false + }, + "decimals": 2, + "displayName": "USDT", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 20, + "x": 4, + "y": 5 + }, "id": 6, "options": { "alignValue": "center", @@ -234,7 +478,7 @@ "h": 10, "w": 12, "x": 0, - "y": 5 + "y": 10 }, "id": 4, "options": { @@ -396,7 +640,7 @@ "h": 10, "w": 12, "x": 12, - "y": 5 + "y": 10 }, "id": 2, "options": { @@ -419,6 +663,7 @@ "type": "postgres", "uid": "P0689FA73A582FCBA" }, + "editorMode": "code", "format": "time_series", "group": [], "metricColumn": "none", @@ -443,6 +688,23 @@ } ] ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, "table": "symbolassets", "timeColumn": "time", "timeColumnType": "timestamp", @@ -467,12 +729,12 @@ "list": [] }, "time": { - "from": "now-30d", + "from": "now-40d", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Dual Investment", "uid": "heg0onYNk", - "version": 24 + "version": 40 } \ No newline at end of file From 7b36f019766c9ffad3ce4c71d4513148b2f1f4b2 Mon Sep 17 00:00:00 2001 From: Matthias Huber Date: Tue, 22 Jul 2025 14:47:14 +0200 Subject: [PATCH 12/13] bump python build version #57 --- .github/workflows/kubot-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/kubot-app.yml b/.github/workflows/kubot-app.yml index bad607f..b94f7de 100644 --- a/.github/workflows/kubot-app.yml +++ b/.github/workflows/kubot-app.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python 3.8 uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.13 - name: Install dependencies run: | python -m pip install --upgrade pip From 08dc073c48a0c136dcffd6678f14b46fde4f48d6 Mon Sep 17 00:00:00 2001 From: Matthias Huber Date: Tue, 22 Jul 2025 14:54:18 +0200 Subject: [PATCH 13/13] update dependencies #57 --- .github/workflows/kubot-app.yml | 2 +- requirements.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/kubot-app.yml b/.github/workflows/kubot-app.yml index b94f7de..29c12e6 100644 --- a/.github/workflows/kubot-app.yml +++ b/.github/workflows/kubot-app.yml @@ -16,7 +16,7 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python 3.13 uses: actions/setup-python@v2 with: python-version: 3.13 diff --git a/requirements.txt b/requirements.txt index f8ed1f3..629c856 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,13 +9,13 @@ kucoin-python @ git+https://github.com/desytech/kucoin-python-sdk.git@master markdown2==2.3.10 packaging==20.7 peewee==3.14.0 -pluggy==0.13.1 +pluggy==1.6.0 psycopg2-binary==2.9.10 pushover @ git+https://github.com/Wyattjoh/pushover@5852545c5b9cf2717e1eafc4c8b134a08b0994da -py==1.9.0 pyarmor==6.6.0 +Pygments==2.19.2 pyparsing==2.4.7 -pytest==6.1.2 +pytest==8.4.1 requests==2.32.3 schema==0.7.2 toml==0.10.2