From 4a00e5180676ec1b700656db888159d4512b527e Mon Sep 17 00:00:00 2001 From: liquid-metal <56035107+liquid-metal@users.noreply.github.com> Date: Wed, 2 Apr 2025 22:02:35 +0200 Subject: [PATCH 01/19] introduce django and serve the same frontend --- .gitignore | 5 + meshping/manage.py | 22 ++ meshping/meshping/__init__.py | 0 meshping/meshping/apps.py | 10 + meshping/meshping/asgi.py | 16 + meshping/meshping/jinja2.py | 15 + meshping/meshping/migrations/0001_initial.py | 54 +++ meshping/meshping/migrations/__init__.py | 0 meshping/meshping/models.py | 34 ++ meshping/meshping/settings.py | 95 +++++ meshping/meshping/templates/index.html.j2 | 216 +++++++++++ meshping/meshping/urls.py | 7 + meshping/meshping/views.py | 28 ++ meshping/meshping/wsgi.py | 16 + meshping/ui/package-lock.json | 359 +++++++++++++++++++ meshping/ui/package.json | 18 + meshping/ui/src/main.js | 184 ++++++++++ requirements.txt | 4 + 18 files changed, 1083 insertions(+) create mode 100755 meshping/manage.py create mode 100644 meshping/meshping/__init__.py create mode 100644 meshping/meshping/apps.py create mode 100644 meshping/meshping/asgi.py create mode 100644 meshping/meshping/jinja2.py create mode 100644 meshping/meshping/migrations/0001_initial.py create mode 100644 meshping/meshping/migrations/__init__.py create mode 100644 meshping/meshping/models.py create mode 100644 meshping/meshping/settings.py create mode 100644 meshping/meshping/templates/index.html.j2 create mode 100644 meshping/meshping/urls.py create mode 100644 meshping/meshping/views.py create mode 100644 meshping/meshping/wsgi.py create mode 100644 meshping/ui/package-lock.json create mode 100644 meshping/ui/package.json create mode 100644 meshping/ui/src/main.js diff --git a/.gitignore b/.gitignore index 6d529b5..0f04a06 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,8 @@ build/ ui/node_modules/ db/ + +meshping/ui/node_modules/ +meshping/db/ + +venv/ diff --git a/meshping/manage.py b/meshping/manage.py new file mode 100755 index 0000000..324b250 --- /dev/null +++ b/meshping/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'meshping.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/meshping/meshping/__init__.py b/meshping/meshping/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/meshping/meshping/apps.py b/meshping/meshping/apps.py new file mode 100644 index 0000000..6b2b4a8 --- /dev/null +++ b/meshping/meshping/apps.py @@ -0,0 +1,10 @@ +from django.apps import AppConfig + + +class MeshpingConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'meshping' + + def ready(self): + # TODO start background threads for ping, traceroute, peers + pass diff --git a/meshping/meshping/asgi.py b/meshping/meshping/asgi.py new file mode 100644 index 0000000..1afae58 --- /dev/null +++ b/meshping/meshping/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for meshping project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/5.1/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'meshping.settings') + +application = get_asgi_application() diff --git a/meshping/meshping/jinja2.py b/meshping/meshping/jinja2.py new file mode 100644 index 0000000..f575cba --- /dev/null +++ b/meshping/meshping/jinja2.py @@ -0,0 +1,15 @@ +from jinja2 import Environment +from django.urls import reverse +from django.templatetags.static import static + + +def environment(**options): + env = Environment( + variable_start_string='{[', + variable_end_string=']}', + **options) + env.globals.update({ + "static": static, + "url": reverse + }) + return env diff --git a/meshping/meshping/migrations/0001_initial.py b/meshping/meshping/migrations/0001_initial.py new file mode 100644 index 0000000..3b3b4b4 --- /dev/null +++ b/meshping/meshping/migrations/0001_initial.py @@ -0,0 +1,54 @@ +# Generated by Django 5.1.7 on 2025-04-02 18:07 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Target', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('addr', models.CharField(max_length=255, unique=True)), + ('name', models.CharField(max_length=255)), + ], + options={ + 'unique_together': {('addr', 'name')}, + }, + ), + migrations.CreateModel( + name='Statistics', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('field', models.CharField(max_length=255)), + ('value', models.FloatField()), + ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='meshping.target')), + ], + ), + migrations.CreateModel( + name='Meta', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('field', models.CharField(max_length=255)), + ('value', models.CharField(max_length=255)), + ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='meshping.target')), + ], + ), + migrations.CreateModel( + name='Histogram', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('timestamp', models.IntegerField()), + ('bucket', models.IntegerField()), + ('count', models.IntegerField(default=1)), + ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='meshping.target')), + ], + ), + ] diff --git a/meshping/meshping/migrations/__init__.py b/meshping/meshping/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/meshping/meshping/models.py b/meshping/meshping/models.py new file mode 100644 index 0000000..aa094f2 --- /dev/null +++ b/meshping/meshping/models.py @@ -0,0 +1,34 @@ +from django.db import models + + +# TODO decide on max_length, even though ignored by sqlite +class Target(models.Model): + addr = models.CharField(max_length=255, unique=True) + name = models.CharField(max_length=255) + + class Meta: + unique_together = ('addr', 'name') + + +# TODO uniqueness constraint `UNIQUE (target_id, timestamp, bucket)` +class Histogram(models.Model): + target = models.ForeignKey(Target, on_delete=models.CASCADE) + timestamp = models.IntegerField() + bucket = models.IntegerField() + count = models.IntegerField(default=1) + + +# TODO decide on max_length, even though ignored by sqlite +# TODO uniqueness constraint `UNIQUE (target_id, field)` +class Statistics(models.Model): + target = models.ForeignKey(Target, on_delete=models.CASCADE) + field = models.CharField(max_length=255) + value = models.FloatField() + + +# TODO check if the CharField maps to SQLite TEXT + decide on max_length +# TODO uniqueness constraint `UNIQUE (target_id, field)` +class Meta(models.Model): + target = models.ForeignKey(Target, on_delete=models.CASCADE) + field = models.CharField(max_length=255) + value = models.CharField(max_length=255) diff --git a/meshping/meshping/settings.py b/meshping/meshping/settings.py new file mode 100644 index 0000000..be9ec64 --- /dev/null +++ b/meshping/meshping/settings.py @@ -0,0 +1,95 @@ +from pathlib import Path +import os + + +BASE_DIR = Path(__file__).resolve().parent.parent + +SECRET_KEY = 'django-insecure-2svcr!j9%^_(^)$%)vvt1j&0$w)&_l&oxacnb1mkurd4f--xmg' +DEBUG = True +ALLOWED_HOSTS = ['*'] + +INSTALLED_APPS = [ +# 'django.contrib.admin', +# 'django.contrib.auth', +# 'django.contrib.contenttypes', +# 'django.contrib.sessions', +# 'django.contrib.messages', + 'django.contrib.staticfiles', + 'meshping', +] + +#MIDDLEWARE = [ +# 'django.middleware.security.SecurityMiddleware', +# 'django.contrib.sessions.middleware.SessionMiddleware', +# 'django.middleware.common.CommonMiddleware', +# 'django.middleware.csrf.CsrfViewMiddleware', +# 'django.contrib.auth.middleware.AuthenticationMiddleware', +# 'django.contrib.messages.middleware.MessageMiddleware', +# 'django.middleware.clickjacking.XFrameOptionsMiddleware', +#] + +ROOT_URLCONF = 'meshping.urls' + +# TODO the APP_DIRS seem to not work as I imagined for the jinja2 backend +TEMPLATES = [ + { + "BACKEND": "django.template.backends.jinja2.Jinja2", + "DIRS": [os.path.join(BASE_DIR, 'meshping/templates'),], + "APP_DIRS": False, + "OPTIONS": { + "environment": "meshping.jinja2.environment", + }, + }, +# { +# 'BACKEND': 'django.template.backends.django.DjangoTemplates', +# 'DIRS': [], +# 'APP_DIRS': True, +# 'OPTIONS': { +# 'context_processors': [ +# 'django.template.context_processors.debug', +# 'django.template.context_processors.request', +## 'django.contrib.auth.context_processors.auth', +## 'django.contrib.messages.context_processors.messages', +# ], +# }, +# }, +] +#WSGI_APPLICATION = 'meshping.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/5.1/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': BASE_DIR / 'db' / 'db.sqlite3', + } +} + +#AUTH_PASSWORD_VALIDATORS = [ +# { +# 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', +# }, +# { +# 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', +# }, +# { +# 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', +# }, +# { +# 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', +# }, +#] + +#LANGUAGE_CODE = 'en-us' +#TIME_ZONE = 'UTC' +#USE_I18N = True +#USE_TZ = True + +STATIC_URL = 'ui/' +STATICFILES_DIRS = [ + BASE_DIR / "ui", +] + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' diff --git a/meshping/meshping/templates/index.html.j2 b/meshping/meshping/templates/index.html.j2 new file mode 100644 index 0000000..fa19cc6 --- /dev/null +++ b/meshping/meshping/templates/index.html.j2 @@ -0,0 +1,216 @@ + + + + {[ Hostname ]} — Meshping + + + + + + + + + + +
+

Meshping: {[ Hostname ]}

+
+ +
+ × + +
+
+ × + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
 TargetAddressSentRecvSuccLossMinAvg15mAvg6hAvg24hMaxLast 
Loading
No targets configured
No targets match your search
+ + {[ icons['check-circle.svg'] ]} + + + {[ icons['arrow-up-right-circle.svg'] ]} + + + {[ icons['x-circle.svg'] ]} + + + {[ icons['arrow-clockwise.svg'] ]} + + + {[ icons['question-circle.svg'] ]} + + + {[ icons['exclamation-circle.svg'] ]} + + {{ target.name }}
{{ target.addr }}		{{ target.addr }}{{ target.sent }}{{ target.recv }}{{ target.succ | prettyFloat }}{{ target.loss | prettyFloat }}{{ target.min | prettyFloat }}{{ target.avg15m | prettyFloat }}{{ target.avg6h | prettyFloat }}{{ target.avg24h | prettyFloat }}{{ target.max | prettyFloat }}{{ target.last | prettyFloat }} + + + graph + + del + + + + +
  + + + + + + +    + + + +
+ + + +
+
+ +
+ github.com/Svedrin/meshping + Meshping by Michael Ziegler +
+ + diff --git a/meshping/meshping/urls.py b/meshping/meshping/urls.py new file mode 100644 index 0000000..d8757d0 --- /dev/null +++ b/meshping/meshping/urls.py @@ -0,0 +1,7 @@ +from django.urls import path +from . import views + + +urlpatterns = [ + path('', views.index, name='index'), +] diff --git a/meshping/meshping/views.py b/meshping/meshping/views.py new file mode 100644 index 0000000..8c5d76b --- /dev/null +++ b/meshping/meshping/views.py @@ -0,0 +1,28 @@ +import os +import socket + +from django.http import HttpResponse +from django.template import loader +from markupsafe import Markup +from .models import Target + + +# TODO do not load icons from disk for every request +# TODO find a better method for finding the icons to not have an absolute path here +def index(request): + template = loader.get_template('index.html.j2') + # icons_dir = "/opt/meshping/ui/node_modules/bootstrap-icons/icons/" + icons_dir = "../ui/node_modules/bootstrap-icons/icons/" + icons_dir = os.path.join(os.path.dirname(__file__), icons_dir) + print(icons_dir) + icons = dict( + icons={ + filename: Markup(open(os.path.join(icons_dir, filename), "r").read()) + for filename in os.listdir(icons_dir) + } + ) + context = { + "Hostname": socket.gethostname(), + "icons": icons, + } + return HttpResponse(template.render(context, request)) diff --git a/meshping/meshping/wsgi.py b/meshping/meshping/wsgi.py new file mode 100644 index 0000000..c227ee0 --- /dev/null +++ b/meshping/meshping/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for meshping project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/5.1/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'meshping.settings') + +application = get_wsgi_application() diff --git a/meshping/ui/package-lock.json b/meshping/ui/package-lock.json new file mode 100644 index 0000000..587746c --- /dev/null +++ b/meshping/ui/package-lock.json @@ -0,0 +1,359 @@ +{ + "name": "meshping-ui", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "meshping-ui", + "version": "0.0.1", + "license": "MIT", + "dependencies": { + "bootstrap": "^4.4.1", + "bootstrap-icons": "^1.11.3", + "jquery": "^3.5.0", + "vue": "^2.6.11", + "vue-resource": "^1.5.3" + } + }, + "node_modules/@sindresorhus/is": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", + "integrity": "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/@szmarczak/http-timer": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.6.tgz", + "integrity": "sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w==", + "dependencies": { + "defer-to-connect": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@types/cacheable-request": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.3.tgz", + "integrity": "sha512-IQ3EbTzGxIigb1I3qPZc1rWJnH0BmSKv5QYTalEwweFvyBDLSAe24zP0le/hyi7ecGfZVlIVAg4BZqb8WBwKqw==", + "dependencies": { + "@types/http-cache-semantics": "*", + "@types/keyv": "^3.1.4", + "@types/node": "*", + "@types/responselike": "^1.0.0" + } + }, + "node_modules/@types/http-cache-semantics": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.1.tgz", + "integrity": "sha512-SZs7ekbP8CN0txVG2xVRH6EgKmEm31BOxA07vkFaETzZz1xh+cbt8BcI0slpymvwhx5dlFnQG2rTlPVQn+iRPQ==" + }, + "node_modules/@types/keyv": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.4.tgz", + "integrity": "sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/node": { + "version": "18.11.18", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.18.tgz", + "integrity": "sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==" + }, + "node_modules/@types/responselike": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.0.tgz", + "integrity": "sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/bootstrap": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-4.4.1.tgz", + "integrity": "sha512-tbx5cHubwE6e2ZG7nqM3g/FZ5PQEDMWmMGNrCUBVRPHXTJaH7CBDdsLeu3eCh3B1tzAxTnAbtmrzvWEvT2NNEA==", + "engines": { + "node": ">=6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/bootstrap" + }, + "peerDependencies": { + "jquery": "1.9.1 - 3", + "popper.js": "^1.16.0" + } + }, + "node_modules/bootstrap-icons": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/bootstrap-icons/-/bootstrap-icons-1.11.3.tgz", + "integrity": "sha512-+3lpHrCw/it2/7lBL15VR0HEumaBss0+f/Lb6ZvHISn1mlK83jjFpooTLsMWbIjJMDjDjOExMsTxnXSIT4k4ww==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/twbs" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/bootstrap" + } + ], + "license": "MIT" + }, + "node_modules/cacheable-lookup": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz", + "integrity": "sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA==", + "engines": { + "node": ">=10.6.0" + } + }, + "node_modules/cacheable-request": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.2.tgz", + "integrity": "sha512-pouW8/FmiPQbuGpkXQ9BAPv/Mo5xDGANgSNXzTzJ8DrKGuXOssM4wIQRjfanNRh3Yu5cfYPvcorqbhg2KIJtew==", + "dependencies": { + "clone-response": "^1.0.2", + "get-stream": "^5.1.0", + "http-cache-semantics": "^4.0.0", + "keyv": "^4.0.0", + "lowercase-keys": "^2.0.0", + "normalize-url": "^6.0.1", + "responselike": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/clone-response": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.3.tgz", + "integrity": "sha512-ROoL94jJH2dUVML2Y/5PEDNaSHgeOdSDicUyS7izcF63G6sTc/FTjLub4b8Il9S8S0beOfYt0TaA5qvFK+w0wA==", + "dependencies": { + "mimic-response": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/decompress-response/node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/defer-to-connect": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz", + "integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==", + "engines": { + "node": ">=10" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/got": { + "version": "11.8.6", + "resolved": "https://registry.npmjs.org/got/-/got-11.8.6.tgz", + "integrity": "sha512-6tfZ91bOr7bOXnK7PRDCGBLa1H4U080YHNaAQ2KsMGlLEzRbk44nsZF2E1IeRc3vtJHPVbKCYgdFbaGO2ljd8g==", + "dependencies": { + "@sindresorhus/is": "^4.0.0", + "@szmarczak/http-timer": "^4.0.5", + "@types/cacheable-request": "^6.0.1", + "@types/responselike": "^1.0.0", + "cacheable-lookup": "^5.0.3", + "cacheable-request": "^7.0.2", + "decompress-response": "^6.0.0", + "http2-wrapper": "^1.0.0-beta.5.2", + "lowercase-keys": "^2.0.0", + "p-cancelable": "^2.0.0", + "responselike": "^2.0.0" + }, + "engines": { + "node": ">=10.19.0" + }, + "funding": { + "url": "https://github.com/sindresorhus/got?sponsor=1" + } + }, + "node_modules/http-cache-semantics": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz", + "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==" + }, + "node_modules/http2-wrapper": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-1.0.3.tgz", + "integrity": "sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg==", + "dependencies": { + "quick-lru": "^5.1.1", + "resolve-alpn": "^1.0.0" + }, + "engines": { + "node": ">=10.19.0" + } + }, + "node_modules/jquery": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.5.0.tgz", + "integrity": "sha512-Xb7SVYMvygPxbFMpTFQiHh1J7HClEaThguL15N/Gg37Lri/qKyhRGZYzHRyLH8Stq3Aow0LsHO2O2ci86fCrNQ==" + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==" + }, + "node_modules/keyv": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.2.tgz", + "integrity": "sha512-5MHbFaKn8cNSmVW7BYnijeAVlE4cYA/SVkifVgrh7yotnfhKmjuXpDKjrABLnT0SfHWV21P8ow07OGfRrNDg8g==", + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/lowercase-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", + "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==", + "engines": { + "node": ">=8" + } + }, + "node_modules/mimic-response": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", + "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", + "engines": { + "node": ">=4" + } + }, + "node_modules/normalize-url": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz", + "integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/p-cancelable": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.1.tgz", + "integrity": "sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/quick-lru": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", + "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/resolve-alpn": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz", + "integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==" + }, + "node_modules/responselike": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.1.tgz", + "integrity": "sha512-4gl03wn3hj1HP3yzgdI7d3lCkF95F21Pz4BPGvKHinyQzALR5CapwC8yIi0Rh58DEMQ/SguC03wFj2k0M/mHhw==", + "dependencies": { + "lowercase-keys": "^2.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/vue": { + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/vue/-/vue-2.6.11.tgz", + "integrity": "sha512-VfPwgcGABbGAue9+sfrD4PuwFar7gPb1yl1UK1MwXoQPAw0BKSqWfoYCT/ThFrdEVWoI51dBuyCoiNU9bZDZxQ==", + "deprecated": "Vue 2 has reached EOL and is no longer actively maintained. See https://v2.vuejs.org/eol/ for more details." + }, + "node_modules/vue-resource": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/vue-resource/-/vue-resource-1.5.3.tgz", + "integrity": "sha512-REhTuEuYSpwmEH/VN4fgDQVC/VXxDK/xsguuiDPnINxOwy1s0CSu//p++osTUkiAXi6d/vptwBpb0AcBIDsXzw==", + "dependencies": { + "got": ">=8.0 <12.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + } + } +} diff --git a/meshping/ui/package.json b/meshping/ui/package.json new file mode 100644 index 0000000..8a7a105 --- /dev/null +++ b/meshping/ui/package.json @@ -0,0 +1,18 @@ +{ + "name": "meshping-ui", + "version": "0.0.1", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "MIT", + "dependencies": { + "bootstrap": "^4.4.1", + "bootstrap-icons": "^1.11.3", + "jquery": "^3.5.0", + "vue": "^2.6.11", + "vue-resource": "^1.5.3" + } +} diff --git a/meshping/ui/src/main.js b/meshping/ui/src/main.js new file mode 100644 index 0000000..3ca4097 --- /dev/null +++ b/meshping/ui/src/main.js @@ -0,0 +1,184 @@ +window.app = new Vue({ + el: '#app', + data: { + hostname: window.meshping_hostname, + error_msg: "", + success_msg: "", + last_update: 0, + search: localStorage.getItem("meshping_search") || "", + targets_all: [], + targets_filtered: [], + add_tgt_name: "", + add_tgt_addr: "", + comparing: false, + creating: false, + route_target: {name: "", "traceroute": []}, + }, + methods: { + update_targets: async function () { + var response = await this.$http.get('/api/targets'); + var json = await response.json(); + this.targets_all = json.targets; + this.last_update = new Date(); + }, + reapply_filters: function() { + if( this.search === "" ){ + // Make a copy of the array, or else chrome goes 100% CPU in sort() :o + this.targets_filtered = this.targets_all.slice(); + } else { + var search = this.search.toLowerCase(); + this.targets_filtered = this.targets_all.filter(function(target){ + return ( + target.name.toLowerCase().includes(search) || + target.addr.includes(search) + ); + }); + } + var ip_as_filled_str = function(ipaddr) { + if (!ipaddr.includes(":")) { + // IPv4 + return (ipaddr + .split(".") + .map(x => x.toString().padStart(3, "0")) + .join("") + ); + } else { + // IPv6 + return (ipaddr + .split(":") + .map(x => x.toString().padStart(4, "0")) + .join("") + ); + } + } + this.targets_filtered.sort(function(a, b){ + return ip_as_filled_str(a.addr).localeCompare(ip_as_filled_str(b.addr)); + }); + }, + delete_target: async function(target) { + var message = `Delete target ${target.name} (${target.addr})?`; + if (confirm(message)) { + var response = await this.$http.delete(`/api/targets/${target.addr}`); + var json = await response.json(); + if (json.success) { + this.show_success(`Success! Deleted target ${target.name} (${target.addr}).`); + this.update_targets(); + } + } + }, + create_target: async function() { + this.creating = true; + var target_str = this.add_tgt_name; + if (this.add_tgt_addr !== "") { + target_str += "@" + this.add_tgt_addr; + } + var response = await this.$http.post('/api/targets', { + "target": target_str + }); + var json = await response.json(); + this.creating = false; + if (json.success) { + this.add_tgt_name = ""; + this.add_tgt_addr = ""; + this.show_success( + "Success! Added targets: " + ); + this.update_targets(); + } else { + this.show_error( + `Error! Could not add target ${json.target}: ` + + json.error + ); + setTimeout(() => $('#add_tgt_name').focus(), 50); + } + }, + clear_stats: async function() { + var response = await this.$http.delete('/api/stats'); + var json = await response.json(); + if (json.success) { + this.show_success( + "Success!Stats are cleared." + ); + this.update_targets(); + } + }, + show_success: function(msg) { + this.success_msg = msg; + setTimeout(function(vue){ + vue.success_msg = ""; + }, 5000, this); + }, + show_error: function(msg) { + this.error_msg = msg; + setTimeout(function(vue){ + vue.error_msg = ""; + }, 5000, this); + }, + on_btn_compare: function() { + if (!this.comparing) { + this.comparing = true; + this.success_msg = "Select targets, then press compare again"; + } else { + var compare_targets = $("input[name=compare_target]:checked").map((_, el) => el.value).toArray(); + if (compare_targets.length == 0) { + this.show_error("Please select a few targets to compare."); + } else if (compare_targets.length > 3) { + this.show_error("We can only compare up to three targets at once."); + } else { + this.success_msg = ""; + this.comparing = false; + window.open( + `/histogram/${this.hostname}/${compare_targets[0]}.png?` + + compare_targets.slice(1).map(el => `compare=${el}`).join('&') + ); + } + } + }, + show_route_for_target: function(target) { + this.route_target = target; + $('#routeModal').modal("show"); + }, + target_from_route_hop: function(hop) { + this.add_tgt_name = hop.name; + this.add_tgt_addr = hop.address; + $('#routeModal').modal("hide"); + } + }, + created: function() { + var self = this; + window.setInterval(function(vue){ + if( new Date() - vue.last_update > 29500 ){ + vue.update_targets(); + } + }, 1000, this); + $(window).keydown(function(ev){ + if (ev.ctrlKey && ev.key === "f") { + ev.preventDefault(); + $("#inpsearch").focus(); + } + else if (ev.key === "Escape") { + $("#inpsearch").blur(); + self.search = ""; + } + }); + }, + watch: { + search: function(search) { + localStorage.setItem("meshping_search", search); + this.reapply_filters(); + }, + targets_all: function() { + this.reapply_filters(); + } + }, + filters: { + prettyFloat: function(value) { + if (value === undefined || typeof value.toFixed !== 'function') { + return '—'; + } + return value.toFixed(2); + } + } +}); diff --git a/requirements.txt b/requirements.txt index 80ce434..43e09a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,7 @@ ipwhois>=1.2.0 netifaces netaddr packaging + + +Django +Jinja2 \ No newline at end of file From 9463062c41308891d8b7c2526b5ed80a15c09ca9 Mon Sep 17 00:00:00 2001 From: liquid-metal <56035107+liquid-metal@users.noreply.github.com> Date: Fri, 4 Apr 2025 17:34:59 +0200 Subject: [PATCH 02/19] change statistics model and fix icon rendering in template --- ...ics_field_remove_statistics_id_and_more.py | 81 +++++++++++++ .../0003_rename_avg5m_statistics_avg15m.py | 18 +++ meshping/meshping/models.py | 16 ++- meshping/meshping/urls.py | 1 + meshping/meshping/views.py | 107 ++++++++++++++++-- 5 files changed, 210 insertions(+), 13 deletions(-) create mode 100644 meshping/meshping/migrations/0002_remove_statistics_field_remove_statistics_id_and_more.py create mode 100644 meshping/meshping/migrations/0003_rename_avg5m_statistics_avg15m.py diff --git a/meshping/meshping/migrations/0002_remove_statistics_field_remove_statistics_id_and_more.py b/meshping/meshping/migrations/0002_remove_statistics_field_remove_statistics_id_and_more.py new file mode 100644 index 0000000..c696471 --- /dev/null +++ b/meshping/meshping/migrations/0002_remove_statistics_field_remove_statistics_id_and_more.py @@ -0,0 +1,81 @@ +# Generated by Django 5.1.7 on 2025-04-02 20:33 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('meshping', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='statistics', + name='field', + ), + migrations.RemoveField( + model_name='statistics', + name='id', + ), + migrations.RemoveField( + model_name='statistics', + name='value', + ), + migrations.AddField( + model_name='statistics', + name='avg24h', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='statistics', + name='avg5m', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='statistics', + name='avg6h', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='statistics', + name='last', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='statistics', + name='lost', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='statistics', + name='max', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='statistics', + name='min', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='statistics', + name='recv', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='statistics', + name='sent', + field=models.FloatField(default=0.0), + ), + migrations.AddField( + model_name='statistics', + name='sum', + field=models.FloatField(default=0.0), + ), + migrations.AlterField( + model_name='statistics', + name='target', + field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='meshping.target'), + ), + ] diff --git a/meshping/meshping/migrations/0003_rename_avg5m_statistics_avg15m.py b/meshping/meshping/migrations/0003_rename_avg5m_statistics_avg15m.py new file mode 100644 index 0000000..4aabbf7 --- /dev/null +++ b/meshping/meshping/migrations/0003_rename_avg5m_statistics_avg15m.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.7 on 2025-04-04 14:44 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('meshping', '0002_remove_statistics_field_remove_statistics_id_and_more'), + ] + + operations = [ + migrations.RenameField( + model_name='statistics', + old_name='avg5m', + new_name='avg15m', + ), + ] diff --git a/meshping/meshping/models.py b/meshping/meshping/models.py index aa094f2..28f9a01 100644 --- a/meshping/meshping/models.py +++ b/meshping/meshping/models.py @@ -18,12 +18,18 @@ class Histogram(models.Model): count = models.IntegerField(default=1) -# TODO decide on max_length, even though ignored by sqlite -# TODO uniqueness constraint `UNIQUE (target_id, field)` class Statistics(models.Model): - target = models.ForeignKey(Target, on_delete=models.CASCADE) - field = models.CharField(max_length=255) - value = models.FloatField() + target = models.OneToOneField(Target, on_delete=models.CASCADE, primary_key=True) + sent = models.FloatField(default=0.0) + lost = models.FloatField(default=0.0) + recv = models.FloatField(default=0.0) + sum = models.FloatField(default=0.0) + last = models.FloatField(default=0.0) + max = models.FloatField(default=0.0) + min = models.FloatField(default=0.0) + avg15m = models.FloatField(default=0.0) + avg6h = models.FloatField(default=0.0) + avg24h = models.FloatField(default=0.0) # TODO check if the CharField maps to SQLite TEXT + decide on max_length diff --git a/meshping/meshping/urls.py b/meshping/meshping/urls.py index d8757d0..c7ad991 100644 --- a/meshping/meshping/urls.py +++ b/meshping/meshping/urls.py @@ -4,4 +4,5 @@ urlpatterns = [ path('', views.index, name='index'), + path('api/targets', views.targets, name='targets'), ] diff --git a/meshping/meshping/views.py b/meshping/meshping/views.py index 8c5d76b..17111e5 100644 --- a/meshping/meshping/views.py +++ b/meshping/meshping/views.py @@ -1,12 +1,15 @@ import os import socket -from django.http import HttpResponse +from django.http import HttpResponse, JsonResponse +from django.views.decorators.http import require_http_methods from django.template import loader from markupsafe import Markup -from .models import Target +from .models import Statistics, Target + +# route / # TODO do not load icons from disk for every request # TODO find a better method for finding the icons to not have an absolute path here def index(request): @@ -15,14 +18,102 @@ def index(request): icons_dir = "../ui/node_modules/bootstrap-icons/icons/" icons_dir = os.path.join(os.path.dirname(__file__), icons_dir) print(icons_dir) - icons = dict( - icons={ - filename: Markup(open(os.path.join(icons_dir, filename), "r").read()) - for filename in os.listdir(icons_dir) - } - ) + icons = { + filename: Markup(open(os.path.join(icons_dir, filename), "r").read()) + for filename in os.listdir(icons_dir) + } context = { "Hostname": socket.gethostname(), "icons": icons, } return HttpResponse(template.render(context, request)) + + +# route /api/targets +@require_http_methods(["GET", "POST"]) +def targets(request): + if request.method == "GET": + targets = [] + + for target in Target.objects.all(): + target_stats = Statistics.objects.filter(target=target).values()[0] + if not target_stats: + target_stats = { + "sent": 0, "lost": 0, "recv": 0, "sum": 0 + } + succ = 0 + loss = 0 + if target_stats["sent"] > 0: + succ = target_stats["recv"] / target_stats["sent"] * 100 + loss = (target_stats["sent"] - target_stats["recv"]) / target_stats["sent"] * 100 + targets.append( + dict( + target_stats, + addr=target.addr, + name=target.name, +# state=target.state, +# error=target.error, + succ=succ, + loss=loss, +# traceroute=target.traceroute, +# route_loop=target.route_loop, + ) + ) + + return JsonResponse({'targets': targets}) + elif request.method == "POST": + pass + +#@app.route("/api/targets", methods=["GET", "POST"]) +# async def targets(): +# if request.method == "GET": +# targets = [] +# +# for target in mp.all_targets(): +# target_stats = target.statistics +# succ = 0 +# loss = 0 +# if target_stats["sent"] > 0: +# succ = target_stats["recv"] / target_stats["sent"] * 100 +# loss = (target_stats["sent"] - target_stats["recv"]) / target_stats["sent"] * 100 +# targets.append( +# dict( +# target_stats, +# addr=target.addr, +# name=target.name, +# state=target.state, +# error=target.error, +# succ=succ, +# loss=loss, +# traceroute=target.traceroute, +# route_loop=target.route_loop, +# ) +# ) +# +# return jsonify(success=True, targets=targets) +# +# if request.method == "POST": +# request_json = await request.get_json() +# if "target" not in request_json: +# return "missing target", 400 +# +# target = request_json["target"] +# added = [] +# +# if "@" not in target: +# try: +# addrinfo = socket.getaddrinfo(target, 0, 0, socket.SOCK_STREAM) +# except socket.gaierror as err: +# return jsonify(success=False, target=target, error=str(err)) +# +# for info in addrinfo: +# target_with_addr = "%s@%s" % (target, info[4][0]) +# mp.add_target(target_with_addr) +# added.append(target_with_addr) +# else: +# mp.add_target(target) +# added.append(target) +# +# return jsonify(success=True, targets=added) +# +# abort(400) From 038ace0b303c2f2b4d8a3f29acce18121e3bb294 Mon Sep 17 00:00:00 2001 From: liquid-metal <56035107+liquid-metal@users.noreply.github.com> Date: Fri, 4 Apr 2025 18:16:15 +0200 Subject: [PATCH 03/19] adding targets via api --- meshping/meshping/views.py | 96 +++++++++++++++----------------------- 1 file changed, 38 insertions(+), 58 deletions(-) diff --git a/meshping/meshping/views.py b/meshping/meshping/views.py index 17111e5..e9b6724 100644 --- a/meshping/meshping/views.py +++ b/meshping/meshping/views.py @@ -1,7 +1,8 @@ +import json import os import socket -from django.http import HttpResponse, JsonResponse +from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse from django.views.decorators.http import require_http_methods from django.template import loader from markupsafe import Markup @@ -12,12 +13,12 @@ # route / # TODO do not load icons from disk for every request # TODO find a better method for finding the icons to not have an absolute path here +# TODO make local development possible when node modules are on disk (relative path) def index(request): template = loader.get_template('index.html.j2') # icons_dir = "/opt/meshping/ui/node_modules/bootstrap-icons/icons/" icons_dir = "../ui/node_modules/bootstrap-icons/icons/" icons_dir = os.path.join(os.path.dirname(__file__), icons_dir) - print(icons_dir) icons = { filename: Markup(open(os.path.join(icons_dir, filename), "r").read()) for filename in os.listdir(icons_dir) @@ -30,17 +31,24 @@ def index(request): # route /api/targets +# TODO add state to response for each target +# TODO add error to response for each target +# TODO add traceroute to response for each target +# TODO add route_loop to response for each target +# TODO do not crash when the uniqueness constraint is not met for new targets @require_http_methods(["GET", "POST"]) def targets(request): if request.method == "GET": targets = [] for target in Target.objects.all(): - target_stats = Statistics.objects.filter(target=target).values()[0] + target_stats = Statistics.objects.filter(target=target).values() if not target_stats: target_stats = { "sent": 0, "lost": 0, "recv": 0, "sum": 0 } + else: + target_stats = target_stats[0] succ = 0 loss = 0 if target_stats["sent"] > 0: @@ -61,59 +69,31 @@ def targets(request): ) return JsonResponse({'targets': targets}) - elif request.method == "POST": - pass -#@app.route("/api/targets", methods=["GET", "POST"]) -# async def targets(): -# if request.method == "GET": -# targets = [] -# -# for target in mp.all_targets(): -# target_stats = target.statistics -# succ = 0 -# loss = 0 -# if target_stats["sent"] > 0: -# succ = target_stats["recv"] / target_stats["sent"] * 100 -# loss = (target_stats["sent"] - target_stats["recv"]) / target_stats["sent"] * 100 -# targets.append( -# dict( -# target_stats, -# addr=target.addr, -# name=target.name, -# state=target.state, -# error=target.error, -# succ=succ, -# loss=loss, -# traceroute=target.traceroute, -# route_loop=target.route_loop, -# ) -# ) -# -# return jsonify(success=True, targets=targets) -# -# if request.method == "POST": -# request_json = await request.get_json() -# if "target" not in request_json: -# return "missing target", 400 -# -# target = request_json["target"] -# added = [] -# -# if "@" not in target: -# try: -# addrinfo = socket.getaddrinfo(target, 0, 0, socket.SOCK_STREAM) -# except socket.gaierror as err: -# return jsonify(success=False, target=target, error=str(err)) -# -# for info in addrinfo: -# target_with_addr = "%s@%s" % (target, info[4][0]) -# mp.add_target(target_with_addr) -# added.append(target_with_addr) -# else: -# mp.add_target(target) -# added.append(target) -# -# return jsonify(success=True, targets=added) -# -# abort(400) + elif request.method == "POST": + request_json = json.loads(request.body) + if "target" not in request_json: + return HttpResponseBadRequest("missing target") + target = request_json["target"] + added = [] + if "@" not in target: + try: + addrinfo = socket.getaddrinfo(target, 0, 0, socket.SOCK_STREAM) + except socket.gaierror as err: + return JsonResponse({ + 'success': False, + 'target': target, + 'error': str(err), + }) + for info in addrinfo: + addr = info[4][0] + Target(name=target, addr=addr).save() + added.append(f"{target}@{addr}") + else: + tname, addr = target.split('@') + Target(name=tname, addr=addr).save() + added.append(target) + return JsonResponse({ + 'success': True, + 'targets': added, + }) From ac780742e27960279dc97e49405636b72fadad99 Mon Sep 17 00:00:00 2001 From: liquid-metal <56035107+liquid-metal@users.noreply.github.com> Date: Fri, 4 Apr 2025 19:14:11 +0200 Subject: [PATCH 04/19] stubs for all api endpoints --- meshping/meshping/urls.py | 9 +++++++++ meshping/meshping/views.py | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/meshping/meshping/urls.py b/meshping/meshping/urls.py index c7ad991..a5772ca 100644 --- a/meshping/meshping/urls.py +++ b/meshping/meshping/urls.py @@ -2,7 +2,16 @@ from . import views +# TODO decide about and deal with caching on /ui resources (django staticfiles) urlpatterns = [ path('', views.index, name='index'), + path('histogram//.png', views.histogram, name='histogram'), + path('metrics', views.metrics, name='metrics'), + path('network.svg', views.network, name='network'), + path('peer', views.peer, name='peer'), + + path('api/resolve/', views.resolve, name='resolve'), + path('api/stats', views.stats, name='stats'), path('api/targets', views.targets, name='targets'), + path('api/targets/', views.edit_target, name='target'), ] diff --git a/meshping/meshping/views.py b/meshping/meshping/views.py index e9b6724..96544a4 100644 --- a/meshping/meshping/views.py +++ b/meshping/meshping/views.py @@ -2,7 +2,7 @@ import os import socket -from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse +from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseServerError, JsonResponse from django.views.decorators.http import require_http_methods from django.template import loader from markupsafe import Markup @@ -30,6 +30,36 @@ def index(request): return HttpResponse(template.render(context, request)) +# route /histogram//.png +def histogram(request, **kwargs): + return HttpResponseServerError('not implemented') + + +# route /metrics +def metrics(request): + return HttpResponseServerError('not implemented') + + +# route /network.svg +def network(request): + return HttpResponseServerError('not implemented') + + +# route /peer +def peer(request): + return HttpResponseServerError('not implemented') + + +# route /api/resolve/ +def resolve(request, **kwargs): + return HttpResponseServerError('not implemented') + + +# route /api/stats +def stats(request): + return HttpResponseServerError('not implemented') + + # route /api/targets # TODO add state to response for each target # TODO add error to response for each target @@ -97,3 +127,8 @@ def targets(request): 'success': True, 'targets': added, }) + + +# route /api/targets/ +def edit_target(request, **kwargs): + return HttpResponseServerError('not implemented') From 1ee340a47bab329a52c2b59bd34daf3f9ac9050c Mon Sep 17 00:00:00 2001 From: liquid-metal <56035107+liquid-metal@users.noreply.github.com> Date: Fri, 4 Apr 2025 19:38:33 +0200 Subject: [PATCH 05/19] format code with the black formatter --- meshping/manage.py | 4 +- meshping/meshping/apps.py | 4 +- meshping/meshping/asgi.py | 11 +--- meshping/meshping/jinja2.py | 21 ++++--- meshping/meshping/models.py | 22 +++---- meshping/meshping/settings.py | 78 +++++++++++++------------ meshping/meshping/urls.py | 19 +++--- meshping/meshping/views.py | 105 +++++++++++++++++++--------------- meshping/meshping/wsgi.py | 11 +--- pyproject.toml | 2 + 10 files changed, 136 insertions(+), 141 deletions(-) create mode 100644 pyproject.toml diff --git a/meshping/manage.py b/meshping/manage.py index 324b250..e96c666 100755 --- a/meshping/manage.py +++ b/meshping/manage.py @@ -6,7 +6,7 @@ def main(): """Run administrative tasks.""" - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'meshping.settings') + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "meshping.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: @@ -18,5 +18,5 @@ def main(): execute_from_command_line(sys.argv) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/meshping/meshping/apps.py b/meshping/meshping/apps.py index 6b2b4a8..16c5a34 100644 --- a/meshping/meshping/apps.py +++ b/meshping/meshping/apps.py @@ -2,8 +2,8 @@ class MeshpingConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'meshping' + default_auto_field = "django.db.models.BigAutoField" + name = "meshping" def ready(self): # TODO start background threads for ping, traceroute, peers diff --git a/meshping/meshping/asgi.py b/meshping/meshping/asgi.py index 1afae58..4b48dcb 100644 --- a/meshping/meshping/asgi.py +++ b/meshping/meshping/asgi.py @@ -1,16 +1,7 @@ -""" -ASGI config for meshping project. - -It exposes the ASGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/5.1/howto/deployment/asgi/ -""" - import os from django.core.asgi import get_asgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'meshping.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "meshping.settings") application = get_asgi_application() diff --git a/meshping/meshping/jinja2.py b/meshping/meshping/jinja2.py index f575cba..aa087bd 100644 --- a/meshping/meshping/jinja2.py +++ b/meshping/meshping/jinja2.py @@ -1,15 +1,14 @@ -from jinja2 import Environment -from django.urls import reverse +from jinja2 import Environment +from django.urls import reverse from django.templatetags.static import static -def environment(**options): - env = Environment( - variable_start_string='{[', - variable_end_string=']}', - **options) - env.globals.update({ - "static": static, - "url": reverse - }) +def environment(**options): + env = Environment(variable_start_string="{[", variable_end_string="]}", **options) + env.globals.update( + { + "static": static, + "url": reverse, + } + ) return env diff --git a/meshping/meshping/models.py b/meshping/meshping/models.py index 28f9a01..5647881 100644 --- a/meshping/meshping/models.py +++ b/meshping/meshping/models.py @@ -7,7 +7,7 @@ class Target(models.Model): name = models.CharField(max_length=255) class Meta: - unique_together = ('addr', 'name') + unique_together = ("addr", "name") # TODO uniqueness constraint `UNIQUE (target_id, timestamp, bucket)` @@ -20,16 +20,16 @@ class Histogram(models.Model): class Statistics(models.Model): target = models.OneToOneField(Target, on_delete=models.CASCADE, primary_key=True) - sent = models.FloatField(default=0.0) - lost = models.FloatField(default=0.0) - recv = models.FloatField(default=0.0) - sum = models.FloatField(default=0.0) - last = models.FloatField(default=0.0) - max = models.FloatField(default=0.0) - min = models.FloatField(default=0.0) - avg15m = models.FloatField(default=0.0) - avg6h = models.FloatField(default=0.0) - avg24h = models.FloatField(default=0.0) + sent = models.FloatField(default=0.0) + lost = models.FloatField(default=0.0) + recv = models.FloatField(default=0.0) + sum = models.FloatField(default=0.0) + last = models.FloatField(default=0.0) + max = models.FloatField(default=0.0) + min = models.FloatField(default=0.0) + avg15m = models.FloatField(default=0.0) + avg6h = models.FloatField(default=0.0) + avg24h = models.FloatField(default=0.0) # TODO check if the CharField maps to SQLite TEXT + decide on max_length diff --git a/meshping/meshping/settings.py b/meshping/meshping/settings.py index be9ec64..8e1b9a1 100644 --- a/meshping/meshping/settings.py +++ b/meshping/meshping/settings.py @@ -4,21 +4,21 @@ BASE_DIR = Path(__file__).resolve().parent.parent -SECRET_KEY = 'django-insecure-2svcr!j9%^_(^)$%)vvt1j&0$w)&_l&oxacnb1mkurd4f--xmg' +SECRET_KEY = "django-insecure-2svcr!j9%^_(^)$%)vvt1j&0$w)&_l&oxacnb1mkurd4f--xmg" DEBUG = True -ALLOWED_HOSTS = ['*'] +ALLOWED_HOSTS = ["*"] INSTALLED_APPS = [ -# 'django.contrib.admin', -# 'django.contrib.auth', -# 'django.contrib.contenttypes', -# 'django.contrib.sessions', -# 'django.contrib.messages', - 'django.contrib.staticfiles', - 'meshping', + # 'django.contrib.admin', + # 'django.contrib.auth', + # 'django.contrib.contenttypes', + # 'django.contrib.sessions', + # 'django.contrib.messages', + "django.contrib.staticfiles", + "meshping", ] -#MIDDLEWARE = [ +# MIDDLEWARE = [ # 'django.middleware.security.SecurityMiddleware', # 'django.contrib.sessions.middleware.SessionMiddleware', # 'django.middleware.common.CommonMiddleware', @@ -26,48 +26,50 @@ # 'django.contrib.auth.middleware.AuthenticationMiddleware', # 'django.contrib.messages.middleware.MessageMiddleware', # 'django.middleware.clickjacking.XFrameOptionsMiddleware', -#] +# ] -ROOT_URLCONF = 'meshping.urls' +ROOT_URLCONF = "meshping.urls" # TODO the APP_DIRS seem to not work as I imagined for the jinja2 backend TEMPLATES = [ { "BACKEND": "django.template.backends.jinja2.Jinja2", - "DIRS": [os.path.join(BASE_DIR, 'meshping/templates'),], + "DIRS": [ + os.path.join(BASE_DIR, "meshping/templates"), + ], "APP_DIRS": False, "OPTIONS": { "environment": "meshping.jinja2.environment", }, }, -# { -# 'BACKEND': 'django.template.backends.django.DjangoTemplates', -# 'DIRS': [], -# 'APP_DIRS': True, -# 'OPTIONS': { -# 'context_processors': [ -# 'django.template.context_processors.debug', -# 'django.template.context_processors.request', -## 'django.contrib.auth.context_processors.auth', -## 'django.contrib.messages.context_processors.messages', -# ], -# }, -# }, + # { + # 'BACKEND': 'django.template.backends.django.DjangoTemplates', + # 'DIRS': [], + # 'APP_DIRS': True, + # 'OPTIONS': { + # 'context_processors': [ + # 'django.template.context_processors.debug', + # 'django.template.context_processors.request', + ## 'django.contrib.auth.context_processors.auth', + ## 'django.contrib.messages.context_processors.messages', + # ], + # }, + # }, ] -#WSGI_APPLICATION = 'meshping.wsgi.application' +# WSGI_APPLICATION = 'meshping.wsgi.application' # Database # https://docs.djangoproject.com/en/5.1/ref/settings/#databases DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': BASE_DIR / 'db' / 'db.sqlite3', + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": BASE_DIR / "db" / "db.sqlite3", } } -#AUTH_PASSWORD_VALIDATORS = [ +# AUTH_PASSWORD_VALIDATORS = [ # { # 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', # }, @@ -80,16 +82,16 @@ # { # 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', # }, -#] +# ] -#LANGUAGE_CODE = 'en-us' -#TIME_ZONE = 'UTC' -#USE_I18N = True -#USE_TZ = True +# LANGUAGE_CODE = 'en-us' +# TIME_ZONE = 'UTC' +# USE_I18N = True +# USE_TZ = True -STATIC_URL = 'ui/' +STATIC_URL = "ui/" STATICFILES_DIRS = [ BASE_DIR / "ui", ] -DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" diff --git a/meshping/meshping/urls.py b/meshping/meshping/urls.py index a5772ca..142e453 100644 --- a/meshping/meshping/urls.py +++ b/meshping/meshping/urls.py @@ -4,14 +4,13 @@ # TODO decide about and deal with caching on /ui resources (django staticfiles) urlpatterns = [ - path('', views.index, name='index'), - path('histogram//.png', views.histogram, name='histogram'), - path('metrics', views.metrics, name='metrics'), - path('network.svg', views.network, name='network'), - path('peer', views.peer, name='peer'), - - path('api/resolve/', views.resolve, name='resolve'), - path('api/stats', views.stats, name='stats'), - path('api/targets', views.targets, name='targets'), - path('api/targets/', views.edit_target, name='target'), + path("", views.index, name="index"), + path("histogram//.png", views.histogram, name="histogram"), + path("metrics", views.metrics, name="metrics"), + path("network.svg", views.network, name="network"), + path("peer", views.peer, name="peer"), + path("api/resolve/", views.resolve, name="resolve"), + path("api/stats", views.stats, name="stats"), + path("api/targets", views.targets, name="targets"), + path("api/targets/", views.edit_target, name="target"), ] diff --git a/meshping/meshping/views.py b/meshping/meshping/views.py index 96544a4..47fd58d 100644 --- a/meshping/meshping/views.py +++ b/meshping/meshping/views.py @@ -2,7 +2,12 @@ import os import socket -from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseServerError, JsonResponse +from django.http import ( + HttpResponse, + HttpResponseBadRequest, + HttpResponseServerError, + JsonResponse, +) from django.views.decorators.http import require_http_methods from django.template import loader from markupsafe import Markup @@ -15,7 +20,7 @@ # TODO find a better method for finding the icons to not have an absolute path here # TODO make local development possible when node modules are on disk (relative path) def index(request): - template = loader.get_template('index.html.j2') + template = loader.get_template("index.html.j2") # icons_dir = "/opt/meshping/ui/node_modules/bootstrap-icons/icons/" icons_dir = "../ui/node_modules/bootstrap-icons/icons/" icons_dir = os.path.join(os.path.dirname(__file__), icons_dir) @@ -32,32 +37,32 @@ def index(request): # route /histogram//.png def histogram(request, **kwargs): - return HttpResponseServerError('not implemented') + return HttpResponseServerError("not implemented") # route /metrics def metrics(request): - return HttpResponseServerError('not implemented') + return HttpResponseServerError("not implemented") # route /network.svg def network(request): - return HttpResponseServerError('not implemented') + return HttpResponseServerError("not implemented") # route /peer def peer(request): - return HttpResponseServerError('not implemented') + return HttpResponseServerError("not implemented") # route /api/resolve/ def resolve(request, **kwargs): - return HttpResponseServerError('not implemented') + return HttpResponseServerError("not implemented") # route /api/stats def stats(request): - return HttpResponseServerError('not implemented') + return HttpResponseServerError("not implemented") # route /api/targets @@ -69,36 +74,38 @@ def stats(request): @require_http_methods(["GET", "POST"]) def targets(request): if request.method == "GET": - targets = [] - - for target in Target.objects.all(): - target_stats = Statistics.objects.filter(target=target).values() - if not target_stats: - target_stats = { - "sent": 0, "lost": 0, "recv": 0, "sum": 0 - } - else: - target_stats = target_stats[0] - succ = 0 - loss = 0 - if target_stats["sent"] > 0: - succ = target_stats["recv"] / target_stats["sent"] * 100 - loss = (target_stats["sent"] - target_stats["recv"]) / target_stats["sent"] * 100 - targets.append( - dict( - target_stats, - addr=target.addr, - name=target.name, -# state=target.state, -# error=target.error, - succ=succ, - loss=loss, -# traceroute=target.traceroute, -# route_loop=target.route_loop, - ) + targets = [] + + for target in Target.objects.all(): + target_stats = Statistics.objects.filter(target=target).values() + if not target_stats: + target_stats = {"sent": 0, "lost": 0, "recv": 0, "sum": 0} + else: + target_stats = target_stats[0] + succ = 0 + loss = 0 + if target_stats["sent"] > 0: + succ = target_stats["recv"] / target_stats["sent"] * 100 + loss = ( + (target_stats["sent"] - target_stats["recv"]) + / target_stats["sent"] + * 100 + ) + targets.append( + dict( + target_stats, + addr=target.addr, + name=target.name, + # state=target.state, + # error=target.error, + succ=succ, + loss=loss, + # traceroute=target.traceroute, + # route_loop=target.route_loop, ) + ) - return JsonResponse({'targets': targets}) + return JsonResponse({"targets": targets}) elif request.method == "POST": request_json = json.loads(request.body) @@ -110,25 +117,29 @@ def targets(request): try: addrinfo = socket.getaddrinfo(target, 0, 0, socket.SOCK_STREAM) except socket.gaierror as err: - return JsonResponse({ - 'success': False, - 'target': target, - 'error': str(err), - }) + return JsonResponse( + { + "success": False, + "target": target, + "error": str(err), + } + ) for info in addrinfo: addr = info[4][0] Target(name=target, addr=addr).save() added.append(f"{target}@{addr}") else: - tname, addr = target.split('@') + tname, addr = target.split("@") Target(name=tname, addr=addr).save() added.append(target) - return JsonResponse({ - 'success': True, - 'targets': added, - }) + return JsonResponse( + { + "success": True, + "targets": added, + } + ) # route /api/targets/ def edit_target(request, **kwargs): - return HttpResponseServerError('not implemented') + return HttpResponseServerError("not implemented") diff --git a/meshping/meshping/wsgi.py b/meshping/meshping/wsgi.py index c227ee0..212c147 100644 --- a/meshping/meshping/wsgi.py +++ b/meshping/meshping/wsgi.py @@ -1,16 +1,7 @@ -""" -WSGI config for meshping project. - -It exposes the WSGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/5.1/howto/deployment/wsgi/ -""" - import os from django.core.wsgi import get_wsgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'meshping.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "meshping.settings") application = get_wsgi_application() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9b5d13e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.black] +exclude = 'meshping/.[^/]+/migrations' From d4695b7895323a467066d93318781f648d2d3632 Mon Sep 17 00:00:00 2001 From: liquid-metal <56035107+liquid-metal@users.noreply.github.com> Date: Fri, 4 Apr 2025 20:00:05 +0200 Subject: [PATCH 06/19] make pylint happy --- .pylintrc | 3 ++- meshping/meshping/urls.py | 2 +- meshping/meshping/views.py | 14 +++++++++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.pylintrc b/.pylintrc index d3d930b..29e283f 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,11 +1,12 @@ [MASTER] extension-pkg-whitelist = netifaces +ignore-paths = meshping/.[^/]+/migrations [FORMAT] max-line-length=120 [MESSAGES CONTROL] -disable=E1101,E1102,E1103,C0111,C0103,W0613,W0108,W0212,R0903 +disable=E1101,E1102,E1103,C0111,C0103,W0613,W0108,W0212,R0903,W0511 [DESIGN] max-public-methods=100 diff --git a/meshping/meshping/urls.py b/meshping/meshping/urls.py index 142e453..6e0b69d 100644 --- a/meshping/meshping/urls.py +++ b/meshping/meshping/urls.py @@ -11,6 +11,6 @@ path("peer", views.peer, name="peer"), path("api/resolve/", views.resolve, name="resolve"), path("api/stats", views.stats, name="stats"), - path("api/targets", views.targets, name="targets"), + path("api/targets", views.targets_endpoint, name="targets"), path("api/targets/", views.edit_target, name="target"), ] diff --git a/meshping/meshping/views.py b/meshping/meshping/views.py index 47fd58d..a65ab8f 100644 --- a/meshping/meshping/views.py +++ b/meshping/meshping/views.py @@ -20,12 +20,16 @@ # TODO find a better method for finding the icons to not have an absolute path here # TODO make local development possible when node modules are on disk (relative path) def index(request): + def read_svg_file(icons_dir, filename): + with open(os.path.join(icons_dir, filename), "r", encoding="utf-8") as f: + return Markup(f.read()) + template = loader.get_template("index.html.j2") # icons_dir = "/opt/meshping/ui/node_modules/bootstrap-icons/icons/" icons_dir = "../ui/node_modules/bootstrap-icons/icons/" icons_dir = os.path.join(os.path.dirname(__file__), icons_dir) icons = { - filename: Markup(open(os.path.join(icons_dir, filename), "r").read()) + filename: read_svg_file(icons_dir, filename) for filename in os.listdir(icons_dir) } context = { @@ -66,13 +70,17 @@ def stats(request): # route /api/targets +# +# django ensures a valid http request method, so we do not need a return value +# pylint: disable=inconsistent-return-statements +# # TODO add state to response for each target # TODO add error to response for each target # TODO add traceroute to response for each target # TODO add route_loop to response for each target # TODO do not crash when the uniqueness constraint is not met for new targets @require_http_methods(["GET", "POST"]) -def targets(request): +def targets_endpoint(request): if request.method == "GET": targets = [] @@ -107,7 +115,7 @@ def targets(request): return JsonResponse({"targets": targets}) - elif request.method == "POST": + if request.method == "POST": request_json = json.loads(request.body) if "target" not in request_json: return HttpResponseBadRequest("missing target") From ddc322be35c6d3bd4de265247021a451c4d510cc Mon Sep 17 00:00:00 2001 From: liquid-metal <56035107+liquid-metal@users.noreply.github.com> Date: Fri, 4 Apr 2025 20:08:21 +0200 Subject: [PATCH 07/19] housekeeping --- meshping/meshping/wsgi.py | 7 ------- requirements.txt | 6 +++--- 2 files changed, 3 insertions(+), 10 deletions(-) delete mode 100644 meshping/meshping/wsgi.py diff --git a/meshping/meshping/wsgi.py b/meshping/meshping/wsgi.py deleted file mode 100644 index 212c147..0000000 --- a/meshping/meshping/wsgi.py +++ /dev/null @@ -1,7 +0,0 @@ -import os - -from django.core.wsgi import get_wsgi_application - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "meshping.settings") - -application = get_wsgi_application() diff --git a/requirements.txt b/requirements.txt index 43e09a5..10f339d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,6 @@ netifaces netaddr packaging - -Django -Jinja2 \ No newline at end of file +black~=25.1.0 +Django~=5.1.7 +Jinja2~=3.1.6 \ No newline at end of file From b804ad24b14f090181d3eefdc501098ed1e46882 Mon Sep 17 00:00:00 2001 From: liquid-metal <56035107+liquid-metal@users.noreply.github.com> Date: Sat, 5 Apr 2025 20:03:34 +0200 Subject: [PATCH 08/19] ping thread and fixes for /api/targets --- .gitignore | 1 + .pylintrc | 2 +- meshping/meshping/apps.py | 13 +- meshping/meshping/meshping/__init__.py | 0 meshping/meshping/meshping/meshping_config.py | 8 ++ meshping/meshping/meshping/ping_thread.py | 130 ++++++++++++++++++ ...move_meta_id_remove_meta_value_and_more.py | 65 +++++++++ .../0005_meta_error_alter_meta_state.py | 35 +++++ ...6_alter_meta_lkgt_alter_meta_traceroute.py | 23 ++++ meshping/meshping/models.py | 24 +++- meshping/meshping/settings.py | 53 ------- meshping/meshping/views.py | 21 ++- 12 files changed, 309 insertions(+), 66 deletions(-) create mode 100644 meshping/meshping/meshping/__init__.py create mode 100644 meshping/meshping/meshping/meshping_config.py create mode 100644 meshping/meshping/meshping/ping_thread.py create mode 100644 meshping/meshping/migrations/0004_remove_meta_field_remove_meta_id_remove_meta_value_and_more.py create mode 100644 meshping/meshping/migrations/0005_meta_error_alter_meta_state.py create mode 100644 meshping/meshping/migrations/0006_alter_meta_lkgt_alter_meta_traceroute.py diff --git a/.gitignore b/.gitignore index 0f04a06..39c250f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ python-ping/ +__pycache__ *.pyc oping.c diff --git a/.pylintrc b/.pylintrc index 29e283f..e0de6a4 100644 --- a/.pylintrc +++ b/.pylintrc @@ -3,7 +3,7 @@ extension-pkg-whitelist = netifaces ignore-paths = meshping/.[^/]+/migrations [FORMAT] -max-line-length=120 +max-line-length=88 [MESSAGES CONTROL] disable=E1101,E1102,E1103,C0111,C0103,W0613,W0108,W0212,R0903,W0511 diff --git a/meshping/meshping/apps.py b/meshping/meshping/apps.py index 16c5a34..e6babc0 100644 --- a/meshping/meshping/apps.py +++ b/meshping/meshping/apps.py @@ -1,10 +1,19 @@ from django.apps import AppConfig +from .meshping.meshping_config import MeshpingConfig as MPCOnfig class MeshpingConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "meshping" + # TODO decide for long-term layout: standard threads, async, scheduling library + # TODO start background threads for traceroute, peers + # TODO create background task for db housekeeping (prune_histograms) def ready(self): - # TODO start background threads for ping, traceroute, peers - pass + # delayed import, otherwise we will get AppRegistryNotReady + # pylint: disable=import-outside-toplevel + from .meshping.ping_thread import PingThread + + mp_config = MPCOnfig() + ping_thread = PingThread(mp_config=mp_config, daemon=True) + ping_thread.start() diff --git a/meshping/meshping/meshping/__init__.py b/meshping/meshping/meshping/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/meshping/meshping/meshping/meshping_config.py b/meshping/meshping/meshping/meshping_config.py new file mode 100644 index 0000000..84498d2 --- /dev/null +++ b/meshping/meshping/meshping/meshping_config.py @@ -0,0 +1,8 @@ +# TODO add implementation for values from environment +# TODO decide about option for config file, and automatic combination with docker +# environment variables +class MeshpingConfig: + def __init__(self): + self.ping_timeout = 1 + self.ping_interval = 1 + self.histogram_period = 1 diff --git a/meshping/meshping/meshping/ping_thread.py b/meshping/meshping/meshping/ping_thread.py new file mode 100644 index 0000000..22301e9 --- /dev/null +++ b/meshping/meshping/meshping/ping_thread.py @@ -0,0 +1,130 @@ +import time +import math +from threading import Thread + +# pylint cannot read the definitions in the shared object +# pylint: disable=no-name-in-module +from oping import PingObj, PingError +from ..models import Histogram, Target, Statistics, Meta, TargetState + + +INTERVAL = 30 +FAC_15m = math.exp(-INTERVAL / (15 * 60.0)) +FAC_6h = math.exp(-INTERVAL / (6 * 60 * 60.0)) +FAC_24h = math.exp(-INTERVAL / (24 * 60 * 60.0)) + + +class PingThread(Thread): + def __init__(self, mp_config, *args, **kwargs): + self.mp_config = mp_config + super().__init__(*args, **kwargs) + + @staticmethod + def exp_avg(current_avg, add_value, factor): + if current_avg is None: + return add_value + return (current_avg * factor) + (add_value * (1 - factor)) + + def process_ping_result(self, timestamp, hostinfo): + target = Target.objects.filter(addr=hostinfo["addr"]).first() + # TODO proper error handling instead of assert + assert target is not None + target_stats, _created = Statistics.objects.get_or_create(target=target) + target_meta, _created = Meta.objects.get_or_create(target=target) + + target_stats.sent += 1 + + if hostinfo["latency"] != -1: + target_meta.state = TargetState.UP + target_stats.recv += 1 + target_stats.last = hostinfo["latency"] + target_stats.sum += target_stats.last + target_stats.max = max(target_stats.max, target_stats.last) + target_stats.min = min(target_stats.min, target_stats.last) + target_stats.avg15m = self.exp_avg( + target_stats.avg15m, target_stats.last, FAC_15m + ) + target_stats.avg6h = self.exp_avg( + target_stats.avg6h, target_stats.last, FAC_6h + ) + target_stats.avg24h = self.exp_avg( + target_stats.avg24h, target_stats.last, FAC_24h + ) + + bucket, _created = Histogram.objects.get_or_create( + target=target, + timestamp=timestamp // 3600 * 3600, + bucket=int(math.log(hostinfo["latency"], 2) * 10), + ) + bucket.count += 1 + bucket.save() + + else: + target_meta.state = TargetState.DOWN + target_stats.lost += 1 + + target_stats.save() + target_meta.save() + + def run(self): + pingobj = PingObj() + pingobj.set_timeout(self.mp_config.ping_timeout) + + next_ping = time.time() + 0.1 + + current_targets = set() + + while True: + now = time.time() + next_ping = now + self.mp_config.ping_interval + + # Run DB housekeeping + # TODO has nothing to do with pings, find a better place (probably new + # housekeeping thread) + Histogram.objects.filter( + timestamp__lt=(now - self.mp_config.histogram_period) + ).delete() + + unseen_targets = current_targets.copy() + for target in Target.objects.all(): + if target.addr not in current_targets: + current_targets.add(target.addr) + try: + pingobj.add_host(target.addr.encode("utf-8")) + except PingError as err: + # TODO make this safe, do not just assume the object exists + target_meta = Meta.objects.filter(target=target)[0] + target_meta.state = TargetState.ERROR + target_meta.error = err.args[0].decode("utf-8") + target_meta.save() + if target.addr in unseen_targets: + unseen_targets.remove(target.addr) + + for target_addr in unseen_targets: + current_targets.remove(target_addr) + try: + pingobj.remove_host(target_addr.encode("utf-8")) + except PingError: + # Host probably not there anyway + pass + + # If we don't have any targets, we're done for now -- just sleep + if not current_targets: + time.sleep(max(0, next_ping - time.time())) + continue + + # We do have targets, so first, let's ping them + pingobj.send() + + for hostinfo in pingobj.get_hosts(): + hostinfo["addr"] = hostinfo["addr"].decode("utf-8") + + try: + self.process_ping_result(now, hostinfo) + except LookupError: + # ping takes a while. it's possible that while we were busy, this + # target has been deleted from the DB. If so, forget about it. + if hostinfo["addr"] in current_targets: + current_targets.remove(hostinfo["addr"]) + + time.sleep(max(0, next_ping - time.time())) diff --git a/meshping/meshping/migrations/0004_remove_meta_field_remove_meta_id_remove_meta_value_and_more.py b/meshping/meshping/migrations/0004_remove_meta_field_remove_meta_id_remove_meta_value_and_more.py new file mode 100644 index 0000000..6f9c26f --- /dev/null +++ b/meshping/meshping/migrations/0004_remove_meta_field_remove_meta_id_remove_meta_value_and_more.py @@ -0,0 +1,65 @@ +# Generated by Django 5.1.7 on 2025-04-05 15:42 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("meshping", "0003_rename_avg5m_statistics_avg15m"), + ] + + operations = [ + migrations.RemoveField( + model_name="meta", + name="field", + ), + migrations.RemoveField( + model_name="meta", + name="id", + ), + migrations.RemoveField( + model_name="meta", + name="value", + ), + migrations.AddField( + model_name="meta", + name="lkgt", + field=models.JSONField(default=[], max_length=2048), + ), + migrations.AddField( + model_name="meta", + name="route_loop", + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name="meta", + name="state", + field=models.CharField( + choices=[("up", "Up"), ("down", "Down"), ("unknown", "Unknown")], + default="down", + max_length=10, + ), + ), + migrations.AddField( + model_name="meta", + name="traceroute", + field=models.JSONField(default=[], max_length=2048), + ), + migrations.AlterField( + model_name="meta", + name="target", + field=models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + serialize=False, + to="meshping.target", + ), + ), + migrations.AlterField( + model_name="statistics", + name="min", + field=models.FloatField(default=float("inf")), + ), + ] diff --git a/meshping/meshping/migrations/0005_meta_error_alter_meta_state.py b/meshping/meshping/migrations/0005_meta_error_alter_meta_state.py new file mode 100644 index 0000000..ab4d553 --- /dev/null +++ b/meshping/meshping/migrations/0005_meta_error_alter_meta_state.py @@ -0,0 +1,35 @@ +# Generated by Django 5.1.7 on 2025-04-05 16:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ( + "meshping", + "0004_remove_meta_field_remove_meta_id_remove_meta_value_and_more", + ), + ] + + operations = [ + migrations.AddField( + model_name="meta", + name="error", + field=models.CharField(default=None, max_length=255, null=True), + ), + migrations.AlterField( + model_name="meta", + name="state", + field=models.CharField( + choices=[ + ("up", "Up"), + ("down", "Down"), + ("unknown", "Unknown"), + ("error", "Error"), + ], + default="down", + max_length=10, + ), + ), + ] diff --git a/meshping/meshping/migrations/0006_alter_meta_lkgt_alter_meta_traceroute.py b/meshping/meshping/migrations/0006_alter_meta_lkgt_alter_meta_traceroute.py new file mode 100644 index 0000000..8e6dd61 --- /dev/null +++ b/meshping/meshping/migrations/0006_alter_meta_lkgt_alter_meta_traceroute.py @@ -0,0 +1,23 @@ +# Generated by Django 5.1.7 on 2025-04-05 16:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("meshping", "0005_meta_error_alter_meta_state"), + ] + + operations = [ + migrations.AlterField( + model_name="meta", + name="lkgt", + field=models.JSONField(default=list, max_length=2048), + ), + migrations.AlterField( + model_name="meta", + name="traceroute", + field=models.JSONField(default=list, max_length=2048), + ), + ] diff --git a/meshping/meshping/models.py b/meshping/meshping/models.py index 5647881..de6792b 100644 --- a/meshping/meshping/models.py +++ b/meshping/meshping/models.py @@ -26,15 +26,29 @@ class Statistics(models.Model): sum = models.FloatField(default=0.0) last = models.FloatField(default=0.0) max = models.FloatField(default=0.0) - min = models.FloatField(default=0.0) + min = models.FloatField(default=float("inf")) avg15m = models.FloatField(default=0.0) avg6h = models.FloatField(default=0.0) avg24h = models.FloatField(default=0.0) -# TODO check if the CharField maps to SQLite TEXT + decide on max_length +# pylint: disable=too-many-ancestors +class TargetState(models.TextChoices): + UP = "up" + DOWN = "down" + UNKNOWN = "unknown" + ERROR = "error" + + +# TODO decide on max_length, even though ignored by sqlite # TODO uniqueness constraint `UNIQUE (target_id, field)` class Meta(models.Model): - target = models.ForeignKey(Target, on_delete=models.CASCADE) - field = models.CharField(max_length=255) - value = models.CharField(max_length=255) + target = models.OneToOneField(Target, on_delete=models.CASCADE, primary_key=True) + state = models.CharField( + max_length=10, choices=TargetState.choices, default=TargetState.DOWN + ) + route_loop = models.BooleanField(default=False) + traceroute = models.JSONField(max_length=2048, default=list) + # lkgt = last known good traceroute + lkgt = models.JSONField(max_length=2048, default=list) + error = models.CharField(max_length=255, null=True, default=None) diff --git a/meshping/meshping/settings.py b/meshping/meshping/settings.py index 8e1b9a1..dafc6cb 100644 --- a/meshping/meshping/settings.py +++ b/meshping/meshping/settings.py @@ -9,25 +9,10 @@ ALLOWED_HOSTS = ["*"] INSTALLED_APPS = [ - # 'django.contrib.admin', - # 'django.contrib.auth', - # 'django.contrib.contenttypes', - # 'django.contrib.sessions', - # 'django.contrib.messages', "django.contrib.staticfiles", "meshping", ] -# MIDDLEWARE = [ -# 'django.middleware.security.SecurityMiddleware', -# 'django.contrib.sessions.middleware.SessionMiddleware', -# 'django.middleware.common.CommonMiddleware', -# 'django.middleware.csrf.CsrfViewMiddleware', -# 'django.contrib.auth.middleware.AuthenticationMiddleware', -# 'django.contrib.messages.middleware.MessageMiddleware', -# 'django.middleware.clickjacking.XFrameOptionsMiddleware', -# ] - ROOT_URLCONF = "meshping.urls" # TODO the APP_DIRS seem to not work as I imagined for the jinja2 backend @@ -42,25 +27,7 @@ "environment": "meshping.jinja2.environment", }, }, - # { - # 'BACKEND': 'django.template.backends.django.DjangoTemplates', - # 'DIRS': [], - # 'APP_DIRS': True, - # 'OPTIONS': { - # 'context_processors': [ - # 'django.template.context_processors.debug', - # 'django.template.context_processors.request', - ## 'django.contrib.auth.context_processors.auth', - ## 'django.contrib.messages.context_processors.messages', - # ], - # }, - # }, ] -# WSGI_APPLICATION = 'meshping.wsgi.application' - - -# Database -# https://docs.djangoproject.com/en/5.1/ref/settings/#databases DATABASES = { "default": { @@ -69,26 +36,6 @@ } } -# AUTH_PASSWORD_VALIDATORS = [ -# { -# 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', -# }, -# { -# 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', -# }, -# { -# 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', -# }, -# { -# 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', -# }, -# ] - -# LANGUAGE_CODE = 'en-us' -# TIME_ZONE = 'UTC' -# USE_I18N = True -# USE_TZ = True - STATIC_URL = "ui/" STATICFILES_DIRS = [ BASE_DIR / "ui", diff --git a/meshping/meshping/views.py b/meshping/meshping/views.py index a65ab8f..6df648e 100644 --- a/meshping/meshping/views.py +++ b/meshping/meshping/views.py @@ -12,7 +12,10 @@ from django.template import loader from markupsafe import Markup -from .models import Statistics, Target +from .models import Statistics, Target, Meta + + +# TODO remove business logic in this file # route / @@ -99,17 +102,25 @@ def targets_endpoint(request): / target_stats["sent"] * 100 ) + + # the browser cannot deserialize JSON with Infinity, but this value is + # very comfortable for comparisons, we want to keep it + if target_stats["min"] == float("inf"): + target_stats["min"] = 0 + + target_meta, _created = Meta.objects.get_or_create(target=target) + targets.append( dict( target_stats, addr=target.addr, name=target.name, - # state=target.state, - # error=target.error, + state=target_meta.state, + error=target_meta.error, succ=succ, loss=loss, - # traceroute=target.traceroute, - # route_loop=target.route_loop, + traceroute=target_meta.traceroute, + route_loop=target_meta.route_loop, ) ) From 0445532bdee881433a77601a9ca3ebbee507f80f Mon Sep 17 00:00:00 2001 From: liquid-metal <56035107+liquid-metal@users.noreply.github.com> Date: Sat, 5 Apr 2025 20:51:51 +0200 Subject: [PATCH 09/19] traceroute background task --- meshping/meshping/apps.py | 6 + meshping/meshping/meshping/meshping_config.py | 16 ++- meshping/meshping/meshping/socklib.py | 109 ++++++++++++++++++ .../meshping/meshping/traceroute_thread.py | 108 +++++++++++++++++ 4 files changed, 236 insertions(+), 3 deletions(-) create mode 100644 meshping/meshping/meshping/socklib.py create mode 100644 meshping/meshping/meshping/traceroute_thread.py diff --git a/meshping/meshping/apps.py b/meshping/meshping/apps.py index e6babc0..2770f6a 100644 --- a/meshping/meshping/apps.py +++ b/meshping/meshping/apps.py @@ -8,12 +8,18 @@ class MeshpingConfig(AppConfig): # TODO decide for long-term layout: standard threads, async, scheduling library # TODO start background threads for traceroute, peers + # TODO with the current thread model, django complains about db access before app + # initialization is completed # TODO create background task for db housekeeping (prune_histograms) def ready(self): # delayed import, otherwise we will get AppRegistryNotReady # pylint: disable=import-outside-toplevel from .meshping.ping_thread import PingThread + from .meshping.traceroute_thread import TracerouteThread mp_config = MPCOnfig() ping_thread = PingThread(mp_config=mp_config, daemon=True) + traceroute_thread = TracerouteThread(mp_config=mp_config) + ping_thread.start() + traceroute_thread.start() diff --git a/meshping/meshping/meshping/meshping_config.py b/meshping/meshping/meshping/meshping_config.py index 84498d2..9154641 100644 --- a/meshping/meshping/meshping/meshping_config.py +++ b/meshping/meshping/meshping/meshping_config.py @@ -1,8 +1,18 @@ # TODO add implementation for values from environment # TODO decide about option for config file, and automatic combination with docker # environment variables +# +# pylint: disable=too-many-instance-attributes class MeshpingConfig: def __init__(self): - self.ping_timeout = 1 - self.ping_interval = 1 - self.histogram_period = 1 + self.ping_timeout = 5 + self.ping_interval = 30 + + self.traceroute_interval = 900 + self.traceroute_timeout = 0.5 + self.traceroute_packets = 1 + self.traceroute_ratelimit_interval = 2 + + self.whois_cache_validiy_h = 72 + + self.histogram_period = 3 diff --git a/meshping/meshping/meshping/socklib.py b/meshping/meshping/meshping/socklib.py new file mode 100644 index 0000000..e569186 --- /dev/null +++ b/meshping/meshping/meshping/socklib.py @@ -0,0 +1,109 @@ +import socket + +from icmplib.sockets import ICMPv4Socket, ICMPv6Socket +from icmplib.exceptions import ICMPSocketError, TimeoutExceeded +from icmplib.models import ICMPRequest +from icmplib.utils import unique_identifier + +# see /usr/include/linux/in.h +IP_MTU_DISCOVER = 10 +IP_PMTUDISC_DO = 2 +IP_MTU = 14 +IP_HEADER_LEN = 20 +ICMP_HEADER_LEN = 8 + +# see /usr/include/linux/in6.h +IPV6_MTU_DISCOVER = 23 +IPV6_PMTUDISC_DO = 2 +IPV6_MTU = 24 +IPV6_HEADER_LEN = 40 +ICMPV6_HEADER_LEN = 8 + + +def reverse_lookup(ip): + try: + return socket.gethostbyaddr(ip)[0] + except socket.herror: + return ip + + +class PMTUDv4Socket(ICMPv4Socket): + # pylint: disable=redefined-builtin + def _create_socket(self, type): + sock = super()._create_socket(type) + sock.setsockopt(socket.IPPROTO_IP, IP_MTU_DISCOVER, IP_PMTUDISC_DO) + return sock + + def get_header_len(self): + return IP_HEADER_LEN + ICMP_HEADER_LEN + + def get_mtu(self): + return self._sock.getsockopt(socket.IPPROTO_IP, IP_MTU) + + def send(self, request): + self._sock.connect((request.destination, 0)) + return super().send(request) + + +class PMTUDv6Socket(ICMPv6Socket): + # pylint: disable=redefined-builtin + def _create_socket(self, type): + sock = super()._create_socket(type) + sock.setsockopt(socket.IPPROTO_IPV6, IPV6_MTU_DISCOVER, IPV6_PMTUDISC_DO) + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_DONTFRAG, 1) + return sock + + def get_header_len(self): + return IPV6_HEADER_LEN + ICMPV6_HEADER_LEN + + def get_mtu(self): + return self._sock.getsockopt(socket.IPPROTO_IPV6, IPV6_MTU) + + def send(self, request): + self._sock.connect((request.destination, 0)) + return super().send(request) + + +def ip_pmtud(ip): + mtu = 9999 + + try: + addrinfo = socket.getaddrinfo(ip, 0, type=socket.SOCK_DGRAM)[0] + except socket.gaierror as err: + return {"state": "error", "error": str(err), "mtu": mtu} + + if addrinfo[0] == socket.AF_INET6: + sock = PMTUDv6Socket(address=None, privileged=True) + else: + sock = PMTUDv4Socket(address=None, privileged=True) + + with sock: + ping_id = unique_identifier() + for sequence in range(30): + request = ICMPRequest( + destination=ip, + id=ping_id, + sequence=sequence, + payload_size=mtu - sock.get_header_len(), + ) + try: + # deliberately send a way-too-large packet to provoke an error. + # if the ping is successful, we found the MTU. + sock.send(request) + sock.receive(request, 1) + return {"state": "up", "mtu": mtu} + + except TimeoutExceeded: + # Target down, but no error -> MTU is probably fine. + return {"state": "down", "mtu": mtu} + + except (ICMPSocketError, OSError) as err: + if "Errno 90" not in str(err): + return {"state": "error", "error": str(err), "mtu": mtu} + + new_mtu = sock.get_mtu() + if new_mtu == mtu: + break + mtu = new_mtu + + return {"state": "ttl_exceeded", "mtu": mtu} diff --git a/meshping/meshping/meshping/traceroute_thread.py b/meshping/meshping/meshping/traceroute_thread.py new file mode 100644 index 0000000..23c0b22 --- /dev/null +++ b/meshping/meshping/meshping/traceroute_thread.py @@ -0,0 +1,108 @@ +import logging +import time +from threading import Thread +from icmplib import traceroute +from ipwhois import IPWhois, IPDefinedError +from netaddr import IPAddress, IPNetwork +from .socklib import reverse_lookup, ip_pmtud +from ..models import Target, Meta + + +class TracerouteThread(Thread): + def __init__(self, mp_config, *args, **kwargs): + self.mp_config = mp_config + self.whois_cache = {} + super().__init__(*args, **kwargs) + + def whois(self, hop_address): + # If we know this address already and it's up-to-date, skip it + now = int(time.time()) + if ( + hop_address in self.whois_cache + and self.whois_cache[hop_address].get("last_check", 0) + + self.mp_config.whois_cache_validiy_h * 3600 + < now + ): + return self.whois_cache[hop_address] + + # Check if the IP is private or reserved + addr = IPAddress(hop_address) + # TODO split out into separate function and allow configuration + # pylint: disable=too-many-boolean-expressions + if ( + addr.version == 4 + and ( + addr in IPNetwork("10.0.0.0/8") + or addr in IPNetwork("172.16.0.0/12") + or addr in IPNetwork("192.168.0.0/16") + or addr in IPNetwork("100.64.0.0/10") + ) + ) or (addr.version == 6 and addr not in IPNetwork("2000::/3")): + return {} + + # It's not, look up whois info + try: + self.whois_cache[hop_address] = dict( + IPWhois(hop_address).lookup_rdap(), last_check=now + ) + except IPDefinedError: + # RFC1918, RFC6598 or something else + return {} + # we do not have a global exception handler atm, thus we want to catch all + # errors here + # pylint: disable=broad-exception-caught + except Exception as err: + logging.warning("Could not query whois for IP %s: %s", hop_address, err) + return self.whois_cache[hop_address] + + def run(self): + while True: + now = time.time() + next_run = now + self.mp_config.traceroute_interval + pmtud_cache = {} + for target in Target.objects.all(): + target_meta, _created = Meta.objects.get_or_create(target=target) + + trace = traceroute( + target.addr, + fast=True, + timeout=self.mp_config.traceroute_timeout, + count=self.mp_config.traceroute_packets, + ) + hopaddrs = [hop.address for hop in trace] + hoaddrs_set = set(hopaddrs) + target_meta.route_loop = ( + len(hopaddrs) != len(hoaddrs_set) and len(hoaddrs_set) > 1 + ) + + trace_hops = [] + for hop in trace: + if hop.address not in pmtud_cache: + pmtud_cache[hop.address] = ip_pmtud(hop.address) + + trace_hops.append( + { + "name": reverse_lookup(hop.address), + "distance": hop.distance, + "address": hop.address, + "max_rtt": hop.max_rtt, + "pmtud": pmtud_cache[hop.address], + "whois": self.whois(hop.address), + "time": now, + } + ) + + target_meta.traceroute = trace_hops + if trace_hops and trace_hops[-1]["address"] == target.addr: + # Store last known good traceroute + target_meta.lkgt = trace_hops + + # Running a bunch'a traceroutes all at once might trigger our default + # gw's rate limiting if it receives too many packets with a ttl of 1 + # too quickly. Let's go a bit slower so that it doesn't stop sending + # "ttl exceeded" replies and messing up our results. + time.sleep(self.mp_config.traceroute_ratelimit_interval) + + target_meta.save() + + time.sleep(max(0, next_run - time.time())) From 6c78e0f6fe10bad5cf8055124a9f26e034dade3d Mon Sep 17 00:00:00 2001 From: liquid-metal <56035107+liquid-metal@users.noreply.github.com> Date: Sat, 5 Apr 2025 21:32:18 +0200 Subject: [PATCH 10/19] peering thread --- meshping/meshping/apps.py | 10 +- meshping/meshping/meshping/ifaces.py | 125 ++++++++++++++++++ meshping/meshping/meshping/meshping_config.py | 4 + meshping/meshping/meshping/peering_thread.py | 70 ++++++++++ .../migrations/0007_meta_is_foreign.py | 18 +++ meshping/meshping/models.py | 1 + requirements.txt | 3 +- 7 files changed, 227 insertions(+), 4 deletions(-) create mode 100644 meshping/meshping/meshping/ifaces.py create mode 100644 meshping/meshping/meshping/peering_thread.py create mode 100644 meshping/meshping/migrations/0007_meta_is_foreign.py diff --git a/meshping/meshping/apps.py b/meshping/meshping/apps.py index 2770f6a..df86e5e 100644 --- a/meshping/meshping/apps.py +++ b/meshping/meshping/apps.py @@ -11,15 +11,19 @@ class MeshpingConfig(AppConfig): # TODO with the current thread model, django complains about db access before app # initialization is completed # TODO create background task for db housekeeping (prune_histograms) + # + # delayed import, otherwise we will get AppRegistryNotReady + # pylint: disable=import-outside-toplevel def ready(self): - # delayed import, otherwise we will get AppRegistryNotReady - # pylint: disable=import-outside-toplevel + from .meshping.peering_thread import PeeringThread from .meshping.ping_thread import PingThread from .meshping.traceroute_thread import TracerouteThread mp_config = MPCOnfig() + peering_thread = PeeringThread(mp_config=mp_config, daemon=True) ping_thread = PingThread(mp_config=mp_config, daemon=True) - traceroute_thread = TracerouteThread(mp_config=mp_config) + traceroute_thread = TracerouteThread(mp_config=mp_config, daemon=True) + peering_thread.start() ping_thread.start() traceroute_thread.start() diff --git a/meshping/meshping/meshping/ifaces.py b/meshping/meshping/meshping/ifaces.py new file mode 100644 index 0000000..90db9c3 --- /dev/null +++ b/meshping/meshping/meshping/ifaces.py @@ -0,0 +1,125 @@ +import socket +import ipaddress +import logging +import netifaces + + +class Ifaces4: + def __init__(self): + self.addrs = [] + self.networks = [] + # Get addresses from our interfaces + for iface in netifaces.interfaces(): + try: + ifaddrs = netifaces.ifaddresses(iface) + except ValueError: + logging.warning( + "Could not retrieve addresses of interface %s, ignoring interface", + iface, + exc_info=True, + ) + continue + + for family, addresses in ifaddrs.items(): + if family != socket.AF_INET: + continue + + for addrinfo in addresses: + self.addrs.append(ipaddress.ip_address(addrinfo["addr"])) + self.networks.append( + ipaddress.IPv4Network( + f"{ipaddress.ip_address(addrinfo['addr'])}/" + f"{ipaddress.ip_address(addrinfo['netmask'])}", + strict=False, + ) + ) + + def find_iface_for_network(self, target): + target = ipaddress.IPv4Address(target) + for addr in self.networks: + if target in addr: + return addr + return None + + def is_local(self, target): + return self.find_iface_for_network(target) is not None + + def is_interface(self, target): + target = ipaddress.IPv4Address(target) + return target in self.addrs + + +class Ifaces6: + def __init__(self): + self.addrs = [] + self.networks = [] + # Get addresses from our interfaces + for iface in netifaces.interfaces(): + try: + ifaddrs = netifaces.ifaddresses(iface) + except ValueError: + logging.warning( + "Could not retrieve addresses of interface %s, ignoring interface", + iface, + exc_info=True, + ) + continue + + for family, addresses in ifaddrs.items(): + if family != socket.AF_INET6: + continue + + for addrinfo in addresses: + if "%" in addrinfo["addr"]: + part_addr, part_iface = addrinfo["addr"].split("%", 1) + assert part_iface == iface + addrinfo["addr"] = part_addr + + # netmask is ffff:ffff:ffff:etc:ffff/128 for some reason, + # we only need the length + addrinfo["netmask"] = int(addrinfo["netmask"].split("/")[1], 10) + + self.addrs.append(ipaddress.ip_address(addrinfo["addr"])) + self.networks.append( + ipaddress.IPv6Network( + f"{ipaddress.ip_address(addrinfo['addr'])}/" + f"{addrinfo['netmask']}", + strict=False, + ) + ) + + def find_iface_for_network(self, target): + target = ipaddress.IPv6Address(target) + for addr in self.networks: + if target in addr: + return addr + return None + + def is_local(self, target): + return self.find_iface_for_network(target) is not None + + def is_interface(self, target): + target = ipaddress.IPv4Address(target) + return target in self.addrs + + +def test(): + if4 = Ifaces4() + for target in ( + "192.168.0.1", + "10.159.1.1", + "10.159.1.2", + "10.5.1.2", + "10.9.9.9", + "8.8.8.8", + "192.168.44.150", + ): + print(f"{target} -> {if4.is_local(target)}") + + if6 = Ifaces6() + for target in ("2001:4860:4860::8888", "2001:4860:4860::8844", "::1"): + print(f"{target} -> {if6.is_local(target)}") + + +if __name__ == "__main__": + test() diff --git a/meshping/meshping/meshping/meshping_config.py b/meshping/meshping/meshping/meshping_config.py index 9154641..f7ae31f 100644 --- a/meshping/meshping/meshping/meshping_config.py +++ b/meshping/meshping/meshping/meshping_config.py @@ -13,6 +13,10 @@ def __init__(self): self.traceroute_packets = 1 self.traceroute_ratelimit_interval = 2 + self.peers = "" + self.peering_interval = 30 + self.peering_timeout = 30 + self.whois_cache_validiy_h = 72 self.histogram_period = 3 diff --git a/meshping/meshping/meshping/peering_thread.py b/meshping/meshping/meshping/peering_thread.py new file mode 100644 index 0000000..cf66524 --- /dev/null +++ b/meshping/meshping/meshping/peering_thread.py @@ -0,0 +1,70 @@ +import json +import logging +import time +from threading import Thread +import requests +from .ifaces import Ifaces4, Ifaces6 +from ..models import Target, Meta + + +# TODO test this code, has not even run once + + +class PeeringThread(Thread): + def __init__(self, mp_config, *args, **kwargs): + self.mp_config = mp_config + super().__init__(*args, **kwargs) + + def run(self): + if self.mp_config.peers: + peers = self.mp_config.peers.split(",") + else: + return + + while True: + if4 = Ifaces4() + if6 = Ifaces6() + + def is_local(addr): + try: + return if4.is_local(addr) + except ValueError: + pass + try: + return if6.is_local(addr) + except ValueError: + pass + return False + + peer_targets = [] + + # TODO feels clumsy to get the meta objects one by one, maybe there is a + # more elegant way, i.e. something that maps to a join under the hood? + for target in Target.objects.all(): + target_meta, _created = Meta.objects.get_or_create(target=target) + if target_meta.is_foreign: + continue + peer_targets.append( + { + "name": target.name, + "addr": target.addr, + "local": is_local(target.addr), + } + ) + + for peer in peers: + try: + requests.post( + f"http://{peer}/peer", + headers={ + "Content-Type": "application/json", + }, + data=json.dumps({"targets": peer_targets}), + timeout=self.mp_config.peering_timeout, + ) + # TODO decide if this general exception catch is the correct way + # pylint: disable=broad-exception-caught + except Exception as err: + logging.warning("Could not connect to peer %s: %s", peer, err) + + time.sleep(self.mp_config.peering_interval) diff --git a/meshping/meshping/migrations/0007_meta_is_foreign.py b/meshping/meshping/migrations/0007_meta_is_foreign.py new file mode 100644 index 0000000..d0c2b0f --- /dev/null +++ b/meshping/meshping/migrations/0007_meta_is_foreign.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.7 on 2025-04-05 19:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("meshping", "0006_alter_meta_lkgt_alter_meta_traceroute"), + ] + + operations = [ + migrations.AddField( + model_name="meta", + name="is_foreign", + field=models.BooleanField(default=False), + ), + ] diff --git a/meshping/meshping/models.py b/meshping/meshping/models.py index de6792b..5d4e18a 100644 --- a/meshping/meshping/models.py +++ b/meshping/meshping/models.py @@ -52,3 +52,4 @@ class Meta(models.Model): # lkgt = last known good traceroute lkgt = models.JSONField(max_length=2048, default=list) error = models.CharField(max_length=255, null=True, default=None) + is_foreign = models.BooleanField(default=False) diff --git a/requirements.txt b/requirements.txt index 10f339d..3982323 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,5 @@ packaging black~=25.1.0 Django~=5.1.7 -Jinja2~=3.1.6 \ No newline at end of file +Jinja2~=3.1.6 +requests~=2.32.3 \ No newline at end of file From 25156b53a116a68a2c824e10fc9e3da95aca5bc1 Mon Sep 17 00:00:00 2001 From: liquid-metal <56035107+liquid-metal@users.noreply.github.com> Date: Sat, 5 Apr 2025 21:46:08 +0200 Subject: [PATCH 11/19] deleting targets --- meshping/meshping/views.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/meshping/meshping/views.py b/meshping/meshping/views.py index 6df648e..bddd37c 100644 --- a/meshping/meshping/views.py +++ b/meshping/meshping/views.py @@ -82,6 +82,7 @@ def stats(request): # TODO add traceroute to response for each target # TODO add route_loop to response for each target # TODO do not crash when the uniqueness constraint is not met for new targets +# TODO nasty race condition, retrieving objects can fail when target was just deleted @require_http_methods(["GET", "POST"]) def targets_endpoint(request): if request.method == "GET": @@ -160,5 +161,8 @@ def targets_endpoint(request): # route /api/targets/ -def edit_target(request, **kwargs): - return HttpResponseServerError("not implemented") +@require_http_methods(["DELETE"]) +def edit_target(request, target): + if request.method == "DELETE": + Target.objects.filter(addr=target).delete() + return JsonResponse({"success": True}) From 637ff774204dabcb17bb1dd4caceca9134beac00 Mon Sep 17 00:00:00 2001 From: liquid-metal <56035107+liquid-metal@users.noreply.github.com> Date: Sat, 5 Apr 2025 22:13:06 +0200 Subject: [PATCH 12/19] first part of metrics output --- meshping/meshping/views.py | 85 +++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/meshping/meshping/views.py b/meshping/meshping/views.py index bddd37c..1e4b7b8 100644 --- a/meshping/meshping/views.py +++ b/meshping/meshping/views.py @@ -2,6 +2,7 @@ import os import socket +from django.forms.models import model_to_dict from django.http import ( HttpResponse, HttpResponseBadRequest, @@ -48,8 +49,90 @@ def histogram(request, **kwargs): # route /metrics +# +# TODO is the metrics output valid prometheus format when no values are present? def metrics(request): - return HttpResponseServerError("not implemented") + respdata = [ + "\n".join( + [ + "# HELP meshping_sent Sent pings", + "# TYPE meshping_sent counter", + "# HELP meshping_recv Received pongs", + "# TYPE meshping_recv counter", + "# HELP meshping_lost Lost pings (actual counter, not just sent-recv)", + "# TYPE meshping_lost counter", + "# HELP meshping_max max ping", + "# TYPE meshping_max gauge", + "# HELP meshping_min min ping", + "# TYPE meshping_min gauge", + "# HELP meshping_pings Pings bucketed by response time", + "# TYPE meshping_pings histogram", + ] + ) + ] + + for target in Target.objects.all(): + target_stats, _created = Statistics.objects.get_or_create(target=target) + target_info = dict( + model_to_dict(target_stats), addr=target.addr, name=target.name + ) + respdata.append( + "\n".join( + [ + 'meshping_sent{name="%(name)s",target="%(addr)s"} %(sent)d', + 'meshping_recv{name="%(name)s",target="%(addr)s"} %(recv)d', + 'meshping_lost{name="%(name)s",target="%(addr)s"} %(lost)d', + ] + ) + % target_info + ) + + if target_info["recv"]: + respdata.append( + "\n".join( + [ + 'meshping_max{name="%(name)s",target="%(addr)s"} %(max).2f', + 'meshping_min{name="%(name)s",target="%(addr)s"} %(min).2f', + ] + ) + % target_info + ) + + respdata.append( + "\n".join( + [ + 'meshping_pings_sum{name="%(name)s",target="%(addr)s"} %(sum)f', + 'meshping_pings_count{name="%(name)s",target="%(addr)s"} %(recv)d', + ] + ) + % target_info + ) + + # TODO add the histogram + # histogram = target.histogram.tail(1) + # count = 0 + # for bucket in histogram.columns: + # if histogram[bucket][0] == 0: + # continue + # nextping = 2 ** ((bucket + 1) / 10.) + # count += histogram[bucket][0] + # respdata.append( + # 'meshping_pings_bucket{name="%(name)s",target="%(addr)s",le="%(le).2f"} %(count)d' % dict( + # addr = target.addr, + # count = count, + # le = nextping, + # name = target.name, + # ) + # ) + # respdata.append( + # 'meshping_pings_bucket{name="%(name)s",target="%(addr)s",le="+Inf"} %(count)d' % dict( + # addr = target.addr, + # count = count, + # name = target.name, + # ) + # ) + + return HttpResponse("\n".join(respdata) + "\n", content_type="text/plain") # route /network.svg From f7ada64c009c0a1f805e173b083f522fc61d1817 Mon Sep 17 00:00:00 2001 From: liquid-metal <56035107+liquid-metal@users.noreply.github.com> Date: Sun, 6 Apr 2025 11:44:30 +0200 Subject: [PATCH 13/19] histogram data in /metrics endpoint --- meshping/meshping/meshping/meshping_config.py | 2 + meshping/meshping/meshping/ping_thread.py | 2 +- meshping/meshping/views.py | 64 +++++++++++-------- 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/meshping/meshping/meshping/meshping_config.py b/meshping/meshping/meshping/meshping_config.py index f7ae31f..df3200f 100644 --- a/meshping/meshping/meshping/meshping_config.py +++ b/meshping/meshping/meshping/meshping_config.py @@ -17,6 +17,8 @@ def __init__(self): self.peering_interval = 30 self.peering_timeout = 30 + # TODO make config options in this object consistent, use seconds self.whois_cache_validiy_h = 72 + # TODO make config options in this object consistent, use seconds self.histogram_period = 3 diff --git a/meshping/meshping/meshping/ping_thread.py b/meshping/meshping/meshping/ping_thread.py index 22301e9..54335d0 100644 --- a/meshping/meshping/meshping/ping_thread.py +++ b/meshping/meshping/meshping/ping_thread.py @@ -82,7 +82,7 @@ def run(self): # TODO has nothing to do with pings, find a better place (probably new # housekeeping thread) Histogram.objects.filter( - timestamp__lt=(now - self.mp_config.histogram_period) + timestamp__lt=(now - self.mp_config.histogram_period * 24 * 3600) ).delete() unseen_targets = current_targets.copy() diff --git a/meshping/meshping/views.py b/meshping/meshping/views.py index 1e4b7b8..2ba81cf 100644 --- a/meshping/meshping/views.py +++ b/meshping/meshping/views.py @@ -9,11 +9,12 @@ HttpResponseServerError, JsonResponse, ) -from django.views.decorators.http import require_http_methods from django.template import loader +from django.views.decorators.http import require_http_methods +from django_pivot.pivot import pivot from markupsafe import Markup -from .models import Statistics, Target, Meta +from .models import Statistics, Target, Meta, Histogram # TODO remove business logic in this file @@ -108,29 +109,42 @@ def metrics(request): % target_info ) - # TODO add the histogram - # histogram = target.histogram.tail(1) - # count = 0 - # for bucket in histogram.columns: - # if histogram[bucket][0] == 0: - # continue - # nextping = 2 ** ((bucket + 1) / 10.) - # count += histogram[bucket][0] - # respdata.append( - # 'meshping_pings_bucket{name="%(name)s",target="%(addr)s",le="%(le).2f"} %(count)d' % dict( - # addr = target.addr, - # count = count, - # le = nextping, - # name = target.name, - # ) - # ) - # respdata.append( - # 'meshping_pings_bucket{name="%(name)s",target="%(addr)s",le="+Inf"} %(count)d' % dict( - # addr = target.addr, - # count = count, - # name = target.name, - # ) - # ) + # TODO add proper explanation + # [ + # { + # "timestamp": 1743883200, + # "47": 2, + # "50": null, + # ... + # }, + # ... + # ] + # + # TODO make sure that result is always ordered by timestamp + # TODO error handling if there is no line in the result + hist = pivot( + Histogram.objects.filter(target=target), "timestamp", "bucket", "count" + )[-1] + count = 0 + for bucket in hist.keys(): + if bucket == "timestamp": + continue + if not hist[bucket]: + continue + nextping = 2 ** ((int(bucket) + 1) / 10.0) + count += hist[bucket] + respdata.append( + ( + f'meshping_pings_bucket{{name="{target.name}",' + f'target="{target.addr}",le="{nextping:.2f}"}} {count}' + ) + ) + respdata.append( + ( + f'meshping_pings_bucket{{name="{target.name}",' + f'target="{target.addr}",le="+Inf"}} {count}' + ) + ) return HttpResponse("\n".join(respdata) + "\n", content_type="text/plain") From 1b15ddef9d004682305b6a5f65a3c817c102733b Mon Sep 17 00:00:00 2001 From: liquid-metal <56035107+liquid-metal@users.noreply.github.com> Date: Sun, 6 Apr 2025 19:37:02 +0200 Subject: [PATCH 14/19] drawing histograms, including comparisons --- meshping/meshping/meshping/histodraw.py | 246 ++++++++++++++++++++++++ meshping/meshping/models.py | 25 +++ meshping/meshping/views.py | 54 ++++-- 3 files changed, 306 insertions(+), 19 deletions(-) create mode 100644 meshping/meshping/meshping/histodraw.py diff --git a/meshping/meshping/meshping/histodraw.py b/meshping/meshping/meshping/histodraw.py new file mode 100644 index 0000000..902407d --- /dev/null +++ b/meshping/meshping/meshping/histodraw.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# kate: space-indent on; indent-width 4; replace-tabs on; + +import socket +import os +from datetime import timedelta +import numpy as np +import pandas + +from PIL import Image, ImageDraw, ImageFont, ImageOps +from ..models import target_histograms + + +# How big do you want the squares to be? +sqsz = 8 + + +# [ +# { +# "timestamp": 1743883200, +# "47": 2, +# "50": null, +# ... +# }, +# ... +# ] +def render_target(target): + histograms_df = target_histograms(target) + if histograms_df.empty: + return None + + # Normalize Buckets by transforming the number of actual pings sent + # into a float [0..1] indicating the grayness of that bucket. + biggestbkt = histograms_df.max().max() + histograms_df = histograms_df.div(biggestbkt, axis="index") + # prune outliers -> keep only values > 5% + histograms_df = histograms_df[histograms_df > 0.05] + # drop columns that contain only NaNs now + histograms_df = histograms_df.dropna(axis="columns", how="all") + # fill missing _rows_ (aka, hours) with rows of just NaN + histograms_df = histograms_df.asfreq("1h") + # replace all the NaNs with 0 + histograms_df = histograms_df.fillna(0) + + # detect dynamic range, and round to the nearest multiple of 10. + # this ensures that the ticks are drawn at powers of 2, which makes + # the graph more easily understandable. (I hope.) + # Btw: 27 // 10 * 10 # = 20 + # -27 // 10 * 10 # = -30 + # hmax needs to be nearest power of 10 + 1 for the top tick to be drawn. + hmin = histograms_df.columns.min() // 10 * 10 + hmax = histograms_df.columns.max() // 10 * 10 + 11 + + # Draw the graph in a pixels array which we then copy to an image + height = hmax - hmin + 1 + width = len(histograms_df) + pixels = np.zeros(width * height) + + for col, (_tstamp, histogram) in enumerate(histograms_df.iterrows()): + for bktval, bktgrayness in histogram.items(): + # ( y ) (x) + pixels[((hmax - bktval) * width) + col] = bktgrayness + + # copy pixels to an Image and paste that into the output image + graph = Image.new("L", (width, height)) + graph.putdata(pixels * 0xFF) + + # Scale graph so each Pixel becomes a square + width *= sqsz + height *= sqsz + + graph = graph.resize((width, height), Image.NEAREST) + graph.hmin = hmin + graph.hmax = hmax + graph.tmin = histograms_df.index.min() + graph.tmax = histograms_df.index.max() + return graph + + +# TODO maybe this method is a bit long? on the other hand, manual rendering code always +# tends to be tedious +# pylint: disable=too-many-locals,too-many-branches,too-many-statements +def render(targets, histogram_period): + rendered_graphs = [] + + for target in targets: + target_graph = render_target(target) + if target_graph is None: + raise ValueError(f"No data available for target {target}") + rendered_graphs.append(target_graph) + + width = histogram_period // 3600 * sqsz + hmin = min(graph.hmin for graph in rendered_graphs) + hmax = max(graph.hmax for graph in rendered_graphs) + tmax = max(graph.tmax for graph in rendered_graphs) + height = (hmax - hmin) * sqsz + + if len(rendered_graphs) == 1: + # Single graph -> use it as-is + graph = Image.new("L", (width, height), "white") + graph.paste( + ImageOps.invert(rendered_graphs[0]), (width - rendered_graphs[0].width, 0) + ) + else: + # Multiple graphs -> merge. + # This width/height may not match what we need for the output. + # Check for which graphs that is the case, and for these, + # create a new image that has the correct size and paste + # the graph into it. + resized_graphs = [] + for graph in rendered_graphs: + dtmax = (tmax - graph.tmax) // pandas.Timedelta(hours=1) + if graph.width != width or graph.height != height: + new_graph = Image.new("L", (width, height), "black") + new_graph.paste( + graph, + (width - graph.width - sqsz * dtmax, (hmax - graph.hmax) * sqsz), + ) + else: + new_graph = graph + + resized_graphs.append(new_graph) + + while len(resized_graphs) != 3: + resized_graphs.append(Image.new("L", (width, height), "black")) + + # Print the graph, on black background still. + graph = Image.merge("RGB", resized_graphs) + + # To get a white background, convert to HSV and set V=1. + # V currently contains the interesting information though, + # so move that to S first. + hsv = np.array(graph.convert("HSV")) + # Add V to S (not sure why adding works better than replacing, but it does) + hsv[:, :, 1] = hsv[:, :, 1] + hsv[:, :, 2] + # Set V to 1 + hsv[:, :, 2] = np.ones((height, width)) * 0xFF + graph = Image.fromarray(hsv, "HSV").convert("RGB") + + # position of the graph + graph_x = 70 + graph_y = 30 * len(targets) + 10 + + # im will hold the output image + im = Image.new("RGB", (graph_x + width + 20, graph_y + height + 100), "white") + im.paste(graph, (graph_x, graph_y)) + + # draw a rect around the graph + draw = ImageDraw.Draw(im) + draw.rectangle( + (graph_x, graph_y, graph_x + width - 1, graph_y + height - 1), outline=0x333333 + ) + + try: + font = ImageFont.truetype("DejaVuSansMono.ttf", 10) + lgfont = ImageFont.truetype("DejaVuSansMono.ttf", 16) + except IOError: + font = ImageFont.truetype( + "/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf", 10 + ) + lgfont = ImageFont.truetype( + "/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf", 16 + ) + + # Headline + if len(targets) == 1: # just black for a single graph + targets_with_colors = zip(targets, (0x000000,)) + else: # red, green, blue for multiple graphs + targets_with_colors = zip(targets, (0x0000FF, 0x00FF00, 0xFF0000)) + + for idx, (target, color) in enumerate(targets_with_colors): + headline_text = f"{socket.gethostname()} → {target.label}" + headline_width = lgfont.getlength(headline_text) + draw.text( + ((graph_x + width + 20 - headline_width) // 2, 30 * idx + 11), + headline_text, + color, + font=lgfont, + ) + + # Y axis ticks and annotations + for hidx in range(hmin, hmax, 5): + bottomrow = hidx - hmin + offset_y = height + graph_y - bottomrow * sqsz - 1 + draw.line((graph_x - 2, offset_y, graph_x + 2, offset_y), fill=0xAAAAAA) + + ping = 2 ** (hidx / 10.0) + label = f"{ping:.2f}" + draw.text( + (graph_x - len(label) * 6 - 10, offset_y - 5), label, 0x333333, font=font + ) + + # Calculate the times at which the histogram begins and ends. + t_hist_end = ( + # Latest hour for which we have data... + tmax.tz_localize("Etc/UTC") + .tz_convert(os.environ.get("TZ", "Etc/UTC")) + .to_pydatetime() + # Plus the current hour which we're also drawing on screen + + timedelta(hours=1) + ) + + td_hours = histogram_period // 3600 + t_hist_begin = t_hist_end - timedelta(hours=td_hours) + + # X axis ticks - one every two hours + for col in range(1, width // sqsz): + # We're now at hour indicated by col + if (t_hist_begin + timedelta(hours=col)).hour % 2 != 0: + continue + offset_x = graph_x + col * sqsz + draw.line( + (offset_x, height + graph_y - 2, offset_x, height + graph_y + 2), + fill=0xAAAAAA, + ) + + # X axis annotations + # Create a temp image for the bottom label that we then rotate by 90° and attach to + # the other one since this stuff is rotated by 90° while we create it, all the + # coordinates are inversed... + tmpim = Image.new("RGB", (80, width + 20), "white") + tmpdraw = ImageDraw.Draw(tmpim) + + # Draw one annotation every four hours + for col in range(0, width // sqsz + 1): + # We're now at hour indicated by col + tstamp = t_hist_begin + timedelta(hours=col) + if tstamp.hour % 4 != 0: + continue + offset_x = col * sqsz + if tstamp.hour == 0: + tmpdraw.text( + (0, offset_x + 4), tstamp.strftime("%m-%d"), 0x333333, font=font + ) + tmpdraw.text((36, offset_x + 4), tstamp.strftime("%H:%M"), 0x333333, font=font) + + im.paste(tmpim.rotate(90, expand=1), (graph_x - 10, height + graph_y + 1)) + + # This worked pretty well for Tobi Oetiker... + tmpim = Image.new("RGB", (170, 13), "white") + tmpdraw = ImageDraw.Draw(tmpim) + tmpdraw.text((0, 0), "Meshping by Michael Ziegler", 0x999999, font=font) + im.paste(tmpim.rotate(270, expand=1), (width + graph_x + 7, graph_y)) + + return im diff --git a/meshping/meshping/models.py b/meshping/meshping/models.py index 5d4e18a..7c546e2 100644 --- a/meshping/meshping/models.py +++ b/meshping/meshping/models.py @@ -1,4 +1,5 @@ from django.db import models +import pandas # TODO decide on max_length, even though ignored by sqlite @@ -9,6 +10,12 @@ class Target(models.Model): class Meta: unique_together = ("addr", "name") + @property + def label(self): + if self.name == self.addr: + return self.name + return f"{self.name} ({self.addr})" + # TODO uniqueness constraint `UNIQUE (target_id, timestamp, bucket)` class Histogram(models.Model): @@ -53,3 +60,21 @@ class Meta(models.Model): lkgt = models.JSONField(max_length=2048, default=list) error = models.CharField(max_length=255, null=True, default=None) is_foreign = models.BooleanField(default=False) + + +# TODO consider making this a Target property, but take care that Histogram is defined +# before Target +# +# pivot method: flip the dataframe: turn each value of the "bucket" DB column into a +# separate column in the DF, using the timestamp as the index and the count for the +# values. None-existing positions in the DF are filled with zero. +def target_histograms(target): + df = pandas.DataFrame.from_dict( + Histogram.objects.filter(target=target).order_by("timestamp", "bucket").values() + ) + df["timestamp"] = pandas.to_datetime(df["timestamp"], unit="s") + return df.pivot( + index="timestamp", + columns="bucket", + values="count", + ).fillna(0) diff --git a/meshping/meshping/views.py b/meshping/meshping/views.py index 2ba81cf..a8ca1e9 100644 --- a/meshping/meshping/views.py +++ b/meshping/meshping/views.py @@ -1,20 +1,22 @@ import json import os import socket +from datetime import datetime from django.forms.models import model_to_dict from django.http import ( HttpResponse, HttpResponseBadRequest, + HttpResponseNotFound, HttpResponseServerError, JsonResponse, ) from django.template import loader from django.views.decorators.http import require_http_methods -from django_pivot.pivot import pivot from markupsafe import Markup -from .models import Statistics, Target, Meta, Histogram +from .meshping import histodraw +from .models import Statistics, Target, Meta, target_histograms # TODO remove business logic in this file @@ -24,6 +26,7 @@ # TODO do not load icons from disk for every request # TODO find a better method for finding the icons to not have an absolute path here # TODO make local development possible when node modules are on disk (relative path) +@require_http_methods(["GET"]) def index(request): def read_svg_file(icons_dir, filename): with open(os.path.join(icons_dir, filename), "r", encoding="utf-8") as f: @@ -45,13 +48,40 @@ def read_svg_file(icons_dir, filename): # route /histogram//.png -def histogram(request, **kwargs): - return HttpResponseServerError("not implemented") +@require_http_methods(["GET"]) +def histogram(request, node, target): + targets = [] + for arg_target in [target] + request.GET.getlist("compare", default=None): + try: + targets.append(Target.objects.get(addr=arg_target)) + except Target.DoesNotExist: + return HttpResponseNotFound(f"Target {arg_target} not found") + + if len(targets) > 3: + # an RGB image only has three channels + return HttpResponseBadRequest("Can only compare up to three targets") + + try: + # TODO reference the configuration, with the current setup there is no + # obvious clean way (at least to me) + img = histodraw.render(targets, 3 * 24 * 3600) + except ValueError as err: + return HttpResponseNotFound(str(err)) + + response = HttpResponse(content_type="image/png") + img.save(response, "png") + response["refresh"] = "300" + response["content-disposition"] = ( + 'inline; filename="meshping_' + f'{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_{target}.png"' + ) + return response # route /metrics # # TODO is the metrics output valid prometheus format when no values are present? +@require_http_methods(["GET"]) def metrics(request): respdata = [ "\n".join( @@ -109,22 +139,8 @@ def metrics(request): % target_info ) - # TODO add proper explanation - # [ - # { - # "timestamp": 1743883200, - # "47": 2, - # "50": null, - # ... - # }, - # ... - # ] - # - # TODO make sure that result is always ordered by timestamp # TODO error handling if there is no line in the result - hist = pivot( - Histogram.objects.filter(target=target), "timestamp", "bucket", "count" - )[-1] + hist = target_histograms(target)[-1] count = 0 for bucket in hist.keys(): if bucket == "timestamp": From ecdf2340d7d7786567c3d3ef7526813296f024b0 Mon Sep 17 00:00:00 2001 From: liquid-metal <56035107+liquid-metal@users.noreply.github.com> Date: Sun, 6 Apr 2025 21:06:07 +0200 Subject: [PATCH 15/19] network diagram endpoint --- meshping/meshping/models.py | 32 +++++++ meshping/meshping/templates/network.puml.j2 | 43 +++++++++ meshping/meshping/views.py | 98 +++++++++++++++++++-- 3 files changed, 167 insertions(+), 6 deletions(-) create mode 100644 meshping/meshping/templates/network.puml.j2 diff --git a/meshping/meshping/models.py b/meshping/meshping/models.py index 7c546e2..397e382 100644 --- a/meshping/meshping/models.py +++ b/meshping/meshping/models.py @@ -1,3 +1,4 @@ +from itertools import zip_longest from django.db import models import pandas @@ -78,3 +79,34 @@ def target_histograms(target): columns="bucket", values="count", ).fillna(0) + + +# TODO consider making this a Target property, but take care that Meta is defined +# before Target +def target_traceroute(target): + target_meta, _created = Meta.objects.get_or_create(target=target) + + curr = target_meta.traceroute + lkgt = target_meta.lkgt # last known good traceroute + if not curr or not lkgt or len(lkgt) < len(curr): + # we probably don't know all the nodes, but the ones we do know are up + return [dict(hop, state="up") for hop in curr] + if curr[-1]["address"] == target.addr: + # Trace has reached the target itself, thus all hops are up + return [dict(hop, state="up") for hop in curr] + + # Check with lkgt to see which hops are still there + result = [] + for lkgt_hop, curr_hop in zip_longest(lkgt, curr): + if lkgt_hop is None: + # This should not be able to happen, because we checked + # len(lkgt) < len(curr) above. + raise ValueError("last known good traceroute: hop is None") + if curr_hop is None: + # hops missing from current traceroute are down + result.append(dict(lkgt_hop, state="down")) + elif curr_hop.get("address") != lkgt_hop.get("address"): + result.append(dict(curr_hop, state="different")) + else: + result.append(dict(curr_hop, state="up")) + return result diff --git a/meshping/meshping/templates/network.puml.j2 b/meshping/meshping/templates/network.puml.j2 new file mode 100644 index 0000000..dfd3bac --- /dev/null +++ b/meshping/meshping/templates/network.puml.j2 @@ -0,0 +1,43 @@ +@startuml + + + +hide <> stereotype +hide <> stereotype +hide <> stereotype + + +title Network Map + +node "{[ hostname ]}" <> as SELF + +{% for hop in uniq_hops_sorted -%} +{% if hop.address -%} +node "{% if hop.target %}{[ hop.target.name ]}{% else %}{[ hop.name ]}{% endif %}\n{[ hop.address ]}{% if hop.whois %}\n{[ hop.whois.network.name ]}{% endif %}{% if hop.target %}\n[[{[ url('histogram', kwargs={'node': hostname, 'target': hop.target.addr})]} Histogram]]{% endif %}" <> as {[ hop.id ]} +{% else -%} +rectangle "?" as {[ hop.id ]} +{% endif -%} +{% endfor -%} + +{% for (lft, rgt) in uniq_links -%} +"{[ lft ]}" -- "{[ rgt ]}" +{% endfor -%} + +footer rendered on {[ now ]} by [[http://github.com/Svedrin/meshping Meshping]] using PlantUML +@enduml diff --git a/meshping/meshping/views.py b/meshping/meshping/views.py index a8ca1e9..ad19e93 100644 --- a/meshping/meshping/views.py +++ b/meshping/meshping/views.py @@ -2,6 +2,8 @@ import os import socket from datetime import datetime +from random import randint +from subprocess import run as run_command from django.forms.models import model_to_dict from django.http import ( @@ -16,7 +18,7 @@ from markupsafe import Markup from .meshping import histodraw -from .models import Statistics, Target, Meta, target_histograms +from .models import Statistics, Target, Meta, target_histograms, target_traceroute # TODO remove business logic in this file @@ -166,8 +168,96 @@ def metrics(request): # route /network.svg +# +# note: plantuml in the ubuntu 24.04 apt repo is version 1.2020.02, which gives a +# broken output. newer version required, known to work with 1.2024.4 +# +# TODO split out some logic here, this probably reduces the amount of local vars +# pylint: disable=too-many-locals +@require_http_methods(["GET"]) def network(request): - return HttpResponseServerError("not implemented") + targets = Target.objects.all() + uniq_hops = {} + uniq_links = set() + + for target in targets: + prev_hop = "SELF" + prev_dist = 0 + for hop in target_traceroute(target): + hop_id = hop["address"].replace(":", "_").replace(".", "_") + + # Check if we know this hop already. If we do, just skip ahead. + if hop_id not in uniq_hops: + # Fill in the blanks for missing hops, if any + while hop["distance"] > prev_dist + 1: + dummy_id = str(randint(10000000, 99999999)) + dummy = { + "id": dummy_id, + "distance": (prev_dist + 1), + "address": None, + "name": None, + "target": None, + "whois": None, + } + uniq_hops[dummy_id] = dummy + uniq_links.add((prev_hop, dummy_id)) + prev_hop = dummy_id + prev_dist += 1 + + # Now render the hop itself + hop_id = hop["address"].replace(":", "_").replace(".", "_") + uniq_hops.setdefault(hop_id, dict(hop, id=hop_id, target=None)) + uniq_links.add((prev_hop, hop_id)) + + # make sure we show the most recent state info + if ( + uniq_hops[hop_id]["state"] != hop["state"] + and uniq_hops[hop_id]["time"] < hop["time"] + ): + uniq_hops[hop_id].update(state=hop["state"], time=hop["time"]) + + if hop["address"] == target.addr: + uniq_hops[hop_id]["target"] = target + + prev_hop = hop_id + prev_dist = hop["distance"] + + now = datetime.now() + + context = { + "hostname": socket.gethostname(), + "now": now.strftime("%Y-%m-%d %H:%M:%S"), + "targets": targets, + "uniq_hops": uniq_hops, + "uniq_links": sorted(uniq_links), + "uniq_hops_sorted": [uniq_hops[hop] for hop in sorted(uniq_hops.keys())], + } + tpl = loader.get_template("network.puml.j2").render(context) + + plantuml = run_command( + ["plantuml", "-tsvg", "-p"], + input=tpl.encode("utf-8"), + capture_output=True, + check=False, + ) + + if plantuml.stderr: + return HttpResponseServerError( + plantuml.stderr.decode("utf-8") + "\n\n===\n\n" + tpl, + content_type="text/plain", + ) + + resp = HttpResponse(plantuml.stdout, content_type="image/svg+xml") + + resp["refresh"] = "43200" # 12h + resp["Cache-Control"] = "max-age=36000, public" # 10h + + resp["content-disposition"] = ( + 'inline; filename="meshping_' + f'{now.strftime("%Y-%m-%d_%H-%M-%S")}_network.svg"' + ) + + return resp # route /peer @@ -190,10 +280,6 @@ def stats(request): # django ensures a valid http request method, so we do not need a return value # pylint: disable=inconsistent-return-statements # -# TODO add state to response for each target -# TODO add error to response for each target -# TODO add traceroute to response for each target -# TODO add route_loop to response for each target # TODO do not crash when the uniqueness constraint is not met for new targets # TODO nasty race condition, retrieving objects can fail when target was just deleted @require_http_methods(["GET", "POST"]) From f4fbf3ecd493184b3481b9b7e6fd638687ef5a6b Mon Sep 17 00:00:00 2001 From: liquid-metal <56035107+liquid-metal@users.noreply.github.com> Date: Sun, 6 Apr 2025 21:11:32 +0200 Subject: [PATCH 16/19] api/resolve endpoint --- meshping/meshping/views.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/meshping/meshping/views.py b/meshping/meshping/views.py index ad19e93..2d23a46 100644 --- a/meshping/meshping/views.py +++ b/meshping/meshping/views.py @@ -266,8 +266,20 @@ def peer(request): # route /api/resolve/ -def resolve(request, **kwargs): - return HttpResponseServerError("not implemented") +@require_http_methods(["GET"]) +def resolve(request, name): + try: + return JsonResponse( + { + "success": True, + "addrs": [ + info[4][0] + for info in socket.getaddrinfo(name, 0, 0, socket.SOCK_STREAM) + ], + } + ) + except socket.gaierror as err: + return JsonResponse({"success": False, "error": str(err)}) # route /api/stats From 6a992192076c82d25ea80ac095946e99f932a46f Mon Sep 17 00:00:00 2001 From: liquid-metal <56035107+liquid-metal@users.noreply.github.com> Date: Sun, 6 Apr 2025 21:21:18 +0200 Subject: [PATCH 17/19] delete stats endpoint --- .../migrations/0008_alter_meta_state.py | 27 +++++++++++++++++++ meshping/meshping/models.py | 2 +- meshping/meshping/views.py | 17 ++++++++++-- 3 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 meshping/meshping/migrations/0008_alter_meta_state.py diff --git a/meshping/meshping/migrations/0008_alter_meta_state.py b/meshping/meshping/migrations/0008_alter_meta_state.py new file mode 100644 index 0000000..3081c20 --- /dev/null +++ b/meshping/meshping/migrations/0008_alter_meta_state.py @@ -0,0 +1,27 @@ +# Generated by Django 5.1.7 on 2025-04-06 19:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("meshping", "0007_meta_is_foreign"), + ] + + operations = [ + migrations.AlterField( + model_name="meta", + name="state", + field=models.CharField( + choices=[ + ("up", "Up"), + ("down", "Down"), + ("unknown", "Unknown"), + ("error", "Error"), + ], + default="unknown", + max_length=10, + ), + ), + ] diff --git a/meshping/meshping/models.py b/meshping/meshping/models.py index 397e382..254585a 100644 --- a/meshping/meshping/models.py +++ b/meshping/meshping/models.py @@ -53,7 +53,7 @@ class TargetState(models.TextChoices): class Meta(models.Model): target = models.OneToOneField(Target, on_delete=models.CASCADE, primary_key=True) state = models.CharField( - max_length=10, choices=TargetState.choices, default=TargetState.DOWN + max_length=10, choices=TargetState.choices, default=TargetState.UNKNOWN ) route_loop = models.BooleanField(default=False) traceroute = models.JSONField(max_length=2048, default=list) diff --git a/meshping/meshping/views.py b/meshping/meshping/views.py index 2d23a46..94cb3ef 100644 --- a/meshping/meshping/views.py +++ b/meshping/meshping/views.py @@ -18,7 +18,14 @@ from markupsafe import Markup from .meshping import histodraw -from .models import Statistics, Target, Meta, target_histograms, target_traceroute +from .models import ( + Statistics, + Target, + TargetState, + Meta, + target_histograms, + target_traceroute, +) # TODO remove business logic in this file @@ -283,8 +290,14 @@ def resolve(request, name): # route /api/stats +# +# TODO possible race condition: upon deletion, the /api/targets endpoint might have +# incorrect assumptions due to the statistics objects disappearing +@require_http_methods(["DELETE"]) def stats(request): - return HttpResponseServerError("not implemented") + Statistics.objects.all().delete() + Meta.objects.all().update(state=TargetState.UNKNOWN) + return JsonResponse({"success": True}) # route /api/targets From 6f34e3fbc4e70b1d0747cbb3ec480aa33da2cd2e Mon Sep 17 00:00:00 2001 From: liquid-metal <56035107+liquid-metal@users.noreply.github.com> Date: Sun, 6 Apr 2025 21:43:24 +0200 Subject: [PATCH 18/19] peer api endpoint --- meshping/meshping/views.py | 62 +++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/meshping/meshping/views.py b/meshping/meshping/views.py index 94cb3ef..5b17ef0 100644 --- a/meshping/meshping/views.py +++ b/meshping/meshping/views.py @@ -17,6 +17,7 @@ from django.views.decorators.http import require_http_methods from markupsafe import Markup +from .meshping.ifaces import Ifaces4 from .meshping import histodraw from .models import ( Statistics, @@ -268,8 +269,67 @@ def network(request): # route /peer +# +# Allows peers to POST a json structure such as this: +# { +# "targets": [ +# { "name": "raspi", "addr": "192.168.0.123", "local": true }, +# { "name": "google", "addr": "8.8.8.8", "local": false } +# ] +# } +# The non-local targets will then be added to our target list +# and stats will be returned for these targets (if known). +# Local targets will only be added if they are also local to us. +@require_http_methods(["POST"]) def peer(request): - return HttpResponseServerError("not implemented") + request_json = json.loads(request.body) + + if request_json is None: + return HttpResponseBadRequest("Please send content-type:application/json") + + if not isinstance(request_json.get("targets"), list): + return HttpResponseBadRequest("need targets as a list") + + ret_stats = [] + if4 = Ifaces4() + + for target in request_json["targets"]: + if not isinstance(target, dict): + return HttpResponseBadRequest("targets must be dicts") + if ( + not target.get("name", "").strip() + or not target.get("addr", "").strip() + or not isinstance(target.get("local"), bool) + ): + return HttpResponseBadRequest("required field missing in target") + + target["name"] = target["name"].strip() + target["addr"] = target["addr"].strip() + + if if4.is_interface(target["addr"]): + # no need to ping my own interfaces, ignore + continue + + if target["local"] and not if4.is_local(target["addr"]): + continue + + # See if we know this target already, otherwise create it. + tgt, created = Target.objects.get_or_create( + addr=target["addr"], name=target["name"] + ) + if created: + tgt_meta, _created = Meta.objects.get_or_create(target=tgt) + tgt_meta.is_foreign = True + tgt_meta.save() + target_stats, _created = Statistics.objects.get_or_create(target=tgt) + ret_stats.append(model_to_dict(target_stats)) + + return JsonResponse( + { + "success": True, + "targets": ret_stats, + } + ) # route /api/resolve/ From b41495b8d7e411f01102da8f720b6d644b45c713 Mon Sep 17 00:00:00 2001 From: liquid-metal <56035107+liquid-metal@users.noreply.github.com> Date: Sun, 6 Apr 2025 22:07:39 +0200 Subject: [PATCH 19/19] make pylint even happier --- .pylintrc | 2 +- meshping/meshping/meshping/histodraw.py | 22 +++++++++---------- meshping/meshping/meshping/meshping_config.py | 2 +- meshping/meshping/models.py | 1 + meshping/meshping/views.py | 8 +++++-- 5 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.pylintrc b/.pylintrc index e0de6a4..e4f3034 100644 --- a/.pylintrc +++ b/.pylintrc @@ -6,7 +6,7 @@ ignore-paths = meshping/.[^/]+/migrations max-line-length=88 [MESSAGES CONTROL] -disable=E1101,E1102,E1103,C0111,C0103,W0613,W0108,W0212,R0903,W0511 +disable=missing-function-docstring,missing-class-docstring,missing-module-docstring,no-member,fixme [DESIGN] max-public-methods=100 diff --git a/meshping/meshping/meshping/histodraw.py b/meshping/meshping/meshping/histodraw.py index 902407d..60d5107 100644 --- a/meshping/meshping/meshping/histodraw.py +++ b/meshping/meshping/meshping/histodraw.py @@ -13,7 +13,7 @@ # How big do you want the squares to be? -sqsz = 8 +SQSZ = 8 # [ @@ -67,8 +67,8 @@ def render_target(target): graph.putdata(pixels * 0xFF) # Scale graph so each Pixel becomes a square - width *= sqsz - height *= sqsz + width *= SQSZ + height *= SQSZ graph = graph.resize((width, height), Image.NEAREST) graph.hmin = hmin @@ -90,11 +90,11 @@ def render(targets, histogram_period): raise ValueError(f"No data available for target {target}") rendered_graphs.append(target_graph) - width = histogram_period // 3600 * sqsz + width = histogram_period // 3600 * SQSZ hmin = min(graph.hmin for graph in rendered_graphs) hmax = max(graph.hmax for graph in rendered_graphs) tmax = max(graph.tmax for graph in rendered_graphs) - height = (hmax - hmin) * sqsz + height = (hmax - hmin) * SQSZ if len(rendered_graphs) == 1: # Single graph -> use it as-is @@ -115,7 +115,7 @@ def render(targets, histogram_period): new_graph = Image.new("L", (width, height), "black") new_graph.paste( graph, - (width - graph.width - sqsz * dtmax, (hmax - graph.hmax) * sqsz), + (width - graph.width - SQSZ * dtmax, (hmax - graph.hmax) * SQSZ), ) else: new_graph = graph @@ -182,7 +182,7 @@ def render(targets, histogram_period): # Y axis ticks and annotations for hidx in range(hmin, hmax, 5): bottomrow = hidx - hmin - offset_y = height + graph_y - bottomrow * sqsz - 1 + offset_y = height + graph_y - bottomrow * SQSZ - 1 draw.line((graph_x - 2, offset_y, graph_x + 2, offset_y), fill=0xAAAAAA) ping = 2 ** (hidx / 10.0) @@ -205,11 +205,11 @@ def render(targets, histogram_period): t_hist_begin = t_hist_end - timedelta(hours=td_hours) # X axis ticks - one every two hours - for col in range(1, width // sqsz): + for col in range(1, width // SQSZ): # We're now at hour indicated by col if (t_hist_begin + timedelta(hours=col)).hour % 2 != 0: continue - offset_x = graph_x + col * sqsz + offset_x = graph_x + col * SQSZ draw.line( (offset_x, height + graph_y - 2, offset_x, height + graph_y + 2), fill=0xAAAAAA, @@ -223,12 +223,12 @@ def render(targets, histogram_period): tmpdraw = ImageDraw.Draw(tmpim) # Draw one annotation every four hours - for col in range(0, width // sqsz + 1): + for col in range(0, width // SQSZ + 1): # We're now at hour indicated by col tstamp = t_hist_begin + timedelta(hours=col) if tstamp.hour % 4 != 0: continue - offset_x = col * sqsz + offset_x = col * SQSZ if tstamp.hour == 0: tmpdraw.text( (0, offset_x + 4), tstamp.strftime("%m-%d"), 0x333333, font=font diff --git a/meshping/meshping/meshping/meshping_config.py b/meshping/meshping/meshping/meshping_config.py index df3200f..1a7f18d 100644 --- a/meshping/meshping/meshping/meshping_config.py +++ b/meshping/meshping/meshping/meshping_config.py @@ -2,7 +2,7 @@ # TODO decide about option for config file, and automatic combination with docker # environment variables # -# pylint: disable=too-many-instance-attributes +# pylint: disable=too-many-instance-attributes,too-few-public-methods class MeshpingConfig: def __init__(self): self.ping_timeout = 5 diff --git a/meshping/meshping/models.py b/meshping/meshping/models.py index 254585a..9ec6ac8 100644 --- a/meshping/meshping/models.py +++ b/meshping/meshping/models.py @@ -8,6 +8,7 @@ class Target(models.Model): addr = models.CharField(max_length=255, unique=True) name = models.CharField(max_length=255) + # pylint: disable=too-few-public-methods class Meta: unique_together = ("addr", "name") diff --git a/meshping/meshping/views.py b/meshping/meshping/views.py index 5b17ef0..bd89400 100644 --- a/meshping/meshping/views.py +++ b/meshping/meshping/views.py @@ -1,3 +1,6 @@ +# We cannot rename view parameters as django spllies them as keyword arguments, so: +# pylint: disable=unused-argument + import json import os import socket @@ -58,6 +61,9 @@ def read_svg_file(icons_dir, filename): # route /histogram//.png +# +# we cannot rename node to _node as django calls by keyword argument, so ignore this +# pylint: disable=unused-argument @require_http_methods(["GET"]) def histogram(request, node, target): targets = [] @@ -89,8 +95,6 @@ def histogram(request, node, target): # route /metrics -# -# TODO is the metrics output valid prometheus format when no values are present? @require_http_methods(["GET"]) def metrics(request): respdata = [