[WIP] Introduce django

.gitignore

@@ -1,5 +1,6 @@
 python-ping/
 
+__pycache__
 *.pyc
 
 oping.c
@@ -8,3 +9,8 @@ build/
 
 ui/node_modules/
 db/
+
+meshping/ui/node_modules/
+meshping/db/
+
+venv/

.pylintrc

@@ -1,11 +1,12 @@
 [MASTER]
 extension-pkg-whitelist = netifaces
+ignore-paths = meshping/.[^/]+/migrations
 
 [FORMAT]
-max-line-length=120
+max-line-length=88
 
 [MESSAGES CONTROL]
-disable=E1101,E1102,E1103,C0111,C0103,W0613,W0108,W0212,R0903
+disable=missing-function-docstring,missing-class-docstring,missing-module-docstring,no-member,fixme
 
 [DESIGN]
 max-public-methods=100

meshping/manage.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""Django's command-line utility for administrative tasks."""
+import os
+import sys
+
+
+def main():
+    """Run administrative tasks."""
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "meshping.settings")
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)
+
+
+if __name__ == "__main__":
+    main()

meshping/meshping/apps.py

@@ -0,0 +1,29 @@
+from django.apps import AppConfig
+from .meshping.meshping_config import MeshpingConfig as MPCOnfig
+
+
+class MeshpingConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "meshping"
+
+    # TODO decide for long-term layout: standard threads, async, scheduling library
+    # TODO start background threads for traceroute, peers
+    # TODO with the current thread model, django complains about db access before app
+    #      initialization is completed
+    # TODO create background task for db housekeeping (prune_histograms)
+    #
+    # delayed import, otherwise we will get AppRegistryNotReady
+    # pylint: disable=import-outside-toplevel
+    def ready(self):
+        from .meshping.peering_thread import PeeringThread
+        from .meshping.ping_thread import PingThread
+        from .meshping.traceroute_thread import TracerouteThread
+
+        mp_config = MPCOnfig()
+        peering_thread = PeeringThread(mp_config=mp_config, daemon=True)
+        ping_thread = PingThread(mp_config=mp_config, daemon=True)
+        traceroute_thread = TracerouteThread(mp_config=mp_config, daemon=True)
+
+        peering_thread.start()
+        ping_thread.start()
+        traceroute_thread.start()

meshping/meshping/asgi.py

@@ -0,0 +1,7 @@
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "meshping.settings")
+
+application = get_asgi_application()

meshping/meshping/jinja2.py

@@ -0,0 +1,14 @@
+from jinja2 import Environment
+from django.urls import reverse
+from django.templatetags.static import static
+
+
+def environment(**options):
+    env = Environment(variable_start_string="{[", variable_end_string="]}", **options)
+    env.globals.update(
+        {
+            "static": static,
+            "url": reverse,
+        }
+    )
+    return env

meshping/meshping/meshping/histodraw.py

@@ -0,0 +1,246 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# kate: space-indent on; indent-width 4; replace-tabs on;
+
+import socket
+import os
+from datetime import timedelta
+import numpy as np
+import pandas
+
+from PIL import Image, ImageDraw, ImageFont, ImageOps
+from ..models import target_histograms
+
+
+# How big do you want the squares to be?
+SQSZ = 8
+
+
+# [
+#   {
+#     "timestamp": 1743883200,
+#     "47": 2,
+#     "50": null,
+#     ...
+#   },
+#   ...
+# ]
+def render_target(target):
+    histograms_df = target_histograms(target)
+    if histograms_df.empty:
+        return None
+
+    # Normalize Buckets by transforming the number of actual pings sent
+    # into a float [0..1] indicating the grayness of that bucket.
+    biggestbkt = histograms_df.max().max()
+    histograms_df = histograms_df.div(biggestbkt, axis="index")
+    # prune outliers -> keep only values > 5%
+    histograms_df = histograms_df[histograms_df > 0.05]
+    # drop columns that contain only NaNs now
+    histograms_df = histograms_df.dropna(axis="columns", how="all")
+    # fill missing _rows_ (aka, hours) with rows of just NaN
+    histograms_df = histograms_df.asfreq("1h")
+    # replace all the NaNs with 0
+    histograms_df = histograms_df.fillna(0)
+
+    # detect dynamic range, and round to the nearest multiple of 10.
+    # this ensures that the ticks are drawn at powers of 2, which makes
+    # the graph more easily understandable. (I hope.)
+    # Btw:  27 // 10 * 10   # =  20
+    #      -27 // 10 * 10   # = -30
+    # hmax needs to be nearest power of 10 + 1 for the top tick to be drawn.
+    hmin = histograms_df.columns.min() // 10 * 10
+    hmax = histograms_df.columns.max() // 10 * 10 + 11
+
+    # Draw the graph in a pixels array which we then copy to an image
+    height = hmax - hmin + 1
+    width = len(histograms_df)
+    pixels = np.zeros(width * height)
+
+    for col, (_tstamp, histogram) in enumerate(histograms_df.iterrows()):
+        for bktval, bktgrayness in histogram.items():
+            #       (     y       )            (x)
+            pixels[((hmax - bktval) * width) + col] = bktgrayness
+
+    # copy pixels to an Image and paste that into the output image
+    graph = Image.new("L", (width, height))
+    graph.putdata(pixels * 0xFF)
+
+    # Scale graph so each Pixel becomes a square
+    width *= SQSZ
+    height *= SQSZ
+
+    graph = graph.resize((width, height), Image.NEAREST)
+    graph.hmin = hmin
+    graph.hmax = hmax
+    graph.tmin = histograms_df.index.min()
+    graph.tmax = histograms_df.index.max()
+    return graph
+
+
+# TODO maybe this method is a bit long? on the other hand, manual rendering code always
+#      tends to be tedious
+# pylint: disable=too-many-locals,too-many-branches,too-many-statements
+def render(targets, histogram_period):
+    rendered_graphs = []
+
+    for target in targets:
+        target_graph = render_target(target)
+        if target_graph is None:
+            raise ValueError(f"No data available for target {target}")
+        rendered_graphs.append(target_graph)
+
+    width = histogram_period // 3600 * SQSZ
+    hmin = min(graph.hmin for graph in rendered_graphs)
+    hmax = max(graph.hmax for graph in rendered_graphs)
+    tmax = max(graph.tmax for graph in rendered_graphs)
+    height = (hmax - hmin) * SQSZ
+
+    if len(rendered_graphs) == 1:
+        # Single graph -> use it as-is
+        graph = Image.new("L", (width, height), "white")
+        graph.paste(
+            ImageOps.invert(rendered_graphs[0]), (width - rendered_graphs[0].width, 0)
+        )
+    else:
+        # Multiple graphs -> merge.
+        # This width/height may not match what we need for the output.
+        # Check for which graphs that is the case, and for these,
+        # create a new image that has the correct size and paste
+        # the graph into it.
+        resized_graphs = []
+        for graph in rendered_graphs:
+            dtmax = (tmax - graph.tmax) // pandas.Timedelta(hours=1)
+            if graph.width != width or graph.height != height:
+                new_graph = Image.new("L", (width, height), "black")
+                new_graph.paste(
+                    graph,
+                    (width - graph.width - SQSZ * dtmax, (hmax - graph.hmax) * SQSZ),
+                )
+            else:
+                new_graph = graph
+
+            resized_graphs.append(new_graph)
+
+        while len(resized_graphs) != 3:
+            resized_graphs.append(Image.new("L", (width, height), "black"))
+
+        # Print the graph, on black background still.
+        graph = Image.merge("RGB", resized_graphs)
+
+        # To get a white background, convert to HSV and set V=1.
+        # V currently contains the interesting information though,
+        # so move that to S first.
+        hsv = np.array(graph.convert("HSV"))
+        # Add V to S (not sure why adding works better than replacing, but it does)
+        hsv[:, :, 1] = hsv[:, :, 1] + hsv[:, :, 2]
+        # Set V to 1
+        hsv[:, :, 2] = np.ones((height, width)) * 0xFF
+        graph = Image.fromarray(hsv, "HSV").convert("RGB")
+
+    # position of the graph
+    graph_x = 70
+    graph_y = 30 * len(targets) + 10
+
+    # im will hold the output image
+    im = Image.new("RGB", (graph_x + width + 20, graph_y + height + 100), "white")
+    im.paste(graph, (graph_x, graph_y))
+
+    # draw a rect around the graph
+    draw = ImageDraw.Draw(im)
+    draw.rectangle(
+        (graph_x, graph_y, graph_x + width - 1, graph_y + height - 1), outline=0x333333
+    )
+
+    try:
+        font = ImageFont.truetype("DejaVuSansMono.ttf", 10)
+        lgfont = ImageFont.truetype("DejaVuSansMono.ttf", 16)
+    except IOError:
+        font = ImageFont.truetype(
+            "/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf", 10
+        )
+        lgfont = ImageFont.truetype(
+            "/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf", 16
+        )
+
+    # Headline
+    if len(targets) == 1:  # just black for a single graph
+        targets_with_colors = zip(targets, (0x000000,))
+    else:  # red, green, blue for multiple graphs
+        targets_with_colors = zip(targets, (0x0000FF, 0x00FF00, 0xFF0000))
+
+    for idx, (target, color) in enumerate(targets_with_colors):
+        headline_text = f"{socket.gethostname()} → {target.label}"
+        headline_width = lgfont.getlength(headline_text)
+        draw.text(
+            ((graph_x + width + 20 - headline_width) // 2, 30 * idx + 11),
+            headline_text,
+            color,
+            font=lgfont,
+        )
+
+    # Y axis ticks and annotations
+    for hidx in range(hmin, hmax, 5):
+        bottomrow = hidx - hmin
+        offset_y = height + graph_y - bottomrow * SQSZ - 1
+        draw.line((graph_x - 2, offset_y, graph_x + 2, offset_y), fill=0xAAAAAA)
+
+        ping = 2 ** (hidx / 10.0)
+        label = f"{ping:.2f}"
+        draw.text(
+            (graph_x - len(label) * 6 - 10, offset_y - 5), label, 0x333333, font=font
+        )
+
+    # Calculate the times at which the histogram begins and ends.
+    t_hist_end = (
+        # Latest hour for which we have data...
+        tmax.tz_localize("Etc/UTC")
+        .tz_convert(os.environ.get("TZ", "Etc/UTC"))
+        .to_pydatetime()
+        # Plus the current hour which we're also drawing on screen
+        + timedelta(hours=1)
+    )
+
+    td_hours = histogram_period // 3600
+    t_hist_begin = t_hist_end - timedelta(hours=td_hours)
+
+    # X axis ticks - one every two hours
+    for col in range(1, width // SQSZ):
+        # We're now at hour indicated by col
+        if (t_hist_begin + timedelta(hours=col)).hour % 2 != 0:
+            continue
+        offset_x = graph_x + col * SQSZ
+        draw.line(
+            (offset_x, height + graph_y - 2, offset_x, height + graph_y + 2),
+            fill=0xAAAAAA,
+        )
+
+    # X axis annotations
+    # Create a temp image for the bottom label that we then rotate by 90° and attach to
+    # the other one since this stuff is rotated by 90° while we create it, all the
+    # coordinates are inversed...
+    tmpim = Image.new("RGB", (80, width + 20), "white")
+    tmpdraw = ImageDraw.Draw(tmpim)
+
+    # Draw one annotation every four hours
+    for col in range(0, width // SQSZ + 1):
+        # We're now at hour indicated by col
+        tstamp = t_hist_begin + timedelta(hours=col)
+        if tstamp.hour % 4 != 0:
+            continue
+        offset_x = col * SQSZ
+        if tstamp.hour == 0:
+            tmpdraw.text(
+                (0, offset_x + 4), tstamp.strftime("%m-%d"), 0x333333, font=font
+            )
+        tmpdraw.text((36, offset_x + 4), tstamp.strftime("%H:%M"), 0x333333, font=font)
+
+    im.paste(tmpim.rotate(90, expand=1), (graph_x - 10, height + graph_y + 1))
+
+    # This worked pretty well for Tobi Oetiker...
+    tmpim = Image.new("RGB", (170, 13), "white")
+    tmpdraw = ImageDraw.Draw(tmpim)
+    tmpdraw.text((0, 0), "Meshping by Michael Ziegler", 0x999999, font=font)
+    im.paste(tmpim.rotate(270, expand=1), (width + graph_x + 7, graph_y))
+
+    return im