From 451113c89af360f9a3eb47782ca39815bc277727 Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 9 Apr 2025 21:47:46 +0000 Subject: [PATCH 001/155] fix name escaping issue in map javascript --- templates/map.html.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/map.html.j2 b/templates/map.html.j2 index f77ce806..cf30326c 100644 --- a/templates/map.html.j2 +++ b/templates/map.html.j2 @@ -149,7 +149,7 @@ nodes['{{ id }}'] = { id: '{{ id }}', short_name: '{{ node.short_name }}', - long_name: '{{ node.long_name }}', + long_name: '{{ node.long_name | escapejs }}', last_seen: '{{ time_ago(node.ts_seen) }}', position: [{{ node.position.longitude_i / 10000000 }}, {{ node.position.latitude_i / 10000000 }}], online: {% if node.active %}true{% else %}false{% endif %}, From 8de0b1e727c087ff0c70a3ff67c669f6946fb47d Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 9 Apr 2025 22:21:08 +0000 Subject: [PATCH 002/155] fixed two map long_name escaping issues --- templates/map.html.j2 | 2 +- templates/node.html.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/map.html.j2 b/templates/map.html.j2 index cf30326c..56104355 100644 --- a/templates/map.html.j2 +++ b/templates/map.html.j2 @@ -149,7 +149,7 @@ nodes['{{ id }}'] = { id: '{{ id }}', short_name: '{{ node.short_name }}', - long_name: '{{ node.long_name | escapejs }}', + long_name: '{{ node.long_name | e }}', last_seen: '{{ time_ago(node.ts_seen) }}', position: [{{ node.position.longitude_i / 10000000 }}, {{ node.position.latitude_i / 10000000 }}], online: {% if node.active %}true{% else %}false{% endif %}, diff --git a/templates/node.html.j2 b/templates/node.html.j2 index 0ff5c092..89cf61bc 100644 --- a/templates/node.html.j2 +++ b/templates/node.html.j2 @@ -517,7 +517,7 @@ var node = { id: '{{ node.id }}', short_name: '{{ node.short_name }}', - long_name: '{{ node.long_name }}', + long_name: '{{ node.long_name | e }}', last_seen: '{{ node.last_seen }}', position: { latitude: 38.575764, From fc8a44acae015ec97a08f8f7c539978e3dbbf611 Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 10 Apr 2025 02:15:41 +0000 Subject: [PATCH 003/155] added caching to pages to reduce load on near-static pages --- meshinfo_web.py | 9 ++++++++- requirements.txt | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/meshinfo_web.py b/meshinfo_web.py index df06127e..19974c5e 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -8,6 +8,7 @@ url_for, abort ) +from flask_caching import Cache from waitress import serve from paste.translogger import TransLogger import configparser @@ -29,7 +30,9 @@ app = Flask(__name__) -# Make timezone utilities available to templates +cache = Cache(app, config={'CACHE_TYPE': 'simple'}) + +# Make globals available to templates app.jinja_env.globals.update(convert_to_local=convert_to_local) app.jinja_env.globals.update(format_timestamp=format_timestamp) app.jinja_env.globals.update(time_ago=time_ago) @@ -60,6 +63,7 @@ def not_found(e): # Serve static files from the root directory @app.route('/') +@cache.cached(timeout=60) # Cache for 60 seconds def serve_index(success_message=None, error_message=None): md = MeshData() nodes = md.get_nodes() @@ -76,6 +80,7 @@ def serve_index(success_message=None, error_message=None): @app.route('/nodes.html') +@cache.cached(timeout=60) # Cache for 60 seconds def nodes(): md = MeshData() nodes = md.get_nodes() @@ -95,6 +100,7 @@ def nodes(): ) @app.route('/allnodes.html') +@cache.cached(timeout=60) # Cache for 60 seconds def allnodes(): md = MeshData() nodes = md.get_nodes() @@ -294,6 +300,7 @@ def traceroute_map(): ) @app.route('/graph.html') +@cache.cached(timeout=60) # Cache for 60 seconds def graph(): md = MeshData() nodes = md.get_nodes() diff --git a/requirements.txt b/requirements.txt index 6e2830a5..3e6368ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,5 @@ rasterio scipy geopy boto3 -pytz \ No newline at end of file +pytz +Flask-Caching \ No newline at end of file From 984d4ccf7673737d3211f6c38413eeadd61946ec Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 10 Apr 2025 17:05:09 +0000 Subject: [PATCH 004/155] fixed get_neighbors sql query logic --- meshdata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meshdata.py b/meshdata.py index 5ff0cdbb..168eea65 100644 --- a/meshdata.py +++ b/meshdata.py @@ -174,7 +174,7 @@ def get_neighbors(self, id): LEFT OUTER JOIN position p1 ON p1.id = a.id LEFT OUTER JOIN position p2 ON p2.id = a.neighbor_id WHERE a.id = %s -AND a.ts_created < NOW() - INTERVAL 1 DAY +AND a.ts_created >= NOW() - INTERVAL 1 DAY """ params = (id, ) cur = self.db.cursor() From aaf7e21373756828f6fe662aa859327379b13eaa Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 13 Apr 2025 03:15:36 +0000 Subject: [PATCH 005/155] profile image changes: fixed emojis in profile images, added config.ini settings, optimized image generation --- Dockerfile | 47 +++++--- config.ini.sample | 7 +- docker-compose.yml | 1 + meshinfo_los_profile.py | 248 +++++++++++++++++++++++++++++----------- meshinfo_web.py | 45 +++++--- requirements.txt | 5 +- templates/node.html.j2 | 2 +- 7 files changed, 252 insertions(+), 103 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6c13aa8a..d02bb4a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,32 +1,53 @@ # trunk-ignore-all(checkov/CKV_DOCKER_3) -FROM python:3.13-slim +FROM python:3.13.3-slim-bookworm -LABEL org.opencontainers.image.source=https://github.com/dadecoza/meshinfo-lite +LABEL org.opencontainers.image.source=https://github.com/agessaman/meshinfo-lite LABEL org.opencontainers.image.description="Realtime web UI to run against a Meshtastic regional or private mesh network." ENV MQTT_TLS=false -ENV PYTHONUNBUFFERED=1 +ENV PYTHONUNBUFFERED=1 \ + # Set standard locations + PATH="/app/.local/bin:${PATH}" \ + # Consistent port for the app + APP_PORT=8000 + +RUN groupadd --system app && \ + useradd --system --gid app --home-dir /app --create-home app # Set the working directory in the container -RUN mkdir /app WORKDIR /app -RUN apt-get update && apt-get -y install \ - libexpat1 libexpat1-dev +# Install system dependencies +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + libexpat1 \ + libcairo2 \ + pkg-config \ + fonts-symbola \ + fontconfig \ + freetype2-demos \ + && apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN fc-cache -fv COPY requirements.txt banner run.sh ./ -COPY *.py ./ -COPY www ./www -COPY templates ./templates -COPY migrations ./migrations RUN pip install --upgrade pip -RUN pip install --no-cache-dir -r requirements.txt +RUN su app -c "pip install --no-cache-dir --user -r requirements.txt" -HEALTHCHECK NONE +COPY --chown=app:app banner run.sh ./ +COPY --chown=app:app *.py ./ +COPY --chown=app:app www ./www +COPY --chown=app:app templates ./templates +COPY --chown=app:app migrations ./migrations -EXPOSE 8080 +HEALTHCHECK NONE RUN chmod +x run.sh +USER app + +EXPOSE ${APP_PORT} + CMD ["./run.sh"] diff --git a/config.ini.sample b/config.ini.sample index 10cc3e26..d46ba32a 100644 --- a/config.ini.sample +++ b/config.ini.sample @@ -42,4 +42,9 @@ jwt_secret=6219c1c2364499639ce9d16c33863c4f6fedb7a4a1a0f29524c20d95cb00e5f5 email=YOUR_EMAIL_ADDRESS password=SMTP_PASSWORD server=SMTP_SERVER -port=SMTP_PORT \ No newline at end of file +port=SMTP_PORT + +[los] +enabled=false +max_distance=10000 +cache_duration=43200 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 2de80d13..83d71ce5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,7 @@ services: - mariadb volumes: - ./config.ini:/app/config.ini + - ./srtm_data:/app/srtm_data environment: - PYTHONUNBUFFERED=1 ports: diff --git a/meshinfo_los_profile.py b/meshinfo_los_profile.py index ffc2c8c6..017e75f3 100644 --- a/meshinfo_los_profile.py +++ b/meshinfo_los_profile.py @@ -3,20 +3,30 @@ import os import rasterio import numpy as np +import matplotlib +import logging import matplotlib.pyplot as plt +from matplotlib.font_manager import FontProperties from scipy.spatial import distance from geopy.distance import geodesic import logging import time import io import base64 +import pandas as pd +import atexit class LOSProfile(): - def __init__(self, nodes={}, node=None): + def __init__(self, nodes={}, node=None, config=None, cache=None): self.nodes = nodes self.node = node self.datasets = [] + self.max_distance = int(config['los']['max_distance']) if config and 'los' in config else 5000 # Default to 5000 if not set + self.cache_duration = int(config['los']['cache_duration']) if config and 'los' in config else 43200 # Default to 43200 if not set + + self.cache = cache # Store the cache object + directory = "srtm_data" try: for filename in os.listdir(directory): @@ -25,8 +35,16 @@ def __init__(self, nodes={}, node=None): dataset = rasterio.open(filepath) self.datasets.append(dataset) except FileNotFoundError as e: - logging.warning("No SRTM data") + logging.warning("No SRTM data found in directory: %s", directory) pass + atexit.register(self.close_datasets) + + def close_datasets(self): + """Close all open rasterio datasets.""" + for ds in self.datasets: + if not ds.closed: + ds.close() + self.datasets = [] def calculate_distance_between_coords(self, coord1, coord2): lat1, lon1 = coord1 @@ -66,19 +84,55 @@ def read_elevation_from_tifb(self, lat, lon): def read_elevation_from_tif(self, lat, lon): """ - Reads elevation data from SRTM .tif files in the given directory for a - specific coordinate. + Reads elevation data from preloaded .tif files for a specific + coordinate using efficient windowed reading. """ for dataset in self.datasets: + # Check if the coordinate is within the bounds of this dataset if dataset.bounds.left <= lon <= dataset.bounds.right \ and dataset.bounds.bottom <= lat <= dataset.bounds.top: - row, col = dataset.index(lon, lat) - elevation = dataset.read(1)[row, col] - return elevation - logging.warning( - f"No elevation data found for coordinates ({lat}, {lon})" - ) - return [] + try: + # Get the row and column index for the coordinate + row, col = dataset.index(lon, lat) + + # Read only the required pixel using a window + # Ensure indices are within dataset dimensions + if 0 <= row < dataset.height and 0 <= col < dataset.width: + # Read a 1x1 window at the specified row, col + elevation = dataset.read( + 1, + window=((row, row + 1), (col, col + 1)) + )[0, 0] # Extract the single value + + # Check for NoData values (important!) + # Use dataset.nodatavals tuple if multiple bands exist, otherwise dataset.nodata + nodata_val = dataset.nodata + # Handle potential float nodata comparison issues + if nodata_val is not None and np.isclose(float(elevation), float(nodata_val)): + logging.warning( + f"NoData value found at ({lat}, {lon}) in {dataset.name}" + ) + return None # Return None or a specific indicator for NoData + elif np.isnan(elevation): + logging.warning( + f"NaN value found at ({lat}, {lon}) in {dataset.name}" + ) + return None # Return None or indicator for NaN + + return elevation # Return the valid elevation + else: + logging.warning(f"Calculated index ({row}, {col}) out of bounds for dataset {dataset.name}") + return None # Index out of bounds + + except Exception as e: + logging.error(f"Error reading elevation for ({lat}, {lon}) from {dataset.name}: {e}") + return None # Error during read + + # If coordinate wasn't found in any dataset bounds + # logging.debug( # Changed to debug as this can be noisy + # f"Coordinate ({lat}, {lon}) not within bounds of any loaded dataset." + # ) + return None # Coordinate not found in any dataset def generate_los_profile(self, coord1, coord2, resolution=100): """ @@ -94,9 +148,24 @@ def generate_los_profile(self, coord1, coord2, resolution=100): longitudes = np.linspace(lon1, lon2, resolution) elevations = [] + valid_points = 0 for lat, lon in zip(latitudes, longitudes): elevation = self.read_elevation_from_tif(lat, lon) - elevations.append(elevation) + # Handle cases where elevation might be None (NoData, NaN, out of bounds) + if elevation is not None: + elevations.append(elevation) + valid_points += 1 + else: + # Decide how to handle missing points: + # Option 1: Append a placeholder like NaN (if plotting handles it) + elevations.append(np.nan) + # Option 2: Skip the point (distances array would need adjustment) + # Option 3: Interpolate (more complex) + + # Check if enough valid points were found + if valid_points < 2: # Need at least start and end for a line + logging.warning(f"Could not retrieve enough elevation points between {coord1[:2]} and {coord2[:2]}. Skipping profile.") + return None, None # Indicate failure # Compute accurate geodesic distances from start point distances = [ @@ -104,68 +173,96 @@ def generate_los_profile(self, coord1, coord2, resolution=100): zip(latitudes, longitudes) ] - profile = [elevation for elevation in elevations] + profile = [elev if not np.isnan(elev) else 0 for elev in elevations] # Replace NaN with 0 for now, adjust if needed # Add altitude to elevation for the final profile - if alt1: - profile[0] = alt1 + # Ensure profile has elements before accessing indices + if profile: + # Use provided altitude if available, otherwise use terrain elevation + profile[0] = alt1 if alt1 is not None else (profile[0] if not np.isnan(profile[0]) else 0) + profile[-1] = alt2 if alt2 is not None else (profile[-1] if not np.isnan(profile[-1]) else 0) + + # Simple linear interpolation for NaN values in the middle + profile_series = pd.Series(profile) + profile_series.interpolate(method='linear', inplace=True) + profile = profile_series.tolist() - if alt2: - profile[-1] = alt2 return distances, profile def plot_los_profile(self, distances, profile, label): - """ - Plots the line-of-sight profile (including altitude) as a solid graph - and saves it as a PNG image. - Additionally, plots a direct straight-line path for comparison. - """ - plt.figure(figsize=(10, 6)) - plt.gca().set_facecolor("cyan") + # Create a unique cache key based on the input parameters + cache_key = f"los_profile_{label}_{hash(tuple(distances))}_{hash(tuple(profile))}" + + # Check if the image is already cached + cached_image = self.cache.get(cache_key) # Use self.cache + if cached_image: + return cached_image # Return the cached image if it exists + + # --- Font Setup --- + # Attempt to load Symbola font + try: + symbola_font_path = '/usr/share/fonts/truetype/ancient-scripts/Symbola_hint.ttf' + symbol_font = FontProperties(fname=symbola_font_path) + except FileNotFoundError: + logging.warning(f"Could not find font file at specified path: {symbola_font_path}. Symbols/Emojis may not render correctly.") + symbol_font = None # Fallback to default + except ValueError: + logging.warning("Error loading font from specified path, even if found. Symbols/Emojis may not render correctly.") + symbol_font = None # Fallback to default + + fig = plt.figure(figsize=(10, 6)) + ax = fig.gca() + ax.set_facecolor("cyan") plt.margins(x=0, y=0, tight=True) - # Direct line (interpolated between start and end altitudes) - direct_line = np.linspace(profile[0], profile[-1], len(profile)) + # Check if profile has valid data + if not profile or len(profile) < 2: + logging.warning(f"Profile data is invalid or too short for plotting label: {label}") + plt.close(fig) + return None + + start_alt = profile[0] + end_alt = profile[-1] + + if np.isnan(start_alt) or np.isnan(end_alt): + logging.warning(f"Cannot plot direct LOS line due to NaN start/end altitude for label: {label}") + direct_line = np.full(len(profile), np.nan) + else: + direct_line = np.linspace(start_alt, end_alt, len(profile)) # Plot the terrain profile - plt.fill_between( - distances, - profile, - color="brown", - alpha=1.0, - label="Terrain Profile" - ) - plt.plot( - distances, - profile, - color="brown", - label="Profile Outline" - ) + if not np.isnan(profile).all(): + plt.fill_between(distances, profile, color="brown", alpha=1.0, label="Terrain Profile") + plt.plot(distances, profile, color="brown", label="Profile Outline") + else: + logging.warning(f"Terrain profile contains only NaN values for label: {label}") # Plot the direct LOS line - plt.plot( - distances, - direct_line, - color="green", - linestyle="dashed", - linewidth=2, - label="Direct LOS Line" - ) + if not np.isnan(direct_line).all(): + plt.plot(distances, direct_line, color="green", linestyle="dashed", linewidth=2, label="Direct LOS Line") - plt.xlabel("Distance (meters)") - plt.ylabel("Elevation + Altitude (meters)") - plt.title(f"{label}") + # --- Apply FontProperties (using symbol_font) --- + plt.xlabel("Distance (meters)", fontproperties=symbol_font) + plt.ylabel("Elevation + Altitude (meters)", fontproperties=symbol_font) + plt.title(f"{label}", fontproperties=symbol_font) + plt.legend(prop=symbol_font) - # Save the plot to a BytesIO buffer buffer = io.BytesIO() - plt.savefig(buffer, format="png", bbox_inches="tight") + try: + plt.savefig(buffer, format="png", bbox_inches="tight") + buffer.seek(0) + img_base64 = base64.b64encode(buffer.getvalue()).decode("utf-8") - # Encode image to Base64 - buffer.seek(0) - img_base64 = base64.b64encode(buffer.getvalue()).decode("utf-8") + # Cache the rendered image for 12 hours (43,200 seconds) + self.cache.set(cache_key, img_base64, timeout=self.cache_duration) - # Print Base64 output (or return it from a function) - return img_base64 + except Exception as e: + logging.error(f"Error saving plot to buffer for label {label}: {e}") + img_base64 = None + finally: + plt.close(fig) + return img_base64 + def get_profiles(self): profiles = {} hexid = utils.convert_node_id_from_int_to_hex(self.node) @@ -197,35 +294,50 @@ def get_profiles(self): (lat, lon) ) - if (dist and dist < 20000): + if (dist and dist < self.max_distance): coord1 = (mylat, mylon, myalt) coord2 = (lat, lon, alt) - output_path = f"altitude_{c}.png" + # output_path = f"altitude_{c}.png" # Not used, can remove c += 1 lname1 = mynode["long_name"] sname1 = mynode["short_name"] lname2 = node["long_name"] sname2 = node["short_name"] + # --- Ensure original label is used --- label = f"{lname1} ({sname1}) <=> {lname2} ({sname2})" + # ------------------------------------ distances, profile = self.generate_los_profile( coord1, coord2 ) - image = self.plot_los_profile( - distances, - profile, - label - ) - profiles[node_id] = { - "image": image, - "distance": dist - } + # Check if profile generation was successful + if distances is not None and profile is not None: + # Pass the original label with emojis + image = self.plot_los_profile( + distances, + profile, + label + ) + # Check if plotting was successful + if image is not None: + profiles[node_id] = { + "image": image, + "distance": dist + } + else: + logging.warning(f"Failed to generate plot for profile: {label}") + else: + logging.warning(f"Failed to generate profile data between {coord1[:2]} and {coord2[:2]}") + except KeyError as e: pass + # logging.warning(f"Missing key 'position' or coordinates for node {node_id} or {hexid}: {e}") except TypeError as e: pass return profiles + def __del__(self): + self.close_datasets() if __name__ == "__main__": from meshdata import MeshData diff --git a/meshinfo_web.py b/meshinfo_web.py index 19974c5e..c6c98ef9 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -797,8 +797,14 @@ def serve_static(filename): node_telemetry = md.get_node_telemetry(node_id) node_route = md.get_route_coordinates(node_id) telemetry_graph = draw_graph(node_telemetry) - lp = LOSProfile(nodes, node_id) - + lp = LOSProfile(nodes, node_id, config, cache) + + # Get the max_distance from config, default to 5000 meters (5 km) + max_distance_km = int(config.get("los", "max_distance", fallback=5000)) / 1000 # Convert to kilometers + + # Check if LOS Profile rendering is enabled + los_enabled = config.getboolean("los", "enabled", fallback=False) + # Get timeout from config zero_hop_timeout = int(config.get("server", "zero_hop_timeout", fallback=43200)) # Default 12 hours cutoff_time = int(time.time()) - zero_hop_timeout @@ -859,23 +865,24 @@ def serve_static(filename): cursor.close() return render_template( - f"node.html.j2", - auth=auth(), - config=config, - node=nodes[node], - nodes=nodes, - hardware=meshtastic_support.HardwareModel, - meshtastic_support=meshtastic_support, - los_profiles=lp.get_profiles(), - telemetry_graph=telemetry_graph, - node_route=node_route, - utils=utils, - datetime=datetime.datetime, - timestamp=datetime.datetime.now(), - zero_hop_heard=zero_hop_heard, - zero_hop_heard_by=zero_hop_heard_by, - zero_hop_timeout=zero_hop_timeout, - ) + f"node.html.j2", + auth=auth(), + config=config, + node=nodes[node], + nodes=nodes, + hardware=meshtastic_support.HardwareModel, + meshtastic_support=meshtastic_support, + los_profiles=lp.get_profiles() if los_enabled else {}, # Render only if enabled + telemetry_graph=telemetry_graph, + node_route=node_route, + utils=utils, + datetime=datetime.datetime, + timestamp=datetime.datetime.now(), + zero_hop_heard=zero_hop_heard, + zero_hop_heard_by=zero_hop_heard_by, + zero_hop_timeout=zero_hop_timeout, + max_distance=max_distance_km + ) if re.match(userp, filename): match = re.match(userp, filename) diff --git a/requirements.txt b/requirements.txt index 3e6368ce..a03d52b3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,9 +9,12 @@ colorlog bcrypt pyjwt matplotlib +cairocffi rasterio scipy geopy boto3 pytz -Flask-Caching \ No newline at end of file +Flask-Caching +pandas +atexit \ No newline at end of file diff --git a/templates/node.html.j2 b/templates/node.html.j2 index 89cf61bc..9c822dd4 100644 --- a/templates/node.html.j2 +++ b/templates/node.html.j2 @@ -464,7 +464,7 @@ {% if los_profiles %} -
Line-of-sight with nodes in 20km radius
+
Line-of-sight with nodes in {{ max_distance | round(2) }} km radius
{% for id, data in los_profiles.items()|sort(attribute='1.distance') %}
From 884cb4ffccc5fd9cf83818d6925889fc8366d459 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 13 Apr 2025 04:36:33 +0000 Subject: [PATCH 006/155] Moved new chat to chat.html, completed work on tooltips. --- meshinfo_web.py | 5 +- requirements.txt | 3 +- templates/chat2.html.j2 | 227 +++++++++++++++++++++++++++++++++------- www/css/meshinfo.css | 1 + 4 files changed, 194 insertions(+), 42 deletions(-) diff --git a/meshinfo_web.py b/meshinfo_web.py index c6c98ef9..2fb653e2 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -120,7 +120,7 @@ def allnodes(): ) -@app.route('/chat.html') +@app.route('/chat-classic.html') def chat(): page = request.args.get('page', 1, type=int) per_page = 50 @@ -146,7 +146,7 @@ def chat(): debug=False, ) -@app.route('/chat2.html') +@app.route('/chat.html') def chat2(): page = request.args.get('page', 1, type=int) per_page = 50 @@ -168,6 +168,7 @@ def chat2(): utils=utils, datetime=datetime.datetime, timestamp=datetime.datetime.now(), + meshtastic_support=meshtastic_support, debug=False, ) diff --git a/requirements.txt b/requirements.txt index a03d52b3..2e85f34c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,5 +16,4 @@ geopy boto3 pytz Flask-Caching -pandas -atexit \ No newline at end of file +pandas \ No newline at end of file diff --git a/templates/chat2.html.j2 b/templates/chat2.html.j2 index 31a8d4e7..1f40557d 100644 --- a/templates/chat2.html.j2 +++ b/templates/chat2.html.j2 @@ -82,46 +82,191 @@ window.addEventListener('load', function() { {% for message in chat %}
-
- - {% if message.from in nodes %} - - {{ nodes[message.from].long_name }} ({{ nodes[message.from].short_name }}) - +
{# Make header a flex container #} + {# Group 1: Sender and optional Recipient #} +
{# Container for left-aligned items #} + {# Sender Span (Always Shown) #} + + {% if message.from in nodes %} + + {{ nodes[message.from].long_name }} ({{ nodes[message.from].short_name }}) + + {% else %} + {{ message.from }} + {% endif %} + + + {# Direct Message Indicator and Recipient (Conditional) #} + {% if message.to != 'ffffffff' and message.to in nodes %} + + + + + + {{ nodes[message.to].long_name }} ({{ nodes[message.to].short_name }}) + + + {% endif %} + {# End DM Indicator #} +
+ + {# Group 2: Channel, Timestamp, Map Icon #} +
{# Container for right-aligned items #} + Ch {{ message.channel }} + + {{ time_ago(message.ts_created) }} + + {% if message.from in nodes and nodes[message.from].position and message.receptions %} + + + {% else %} - {{ message.from }} + {% endif %} - - Ch {{ message.channel }} - - {{ time_ago(message.ts_created) }} - - {% if message.from in nodes and nodes[message.from].position and message.receptions %} - - - - {% else %} - - {% endif %} +
{{ message.text }} @@ -200,5 +345,11 @@ window.addEventListener('load', function() {
{% endif %} + +
+ + Classic Chat + +
{% endblock %} \ No newline at end of file diff --git a/www/css/meshinfo.css b/www/css/meshinfo.css index d7cd9064..2ba55500 100644 --- a/www/css/meshinfo.css +++ b/www/css/meshinfo.css @@ -301,6 +301,7 @@ nav { background: rgba(0, 0, 0, 0.05); border-radius: 4px; white-space: nowrap; + margin-right: 0.5em; } .btn-group .btn { From c6a296cc9c62925617d457d12b506f383d8919bf Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 13 Apr 2025 16:26:35 +0000 Subject: [PATCH 007/155] switched caching to disk instead of memory cache --- main.py | 10 ++++++---- meshinfo_web.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/main.py b/main.py index c2955c9b..15bba611 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,3 @@ -import meshinfo_web -import meshinfo_mqtt -from meshdata import MeshData, create_database import threading import logging import colorlog @@ -45,6 +42,11 @@ def setup_logger(): return logger +logger = setup_logger() + +import meshinfo_web +import meshinfo_mqtt +from meshdata import MeshData, create_database def check_pid(pid): """ Check For the existence of a unix pid. """ @@ -85,7 +87,7 @@ def wrapper(): fh.write(str(os.getpid())) fh.close() -logger = setup_logger() + config_file = "config.ini" if not os.path.isfile(config_file): diff --git a/meshinfo_web.py b/meshinfo_web.py index 2fb653e2..1b70b448 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -30,7 +30,36 @@ app = Flask(__name__) -cache = Cache(app, config={'CACHE_TYPE': 'simple'}) +cache_dir = os.path.join(os.path.dirname(__file__), 'runtime_cache') + +# Ensure the cache directory exists +if not os.path.exists(cache_dir): + try: + os.makedirs(cache_dir) + logging.info(f"Created cache directory: {cache_dir}") + except OSError as e: + logging.error(f"Could not create cache directory {cache_dir}: {e}") + cache_dir = None # Indicate failure + +# Configure Flask-Caching +cache_config = { + 'CACHE_TYPE': 'SimpleCache', # Default fallback + 'CACHE_DEFAULT_TIMEOUT': 300 # Default timeout 5 minutes +} +if cache_dir: + cache_config = { + 'CACHE_TYPE': 'FileSystemCache', + 'CACHE_DIR': cache_dir, + 'CACHE_THRESHOLD': 1000, # Max number of items (optional, adjust as needed) + 'CACHE_DEFAULT_TIMEOUT': 60 # Keep your 60 second default timeout + } + logging.info(f"Using FileSystemCache with directory: {cache_dir}") +else: + logging.warning("Falling back to SimpleCache due to directory creation issues.") + + +# Initialize Cache with the chosen config +cache = Cache(app, config=cache_config) # Make globals available to templates app.jinja_env.globals.update(convert_to_local=convert_to_local) From 54598d06b13da347c91e70fa24b82214a095d15c Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 13 Apr 2025 18:44:36 +0000 Subject: [PATCH 008/155] message_map now uses the sender and reciever's nearest position update for mapping --- meshdata.py | 46 ++++++ meshinfo_web.py | 149 ++++++++++++++----- templates/message_map.html.j2 | 260 +++++++++++++++++++++++----------- 3 files changed, 338 insertions(+), 117 deletions(-) diff --git a/meshdata.py b/meshdata.py index 168eea65..903c4723 100644 --- a/meshdata.py +++ b/meshdata.py @@ -160,6 +160,52 @@ def get_position(self, id): cur.close() return position + def get_position_at_time(self, node_id, target_timestamp): + """ + Retrieves the position record from positionlog for a node + that is closest to, but not after, the target timestamp. + """ + position = {} + # Convert Unix timestamp to datetime object for SQL comparison + target_dt = datetime.datetime.fromtimestamp(target_timestamp) + + # Query positionlog for the latest entry at or before the target time + sql = """SELECT latitude_i, longitude_i, ts_created + FROM positionlog + WHERE id = %s + ORDER BY ABS(TIMESTAMPDIFF(SECOND, ts_created, %s)) ASC + LIMIT 1""" + params = (node_id, target_dt) + cur = None + try: + cur = self.db.cursor(dictionary=True) + cur.execute(sql, params) + row = cur.fetchone() + if row: + position = { + "latitude_i": row["latitude_i"], + "longitude_i": row["longitude_i"], + "position_time": row["ts_created"].timestamp() # Still store the actual timestamp of the position found + } + # Add derived lat/lon + if position["latitude_i"]: + position["latitude"] = position["latitude_i"] / 10000000 + else: + position["latitude"] = None + if position["longitude_i"]: + position["longitude"] = position["longitude_i"] / 10000000 + else: + position["longitude"] = None + + except mysql.connector.Error as err: + logging.error(f"Database error fetching nearest position for {node_id}: {err}") + except Exception as e: + logging.error(f"Error fetching nearest position for {node_id}: {e}") + finally: + if cur: + cur.close() + return position + def get_neighbors(self, id): neighbors = [] sql = """SELECT diff --git a/meshinfo_web.py b/meshinfo_web.py index 1b70b448..446a6072 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -6,7 +6,8 @@ make_response, redirect, url_for, - abort + abort, + g ) from flask_caching import Cache from waitress import serve @@ -72,6 +73,32 @@ config.read("config.ini") +# --- Add these MeshData Management functions --- +def get_meshdata(): + """Opens a new MeshData connection if there is none yet for the + current application context. + """ + if 'meshdata' not in g: + try: + g.meshdata = MeshData() + logging.debug("MeshData instance created for request context.") + except Exception as e: + logging.error(f"Failed to create MeshData for request context: {e}") + # Indicate failure + g.meshdata = None + return g.meshdata + +@app.teardown_appcontext +def teardown_meshdata(exception): + """Closes the MeshData connection at the end of the request.""" + md = g.pop('meshdata', None) + if md is not None: + # MeshData.__del__ should handle closing connection if implemented + # Add explicit md.db.close() if __del__ doesn't or isn't reliable + logging.debug("MeshData instance removed from request context.") +# --- End MeshData Management functions --- + + def auth(): jwt = request.cookies.get('jwt') if not jwt: @@ -206,60 +233,110 @@ def message_map(): message_id = request.args.get('id') if not message_id: abort(404) - - md = MeshData() - nodes = md.get_nodes() - - # Get message and reception data + + md = get_meshdata() # Use application context + if not md: abort(503, description="Database connection unavailable") + + nodes = md.get_nodes() # Still get latest node info for names etc. + + # Get message and basic reception data cursor = md.db.cursor(dictionary=True) cursor.execute(""" - SELECT t.*, r.* + SELECT t.*, GROUP_CONCAT(r.received_by_id) as receiver_ids FROM text t LEFT JOIN message_reception r ON t.message_id = r.message_id WHERE t.message_id = %s + GROUP BY t.message_id """, (message_id,)) - - message_data = cursor.fetchall() - if not message_data: + + message_base = cursor.fetchone() + cursor.close() + + if not message_base: abort(404) - - # Process message data + + # Get the precise message time (assuming ts_created is Unix timestamp) + message_time = message_base['ts_created'].timestamp() # Convert from DB datetime to Unix timestamp + + # Fetch historical position for the sender + sender_position = md.get_position_at_time(message_base['from_id'], message_time) + + # Fetch historical positions for receivers + receiver_positions = {} + receiver_details = {} # Store SNR etc. associated with the position + if message_base['receiver_ids']: + receiver_ids_list = [int(r_id) for r_id in message_base['receiver_ids'].split(',')] + + # Fetch full reception details for these receivers for THIS message + # Need a separate query as GROUP_CONCAT loses details + query_placeholders = ', '.join(['%s'] * len(receiver_ids_list)) + sql_reception = f""" + SELECT * FROM message_reception + WHERE message_id = %s AND received_by_id IN ({query_placeholders}) + """ + params_reception = [message_id] + receiver_ids_list + cursor = md.db.cursor(dictionary=True) + cursor.execute(sql_reception, params_reception) + receptions = cursor.fetchall() + cursor.close() + + for reception in receptions: # Iterate through detailed reception info + receiver_id = reception['received_by_id'] + logging.info(f"Attempting to find position for receiver: {receiver_id} at time {message_time}") # Log attempt + pos = md.get_position_at_time(receiver_id, message_time) + logging.info(f"Position found for receiver {receiver_id}: {pos}") # Log result + if pos: # Only store if position was actually found + receiver_positions[receiver_id] = pos + # Store details associated with this reception + receiver_details[receiver_id] = { + 'rx_snr': reception['rx_snr'], + 'rx_rssi': reception['rx_rssi'], + 'hop_start': reception['hop_start'], + 'hop_limit': reception['hop_limit'], + 'rx_time': reception['rx_time'] + } + else: + logging.warning(f"No position found for receiver {receiver_id} near time {message_time}") + + + # Prepare message object for template message = { 'id': message_id, - 'from_id': message_data[0]['from_id'], - 'text': message_data[0]['text'], - 'ts_created': message_data[0]['ts_created'], - 'receptions': [] + 'from_id': message_base['from_id'], + 'text': message_base['text'], + 'ts_created': message_time, # Pass Unix timestamp + # Pass receiver IDs and their corresponding details + 'receiver_ids': receiver_ids_list if message_base['receiver_ids'] else [] } - - # Only process receptions if they exist - for reception in message_data: - if reception['received_by_id'] is not None: # Add this check - message['receptions'].append({ - 'node_id': reception['received_by_id'], - 'rx_snr': reception['rx_snr'], - 'rx_rssi': reception['rx_rssi'], - 'hop_start': reception['hop_start'], - 'hop_limit': reception['hop_limit'], - 'rx_time': reception['rx_time'] - }) - - cursor.close() + # Check if sender has position data before rendering map - from_id = utils.convert_node_id_from_int_to_hex(message['from_id']) - if from_id not in nodes or not nodes[from_id].get('position'): - abort(404, description="Sender position data not available") - + if not sender_position: + # Decide how to handle - show map without sender? Show error? + # For now, let's allow rendering but template must handle missing sender pos + logging.warning(f"Sender position at time {message_time} not found for node {message['from_id']}") + # abort(404, description="Sender position data not available for message time") + + # --- Add this logging --- + logging.debug(f"Data for message {message_id} map:") + logging.debug(f" Sender Position: {sender_position}") + logging.debug(f" Receiver Positions Dict: {receiver_positions}") + logging.debug(f" Receiver Details Dict: {receiver_details}") + logging.debug(f" Receiver IDs List: {message.get('receiver_ids', [])}") + # --- End logging --- + return render_template( "message_map.html.j2", auth=auth(), config=config, - nodes=nodes, + nodes=nodes, # Pass current node info for names message=message, + sender_position=sender_position, # Pass historical sender position + receiver_positions=receiver_positions, # Pass dict of historical receiver positions + receiver_details=receiver_details, # Pass dict of receiver SNR/hop details utils=utils, datetime=datetime.datetime, - timestamp=datetime.datetime.now() + timestamp=datetime.datetime.now() # Page generation time ) @app.route('/traceroute_map.html') diff --git a/templates/message_map.html.j2 b/templates/message_map.html.j2 index aaa92f02..abbaa042 100644 --- a/templates/message_map.html.j2 +++ b/templates/message_map.html.j2 @@ -94,82 +94,101 @@ const features = []; const lines = []; const labels = []; - - // Add sender - {% set from_id = utils.convert_node_id_from_int_to_hex(message.from_id) %} - {% if from_id in nodes and nodes[from_id].position and nodes[from_id].position.longitude_i is not none and nodes[from_id].position.latitude_i is not none %} - var sender = new ol.Feature({ + let senderFeature = null; + + // Add sender using sender_position + {% set from_id_hex = utils.convert_node_id_from_int_to_hex(message.from_id) %} + {% if sender_position and sender_position.longitude_i is not none and sender_position.latitude_i is not none %} + senderFeature = new ol.Feature({ geometry: new ol.geom.Point(ol.proj.fromLonLat([ - {{ nodes[from_id].position.longitude_i / 10000000 }}, - {{ nodes[from_id].position.latitude_i / 10000000 }} + {{ sender_position.longitude_i / 10000000 }}, + {{ sender_position.latitude_i / 10000000 }} ])), node: { - id: '{{ from_id }}', - name: '{{ nodes[from_id].short_name }}', - longName: '{{ nodes[from_id].long_name }}', + id: '{{ from_id_hex }}', + {# Get names from the main nodes dict #} + name: '{{ nodes[from_id_hex].short_name if from_id_hex in nodes else "???" }}', + longName: '{{ nodes[from_id_hex].long_name if from_id_hex in nodes else "Unknown Sender" }}', type: 'sender', - lat: {{ nodes[from_id].position.latitude_i / 10000000 }}, - lon: {{ nodes[from_id].position.longitude_i / 10000000 }} + lat: {{ sender_position.latitude_i / 10000000 }}, + lon: {{ sender_position.longitude_i / 10000000 }}, + positionTime: {{ sender_position.position_time }} } - }); - sender.setStyle(senderStyle); - features.push(sender); + }); + senderFeature.setStyle(senderStyle); + features.push(senderFeature); - // Add receivers and lines - {% for reception in message.receptions %} - {% set rid = utils.convert_node_id_from_int_to_hex(reception.node_id) %} - {% if rid in nodes and nodes[rid].position and nodes[rid].position.longitude_i is not none and nodes[rid].position.latitude_i is not none %} + // Add receivers and lines - only if sender position is known + {% for receiver_id in message.receiver_ids %} + var currentReceiverId = {{ receiver_id }}; // Store JS variable for clarity + console.log("Processing Receiver ID:", currentReceiverId); + + {# Use receiver_positions dict for location #} + {% set receiver_pos = receiver_positions.get(receiver_id) %} + var receiverPosData = {{ receiver_pos | tojson if receiver_pos is defined else 'null' }}; + console.log(" -> Position data from backend:", receiverPosData); + {% if receiver_pos and receiver_pos.longitude_i is not none and receiver_pos.latitude_i is not none %} + console.log(" -> Position check PASSED for", currentReceiverId); + try { + var lon = {{ receiver_pos.longitude_i / 10000000 }}; + var lat = {{ receiver_pos.latitude_i / 10000000 }}; + console.log(` Coords: lon=${lon}, lat=${lat}`); + + var pointGeom = new ol.geom.Point(ol.proj.fromLonLat([lon, lat])); + console.log(" Created ol.geom.Point"); + + {% set rid_hex = utils.convert_node_id_from_int_to_hex(receiver_id) %} + {% set details = receiver_details.get(receiver_id) %} + var nodeData = { + id: '{{ rid_hex }}', + name: '{{ nodes[rid_hex].short_name if rid_hex in nodes else "???" }}', + longName: '{{ nodes[rid_hex].long_name if rid_hex in nodes else "Unknown Receiver" }}', + snr: {{ details.rx_snr if details else 'null' }}, + hops: {% if details and details.hop_start is not none and details.hop_limit is not none %}{{ details.hop_start - details.hop_limit }}{% else %}0{% endif %}, + type: 'receiver', + lat: lat, // Use JS var + lon: lon, // Use JS var + positionTime: {{ receiver_pos.position_time }} + }; + console.log(" Created nodeData object:", nodeData); + var receiver = new ol.Feature({ - geometry: new ol.geom.Point(ol.proj.fromLonLat([ - {{ nodes[rid].position.longitude_i / 10000000 }}, - {{ nodes[rid].position.latitude_i / 10000000 }} - ])), - node: { - id: '{{ rid }}', - name: '{{ nodes[rid].short_name }}', - longName: '{{ nodes[rid].long_name }}', - snr: {{ reception.rx_snr }}, - hops: {% if reception.hop_start is not none %}{{ reception.hop_start - reception.hop_limit }}{% else %}0{% endif %}, - type: 'receiver', - lat: {{ nodes[rid].position.latitude_i / 10000000 }}, - lon: {{ nodes[rid].position.longitude_i / 10000000 }} - } - }); + geometry: pointGeom, + node: nodeData + }); + console.log(" Created ol.Feature"); + receiver.setStyle(receiverStyle); + console.log(" Set receiver style"); + features.push(receiver); + console.log(" Pushed receiver feature to array. Current features count:", features.length); + + // Line and Label creation (can add logs here too if needed) + var senderCoords = senderFeature.getGeometry().getCoordinates(); // Get projected coords + // Transform sender's projected coords back to LonLat for fromCoord + var fromCoordLonLat = ol.proj.transform(senderCoords, 'EPSG:3857', 'EPSG:4326'); + var fromCoord = fromCoordLonLat; // Use LonLat for distance calculation - // Calculate coordinates and distance - var fromCoord = [ - {{ nodes[from_id].position.longitude_i / 10000000 }}, - {{ nodes[from_id].position.latitude_i / 10000000 }} - ]; - var toCoord = [ - {{ nodes[rid].position.longitude_i / 10000000 }}, - {{ nodes[rid].position.latitude_i / 10000000 }} - ]; - + var toCoord = [lon, lat]; var distance = calculateDistance(fromCoord[1], fromCoord[0], toCoord[1], toCoord[0]); - var hops = {% if reception.hop_start is not none %}{{ reception.hop_start - reception.hop_limit }}{% else %}0{% endif %}; - - // Create line + var hops = nodeData.hops; // Use hops from nodeData + var points = [fromCoord, toCoord]; - for (var i = 0; i < points.length; i++) { - points[i] = ol.proj.transform(points[i], 'EPSG:4326', 'EPSG:3857'); - } - - var line = new ol.Feature({ - geometry: new ol.geom.LineString(points) - }); - + for (var i = 0; i < points.length; i++) { points[i] = ol.proj.transform(points[i], 'EPSG:4326', 'EPSG:3857'); } + var line = new ol.Feature({ geometry: new ol.geom.LineString(points) }); + var lineStyle = new ol.style.Style({ stroke: new ol.style.Stroke({ - color: {% if reception.hop_start is not none and reception.hop_start - reception.hop_limit > 0 %}'#6666FF'{% else %}'#FF6666'{% endif %}, + color: {% if details and details.hop_start is not none and details.hop_limit is not none and (details.hop_start - details.hop_limit) > 0 %}'#6666FF'{% else %}'#FF6666'{% endif %}, // <-- Added comma here width: 2 }) }); - + line.setStyle(lineStyle); lines.push(line); + console.log(" Pushed line feature to array."); + // Add label at the middle of the line var midPoint = [ @@ -209,40 +228,119 @@ })); labels.push(label); + console.log(" Pushed label feature to array."); + + } catch (e) { + console.error(" ERROR processing receiver", currentReceiverId, ":", e); + } + {% else %} + // This block is entered - position is missing or invalid + console.warn(" -> Position check FAILED for", currentReceiverId); + if (!receiverPosData) { + console.warn(" Reason: No position data found in receiver_positions dict."); + } else if (receiverPosData.longitude_i === null || receiverPosData.longitude_i === undefined ) { + console.warn(" Reason: longitude_i is null or undefined."); + } else if (receiverPosData.latitude_i === null || receiverPosData.latitude_i === undefined) { + console.warn(" Reason: latitude_i is null or undefined."); + } else { + console.warn(" Reason: Unknown (check receiverPosData object)."); + } {% endif %} {% endfor %} {% endif %} - // Add features to map + console.log("Finished processing features. Total points:", features.length, "Total lines:", lines.length, "Total labels:", labels.length); + + // Combine features + const allFeatures = features.concat(lines).concat(labels); + console.log("Combined all features. Total count:", allFeatures.length); + + // Create empty source first + const vectorSource = new ol.source.Vector(); + console.log("Created empty vector source."); + + // Add features explicitly AFTER loops finish + if (allFeatures.length > 0) { + vectorSource.addFeatures(allFeatures); + console.log(`Added ${allFeatures.length} features to vector source. Source now has ${vectorSource.getFeatures().length} features.`); + } else { + console.warn("No features were created to add to the source."); + } + + // Create layer with the source const vectorLayer = new ol.layer.Vector({ - source: new ol.source.Vector({ - features: features.concat(lines).concat(labels) - }) + source: vectorSource, + // Declutter labels slightly + declutter: true }); + console.log("Created vector layer."); + map.addLayer(vectorLayer); - - // Fit map to features with minimum zoom - if (features.length > 0) { - var extent = vectorLayer.getSource().getExtent(); - - // Get the current view - var view = map.getView(); - - // Fit to extent with padding - view.fit(extent, { - padding: [100, 100, 100, 100], // Increased padding - minResolution: 0.1, // Ensures we don't zoom in too far - maxZoom: 19, // Allow higher zoom for close nodes - duration: 500 // Smooth animation - }); + console.log("Added vector layer to map."); + + // --- Debugging Map View --- + var view = map.getView(); + console.log("Initial/Current Map View State:", JSON.stringify(view.getProperties())); + + if (features.length > 0) { + console.log("Attempting to fit map view based on point features..."); + const pointSourceForExtent = new ol.source.Vector({ features: features }); + var extent = pointSourceForExtent.getExtent(); + console.log("Calculated extent from points:", extent); + + if (!ol.extent.isEmpty(extent) && isFinite(extent[0]) && isFinite(extent[1]) && isFinite(extent[2]) && isFinite(extent[3]) && extent[0] <= extent[2] && extent[1] <= extent[3]) + { + console.log("Extent is valid. Fitting view..."); + console.log("View state BEFORE fit:", JSON.stringify(view.getProperties())); + view.fit(extent, { + padding: [100, 100, 100, 100], + minResolution: view.getResolutionForZoom(19), // Use maxZoom resolution + maxZoom: 19, + duration: 500 + }); + console.log("Fit view called."); + // Log state immediately after fit call (may not reflect final state yet) + console.log("View state immediately AFTER fit call:", JSON.stringify(view.getProperties())); - // If nodes are very close, ensure minimum zoom level - var resolution = view.getResolution(); - if (resolution < 0.1) { // If zoomed in too far - view.setResolution(0.1); // Set minimum resolution - view.setCenter(ol.extent.getCenter(extent)); // Center on the features + // Log state after fit animation duration + setTimeout(() => { + console.log("View state AFTER fit timeout:", JSON.stringify(view.getProperties())); + }, 600); // Wait slightly longer than duration + } else { + console.warn("Calculated extent is invalid or empty. Cannot fit view. Extent:", extent); + // ... (Fallback logic remains the same) ... } + } else { + console.log("No point features found to fit view."); + } + + // --- Add Render Complete Listener --- + map.once('rendercomplete', function() { + console.log("--- Map Render Complete ---"); + try { + const mapLayers = map.getLayers().getArray(); + console.log(`Map has ${mapLayers.length} layers.`); + if (mapLayers.length > 1) { + const potentialVectorLayer = mapLayers[1]; + if (potentialVectorLayer instanceof ol.layer.Vector) { + const source = potentialVectorLayer.getSource(); + const layerFeatures = source.getFeatures(); + console.log(`Vector layer found. Contains ${layerFeatures.length} features.`); + if (layerFeatures.length > 0) { + console.log("First feature geom in layer:", layerFeatures[0].getGeometry().getCoordinates()); + } + } else { + console.warn("Second layer is not a Vector layer:", potentialVectorLayer); + } + } else { + console.warn("Vector layer not found or only base layer exists."); + } + console.log("Final Map View State:", JSON.stringify(map.getView().getProperties())); + } catch(e) { + console.error("Error during rendercomplete check:", e); } + console.log("--- End Render Complete ---"); + }); // Add popup for node info var container = document.createElement('div'); From 64f884450c26825642b8d255d4af6d3e365e9bac Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 14 Apr 2025 03:48:52 +0000 Subject: [PATCH 009/155] added a robots.txt file --- templates/message_map.html.j2 | 3 +-- www/robots.txt | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 www/robots.txt diff --git a/templates/message_map.html.j2 b/templates/message_map.html.j2 index abbaa042..3db54491 100644 --- a/templates/message_map.html.j2 +++ b/templates/message_map.html.j2 @@ -169,7 +169,7 @@ // Transform sender's projected coords back to LonLat for fromCoord var fromCoordLonLat = ol.proj.transform(senderCoords, 'EPSG:3857', 'EPSG:4326'); var fromCoord = fromCoordLonLat; // Use LonLat for distance calculation - + var toCoord = [lon, lat]; var distance = calculateDistance(fromCoord[1], fromCoord[0], toCoord[1], toCoord[0]); var hops = nodeData.hops; // Use hops from nodeData @@ -308,7 +308,6 @@ }, 600); // Wait slightly longer than duration } else { console.warn("Calculated extent is invalid or empty. Cannot fit view. Extent:", extent); - // ... (Fallback logic remains the same) ... } } else { console.log("No point features found to fit view."); diff --git a/www/robots.txt b/www/robots.txt new file mode 100644 index 00000000..77470cb3 --- /dev/null +++ b/www/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / \ No newline at end of file From 08d6d8b5a2acf4aafd9ad3617f0d4bf57e59ee13 Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 14 Apr 2025 18:52:44 +0000 Subject: [PATCH 010/155] custom config for mariadb; routes and mqtt processing use one instance of MeshData instead of recreating repeatedly; more efficient neighbors.html, more efficient get_nodes(). --- custom.cnf | 6 + docker-compose.yml | 1 + meshdata.py | 71 +++++++- meshinfo_mqtt.py | 98 ++++++++--- meshinfo_web.py | 402 +++++++++++++++++++++++++-------------------- process_payload.py | 19 ++- 6 files changed, 392 insertions(+), 205 deletions(-) create mode 100644 custom.cnf diff --git a/custom.cnf b/custom.cnf new file mode 100644 index 00000000..07c8e618 --- /dev/null +++ b/custom.cnf @@ -0,0 +1,6 @@ +[mysqld] +aria_pagecache_buffer_size = 16M +aria_sort_buffer_size = 32M +innodb_buffer_pool_size = 32M +key_buffer_size = 16M +myisam_sort_buffer_size = 16M \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 83d71ce5..0eab2cca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ services: MYSQL_PASSWORD: passw0rd volumes: - ./mysql_data:/var/lib/mysql + - ./custom.cnf:/etc/mysql/conf.d/custom.cnf networks: - backend meshinfo: diff --git a/meshdata.py b/meshdata.py index 903c4723..78e1ff1d 100644 --- a/meshdata.py +++ b/meshdata.py @@ -65,23 +65,77 @@ def connect_db(self): for attempt in range(max_retries): try: + # Ensure any existing connection is closed before creating a new one + if self.db and self.db.is_connected(): + try: + self.db.close() + logging.debug("Closed existing DB connection before reconnecting.") + except mysql.connector.Error as close_err: + logging.warning(f"Error closing existing DB connection: {close_err}") + self.db = mysql.connector.connect( host=self.config["database"]["host"], user=self.config["database"]["username"], password=self.config["database"]["password"], database=self.config["database"]["database"], - charset="utf8mb4" + charset="utf8mb4", + # Add connection timeout (e.g., 10 seconds) + connection_timeout=10 ) - cur = self.db.cursor() - cur.execute("SET NAMES utf8mb4;") - cur.close() - return + if self.db.is_connected(): + cur = self.db.cursor() + cur.execute("SET NAMES utf8mb4;") + cur.close() + logging.info(f"Database connection successful (Attempt {attempt + 1}).") + return + else: + # This case might not be reached if connect throws error, but good practice + raise mysql.connector.Error("Connection attempt returned but not connected.") + except mysql.connector.Error as err: + logging.warning(f"Database connection attempt {attempt + 1}/{max_retries} failed: {err}") if attempt < max_retries - 1: - logging.warning(f"Waiting for database to become ready. Attempt {attempt + 1}/{max_retries}") + logging.info(f"Retrying connection in {retry_delay} seconds...") time.sleep(retry_delay) else: - raise + logging.error("Maximum database connection retries reached. Raising error.") + raise # Re-raise the last error after all retries fail + + def ping_db(self): + """Checks connection and attempts reconnect if needed.""" + if self.db is None: + logging.warning("Database object is None. Attempting reconnect.") + try: + self.connect_db() + return self.db is not None and self.db.is_connected() + except Exception as e: + logging.error(f"Reconnect failed during ping: {e}") + return False + + try: + # Check if connected first, then try ping with reconnect=True + if not self.db.is_connected(): + logging.warning("DB connection reported as not connected. Attempting ping/reconnect.") + # The ping=True argument attempts to reconnect if connection is lost. + self.db.ping(reconnect=True, attempts=3, delay=2) + logging.debug("Database connection verified via ping.") + return True + except mysql.connector.Error as err: + logging.error(f"Database ping/reconnect failed: {err}") + # Attempt a full reconnect as a final measure if ping fails + try: + logging.warning("Ping failed. Attempting full database reconnect...") + self.connect_db() # Use the existing connect method + # Check connection status again after attempting connect_db + if self.db and self.db.is_connected(): + logging.info("Full database reconnect successful.") + return True + else: + logging.error("Full database reconnect attempt failed to establish connection.") + return False + except Exception as e: + logging.error(f"Full database reconnect attempt raised an exception: {e}") + return False def get_telemetry(self, id): telemetry = {} @@ -1061,6 +1115,9 @@ def log_position(self, id, lat, lon, source): logging.info(f"Position updated for {id}") def store(self, data, topic): + if not self.ping_db(): + logging.error("Database connection is not active. Skipping store operation.") + return # Stop processing if DB is unavailable if not data: return self.log_data(topic, data) diff --git a/meshinfo_mqtt.py b/meshinfo_mqtt.py index 151d2b53..c8b7a56d 100644 --- a/meshinfo_mqtt.py +++ b/meshinfo_mqtt.py @@ -1,6 +1,7 @@ import logging from paho.mqtt import client as mqtt_client from process_payload import process_payload +from meshdata import MeshData # Import MeshData import configparser import time @@ -9,53 +10,110 @@ config.read("config.ini") logger = logging.getLogger(__name__) +try: + mesh_data_instance = MeshData() + logger.info("MeshData instance created successfully.") +except Exception as e: + logger.error(f"Fatal error: Could not initialize MeshData. Exiting. Error: {e}") + exit(1) # Exit if we can't connect to the DB def connect_mqtt() -> mqtt_client: def on_connect(client, userdata, flags, rc): if rc == 0: - logger.info("Connected to MQTT Broker!") + logger.info("Connected to MQTT Broker! Return Code: %d", rc) + # --- Add log before calling subscribe --- + logger.info("on_connect: Attempting to subscribe...") + try: + subscribe(client, mesh_data_instance) + logger.info("on_connect: subscribe() call completed.") + except Exception as e: + logger.exception("on_connect: Error calling subscribe()") # Log exception if subscribe fails + # --- End log --- else: - logger.error("Failed to connect, return code %d\n", rc) + logger.error("Failed to connect, return code %d", rc) def on_disconnect(client, userdata, rc): - logger.warning(f"Disconnected with result code {rc}. Reconnecting...") - while True: - try: - client.reconnect() - logger.info("Reconnected successfully!") - break - except Exception as e: - logger.error(f"Reconnection failed: {e}") - time.sleep(5) # Wait before retrying - client = mqtt_client.Client() + logger.warning(f"Disconnected with result code {rc}. Will attempt reconnect.") + # No need for manual reconnect loop, paho handles it with reconnect_delay_set + + client = mqtt_client.Client(client_id="", clean_session=True, userdata=mesh_data_instance) # Ensure clean session if needed + client.user_data_set(mesh_data_instance) # Redundant if passed in constructor, but safe if "username" in config["mqtt"] \ and config["mqtt"]["username"] \ and "password" in config["mqtt"] \ and config["mqtt"]["password"]: + logger.info("Setting MQTT username and password.") client.username_pw_set( config["mqtt"]["username"], config["mqtt"]["password"] ) client.on_connect = on_connect client.on_disconnect = on_disconnect - client.connect(config["mqtt"]["broker"], int(config["mqtt"]["port"])) + client.reconnect_delay_set(min_delay=5, max_delay=120) # Increased min delay slightly + + broker_address = config["mqtt"]["broker"] + broker_port = int(config["mqtt"]["port"]) + logger.info(f"Connecting to MQTT broker at {broker_address}:{broker_port}...") + try: + client.connect(broker_address, broker_port, 60) # Keepalive 60 seconds + except Exception as e: + logger.exception(f"Failed to connect to MQTT broker: {e}") + raise # Reraise the exception to prevent starting loop_forever on failed connect return client -def subscribe(client: mqtt_client): +def subscribe(client: mqtt_client, md_instance: MeshData): + # --- Add log at the start --- + logger.info("subscribe: Entered function.") + # --- End log --- def on_message(client, userdata, msg): + # --- Add log for every message received --- + logger.debug(f"on_message: Received message on topic: {msg.topic}") + # --- End log --- + + # Filter for relevant topics if "/2/e/" in msg.topic or "/2/map/" in msg.topic: - process_payload(msg.payload, msg.topic) + logger.debug(f"on_message: Processing message from relevant topic: {msg.topic}") + try: + # Pass the existing MeshData instance + process_payload(msg.payload, msg.topic, md_instance) + except Exception as e: + logger.exception(f"on_message: Error calling process_payload for topic {msg.topic}") + else: + logger.debug(f"on_message: Skipping message from topic: {msg.topic}") + + + topic_to_subscribe = config["mqtt"]["topic"] + logger.info(f"subscribe: Subscribing to topic: {topic_to_subscribe}") + try: + result, mid = client.subscribe(topic_to_subscribe) + if result == mqtt_client.MQTT_ERR_SUCCESS: + logger.info(f"subscribe: Successfully initiated subscription to {topic_to_subscribe} (MID: {mid})") + else: + logger.error(f"subscribe: Failed to initiate subscription to {topic_to_subscribe}, Error code: {result}") + return # Don't set on_message if subscribe failed + + logger.info("subscribe: Setting on_message callback.") + client.on_message = on_message + logger.info("subscribe: on_message callback set.") + except Exception as e: + logger.exception(f"subscribe: Error during subscribe call or setting on_message for topic {topic_to_subscribe}") - client.subscribe(config["mqtt"]["topic"]) - client.on_message = on_message def run(): - client = connect_mqtt() - subscribe(client) - client.loop_forever() + logger.info("Starting MQTT client run sequence...") + try: + client = connect_mqtt() + logger.info("Entering MQTT client loop (loop_forever)...") + client.loop_forever() + except Exception as e: + logger.exception("An error occurred during MQTT client execution.") + finally: + logger.info("Exited MQTT client loop.") if __name__ == '__main__': + # Configure logging (ensure level allows DEBUG if you want to see the on_message logs) + logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - [%(name)s] - %(message)s') run() diff --git a/meshinfo_web.py b/meshinfo_web.py index 446a6072..58ecdc88 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -121,7 +121,9 @@ def not_found(e): @app.route('/') @cache.cached(timeout=60) # Cache for 60 seconds def serve_index(success_message=None, error_message=None): - md = MeshData() + md = get_meshdata() + if not md: # Check if MeshData failed to initialize + abort(503, description="Database connection unavailable") nodes = md.get_nodes() return render_template( "index.html.j2", @@ -138,8 +140,11 @@ def serve_index(success_message=None, error_message=None): @app.route('/nodes.html') @cache.cached(timeout=60) # Cache for 60 seconds def nodes(): - md = MeshData() + md = get_meshdata() + if not md: # Check if MeshData failed to initialize + abort(503, description="Database connection unavailable") nodes = md.get_nodes() + logging.info(f"/nodes.html: Loaded {len(nodes)} nodes.") # Add this latest = md.get_latest_node() return render_template( "nodes.html.j2", @@ -158,7 +163,9 @@ def nodes(): @app.route('/allnodes.html') @cache.cached(timeout=60) # Cache for 60 seconds def allnodes(): - md = MeshData() + md = get_meshdata() + if not md: # Check if MeshData failed to initialize + abort(503, description="Database connection unavailable") nodes = md.get_nodes() latest = md.get_latest_node() return render_template( @@ -181,7 +188,9 @@ def chat(): page = request.args.get('page', 1, type=int) per_page = 50 - md = MeshData() + md = get_meshdata() + if not md: # Check if MeshData failed to initialize + abort(503, description="Database connection unavailable") nodes = md.get_nodes() chat_data = md.get_chat(page=page, per_page=per_page) @@ -207,7 +216,9 @@ def chat2(): page = request.args.get('page', 1, type=int) per_page = 50 - md = MeshData() + md = get_meshdata() + if not md: # Check if MeshData failed to initialize + abort(503, description="Database connection unavailable") nodes = md.get_nodes() chat_data = md.get_chat(page=page, per_page=per_page) @@ -345,7 +356,9 @@ def traceroute_map(): if not traceroute_id: abort(404) - md = MeshData() + md = get_meshdata() + if not md: # Check if MeshData failed to initialize + abort(503, description="Database connection unavailable") nodes = md.get_nodes() # Get traceroute data @@ -409,7 +422,9 @@ def traceroute_map(): @app.route('/graph.html') @cache.cached(timeout=60) # Cache for 60 seconds def graph(): - md = MeshData() + md = get_meshdata() + if not md: # Check if MeshData failed to initialize + abort(503, description="Database connection unavailable") nodes = md.get_nodes() graph = md.graph_nodes() return render_template( @@ -426,7 +441,9 @@ def graph(): @app.route('/map.html') def map(): - md = MeshData() + md = get_meshdata() + if not md: # Check if MeshData failed to initialize + abort(503, description="Database connection unavailable") nodes = md.get_nodes() # Get timeout from config @@ -505,182 +522,205 @@ def map(): @app.route('/neighbors.html') def neighbors(): - view_type = request.args.get('view', 'neighbor_info') - md = MeshData() + view_type = request.args.get('view', 'neighbor_info') # Default to neighbor_info + md = get_meshdata() + if not md: + abort(503, description="Database connection unavailable") + + # Get base node data (already optimized) nodes = md.get_nodes() - + if not nodes: + # Handle case with no nodes gracefully + return render_template( + "neighbors.html.j2", + auth=auth(), config=config, nodes={}, + active_nodes_with_connections={}, view_type=view_type, + utils=utils, datetime=datetime.datetime, timestamp=datetime.datetime.now() + ) + + zero_hop_timeout = int(config.get("server", "zero_hop_timeout", fallback=43200)) cutoff_time = int(time.time()) - zero_hop_timeout - - active_nodes_with_connections = {} - - if view_type in ['neighbor_info', 'merged']: - neighbor_info_nodes = {} - for node_id, node in nodes.items(): - if node.get("active") and node.get("neighbors"): - node_data = dict(node) - if 'last_heard' not in node_data: - node_data['last_heard'] = datetime.datetime.now() - elif isinstance(node_data['last_heard'], (int, float)): - node_data['last_heard'] = datetime.datetime.fromtimestamp(node_data['last_heard']) - - # Add heard-by data from neighbor info - node_data['heard_by_neighbors'] = [] - for other_id, other_node in nodes.items(): - if other_node.get("neighbors"): - for neighbor in other_node["neighbors"]: - if utils.convert_node_id_from_int_to_hex(neighbor["neighbor_id"]) == node_id: - node_data['heard_by_neighbors'].append({ - 'neighbor_id': utils.convert_node_id_from_hex_to_int(other_id), - 'snr': neighbor["snr"], - 'distance': neighbor.get("distance") - }) - - neighbor_info_nodes[node_id] = node_data - - if view_type == 'neighbor_info': - active_nodes_with_connections = neighbor_info_nodes - + + # Dictionary to hold the final data for active nodes + active_nodes_data = {} + + # --- Pre-fetch and process Neighbor Info --- + neighbor_info_links = {} # {node_id: {'heard': {neighbor_id: {data}}, 'heard_by': {neighbor_id: {data}}} } + + # Fetch all recent neighbor info links once + cursor = md.db.cursor(dictionary=True) + cursor.execute(""" + SELECT + ni.id, ni.neighbor_id, ni.snr, + p1.latitude_i as lat1_i, p1.longitude_i as lon1_i, + p2.latitude_i as lat2_i, p2.longitude_i as lon2_i + FROM neighborinfo ni + LEFT OUTER JOIN position p1 ON p1.id = ni.id + LEFT OUTER JOIN position p2 ON p2.id = ni.neighbor_id + WHERE ni.ts_created >= NOW() - INTERVAL 1 DAY + """) + + for row in cursor.fetchall(): + node_id = row['id'] + neighbor_id = row['neighbor_id'] + distance = None + if (row['lat1_i'] and row['lon1_i'] and row['lat2_i'] and row['lon2_i']): + distance = round(utils.distance_between_two_points( + row['lat1_i'] / 10000000, row['lon1_i'] / 10000000, + row['lat2_i'] / 10000000, row['lon2_i'] / 10000000 + ), 2) + + link_data = {'snr': row['snr'], 'distance': distance, 'neighbor_id': neighbor_id} + heard_by_data = {'snr': row['snr'], 'distance': distance, 'neighbor_id': node_id} + + # Add to node_id's 'heard' list + if node_id not in neighbor_info_links: neighbor_info_links[node_id] = {'heard': {}, 'heard_by': {}} + neighbor_info_links[node_id]['heard'][neighbor_id] = link_data + + # Add to neighbor_id's 'heard_by' list + if neighbor_id not in neighbor_info_links: neighbor_info_links[neighbor_id] = {'heard': {}, 'heard_by': {}} + neighbor_info_links[neighbor_id]['heard_by'][node_id] = heard_by_data + cursor.close() + + + # --- Pre-fetch and process Zero Hop Info --- + zero_hop_links = {} # {node_id: {'heard': {neighbor_id: {data}}, 'heard_by': {neighbor_id: {data}}} } + zero_hop_last_heard = {} # Keep track of last heard time for sorting + if view_type in ['zero_hop', 'merged']: cursor = md.db.cursor(dictionary=True) - - # First get nodes that have heard messages cursor.execute(""" - SELECT DISTINCT - received_by_id as node_id, - MAX(rx_time) as last_heard - FROM message_reception - WHERE rx_time > %s + SELECT + m.from_id, + m.received_by_id, + MAX(m.rx_snr) as snr, + COUNT(*) as message_count, + MAX(m.rx_time) as last_heard_time, + p_sender.latitude_i as lat_sender_i, + p_sender.longitude_i as lon_sender_i, + p_receiver.latitude_i as lat_receiver_i, + p_receiver.longitude_i as lon_receiver_i + FROM message_reception m + LEFT OUTER JOIN position p_sender ON p_sender.id = m.from_id + LEFT OUTER JOIN position p_receiver ON p_receiver.id = m.received_by_id + WHERE m.rx_time > %s AND ( - (hop_limit IS NULL AND hop_start IS NULL) + (m.hop_limit IS NULL AND m.hop_start IS NULL) OR - (hop_start - hop_limit = 0) + (m.hop_start - m.hop_limit = 0) ) - GROUP BY received_by_id + GROUP BY m.from_id, m.received_by_id, + p_sender.latitude_i, p_sender.longitude_i, + p_receiver.latitude_i, p_receiver.longitude_i """, (cutoff_time,)) - - zero_hop_nodes = {} + for row in cursor.fetchall(): - node_id = utils.convert_node_id_from_int_to_hex(row['node_id']) - if node_id in nodes and nodes[node_id].get("active"): - node_data = dict(nodes[node_id]) - node_data['zero_hop_neighbors'] = [] - node_data['heard_by_zero_hop'] = [] - node_data['last_heard'] = datetime.datetime.fromtimestamp(row['last_heard']) - - # Get nodes this node heard - cursor.execute(""" - SELECT - from_id as neighbor_id, - MAX(rx_snr) as snr, - COUNT(*) as message_count, - MAX(rx_time) as last_heard, - p1.latitude_i as lat1_i, - p1.longitude_i as lon1_i, - p2.latitude_i as lat2_i, - p2.longitude_i as lon2_i - FROM message_reception m - LEFT OUTER JOIN position p1 ON p1.id = m.received_by_id - LEFT OUTER JOIN position p2 ON p2.id = m.from_id - WHERE m.received_by_id = %s - AND m.rx_time > %s - AND ( - (m.hop_limit IS NULL AND m.hop_start IS NULL) - OR - (m.hop_start - m.hop_limit = 0) - ) - GROUP BY from_id, p1.latitude_i, p1.longitude_i, p2.latitude_i, p2.longitude_i - """, (row['node_id'], cutoff_time)) - - # Process nodes this node heard - for neighbor in cursor.fetchall(): - distance = None - if (neighbor['lat1_i'] and neighbor['lon1_i'] and - neighbor['lat2_i'] and neighbor['lon2_i']): - distance = round(utils.distance_between_two_points( - neighbor['lat1_i'] / 10000000, - neighbor['lon1_i'] / 10000000, - neighbor['lat2_i'] / 10000000, - neighbor['lon2_i'] / 10000000 - ), 2) - - node_data['zero_hop_neighbors'].append({ - 'neighbor_id': neighbor['neighbor_id'], - 'snr': neighbor['snr'], - 'message_count': neighbor['message_count'], - 'distance': distance, - 'last_heard': datetime.datetime.fromtimestamp(neighbor['last_heard']) - }) - - # Get nodes that heard this node - cursor.execute(""" - SELECT - received_by_id as neighbor_id, - MAX(rx_snr) as snr, - COUNT(*) as message_count, - MAX(rx_time) as last_heard, - p1.latitude_i as lat1_i, - p1.longitude_i as lon1_i, - p2.latitude_i as lat2_i, - p2.longitude_i as lon2_i - FROM message_reception m - LEFT OUTER JOIN position p1 ON p1.id = m.received_by_id - LEFT OUTER JOIN position p2 ON p2.id = m.from_id - WHERE m.from_id = %s - AND m.rx_time > %s - AND ( - (m.hop_limit IS NULL AND m.hop_start IS NULL) - OR - (m.hop_start - m.hop_limit = 0) - ) - GROUP BY received_by_id, p1.latitude_i, p1.longitude_i, p2.latitude_i, p2.longitude_i - """, (row['node_id'], cutoff_time)) - - # Process nodes that heard this node - for neighbor in cursor.fetchall(): - distance = None - if (neighbor['lat1_i'] and neighbor['lon1_i'] and - neighbor['lat2_i'] and neighbor['lon2_i']): - distance = round(utils.distance_between_two_points( - neighbor['lat1_i'] / 10000000, - neighbor['lon1_i'] / 10000000, - neighbor['lat2_i'] / 10000000, - neighbor['lon2_i'] / 10000000 - ), 2) - - node_data['heard_by_zero_hop'].append({ - 'neighbor_id': neighbor['neighbor_id'], - 'snr': neighbor['snr'], - 'message_count': neighbor['message_count'], - 'distance': distance, - 'last_heard': datetime.datetime.fromtimestamp(neighbor['last_heard']) - }) - - if node_data['zero_hop_neighbors'] or node_data['heard_by_zero_hop']: - zero_hop_nodes[node_id] = node_data - - if view_type == 'zero_hop': - active_nodes_with_connections = zero_hop_nodes - else: # merged view - active_nodes_with_connections = neighbor_info_nodes.copy() - for node_id, node_data in zero_hop_nodes.items(): - if node_id in active_nodes_with_connections: - active_nodes_with_connections[node_id]['zero_hop_neighbors'] = node_data['zero_hop_neighbors'] - active_nodes_with_connections[node_id]['heard_by_zero_hop'] = node_data['heard_by_zero_hop'] - if (node_data['last_heard'] > - active_nodes_with_connections[node_id]['last_heard']): - active_nodes_with_connections[node_id]['last_heard'] = node_data['last_heard'] - else: - active_nodes_with_connections[node_id] = node_data - + sender_id = row['from_id'] + receiver_id = row['received_by_id'] + last_heard_dt = datetime.datetime.fromtimestamp(row['last_heard_time']) + + # Update last heard time for involved nodes + zero_hop_last_heard[sender_id] = max(zero_hop_last_heard.get(sender_id, datetime.datetime.min), last_heard_dt) + zero_hop_last_heard[receiver_id] = max(zero_hop_last_heard.get(receiver_id, datetime.datetime.min), last_heard_dt) + + distance = None + if (row['lat_sender_i'] and row['lon_sender_i'] and + row['lat_receiver_i'] and row['lon_receiver_i']): + distance = round(utils.distance_between_two_points( + row['lat_sender_i'] / 10000000, row['lon_sender_i'] / 10000000, + row['lat_receiver_i'] / 10000000, row['lon_receiver_i'] / 10000000 + ), 2) + + link_data = { + 'snr': row['snr'], + 'message_count': row['message_count'], + 'distance': distance, + 'last_heard': last_heard_dt, + 'neighbor_id': sender_id # For receiver, neighbor is sender + } + heard_by_data = { + 'snr': row['snr'], + 'message_count': row['message_count'], + 'distance': distance, + 'last_heard': last_heard_dt, + 'neighbor_id': receiver_id # For sender, neighbor is receiver + } + + # Add to receiver's 'heard' list + if receiver_id not in zero_hop_links: zero_hop_links[receiver_id] = {'heard': {}, 'heard_by': {}} + zero_hop_links[receiver_id]['heard'][sender_id] = link_data + + # Add to sender's 'heard_by' list + if sender_id not in zero_hop_links: zero_hop_links[sender_id] = {'heard': {}, 'heard_by': {}} + zero_hop_links[sender_id]['heard_by'][receiver_id] = heard_by_data + cursor.close() + + # --- Combine data for active nodes based on view type --- + for node_id_hex, node_base_data in nodes.items(): + if not node_base_data.get("active"): + continue # Skip inactive nodes + + node_id_int = utils.convert_node_id_from_hex_to_int(node_id_hex) + final_node_data = dict(node_base_data) # Start with base data + + # Initialize lists + final_node_data['neighbors'] = [] + final_node_data['heard_by_neighbors'] = [] + final_node_data['zero_hop_neighbors'] = [] + final_node_data['heard_by_zero_hop'] = [] + + has_neighbor_info = node_id_int in neighbor_info_links + has_zero_hop_info = node_id_int in zero_hop_links + + # Determine overall last heard time for sorting + last_heard_zero_hop = max([d['last_heard'] for d in zero_hop_links[node_id_int]['heard'].values()], default=datetime.datetime.min) if has_zero_hop_info else datetime.datetime.min + last_heard_by_zero_hop = max([d['last_heard'] for d in zero_hop_links[node_id_int]['heard_by'].values()], default=datetime.datetime.min) if has_zero_hop_info else datetime.datetime.min + last_heard_zero_hop = max([d['last_heard'] for d in zero_hop_links[node_id_int]['heard'].values()], default=datetime.datetime.min) if has_zero_hop_info else datetime.datetime.min + last_heard_by_zero_hop = max([d['last_heard'] for d in zero_hop_links[node_id_int]['heard_by'].values()], default=datetime.datetime.min) if has_zero_hop_info else datetime.datetime.min + + node_ts_seen = datetime.datetime.fromtimestamp(node_base_data['ts_seen']) if node_base_data.get('ts_seen') else datetime.datetime.min + + final_node_data['last_heard'] = max( + node_ts_seen, + zero_hop_last_heard.get(node_id_int, datetime.datetime.min) # Use precalculated zero hop time + # We don't need to include neighbor info times here as they aren't distinct per-link + ) + + + include_node = False + if view_type in ['neighbor_info', 'merged']: + if has_neighbor_info: + final_node_data['neighbors'] = list(neighbor_info_links[node_id_int]['heard'].values()) + final_node_data['heard_by_neighbors'] = list(neighbor_info_links[node_id_int]['heard_by'].values()) + if final_node_data['neighbors'] or final_node_data['heard_by_neighbors']: + include_node = True + + if view_type in ['zero_hop', 'merged']: + if has_zero_hop_info: + final_node_data['zero_hop_neighbors'] = list(zero_hop_links[node_id_int]['heard'].values()) + final_node_data['heard_by_zero_hop'] = list(zero_hop_links[node_id_int]['heard_by'].values()) + if final_node_data['zero_hop_neighbors'] or final_node_data['heard_by_zero_hop']: + include_node = True + + if include_node: + active_nodes_data[node_id_hex] = final_node_data + + # Sort final results by last heard time + active_nodes_data = dict(sorted( + active_nodes_data.items(), + key=lambda item: item[1].get('last_heard', datetime.datetime.min), + reverse=True + )) + return render_template( "neighbors.html.j2", auth=auth(), config=config, - nodes=nodes, - active_nodes_with_connections=active_nodes_with_connections, + nodes=nodes, # Pass full nodes list for lookups in template + active_nodes_with_connections=active_nodes_data, # Pass the processed data view_type=view_type, utils=utils, datetime=datetime.datetime, @@ -689,7 +729,9 @@ def neighbors(): @app.route('/telemetry.html') def telemetry(): - md = MeshData() + md = get_meshdata() + if not md: # Check if MeshData failed to initialize + abort(503, description="Database connection unavailable") nodes = md.get_nodes() telemetry = md.get_telemetry_all() return render_template( @@ -706,7 +748,9 @@ def telemetry(): @app.route('/traceroutes.html') def traceroutes(): - md = MeshData() + md = get_meshdata() + if not md: # Check if MeshData failed to initialize + abort(503, description="Database connection unavailable") page = request.args.get('page', 1, type=int) per_page = 100 @@ -748,7 +792,9 @@ def traceroutes(): @app.route('/logs.html') def logs(): - md = MeshData() + md = get_meshdata() + if not md: # Check if MeshData failed to initialize + abort(503, description="Database connection unavailable") logs = md.get_logs() return render_template( "logs.html.j2", @@ -764,7 +810,9 @@ def logs(): @app.route('/monday.html') def monday(): - md = MeshData() + md = get_meshdata() + if not md: # Check if MeshData failed to initialize + abort(503, description="Database connection unavailable") nodes = md.get_nodes() chat = md.get_chat() monday = MeshtasticMonday(chat).get_data() @@ -782,7 +830,9 @@ def monday(): @app.route('/mynodes.html') def mynodes(): - md = MeshData() + md = get_meshdata() + if not md: # Check if MeshData failed to initialize + abort(503, description="Database connection unavailable") nodes = md.get_nodes() owner = auth() if not owner: @@ -894,7 +944,9 @@ def serve_static(filename): userp = r"user\_(\w+)\.html" if re.match(nodep, filename): - md = MeshData() + md = get_meshdata() + if not md: # Check if MeshData failed to initialize + abort(503, description="Database connection unavailable") match = re.match(nodep, filename) node = match.group(1) nodes = md.get_nodes() @@ -994,7 +1046,9 @@ def serve_static(filename): if re.match(userp, filename): match = re.match(userp, filename) username = match.group(1) - md = MeshData() + md = get_meshdata() + if not md: # Check if MeshData failed to initialize + abort(503, description="Database connection unavailable") owner = md.get_user(username) if not owner: abort(404) diff --git a/process_payload.py b/process_payload.py index 6e591ce4..248e44f1 100644 --- a/process_payload.py +++ b/process_payload.py @@ -183,17 +183,28 @@ def get_data(msg): return None -def process_payload(payload, topic): - md = MeshData() +def process_payload(payload, topic, md: MeshData): + # --- Add log at the start --- + logger = logging.getLogger(__name__) # Get logger instance + logger.debug(f"process_payload: Entered function for topic: {topic}") + # --- End log --- mp = get_packet(payload) if mp: try: data = get_data(mp) if data: # Only store if we got valid data + logger.debug(f"process_payload: Calling md.store() for topic {topic}") + # Use the passed-in MeshData instance md.store(data, topic) else: - logging.warning(f"Received invalid or unsupported message type on topic {topic}") + # Log topic only if debug is enabled or if it's an unsupported type + if config.get("server", "debug") == "true": + logging.warning(f"Received invalid or unsupported message type on topic {topic}. Payload: {payload[:100]}...") # Log partial payload for debug + else: + logger.warning(f"process_payload: get_packet returned None for topic {topic}") + except KeyError as e: logging.warning(f"Failed to process message: Missing key {str(e)} in payload on topic {topic}") except Exception as e: - logging.error(f"Unexpected error processing message: {str(e)}") \ No newline at end of file + # Log the full traceback for unexpected errors + logging.exception(f"Unexpected error processing message on topic {topic}: {str(e)}") # Use logging.exception \ No newline at end of file From 6140bfa56ac94365416b1603f649f9968dc5b4e3 Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 17 Apr 2025 00:55:41 +0000 Subject: [PATCH 011/155] added zero hop and merged graph options; added two other graph options for testing --- meshdata.py | 368 +++++++++++++++++++++++++++++++++++++++ meshinfo_web.py | 241 +++++++------------------ templates/graph.html.j2 | 196 +++++++++++++++++++-- templates/graph2.html.j2 | 292 +++++++++++++++++++++++++++++++ templates/graph3.html.j2 | 334 +++++++++++++++++++++++++++++++++++ 5 files changed, 1245 insertions(+), 186 deletions(-) create mode 100644 templates/graph2.html.j2 create mode 100644 templates/graph3.html.j2 diff --git a/meshdata.py b/meshdata.py index 78e1ff1d..c7a17ce6 100644 --- a/meshdata.py +++ b/meshdata.py @@ -6,6 +6,7 @@ import utils import logging import re +from timezone_utils import time_ago # Import time_ago from timezone_utils class CustomJSONEncoder(json.JSONEncoder): @@ -1411,6 +1412,373 @@ def import_chat(self, filename): print(f"failed to write record.") self.db.commit() + def get_neighbor_info_links(self, days=1): + """ + Fetch neighbor info links from the database. + + Args: + days: Number of days to look back for neighbor info data + + Returns: + Dictionary with node_id_int as keys and neighbor info data as values + """ + neighbor_info_links = {} # {node_id_int: {'heard': {neighbor_id_int: {data}}, ...}} + + cursor = self.db.cursor(dictionary=True) + # Fetch links from the specified days, adjust interval as needed + cursor.execute(""" + SELECT + ni.id, ni.neighbor_id, ni.snr, + p1.latitude_i as lat1_i, p1.longitude_i as lon1_i, + p2.latitude_i as lat2_i, p2.longitude_i as lon2_i + FROM neighborinfo ni + LEFT OUTER JOIN position p1 ON p1.id = ni.id + LEFT OUTER JOIN position p2 ON p2.id = ni.neighbor_id + WHERE ni.ts_created >= NOW() - INTERVAL %s DAY + """, (days,)) + + for row in cursor.fetchall(): + node_id_int = row['id'] + neighbor_id_int = row['neighbor_id'] + distance = None + if (row['lat1_i'] and row['lon1_i'] and row['lat2_i'] and row['lon2_i']): + distance = round(utils.distance_between_two_points( + row['lat1_i'] / 10000000, row['lon1_i'] / 10000000, + row['lat2_i'] / 10000000, row['lon2_i'] / 10000000 + ), 2) + + link_data = { + 'snr': row['snr'], + 'distance': distance, + 'neighbor_id': neighbor_id_int, + 'source_type': 'neighbor_info' + } + + if node_id_int not in neighbor_info_links: + neighbor_info_links[node_id_int] = {'heard': {}, 'heard_by': {}} + + # Add to node_id's 'heard' list + neighbor_info_links[node_id_int]['heard'][neighbor_id_int] = link_data + + # Add to neighbor_id's 'heard_by' list + if neighbor_id_int not in neighbor_info_links: + neighbor_info_links[neighbor_id_int] = {'heard': {}, 'heard_by': {}} + + heard_by_data = { + 'snr': row['snr'], + 'distance': distance, + 'neighbor_id': node_id_int, + 'source_type': 'neighbor_info' + } + neighbor_info_links[neighbor_id_int]['heard_by'][node_id_int] = heard_by_data + + cursor.close() + return neighbor_info_links + + def get_zero_hop_links(self, cutoff_time): + """ + Fetch zero-hop links from the database. + + Args: + cutoff_time: Unix timestamp for the cutoff time + + Returns: + Dictionary with node_id_int as keys and zero-hop data as values + """ + zero_hop_links = {} # {node_id_int: {'heard': {neighbor_id_int: {data}}, 'heard_by': {neighbor_id_int: {data}}}} + zero_hop_last_heard = {} # Keep track of last heard time for sorting + + cursor = self.db.cursor(dictionary=True) + cursor.execute(""" + SELECT + m.from_id, + m.received_by_id, + MAX(m.rx_snr) as snr, + COUNT(*) as message_count, + MAX(m.rx_time) as last_heard_time, + p_sender.latitude_i as lat_sender_i, + p_sender.longitude_i as lon_sender_i, + p_receiver.latitude_i as lat_receiver_i, + p_receiver.longitude_i as lon_receiver_i + FROM message_reception m + LEFT OUTER JOIN position p_sender ON p_sender.id = m.from_id + LEFT OUTER JOIN position p_receiver ON p_receiver.id = m.received_by_id + WHERE m.rx_time > %s + AND ( + (m.hop_limit IS NULL AND m.hop_start IS NULL) + OR + (m.hop_start - m.hop_limit = 0) + ) + GROUP BY m.from_id, m.received_by_id, + p_sender.latitude_i, p_sender.longitude_i, + p_receiver.latitude_i, p_receiver.longitude_i + """, (cutoff_time,)) + + for row in cursor.fetchall(): + sender_id = row['from_id'] + receiver_id = row['received_by_id'] + last_heard_dt = datetime.datetime.fromtimestamp(row['last_heard_time']) + + # Update last heard time for involved nodes + zero_hop_last_heard[sender_id] = max(zero_hop_last_heard.get(sender_id, datetime.datetime.min), last_heard_dt) + zero_hop_last_heard[receiver_id] = max(zero_hop_last_heard.get(receiver_id, datetime.datetime.min), last_heard_dt) + + distance = None + if (row['lat_sender_i'] and row['lon_sender_i'] and + row['lat_receiver_i'] and row['lon_receiver_i']): + distance = round(utils.distance_between_two_points( + row['lat_sender_i'] / 10000000, row['lon_sender_i'] / 10000000, + row['lat_receiver_i'] / 10000000, row['lon_receiver_i'] / 10000000 + ), 2) + + link_data = { + 'snr': row['snr'], + 'message_count': row['message_count'], + 'distance': distance, + 'last_heard': last_heard_dt, + 'neighbor_id': sender_id, # For receiver, neighbor is sender + 'source_type': 'zero_hop' + } + + heard_by_data = { + 'snr': row['snr'], + 'message_count': row['message_count'], + 'distance': distance, + 'last_heard': last_heard_dt, + 'neighbor_id': receiver_id, # For sender, neighbor is receiver + 'source_type': 'zero_hop' + } + + # Add to receiver's 'heard' list + if receiver_id not in zero_hop_links: + zero_hop_links[receiver_id] = {'heard': {}, 'heard_by': {}} + zero_hop_links[receiver_id]['heard'][sender_id] = link_data + + # Add to sender's 'heard_by' list + if sender_id not in zero_hop_links: + zero_hop_links[sender_id] = {'heard': {}, 'heard_by': {}} + zero_hop_links[sender_id]['heard_by'][receiver_id] = heard_by_data + + cursor.close() + return zero_hop_links, zero_hop_last_heard + + def get_graph_data(self, view_type='merged', days=1, zero_hop_timeout=43200): + """ + Get graph data for visualization. + + Args: + view_type: 'neighbor_info', 'zero_hop', or 'merged' + days: Number of days to look back for neighbor info data + zero_hop_timeout: Timeout in seconds for zero-hop data + + Returns: + Dictionary with nodes and edges for graph visualization + """ + nodes = self.get_nodes() + nodes_for_graph = [] + edges_for_graph = [] + active_node_ids_hex = set() # Keep track of nodes to include in the graph + + # Get neighbor info links + neighbor_info_links = {} + if view_type in ['neighbor_info', 'merged']: + neighbor_info_links = self.get_neighbor_info_links(days) + + # Add involved nodes to the active set if they exist in our main nodes list + for node_id_int, links in neighbor_info_links.items(): + node_id_hex = utils.convert_node_id_from_int_to_hex(node_id_int) + if node_id_hex in nodes and nodes[node_id_hex].get("active"): + active_node_ids_hex.add(node_id_hex) + + for neighbor_id_int in links.get('heard', {}): + neighbor_id_hex = utils.convert_node_id_from_int_to_hex(neighbor_id_int) + if neighbor_id_hex in nodes and nodes[neighbor_id_hex].get("active"): + active_node_ids_hex.add(neighbor_id_hex) + + # Get zero-hop links + zero_hop_links = {} + if view_type in ['zero_hop', 'merged']: + cutoff_time = int(time.time()) - zero_hop_timeout + zero_hop_links, _ = self.get_zero_hop_links(cutoff_time) + + # Add involved nodes to the active set if they exist + for node_id_int, links in zero_hop_links.items(): + node_id_hex = utils.convert_node_id_from_int_to_hex(node_id_int) + if node_id_hex in nodes and nodes[node_id_hex].get("active"): + active_node_ids_hex.add(node_id_hex) + + for neighbor_id_int in links.get('heard', {}): + neighbor_id_hex = utils.convert_node_id_from_int_to_hex(neighbor_id_int) + if neighbor_id_hex in nodes and nodes[neighbor_id_hex].get("active"): + active_node_ids_hex.add(neighbor_id_hex) + + # Build nodes for graph + for node_id_hex in active_node_ids_hex: + node_data = nodes[node_id_hex] + + # Get HW Model Name safely + hw_model = node_data.get('hw_model') + hw_model_name = 'Unknown HW' + if hw_model is not None: + try: + from meshtastic_support import HardwareModel + hw_model_name = HardwareModel(hw_model).name + hw_model_name = hw_model_name.replace('_', ' ').title() + except (ValueError, ImportError) as e: + logging.warning(f"Could not resolve hardware model {hw_model} for node {node_id_hex}: {e}") + + # Get Icon URL + node_name_for_icon = node_data.get('long_name', node_data.get('short_name', '')) + icon_url = utils.graph_icon(node_name_for_icon) + + nodes_for_graph.append({ + 'id': node_id_hex, + 'short': node_data.get('short_name', 'UNK'), + 'icon_url': icon_url, + 'node_data': { + 'long_name': node_data.get('long_name', 'Unknown Name'), + 'hw_model': hw_model_name, + 'last_seen': time_ago(node_data.get('ts_seen')) if node_data.get('ts_seen') else 'Never' + } + }) + + # Build edges for graph + added_node_pairs = set() + + # Add Neighbor Info Edges + if view_type in ['neighbor_info', 'merged']: + for node_id_int, links in neighbor_info_links.items(): + for neighbor_id_int, data in links.get('heard', {}).items(): + from_node_hex = utils.convert_node_id_from_int_to_hex(node_id_int) + to_node_hex = utils.convert_node_id_from_int_to_hex(neighbor_id_int) + + # Ensure both nodes are active and in our graph node list + if from_node_hex in active_node_ids_hex and to_node_hex in active_node_ids_hex: + node_pair = tuple(sorted((from_node_hex, to_node_hex))) + # Add edge only if this node pair hasn't been added yet + if node_pair not in added_node_pairs: + edges_for_graph.append({ + 'from': from_node_hex, + 'to': to_node_hex, + 'edge_data': data # Contains snr, distance, source_type='neighbor_info' + }) + added_node_pairs.add(node_pair) # Mark pair as added + + # Add Zero Hop Edges (only if not already added via Neighbor Info) + if view_type in ['zero_hop', 'merged']: + for receiver_id_int, links in zero_hop_links.items(): + for sender_id_int, data in links.get('heard', {}).items(): + # For zero hop, 'from' is sender, 'to' is receiver + from_node_hex = utils.convert_node_id_from_int_to_hex(sender_id_int) + to_node_hex = utils.convert_node_id_from_int_to_hex(receiver_id_int) + + # Ensure both nodes are active and in our graph node list + if from_node_hex in active_node_ids_hex and to_node_hex in active_node_ids_hex: + node_pair = tuple(sorted((from_node_hex, to_node_hex))) + # Add edge only if this node pair hasn't been added yet + if node_pair not in added_node_pairs: + edges_for_graph.append({ + 'from': from_node_hex, + 'to': to_node_hex, + 'edge_data': data # Contains snr, distance, source_type='zero_hop' + }) + added_node_pairs.add(node_pair) # Mark pair as added + + # Combine nodes and edges + graph_data = { + 'nodes': nodes_for_graph, + 'edges': edges_for_graph + } + + return graph_data + + def get_neighbors_data(self, view_type='neighbor_info', days=1, zero_hop_timeout=43200): + """ + Get neighbors data for the neighbors page. + + Args: + view_type: 'neighbor_info', 'zero_hop', or 'merged' + days: Number of days to look back for neighbor info data + zero_hop_timeout: Timeout in seconds for zero-hop data + + Returns: + Dictionary with node_id_hex as keys and neighbor data as values + """ + nodes = self.get_nodes() + if not nodes: + return {} + + # Get neighbor info links + neighbor_info_links = {} + if view_type in ['neighbor_info', 'merged']: + neighbor_info_links = self.get_neighbor_info_links(days) + + # Get zero-hop links + zero_hop_links = {} + zero_hop_last_heard = {} + if view_type in ['zero_hop', 'merged']: + cutoff_time = int(time.time()) - zero_hop_timeout + zero_hop_links, zero_hop_last_heard = self.get_zero_hop_links(cutoff_time) + + # Dictionary to hold the final data for active nodes + active_nodes_data = {} + + # Combine data for active nodes based on view type + for node_id_hex, node_base_data in nodes.items(): + if not node_base_data.get("active"): + continue # Skip inactive nodes + + node_id_int = utils.convert_node_id_from_hex_to_int(node_id_hex) + final_node_data = dict(node_base_data) # Start with base data + + # Initialize lists + final_node_data['neighbors'] = [] + final_node_data['heard_by_neighbors'] = [] + final_node_data['zero_hop_neighbors'] = [] + final_node_data['heard_by_zero_hop'] = [] + + has_neighbor_info = node_id_int in neighbor_info_links + has_zero_hop_info = node_id_int in zero_hop_links + + # Determine overall last heard time for sorting + last_heard_zero_hop = max([d['last_heard'] for d in zero_hop_links[node_id_int]['heard'].values()], default=datetime.datetime.min) if has_zero_hop_info else datetime.datetime.min + last_heard_by_zero_hop = max([d['last_heard'] for d in zero_hop_links[node_id_int]['heard_by'].values()], default=datetime.datetime.min) if has_zero_hop_info else datetime.datetime.min + + node_ts_seen = datetime.datetime.fromtimestamp(node_base_data['ts_seen']) if node_base_data.get('ts_seen') else datetime.datetime.min + + final_node_data['last_heard'] = max( + node_ts_seen, + zero_hop_last_heard.get(node_id_int, datetime.datetime.min) # Use precalculated zero hop time + # We don't need to include neighbor info times here as they aren't distinct per-link + ) + + include_node = False + if view_type in ['neighbor_info', 'merged']: + if has_neighbor_info: + final_node_data['neighbors'] = list(neighbor_info_links[node_id_int]['heard'].values()) + final_node_data['heard_by_neighbors'] = list(neighbor_info_links[node_id_int]['heard_by'].values()) + if final_node_data['neighbors'] or final_node_data['heard_by_neighbors']: + include_node = True + + if view_type in ['zero_hop', 'merged']: + if has_zero_hop_info: + final_node_data['zero_hop_neighbors'] = list(zero_hop_links[node_id_int]['heard'].values()) + final_node_data['heard_by_zero_hop'] = list(zero_hop_links[node_id_int]['heard_by'].values()) + if final_node_data['zero_hop_neighbors'] or final_node_data['heard_by_zero_hop']: + include_node = True + + if include_node: + active_nodes_data[node_id_hex] = final_node_data + + # Sort final results by last heard time + active_nodes_data = dict(sorted( + active_nodes_data.items(), + key=lambda item: item[1].get('last_heard', datetime.datetime.min), + reverse=True + )) + + return active_nodes_data + def create_database(): config = configparser.ConfigParser() diff --git a/meshinfo_web.py b/meshinfo_web.py index 58ecdc88..ed2a3d04 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -420,24 +420,82 @@ def traceroute_map(): ) @app.route('/graph.html') -@cache.cached(timeout=60) # Cache for 60 seconds +@cache.cached(timeout=60, query_string=True) # Cache based on query string (view type) def graph(): + view_type = request.args.get('view', 'merged') # Default to merged view md = get_meshdata() - if not md: # Check if MeshData failed to initialize + if not md: abort(503, description="Database connection unavailable") - nodes = md.get_nodes() - graph = md.graph_nodes() + + # Get graph data using the new method + graph_data = md.get_graph_data(view_type=view_type) + + # Log graph data size for debugging + logging.debug(f"Graph data for view '{view_type}': {len(graph_data['nodes'])} nodes, {len(graph_data['edges'])} edges.") + return render_template( "graph.html.j2", auth=auth(), config=config, - nodes=nodes, - graph=graph, + nodes=md.get_nodes(), # Pass full nodes list for lookups in template + graph=graph_data, + view_type=view_type, + utils=utils, + datetime=datetime.datetime, + timestamp=datetime.datetime.now(), + ) + +@app.route('/graph2.html') +@cache.cached(timeout=60, query_string=True) # Cache based on query string (view type) +def graph2(): + view_type = request.args.get('view', 'merged') # Default to merged view + md = get_meshdata() + if not md: + abort(503, description="Database connection unavailable") + + # Get graph data using the new method + graph_data = md.get_graph_data(view_type=view_type) + + # Log graph data size for debugging + logging.debug(f"Graph data for view '{view_type}': {len(graph_data['nodes'])} nodes, {len(graph_data['edges'])} edges.") + + return render_template( + "graph2.html.j2", + auth=auth(), + config=config, + nodes=md.get_nodes(), # Pass full nodes list for lookups in template + graph=graph_data, + view_type=view_type, utils=utils, datetime=datetime.datetime, timestamp=datetime.datetime.now(), ) +@app.route('/graph3.html') +@cache.cached(timeout=60, query_string=True) # Cache based on query string (view type) +def graph3(): + view_type = request.args.get('view', 'merged') # Default to merged view + md = get_meshdata() + if not md: + abort(503, description="Database connection unavailable") + + # Get graph data using the new method + graph_data = md.get_graph_data(view_type=view_type) + + # Log graph data size for debugging + logging.debug(f"Graph data for view '{view_type}': {len(graph_data['nodes'])} nodes, {len(graph_data['edges'])} edges.") + + return render_template( + "graph3.html.j2", + auth=auth(), + config=config, + nodes=md.get_nodes(), # Pass full nodes list for lookups in template + graph=graph_data, + view_type=view_type, + utils=utils, + datetime=datetime.datetime, + timestamp=datetime.datetime.now(), + ) @app.route('/map.html') def map(): @@ -538,175 +596,8 @@ def neighbors(): utils=utils, datetime=datetime.datetime, timestamp=datetime.datetime.now() ) - - zero_hop_timeout = int(config.get("server", "zero_hop_timeout", fallback=43200)) - cutoff_time = int(time.time()) - zero_hop_timeout - - # Dictionary to hold the final data for active nodes - active_nodes_data = {} - - # --- Pre-fetch and process Neighbor Info --- - neighbor_info_links = {} # {node_id: {'heard': {neighbor_id: {data}}, 'heard_by': {neighbor_id: {data}}} } - - # Fetch all recent neighbor info links once - cursor = md.db.cursor(dictionary=True) - cursor.execute(""" - SELECT - ni.id, ni.neighbor_id, ni.snr, - p1.latitude_i as lat1_i, p1.longitude_i as lon1_i, - p2.latitude_i as lat2_i, p2.longitude_i as lon2_i - FROM neighborinfo ni - LEFT OUTER JOIN position p1 ON p1.id = ni.id - LEFT OUTER JOIN position p2 ON p2.id = ni.neighbor_id - WHERE ni.ts_created >= NOW() - INTERVAL 1 DAY - """) - - for row in cursor.fetchall(): - node_id = row['id'] - neighbor_id = row['neighbor_id'] - distance = None - if (row['lat1_i'] and row['lon1_i'] and row['lat2_i'] and row['lon2_i']): - distance = round(utils.distance_between_two_points( - row['lat1_i'] / 10000000, row['lon1_i'] / 10000000, - row['lat2_i'] / 10000000, row['lon2_i'] / 10000000 - ), 2) - - link_data = {'snr': row['snr'], 'distance': distance, 'neighbor_id': neighbor_id} - heard_by_data = {'snr': row['snr'], 'distance': distance, 'neighbor_id': node_id} - - # Add to node_id's 'heard' list - if node_id not in neighbor_info_links: neighbor_info_links[node_id] = {'heard': {}, 'heard_by': {}} - neighbor_info_links[node_id]['heard'][neighbor_id] = link_data - - # Add to neighbor_id's 'heard_by' list - if neighbor_id not in neighbor_info_links: neighbor_info_links[neighbor_id] = {'heard': {}, 'heard_by': {}} - neighbor_info_links[neighbor_id]['heard_by'][node_id] = heard_by_data - cursor.close() - - - # --- Pre-fetch and process Zero Hop Info --- - zero_hop_links = {} # {node_id: {'heard': {neighbor_id: {data}}, 'heard_by': {neighbor_id: {data}}} } - zero_hop_last_heard = {} # Keep track of last heard time for sorting - - if view_type in ['zero_hop', 'merged']: - cursor = md.db.cursor(dictionary=True) - cursor.execute(""" - SELECT - m.from_id, - m.received_by_id, - MAX(m.rx_snr) as snr, - COUNT(*) as message_count, - MAX(m.rx_time) as last_heard_time, - p_sender.latitude_i as lat_sender_i, - p_sender.longitude_i as lon_sender_i, - p_receiver.latitude_i as lat_receiver_i, - p_receiver.longitude_i as lon_receiver_i - FROM message_reception m - LEFT OUTER JOIN position p_sender ON p_sender.id = m.from_id - LEFT OUTER JOIN position p_receiver ON p_receiver.id = m.received_by_id - WHERE m.rx_time > %s - AND ( - (m.hop_limit IS NULL AND m.hop_start IS NULL) - OR - (m.hop_start - m.hop_limit = 0) - ) - GROUP BY m.from_id, m.received_by_id, - p_sender.latitude_i, p_sender.longitude_i, - p_receiver.latitude_i, p_receiver.longitude_i - """, (cutoff_time,)) - - for row in cursor.fetchall(): - sender_id = row['from_id'] - receiver_id = row['received_by_id'] - last_heard_dt = datetime.datetime.fromtimestamp(row['last_heard_time']) - - # Update last heard time for involved nodes - zero_hop_last_heard[sender_id] = max(zero_hop_last_heard.get(sender_id, datetime.datetime.min), last_heard_dt) - zero_hop_last_heard[receiver_id] = max(zero_hop_last_heard.get(receiver_id, datetime.datetime.min), last_heard_dt) - - distance = None - if (row['lat_sender_i'] and row['lon_sender_i'] and - row['lat_receiver_i'] and row['lon_receiver_i']): - distance = round(utils.distance_between_two_points( - row['lat_sender_i'] / 10000000, row['lon_sender_i'] / 10000000, - row['lat_receiver_i'] / 10000000, row['lon_receiver_i'] / 10000000 - ), 2) - - link_data = { - 'snr': row['snr'], - 'message_count': row['message_count'], - 'distance': distance, - 'last_heard': last_heard_dt, - 'neighbor_id': sender_id # For receiver, neighbor is sender - } - heard_by_data = { - 'snr': row['snr'], - 'message_count': row['message_count'], - 'distance': distance, - 'last_heard': last_heard_dt, - 'neighbor_id': receiver_id # For sender, neighbor is receiver - } - - # Add to receiver's 'heard' list - if receiver_id not in zero_hop_links: zero_hop_links[receiver_id] = {'heard': {}, 'heard_by': {}} - zero_hop_links[receiver_id]['heard'][sender_id] = link_data - - # Add to sender's 'heard_by' list - if sender_id not in zero_hop_links: zero_hop_links[sender_id] = {'heard': {}, 'heard_by': {}} - zero_hop_links[sender_id]['heard_by'][receiver_id] = heard_by_data - - cursor.close() - - - # --- Combine data for active nodes based on view type --- - for node_id_hex, node_base_data in nodes.items(): - if not node_base_data.get("active"): - continue # Skip inactive nodes - - node_id_int = utils.convert_node_id_from_hex_to_int(node_id_hex) - final_node_data = dict(node_base_data) # Start with base data - - # Initialize lists - final_node_data['neighbors'] = [] - final_node_data['heard_by_neighbors'] = [] - final_node_data['zero_hop_neighbors'] = [] - final_node_data['heard_by_zero_hop'] = [] - - has_neighbor_info = node_id_int in neighbor_info_links - has_zero_hop_info = node_id_int in zero_hop_links - - # Determine overall last heard time for sorting - last_heard_zero_hop = max([d['last_heard'] for d in zero_hop_links[node_id_int]['heard'].values()], default=datetime.datetime.min) if has_zero_hop_info else datetime.datetime.min - last_heard_by_zero_hop = max([d['last_heard'] for d in zero_hop_links[node_id_int]['heard_by'].values()], default=datetime.datetime.min) if has_zero_hop_info else datetime.datetime.min - last_heard_zero_hop = max([d['last_heard'] for d in zero_hop_links[node_id_int]['heard'].values()], default=datetime.datetime.min) if has_zero_hop_info else datetime.datetime.min - last_heard_by_zero_hop = max([d['last_heard'] for d in zero_hop_links[node_id_int]['heard_by'].values()], default=datetime.datetime.min) if has_zero_hop_info else datetime.datetime.min - - node_ts_seen = datetime.datetime.fromtimestamp(node_base_data['ts_seen']) if node_base_data.get('ts_seen') else datetime.datetime.min - - final_node_data['last_heard'] = max( - node_ts_seen, - zero_hop_last_heard.get(node_id_int, datetime.datetime.min) # Use precalculated zero hop time - # We don't need to include neighbor info times here as they aren't distinct per-link - ) - - - include_node = False - if view_type in ['neighbor_info', 'merged']: - if has_neighbor_info: - final_node_data['neighbors'] = list(neighbor_info_links[node_id_int]['heard'].values()) - final_node_data['heard_by_neighbors'] = list(neighbor_info_links[node_id_int]['heard_by'].values()) - if final_node_data['neighbors'] or final_node_data['heard_by_neighbors']: - include_node = True - - if view_type in ['zero_hop', 'merged']: - if has_zero_hop_info: - final_node_data['zero_hop_neighbors'] = list(zero_hop_links[node_id_int]['heard'].values()) - final_node_data['heard_by_zero_hop'] = list(zero_hop_links[node_id_int]['heard_by'].values()) - if final_node_data['zero_hop_neighbors'] or final_node_data['heard_by_zero_hop']: - include_node = True - - if include_node: - active_nodes_data[node_id_hex] = final_node_data + # Get neighbors data using the new method + active_nodes_data = md.get_neighbors_data(view_type=view_type) # Sort final results by last heard time active_nodes_data = dict(sorted( diff --git a/templates/graph.html.j2 b/templates/graph.html.j2 index 77514494..f6fff35a 100644 --- a/templates/graph.html.j2 +++ b/templates/graph.html.j2 @@ -5,27 +5,201 @@ {% block content %}
{{ this_page.title() }}
+ +
+ + Neighbor Info + + + Zero Hop + + + Merged + +
-{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/templates/graph2.html.j2 b/templates/graph2.html.j2 new file mode 100644 index 00000000..a0942757 --- /dev/null +++ b/templates/graph2.html.j2 @@ -0,0 +1,292 @@ +{% set this_page = "graph" %} +{% extends "layout.html.j2" %} + +{% block title %}Graph | MeshInfo{% endblock %} +{% block content %} +
+
{{ this_page.title() }}
+ +
+ + Neighbor Info + + + Zero Hop + + + Merged + +
+

+ Connections shown: Blue solid lines = Neighbor Info module messages, + Green dashed lines = LoRa message reception (Zero Hop). + Merged view shows both types. +

+
+
+ + + + + + + + + + +{% endblock %} \ No newline at end of file diff --git a/templates/graph3.html.j2 b/templates/graph3.html.j2 new file mode 100644 index 00000000..6534b60c --- /dev/null +++ b/templates/graph3.html.j2 @@ -0,0 +1,334 @@ +{% set this_page = "graph" %} +{% extends "layout.html.j2" %} + +{% block title %}Graph | MeshInfo{% endblock %} +{% block content %} +
+
{{ this_page.title() }}
+ +
+ + Neighbor Info + + + Zero Hop + + + Merged + +
+

+ Connections shown: Blue solid lines = Neighbor Info module messages, + Green dashed lines = LoRa message reception (Zero Hop). + Merged view shows both types. +

+
+
+ + + + + + + + + + + + + + +{% endblock %} \ No newline at end of file From 3e157bb8abf26d51a9dacb6d6a3fc3413f4325e5 Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 17 Apr 2025 05:12:28 +0000 Subject: [PATCH 012/155] more revisions to possible graph updates --- meshinfo_web.py | 26 ++++ templates/graph2.html.j2 | 6 +- templates/graph3.html.j2 | 7 +- templates/graph4.html.j2 | 329 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 362 insertions(+), 6 deletions(-) create mode 100644 templates/graph4.html.j2 diff --git a/meshinfo_web.py b/meshinfo_web.py index ed2a3d04..325f79e1 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -497,6 +497,32 @@ def graph3(): timestamp=datetime.datetime.now(), ) +@app.route('/graph4.html') +@cache.cached(timeout=60, query_string=True) # Cache based on query string (view type) +def graph4(): + view_type = request.args.get('view', 'merged') # Default to merged view + md = get_meshdata() + if not md: + abort(503, description="Database connection unavailable") + + # Get graph data using the new method + graph_data = md.get_graph_data(view_type=view_type) + + # Log graph data size for debugging + logging.debug(f"Graph data for view '{view_type}': {len(graph_data['nodes'])} nodes, {len(graph_data['edges'])} edges.") + + return render_template( + "graph4.html.j2", + auth=auth(), + config=config, + nodes=md.get_nodes(), # Pass full nodes list for lookups in template + graph=graph_data, + view_type=view_type, + utils=utils, + datetime=datetime.datetime, + timestamp=datetime.datetime.now(), + ) + @app.route('/map.html') def map(): md = get_meshdata() diff --git a/templates/graph2.html.j2 b/templates/graph2.html.j2 index a0942757..0d1bb3c9 100644 --- a/templates/graph2.html.j2 +++ b/templates/graph2.html.j2 @@ -7,15 +7,15 @@
{{ this_page.title() }}
- Neighbor Info - Zero Hop - Merged diff --git a/templates/graph3.html.j2 b/templates/graph3.html.j2 index 6534b60c..06eee8c4 100644 --- a/templates/graph3.html.j2 +++ b/templates/graph3.html.j2 @@ -7,15 +7,15 @@
{{ this_page.title() }}
- Neighbor Info - Zero Hop - Merged @@ -253,6 +253,7 @@ fit: false, padding: 100, nodeDimensionsIncludeLabels: true, + // Increase repulsion to prevent overlap nodeRepulsion: function(node) { const baseRepulsion = 35000; diff --git a/templates/graph4.html.j2 b/templates/graph4.html.j2 new file mode 100644 index 00000000..be0a543d --- /dev/null +++ b/templates/graph4.html.j2 @@ -0,0 +1,329 @@ +{% set this_page = "graph" %} +{% extends "layout.html.j2" %} + +{% block title %}Graph | MeshInfo{% endblock %} +{% block content %} +
+
{{ this_page.title() }}
+ +
+ + Neighbor Info + + + Zero Hop + + + Merged + +
+

+ Connections shown: Blue solid lines = Neighbor Info module messages, + Green dashed lines = LoRa message reception (Zero Hop). + Merged view shows both types. +

+
+
+ + + + + + + + + + + + + +{% endblock %} \ No newline at end of file From 6d9dd6b609575017d540bf8ccddbe4987e2dcbae Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Apr 2025 03:00:13 +0000 Subject: [PATCH 013/155] added time of most adjacent position update to the message propagation map tooltips --- templates/message_map.html.j2 | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/templates/message_map.html.j2 b/templates/message_map.html.j2 index 3db54491..a41199e2 100644 --- a/templates/message_map.html.j2 +++ b/templates/message_map.html.j2 @@ -26,6 +26,10 @@
Time: {{ format_timestamp(message.ts_created) }}
Message: {{ message.text }} + {% if sender_position and sender_position.position_time %} +
+ Sender Position Updated: {{ format_timestamp(sender_position.position_time) }} + {% endif %}

@@ -91,6 +95,13 @@ ) * 111.32 * 100) / 100; // Convert to km and round to 2 decimal places } + // Add function to format timestamp + function formatTimestamp(timestamp) { + if (!timestamp) return 'Unknown'; + const date = new Date(timestamp * 1000); + return date.toLocaleString(); + } + const features = []; const lines = []; const labels = []; @@ -398,6 +409,11 @@ } } + // Add position time information + if (node.positionTime) { + html += `Position Updated: ${formatTimestamp(node.positionTime)}
`; + } + html += '
'; content.innerHTML = html; From 1cb01be13d880fc8046b5453f3ea160649cf576a Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Apr 2025 03:50:12 +0000 Subject: [PATCH 014/155] fixed heard chat notification wrapping --- templates/chat2.html.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/chat2.html.j2 b/templates/chat2.html.j2 index 1f40557d..09311e5d 100644 --- a/templates/chat2.html.j2 +++ b/templates/chat2.html.j2 @@ -272,9 +272,9 @@ window.addEventListener('load', function() { {{ message.text }}
{% if message.receptions %} -
+
-
+
{% for reception in message.receptions %} {% set node_id = utils.convert_node_id_from_int_to_hex(reception.node_id) %} {% if node_id in nodes %} From f4bccf99d5460c4756173284fe0bc6183bbaa784 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Apr 2025 19:11:03 +0000 Subject: [PATCH 015/155] alpha metrics feature added --- app.py | 145 +++++++ config.ini.sample | 2 + meshdata.py | 24 +- meshinfo_web.py | 455 +++++++++++++++++++++- templates/layout.html.j2 | 2 +- templates/metrics.html.j2 | 779 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 1399 insertions(+), 8 deletions(-) create mode 100644 app.py create mode 100644 templates/metrics.html.j2 diff --git a/app.py b/app.py new file mode 100644 index 00000000..ffbea7af --- /dev/null +++ b/app.py @@ -0,0 +1,145 @@ +@app.route('/metrics.html') +def metrics(): + md = get_meshdata() + if not md: # Check if MeshData failed to initialize + abort(503, description="Database connection unavailable") + return render_template( + "metrics.html.j2", + auth=auth(), + config=config, + this_page="metrics", + utils=utils, + datetime=datetime.datetime, + timestamp=datetime.datetime.now() + ) + +@app.route('/api/metrics') +def get_metrics(): + db = MeshData() + + # Get the last 24 hours of data + end_time = datetime.now() + start_time = end_time - timedelta(hours=24) + + # Nodes Online (per hour) + nodes_online = db.execute(""" + SELECT + strftime('%Y-%m-%d %H:00', datetime(timestamp, 'unixepoch')) as hour, + COUNT(DISTINCT id) as node_count + FROM telemetry + WHERE timestamp >= ? AND timestamp <= ? + GROUP BY hour + ORDER BY hour + """, (int(start_time.timestamp()), int(end_time.timestamp()))) + + # Message Traffic (per hour) + message_traffic = db.execute(""" + SELECT + strftime('%Y-%m-%d %H:00', datetime(timestamp, 'unixepoch')) as hour, + COUNT(*) as message_count + FROM text + WHERE timestamp >= ? AND timestamp <= ? + GROUP BY hour + ORDER BY hour + """, (int(start_time.timestamp()), int(end_time.timestamp()))) + + # Channel Utilization + channel_util = db.execute(""" + SELECT + strftime('%Y-%m-%d %H:00', datetime(timestamp, 'unixepoch')) as hour, + AVG(channel_utilization) as avg_util + FROM telemetry + WHERE timestamp >= ? AND timestamp <= ? + GROUP BY hour + ORDER BY hour + """, (int(start_time.timestamp()), int(end_time.timestamp()))) + + # Battery Levels (for each node) + battery_levels = db.execute(""" + SELECT + id, + strftime('%Y-%m-%d %H:00', datetime(timestamp, 'unixepoch')) as hour, + AVG(battery_level) as avg_battery + FROM telemetry + WHERE timestamp >= ? AND timestamp <= ? + GROUP BY id, hour + ORDER BY hour + """, (int(start_time.timestamp()), int(end_time.timestamp()))) + + # Temperature Readings + temperature = db.execute(""" + SELECT + id, + strftime('%Y-%m-%d %H:00', datetime(timestamp, 'unixepoch')) as hour, + AVG(temperature) as avg_temp + FROM telemetry + WHERE timestamp >= ? AND timestamp <= ? + GROUP BY id, hour + ORDER BY hour + """, (int(start_time.timestamp()), int(end_time.timestamp()))) + + # Signal Strength (SNR) + snr = db.execute(""" + SELECT + strftime('%Y-%m-%d %H:00', datetime(timestamp, 'unixepoch')) as hour, + AVG(snr) as avg_snr + FROM message_reception + WHERE timestamp >= ? AND timestamp <= ? + GROUP BY hour + ORDER BY hour + """, (int(start_time.timestamp()), int(end_time.timestamp()))) + + # Process battery levels data + battery_data = {} + for row in battery_levels: + if row['id'] not in battery_data: + battery_data[row['id']] = {'hours': [], 'levels': []} + battery_data[row['id']]['hours'].append(row['hour']) + battery_data[row['id']]['levels'].append(row['avg_battery']) + + # Process temperature data + temp_data = {} + for row in temperature: + if row['id'] not in temp_data: + temp_data[row['id']] = {'hours': [], 'temps': []} + temp_data[row['id']]['hours'].append(row['hour']) + temp_data[row['id']]['temps'].append(row['avg_temp']) + + return jsonify({ + 'nodes_online': { + 'labels': [row['hour'] for row in nodes_online], + 'data': [row['node_count'] for row in nodes_online] + }, + 'message_traffic': { + 'labels': [row['hour'] for row in message_traffic], + 'data': [row['message_count'] for row in message_traffic] + }, + 'channel_util': { + 'labels': [row['hour'] for row in channel_util], + 'data': [row['avg_util'] for row in channel_util] + }, + 'battery_levels': { + 'labels': list(set([hour for data in battery_data.values() for hour in data['hours']])), + 'datasets': [ + { + 'node_id': node_id, + 'data': data['levels'] + } + for node_id, data in battery_data.items() + ] + }, + 'temperature': { + 'labels': list(set([hour for data in temp_data.values() for hour in data['hours']])), + 'datasets': [ + { + 'node_id': node_id, + 'data': data['temps'] + } + for node_id, data in temp_data.items() + ] + }, + 'snr': { + 'labels': [row['hour'] for row in snr], + 'data': [row['avg_snr'] for row in snr] + } + }) \ No newline at end of file diff --git a/config.ini.sample b/config.ini.sample index d46ba32a..03def88b 100644 --- a/config.ini.sample +++ b/config.ini.sample @@ -1,5 +1,6 @@ [mesh] name=ZA Mesh +short_name=ZA Mesh description=Serving Meshtastic to South Africa. contact=dade@dade.co.za url=https://mesh.zr1rf.za.net @@ -29,6 +30,7 @@ render_interval=15 debug=true timezone=America/Los_Angeles zero_hop_timeout=43200 +telemetry_retention_days = 30 [geocoding] enabled=false diff --git a/meshdata.py b/meshdata.py index c7a17ce6..66bc6564 100644 --- a/meshdata.py +++ b/meshdata.py @@ -911,12 +911,26 @@ def get_successful_traceroutes(self): return results def store_telemetry(self, data): + # Use class-level config that's already loaded + retention_days = self.config.getint("server", "telemetry_retention_days", fallback=None) if self.config else None + + # Only check for cleanup every N records (using modulo on auto-increment ID or timestamp) cur = self.db.cursor() - cur.execute(f"SELECT COUNT(*) FROM telemetry") - count = cur.fetchone()[0] - if count >= 20000: - cur.execute(f"""DELETE FROM telemetry -ORDER BY ts_created ASC LIMIT 1""") + cur.execute("SELECT COUNT(*) FROM telemetry WHERE ts_created > DATE_SUB(NOW(), INTERVAL 1 HOUR)") + recent_count = cur.fetchone()[0] + + # Only run cleanup if we have accumulated enough recent records + if recent_count > 200: # Adjust threshold as needed + if retention_days: + cur.execute("""DELETE FROM telemetry + WHERE ts_created < DATE_SUB(NOW(), INTERVAL %s DAY)""", + (retention_days,)) + else: + # More efficient count-based cleanup + cur.execute("""DELETE FROM telemetry WHERE ts_created <= + (SELECT ts_created FROM + (SELECT ts_created FROM telemetry ORDER BY ts_created DESC LIMIT 1 OFFSET 20000) t)""") + cur.close() self.db.commit() diff --git a/meshinfo_web.py b/meshinfo_web.py index 325f79e1..8068b35e 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -7,7 +7,8 @@ redirect, url_for, abort, - g + g, + jsonify ) from flask_caching import Cache from waitress import serve @@ -599,7 +600,7 @@ def map(): zero_hop_data=zero_hop_data, zero_hop_timeout=zero_hop_timeout, utils=utils, - datetime=datetime, + datetime=datetime.datetime, timestamp=datetime.datetime.now() ) @@ -988,6 +989,456 @@ def serve_static(filename): return send_from_directory("www", filename) +@app.route('/metrics.html') +@cache.cached(timeout=60) # Cache for 60 seconds +def metrics(): + md = get_meshdata() + if not md: # Check if MeshData failed to initialize + abort(503, description="Database connection unavailable") + return render_template( + "metrics.html.j2", + auth=auth(), + config=config, + this_page="metrics", + utils=utils, + datetime=datetime.datetime, + timestamp=datetime.datetime.now() + ) + +@app.route('/api/metrics') +def get_metrics(): + md = get_meshdata() + if not md: # Check if MeshData failed to initialize + abort(503, description="Database connection unavailable") + + try: + # Get time range from request parameters + time_range = request.args.get('time_range', 'day') # day, week, month, year, all + + # Set time range based on parameter + end_time = datetime.datetime.now() + if time_range == 'week': + start_time = end_time - datetime.timedelta(days=7) + bucket_size = 180 # 3 hours in minutes + elif time_range == 'month': + start_time = end_time - datetime.timedelta(days=30) + bucket_size = 720 # 12 hours in minutes + elif time_range == 'year': + start_time = end_time - datetime.timedelta(days=365) + bucket_size = 2880 # 2 days in minutes + elif time_range == 'all': + # For 'all', we'll first check the data range in the database + cursor = md.db.cursor(dictionary=True) + cursor.execute("SELECT MIN(ts_created) as min_time FROM telemetry") + min_time = cursor.fetchone()['min_time'] + cursor.close() + + if min_time: + start_time = min_time + else: + # Default to 1 year if no data + start_time = end_time - datetime.timedelta(days=365) + + bucket_size = 10080 # 7 days in minutes + else: # Default to 'day' + start_time = end_time - datetime.timedelta(hours=24) + bucket_size = 30 # 30 minutes + + logging.info(f"Fetching metrics data from {start_time} to {end_time} with bucket size {bucket_size} minutes") + + # Check if we have any data in the tables + cursor = md.db.cursor(dictionary=True) + + # Get the actual time range of data in the database + cursor.execute("SELECT MIN(ts_created) as min_time, MAX(ts_created) as max_time FROM telemetry") + telemetry_time_range = cursor.fetchone() + logging.info(f"Telemetry data time range: {telemetry_time_range['min_time']} to {telemetry_time_range['max_time']}") + + cursor.execute("SELECT MIN(ts_created) as min_time, MAX(ts_created) as max_time FROM text") + text_time_range = cursor.fetchone() + logging.info(f"Text data time range: {text_time_range['min_time']} to {text_time_range['max_time']}") + + cursor.execute("SELECT MIN(ts_created) as min_time, MAX(ts_created) as max_time FROM message_reception") + reception_time_range = cursor.fetchone() + logging.info(f"Message reception data time range: {reception_time_range['min_time']} to {reception_time_range['max_time']}") + + # Check if tables exist and have data + cursor.execute("SELECT COUNT(*) as count FROM telemetry") + telemetry_count = cursor.fetchone()['count'] + logging.info(f"Telemetry table has {telemetry_count} records") + + cursor.execute("SELECT COUNT(*) as count FROM text") + text_count = cursor.fetchone()['count'] + logging.info(f"Text table has {text_count} records") + + cursor.execute("SELECT COUNT(*) as count FROM message_reception") + reception_count = cursor.fetchone()['count'] + logging.info(f"Message_reception table has {reception_count} records") + + # Add warnings for metrics with incomplete data + warnings = [] + if telemetry_time_range['min_time'] and telemetry_time_range['min_time'] > start_time: + warnings.append( + f"Telemetry data (nodes online, channel utilization, battery, temperature) " + f"only available from {telemetry_time_range['min_time'].strftime('%Y-%m-%d %H:%M:%S')}" + ) + + # Convert timestamps to the correct format for MySQL + start_timestamp = start_time.strftime('%Y-%m-%d %H:%M:%S') + end_timestamp = end_time.strftime('%Y-%m-%d %H:%M:%S') + + logging.info(f"Querying with timestamps: {start_timestamp} to {end_timestamp}") + + # Format string for time buckets based on bucket size + if bucket_size >= 10080: # 7 days or more + time_format = '%Y-%m-%d' # Daily format + elif bucket_size >= 1440: # 1 day or more + time_format = '%Y-%m-%d %H:00' # Hourly format + else: + time_format = '%Y-%m-%d %H:%i' # Minute format + + # Nodes Online + nodes_online_query = f""" + SELECT + DATE_FORMAT( + DATE_ADD( + ts_created, + INTERVAL -MOD(MINUTE(ts_created), {bucket_size}) MINUTE + ), + '{time_format}' + ) as time_slot, + COUNT(DISTINCT id) as node_count + FROM telemetry + WHERE ts_created >= %s AND ts_created <= %s + GROUP BY time_slot + ORDER BY time_slot + """ + logging.debug(f"Executing nodes online query with params: {start_timestamp}, {end_timestamp}") + cursor.execute(nodes_online_query, (start_timestamp, end_timestamp)) + nodes_online = cursor.fetchall() + logging.debug(f"Nodes online query returned {len(nodes_online)} records. First record: {nodes_online[0] if nodes_online else 'None'}") + + # Message Traffic + # First, generate a series of time slots + time_slots_query = f""" + WITH RECURSIVE time_slots AS ( + SELECT DATE_FORMAT( + DATE_ADD(%s, INTERVAL -MOD(MINUTE(%s), {bucket_size}) MINUTE), + '{time_format}' + ) as time_slot + UNION ALL + SELECT DATE_FORMAT( + DATE_ADD( + STR_TO_DATE(time_slot, '{time_format}'), + INTERVAL {bucket_size} MINUTE + ), + '{time_format}' + ) + FROM time_slots + WHERE STR_TO_DATE(time_slot, '{time_format}') < %s + ) + SELECT t.time_slot, + COALESCE(m.message_count, 0) as message_count + FROM time_slots t + LEFT JOIN ( + SELECT DATE_FORMAT( + DATE_ADD( + ts_created, + INTERVAL -MOD(MINUTE(ts_created), {bucket_size}) MINUTE + ), + '{time_format}' + ) as time_slot, + COUNT(*) as message_count + FROM text + WHERE ts_created >= %s AND ts_created <= %s + GROUP BY time_slot + ) m ON t.time_slot = m.time_slot + ORDER BY t.time_slot; + """ + logging.debug(f"Executing message traffic query with params: {start_timestamp}, {end_timestamp}") + cursor.execute(time_slots_query, (start_timestamp, start_timestamp, end_timestamp, start_timestamp, end_timestamp)) + message_traffic = cursor.fetchall() + logging.debug(f"Message traffic query returned {len(message_traffic)} records. First record: {message_traffic[0] if message_traffic else 'None'}") + + # Channel Utilization + channel_util_query = f""" + SELECT + DATE_FORMAT( + DATE_ADD( + ts_created, + INTERVAL -MOD(MINUTE(ts_created), {bucket_size}) MINUTE + ), + '{time_format}' + ) as time_slot, + AVG(channel_utilization) as avg_util + FROM telemetry + WHERE ts_created >= %s AND ts_created <= %s + GROUP BY time_slot + ORDER BY time_slot + """ + logging.debug(f"Executing channel utilization query with params: {start_timestamp}, {end_timestamp}") + cursor.execute(channel_util_query, (start_timestamp, end_timestamp)) + channel_util = cursor.fetchall() + logging.debug(f"Channel utilization query returned {len(channel_util)} records. First record: {channel_util[0] if channel_util else 'None'}") + + # Average Battery Level + battery_query = f""" + SELECT + DATE_FORMAT( + DATE_ADD( + ts_created, + INTERVAL -MOD(MINUTE(ts_created), {bucket_size}) MINUTE + ), + '{time_format}' + ) as time_slot, + AVG(battery_level) as avg_battery + FROM telemetry + WHERE ts_created >= %s AND ts_created <= %s + GROUP BY time_slot + ORDER BY time_slot + """ + logging.debug(f"Executing battery level query with params: {start_timestamp}, {end_timestamp}") + cursor.execute(battery_query, (start_timestamp, end_timestamp)) + battery_levels = cursor.fetchall() + logging.debug(f"Battery level query returned {len(battery_levels)} records. First record: {battery_levels[0] if battery_levels else 'None'}") + + # Average Temperature + temperature_query = f""" + SELECT + DATE_FORMAT( + DATE_ADD( + ts_created, + INTERVAL -MOD(MINUTE(ts_created), {bucket_size}) MINUTE + ), + '{time_format}' + ) as time_slot, + AVG(temperature) as avg_temp + FROM telemetry + WHERE ts_created >= %s AND ts_created <= %s + GROUP BY time_slot + ORDER BY time_slot + """ + logging.debug(f"Executing temperature query with params: {start_timestamp}, {end_timestamp}") + cursor.execute(temperature_query, (start_timestamp, end_timestamp)) + temperature = cursor.fetchall() + logging.debug(f"Temperature query returned {len(temperature)} records. First record: {temperature[0] if temperature else 'None'}") + + # Signal Strength (SNR) + snr_query = f""" + SELECT + DATE_FORMAT( + DATE_ADD( + ts_created, + INTERVAL -MOD(MINUTE(ts_created), {bucket_size}) MINUTE + ), + '{time_format}' + ) as time_slot, + AVG(rx_snr) as avg_snr + FROM message_reception + WHERE ts_created >= %s AND ts_created <= %s + GROUP BY time_slot + ORDER BY time_slot + """ + logging.debug(f"Executing SNR query with params: {start_timestamp}, {end_timestamp}") + cursor.execute(snr_query, (start_timestamp, end_timestamp)) + snr = cursor.fetchall() + logging.debug(f"SNR query returned {len(snr)} records. First record: {snr[0] if snr else 'None'}") + + cursor.close() + + # If we have no data, return a more informative response + if not any([nodes_online, message_traffic, channel_util, battery_levels, temperature, snr]): + logging.warning("No data found for any metrics in the specified time range") + return jsonify({ + 'error': 'No data available for the specified time range', + 'time_range': { + 'start': start_time.isoformat(), + 'end': end_time.isoformat(), + 'selected': time_range + }, + 'table_counts': { + 'telemetry': telemetry_count, + 'text': text_count, + 'message_reception': reception_count + }, + 'data_ranges': { + 'telemetry': { + 'min': telemetry_time_range['min_time'].isoformat() if telemetry_time_range['min_time'] else None, + 'max': telemetry_time_range['max_time'].isoformat() if telemetry_time_range['max_time'] else None + }, + 'text': { + 'min': text_time_range['min_time'].isoformat() if text_time_range['min_time'] else None, + 'max': text_time_range['max_time'].isoformat() if text_time_range['max_time'] else None + }, + 'message_reception': { + 'min': reception_time_range['min_time'].isoformat() if reception_time_range['min_time'] else None, + 'max': reception_time_range['max_time'].isoformat() if reception_time_range['max_time'] else None + } + } + }) + + return jsonify({ + 'nodes_online': { + 'labels': [row['time_slot'] for row in nodes_online], + 'data': [row['node_count'] for row in nodes_online] + }, + 'message_traffic': { + 'labels': [row['time_slot'] for row in message_traffic], + 'data': [row['message_count'] for row in message_traffic] + }, + 'channel_util': { + 'labels': [row['time_slot'] for row in channel_util], + 'data': [row['avg_util'] for row in channel_util] + }, + 'battery_levels': { + 'labels': [row['time_slot'] for row in battery_levels], + 'data': [row['avg_battery'] for row in battery_levels] + }, + 'temperature': { + 'labels': [row['time_slot'] for row in temperature], + 'data': [row['avg_temp'] for row in temperature] + }, + 'snr': { + 'labels': [row['time_slot'] for row in snr], + 'data': [row['avg_snr'] for row in snr] + }, + 'time_range': { + 'start': start_time.isoformat(), + 'end': end_time.isoformat(), + 'selected': time_range, + 'data_ranges': { + 'telemetry': { + 'min': telemetry_time_range['min_time'].isoformat() if telemetry_time_range['min_time'] else None, + 'max': telemetry_time_range['max_time'].isoformat() if telemetry_time_range['max_time'] else None + }, + 'text': { + 'min': text_time_range['min_time'].isoformat() if text_time_range['min_time'] else None, + 'max': text_time_range['max_time'].isoformat() if text_time_range['max_time'] else None + }, + 'message_reception': { + 'min': reception_time_range['min_time'].isoformat() if reception_time_range['min_time'] else None, + 'max': reception_time_range['max_time'].isoformat() if reception_time_range['max_time'] else None + } + } + }, + 'warnings': warnings + }) + except Exception as e: + logging.error(f"Error fetching metrics data: {str(e)}") + return jsonify({ + 'error': f'Error fetching metrics data: {str(e)}' + }), 500 + +@app.route('/api/chattiest-nodes') +def get_chattiest_nodes(): + md = get_meshdata() + if not md: # Check if MeshData failed to initialize + abort(503, description="Database connection unavailable") + + # Get filter parameters from request + time_frame = request.args.get('time_frame', 'day') # day, week, month, year, all + message_type = request.args.get('message_type', 'all') # all, text, position, telemetry + + try: + cursor = md.db.cursor(dictionary=True) + + # Build the time frame condition + time_condition = "" + if time_frame == 'year': + time_condition = "WHERE ts_created >= DATE_SUB(NOW(), INTERVAL 1 YEAR)" + elif time_frame == 'month': + time_condition = "WHERE ts_created >= DATE_SUB(NOW(), INTERVAL 1 MONTH)" + elif time_frame == 'week': + time_condition = "WHERE ts_created >= DATE_SUB(NOW(), INTERVAL 7 DAY)" + elif time_frame == 'day': + time_condition = "WHERE ts_created >= DATE_SUB(NOW(), INTERVAL 24 HOUR)" + + # Build the message type query based on the selected type + if message_type == 'all': + message_query = """ + SELECT from_id as node_id, ts_created + FROM message_reception + {time_condition} + """.format(time_condition=time_condition) + elif message_type == 'text': + message_query = """ + SELECT from_id as node_id, ts_created + FROM text + {time_condition} + """.format(time_condition=time_condition) + elif message_type == 'position': + message_query = """ + SELECT id as node_id, ts_created + FROM positionlog + {time_condition} + """.format(time_condition=time_condition) + elif message_type == 'telemetry': + message_query = """ + SELECT id as node_id, ts_created + FROM telemetry + {time_condition} + """.format(time_condition=time_condition) + else: + return jsonify({ + 'error': f'Invalid message type: {message_type}' + }), 400 + + # Query to get the top 20 nodes by message count, including node names + query = """ + WITH messages AS ({message_query}) + SELECT + m.node_id as from_id, + n.long_name, + n.short_name, + COUNT(*) as message_count, + COUNT(DISTINCT DATE_FORMAT(m.ts_created, '%Y-%m-%d')) as active_days, + MIN(m.ts_created) as first_message, + MAX(m.ts_created) as last_message + FROM + messages m + LEFT JOIN + nodeinfo n ON m.node_id = n.id + GROUP BY + m.node_id, n.long_name + ORDER BY + message_count DESC + LIMIT 20 + """.format(message_query=message_query) + + cursor.execute(query) + chattiest_nodes = cursor.fetchall() + cursor.close() + + # Format the data for the response + formatted_nodes = [] + for node in chattiest_nodes: + # Convert node ID to hex format for the link + node_id_hex = utils.convert_node_id_from_int_to_hex(node['from_id']) + + formatted_nodes.append({ + 'node_id': node['from_id'], + 'node_id_hex': node_id_hex, + 'long_name': node['long_name'] or f"Node {node['from_id']}", # Fallback if no long name + 'short_name': node['short_name'] or f"Node {node['from_id']}", # Fallback if no short name + 'message_count': node['message_count'], + 'active_days': node['active_days'], + 'first_message': node['first_message'].isoformat() if node['first_message'] else None, + 'last_message': node['last_message'].isoformat() if node['last_message'] else None + }) + + return jsonify({ + 'chattiest_nodes': formatted_nodes, + 'filters': { + 'time_frame': time_frame, + 'message_type': message_type + } + }) + except Exception as e: + logging.error(f"Error fetching chattiest nodes: {str(e)}") + return jsonify({ + 'error': f'Error fetching chattiest nodes: {str(e)}' + }), 500 + def run(): # Enable Waitress logging config = configparser.ConfigParser() diff --git a/templates/layout.html.j2 b/templates/layout.html.j2 index 159e68ff..bf837591 100644 --- a/templates/layout.html.j2 +++ b/templates/layout.html.j2 @@ -36,7 +36,7 @@ @@ -219,36 +225,45 @@ Mesh Metrics
-

Top 20 Chattiest Nodes

+

Chattiest Nodes

+
- +
- + + - + + +
Rank# Node Role Messages Active Days First Message Last MessageChannels
Loading...
@@ -523,6 +538,7 @@ Mesh Metrics // Update metrics based on selected time range function updateMetrics() { const timeRange = document.getElementById('timeRangeFilter').value; + const channel = document.getElementById('channelFilter').value; // Show loading state document.querySelectorAll('.chart-container').forEach(container => { @@ -530,7 +546,7 @@ Mesh Metrics }); // Fetch data from API - fetch(`/api/metrics?time_range=${timeRange}`) + fetch(`/api/metrics?time_range=${timeRange}&channel=${channel}`) .then(response => { if (!response.ok) { throw new Error('Network response was not ok'); @@ -703,6 +719,7 @@ Mesh Metrics function updateChattiestNodes() { const timeFrame = document.getElementById('timeFrameFilter').value; const messageType = document.getElementById('messageTypeFilter').value; + const channel = document.getElementById('nodeChannelFilter').value; // Add loading indicator to the table without clearing it const tableBody = document.getElementById('chattiestNodesBody'); @@ -711,14 +728,14 @@ Mesh Metrics loadingRow.style.backgroundColor = 'rgba(255, 255, 255, 0.8)'; loadingRow.style.position = 'absolute'; loadingRow.style.width = '100%'; - loadingRow.innerHTML = 'Loading...'; + loadingRow.innerHTML = 'Loading...'; // Make the table container relative for absolute positioning of loading indicator tableBody.parentElement.style.position = 'relative'; tableBody.parentElement.appendChild(loadingRow); // Fetch data from API - fetch(`/api/chattiest-nodes?time_frame=${timeFrame}&message_type=${messageType}`) + fetch(`/api/chattiest-nodes?time_frame=${timeFrame}&message_type=${messageType}&channel=${channel}`) .then(response => { if (!response.ok) { throw new Error('Network response was not ok'); @@ -728,15 +745,15 @@ Mesh Metrics .then(data => { if (data.error) { console.error('Error fetching chattiest nodes:', data.error); - tableBody.innerHTML = 'Error: ' + data.error + ''; + tableBody.innerHTML = 'Error: ' + data.error + ''; return; } - // Only clear and update the table once we have new data + // Clear the table body tableBody.innerHTML = ''; if (!data.chattiest_nodes || data.chattiest_nodes.length === 0) { - tableBody.innerHTML = 'No data available'; + tableBody.innerHTML = 'No data available'; } else { data.chattiest_nodes.forEach((node, index) => { const row = document.createElement('tr'); @@ -745,6 +762,14 @@ Mesh Metrics const firstMessage = new Date(node.first_message).toLocaleString(); const lastMessage = new Date(node.last_message).toLocaleString(); + // Format channels + const channelsText = node.channels && node.channels.length > 0 + ? node.channels.map(ch => { + const channelName = ch.name || `Channel ${ch.id}`; + return `${channelName}`; + }).join('') + : '-'; + row.innerHTML = ` ${index + 1} @@ -759,6 +784,7 @@ Mesh Metrics ${node.active_days} ${firstMessage} ${lastMessage} + ${channelsText} `; tableBody.appendChild(row); @@ -767,7 +793,7 @@ Mesh Metrics }) .catch(error => { console.error('Error fetching chattiest nodes:', error); - tableBody.innerHTML = 'Error loading data'; + tableBody.innerHTML = 'Error loading data'; }) .finally(() => { // Remove loading indicator @@ -777,5 +803,18 @@ Mesh Metrics } }); } + + // Add this function to get channel colors + function getChannelColor(channelId) { + // Define channel colors based on the same logic used in utils.get_channel_color + const channelColors = { + 8: '#4CAF50', // Long Fast + 31: '#2196F3', // Medium Fast + 112: '#FF9800', // Short Fast + // Add more channels as needed + }; + + return channelColors[channelId] || '#9E9E9E'; // Default gray for unknown channels + } {% endblock %} \ No newline at end of file diff --git a/templates/telemetry.html.j2 b/templates/telemetry.html.j2 index a3b13dbe..2885236b 100644 --- a/templates/telemetry.html.j2 +++ b/templates/telemetry.html.j2 @@ -11,6 +11,7 @@ Timestamp Node + Ch Air Util TX @@ -58,6 +59,14 @@ UNK {% endif %} + + {% if item.channel is not none %} + + {{ utils.get_channel_name(item.channel, use_short_names=True) }} + + {% endif %} + {% if item.air_util_tx is not none %} {{ item.air_util_tx | round(2) }}% diff --git a/templates/traceroutes.html.j2 b/templates/traceroutes.html.j2 index 4befca95..8d05f994 100644 --- a/templates/traceroutes.html.j2 +++ b/templates/traceroutes.html.j2 @@ -15,6 +15,7 @@ To Route Hops + Ch {# Success #} Actions @@ -76,17 +77,16 @@ {% endif %}
{% endfor %} - {% if item.route_back %} - {# Destination node for forward path #} - ⇢ -
- {% if tnodeid in nodes %} - - {{ nodes[tnodeid].short_name }} - - {% else %} - UNK - {% endif %} + + {# Destination node for forward path #} + ⇢ +
+ {% if tnodeid in nodes %} + + {{ nodes[tnodeid].short_name }} + + {% else %} + UNK {% endif %}
@@ -156,6 +156,13 @@ {{ item.route|length }} {% endif %} + + {% if item.channel is not none %} + + {{ utils.get_channel_name(item.channel, use_short_names=True) }} + + {% endif %} + {# {% if item.success %} diff --git a/utils.py b/utils.py index 8fc70c36..c67c5480 100644 --- a/utils.py +++ b/utils.py @@ -13,7 +13,8 @@ from email.mime.text import MIMEText import configparser import logging -from meshtastic_support import Role +from meshtastic_support import Role, Channel, ShortChannel +import hashlib def distance_between_two_points(lat1, lon1, lat2, lon2): @@ -250,3 +251,83 @@ def send_email(recipient_email, subject, message): except Exception as e: logging.error(str(e)) + + +def get_channel_name(channel_value, use_short_names=False): + """Convert a channel number to its human-readable name.""" + if channel_value is None: + return "Unknown" + try: + if use_short_names: + channel_name = ShortChannel(channel_value).name + return channel_name + else: + channel_name = Channel(channel_value).name + # Keep the underscores but capitalize each word + words = channel_name.split('_') + formatted_words = [word.capitalize() for word in words] + return ''.join(formatted_words) + except (ValueError, TypeError): + return f"Unknown ({channel_value})" + + +def get_channel_color(channel_value): + """ + Generate a consistent, visually pleasing color for a channel. + + Args: + channel_value: The numeric channel value + + Returns: + A hex color code (e.g., "#FF5733") + """ + if channel_value is None: + return "#808080" # Gray for unknown channels + + # Define a set of visually pleasing colors for known channels + channel_colors = { + 8: "#4CAF50", # Green for LongFast + 31: "#2196F3", # Blue for MediumFast + 112: "#FF9800", # Orange for ShortFast + # Add more channels as they are discovered + } + + # If we have a predefined color for this channel, use it + if channel_value in channel_colors: + return channel_colors[channel_value] + + # For unknown channels, generate a consistent color based on the channel value + # Using a hash function to convert the channel value to a color + + # Create a hash of the channel value + hash_object = hashlib.md5(str(channel_value).encode()) + hex_dig = hash_object.hexdigest() + + # Use the first 6 characters of the hash as the color + # This ensures the same channel always gets the same color + color = f"#{hex_dig[:6]}" + + # Adjust the color to ensure it's visually pleasing + # Convert to RGB, adjust saturation and brightness, then back to hex + r = int(color[1:3], 16) + g = int(color[3:5], 16) + b = int(color[5:7], 16) + + # Increase saturation and brightness + max_val = max(r, g, b) + min_val = min(r, g, b) + + # If the color is too dark, lighten it + if max_val < 100: + r = min(255, r + 100) + g = min(255, g + 100) + b = min(255, b + 100) + + # If the color is too light, darken it slightly + if min_val > 200: + r = max(0, r - 50) + g = max(0, g - 50) + b = max(0, b - 50) + + # Convert back to hex + return f"#{r:02x}{g:02x}{b:02x}" From 1cae59c15fedeba744b8ecd75c9ddbf613c89cd0 Mon Sep 17 00:00:00 2001 From: agessaman Date: Tue, 22 Apr 2025 04:48:27 +0000 Subject: [PATCH 018/155] added arrow to the traceroute_map --- templates/traceroute_map.html.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/traceroute_map.html.j2 b/templates/traceroute_map.html.j2 index 407aa9a9..3da844f7 100644 --- a/templates/traceroute_map.html.j2 +++ b/templates/traceroute_map.html.j2 @@ -77,6 +77,7 @@ {# Destination node for forward path #}
+ ⇢
{% if traceroute.to_id_hex in nodes %} From 6194267b300501740d54fcd653d95e43f6164bdf Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 23 Apr 2025 02:25:34 +0000 Subject: [PATCH 019/155] fixed snr scaling issue, display issues in traceroutes --- meshdata.py | 24 +++++--- meshinfo_web.py | 8 +-- migrations/add_traceroute_snr.py | 100 +++++++++++++++++-------------- process_payload.py | 27 +++++++-- templates/layout.html.j2 | 8 +-- templates/traceroutes.html.j2 | 8 +-- 6 files changed, 106 insertions(+), 69 deletions(-) diff --git a/meshdata.py b/meshdata.py index c96ed032..91665a92 100644 --- a/meshdata.py +++ b/meshdata.py @@ -335,8 +335,8 @@ def get_traceroutes(self, page=1, per_page=25): "to_id": row[2], "route": [int(a) for a in row[3].split(";")] if row[3] else [], "route_back": [int(a) for a in row[4].split(";")] if row[4] else [], - "snr_towards": [float(s) for s in row[5].split(";")] if row[5] else [], - "snr_back": [float(s) for s in row[6].split(";")] if row[6] else [], + "snr_towards": [float(s)/4.0 for s in row[5].split(";")] if row[5] else [], + "snr_back": [float(s)/4.0 for s in row[6].split(";")] if row[6] else [], "success": row[7], "channel": row[8], "hop_limit": row[9], @@ -825,7 +825,8 @@ def store_traceroute(self, data): if "route" in payload: route = ";".join(str(r) for r in payload["route"]) if "snr_towards" in payload: - snr_towards = ";".join(str(s) for s in payload["snr_towards"]) + # Store raw SNR values directly + snr_towards = ";".join(str(float(s)) for s in payload["snr_towards"]) # Process return route and SNR route_back = None @@ -833,7 +834,8 @@ def store_traceroute(self, data): if "route_back" in payload: route_back = ";".join(str(r) for r in payload["route_back"]) if "snr_back" in payload: - snr_back = ";".join(str(s) for s in payload["snr_back"]) + # Store raw SNR values directly + snr_back = ";".join(str(float(s)) for s in payload["snr_back"]) # A traceroute is successful if we have either: # 1. A direct connection with SNR data in both directions @@ -883,7 +885,7 @@ def get_successful_traceroutes(self): t.to_id, t.route, t.route_back, - t.snr, + t.snr_towards, t.snr_back, t.ts_created, n1.short_name as from_name, @@ -891,8 +893,7 @@ def get_successful_traceroutes(self): FROM traceroute t JOIN nodeinfo n1 ON t.from_id = n1.id JOIN nodeinfo n2 ON t.to_id = n2.id - WHERE t.route_back IS NOT NULL - AND t.route_back != '' + WHERE t.success = TRUE ORDER BY t.ts_created DESC """ cur = self.db.cursor() @@ -905,6 +906,15 @@ def get_successful_traceroutes(self): # Convert timestamp to Unix timestamp if needed if isinstance(result['ts_created'], datetime.datetime): result['ts_created'] = result['ts_created'].timestamp() + # Parse route and SNR data + if result['route']: + result['route'] = [int(a) for a in result['route'].split(";")] + if result['route_back']: + result['route_back'] = [int(a) for a in result['route_back'].split(";")] + if result['snr_towards']: + result['snr_towards'] = [float(s) for s in result['snr_towards'].split(";")] + if result['snr_back']: + result['snr_back'] = [float(s) for s in result['snr_back'].split(";")] results.append(result) cur.close() diff --git a/meshinfo_web.py b/meshinfo_web.py index f15984ee..68638ca1 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -382,15 +382,15 @@ def traceroute_map(): if traceroute_data['route_back']: route_back = [int(hop) for hop in traceroute_data['route_back'].split(';')] - # Format the forward SNR values + # Format the forward SNR values and scale by dividing by 4 snr_towards = [] if traceroute_data['snr_towards']: - snr_towards = [float(s) for s in traceroute_data['snr_towards'].split(';')] + snr_towards = [float(s)/4.0 for s in traceroute_data['snr_towards'].split(';')] - # Format the return SNR values + # Format the return SNR values and scale by dividing by 4 snr_back = [] if traceroute_data['snr_back']: - snr_back = [float(s) for s in traceroute_data['snr_back'].split(';')] + snr_back = [float(s)/4.0 for s in traceroute_data['snr_back'].split(';')] # Create a clean traceroute object for the template traceroute = { diff --git a/migrations/add_traceroute_snr.py b/migrations/add_traceroute_snr.py index beebef1a..fdfce62f 100644 --- a/migrations/add_traceroute_snr.py +++ b/migrations/add_traceroute_snr.py @@ -2,7 +2,7 @@ def migrate(db): """ - Improve SNR storage in traceroute table + Update traceroute table to store separate SNR values for forward and return paths """ try: cursor = db.cursor() @@ -19,19 +19,26 @@ def migrate(db): logging.info("Creating traceroute table...") cursor.execute(""" CREATE TABLE traceroute ( + traceroute_id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, from_id INT UNSIGNED NOT NULL, to_id INT UNSIGNED NOT NULL, - route VARCHAR(255), - snr TEXT COMMENT 'Semicolon-separated SNR values, stored as integers (actual_value * 4)', + route TEXT, + route_back TEXT, + snr_towards TEXT COMMENT 'Semicolon-separated SNR values for forward path', + snr_back TEXT COMMENT 'Semicolon-separated SNR values for return path', + success BOOLEAN DEFAULT FALSE, + channel TINYINT UNSIGNED, + hop_limit TINYINT UNSIGNED, ts_created TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - INDEX idx_traceroute_nodes (from_id, to_id) + INDEX idx_traceroute_nodes (from_id, to_id), + INDEX idx_traceroute_time (ts_created) ) """) db.commit() logging.info("Created traceroute table successfully") return - # Check if SNR column exists + # Check if old SNR column exists cursor.execute(""" SELECT COLUMN_TYPE FROM information_schema.COLUMNS @@ -40,75 +47,76 @@ def migrate(db): """) result = cursor.fetchone() - if not result: - # SNR column doesn't exist, add it - logging.info("Adding SNR column to traceroute table...") - cursor.execute(""" - ALTER TABLE traceroute - ADD COLUMN snr TEXT COMMENT 'Semicolon-separated SNR values, stored as integers (actual_value * 4)' - """) - db.commit() - logging.info("Added SNR column successfully") - return - - current_type = result[0] - - if current_type.upper() == 'VARCHAR(255)': - logging.info("Converting traceroute SNR column to more efficient format...") + if result: + # Old SNR column exists, need to migrate to new format + logging.info("Migrating from single SNR column to separate forward/return SNR columns...") - # Create temporary column with same type to preserve data + # Add new columns if they don't exist cursor.execute(""" ALTER TABLE traceroute - ADD COLUMN snr_temp VARCHAR(255) + ADD COLUMN IF NOT EXISTS snr_towards TEXT COMMENT 'Semicolon-separated SNR values for forward path', + ADD COLUMN IF NOT EXISTS snr_back TEXT COMMENT 'Semicolon-separated SNR values for return path' """) - # Copy existing data + # Copy existing SNR data to snr_towards (since historically it was forward path only) cursor.execute(""" UPDATE traceroute - SET snr_temp = snr + SET snr_towards = snr WHERE snr IS NOT NULL """) - # Drop old column and create new one with better type - cursor.execute(""" - ALTER TABLE traceroute - DROP COLUMN snr, - ADD COLUMN snr TEXT COMMENT 'Semicolon-separated SNR values, stored as integers (actual_value * 4)' - """) - - # Copy data back - cursor.execute(""" - UPDATE traceroute - SET snr = snr_temp - WHERE snr_temp IS NOT NULL - """) - - # Drop temporary column + # Drop old column cursor.execute(""" ALTER TABLE traceroute - DROP COLUMN snr_temp + DROP COLUMN snr """) db.commit() - logging.info("Converted SNR column successfully") + logging.info("Migrated SNR columns successfully") - # Add index if it doesn't exist + # Add other necessary columns if they don't exist + cursor.execute(""" + ALTER TABLE traceroute + ADD COLUMN IF NOT EXISTS traceroute_id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST, + ADD COLUMN IF NOT EXISTS route_back TEXT AFTER route, + ADD COLUMN IF NOT EXISTS success BOOLEAN DEFAULT FALSE AFTER snr_back, + ADD COLUMN IF NOT EXISTS channel TINYINT UNSIGNED AFTER success, + ADD COLUMN IF NOT EXISTS hop_limit TINYINT UNSIGNED AFTER channel + """) + + # Add indices if they don't exist cursor.execute(""" SELECT COUNT(*) FROM information_schema.STATISTICS WHERE TABLE_NAME = 'traceroute' AND INDEX_NAME = 'idx_traceroute_nodes' """) - has_index = cursor.fetchone()[0] > 0 + has_node_index = cursor.fetchone()[0] > 0 - if not has_index: + if not has_node_index: logging.info("Adding index on from_id, to_id...") cursor.execute(""" ALTER TABLE traceroute ADD INDEX idx_traceroute_nodes (from_id, to_id) """) - db.commit() - logging.info("Added index successfully") + + cursor.execute(""" + SELECT COUNT(*) + FROM information_schema.STATISTICS + WHERE TABLE_NAME = 'traceroute' + AND INDEX_NAME = 'idx_traceroute_time' + """) + has_time_index = cursor.fetchone()[0] > 0 + + if not has_time_index: + logging.info("Adding index on ts_created...") + cursor.execute(""" + ALTER TABLE traceroute + ADD INDEX idx_traceroute_time (ts_created) + """) + + db.commit() + logging.info("Migration completed successfully") except Exception as e: logging.error(f"Error performing traceroute SNR migration: {e}") diff --git a/process_payload.py b/process_payload.py index 248e44f1..0747355a 100644 --- a/process_payload.py +++ b/process_payload.py @@ -124,11 +124,30 @@ def get_data(msg): ) elif portnum == portnums_pb2.TRACEROUTE_APP: j["type"] = "traceroute" + + # Log the raw payload before parsing + logging.info(f"Raw traceroute payload before parsing: {msg.decoded.payload[:100]}...") + route_discovery = mesh_pb2.RouteDiscovery().FromString(msg.decoded.payload) + + # Log the entire route_discovery object + logging.info(f"Full RouteDiscovery object: {route_discovery}") + + # Log the raw protobuf object before conversion + #logging.info("Raw RouteDiscovery protobuf values:") + #if hasattr(route_discovery, 'snr_towards'): + # logging.info(f" snr_towards (raw protobuf): {route_discovery.snr_towards}") + #if hasattr(route_discovery, 'snr_back'): + # logging.info(f" snr_back (raw protobuf): {route_discovery.snr_back}") + route_data = to_json(route_discovery) - # Log the raw route data for debugging - logging.debug(f"Raw traceroute data: {json.dumps(route_data, indent=2)}") + # Log SNR values + #logging.info("SNR values from protobuf:") + #if "snr_towards" in route_data: + # logging.info(f" snr_towards: {route_data['snr_towards']}") + #if "snr_back" in route_data: + # logging.info(f" snr_back: {route_data['snr_back']}") # Ensure we have all required fields with proper defaults route_data.setdefault("route", []) @@ -146,8 +165,8 @@ def get_data(msg): j["decoded"]["json_payload"] = route_data - # Log the processed payload - logging.debug(f"Processed traceroute payload: {json.dumps(j['decoded']['json_payload'], indent=2)}") + # Log the final data that will be stored + #logging.info(f"Final traceroute data to be stored: {json.dumps(j['decoded']['json_payload'], indent=2)}") elif portnum == portnums_pb2.POSITION_APP: j["type"] = "position" diff --git a/templates/layout.html.j2 b/templates/layout.html.j2 index bf837591..90444a4e 100644 --- a/templates/layout.html.j2 +++ b/templates/layout.html.j2 @@ -4,13 +4,13 @@ {% macro snr_badge(snr) %} {% if snr is not none %} {# Check if snr exists and is not None #} {% if snr > 0 %} - {{ "%.1f"|format(snr) }} dB + {{ "%.1f"|format(snr) }} dB {% elif snr > -5 %} - {{ "%.1f"|format(snr) }} dB + {{ "%.1f"|format(snr) }} dB {% elif snr > -10 %} - {{ "%.1f"|format(snr) }} dB + {{ "%.1f"|format(snr) }} dB {% else %} - {{ "%.1f"|format(snr) }} dB + {{ "%.1f"|format(snr) }} dB {% endif %} {% endif %} {% endmacro %} diff --git a/templates/traceroutes.html.j2 b/templates/traceroutes.html.j2 index 8d05f994..f0a6c9f5 100644 --- a/templates/traceroutes.html.j2 +++ b/templates/traceroutes.html.j2 @@ -52,7 +52,7 @@ {{ nodes[fnodeid].short_name }} {% if item.route|length == 0 and item.snr_towards %} - {{ snr_badge(item.snr_towards[0]) }} + {{ snr_badge(item.snr_towards[0]) }} {% endif %} {% else %} UNK @@ -73,7 +73,7 @@ UNK {% endif %} {% if item.snr_towards and loop.index0 < item.snr_towards|length %} - {{ snr_badge(item.snr_towards[loop.index0]) }} + {{ snr_badge(item.snr_towards[loop.index0]) }} {% endif %}
{% endfor %} @@ -104,7 +104,7 @@ {{ nodes[tnodeid].short_name }} {% if item.route_back|length == 0 and item.snr_back %} - {{ snr_badge(item.snr_back[0]) }} + {{ snr_badge(item.snr_back[0]) }} {% endif %} {% else %} UNK @@ -125,7 +125,7 @@ UNK {% endif %} {% if item.snr_back and loop.index0 < item.snr_back|length %} - {{ snr_badge(item.snr_back[loop.index0]) }} + {{ snr_badge(item.snr_back[loop.index0]) }} {% endif %}
{% endfor %} From 8ae24b1834c291469ecdf818a11971a57d6a8636 Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 23 Apr 2025 02:49:17 +0000 Subject: [PATCH 020/155] upgraded traceroute UI to use chat2 badges --- meshinfo_web.py | 10 + process_payload.py | 22 +- templates/traceroute_map.html.j2 | 351 ++++++++++++++++++++----------- templates/traceroutes.html.j2 | 296 ++++++++++++++++++++------ www/css/meshinfo.css | 5 +- 5 files changed, 471 insertions(+), 213 deletions(-) diff --git a/meshinfo_web.py b/meshinfo_web.py index 68638ca1..ce099c05 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -70,6 +70,14 @@ app.jinja_env.globals.update(min=min) app.jinja_env.globals.update(max=max) +# Add template filters +@app.template_filter('safe_hw_model') +def safe_hw_model(value): + try: + return meshtastic_support.HardwareModel(value).name.replace('_', ' ') + except (ValueError, AttributeError): + return f"Unknown ({value})" + config = configparser.ConfigParser() config.read("config.ini") @@ -416,6 +424,7 @@ def traceroute_map(): nodes=nodes, traceroute=traceroute, utils=utils, + meshtastic_support=meshtastic_support, datetime=datetime.datetime, timestamp=datetime.datetime.now() ) @@ -703,6 +712,7 @@ def traceroutes(): traceroutes=traceroute_data['items'], pagination=pagination, utils=utils, + meshtastic_support=meshtastic_support, datetime=datetime.datetime, timestamp=datetime.datetime.now(), ) diff --git a/process_payload.py b/process_payload.py index 0747355a..ceae72e0 100644 --- a/process_payload.py +++ b/process_payload.py @@ -124,31 +124,11 @@ def get_data(msg): ) elif portnum == portnums_pb2.TRACEROUTE_APP: j["type"] = "traceroute" - - # Log the raw payload before parsing - logging.info(f"Raw traceroute payload before parsing: {msg.decoded.payload[:100]}...") - + route_discovery = mesh_pb2.RouteDiscovery().FromString(msg.decoded.payload) - # Log the entire route_discovery object - logging.info(f"Full RouteDiscovery object: {route_discovery}") - - # Log the raw protobuf object before conversion - #logging.info("Raw RouteDiscovery protobuf values:") - #if hasattr(route_discovery, 'snr_towards'): - # logging.info(f" snr_towards (raw protobuf): {route_discovery.snr_towards}") - #if hasattr(route_discovery, 'snr_back'): - # logging.info(f" snr_back (raw protobuf): {route_discovery.snr_back}") - route_data = to_json(route_discovery) - # Log SNR values - #logging.info("SNR values from protobuf:") - #if "snr_towards" in route_data: - # logging.info(f" snr_towards: {route_data['snr_towards']}") - #if "snr_back" in route_data: - # logging.info(f" snr_back: {route_data['snr_back']}") - # Ensure we have all required fields with proper defaults route_data.setdefault("route", []) route_data.setdefault("route_back", []) diff --git a/templates/traceroute_map.html.j2 b/templates/traceroute_map.html.j2 index 3da844f7..a6f7622b 100644 --- a/templates/traceroute_map.html.j2 +++ b/templates/traceroute_map.html.j2 @@ -4,6 +4,47 @@ {% block head %} + + {% endblock %} {% block content %} @@ -15,146 +56,214 @@
Traceroute Details
-

- From: - {% if traceroute.from_id_hex in nodes %} - {{ nodes[traceroute.from_id_hex].long_name }} ({{ nodes[traceroute.from_id_hex].short_name }}) - {% else %} - Unknown - {% endif %} -
- To: - {% if traceroute.to_id_hex in nodes %} - {{ nodes[traceroute.to_id_hex].long_name }} ({{ nodes[traceroute.to_id_hex].short_name }}) - {% else %} - Unknown - {% endif %} -
- Time: {{ format_timestamp(traceroute.ts_created) }}
- Forward Route: -

- {# Source node #} -
-
- {% if traceroute.from_id_hex in nodes %} - - {{ nodes[traceroute.from_id_hex].short_name }} - - {% if traceroute.route|length == 0 and traceroute.snr_towards %} - {{ snr_badge(traceroute.snr_towards[0]) }} - {% endif %} - {% else %} - UNK - {% endif %} -
- ⇢ -
+
+
+ Outbound: +
+ {# Source node #} + {% if traceroute.from_id_hex in nodes %} + + {% if traceroute.route|length == 0 and traceroute.snr_towards %}{% endif %} + {{ nodes[traceroute.from_id_hex].short_name }} + + {% else %} + UNK + {% endif %} - {# Forward hop nodes - only if there are hops #} - {% if traceroute.route|length > 0 %} + {# Forward hops #} {% for hop in traceroute.route %} -
- {% set hnodeid = utils.convert_node_id_from_int_to_hex(hop) %} - {% set hnode = nodes[hnodeid] if hnodeid in nodes else None %} -
- {% if hnode %} - - {{ hnode.short_name }} - - {% else %} - UNK - {% endif %} - {% if traceroute.snr_towards and loop.index0 < traceroute.snr_towards|length %} - {{ snr_badge(traceroute.snr_towards[loop.index0]) }} - {% endif %} -
- {% if not loop.last %} - ⇢ - {% endif %} -
- {% endfor %} - {% endif %} - - {# Destination node for forward path #} -
- ⇢ -
- {% if traceroute.to_id_hex in nodes %} - - {{ nodes[traceroute.to_id_hex].short_name }} + ⇢ + {% set hnodeid = utils.convert_node_id_from_int_to_hex(hop) %} + {% set hnode = nodes[hnodeid] if hnodeid in nodes else None %} + {% if hnode %} + + {% if traceroute.snr_towards and loop.index0 < traceroute.snr_towards|length %}{% endif %} + {{ hnode.short_name }} {% else %} - UNK + UNK {% endif %} -
+ {% endfor %} + + {# Destination node for forward path #} + ⇢ + {% if traceroute.to_id_hex in nodes %} + + {{ nodes[traceroute.to_id_hex].short_name }} + + {% else %} + UNK + {% endif %}
{% if traceroute.route_back %} - Return Route: -
- {# Source node for return path #} -
-
- {% if traceroute.to_id_hex in nodes %} - - {{ nodes[traceroute.to_id_hex].short_name }} - - {% if traceroute.route_back|length == 0 and traceroute.snr_back %} - {{ snr_badge(traceroute.snr_back[0]) }} - {% endif %} - {% else %} - UNK - {% endif %} -
- {% if traceroute.route_back|length > 0 %} - ⇢ +
+ Return: +
+ {# Return source (destination node) #} + {% if traceroute.to_id_hex in nodes %} + + {% if traceroute.route_back|length == 0 and traceroute.snr_back %}{% endif %} + {{ nodes[traceroute.to_id_hex].short_name }} + + {% else %} + UNK {% endif %} -
- {# Return hop nodes #} - {% if traceroute.route_back|length > 0 %} + {# Return hops #} {% for hop in traceroute.route_back %} -
- {% set hnodeid = utils.convert_node_id_from_int_to_hex(hop) %} - {% set hnode = nodes[hnodeid] if hnodeid in nodes else None %} -
- {% if hnode %} - - {{ hnode.short_name }} - - {% else %} - UNK - {% endif %} - {% if traceroute.snr_back and loop.index0 < traceroute.snr_back|length %} - {{ snr_badge(traceroute.snr_back[loop.index0]) }} - {% endif %} -
- {% if not loop.last %} - ⇢ - {% endif %} -
- {% endfor %} - {% endif %} - - {# Final destination (original source) for return path #} -
-
- {% if traceroute.from_id_hex in nodes %} - ⇢ - - {{ nodes[traceroute.from_id_hex].short_name }} + ⇢ + {% set hnodeid = utils.convert_node_id_from_int_to_hex(hop) %} + {% set hnode = nodes[hnodeid] if hnodeid in nodes else None %} + {% if hnode %} + + {% if traceroute.snr_back and loop.index0 < traceroute.snr_back|length %}{% endif %} + {{ hnode.short_name }} {% else %} - UNK + UNK {% endif %} -
+ {% endfor %} + + {# Return destination (source node) #} + ⇢ + {% if traceroute.from_id_hex in nodes %} + + {{ nodes[traceroute.from_id_hex].short_name }} + + {% else %} + UNK + {% endif %}
{% endif %} - Total Hops: {{ traceroute.route|length }} forward{% if traceroute.route_back %}, {{ traceroute.route_back|length }} return{% endif %}
-

+
+ Time: {{ format_timestamp(traceroute.ts_created) }}
+ Total Hops: {{ traceroute.route|length }} forward{% if traceroute.route_back %}, {{ traceroute.route_back|length }} return{% endif %} +
+
diff --git a/templates/traceroutes.html.j2 b/templates/traceroutes.html.j2 index f0a6c9f5..b96ad5a7 100644 --- a/templates/traceroutes.html.j2 +++ b/templates/traceroutes.html.j2 @@ -3,6 +3,49 @@ {% block title %}Traceroutes | MeshInfo{% endblock %} +{% block head %} + +{% endblock %} + {% block content %}
{{ this_page.title() }}
@@ -46,49 +89,105 @@ Outbound:
{# Source node #} -
- {% if fnodeid in nodes %} - - {{ nodes[fnodeid].short_name }} - - {% if item.route|length == 0 and item.snr_towards %} - {{ snr_badge(item.snr_towards[0]) }} - {% endif %} - {% else %} - UNK - {% endif %} -
+ {% if fnodeid in nodes %} + + {% if item.route|length == 0 and item.snr_towards %}{% endif %} + {{ nodes[fnodeid].short_name }} + + {% else %} + + {{ "UNK" }} + + {% endif %} {# Forward hops #} {% for hop in item.route %} ⇢ -
- {% set hnodeid = utils.convert_node_id_from_int_to_hex(hop) %} - {% set hnode = nodes[hnodeid] if hnodeid in nodes else None %} - {% if hnode %} - - {{ hnode.short_name }} - - {% else %} - UNK - {% endif %} - {% if item.snr_towards and loop.index0 < item.snr_towards|length %} - {{ snr_badge(item.snr_towards[loop.index0]) }} - {% endif %} -
+ {% set hnodeid = utils.convert_node_id_from_int_to_hex(hop) %} + {% set hnode = nodes[hnodeid] if hnodeid in nodes else None %} + {% if hnode %} + + {% if item.snr_towards and loop.index0 < item.snr_towards|length %}{% endif %} + {{ hnode.short_name }} + + {% else %} + + {{ "UNK" }} + + {% endif %} {% endfor %} {# Destination node for forward path #} ⇢ -
- {% if tnodeid in nodes %} - - {{ nodes[tnodeid].short_name }} - - {% else %} - UNK - {% endif %} -
+ {% if tnodeid in nodes %} + + {{ nodes[tnodeid].short_name }} + + {% else %} + + {{ "UNK" }} + + {% endif %}
@@ -98,49 +197,106 @@ Return:
{# Return source (destination node) #} -
- {% if tnodeid in nodes %} - - {{ nodes[tnodeid].short_name }} - - {% if item.route_back|length == 0 and item.snr_back %} - {{ snr_badge(item.snr_back[0]) }} - {% endif %} - {% else %} - UNK - {% endif %} -
+ {% if tnodeid in nodes %} + + {% if item.route_back|length == 0 and item.snr_back %}{% endif %} + {{ nodes[tnodeid].short_name }} + + {% else %} + + {{ "UNK" }} + + {% endif %} {# Return hops #} {% for hop in item.route_back %} ⇢ -
- {% set hnodeid = utils.convert_node_id_from_int_to_hex(hop) %} - {% set hnode = nodes[hnodeid] if hnodeid in nodes else None %} - {% if hnode %} - - {{ hnode.short_name }} - - {% else %} - UNK - {% endif %} - {% if item.snr_back and loop.index0 < item.snr_back|length %} - {{ snr_badge(item.snr_back[loop.index0]) }} - {% endif %} -
+ {% set hnodeid = utils.convert_node_id_from_int_to_hex(hop) %} + {% set hnode = nodes[hnodeid] if hnodeid in nodes else None %} + {% if hnode %} + + {% if item.snr_back and loop.index0 < item.snr_back|length %}{% endif %} + {{ hnode.short_name }} + + {% else %} + + {{ "UNK" }} + + {% endif %} {% endfor %} {# Return destination (source node) #} ⇢ -
- {% if fnodeid in nodes %} - - {{ nodes[fnodeid].short_name }} - - {% else %} - UNK - {% endif %} -
+ {% if fnodeid in nodes %} + + {% if item.route_back|length == 0 and item.snr_back %}{% endif %} + {{ nodes[fnodeid].short_name }} + + {% else %} + + {{ "UNK" }} + + {% endif %}
{% endif %} diff --git a/www/css/meshinfo.css b/www/css/meshinfo.css index 2ba55500..844f94ff 100644 --- a/www/css/meshinfo.css +++ b/www/css/meshinfo.css @@ -211,13 +211,16 @@ nav { border-radius: 12px; padding: 4px 10px; font-size: 0.85em; - gap: 6px; text-decoration: none; color: inherit; transition: background-color 0.2s; margin: 2px; } +.reception-badge .snr-indicator { + margin-right: 6px; +} + .reception-badge:hover { background-color: #e9ecef; text-decoration: none; From f5ef194c22c1059ae548c83f469e0e1075244938 Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 23 Apr 2025 03:22:21 +0000 Subject: [PATCH 021/155] fixed escaping of node names in json --- templates/message_map.html.j2 | 11 +- templates/traceroute_map.html.j2 | 566 ++++++++++++++++--------------- 2 files changed, 292 insertions(+), 285 deletions(-) diff --git a/templates/message_map.html.j2 b/templates/message_map.html.j2 index a41199e2..bb791c39 100644 --- a/templates/message_map.html.j2 +++ b/templates/message_map.html.j2 @@ -36,7 +36,7 @@
-
+
LEGEND
Direct Reception @@ -429,17 +429,12 @@ height: 70vh; width: 100%; } - #legend { - position: absolute; - bottom: 20px; - right: 20px; - z-index: 1000; - } + .ol-popup { position: absolute; background-color: white; box-shadow: 0 1px 4px rgba(0,0,0,0.2); - padding: 0; // Removed padding as it's now in the content div + padding: 0; border-radius: 10px; border: 1px solid #cccccc; bottom: 12px; diff --git a/templates/traceroute_map.html.j2 b/templates/traceroute_map.html.j2 index a6f7622b..93c0d25b 100644 --- a/templates/traceroute_map.html.j2 +++ b/templates/traceroute_map.html.j2 @@ -426,293 +426,295 @@ // Add source node (from_id) {% set from_id = traceroute.from_id_hex %} - {% if from_id in nodes and nodes[from_id].position and nodes[from_id].position.longitude_i is not none and nodes[from_id].position.latitude_i is not none %} var source = new ol.Feature({ - geometry: new ol.geom.Point(ol.proj.fromLonLat([ - {{ nodes[from_id].position.longitude_i / 10000000 }}, - {{ nodes[from_id].position.latitude_i / 10000000 }} - ])), - node: { - id: '{{ from_id }}', - name: '{{ nodes[from_id].short_name }}', - longName: '{{ nodes[from_id].long_name }}', - type: 'source', - lat: {{ nodes[from_id].position.latitude_i / 10000000 }}, - lon: {{ nodes[from_id].position.longitude_i / 10000000 }} - } - }); - source.setStyle(sourceStyle); - features.push(source); - - // Initialize points tracking - var allpoints = []; - var lastKnownPoint = null; - var unknownSequence = []; + {% if from_id in nodes and nodes[from_id].position and nodes[from_id].position.longitude_i is not none and nodes[from_id].position.latitude_i is not none %} + var source = new ol.Feature({ + geometry: new ol.geom.Point(ol.proj.fromLonLat([ + {{ nodes[from_id].position.longitude_i / 10000000 }}, + {{ nodes[from_id].position.latitude_i / 10000000 }} + ])), + node: { + id: '{{ from_id }}', + name: {{ nodes[from_id].short_name|tojson }}, + longName: {{ nodes[from_id].long_name|tojson }}, + type: 'source', + lat: {{ nodes[from_id].position.latitude_i / 10000000 }}, + lon: {{ nodes[from_id].position.longitude_i / 10000000 }} + } + }); + source.setStyle(sourceStyle); + features.push(source); + + // Initialize points tracking + var allpoints = []; + var lastKnownPoint = null; + var unknownSequence = []; - // Start with source node - allpoints.push([ - {{ nodes[from_id].position.longitude_i / 10000000 }}, - {{ nodes[from_id].position.latitude_i / 10000000 }} - ]); - lastKnownPoint = [ - {{ nodes[from_id].position.longitude_i / 10000000 }}, - {{ nodes[from_id].position.latitude_i / 10000000 }} - ]; + // Start with source node + allpoints.push([ + {{ nodes[from_id].position.longitude_i / 10000000 }}, + {{ nodes[from_id].position.latitude_i / 10000000 }} + ]); + lastKnownPoint = [ + {{ nodes[from_id].position.longitude_i / 10000000 }}, + {{ nodes[from_id].position.latitude_i / 10000000 }} + ]; - // Add all forward hops in the route - {% for hop in traceroute.route %} - {% set hop_id = utils.convert_node_id_from_int_to_hex(hop) %} - {% if hop_id in nodes and nodes[hop_id].position and nodes[hop_id].position.longitude_i is not none and nodes[hop_id].position.latitude_i is not none %} // If we had unknown nodes before this known node, place them now - if (unknownSequence.length > 0) { - var nextKnownPoint = [ - {{ nodes[hop_id].position.longitude_i / 10000000 }}, - {{ nodes[hop_id].position.latitude_i / 10000000 }} - ]; - - // Calculate positions for all unknown nodes in sequence - for (var i = 0; i < unknownSequence.length; i++) { - var fraction = (i + 1) / (unknownSequence.length + 1); - var midPoint = [ - lastKnownPoint[0] + (nextKnownPoint[0] - lastKnownPoint[0]) * fraction, - lastKnownPoint[1] + (nextKnownPoint[1] - lastKnownPoint[1]) * fraction - ]; - - // Create unknown node feature - var unknownNode = new ol.Feature({ - geometry: new ol.geom.Point(ol.proj.fromLonLat(midPoint)), - node: { - ...unknownSequence[i], - direction: 'forward' - } - }); - unknownNode.setStyle(unknownStyle); - features.push(unknownNode); - allpoints.push(midPoint); - } - - // Clear the sequence - unknownSequence = []; - } + // Add all forward hops in the route + {% for hop in traceroute.route %} + {% set hop_id = utils.convert_node_id_from_int_to_hex(hop) %} + {% if hop_id in nodes and nodes[hop_id].position and nodes[hop_id].position.longitude_i is not none and nodes[hop_id].position.latitude_i is not none %} + // If we had unknown nodes before this known node, place them now + if (unknownSequence.length > 0) { + var nextKnownPoint = [ + {{ nodes[hop_id].position.longitude_i / 10000000 }}, + {{ nodes[hop_id].position.latitude_i / 10000000 }} + ]; + + // Calculate positions for all unknown nodes in sequence + for (var i = 0; i < unknownSequence.length; i++) { + var fraction = (i + 1) / (unknownSequence.length + 1); + var midPoint = [ + lastKnownPoint[0] + (nextKnownPoint[0] - lastKnownPoint[0]) * fraction, + lastKnownPoint[1] + (nextKnownPoint[1] - lastKnownPoint[1]) * fraction + ]; + + // Create unknown node feature + var unknownNode = new ol.Feature({ + geometry: new ol.geom.Point(ol.proj.fromLonLat(midPoint)), + node: { + ...unknownSequence[i], + direction: 'forward' + } + }); + unknownNode.setStyle(unknownStyle); + features.push(unknownNode); + allpoints.push(midPoint); + } + + // Clear the sequence + unknownSequence = []; + } - // Add the known forward hop node - var hopNode = new ol.Feature({ - geometry: new ol.geom.Point(ol.proj.fromLonLat([ - {{ nodes[hop_id].position.longitude_i / 10000000 }}, - {{ nodes[hop_id].position.latitude_i / 10000000 }} - ])), - node: { - id: '{{ hop_id }}', - name: '{{ nodes[hop_id].short_name }}', - longName: '{{ nodes[hop_id].long_name }}', - snr: {{ traceroute.snr_towards[loop.index0] if traceroute.snr_towards and loop.index0 < traceroute.snr_towards|length else 'null' }}, - type: 'hop', - direction: 'forward', - hopNumber: {{ loop.index }}, - lat: {{ nodes[hop_id].position.latitude_i / 10000000 }}, - lon: {{ nodes[hop_id].position.longitude_i / 10000000 }} - } - }); - hopNode.setStyle(createHopStyle({{ loop.index }}, false)); // false for forward hop - features.push(hopNode); + // Add the known forward hop node + var hopNode = new ol.Feature({ + geometry: new ol.geom.Point(ol.proj.fromLonLat([ + {{ nodes[hop_id].position.longitude_i / 10000000 }}, + {{ nodes[hop_id].position.latitude_i / 10000000 }} + ])), + node: { + id: '{{ hop_id }}', + name: {{ nodes[hop_id].short_name|tojson }}, + longName: {{ nodes[hop_id].long_name|tojson }}, + snr: {{ traceroute.snr_towards[loop.index0] if traceroute.snr_towards and loop.index0 < traceroute.snr_towards|length else 'null' }}, + type: 'hop', + direction: 'forward', + hopNumber: {{ loop.index }}, + lat: {{ nodes[hop_id].position.latitude_i / 10000000 }}, + lon: {{ nodes[hop_id].position.longitude_i / 10000000 }} + } + }); + hopNode.setStyle(createHopStyle({{ loop.index }}, false)); // false for forward hop + features.push(hopNode); - // Add the point to allpoints and update lastKnownPoint - var currentPoint = [ - {{ nodes[hop_id].position.longitude_i / 10000000 }}, - {{ nodes[hop_id].position.latitude_i / 10000000 }} - ]; - allpoints.push(currentPoint); - lastKnownPoint = currentPoint; - {% else %} - // Add to sequence of unknown nodes - unknownSequence.push({ - id: '{{ hop_id }}', - name: '{{ nodes[hop_id].short_name if hop_id in nodes else "Unknown" }}', - longName: '{{ nodes[hop_id].long_name if hop_id in nodes else "Unknown Node" }}', - type: 'unknown', - direction: 'forward', - hopNumber: {{ loop.index }}, - snr: {{ traceroute.snr_towards[loop.index0] if traceroute.snr_towards and loop.index0 < traceroute.snr_towards|length else 'null' }} - }); - {% endif %} - {% endfor %} - - // Add destination node - {% set to_id = traceroute.to_id_hex %} - {% if to_id in nodes and nodes[to_id].position and nodes[to_id].position.longitude_i is not none and nodes[to_id].position.latitude_i is not none %} - // Handle any remaining unknown nodes before destination - if (unknownSequence.length > 0) { - var nextKnownPoint = [ - {{ nodes[to_id].position.longitude_i / 10000000 }}, - {{ nodes[to_id].position.latitude_i / 10000000 }} - ]; - - // Calculate positions for all unknown nodes in sequence - for (var i = 0; i < unknownSequence.length; i++) { - var fraction = (i + 1) / (unknownSequence.length + 1); - var midPoint = [ - lastKnownPoint[0] + (nextKnownPoint[0] - lastKnownPoint[0]) * fraction, - lastKnownPoint[1] + (nextKnownPoint[1] - lastKnownPoint[1]) * fraction - ]; - - // Create unknown node feature - var unknownNode = new ol.Feature({ - geometry: new ol.geom.Point(ol.proj.fromLonLat(midPoint)), - node: unknownSequence[i] - }); - unknownNode.setStyle(unknownStyle); - features.push(unknownNode); - allpoints.push(midPoint); - } - } + // Add the point to allpoints and update lastKnownPoint + var currentPoint = [ + {{ nodes[hop_id].position.longitude_i / 10000000 }}, + {{ nodes[hop_id].position.latitude_i / 10000000 }} + ]; + allpoints.push(currentPoint); + lastKnownPoint = currentPoint; + {% else %} + // Add to sequence of unknown nodes + unknownSequence.push({ + id: '{{ hop_id }}', + name: {{ nodes[hop_id].short_name|tojson if hop_id in nodes else '"Unknown"' }}, + longName: {{ nodes[hop_id].long_name|tojson if hop_id in nodes else '"Unknown Node"' }}, + type: 'unknown', + direction: 'forward', + hopNumber: {{ loop.index }}, + snr: {{ traceroute.snr_towards[loop.index0] if traceroute.snr_towards and loop.index0 < traceroute.snr_towards|length else 'null' }} + }); + {% endif %} + {% endfor %} + + // Add destination node + {% set to_id = traceroute.to_id_hex %} + {% if to_id in nodes and nodes[to_id].position and nodes[to_id].position.longitude_i is not none and nodes[to_id].position.latitude_i is not none %} + // Handle any remaining unknown nodes before destination + if (unknownSequence.length > 0) { + var nextKnownPoint = [ + {{ nodes[to_id].position.longitude_i / 10000000 }}, + {{ nodes[to_id].position.latitude_i / 10000000 }} + ]; + + // Calculate positions for all unknown nodes in sequence + for (var i = 0; i < unknownSequence.length; i++) { + var fraction = (i + 1) / (unknownSequence.length + 1); + var midPoint = [ + lastKnownPoint[0] + (nextKnownPoint[0] - lastKnownPoint[0]) * fraction, + lastKnownPoint[1] + (nextKnownPoint[1] - lastKnownPoint[1]) * fraction + ]; + + // Create unknown node feature + var unknownNode = new ol.Feature({ + geometry: new ol.geom.Point(ol.proj.fromLonLat(midPoint)), + node: unknownSequence[i] + }); + unknownNode.setStyle(unknownStyle); + features.push(unknownNode); + allpoints.push(midPoint); + } + } - // Add destination node - var dest = new ol.Feature({ - geometry: new ol.geom.Point(ol.proj.fromLonLat([ - {{ nodes[to_id].position.longitude_i / 10000000 }}, - {{ nodes[to_id].position.latitude_i / 10000000 }} - ])), - node: { - id: '{{ to_id }}', - name: '{{ nodes[to_id].short_name }}', - longName: '{{ nodes[to_id].long_name }}', - type: 'destination', - lat: {{ nodes[to_id].position.latitude_i / 10000000 }}, - lon: {{ nodes[to_id].position.longitude_i / 10000000 }} - } - }); - dest.setStyle(destStyle); - features.push(dest); - allpoints.push([ - {{ nodes[to_id].position.longitude_i / 10000000 }}, - {{ nodes[to_id].position.latitude_i / 10000000 }} - ]); + // Add destination node + var dest = new ol.Feature({ + geometry: new ol.geom.Point(ol.proj.fromLonLat([ + {{ nodes[to_id].position.longitude_i / 10000000 }}, + {{ nodes[to_id].position.latitude_i / 10000000 }} + ])), + node: { + id: '{{ to_id }}', + name: {{ nodes[to_id].short_name|tojson }}, + longName: {{ nodes[to_id].long_name|tojson }}, + type: 'destination', + lat: {{ nodes[to_id].position.latitude_i / 10000000 }}, + lon: {{ nodes[to_id].position.longitude_i / 10000000 }} + } + }); + dest.setStyle(destStyle); + features.push(dest); + allpoints.push([ + {{ nodes[to_id].position.longitude_i / 10000000 }}, + {{ nodes[to_id].position.latitude_i / 10000000 }} + ]); - // Reset for return path - lastKnownPoint = [ - {{ nodes[to_id].position.longitude_i / 10000000 }}, - {{ nodes[to_id].position.latitude_i / 10000000 }} - ]; - unknownSequence = []; - {% else %} - // Add destination to unknown sequence if it exists in nodes but has no position - {% if to_id in nodes %} - unknownSequence.push({ - id: '{{ to_id }}', - name: '{{ nodes[to_id].short_name }}', - longName: '{{ nodes[to_id].long_name }}', - type: 'unknown', - direction: 'forward', - hopNumber: {{ traceroute.route|length + 1 }} - }); - {% endif %} - {% endif %} + // Reset for return path + lastKnownPoint = [ + {{ nodes[to_id].position.longitude_i / 10000000 }}, + {{ nodes[to_id].position.latitude_i / 10000000 }} + ]; + unknownSequence = []; + {% else %} + // Add destination to unknown sequence if it exists in nodes but has no position + {% if to_id in nodes %} + unknownSequence.push({ + id: '{{ to_id }}', + name: {{ nodes[to_id].short_name|tojson }}, + longName: {{ nodes[to_id].long_name|tojson }}, + type: 'unknown', + direction: 'forward', + hopNumber: {{ traceroute.route|length + 1 }} + }); + {% endif %} + {% endif %} - // Add return hops - {% if traceroute.route_back %} - {% for hop in traceroute.route_back %} - {% set hop_id = utils.convert_node_id_from_int_to_hex(hop) %} - {% if hop_id in nodes and nodes[hop_id].position and nodes[hop_id].position.longitude_i is not none and nodes[hop_id].position.latitude_i is not none %} - // Handle any unknown nodes in sequence - if (unknownSequence.length > 0) { - var nextKnownPoint = [ - {{ nodes[hop_id].position.longitude_i / 10000000 }}, - {{ nodes[hop_id].position.latitude_i / 10000000 }} - ]; - - // Place unknown nodes - for (var i = 0; i < unknownSequence.length; i++) { - var fraction = (i + 1) / (unknownSequence.length + 1); - var midPoint = [ - lastKnownPoint[0] + (nextKnownPoint[0] - lastKnownPoint[0]) * fraction, - lastKnownPoint[1] + (nextKnownPoint[1] - lastKnownPoint[1]) * fraction - ]; - - var unknownNode = new ol.Feature({ - geometry: new ol.geom.Point(ol.proj.fromLonLat(midPoint)), - node: { - ...unknownSequence[i], - direction: 'return' - } - }); - unknownNode.setStyle(unknownStyle); - features.push(unknownNode); - allpoints.push(midPoint); - } - unknownSequence = []; - } + // Add return hops + {% if traceroute.route_back %} + {% for hop in traceroute.route_back %} + {% set hop_id = utils.convert_node_id_from_int_to_hex(hop) %} + {% if hop_id in nodes and nodes[hop_id].position and nodes[hop_id].position.longitude_i is not none and nodes[hop_id].position.latitude_i is not none %} + // Handle any unknown nodes in sequence + if (unknownSequence.length > 0) { + var nextKnownPoint = [ + {{ nodes[hop_id].position.longitude_i / 10000000 }}, + {{ nodes[hop_id].position.latitude_i / 10000000 }} + ]; + + // Place unknown nodes + for (var i = 0; i < unknownSequence.length; i++) { + var fraction = (i + 1) / (unknownSequence.length + 1); + var midPoint = [ + lastKnownPoint[0] + (nextKnownPoint[0] - lastKnownPoint[0]) * fraction, + lastKnownPoint[1] + (nextKnownPoint[1] - lastKnownPoint[1]) * fraction + ]; + + var unknownNode = new ol.Feature({ + geometry: new ol.geom.Point(ol.proj.fromLonLat(midPoint)), + node: { + ...unknownSequence[i], + direction: 'return' + } + }); + unknownNode.setStyle(unknownStyle); + features.push(unknownNode); + allpoints.push(midPoint); + } + unknownSequence = []; + } - // Add return hop node - var returnHopNode = new ol.Feature({ - geometry: new ol.geom.Point(ol.proj.fromLonLat([ - {{ nodes[hop_id].position.longitude_i / 10000000 }}, - {{ nodes[hop_id].position.latitude_i / 10000000 }} - ])), - node: { - id: '{{ hop_id }}', - name: '{{ nodes[hop_id].short_name }}', - longName: '{{ nodes[hop_id].long_name }}', - snr: {{ traceroute.snr_back[loop.index0] if traceroute.snr_back and loop.index0 < traceroute.snr_back|length else 'null' }}, - type: 'hop', - direction: 'return', - hopNumber: {{ loop.index }}, - lat: {{ nodes[hop_id].position.latitude_i / 10000000 }}, - lon: {{ nodes[hop_id].position.longitude_i / 10000000 }} - } - }); - returnHopNode.setStyle(createHopStyle({{ loop.index }}, true)); // true for return hop - features.push(returnHopNode); + // Add return hop node + var returnHopNode = new ol.Feature({ + geometry: new ol.geom.Point(ol.proj.fromLonLat([ + {{ nodes[hop_id].position.longitude_i / 10000000 }}, + {{ nodes[hop_id].position.latitude_i / 10000000 }} + ])), + node: { + id: '{{ hop_id }}', + name: {{ nodes[hop_id].short_name|tojson }}, + longName: {{ nodes[hop_id].long_name|tojson }}, + snr: {{ traceroute.snr_back[loop.index0] if traceroute.snr_back and loop.index0 < traceroute.snr_back|length else 'null' }}, + type: 'hop', + direction: 'return', + hopNumber: {{ loop.index }}, + lat: {{ nodes[hop_id].position.latitude_i / 10000000 }}, + lon: {{ nodes[hop_id].position.longitude_i / 10000000 }} + } + }); + returnHopNode.setStyle(createHopStyle({{ loop.index }}, true)); // true for return hop + features.push(returnHopNode); - var currentPoint = [ - {{ nodes[hop_id].position.longitude_i / 10000000 }}, - {{ nodes[hop_id].position.latitude_i / 10000000 }} - ]; - allpoints.push(currentPoint); - lastKnownPoint = currentPoint; + var currentPoint = [ + {{ nodes[hop_id].position.longitude_i / 10000000 }}, + {{ nodes[hop_id].position.latitude_i / 10000000 }} + ]; + allpoints.push(currentPoint); + lastKnownPoint = currentPoint; {% else %} // Add unknown return hop to sequence unknownSequence.push({ id: '{{ hop_id }}', - name: '{{ nodes[hop_id].short_name if hop_id in nodes else "Unknown" }}', - longName: '{{ nodes[hop_id].long_name if hop_id in nodes else "Unknown Node" }}', + name: {{ nodes[hop_id].short_name|tojson if hop_id in nodes else '"Unknown"' }}, + longName: {{ nodes[hop_id].long_name|tojson if hop_id in nodes else '"Unknown Node"' }}, type: 'unknown', direction: 'return', hopNumber: {{ loop.index }}, snr: {{ traceroute.snr_back[loop.index0] if traceroute.snr_back and loop.index0 < traceroute.snr_back|length else 'null' }} }); {% endif %} - {% endfor %} + {% endfor %} - // Handle any remaining unknown nodes before final return to source - if (unknownSequence.length > 0) { - var sourcePoint = [ - {{ nodes[from_id].position.longitude_i / 10000000 }}, - {{ nodes[from_id].position.latitude_i / 10000000 }} - ]; - - // Place remaining unknown nodes between last known point and source - for (var i = 0; i < unknownSequence.length; i++) { - var fraction = (i + 1) / (unknownSequence.length + 1); - var midPoint = [ - lastKnownPoint[0] + (sourcePoint[0] - lastKnownPoint[0]) * fraction, - lastKnownPoint[1] + (sourcePoint[1] - lastKnownPoint[1]) * fraction - ]; - - var unknownNode = new ol.Feature({ - geometry: new ol.geom.Point(ol.proj.fromLonLat(midPoint)), - node: unknownSequence[i] // Include all node properties directly - }); - unknownNode.setStyle(unknownStyle); - features.push(unknownNode); - allpoints.push(midPoint); - } - unknownSequence = []; - } + // Handle any remaining unknown nodes before final return to source + if (unknownSequence.length > 0) { + var sourcePoint = [ + {{ nodes[from_id].position.longitude_i / 10000000 }}, + {{ nodes[from_id].position.latitude_i / 10000000 }} + ]; + + // Place remaining unknown nodes between last known point and source + for (var i = 0; i < unknownSequence.length; i++) { + var fraction = (i + 1) / (unknownSequence.length + 1); + var midPoint = [ + lastKnownPoint[0] + (sourcePoint[0] - lastKnownPoint[0]) * fraction, + lastKnownPoint[1] + (sourcePoint[1] - lastKnownPoint[1]) * fraction + ]; + + var unknownNode = new ol.Feature({ + geometry: new ol.geom.Point(ol.proj.fromLonLat(midPoint)), + node: unknownSequence[i] // Include all node properties directly + }); + unknownNode.setStyle(unknownStyle); + features.push(unknownNode); + allpoints.push(midPoint); + } + unknownSequence = []; + } - // After all return hops and unknown nodes, add the source point to complete the return path - allpoints.push([ - {{ nodes[from_id].position.longitude_i / 10000000 }}, - {{ nodes[from_id].position.latitude_i / 10000000 }} - ]); - {% endif %} + // After all return hops and unknown nodes, add the source point to complete the return path + allpoints.push([ + {{ nodes[from_id].position.longitude_i / 10000000 }}, + {{ nodes[from_id].position.latitude_i / 10000000 }} + ]); + {% endif %} {% endif %} // Create lines between nodes @@ -1043,14 +1045,24 @@ for (var i = 0; i < allpoints.length - 1; i++) { margin-bottom: 60px; // Add margin to prevent legend overlap } #legend { - position: relative; // Change from absolute to relative background-color: #ffffff; padding: 8px; border-radius: 4px; box-shadow: 0 1px 4px rgba(0,0,0,0.2); - margin-bottom: 20px; // Add some margin at the bottom + margin-bottom: 20px; +} + +@media screen and (min-width: 768px) { + #legend { + position: absolute; + bottom: 20px; + right: 20px; + z-index: 1000; + margin-bottom: 0; } - .ol-popup { +} + +.ol-popup { position: absolute; background-color: white; box-shadow: 0 1px 4px rgba(0,0,0,0.2); @@ -1061,10 +1073,10 @@ for (var i = 0; i < allpoints.length - 1; i++) { left: -50px; min-width: 200px; max-width: 300px; - } +} - // Add badge styling if not already present - .badge { +// Add badge styling if not already present +.badge { display: inline-block; padding: 0.25em 0.4em; font-size: 75%; @@ -1075,14 +1087,14 @@ for (var i = 0; i < allpoints.length - 1; i++) { vertical-align: baseline; border-radius: 0.25rem; margin-bottom: 0.5rem; - } - .bg-danger { +} +.bg-danger { background-color: #dc3545; color: white; - } - .bg-primary { +} +.bg-primary { background-color: #0d6efd; color: white; - } +} {% endblock %} \ No newline at end of file From be8d40e522b32f131bc9a8b9e893c344c105e0e5 Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 23 Apr 2025 03:54:55 +0000 Subject: [PATCH 022/155] options to hide position and telemetry in nodes table --- templates/node_table.html.j2 | 126 +++++++++++++++++++++++++---------- 1 file changed, 89 insertions(+), 37 deletions(-) diff --git a/templates/node_table.html.j2 b/templates/node_table.html.j2 index d0441e90..19f2f5df 100644 --- a/templates/node_table.html.j2 +++ b/templates/node_table.html.j2 @@ -1,3 +1,14 @@ +
+
+ + +
+
+ + +
+
+
@@ -7,9 +18,8 @@ - - - + + @@ -21,18 +31,13 @@ - - - - - - - - - - - + + + + + + + @@ -107,30 +112,25 @@ {% endif %} {% if node.position %} - - - - {% else %} - - - - {% endif %} - {% if node.neighbors %} - + + {% else %} - + + + {% endif %} {% if node.telemetry %} - - - - {% else %} - - - - + + + + {% endif %}
HW FW RoleLast PositionNeighborsTelemetryLast PositionTelemetry Seen Owner
     AltitudeLatitudeLongitudeCountBatteryVoltageAir Util TXChannel Util - AltitudeLatitudeLongitudeBatteryVoltageAir Util TXChannel Util Since
+ {% if node.position.altitude %} {{ node.position.altitude }} m {% endif %} {{ node.position.latitude or "" }}{{ node.position.longitude or "" }}{{ node.neighbors|length or "" }}{{ node.position.latitude or "" }}{{ node.position.longitude or "" }} + {% if 'battery_level' in node.telemetry %} {{ node.telemetry.battery_level }}% {% endif %} + {% if 'voltage' in node.telemetry %} {% if node.telemetry.voltage is number %} {{ node.telemetry.voltage|round(2) }}V @@ -139,7 +139,7 @@ {% endif %} {% endif %} + {% if 'air_util_tx' in node.telemetry %} {% if node.telemetry.air_util_tx is number %} {{ node.telemetry.air_util_tx|round(1) }}% @@ -148,7 +148,7 @@ {% endif %} {% endif %} + {% if 'channel_utilization' in node.telemetry %} {% if node.telemetry.channel_utilization is number %} {{ node.telemetry.channel_utilization|round(1) }}% @@ -158,10 +158,10 @@ {% endif %} {{ time_ago(node.ts_seen) }} @@ -173,7 +173,59 @@ {% endif %} {% endfor %} - -
-
\ No newline at end of file +
+ + + + \ No newline at end of file From 82df55d55d440352a69b4ba0e5b4ec6ae18dd42a Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 27 Apr 2025 03:39:34 +0000 Subject: [PATCH 023/155] metrics now use a moving average for smoothing --- config.ini.sample | 1 + meshinfo_web.py | 84 ++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/config.ini.sample b/config.ini.sample index 03def88b..23bfd46c 100644 --- a/config.ini.sample +++ b/config.ini.sample @@ -31,6 +31,7 @@ debug=true timezone=America/Los_Angeles zero_hop_timeout=43200 telemetry_retention_days = 30 +metrics_average_interval=7200 [geocoding] enabled=false diff --git a/meshinfo_web.py b/meshinfo_web.py index ce099c05..7460aee0 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -1218,11 +1218,85 @@ def get_metrics(): cursor.close() - # Fill in missing time slots with zeros + # --- Moving Average Helper --- + def moving_average_centered(data_list, window_minutes, bucket_size_minutes): + # data_list: list of floats (same order as time_slots) + # window_minutes: total window size (e.g., 120 for 2 hours) + # bucket_size_minutes: size of each bucket (e.g., 30 for 30 min) + n = len(data_list) + result = [] + half_window = window_minutes // 2 + buckets_per_window = max(1, window_minutes // bucket_size_minutes) + for i in range(n): + # Centered window: find all indices within window centered at i + center_time = i + window_indices = [] + for j in range(n): + if abs(j - center_time) * bucket_size_minutes <= half_window: + window_indices.append(j) + if window_indices: + avg = sum(data_list[j] for j in window_indices) / len(window_indices) + else: + avg = data_list[i] + result.append(avg) + return result + + # --- Get metrics_average_interval from config --- + metrics_avg_interval = int(config.get('server', 'metrics_average_interval', fallback=7200)) # seconds + metrics_avg_minutes = metrics_avg_interval // 60 + + # --- Calculate moving averages for relevant metrics --- + # Determine bucket_size_minutes from bucket_size + bucket_size_minutes = bucket_size + + # Prepare raw data lists + nodes_online_raw = [nodes_online_data.get(slot, 0) for slot in time_slots] + battery_levels_raw = [battery_data.get(slot, 0) for slot in time_slots] + temperature_raw = [temperature_data.get(slot, 0) for slot in time_slots] + snr_raw = [snr_data.get(slot, 0) for slot in time_slots] + + # --- Get node_activity_prune_threshold from config --- + node_activity_prune_threshold = int(config.get('server', 'node_activity_prune_threshold', fallback=7200)) # seconds + + # --- For each time slot, count unique nodes heard in the preceding activity window --- + # Fetch all telemetry records in the full time range + cursor = md.db.cursor(dictionary=True) + cursor.execute(f""" + SELECT id, ts_created + FROM telemetry + WHERE ts_created >= %s AND ts_created <= %s {channel_condition_telemetry} + ORDER BY ts_created + """, (start_timestamp, end_timestamp)) + all_telemetry = list(cursor.fetchall()) + # Convert ts_created to datetime for easier comparison + for row in all_telemetry: + if isinstance(row['ts_created'], str): + row['ts_created'] = datetime.datetime.strptime(row['ts_created'], '%Y-%m-%d %H:%M:%S') + # Precompute for each time slot + nodes_heard_per_slot = [] + for slot in time_slots: + # slot is a string, convert to datetime + if '%H:%M' in time_format or '%H:%i' in time_format: + slot_time = datetime.datetime.strptime(slot, '%Y-%m-%d %H:%M') + elif '%H:00' in time_format: + slot_time = datetime.datetime.strptime(slot, '%Y-%m-%d %H:%M') + else: + slot_time = datetime.datetime.strptime(slot, '%Y-%m-%d') + window_start = slot_time - datetime.timedelta(seconds=node_activity_prune_threshold) + # Find all node ids with telemetry in [window_start, slot_time] + active_nodes = set() + for row in all_telemetry: + if window_start < row['ts_created'] <= slot_time: + active_nodes.add(row['id']) + nodes_heard_per_slot.append(len(active_nodes)) + # Now apply moving average and round to nearest integer + nodes_online_smoothed = [round(x) for x in moving_average_centered(nodes_heard_per_slot, metrics_avg_minutes, bucket_size_minutes)] + + # Fill in missing time slots with zeros (for non-averaged metrics) result = { 'nodes_online': { 'labels': time_slots, - 'data': [nodes_online_data.get(slot, 0) for slot in time_slots] + 'data': nodes_online_smoothed }, 'message_traffic': { 'labels': time_slots, @@ -1234,15 +1308,15 @@ def get_metrics(): }, 'battery_levels': { 'labels': time_slots, - 'data': [battery_data.get(slot, 0) for slot in time_slots] + 'data': moving_average_centered(battery_levels_raw, metrics_avg_minutes, bucket_size_minutes) }, 'temperature': { 'labels': time_slots, - 'data': [temperature_data.get(slot, 0) for slot in time_slots] + 'data': moving_average_centered(temperature_raw, metrics_avg_minutes, bucket_size_minutes) }, 'snr': { 'labels': time_slots, - 'data': [snr_data.get(slot, 0) for slot in time_slots] + 'data': moving_average_centered(snr_raw, metrics_avg_minutes, bucket_size_minutes) } } From db851e106d5af5d5cfc4338dc7ef3a3176a7ebf6 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 27 Apr 2025 18:17:46 +0000 Subject: [PATCH 024/155] metrics data cleanup --- app.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/app.py b/app.py index ffbea7af..bb82e273 100644 --- a/app.py +++ b/app.py @@ -15,12 +15,18 @@ def metrics(): @app.route('/api/metrics') def get_metrics(): + node_activity_prune_threshold = int(config.get('server', 'node_activity_prune_threshold', fallback=7200)) + metrics_avg_interval = int(config.get('server', 'metrics_average_interval', fallback=7200)) + metrics_avg_minutes = metrics_avg_interval // 60 + db = MeshData() - - # Get the last 24 hours of data + # Get the last 24 hours of data, but extend for smoothing end_time = datetime.now() start_time = end_time - timedelta(hours=24) - + extra_minutes = metrics_avg_minutes // 2 + extended_start_time = start_time - timedelta(minutes=extra_minutes) + extended_end_time = end_time + timedelta(minutes=extra_minutes) + # Nodes Online (per hour) nodes_online = db.execute(""" SELECT @@ -30,7 +36,7 @@ def get_metrics(): WHERE timestamp >= ? AND timestamp <= ? GROUP BY hour ORDER BY hour - """, (int(start_time.timestamp()), int(end_time.timestamp()))) + """, (int(extended_start_time.timestamp()), int(extended_end_time.timestamp()))) # Message Traffic (per hour) message_traffic = db.execute(""" @@ -41,7 +47,7 @@ def get_metrics(): WHERE timestamp >= ? AND timestamp <= ? GROUP BY hour ORDER BY hour - """, (int(start_time.timestamp()), int(end_time.timestamp()))) + """, (int(extended_start_time.timestamp()), int(extended_end_time.timestamp()))) # Channel Utilization channel_util = db.execute(""" @@ -52,7 +58,7 @@ def get_metrics(): WHERE timestamp >= ? AND timestamp <= ? GROUP BY hour ORDER BY hour - """, (int(start_time.timestamp()), int(end_time.timestamp()))) + """, (int(extended_start_time.timestamp()), int(extended_end_time.timestamp()))) # Battery Levels (for each node) battery_levels = db.execute(""" @@ -64,7 +70,7 @@ def get_metrics(): WHERE timestamp >= ? AND timestamp <= ? GROUP BY id, hour ORDER BY hour - """, (int(start_time.timestamp()), int(end_time.timestamp()))) + """, (int(extended_start_time.timestamp()), int(extended_end_time.timestamp()))) # Temperature Readings temperature = db.execute(""" @@ -76,7 +82,7 @@ def get_metrics(): WHERE timestamp >= ? AND timestamp <= ? GROUP BY id, hour ORDER BY hour - """, (int(start_time.timestamp()), int(end_time.timestamp()))) + """, (int(extended_start_time.timestamp()), int(extended_end_time.timestamp()))) # Signal Strength (SNR) snr = db.execute(""" @@ -87,7 +93,7 @@ def get_metrics(): WHERE timestamp >= ? AND timestamp <= ? GROUP BY hour ORDER BY hour - """, (int(start_time.timestamp()), int(end_time.timestamp()))) + """, (int(extended_start_time.timestamp()), int(extended_end_time.timestamp()))) # Process battery levels data battery_data = {} @@ -105,10 +111,39 @@ def get_metrics(): temp_data[row['id']]['hours'].append(row['hour']) temp_data[row['id']]['temps'].append(row['avg_temp']) + # Build extended labels and data for nodes_online + extended_labels = [row['hour'] for row in nodes_online] + extended_data = [row['node_count'] for row in nodes_online] + + # Moving average helper (centered) + def moving_average_centered(data, window): + n = len(data) + result = [] + half = window // 2 + for i in range(n): + left = max(0, i - half) + right = min(n, i + half + 1) + window_data = data[left:right] + result.append(sum(window_data) / len(window_data)) + return result + + # Apply smoothing to extended data + window_buckets = max(1, metrics_avg_minutes // 60) + smoothed_data = moving_average_centered(extended_data, window_buckets) + + # Trim to original 24-hour window + trimmed_labels = [] + trimmed_data = [] + for label, value in zip(extended_labels, smoothed_data): + label_dt = datetime.strptime(label, '%Y-%m-%d %H:%M') + if start_time <= label_dt <= end_time: + trimmed_labels.append(label) + trimmed_data.append(value) + return jsonify({ 'nodes_online': { - 'labels': [row['hour'] for row in nodes_online], - 'data': [row['node_count'] for row in nodes_online] + 'labels': trimmed_labels, + 'data': trimmed_data }, 'message_traffic': { 'labels': [row['hour'] for row in message_traffic], From f2c4fb363b49f02b0cda60fdb9ee6e63086c83dd Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 27 Apr 2025 18:21:56 +0000 Subject: [PATCH 025/155] column filter state persistence works now --- templates/node_table.html.j2 | 37 +++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/templates/node_table.html.j2 b/templates/node_table.html.j2 index 19f2f5df..404ee0dc 100644 --- a/templates/node_table.html.j2 +++ b/templates/node_table.html.j2 @@ -183,36 +183,39 @@ document.addEventListener('DOMContentLoaded', function() { const positionCols = document.querySelectorAll('.position-col'); const telemetryCols = document.querySelectorAll('.telemetry-col'); - // Clear any existing preferences - localStorage.removeItem('showPosition'); - localStorage.removeItem('showTelemetry'); - - // Load saved preferences - default to false (hidden) if not set - const showPosition = localStorage.getItem('showPosition') === 'true'; - const showTelemetry = localStorage.getItem('showTelemetry') === 'true'; - - positionSwitch.checked = showPosition; - telemetrySwitch.checked = showTelemetry; - function toggleColumns(columns, show) { columns.forEach(col => { col.style.display = show ? '' : 'none'; }); } - // Initial state + // Load saved preferences from localStorage + let showPosition = localStorage.getItem('showPosition'); + let showTelemetry = localStorage.getItem('showTelemetry'); + + // Convert string 'true'/'false' to boolean, default to false if not set + showPosition = showPosition === 'true'; + showTelemetry = showTelemetry === 'true'; + + // Set initial checkbox states + positionSwitch.checked = showPosition; + telemetrySwitch.checked = showTelemetry; + + // Set initial column visibility toggleColumns(positionCols, showPosition); toggleColumns(telemetryCols, showTelemetry); - // Event listeners + // Event listeners for changes positionSwitch.addEventListener('change', function() { - toggleColumns(positionCols, this.checked); - localStorage.setItem('showPosition', this.checked); + const isChecked = this.checked; + toggleColumns(positionCols, isChecked); + localStorage.setItem('showPosition', isChecked); }); telemetrySwitch.addEventListener('change', function() { - toggleColumns(telemetryCols, this.checked); - localStorage.setItem('showTelemetry', this.checked); + const isChecked = this.checked; + toggleColumns(telemetryCols, isChecked); + localStorage.setItem('showTelemetry', isChecked); }); }); From fcf222f241f02c8dbb16e23c4776712699b43c1e Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 30 Apr 2025 03:25:51 +0000 Subject: [PATCH 026/155] adjustments to edge effects in graph calculations; glow effect and placement update in traceroute map --- app.py | 199 ++++++++++++++++++++++++++----- templates/metrics.html.j2 | 49 +++++--- templates/traceroute_map.html.j2 | 138 ++++++++++++++++----- 3 files changed, 305 insertions(+), 81 deletions(-) diff --git a/app.py b/app.py index bb82e273..a257b7a4 100644 --- a/app.py +++ b/app.py @@ -15,17 +15,36 @@ def metrics(): @app.route('/api/metrics') def get_metrics(): - node_activity_prune_threshold = int(config.get('server', 'node_activity_prune_threshold', fallback=7200)) + db = MeshData() + # Get time_range from query params + time_range = request.args.get('time_range', 'day') # day, week, month + end_time = datetime.now() + if time_range == 'week': + start_time = end_time - timedelta(days=7) + elif time_range == 'month': + start_time = end_time - timedelta(days=30) + else: # default to day + start_time = end_time - timedelta(hours=24) metrics_avg_interval = int(config.get('server', 'metrics_average_interval', fallback=7200)) metrics_avg_minutes = metrics_avg_interval // 60 + node_activity_prune_threshold = int(config.get('server', 'node_activity_prune_threshold', fallback=7200)) - db = MeshData() - # Get the last 24 hours of data, but extend for smoothing - end_time = datetime.now() - start_time = end_time - timedelta(hours=24) - extra_minutes = metrics_avg_minutes // 2 - extended_start_time = start_time - timedelta(minutes=extra_minutes) - extended_end_time = end_time + timedelta(minutes=extra_minutes) + # For day view, calculate the correct extended_start_time for all queries + if time_range == 'day': + window_buckets = max(1, metrics_avg_minutes // 60) + half_window = window_buckets // 2 + extended_bucket_start = start_time - timedelta(hours=half_window) + # Extend by even more - go back at least 48 hours to ensure we have sufficient history + fetch_start_time = min( + extended_bucket_start - timedelta(seconds=3 * node_activity_prune_threshold), + start_time - timedelta(hours=48) + ) + extended_start_time = fetch_start_time + extended_end_time = end_time + timedelta(hours=half_window) + else: + extra_minutes = metrics_avg_minutes // 2 + extended_start_time = start_time - timedelta(minutes=extra_minutes) + extended_end_time = end_time + timedelta(minutes=extra_minutes) # Nodes Online (per hour) nodes_online = db.execute(""" @@ -115,35 +134,112 @@ def get_metrics(): extended_labels = [row['hour'] for row in nodes_online] extended_data = [row['node_count'] for row in nodes_online] - # Moving average helper (centered) - def moving_average_centered(data, window): - n = len(data) - result = [] - half = window // 2 - for i in range(n): - left = max(0, i - half) - right = min(n, i + half + 1) - window_data = data[left:right] - result.append(sum(window_data) / len(window_data)) - return result + # For day view, use 'active in window' approach for nodes heard with true centered smoothing + if time_range == 'day': + # Calculate how many extra buckets are needed for smoothing + window_buckets = max(1, metrics_avg_minutes // 60) + half_window = window_buckets // 2 + # Generate all expected hourly labels from (start_time - half_window*1hr) to (end_time + half_window*1hr) + extended_bucket_start = start_time - timedelta(hours=half_window) + extended_bucket_end = end_time + timedelta(hours=half_window) + expected_labels = [] + current = extended_bucket_start.replace(minute=0, second=0, microsecond=0) + while current <= extended_bucket_end: + expected_labels.append(current.strftime('%Y-%m-%d %H:%M')) + current += timedelta(hours=1) + # Extend the fetch window to cover the full activity window before the first extended bucket + fetch_start_time = extended_bucket_start - timedelta(seconds=node_activity_prune_threshold) + # Fetch all telemetry records in the full time range + all_telemetry = db.execute(""" + SELECT id, timestamp FROM telemetry + WHERE timestamp >= ? AND timestamp <= ? + ORDER BY timestamp + """, (int(fetch_start_time.timestamp()), int(extended_bucket_end.timestamp()))) + # Convert to list of (id, datetime) + telemetry_points = [(row['id'], datetime.fromtimestamp(row['timestamp'])) for row in all_telemetry] + # First, get all node IDs that appeared in our data + all_node_ids = set(node_id for node_id, _ in telemetry_points) - # Apply smoothing to extended data - window_buckets = max(1, metrics_avg_minutes // 60) - smoothed_data = moving_average_centered(extended_data, window_buckets) + # Find the earliest telemetry timestamp + earliest_telemetry_time = min((ts for _, ts in telemetry_points), default=extended_bucket_start) - # Trim to original 24-hour window - trimmed_labels = [] - trimmed_data = [] - for label, value in zip(extended_labels, smoothed_data): - label_dt = datetime.strptime(label, '%Y-%m-%d %H:%M') - if start_time <= label_dt <= end_time: - trimmed_labels.append(label) - trimmed_data.append(value) + # For the main visible time range (not the extended range), calculate the average number of + # nodes that are typically active in any given hour + visible_range_active_nodes = [] + for label in expected_labels: + bucket_time = datetime.strptime(label, '%Y-%m-%d %H:%M') + # Only consider buckets within our actual display window (not extended) + if start_time <= bucket_time <= end_time: + window_start = bucket_time - timedelta(seconds=node_activity_prune_threshold) + # Only count this bucket if we have complete history for it + if window_start >= earliest_telemetry_time: + active_nodes = set() + for node_id, ts in telemetry_points: + if window_start < ts <= bucket_time: + active_nodes.add(node_id) + visible_range_active_nodes.append(len(active_nodes)) + + # Calculate the median of fully visible buckets with sufficient history + if visible_range_active_nodes: + median_active_nodes = sorted(visible_range_active_nodes)[len(visible_range_active_nodes) // 2] + else: + # Fallback if we somehow don't have any good buckets + median_active_nodes = len(all_node_ids) + + # Now process all buckets, but force buckets with insufficient history to use the median + extended_data = [] + for i, label in enumerate(expected_labels): + bucket_time = datetime.strptime(label, '%Y-%m-%d %H:%M') + window_start = bucket_time - timedelta(seconds=node_activity_prune_threshold) + + # Check if we have enough history for this bucket + if window_start < earliest_telemetry_time: + # No sufficient history - use the median value from good buckets + extended_data.append(median_active_nodes) + else: + # Normal calculation for buckets with good history + active_nodes = set() + for node_id, ts in telemetry_points: + if window_start < ts <= bucket_time: + active_nodes.add(node_id) + extended_data.append(len(active_nodes)) + + if not has_sufficient_history: + insufficient_history_buckets.append(i) + + # Fix the buckets with insufficient history by estimating based on later data + if insufficient_history_buckets and len(extended_data) > max(insufficient_history_buckets) + 1: + # Find the median value from buckets with sufficient history + sufficient_buckets = [i for i in range(len(extended_data)) if i not in insufficient_history_buckets] + if sufficient_buckets: + sufficient_values = [extended_data[i] for i in sufficient_buckets] + median_value = sorted(sufficient_values)[len(sufficient_values) // 2] + + # Replace insufficient bucket values with this median + for i in insufficient_history_buckets: + if i < len(extended_data): + extended_data[i] = median_value + + # Apply smoothing to extended data with improved edge handling + smoothed_data = moving_average_centered_with_edge_handling(extended_data, window_buckets) + + # Trim to original window + trimmed_labels = [] + trimmed_data = [] + for label, value in zip(extended_labels, smoothed_data): + label_dt = datetime.strptime(label, '%Y-%m-%d %H:%M') + if start_time <= label_dt <= end_time: + trimmed_labels.append(label) + trimmed_data.append(value) + + # Overwrite for output + extended_labels = trimmed_labels + extended_data = trimmed_data return jsonify({ 'nodes_online': { - 'labels': trimmed_labels, - 'data': trimmed_data + 'labels': extended_labels, + 'data': extended_data }, 'message_traffic': { 'labels': [row['hour'] for row in message_traffic], @@ -177,4 +273,41 @@ def moving_average_centered(data, window): 'labels': [row['hour'] for row in snr], 'data': [row['avg_snr'] for row in snr] } - }) \ No newline at end of file + }) + +# Moving average helper (centered, with mirrored edge handling) +def moving_average_centered_with_edge_handling(data, window_size): + """ + Apply centered moving average with proper edge handling. + Uses mirroring at the edges to ensure consistent smoothing. + """ + # Get the median value for more robust padding + # This helps avoid using outliers at the edges + sorted_data = sorted(data) + median_value = sorted_data[len(sorted_data) // 2] + + result = [] + half_window = window_size // 2 + + # For each point, calculate the average of the window centered on that point + for i in range(len(data)): + window_sum = 0 + window_count = 0 + + for j in range(i - half_window, i + half_window + 1): + if 0 <= j < len(data): + # Within bounds, use actual data + window_sum += data[j] + window_count += 1 + elif j < 0: + # Before start, use median or first value + window_sum += median_value # Alternative: data[0] + window_count += 1 + else: # j >= len(data) + # After end, use median or last value + window_sum += median_value # Alternative: data[-1] + window_count += 1 + + result.append(window_sum / window_count) + + return result \ No newline at end of file diff --git a/templates/metrics.html.j2 b/templates/metrics.html.j2 index 240c8739..64e13cd6 100644 --- a/templates/metrics.html.j2 +++ b/templates/metrics.html.j2 @@ -653,7 +653,9 @@ Mesh Metrics day: 'numeric' }) + '\n' + timeStr, isDateLabel: true, - isFirstOfDate: true + isFirstOfDate: true, + date: utcDate, + timeStr: timeStr }; } @@ -661,7 +663,9 @@ Mesh Metrics return { display: timeStr, isDateLabel: false, - isFirstOfDate: false + isFirstOfDate: false, + date: utcDate, + timeStr: timeStr }; }); @@ -670,6 +674,9 @@ Mesh Metrics chart.data.datasets[0].data = data.data; chart.data.datasets[0].label = label; + // Get the current time range from the filter + const timeRange = document.getElementById('timeRangeFilter') ? document.getElementById('timeRangeFilter').value : 'day'; + // Update y-axis label chart.options.scales.y.title.text = yAxisLabel; @@ -679,33 +686,45 @@ Mesh Metrics ticks: { ...chart.options.scales.x.ticks, callback: function(val, index) { - // Get the actual label const label = localLabels[index]; if (!label) return ''; - - // Calculate label density const labelCount = localLabels.length; - const skipFactor = Math.ceil(labelCount / 15); // Target showing ~15 labels - - // Always show date labels + // For 'month' view: only show the date (no time) + if (timeRange === 'month') { + if (label.isFirstOfDate) { + const day = label.date.getDate(); + // Show label for the 1st, 4th, 7th, ... day (every 3rd day, including the 1st) + if (day === 1 || day % 3 === 0) { + return label.date.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }); + } + } + return ''; + } + // For 'week' view: show date at start of day, and only show '12:00' (noon) for other times + if (timeRange === 'week') { + if (label.isFirstOfDate) { + return label.date.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }); + } + // Only show '12:00' (noon) + if (label.timeStr === '12:00') { + return '12:00'; + } + return ''; + } + // For 'day' view: keep current behavior (show date at start of day, otherwise show time at regular intervals) if (label.isFirstOfDate) { return label.display.split('\n'); } - - // For time labels, check if we're too close to a date label + // Show other time labels at regular intervals + const skipFactor = Math.ceil(labelCount / 15); // Target showing ~15 labels const prevLabel = index > 0 ? localLabels[index - 1] : null; const nextLabel = index < localLabels.length - 1 ? localLabels[index + 1] : null; - - // Skip if adjacent to a date label if (prevLabel?.isFirstOfDate || nextLabel?.isFirstOfDate) { return ''; } - - // Show other time labels at regular intervals if (index === 0 || index % skipFactor === 0) { return label.display; } - return ''; } } diff --git a/templates/traceroute_map.html.j2 b/templates/traceroute_map.html.j2 index 93c0d25b..63daec27 100644 --- a/templates/traceroute_map.html.j2 +++ b/templates/traceroute_map.html.j2 @@ -769,7 +769,9 @@ for (var i = 0; i < allpoints.length - 1; i++) { // Create line feature var line = new ol.Feature({ - geometry: new ol.geom.LineString(points) + geometry: new ol.geom.LineString(points), + segmentIndex: i, + isForwardPath: isForwardPath }); // Style the line based on path type @@ -812,14 +814,32 @@ for (var i = 0; i < allpoints.length - 1; i++) { lines.push(line); - // Add label at the middle of the line - var midPoint = [ - (points[0][0] + points[1][0]) / 2, - (points[0][1] + points[1][1]) / 2 - ]; - + // Add label at a fraction along the line with perpendicular offset + var fraction = (i + 1) / (allpoints.length); // Spread labels along the line + var x = points[0][0] + (points[1][0] - points[0][0]) * fraction; + var y = points[0][1] + (points[1][1] - points[0][1]) * fraction; + + // Calculate perpendicular offset + var dx = points[1][0] - points[0][0]; + var dy = points[1][1] - points[0][1]; + var length = Math.sqrt(dx*dx + dy*dy); + var offset = 40; // in map units (adjust as needed) + var perpX = -dy / length * offset; + var perpY = dx / length * offset; + + // Direction: forward = +offset, return = -offset + var isReturn = !isForwardPath; + if (isReturn) { + perpX = -perpX; + perpY = -perpY; + } + + var labelPoint = [x + perpX, y + perpY]; + var label = new ol.Feature({ - geometry: new ol.geom.Point(midPoint) + geometry: new ol.geom.Point(labelPoint), + segmentIndex: i, + isForwardPath: isForwardPath }); var labelText = ''; @@ -867,31 +887,8 @@ for (var i = 0; i < allpoints.length - 1; i++) { // Only create label if there's text to show if (labelText) { - label.setStyle(new ol.style.Style({ - text: new ol.style.Text({ - text: labelText, - font: '12px Arial', - fill: new ol.style.Fill({ color: '#000000' }), - stroke: new ol.style.Stroke({ - color: '#ffffff', - width: 3 - }), - padding: [3, 5, 3, 5], - offsetY: -20, - backgroundFill: new ol.style.Fill({ - color: 'rgba(255, 255, 255, 0.85)' - }), - backgroundStroke: new ol.style.Stroke({ - color: 'rgba(0, 0, 0, 0.4)', - width: 1, - lineCap: 'round', - lineJoin: 'round' - }), - placement: 'point', - overflow: true - }) - })); - + label.set('labelText', labelText); + label.setStyle(getNormalLabelStyle(labelText)); labels.push(label); } } @@ -1036,6 +1033,81 @@ for (var i = 0; i < allpoints.length - 1; i++) { map.getTargetElement().style.cursor = hit ? 'pointer' : ''; }); + + // Define glow box styles for labels + function getGlowBoxStyle(isForward, labelText) { + var color = isForward ? 'rgba(68,170,68,0.35)' : 'rgba(170,68,170,0.35)'; + return new ol.style.Style({ + text: new ol.style.Text({ + text: labelText, + font: '12px Arial', + fill: new ol.style.Fill({ color: 'rgba(0,0,0,0)' }), // invisible text + backgroundFill: new ol.style.Fill({ color: color }), + padding: [4, 6, 4, 6], // tighter padding for glow + offsetY: -20, + backgroundStroke: new ol.style.Stroke({ + color: color, + width: 5 // tighter stroke + }), + placement: 'point', + overflow: true + }) + }); + } + + // Save the normal label style for later use + function getNormalLabelStyle(labelText) { + return new ol.style.Style({ + text: new ol.style.Text({ + text: labelText, + font: '12px Arial', + fill: new ol.style.Fill({ color: '#000000' }), + stroke: new ol.style.Stroke({ + color: '#ffffff', + width: 3 + }), + padding: [3, 5, 3, 5], + offsetY: -20, + backgroundFill: new ol.style.Fill({ + color: 'rgba(255, 255, 255, 0.85)' + }), + backgroundStroke: new ol.style.Stroke({ + color: 'rgba(0, 0, 0, 0.4)', + width: 1, + lineCap: 'round', + lineJoin: 'round' + }), + placement: 'point', + overflow: true + }) + }); + } + + // Add hover effect for lines to glow the associated label with a box + map.on('pointermove', function(e) { + var pixel = map.getEventPixel(e.originalEvent); + var feature = map.forEachFeatureAtPixel(pixel, function(feature) { + return feature; + }); + // Remove glow from all labels first + labels.forEach(function(label) { + var labelText = label.get('labelText'); + if (labelText) { + label.setStyle(getNormalLabelStyle(labelText)); + } + }); + if (feature && feature.getGeometry() instanceof ol.geom.LineString) { + var segIdx = feature.get('segmentIndex'); + var isForward = feature.get('isForwardPath'); + var label = labels.find(l => l.get('segmentIndex') === segIdx); + if (label) { + var labelText = label.get('labelText'); + var glowBox = getGlowBoxStyle(isForward, labelText); + var normal = getNormalLabelStyle(labelText); + label.setStyle([glowBox, normal]); + } + } + }); {% endblock %} \ No newline at end of file From 15596c07567e971d5ca3292dfd3da5818930e780 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 18 May 2025 23:12:29 +0000 Subject: [PATCH 032/155] fix for transposed gps coordinates --- templates/map.html.j2 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/templates/map.html.j2 b/templates/map.html.j2 index 8595586e..dc69aba2 100644 --- a/templates/map.html.j2 +++ b/templates/map.html.j2 @@ -289,6 +289,9 @@ var neighborLayers = []; {% for id, node in nodes.items() %} {% if node.position is defined and node.position and node.position.longitude_i and node.position.latitude_i %} + {% set lat = node.position.latitude_i / 10000000 %} + {% set lon = node.position.longitude_i / 10000000 %} + {% if -90 <= lat <= 90 and -180 <= lon <= 180 %} nodes['{{ id }}'] = { id: '{{ id }}', short_name: '{{ node.short_name }}', @@ -296,7 +299,7 @@ last_seen: '{{ time_ago(node.ts_seen) }}', ts_seen: {{ node.ts_seen }}, ts_uplink: {% if node.ts_uplink is defined and node.ts_uplink is not none %}{{ node.ts_uplink }}{% else %}null{% endif %}, - position: [{{ node.position.longitude_i / 10000000 }}, {{ node.position.latitude_i / 10000000 }}], + position: [{{ lon }}, {{ lat }}], online: {% if node.active %}true{% else %}false{% endif %}, channel: {% if node.channel is not none %}{{ node.channel }}{% else %}null{% endif %}, zero_hop_data: {{ zero_hop_data.get(id, {'heard': [], 'heard_by': []}) | tojson | safe }} From 3ee0f11ebc6568bf211334f79a5fa43f6439d4b4 Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 22 May 2025 03:40:33 +0000 Subject: [PATCH 033/155] add channel filtering to reduce logging of errors on private channels --- main.py | 77 ++++++++++++++++++++++++++++++++++------------ meshdata.py | 53 +++++++++++++++++++++++++++++-- process_payload.py | 9 ++++++ 3 files changed, 117 insertions(+), 22 deletions(-) diff --git a/main.py b/main.py index 15bba611..0f0c14bd 100644 --- a/main.py +++ b/main.py @@ -5,6 +5,8 @@ import time import sys import os +import atexit +import signal def setup_logger(): @@ -49,14 +51,34 @@ def setup_logger(): from meshdata import MeshData, create_database def check_pid(pid): - """ Check For the existence of a unix pid. """ + """Check if the process with the given PID is running and is our process.""" try: + # Check if process exists os.kill(pid, 0) + # Check if it's our process by reading /proc//cmdline + try: + with open(f'/proc/{pid}/cmdline', 'rb') as f: + cmdline = f.read().decode('utf-8', errors='ignore') + return 'python' in cmdline and 'main.py' in cmdline + except (IOError, PermissionError): + return False except OSError: return False - else: - return True +def cleanup_pidfile(): + """Remove the PID file on exit.""" + try: + if os.path.exists(pidfile): + os.remove(pidfile) + logger.info("Cleaned up PID file") + except Exception as e: + logger.error(f"Error cleaning up PID file: {e}") + +def handle_signal(signum, frame): + """Handle termination signals gracefully.""" + logger.info(f"Received signal {signum}, cleaning up...") + cleanup_pidfile() + sys.exit(0) def threadwrap(threadfunc): def wrapper(): @@ -69,34 +91,49 @@ def wrapper(): logger.error('exited normally, bad thread; restarting') return wrapper - pidfile = "meshinfo.pid" pid = None -try: - fh = open(pidfile, "r") - pid = int(fh.read()) - fh.close() -except Exception as e: - pass -if pid and check_pid(pid): - print("already running") - sys.exit(0) +# Register signal handlers +signal.signal(signal.SIGTERM, handle_signal) +signal.signal(signal.SIGINT, handle_signal) -fh = open(pidfile, "w") -fh.write(str(os.getpid())) -fh.close() +# Register cleanup function +atexit.register(cleanup_pidfile) +try: + if os.path.exists(pidfile): + with open(pidfile, "r") as fh: + pid = int(fh.read().strip()) + if check_pid(pid): + logger.info("Process already running with PID %d", pid) + sys.exit(0) + else: + logger.warning("Found stale PID file, removing it") + os.remove(pidfile) +except Exception as e: + logger.warning(f"Error reading PID file: {e}") + +# Write our PID +try: + with open(pidfile, "w") as fh: + fh.write(str(os.getpid())) + logger.info("Wrote PID file") +except Exception as e: + logger.error(f"Error writing PID file: {e}") + sys.exit(1) config_file = "config.ini" if not os.path.isfile(config_file): - print(f"Error: Configuration file '{config_file}' not found!") + logger.error(f"Error: Configuration file '{config_file}' not found!") sys.exit(1) -fh = open("banner", "r") -logger.info(fh.read()) -fh.close() +try: + with open("banner", "r") as fh: + logger.info(fh.read()) +except Exception as e: + logger.warning(f"Error reading banner file: {e}") logger.info("Setting up database") db_connected = False diff --git a/meshdata.py b/meshdata.py index f9e831fe..9309ed91 100644 --- a/meshdata.py +++ b/meshdata.py @@ -968,7 +968,7 @@ def store_telemetry(self, data): # Extract channel from the root of the telemetry data channel = data.get("channel") - def validate_telemetry_value(value): + def validate_telemetry_value(value, field_name=None): """Validate telemetry values, converting invalid values to None.""" if value is None: return None @@ -977,6 +977,55 @@ def validate_telemetry_value(value): float_val = float(value) if float_val == float('inf') or float_val == float('-inf') or str(float_val).lower() == 'nan': return None + + # Apply field-specific validation + if field_name: + if field_name == 'battery_level': + # Battery level should be positive and an integer + # Allow values > 100% due to calibration issues + if float_val < 0 or not float_val.is_integer(): + return None + return int(float_val) # Convert to int since DB expects INT + elif field_name == 'air_util_tx' or field_name == 'channel_utilization': + # Utilization values should be positive + # Allow > 100% due to network conditions + if float_val < 0: + return None + elif field_name == 'uptime_seconds': + # Uptime should be positive + if float_val < 0: + return None + elif field_name == 'voltage': + # Voltage should be positive and reasonable + # Allow up to 100V to accommodate various power systems + if float_val < 0 or float_val > 100: + return None + elif field_name == 'temperature': + # Temperature should be within reasonable sensor range + # Allow -100°C to 150°C to accommodate various sensors and conditions + if not -100 <= float_val <= 150: + return None + elif field_name == 'relative_humidity': + # Humidity should be reasonable + # Allow slight overshoot due to calibration + if not -5 <= float_val <= 105: + return None + elif field_name == 'barometric_pressure': + # Pressure should be positive and reasonable + # Allow wider range for different altitudes and conditions + if float_val < 0 or float_val > 2000: + return None + elif field_name == 'gas_resistance': + # Gas resistance should be positive + # Allow up to 1e308 (near DOUBLE_MAX) for various sensors + if float_val <= 0 or float_val > 1e308: + return None + elif field_name == 'current': + # Current should be reasonable + # Allow wider range for various power monitoring setups + if not -50 <= float_val <= 50: + return None + return float_val except (ValueError, TypeError): return None @@ -1004,7 +1053,7 @@ def validate_telemetry_value(value): continue for key in data: if key in payload[metric]: - data[key] = validate_telemetry_value(payload[metric][key]) + data[key] = validate_telemetry_value(payload[metric][key], key) sql = """INSERT INTO telemetry (id, air_util_tx, battery_level, channel_utilization, diff --git a/process_payload.py b/process_payload.py index ceae72e0..8bdfc52c 100644 --- a/process_payload.py +++ b/process_payload.py @@ -186,6 +186,15 @@ def process_payload(payload, topic, md: MeshData): # --- Add log at the start --- logger = logging.getLogger(__name__) # Get logger instance logger.debug(f"process_payload: Entered function for topic: {topic}") + + # Check if this is an ignored channel + if "/2/e/" in topic: + channel_name = topic.split("/")[-2] # Get channel name from topic + ignored_channels = config.get("channels", "ignored_channels", fallback="").split(",") + if channel_name in ignored_channels: + logger.debug(f"Ignoring message from channel: {channel_name}") + return + # --- End log --- mp = get_packet(payload) if mp: From 3cf02f2fe7e424b422840f4bcc44fa190fc943ca Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 24 May 2025 15:23:43 +0000 Subject: [PATCH 034/155] updated get_nodes() query to be more responsive --- meshdata.py | 236 +++++++++++++++++++++++++++-------- templates/map.html.j2 | 51 ++++---- templates/mynodes.html.j2 | 8 +- templates/node.html.j2 | 62 ++++----- templates/node_table.html.j2 | 8 +- templates/nodes.html.j2 | 6 +- utils.py | 37 +++--- 7 files changed, 275 insertions(+), 133 deletions(-) diff --git a/meshdata.py b/meshdata.py index 9309ed91..f9f09319 100644 --- a/meshdata.py +++ b/meshdata.py @@ -370,70 +370,206 @@ def iter_pages(current_page, total_pages, left_edge=2, left_current=2, right_cur last = num def get_nodes(self, active=False): + """ + Retrieve all nodes from the database, including their latest telemetry, position, and channel data. + + This method uses a single optimized SQL query with Common Table Expressions (CTEs) to join the latest telemetry, + position, and channel information for each node. To avoid column name collisions (especially for the 'id' field), + all joined tables alias their 'id' columns (e.g., 'telemetry_id', 'position_id', 'channel_id') and only the + primary node ID from 'nodeinfo' is selected as 'id'. + + This explicit column selection and aliasing is critical: if the joined tables' 'id' columns were not aliased or + omitted from the SELECT list, they could overwrite the real node ID in the result set with NULL for nodes that + lack telemetry or position data. This would cause many nodes to be skipped in the final output, leading to an + incorrect and reduced node count. By selecting only 'n.id' as 'id', we ensure all nodes from 'nodeinfo' are + included, regardless of whether they have telemetry, position, or channel data. + + Args: + active (bool): Unused, present for compatibility. + Returns: + dict: A dictionary of nodes keyed by their hex ID, with all relevant data included. + """ nodes = {} active_threshold = int( self.config["server"]["node_activity_prune_threshold"] ) - # Modified to include all nodes but still mark active status - all_sql = """SELECT n.*, u.username owner_username, + + # Combined query to get all node data in one go + sql = """ + WITH latest_telemetry AS ( + SELECT id as telemetry_id, + air_util_tx, + battery_level, + channel_utilization, + uptime_seconds, + voltage, + temperature, + relative_humidity, + barometric_pressure, + gas_resistance, + current, + telemetry_time, + channel as telemetry_channel, + ROW_NUMBER() OVER (PARTITION BY id ORDER BY telemetry_time DESC) as rn + FROM telemetry + WHERE battery_level IS NOT NULL + ), + latest_position AS ( + SELECT id as position_id, + altitude, + ground_speed, + ground_track, + latitude_i, + longitude_i, + location_source, + precision_bits, + position_time, + geocoded, + ROW_NUMBER() OVER (PARTITION BY id ORDER BY position_time DESC) as rn + FROM position + ), + latest_channel AS ( + SELECT id as channel_id, channel, ts_created + FROM ( + SELECT id, channel, ts_created + FROM telemetry + WHERE channel IS NOT NULL + UNION ALL + SELECT from_id as id, channel, ts_created + FROM text + WHERE channel IS NOT NULL + ) combined + WHERE id IS NOT NULL + GROUP BY id + ORDER BY ts_created DESC + ) + SELECT + n.id, + n.long_name, + n.short_name, + n.hw_model, + n.role, + n.firmware_version, + n.owner, + n.updated_via, + n.ts_seen, + n.ts_created, + n.ts_updated, + u.username as owner_username, CASE WHEN n.ts_seen > FROM_UNIXTIME(%s) THEN 1 ELSE 0 END as is_active, - UNIX_TIMESTAMP(n.ts_uplink) as ts_uplink - FROM nodeinfo n - LEFT OUTER JOIN meshuser u ON n.owner = u.email""" + UNIX_TIMESTAMP(n.ts_uplink) as ts_uplink, + -- Telemetry fields + t.air_util_tx, + t.battery_level, + t.channel_utilization, + t.uptime_seconds, + t.voltage, + t.temperature, + t.relative_humidity, + t.barometric_pressure, + t.gas_resistance, + t.current, + t.telemetry_time, + t.telemetry_channel, + -- Position fields + p.altitude, + p.ground_speed, + p.ground_track, + p.latitude_i, + p.longitude_i, + p.location_source, + p.precision_bits, + p.position_time, + p.geocoded, + -- Channel + c.channel + FROM nodeinfo n + LEFT OUTER JOIN meshuser u ON n.owner = u.email + LEFT OUTER JOIN latest_telemetry t ON n.id = t.telemetry_id AND t.rn = 1 + LEFT OUTER JOIN latest_position p ON n.id = p.position_id AND p.rn = 1 + LEFT OUTER JOIN latest_channel c ON n.id = c.channel_id + WHERE n.id IS NOT NULL + """ - cur = self.db.cursor() + cur = self.db.cursor(dictionary=True) timeout = time.time() - active_threshold - params = (timeout, ) - cur.execute(all_sql, params) + cur.execute(sql, (timeout,)) rows = cur.fetchall() - column_names = [desc[0] for desc in cur.description] + print("Fetched rows:", len(rows)) + skipped = 0 for row in rows: + if not row or row.get('id') is None: + skipped += 1 + continue + if not row or row.get('id') is None: + continue # Skip rows with no ID + record = {} - for i in range(len(row)): - if isinstance(row[i], datetime.datetime): - record[column_names[i]] = row[i].timestamp() + # Convert datetime fields to timestamps + for key, value in row.items(): + if isinstance(value, datetime.datetime): + record[key] = value.timestamp() else: - record[column_names[i]] = row[i] + record[key] = value - # Use the is_active field from the query - is_active = bool(record.get("is_active", 0)) - record["telemetry"] = self.get_telemetry(row[0]) - record["neighbors"] = self.get_neighbors(row[0]) - record["position"] = self.get_position(row[0]) - if record["position"]: - if record["position"]["latitude_i"]: - record["position"]["latitude"] = \ - record["position"]["latitude_i"] / 10000000 - else: - record["position"]["latitude"] = None - if record["position"]["longitude_i"]: - record["position"]["longitude"] = \ - record["position"]["longitude_i"] / 10000000 - else: - record["position"]["longitude"] = None - record["role"] = record["role"] or 0 - record["active"] = is_active - record["last_seen"] = utils.time_since(record["ts_seen"]) - # --- Add: Find most recent channel --- - node_id = row[0] - cur2 = self.db.cursor() - cur2.execute(""" - (SELECT channel, ts_created FROM telemetry WHERE id = %s AND channel IS NOT NULL ORDER BY ts_created DESC LIMIT 1) - UNION ALL - (SELECT channel, ts_created FROM text WHERE from_id = %s AND channel IS NOT NULL ORDER BY ts_created DESC LIMIT 1) - ORDER BY ts_created DESC LIMIT 1 - """, (node_id, node_id)) - channel_row = cur2.fetchone() - if channel_row and channel_row[0] is not None: - record["channel"] = channel_row[0] - else: - record["channel"] = None - cur2.close() - # --- End add --- - node_id_hex = utils.convert_node_id_from_int_to_hex(row[0]) - nodes[node_id_hex] = record + # Process telemetry data + telemetry = type('Telemetry', (), {})() # Create an object with attributes + telemetry_fields = [ + 'air_util_tx', 'battery_level', 'channel_utilization', + 'uptime_seconds', 'voltage', 'temperature', 'relative_humidity', + 'barometric_pressure', 'gas_resistance', 'current', + 'telemetry_time', 'telemetry_channel' + ] + # Initialize all telemetry fields to None first + for field in telemetry_fields: + setattr(telemetry, field, None) + # Then set values from row if they exist + for field in telemetry_fields: + if field in row and row[field] is not None: + setattr(telemetry, field, row[field]) + record['telemetry'] = telemetry + + # Process position data + position = type('Position', (), {})() # Create an object with attributes + position_fields = [ + 'altitude', 'ground_speed', 'ground_track', 'latitude_i', + 'longitude_i', 'location_source', 'precision_bits', + 'position_time', 'geocoded' + ] + # Initialize all position fields to None first + for field in position_fields: + setattr(position, field, None) + # Then set values from row if they exist + for field in position_fields: + if field in row and row[field] is not None: + setattr(position, field, row[field]) + + # Always set latitude and longitude attributes + position.latitude = position.latitude_i / 10000000 if position.latitude_i else None + position.longitude = position.longitude_i / 10000000 if position.longitude_i else None + record['position'] = position + + # Get neighbors data + record['neighbors'] = self.get_neighbors(row['id']) + + # Set other fields + record['role'] = record.get('role', 0) + record['active'] = bool(record.get('is_active', 0)) + record['last_seen'] = utils.time_since(record['ts_seen']) + record['channel'] = row.get('channel') + + try: + # Convert node ID to hex string and ensure it's properly formatted + node_id_hex = utils.convert_node_id_from_int_to_hex(row['id']) + if node_id_hex: # Only add if conversion was successful + nodes[node_id_hex] = record + except (TypeError, ValueError) as e: + logging.error(f"Error converting node ID {row['id']} to hex: {e}") + continue + print("Skipped rows due to missing id:", skipped) + print("Final nodes count:", len(nodes)) cur.close() return nodes diff --git a/templates/map.html.j2 b/templates/map.html.j2 index dc69aba2..7b5df395 100644 --- a/templates/map.html.j2 +++ b/templates/map.html.j2 @@ -292,31 +292,32 @@ {% set lat = node.position.latitude_i / 10000000 %} {% set lon = node.position.longitude_i / 10000000 %} {% if -90 <= lat <= 90 and -180 <= lon <= 180 %} - nodes['{{ id }}'] = { - id: '{{ id }}', - short_name: '{{ node.short_name }}', - long_name: '{{ node.long_name | e }}', - last_seen: '{{ time_ago(node.ts_seen) }}', - ts_seen: {{ node.ts_seen }}, - ts_uplink: {% if node.ts_uplink is defined and node.ts_uplink is not none %}{{ node.ts_uplink }}{% else %}null{% endif %}, - position: [{{ lon }}, {{ lat }}], - online: {% if node.active %}true{% else %}false{% endif %}, - channel: {% if node.channel is not none %}{{ node.channel }}{% else %}null{% endif %}, - zero_hop_data: {{ zero_hop_data.get(id, {'heard': [], 'heard_by': []}) | tojson | safe }} - }; - {% if node.neighbors %} - nodes['{{ id }}'].neighbors = [ - {% for neighbor in node.neighbors %} - { - id: '{{ utils.convert_node_id_from_int_to_hex(neighbor.neighbor_id) }}', - snr: '{{ neighbor.snr }}', - distance: '{{ neighbor.distance }}', - }, - {% endfor %} - ]; - {% else %} - nodes['{{ id }}'].neighbors = []; - {% endif %} + nodes['{{ id }}'] = { + id: '{{ id }}', + short_name: '{{ node.short_name }}', + long_name: '{{ node.long_name | e }}', + last_seen: '{{ time_ago(node.ts_seen) }}', + ts_seen: {{ node.ts_seen }}, + ts_uplink: {% if node.ts_uplink is defined and node.ts_uplink is not none %}{{ node.ts_uplink }}{% else %}null{% endif %}, + position: [{{ lon }}, {{ lat }}], + online: {% if node.active %}true{% else %}false{% endif %}, + channel: {% if node.channel is not none %}{{ node.channel }}{% else %}null{% endif %}, + zero_hop_data: {{ zero_hop_data.get(id, {'heard': [], 'heard_by': []}) | tojson | safe }} + }; + {% if node.neighbors %} + nodes['{{ id }}'].neighbors = [ + {% for neighbor in node.neighbors %} + { + id: '{{ utils.convert_node_id_from_int_to_hex(neighbor.neighbor_id) }}', + snr: '{{ neighbor.snr }}', + distance: '{{ neighbor.distance }}', + }, + {% endfor %} + ]; + {% else %} + nodes['{{ id }}'].neighbors = []; + {% endif %} + {% endif %} {% endif %} {% endfor %} diff --git a/templates/mynodes.html.j2 b/templates/mynodes.html.j2 index 26d805ac..01cb5019 100644 --- a/templates/mynodes.html.j2 +++ b/templates/mynodes.html.j2 @@ -86,7 +86,7 @@
Telemetry
- {% if node.telemetry.battery_level %} + {% if node.telemetry.battery_level is not none %}
Battery @@ -94,7 +94,7 @@
{% endif %} - {% if node.telemetry.voltage %} + {% if node.telemetry.voltage is not none %}
Voltage @@ -102,7 +102,7 @@
{% endif %} - {% if node.telemetry.air_util_tx %} + {% if node.telemetry.air_util_tx is not none %}
Air Util TX @@ -110,7 +110,7 @@
{% endif %} - {% if node.telemetry.channel_utilization %} + {% if node.telemetry.channel_utilization is not none %}
Channel Util diff --git a/templates/node.html.j2 b/templates/node.html.j2 index 63225157..174971d9 100644 --- a/templates/node.html.j2 +++ b/templates/node.html.j2 @@ -23,7 +23,7 @@ {% if node.telemetry %} - {% if node.telemetry.air_util_tx %} + {% if node.telemetry.air_util_tx is not none %} Air Util TX {{ node.telemetry.air_util_tx | round(1) }}% @@ -31,7 +31,7 @@ {% endif %} - {% if node.telemetry.channel_utilization %} + {% if node.telemetry.channel_utilization is not none %} Channel Util {{ node.telemetry.channel_utilization | round(1) }}% @@ -39,7 +39,7 @@ {% endif %} - {% if node.telemetry.battery_level %} + {% if node.telemetry.battery_level is not none %} Battery {{ node.telemetry.battery_level | round(0) }}% @@ -47,7 +47,7 @@ {% endif %} - {% if node.telemetry.voltage %} + {% if node.telemetry.voltage is not none %} Voltage {% if node.telemetry.voltage is number %} @@ -305,19 +305,19 @@ {% if nid in nodes %} - {% set dist = utils.calculate_distance_between_nodes(nodes[nid], node) %} - {% if dist %} - {% if nodes[nid].position and nodes[nid].position.latitude_i and nodes[nid].position.longitude_i and node.position and node.position.latitude_i and node.position.longitude_i %} - - {{ dist }} km 🏔️ - - {% else %} - {{ dist }} km + {% if nodes[nid].position and node.position %} + {% if nodes[nid].position.latitude_i is not none and nodes[nid].position.longitude_i is not none and node.position.latitude_i is not none and node.position.longitude_i is not none %} + {% set dist = utils.calculate_distance_between_nodes(nodes[nid], node) %} + {% if dist %} + + {{ dist|round(2) }} km 🏔️ + + {% endif %} + {% endif %} {% endif %} {% endif %} - {% endif %} N @@ -349,10 +349,10 @@ - {{ dist }} km 🏔️ + {{ dist|round(2) }} km 🏔️ {% else %} - {{ dist }} km + {{ dist|round(2) }} km {% endif %} {% endif %} {% endif %} @@ -398,20 +398,20 @@ SNR: {{ neighbor.snr }} - {% if nid in nodes %} - {% set dist = utils.calculate_distance_between_nodes(nodes[nid], node) %} - {% if dist %} - {% if nodes[nid].position and nodes[nid].position.latitude_i and nodes[nid].position.longitude_i and node.position and node.position.latitude_i and node.position.longitude_i %} - - {{ dist }} km 🏔️ - - {% else %} - {{ dist }} km + {% if id in nodes %} + {% if nodes[id].position and node.position %} + {% if nodes[id].position.latitude_i is not none and nodes[id].position.longitude_i is not none and node.position.latitude_i is not none and node.position.longitude_i is not none %} + {% set dist = utils.calculate_distance_between_nodes(nodes[id], node) %} + {% if dist %} + + {{ dist|round(2) }} km 🏔️ + + {% endif %} + {% endif %} {% endif %} {% endif %} - {% endif %} N @@ -445,10 +445,10 @@ - {{ dist }} km 🏔️ + {{ dist|round(2) }} km 🏔️ {% else %} - {{ dist }} km + {{ dist|round(2) }} km {% endif %} {% endif %} {% endif %} diff --git a/templates/node_table.html.j2 b/templates/node_table.html.j2 index 2ecbe224..36d9944f 100644 --- a/templates/node_table.html.j2 +++ b/templates/node_table.html.j2 @@ -133,12 +133,12 @@ {% endif %} {% if node.telemetry %} - {% if 'battery_level' in node.telemetry %} + {% if node.telemetry and node.telemetry.battery_level is not none %} {{ node.telemetry.battery_level }}% {% endif %} - {% if 'voltage' in node.telemetry %} + {% if node.telemetry and node.telemetry.voltage is not none %} {% if node.telemetry.voltage is number %} {{ node.telemetry.voltage|round(2) }}V {% else %} @@ -147,7 +147,7 @@ {% endif %} - {% if 'air_util_tx' in node.telemetry %} + {% if node.telemetry and node.telemetry.air_util_tx is not none %} {% if node.telemetry.air_util_tx is number %} {{ node.telemetry.air_util_tx|round(1) }}% {% else %} @@ -156,7 +156,7 @@ {% endif %} - {% if 'channel_utilization' in node.telemetry %} + {% if node.telemetry and node.telemetry.channel_utilization is not none %} {% if node.telemetry.channel_utilization is number %} {{ node.telemetry.channel_utilization|round(1) }}% {% else %} diff --git a/templates/nodes.html.j2 b/templates/nodes.html.j2 index 16acbe4a..375cc42e 100644 --- a/templates/nodes.html.j2 +++ b/templates/nodes.html.j2 @@ -4,13 +4,15 @@ {% block title %}Nodes | MeshInfo{% endblock %} {% block content %} -{% if latest %} -{% set lnodeid = utils.convert_node_id_from_int_to_hex(latest["id"]) %} +{% if latest and latest.id is not none %} +{% set lnodeid = utils.convert_node_id_from_int_to_hex(latest.id) %} +{% if lnodeid and lnodeid in nodes %} {% set lnode = nodes[lnodeid] %}
📌 Welcome to our newest node, {{ lnode.long_name }} ({{ lnode.short_name }}).👋
{% endif %} +{% endif %}
{{ this_page.title() }}
{% include 'node_search.html.j2' %} diff --git a/utils.py b/utils.py index c67c5480..6c2ad25f 100644 --- a/utils.py +++ b/utils.py @@ -31,25 +31,28 @@ def distance_between_two_points(lat1, lon1, lat2, lon2): def calculate_distance_between_nodes(node1, node2): - """Calculate the distance between two nodes, ensuring data integrity.""" - if not node1 or not node2: - return None - if not node1.get("position") or not node2.get("position"): + """Calculate distance between two nodes in kilometers.""" + # Handle both dictionary and object access + pos1 = node1.get("position") if isinstance(node1, dict) else node1.position + pos2 = node2.get("position") if isinstance(node2, dict) else node2.position + + if not pos1 or not pos2: return None - if any( - key not in node1["position"] or key not in node2["position"] - or node1["position"][key] is None or node2["position"][key] is None - for key in ["latitude_i", "longitude_i"] - ): + + # Handle both dictionary and object access for latitude_i and longitude_i + lat1 = pos1.get("latitude_i") if isinstance(pos1, dict) else pos1.latitude_i + lon1 = pos1.get("longitude_i") if isinstance(pos1, dict) else pos1.longitude_i + lat2 = pos2.get("latitude_i") if isinstance(pos2, dict) else pos2.latitude_i + lon2 = pos2.get("longitude_i") if isinstance(pos2, dict) else pos2.longitude_i + + if lat1 is None or lon1 is None or lat2 is None or lon2 is None: return None - return round( - distance_between_two_points( - node1["position"]["latitude_i"] / 10000000, - node1["position"]["longitude_i"] / 10000000, - node2["position"]["latitude_i"] / 10000000, - node2["position"]["longitude_i"] / 10000000, - ), - 2, + + return distance_between_two_points( + lat1 / 1e7, + lon1 / 1e7, + lat2 / 1e7, + lon2 / 1e7 ) From 74156ebd3776b53d6c8544d7b367e1f6188b95fd Mon Sep 17 00:00:00 2001 From: agessaman Date: Tue, 27 May 2025 03:20:21 +0000 Subject: [PATCH 035/155] Add traceroute improvements and update mesh data handling --- meshdata.py | 248 +++++++--- meshinfo_web.py | 11 +- meshtastic_support.py | 2 + migrations/__init__.py | 4 +- migrations/add_traceroute_improvements.py | 75 +++ templates/traceroutes.html.j2 | 532 +++++++++++++--------- 6 files changed, 590 insertions(+), 282 deletions(-) create mode 100644 migrations/add_traceroute_improvements.py diff --git a/meshdata.py b/meshdata.py index f9f09319..772181fa 100644 --- a/meshdata.py +++ b/meshdata.py @@ -311,42 +311,107 @@ def get_neighbors(self, id): return neighbors def get_traceroutes(self, page=1, per_page=25): - """Get paginated traceroutes with SNR information.""" - # Get total count first + """Get paginated traceroutes with SNR information, grouping all attempts (request and reply) together.""" cur = self.db.cursor() - cur.execute("SELECT COUNT(*) FROM traceroute") + cur.execute("SELECT COUNT(DISTINCT request_id) FROM traceroute") total = cur.fetchone()[0] - # Get paginated results with all fields - sql = """SELECT traceroute_id, from_id, to_id, route, route_back, - snr_towards, snr_back, success, channel, hop_limit, ts_created - FROM traceroute - ORDER BY ts_created DESC - LIMIT %s OFFSET %s""" + # Get paginated request_ids + cur.execute(""" + SELECT request_id + FROM traceroute + GROUP BY request_id + ORDER BY MAX(ts_created) DESC + LIMIT %s OFFSET %s + """, (per_page, (page - 1) * per_page)) + request_ids = [row[0] for row in cur.fetchall()] - offset = (page - 1) * per_page - cur.execute(sql, (per_page, offset)) + # Fetch all traceroute rows for these request_ids + if not request_ids: + return { + "items": [], + "page": page, + "per_page": per_page, + "total": total, + "pages": (total + per_page - 1) // per_page, + "has_prev": page > 1, + "has_next": page * per_page < total, + "prev_num": page - 1, + "next_num": page + 1 + } + format_strings = ','.join(['%s'] * len(request_ids)) + sql = f""" + SELECT + t.request_id, + t.from_id, + t.to_id, + t.route, + t.route_back, + t.snr_towards, + t.snr_back, + t.success, + t.channel, + t.hop_limit, + t.ts_created, + t.is_reply, + t.error_reason, + t.attempt_number, + t.traceroute_id + FROM traceroute t + WHERE t.request_id IN ({format_strings}) + ORDER BY t.request_id DESC, t.ts_created ASC + """ + cur.execute(sql, tuple(request_ids)) rows = cur.fetchall() - traceroutes = [] + # Group by request_id + from collections import defaultdict + grouped = defaultdict(list) for row in rows: - traceroutes.append({ - "id": row[0], - "from_id": row[1], - "to_id": row[2], - "route": [int(a) for a in row[3].split(";")] if row[3] else [], - "route_back": [int(a) for a in row[4].split(";")] if row[4] else [], - "snr_towards": [float(s)/4.0 for s in row[5].split(";")] if row[5] else [], - "snr_back": [float(s)/4.0 for s in row[6].split(";")] if row[6] else [], + route = [int(a) for a in row[3].split(";")] if row[3] else [] + route_back = [int(a) for a in row[4].split(";")] if row[4] else [] + snr_towards = [float(s)/4.0 for s in row[5].split(";")] if row[5] else [] + snr_back = [float(s)/4.0 for s in row[6].split(";")] if row[6] else [] + from_id = row[1] + to_id = row[2] + # For zero-hop, do NOT set route or route_back to endpoints; leave as empty lists + grouped[row[0]].append({ + "id": row[14], # Use traceroute_id as the unique id + "from_id": from_id, + "to_id": to_id, + "route": route, + "route_back": route_back, + "snr_towards": snr_towards, + "snr_back": snr_back, "success": row[7], "channel": row[8], "hop_limit": row[9], - "ts_created": row[10].timestamp() + "ts_created": row[10].timestamp(), + "is_reply": row[11], + "error_reason": row[12], + "attempt_number": row[13], + "traceroute_id": row[14] }) + # Prepare items as a list of dicts, each with all attempts for a request_id + items = [] + for req_id in request_ids: + attempts = grouped[req_id] + # Group status logic: prefer success, then error, then incomplete + summary = dict(attempts[0]) + summary['attempts'] = attempts + summary['success'] = any(a['success'] for a in attempts) + summary['error_reason'] = next((a['error_reason'] for a in attempts if a['error_reason']), None) + # If any attempt is successful, set status to success + if summary['success']: + summary['status'] = 'success' + elif summary['error_reason']: + summary['status'] = 'error' + else: + summary['status'] = 'incomplete' + items.append(summary) cur.close() - return { - "items": traceroutes, + "items": items, "page": page, "per_page": per_page, "total": total, @@ -969,52 +1034,91 @@ def store_neighborinfo(self, data): self.db.commit() def store_traceroute(self, data): + import logging from_id = self.verify_node(data["from"]) to_id = self.verify_node(data["to"]) payload = dict(data["decoded"]["json_payload"]) - - # Process forward route and SNR + + # Always use the payload's request_id if present, else fallback to message id + payload_request_id = data["decoded"].get("request_id") + message_id = data.get("id") + request_id = payload_request_id or message_id + + # Determine if this is a reply and get error reason + is_reply = False + error_reason = None + + # Check if this is a routing message with error + if "error_reason" in payload: + error_reason = payload["error_reason"] + + # Check if this is a route reply (legacy Meshtastic format) + if "route_reply" in payload: + is_reply = True + payload = payload["route_reply"] + elif "route_request" in payload: + payload = payload["route_request"] + + # Robust reply detection: if payload_request_id is present and does not match message_id, treat as reply + if payload_request_id and message_id and str(payload_request_id) != str(message_id): + is_reply = True + + # Insert the traceroute row route = None snr_towards = None if "route" in payload: route = ";".join(str(r) for r in payload["route"]) if "snr_towards" in payload: - # Store raw SNR values directly snr_towards = ";".join(str(float(s)) for s in payload["snr_towards"]) - - # Process return route and SNR route_back = None snr_back = None if "route_back" in payload: route_back = ";".join(str(r) for r in payload["route_back"]) if "snr_back" in payload: - # Store raw SNR values directly snr_back = ";".join(str(float(s)) for s in payload["snr_back"]) - # A traceroute is successful if we have either: - # 1. A direct connection with SNR data in both directions - # 2. A multi-hop route with SNR data in both directions - is_direct = not bool(route and route_back) # True if no hops in either direction - - success = False - if is_direct: - # For direct connections, we just need SNR data in both directions - success = bool(snr_towards and snr_back) + # Parse as lists for logic + route_list = payload.get("route", []) + route_back_list = payload.get("route_back", []) + snr_towards_list = payload.get("snr_towards", []) + snr_back_list = payload.get("snr_back", []) + + # Only mark as success if: + # For zero-hop direct connection: + if not route_list and not route_back_list: + # Must have SNR values in either direction + success = bool(snr_towards_list or snr_back_list) + # For multi-hop: + elif route_list and route_back_list: + # Must have SNR values in both directions + success = bool(snr_towards_list and snr_back_list) else: - # For multi-hop routes, we need both routes and their SNR data - success = bool(route and route_back and snr_towards and snr_back) + success = False + + # Now join for DB storage + route = None + snr_towards = None + if "route" in payload: + route = ";".join(str(r) for r in payload["route"]) + if "snr_towards" in payload: + snr_towards = ";".join(str(float(s)) for s in payload["snr_towards"]) + route_back = None + snr_back = None + if "route_back" in payload: + route_back = ";".join(str(r) for r in payload["route_back"]) + if "snr_back" in payload: + snr_back = ";".join(str(float(s)) for s in payload["snr_back"]) + + # --- FIX: Define attempt_number before use --- + attempt_number = data.get("attempt_number") or payload.get("attempt_number") - # Extract additional metadata channel = data.get("channel", None) hop_limit = data.get("hop_limit", None) - request_id = data.get("id", None) traceroute_time = payload.get("time", None) - sql = """INSERT INTO traceroute (from_id, to_id, channel, hop_limit, success, request_id, route, route_back, - snr_towards, snr_back, time, ts_created) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, FROM_UNIXTIME(%s), NOW())""" - + snr_towards, snr_back, time, ts_created, is_reply, error_reason, attempt_number) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, FROM_UNIXTIME(%s), NOW(), %s, %s, %s)""" params = ( from_id, to_id, @@ -1026,11 +1130,38 @@ def store_traceroute(self, data): route_back, snr_towards, snr_back, - traceroute_time + traceroute_time, + is_reply, + error_reason, + attempt_number ) self.db.cursor().execute(sql, params) self.db.commit() + # --- Robust consolidation logic: ensure all related rows use the same request_id --- + cur = self.db.cursor() + ids_to_check = set() + if message_id: + ids_to_check.add(message_id) + if payload_request_id: + ids_to_check.add(payload_request_id) + if request_id: + ids_to_check.add(request_id) + # Find all request_ids for these ids (as traceroute_id or request_id) + all_request_ids = set() + for idval in ids_to_check: + if idval: + cur.execute("SELECT request_id FROM traceroute WHERE traceroute_id = %s OR request_id = %s", (idval, idval)) + all_request_ids.update([row[0] for row in cur.fetchall() if row[0]]) + if all_request_ids: + canonical_request_id = min(all_request_ids) + cur.execute(f"UPDATE traceroute SET request_id = %s WHERE request_id IN ({','.join(['%s']*len(all_request_ids))})", (canonical_request_id, *all_request_ids)) + self.db.commit() + logging.info(f"Consolidated traceroute rows: set request_id={canonical_request_id} for ids {all_request_ids}") + else: + logging.warning(f"No related traceroute rows found for ids: {ids_to_check}") + cur.close() + def get_successful_traceroutes(self): sql = """ SELECT @@ -1547,22 +1678,23 @@ def setup_database(self): import sys migrations_path = os.path.join(os.path.dirname(__file__), 'migrations') sys.path.insert(0, os.path.dirname(__file__)) - import migrations.add_message_reception as add_message_reception - import migrations.add_traceroute_snr as add_traceroute_snr - import migrations.add_traceroute_id as add_traceroute_id - import migrations.add_channel_info as add_channel_info - import migrations.add_ts_uplink as add_ts_uplink - add_message_reception.migrate(self.db) - add_traceroute_snr.migrate(self.db) - add_traceroute_id.migrate(self.db) - add_channel_info.migrate(self.db) - add_ts_uplink.migrate(self.db) + from migrations import MIGRATIONS + + # Run all migrations in order + for migration in MIGRATIONS: + try: + migration(self.db) + logging.info(f"Successfully ran migration: {migration.__name__}") + except Exception as e: + logging.error(f"Failed to run migration {migration.__name__}: {e}") + raise + except ImportError as e: logging.error(f"Failed to import migration module: {e}") # Continue with database setup even if migration fails pass except Exception as e: - logging.error(f"Failed to run migration: {e}") + logging.error(f"Failed to run migrations: {e}") raise self.db.commit() diff --git a/meshinfo_web.py b/meshinfo_web.py index 89ca2118..d87394a1 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -370,12 +370,11 @@ def traceroute_map(): abort(503, description="Database connection unavailable") nodes = md.get_nodes() - # Get traceroute data + # Get traceroute attempt by unique id cursor = md.db.cursor(dictionary=True) cursor.execute(""" SELECT * FROM traceroute WHERE traceroute_id = %s """, (traceroute_id,)) - traceroute_data = cursor.fetchone() if not traceroute_data: abort(404) @@ -383,22 +382,22 @@ def traceroute_map(): # Format the forward route data route = [] if traceroute_data['route']: - route = [int(hop) for hop in traceroute_data['route'].split(';')] + route = [int(hop) for hop in traceroute_data['route'].split(';') if hop] # Format the return route data route_back = [] if traceroute_data['route_back']: - route_back = [int(hop) for hop in traceroute_data['route_back'].split(';')] + route_back = [int(hop) for hop in traceroute_data['route_back'].split(';') if hop] # Format the forward SNR values and scale by dividing by 4 snr_towards = [] if traceroute_data['snr_towards']: - snr_towards = [float(s)/4.0 for s in traceroute_data['snr_towards'].split(';')] + snr_towards = [float(s)/4.0 for s in traceroute_data['snr_towards'].split(';') if s] # Format the return SNR values and scale by dividing by 4 snr_back = [] if traceroute_data['snr_back']: - snr_back = [float(s)/4.0 for s in traceroute_data['snr_back'].split(';')] + snr_back = [float(s)/4.0 for s in traceroute_data['snr_back'].split(';') if s] # Create a clean traceroute object for the template traceroute = { diff --git a/meshtastic_support.py b/meshtastic_support.py index 9f2d8ab5..11db490d 100644 --- a/meshtastic_support.py +++ b/meshtastic_support.py @@ -122,6 +122,7 @@ class Channel(Enum): LONG_FAST = 8 MEDIUM_FAST = 31 SHORT_FAST = 112 + LONG_MODERATE = 88 # Additional channels will be added as they are discovered class ShortChannel(Enum): @@ -132,6 +133,7 @@ class ShortChannel(Enum): LF = 8 MF = 31 SF = 112 + LM = 88 # Additional channels will be added as they are discovered def get_channel_name(channel_value, use_short_names=False): diff --git a/migrations/__init__.py b/migrations/__init__.py index b318b9ce..0c16c312 100644 --- a/migrations/__init__.py +++ b/migrations/__init__.py @@ -3,11 +3,13 @@ from migrations.add_traceroute_snr import migrate as migrate_traceroute_snr from migrations.add_traceroute_id import migrate as migrate_traceroute_id from migrations.add_channel_info import migrate as migrate_channel_info +from migrations.add_traceroute_improvements import migrate as migrate_traceroute_improvements # List of migrations to run in order MIGRATIONS = [ migrate_message_reception, migrate_traceroute_snr, migrate_traceroute_id, - migrate_channel_info + migrate_channel_info, + migrate_traceroute_improvements ] diff --git a/migrations/add_traceroute_improvements.py b/migrations/add_traceroute_improvements.py new file mode 100644 index 00000000..f523c36a --- /dev/null +++ b/migrations/add_traceroute_improvements.py @@ -0,0 +1,75 @@ +import logging +import mysql.connector +from mysql.connector import Error + +def migrate(db): + """ + Add improvements to traceroute table: + - request_id to group related attempts + - is_reply to distinguish requests from replies + - error_reason to track failure reasons + - attempt_number to track attempt sequence + """ + cursor = None + try: + cursor = db.cursor() + + # Check which columns already exist + cursor.execute("SHOW COLUMNS FROM traceroute") + existing_columns = [column[0] for column in cursor.fetchall()] + + # Build ALTER TABLE statement only for missing columns + alter_statements = [] + if 'request_id' not in existing_columns: + alter_statements.append("ADD COLUMN request_id BIGINT") + if 'is_reply' not in existing_columns: + alter_statements.append("ADD COLUMN is_reply BOOLEAN DEFAULT FALSE") + if 'error_reason' not in existing_columns: + alter_statements.append("ADD COLUMN error_reason INT") + if 'attempt_number' not in existing_columns: + alter_statements.append("ADD COLUMN attempt_number INT") + + # Only execute ALTER TABLE if we have columns to add + if alter_statements: + alter_sql = f"ALTER TABLE traceroute {', '.join(alter_statements)}" + cursor.execute(alter_sql) + + # Check if index exists before creating it + cursor.execute("SHOW INDEX FROM traceroute WHERE Key_name = 'idx_traceroute_request_id'") + if not cursor.fetchone(): + cursor.execute(""" + CREATE INDEX idx_traceroute_request_id + ON traceroute(request_id) + """) + + # Update existing records to use message_id as request_id if needed + if 'request_id' not in existing_columns: + cursor.execute(""" + UPDATE traceroute + SET request_id = message_id + WHERE request_id IS NULL + """) + + # Calculate attempt numbers for existing records if needed + if 'attempt_number' not in existing_columns: + cursor.execute(""" + UPDATE traceroute t1 + JOIN ( + SELECT request_id, COUNT(*) as attempt_count + FROM traceroute + GROUP BY request_id + ) t2 ON t1.request_id = t2.request_id + SET t1.attempt_number = t2.attempt_count + WHERE t1.attempt_number IS NULL + """) + + db.commit() + logging.info("Successfully added traceroute improvements") + + except Error as e: + logging.error(f"Error during traceroute improvements migration: {e}") + db.rollback() + raise + finally: + if cursor: + cursor.close() \ No newline at end of file diff --git a/templates/traceroutes.html.j2 b/templates/traceroutes.html.j2 index b96ad5a7..75a8c86b 100644 --- a/templates/traceroutes.html.j2 +++ b/templates/traceroutes.html.j2 @@ -59,18 +59,46 @@ window.addEventListener('load', function() { Route Hops Ch - {# Success #} + Status + Attempts Actions {% for item in traceroutes %} - - {{ format_timestamp(item.ts_created) }} + {% set successful_attempts = item.attempts | selectattr('success') | list %} + {% set best_attempt = (successful_attempts | selectattr('route_back', '!=', []) | list | first) or (successful_attempts | first) or item.attempts[0] %} + {% set is_zero_hop = (best_attempt.route|length == 0 and best_attempt.snr_towards|length > 0) %} + {% set any_success = item.attempts | selectattr('success') | list | length > 0 %} + {% set best_successful_reply = (item.attempts | selectattr('is_reply') | selectattr('success') | list | first) %} + + + {{ format_timestamp(item.ts_created) }} + {% if item.first_attempt and item.last_attempt and item.first_attempt != item.last_attempt %} +
+ + First: {{ format_timestamp(item.first_attempt) }}
+ Last: {{ format_timestamp(item.last_attempt) }} + + {% endif %} + {% set fnodeid = utils.convert_node_id_from_int_to_hex(item["from_id"]) %} {% if fnodeid in nodes %} - {{ nodes[fnodeid].short_name }} +
+ {% if best_attempt.snr_towards %}SNR: {{ '%.1f'|format(best_attempt.snr_towards[0]) }}dB
{% endif %} + {% if nodes[fnodeid].hw_model %}HW: {{ nodes[fnodeid].hw_model|safe_hw_model }}
{% endif %} + {% if nodes[fnodeid].firmware_version %}FW: {{ nodes[fnodeid].firmware_version }}
{% endif %} + {% if nodes[fnodeid].role is not none %}Role: {{ utils.get_role_name(nodes[fnodeid].role) }}
{% endif %} + {% if nodes[fnodeid].owner_username %}Owner: {{ nodes[fnodeid].owner_username }}
{% endif %} + Last Seen: {{ time_ago(nodes[fnodeid].ts_seen) }} +
"> + {{ nodes[fnodeid].short_name }} + {% else %} UNK {% endif %} @@ -78,263 +106,305 @@ window.addEventListener('load', function() { {% set tnodeid = utils.convert_node_id_from_int_to_hex(item["to_id"]) %} {% if tnodeid in nodes %} - {{ nodes[tnodeid].short_name }} +
+ {% if nodes[tnodeid].hw_model %}HW: {{ nodes[tnodeid].hw_model|safe_hw_model }}
{% endif %} + {% if nodes[tnodeid].firmware_version %}FW: {{ nodes[tnodeid].firmware_version }}
{% endif %} + {% if nodes[tnodeid].role is not none %}Role: {{ utils.get_role_name(nodes[tnodeid].role) }}
{% endif %} + {% if nodes[tnodeid].owner_username %}Owner: {{ nodes[tnodeid].owner_username }}
{% endif %} + Last Seen: {{ time_ago(nodes[tnodeid].ts_seen) }} +
"> + {{ nodes[tnodeid].short_name }} + {% else %} UNK {% endif %} - {# Forward Route #} + {# Outbound and Inbound Route for best successful reply, else fallback to best_attempt #}
- Outbound:
- {# Source node #} - {% if fnodeid in nodes %} - - {% if item.route|length == 0 and item.snr_towards %}{% endif %} - {{ nodes[fnodeid].short_name }} - - {% else %} - - {{ "UNK" }} - - {% endif %} - - {# Forward hops #} - {% for hop in item.route %} - ⇢ - {% set hnodeid = utils.convert_node_id_from_int_to_hex(hop) %} - {% set hnode = nodes[hnodeid] if hnodeid in nodes else None %} - {% if hnode %} - - {% if item.snr_towards and loop.index0 < item.snr_towards|length %}{% endif %} - {{ hnode.short_name }} - - {% else %} - - {{ "UNK" }} - - {% endif %} - {% endfor %} - - {# Destination node for forward path #} - ⇢ - {% if tnodeid in nodes %} - - {{ nodes[tnodeid].short_name }} - - {% else %} - - {{ "UNK" }} - - {% endif %} -
-
- - {# Return Route - only show if exists #} - {% if item.route_back %} -
- Return: -
- {# Return source (destination node) #} - {% if tnodeid in nodes %} - + Outbound: +
+ {% if srcid in nodes %} +
+ {% if show_attempt.snr_towards %}SNR: {{ '%.1f'|format(show_attempt.snr_towards[0]) }}dB
{% endif %} + {% if nodes[srcid].hw_model %}HW: {{ nodes[srcid].hw_model|safe_hw_model }}
{% endif %} + {% if nodes[srcid].firmware_version %}FW: {{ nodes[srcid].firmware_version }}
{% endif %} + {% if nodes[srcid].role is not none %}Role: {{ utils.get_role_name(nodes[srcid].role) }}
{% endif %} + {% if nodes[srcid].owner_username %}Owner: {{ nodes[srcid].owner_username }}
{% endif %} + Last Seen: {{ time_ago(nodes[srcid].ts_seen) }} +
"> + {{ nodes[srcid].short_name }} + + {% else %}UNK{% endif %} + {% for hop in show_attempt.route %} + → + {% set hopid = utils.convert_node_id_from_int_to_hex(hop) %} + {% if hopid in nodes %} +
+ {% if show_attempt.snr_towards and loop.index < show_attempt.snr_towards|length %}SNR: {{ '%.1f'|format(show_attempt.snr_towards[loop.index]) }}dB
{% endif %} + {% if nodes[hopid].hw_model %}HW: {{ nodes[hopid].hw_model|safe_hw_model }}
{% endif %} + {% if nodes[hopid].firmware_version %}FW: {{ nodes[hopid].firmware_version }}
{% endif %} + {% if nodes[hopid].role is not none %}Role: {{ utils.get_role_name(nodes[hopid].role) }}
{% endif %} + {% if nodes[hopid].owner_username %}Owner: {{ nodes[hopid].owner_username }}
{% endif %} + Last Seen: {{ time_ago(nodes[hopid].ts_seen) }} +
"> + {% if show_attempt.snr_towards and loop.index < show_attempt.snr_towards|length %}{% endif %} + {{ nodes[hopid].short_name }} + + {% else %}UNK{% endif %} + {% endfor %} + {# Only show destination if successful #} + {% if show_attempt.success and dstid in nodes %} + → +
+ {% if nodes[dstid].hw_model %}HW: {{ nodes[dstid].hw_model|safe_hw_model }}
{% endif %} + {% if nodes[dstid].firmware_version %}FW: {{ nodes[dstid].firmware_version }}
{% endif %} + {% if nodes[dstid].role is not none %}Role: {{ utils.get_role_name(nodes[dstid].role) }}
{% endif %} + {% if nodes[dstid].owner_username %}Owner: {{ nodes[dstid].owner_username }}
{% endif %} + Last Seen: {{ time_ago(nodes[dstid].ts_seen) }} +
"> + {{ nodes[dstid].short_name }} + + {% endif %} +
+
+ {# Return Route: Only if successful and route_back is non-empty #} + {% if show_attempt.success and show_attempt.route_back|length > 0 %} + "> - {% if item.route_back|length == 0 and item.snr_back %}{% endif %} - {{ nodes[tnodeid].short_name }} + {{ nodes[dstid].short_name }} - {% else %} - - {{ "UNK" }} - - {% endif %} - - {# Return hops #} - {% for hop in item.route_back %} - ⇢ - {% set hnodeid = utils.convert_node_id_from_int_to_hex(hop) %} - {% set hnode = nodes[hnodeid] if hnodeid in nodes else None %} - {% if hnode %} -
+ {% if nodes[hopid].hw_model %}HW: {{ nodes[hopid].hw_model|safe_hw_model }}
{% endif %} + {% if nodes[hopid].firmware_version %}FW: {{ nodes[hopid].firmware_version }}
{% endif %} + {% if nodes[hopid].role is not none %}Role: {{ utils.get_role_name(nodes[hopid].role) }}
{% endif %} + {% if nodes[hopid].owner_username %}Owner: {{ nodes[hopid].owner_username }}
{% endif %} + Last Seen: {{ time_ago(nodes[hopid].ts_seen) }}
"> - {% if item.snr_back and loop.index0 < item.snr_back|length %}{% endif %} - {{ hnode.short_name }} + {{ nodes[hopid].short_name }} - {% else %} - - {{ "UNK" }} - - {% endif %} + {% else %}UNK{% endif %} {% endfor %} - - {# Return destination (source node) #} - ⇢ - {% if fnodeid in nodes %} -
+ {% if nodes[srcid].hw_model %}HW: {{ nodes[srcid].hw_model|safe_hw_model }}
{% endif %} + {% if nodes[srcid].firmware_version %}FW: {{ nodes[srcid].firmware_version }}
{% endif %} + {% if nodes[srcid].role is not none %}Role: {{ utils.get_role_name(nodes[srcid].role) }}
{% endif %} + {% if nodes[srcid].owner_username %}Owner: {{ nodes[srcid].owner_username }}
{% endif %} + Last Seen: {{ time_ago(nodes[srcid].ts_seen) }}
"> - {% if item.route_back|length == 0 and item.snr_back %}{% endif %} - {{ nodes[fnodeid].short_name }} + {{ nodes[srcid].short_name }} - {% else %} - - {{ "UNK" }} - - {% endif %} -
+ {% else %}UNK{% endif %} +
+ {% endif %}
- {% endif %} +
- {% if item.route_back %} + {% if best_attempt.route_back %}
- Out: {{ item.route|length }} + Out: {{ best_attempt.route|length }}
- Return: {{ item.route_back|length }} + Return: {{ best_attempt.route_back|length }}
{% else %} - {{ item.route|length }} + {{ best_attempt.route|length }} {% endif %} - {% if item.channel is not none %} - - {{ utils.get_channel_name(item.channel, use_short_names=True) }} + {% if best_attempt.channel is not none %} + + {{ utils.get_channel_name(best_attempt.channel, use_short_names=True) }} {% endif %} - {# - {% if item.success %} - + + {% if any_success %} + + Success + + {% elif best_attempt.error_reason %} + + Error {{ best_attempt.error_reason }} + + {% else %} + + Incomplete + {% endif %} - #} + + + {% if item.attempts|length > 1 %} + + {{ item.attempts|length }} attempts + + {% endif %} + {% set from_id_hex = utils.convert_node_id_from_int_to_hex(item["from_id"]) %} {% set to_id_hex = utils.convert_node_id_from_int_to_hex(item["to_id"]) %} {% if from_id_hex in nodes and to_id_hex in nodes and - nodes[from_id_hex].position and nodes[to_id_hex].position %} - + nodes[from_id_hex].position and nodes[from_id_hex].position.latitude is not none and nodes[from_id_hex].position.longitude is not none and + nodes[to_id_hex].position and nodes[to_id_hex].position.latitude is not none and nodes[to_id_hex].position.longitude is not none %} + Map {% endif %} + + +
+ Traceroute Attempts: + + + + + + + + + + + + + + + {% for attempt in item.attempts %} + {% set has_successful_reply = item.attempts | selectattr('is_reply') | selectattr('success') | list | length > 0 %} + + + + + + + + + + + + {% endfor %} + +
#TypeTimestampStatusErrorRouteReturn RouteChannel
{{ loop.index }}{% if attempt.is_reply %}Reply{% else %}Request{% endif %}{{ format_timestamp(attempt.ts_created) }} + {% if attempt.is_reply %} + {% if attempt.success or (attempt.is_reply and attempt.route|length == 0 and attempt.snr_towards|length > 0) %} + Success (reply) + {% else %} + Incomplete (reply) + {% endif %} + {% else %} + {% if attempt.success %} + Success + {% elif attempt.error_reason %} + Error + {% else %} + Request + {% endif %} + {% endif %} + + {% set error_map = {0: 'Timeout', 1: 'No Route', 2: 'Network Error', 3: 'Other'} %} + {% if attempt.error_reason is not none %} + {{ error_map.get(attempt.error_reason, attempt.error_reason) }} + {% else %}-{% endif %} + + {# Zero-hop: SRC → DST with SNR #} + {% set srcid = utils.convert_node_id_from_int_to_hex(attempt.from_id) %} + {% set dstid = utils.convert_node_id_from_int_to_hex(attempt.to_id) %} + {% if attempt.route|length == 0 and attempt.snr_towards|length > 0 %} + {% if srcid in nodes %}{{ nodes[srcid].short_name }}{% else %}UNK{% endif %} + → + {% if dstid in nodes %}{{ nodes[dstid].short_name }}{% else %}UNK{% endif %} + {% elif attempt.route|length > 0 %} + {% for hop in attempt.route %} + {% set hopid = utils.convert_node_id_from_int_to_hex(hop) %} + {% if hopid in nodes %} + {{ nodes[hopid].short_name }} + {% else %} + UNK + {% endif %} + {% if not loop.last %} → {% endif %} + {% endfor %} + {% else %}-{% endif %} + + {# For reply, if zero-hop, show return as target → initiator with SNR #} + {% set srcid = utils.convert_node_id_from_int_to_hex(attempt.from_id) %} + {% set dstid = utils.convert_node_id_from_int_to_hex(attempt.to_id) %} + {% if attempt.is_reply and attempt.route|length == 0 and attempt.snr_towards|length > 0 %} + {# Show as destination → source for zero-hop reply #} + {% if dstid in nodes %}{{ nodes[dstid].short_name }}{% else %}UNK{% endif %} + → + {% if srcid in nodes %}{{ nodes[srcid].short_name }}{% else %}UNK{% endif %} + {% elif attempt.route_back|length == 0 and attempt.snr_back|length > 0 %} + {% if dstid in nodes %}{{ nodes[dstid].short_name }}{% else %}UNK{% endif %} + → + {% if srcid in nodes %}{{ nodes[srcid].short_name }}{% else %}UNK{% endif %} + {% elif attempt.route_back|length > 0 %} + {% for hop in attempt.route_back %} + {% set hopid = utils.convert_node_id_from_int_to_hex(hop) %} + {% if hopid in nodes %} + {{ nodes[hopid].short_name }} + {% else %} + UNK + {% endif %} + {% if not loop.last %} → {% endif %} + {% endfor %} + {% else %}-{% endif %} + {{ attempt.channel }} + + + +
+
+ + {% endfor %} @@ -379,4 +449,32 @@ window.addEventListener('load', function() {
{% endif %}
+ + {% endblock %} \ No newline at end of file From b68b4831f04d93dd5d4a39a4c730293c90d3f331 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 30 May 2025 02:35:31 +0000 Subject: [PATCH 036/155] Enhance traceroute map display with improved node information and position tracking. Added HTML escaping for node details, updated popover content to include position timestamps, and refined map interaction handling. Adjusted styles for better visual clarity and responsiveness. --- templates/traceroute_map.html.j2 | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/templates/traceroute_map.html.j2 b/templates/traceroute_map.html.j2 index 63daec27..cfc7e3a5 100644 --- a/templates/traceroute_map.html.j2 +++ b/templates/traceroute_map.html.j2 @@ -312,6 +312,12 @@ center: ol.proj.fromLonLat([0, 0]), zoom: 2, }), + // Add event handling configuration + eventHandling: { + touchstart: { passive: true }, + touchmove: { passive: true }, + touchend: { passive: true } + } }); var forwardLineStyle = new ol.style.Style({ @@ -1114,7 +1120,7 @@ for (var i = 0; i < allpoints.length - 1; i++) { #map { height: 70vh; width: 100%; - margin-bottom: 60px; // Add margin to prevent legend overlap + margin-bottom: 60px; /* Add margin to prevent legend overlap */ } #legend { background-color: #ffffff; From a8764b5dd4f69fb8fe4ac77528e5cb604fec584e Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 30 May 2025 02:36:00 +0000 Subject: [PATCH 037/155] Add position freshness check for nodes in traceroute display. Enhanced node information popovers to include position timestamps and updated styles for badges. Improved map interaction and responsiveness. --- meshdata.py | 20 + meshinfo_web.py | 32 + templates/traceroute_map.html.j2 | 1183 +++++++++++++++--------------- templates/traceroutes.html.j2 | 4 +- www/css/meshinfo.css | 11 + 5 files changed, 670 insertions(+), 580 deletions(-) diff --git a/meshdata.py b/meshdata.py index 772181fa..6bc7b546 100644 --- a/meshdata.py +++ b/meshdata.py @@ -2152,6 +2152,26 @@ def get_neighbors_data(self, view_type='neighbor_info', days=1, zero_hop_timeout return active_nodes_data + def is_position_fresh(self, position, prune_threshold, now=None): + """ + Returns True if the position dict/object has a position_time within prune_threshold seconds of now. + """ + if not position: + return False + # Accept both dict and object + position_time = None + if isinstance(position, dict): + position_time = position.get('position_time') + else: + position_time = getattr(position, 'position_time', None) + if not position_time: + return False + if isinstance(position_time, datetime.datetime): + position_time = position_time.timestamp() + if now is None: + now = time.time() + return (now - position_time) <= prune_threshold + def create_database(): config = configparser.ConfigParser() diff --git a/meshinfo_web.py b/meshinfo_web.py index d87394a1..ab957d7a 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -415,6 +415,36 @@ def traceroute_map(): } cursor.close() + + # --- Build traceroute_positions dict for historical accuracy --- + node_ids = set([traceroute['from_id'], traceroute['to_id']] + traceroute['route'] + traceroute['route_back']) + traceroute_positions = {} + ts_created = traceroute['ts_created'] + # If ts_created is a datetime, convert to timestamp + if hasattr(ts_created, 'timestamp'): + ts_created = ts_created.timestamp() + for node_id in node_ids: + pos = md.get_position_at_time(node_id, ts_created) + if not pos and node_id in nodes and nodes[node_id].position: + pos_obj = nodes[node_id].position + # Convert to dict if needed + if hasattr(pos_obj, '__dict__'): + pos = dict(pos_obj.__dict__) + else: + pos = dict(pos_obj) + # Ensure position_time is present and properly formatted + if 'position_time' not in pos or not pos['position_time']: + if hasattr(pos_obj, 'position_time') and pos_obj.position_time: + pt = pos_obj.position_time + if isinstance(pt, datetime.datetime): + pos['position_time'] = pt.timestamp() + else: + pos['position_time'] = pt + else: + pos['position_time'] = None + if pos: + traceroute_positions[node_id] = pos + return render_template( "traceroute_map.html.j2", @@ -422,6 +452,7 @@ def traceroute_map(): config=config, nodes=nodes, traceroute=traceroute, + traceroute_positions=traceroute_positions, # <-- pass to template utils=utils, meshtastic_support=meshtastic_support, datetime=datetime.datetime, @@ -715,6 +746,7 @@ def traceroutes(): meshtastic_support=meshtastic_support, datetime=datetime.datetime, timestamp=datetime.datetime.now(), + meshdata=md # Add meshdata to template context ) diff --git a/templates/traceroute_map.html.j2 b/templates/traceroute_map.html.j2 index cfc7e3a5..4016da63 100644 --- a/templates/traceroute_map.html.j2 +++ b/templates/traceroute_map.html.j2 @@ -62,28 +62,32 @@
{# Source node #} {% if traceroute.from_id_hex in nodes %} + {% set from_id_int = traceroute.from_id %} + {{ nodes[traceroute.from_id_hex].long_name|e }}
{% if traceroute.route|length == 0 and traceroute.snr_towards %} SNR: {{ '%.1f'|format(traceroute.snr_towards[0]) }}dB
{% endif %} {% if nodes[traceroute.from_id_hex].hw_model %} - HW: {{ nodes[traceroute.from_id_hex].hw_model | safe_hw_model }}
+ HW: {{ nodes[traceroute.from_id_hex].hw_model|safe_hw_model|e }}
{% endif %} {% if nodes[traceroute.from_id_hex].firmware_version %} - FW: {{ nodes[traceroute.from_id_hex].firmware_version }}
+ FW: {{ nodes[traceroute.from_id_hex].firmware_version|e }}
{% endif %} {% if nodes[traceroute.from_id_hex].role is not none %} - Role: {{ utils.get_role_name(nodes[traceroute.from_id_hex].role) }}
+ Role: {{ utils.get_role_name(nodes[traceroute.from_id_hex].role)|e }}
{% endif %} {% if nodes[traceroute.from_id_hex].owner_username %} - Owner: {{ nodes[traceroute.from_id_hex].owner_username }}
+ Owner: {{ nodes[traceroute.from_id_hex].owner_username|e }}
{% endif %} - Last Seen: {{ time_ago(nodes[traceroute.from_id_hex].ts_seen) }} -
"> + Last Seen: {{ time_ago(nodes[traceroute.from_id_hex].ts_seen)|e }}
+ {% if from_id_int in traceroute_positions and traceroute_positions[from_id_int]['position_time'] is not none %} + Position as of: {{ format_timestamp(traceroute_positions[from_id_int]['position_time']) }} + {% endif %} +
{% endfilter %}"> {% if traceroute.route|length == 0 and traceroute.snr_towards %}{% endif %} {{ nodes[traceroute.from_id_hex].short_name }} @@ -100,25 +104,31 @@ + {{ hnode.long_name|e }}
+ {% if hop in traceroute_positions and traceroute_positions[hop]['position_time'] is not none %} + Position as of: {{ format_timestamp(traceroute_positions[hop]['position_time']) }}
+ {% endif %} {% if traceroute.snr_towards and loop.index0 < traceroute.snr_towards|length %} - SNR: {{ '%.1f'|format(traceroute.snr_towards[loop.index0]) }}dB
+ SNR: {{ '%.1f'|format(traceroute.snr_towards[loop.index0]) }}dB
{% endif %} {% if hnode.hw_model %} - HW: {{ hnode.hw_model | safe_hw_model }}
+ HW: {{ hnode.hw_model|safe_hw_model|e }}
{% endif %} {% if hnode.firmware_version %} - FW: {{ hnode.firmware_version }}
+ FW: {{ hnode.firmware_version|e }}
{% endif %} {% if hnode.role is not none %} - Role: {{ utils.get_role_name(hnode.role) }}
+ Role: {{ utils.get_role_name(hnode.role)|e }}
{% endif %} {% if hnode.owner_username %} - Owner: {{ hnode.owner_username }}
+ Owner: {{ hnode.owner_username|e }}
+ {% endif %} + Last Seen: {{ time_ago(hnode.ts_seen)|e }}
+ {% if hop in traceroute_positions and traceroute_positions[hop]['position_time'] is not none %} + Position as of: {{ format_timestamp(traceroute_positions[hop]['position_time']) }} {% endif %} - Last Seen: {{ time_ago(hnode.ts_seen) }} -
"> + {% endfilter %}"> {% if traceroute.snr_towards and loop.index0 < traceroute.snr_towards|length %}{% endif %} {{ hnode.short_name }} @@ -130,25 +140,29 @@ {# Destination node for forward path #} ⇢ {% if traceroute.to_id_hex in nodes %} + {% set to_id_int = traceroute.to_id %} + {{ nodes[traceroute.to_id_hex].long_name|e }}
{% if nodes[traceroute.to_id_hex].hw_model %} - HW: {{ nodes[traceroute.to_id_hex].hw_model | safe_hw_model }}
+ HW: {{ nodes[traceroute.to_id_hex].hw_model|safe_hw_model|e }}
{% endif %} {% if nodes[traceroute.to_id_hex].firmware_version %} - FW: {{ nodes[traceroute.to_id_hex].firmware_version }}
+ FW: {{ nodes[traceroute.to_id_hex].firmware_version|e }}
{% endif %} {% if nodes[traceroute.to_id_hex].role is not none %} - Role: {{ utils.get_role_name(nodes[traceroute.to_id_hex].role) }}
+ Role: {{ utils.get_role_name(nodes[traceroute.to_id_hex].role)|e }}
{% endif %} {% if nodes[traceroute.to_id_hex].owner_username %} - Owner: {{ nodes[traceroute.to_id_hex].owner_username }}
+ Owner: {{ nodes[traceroute.to_id_hex].owner_username|e }}
{% endif %} - Last Seen: {{ time_ago(nodes[traceroute.to_id_hex].ts_seen) }} - "> + Last Seen: {{ time_ago(nodes[traceroute.to_id_hex].ts_seen)|e }}
+ {% if to_id_int in traceroute_positions and traceroute_positions[to_id_int]['position_time'] is not none %} + Position as of: {{ format_timestamp(traceroute_positions[to_id_int]['position_time']) }} + {% endif %} + {% endfilter %}"> {{ nodes[traceroute.to_id_hex].short_name }} {% else %} @@ -166,25 +180,25 @@ + {{ nodes[traceroute.to_id_hex].long_name|e }}
{% if traceroute.route_back|length == 0 and traceroute.snr_back %} - SNR: {{ '%.1f'|format(traceroute.snr_back[0]) }}dB
+ SNR: {{ '%.1f'|format(traceroute.snr_back[0]) }}dB
{% endif %} {% if nodes[traceroute.to_id_hex].hw_model %} - HW: {{ nodes[traceroute.to_id_hex].hw_model | safe_hw_model }}
+ HW: {{ nodes[traceroute.to_id_hex].hw_model|safe_hw_model|e }}
{% endif %} {% if nodes[traceroute.to_id_hex].firmware_version %} - FW: {{ nodes[traceroute.to_id_hex].firmware_version }}
+ FW: {{ nodes[traceroute.to_id_hex].firmware_version|e }}
{% endif %} {% if nodes[traceroute.to_id_hex].role is not none %} - Role: {{ utils.get_role_name(nodes[traceroute.to_id_hex].role) }}
+ Role: {{ utils.get_role_name(nodes[traceroute.to_id_hex].role)|e }}
{% endif %} {% if nodes[traceroute.to_id_hex].owner_username %} - Owner: {{ nodes[traceroute.to_id_hex].owner_username }}
+ Owner: {{ nodes[traceroute.to_id_hex].owner_username|e }}
{% endif %} - Last Seen: {{ time_ago(nodes[traceroute.to_id_hex].ts_seen) }} - "> + Last Seen: {{ time_ago(nodes[traceroute.to_id_hex].ts_seen)|e }} + {% endfilter %}"> {% if traceroute.route_back|length == 0 and traceroute.snr_back %}{% endif %} {{ nodes[traceroute.to_id_hex].short_name }} @@ -201,25 +215,28 @@ + {{ hnode.long_name|e }}
{% if traceroute.snr_back and loop.index0 < traceroute.snr_back|length %} - SNR: {{ '%.1f'|format(traceroute.snr_back[loop.index0]) }}dB
+ SNR: {{ '%.1f'|format(traceroute.snr_back[loop.index0]) }}dB
{% endif %} {% if hnode.hw_model %} - HW: {{ hnode.hw_model | safe_hw_model }}
+ HW: {{ hnode.hw_model|safe_hw_model|e }}
{% endif %} {% if hnode.firmware_version %} - FW: {{ hnode.firmware_version }}
+ FW: {{ hnode.firmware_version|e }}
{% endif %} {% if hnode.role is not none %} - Role: {{ utils.get_role_name(hnode.role) }}
+ Role: {{ utils.get_role_name(hnode.role)|e }}
{% endif %} {% if hnode.owner_username %} - Owner: {{ hnode.owner_username }}
+ Owner: {{ hnode.owner_username|e }}
+ {% endif %} + Last Seen: {{ time_ago(hnode.ts_seen)|e }}
+ {% if hop in traceroute_positions and traceroute_positions[hop]['position_time'] is not none %} + Position as of: {{ format_timestamp(traceroute_positions[hop]['position_time']) }} {% endif %} - Last Seen: {{ time_ago(hnode.ts_seen) }} - "> + {% endfilter %}"> {% if traceroute.snr_back and loop.index0 < traceroute.snr_back|length %}{% endif %} {{ hnode.short_name }} @@ -231,25 +248,29 @@ {# Return destination (source node) #} ⇢ {% if traceroute.from_id_hex in nodes %} + {% set from_id_int = traceroute.from_id %} + {{ nodes[traceroute.from_id_hex].long_name|e }}
{% if nodes[traceroute.from_id_hex].hw_model %} - HW: {{ nodes[traceroute.from_id_hex].hw_model | safe_hw_model }}
+ HW: {{ nodes[traceroute.from_id_hex].hw_model|safe_hw_model|e }}
{% endif %} {% if nodes[traceroute.from_id_hex].firmware_version %} - FW: {{ nodes[traceroute.from_id_hex].firmware_version }}
+ FW: {{ nodes[traceroute.from_id_hex].firmware_version|e }}
{% endif %} {% if nodes[traceroute.from_id_hex].role is not none %} - Role: {{ utils.get_role_name(nodes[traceroute.from_id_hex].role) }}
+ Role: {{ utils.get_role_name(nodes[traceroute.from_id_hex].role)|e }}
{% endif %} {% if nodes[traceroute.from_id_hex].owner_username %} - Owner: {{ nodes[traceroute.from_id_hex].owner_username }}
+ Owner: {{ nodes[traceroute.from_id_hex].owner_username|e }}
{% endif %} - Last Seen: {{ time_ago(nodes[traceroute.from_id_hex].ts_seen) }} - "> + Last Seen: {{ time_ago(nodes[traceroute.from_id_hex].ts_seen)|e }}
+ {% if from_id_int in traceroute_positions and traceroute_positions[from_id_int]['position_time'] is not none %} + Position as of: {{ format_timestamp(traceroute_positions[from_id_int]['position_time']) }} + {% endif %} + {% endfilter %}"> {{ nodes[traceroute.from_id_hex].short_name }} {% else %} @@ -301,108 +322,89 @@ - + + {% endblock %} {% block content %} @@ -132,6 +143,24 @@ - + + {% endblock %} {% block content %} @@ -48,39 +59,57 @@ - + + {% endblock %} @@ -169,56 +180,68 @@ {% endif %} - - + + {% endblock %} {% block content %} @@ -474,131 +485,136 @@ {% endif %} +{% endblock %} + {% block content %}
MQTT Messages
@@ -10,6 +25,50 @@ All messages received by MQTT. Only the messages received since this server was last restarted are shown.

+ + +
+
+
+
+
+ + +
+
+ + +
+
+ + +
+
+ Showing all messages + +
+
+
+
+
+
@@ -19,9 +78,9 @@ - + {% for message in logs %} - +
Message
{% if message.ts_created %} {{ format_timestamp(message.ts_created) }} @@ -37,4 +96,266 @@
+ + {% endblock %} \ No newline at end of file diff --git a/templates/mynodes.html.j2 b/templates/mynodes.html.j2 index 0d501b21..917eab4a 100644 --- a/templates/mynodes.html.j2 +++ b/templates/mynodes.html.j2 @@ -77,6 +77,8 @@ {% elif node.role == 8 %}Client Hidden {% elif node.role == 9 %}Lost and Found {% elif node.role == 10 %}ATAK Tracker + {% elif node.role == 11 %}Router Late + {% else %}Unknown ({{ node.role }}) {% endif %} {% endif %} diff --git a/templates/node.html.j2 b/templates/node.html.j2 index 99d7d195..5fae6cb1 100644 --- a/templates/node.html.j2 +++ b/templates/node.html.j2 @@ -248,6 +248,9 @@ {% else %} Unknown {% endif %} + + + @@ -276,8 +279,10 @@ Lost and Found {% elif node.role == 10 %} ATAK Tracker + {% elif node.role == 11 %} + Router Late {% else %} - Unknown + Unknown ({{ node.role }}) {% endif %} {% else %} Unknown diff --git a/templates/node_table.html.j2 b/templates/node_table.html.j2 index 36d9944f..94703d10 100644 --- a/templates/node_table.html.j2 +++ b/templates/node_table.html.j2 @@ -115,6 +115,10 @@ LF {% elif node.role == 10 %} AT + {% elif node.role == 11 %} + RL + {% else %} + ? {% endif %} {% endif %} diff --git a/utils.py b/utils.py index 6c2ad25f..90a5ad2a 100644 --- a/utils.py +++ b/utils.py @@ -173,7 +173,7 @@ def time_since(epoch_timestamp): def active_nodes(nodes): return { - node: dict(nodes[node]) + node: nodes[node] # Return reference instead of copying for node in nodes if nodes[node]["active"] } @@ -199,7 +199,7 @@ def get_role_name(role_value): def get_owner_nodes(nodes, owner): return { - node: dict(nodes[node]) + node: nodes[node] # Return reference instead of copying for node in nodes if nodes[node]["owner"] == owner } diff --git a/www/images/hardware/HELTEC_MESH_POCKET.webp b/www/images/hardware/HELTEC_MESH_POCKET.webp new file mode 100644 index 0000000000000000000000000000000000000000..59635fea16c20610d77e76becd78678a229f5671 GIT binary patch literal 7136 zcmbVwWlS6ZkoDs3P^>Ht#kFYB;w3ks;IIiGr*(9z5t zrc{F$>hrmQm}?O6)lnEYKY-xc@j8C)xl2$kddXn3_{I8}8v1n+_6S?8al)CO%!C?? zS%Zy5*19*qBlof|JkaEIgr|!;99`s?c#$mLLUaZqbZl4gP-T#l%dsQ2Y9Fa^WdcbZE)xv5!82j zstSDa)+3e#lZ0tJHD8H6U)*--z?@+jrzo#e&z`SSx0D%T6yQs6|@m{wHn5gch4;KL6mbcZNv$G$qFqSrtbD{n%F zjG{JBv*VZ8_xk?rYc1$#OHy`~dI+)ENzV=>`{Q$oO<9_Y``&J$Mm^m-9Pg1QQxOs( z*frxhV!^oP2IoDx$2ow+=SOg>0s|7hZsrrzJ@2nuvom8pYCv1VkX0Ogj^|UNa^?j7 zLEg&neV((uLwM&3gLf^kTxnsegT7=_LJQe8W6q694&B}IhRanH08*9RJ z=^`3yH(q6FUFQwKil~$U;b{=XCf=aWBs<+igjlSTW(txDqi|s82jWTW0{g z-IPT2{;ElMlK)*eHqD=iwZzENQ}kCd|DR5B&UdtM{ahUAWgfn#arpl(`7&SfcD8ia z4?HZdYT>hi6=}Vo8TowuM=~N@U|Pb4zDRhl(2Y>5(?T@ zwUiDMg_rWi$-9wJH+{R_IF(hHIj>BmX3E)`Vv0@JAI@D{IX@RaSnl>I;XD1(Kg$&u zTJZ5Jg={-i^@7EEwk%yAjW#JWaw1rmxB9%o=W_-G;RyjDpn&4itlbl2g1jte4>3OT zf-mjW;@djErke&5QK}SamcWz#s@>sNFiRLhVX&j)Yl|Amv?6Mux~tVSE_GJ}8BDtP z*)0E)Xe9RD1YHvkBt*=DIy*&M#B0={Yq=VJqm_5@jl$?61*^$DJKr;*8RBjq3umY8#xc_ra3O{LwLY#Ol>e>bDT=gl!H1r<+vBFO z?fwz7&glHe+`Ogq zvXLN%6O$sq^q~6?6VYN!@M1VTre0E}h~o0!^6kLTTEi(*Ds8gVKbnq$O(lpTxTw6S z@_*zk*2LlB`t-iqe>0W4LyF{8M2nr{`np*fd{%(-8izzOP=7Fc01U$8i6}S1oK8p_ z+C=oGoi^oUh<9?LzCZqQ_JTiQ1jmh&l)I5cGDThC<Z8G}a| z&eyai^g%|CwT{7@Fn6+jM72Y301{<6Cl{h)U0X5*vrq<)-d$p(mn>iMK@L{zcL>5#aLmTDif%a`pnrU#01rOAQ_rb2in_0#mY>!~#j^pXVk@h?!-Xt%jw^<=T%YGci~ZLwyi_2r1M`hKC%Fxba@x%7K z@yMV?F-RUWxx?czf;Jcq@O#3yvY^n4Yl*a}(` zj&*RlxvoK6A5!4HX^Vd>meIwQ@q&t~FHPeYO4+-*s^zq^5-ts!`nSB!>=9KhZ)-65 znImo*kkhzE-R;+ED?AI1YwK72^6{BDa1&ix<)R zk#eD)>{GEWXX=x29xs#aQ|#j6TG*o``0YE`JjO_wmgLj13ra!Yrjq`q%Z3+KLl?X7^H`l!Wj7Ie4y27sZQ|{S3`mbwYk=%ugTHBo zcSCh05?VwI-hl=NUy^w?1j# zSr<;hwMEmao`eQ2RZAd(Hh@z6q)q97-l{ zMA%Z`Zzg;aHZzG_STl2z(ev{q6Y_2u2>&}Nsj%HSuxt*~7?AcY&+)m74nQv#!tK(N z3l6jv?O-*Nw4L$+x*rvfnca+r^+c2&Ld@~IJ}#kg6NL38pG=X?F(44;Y@G zOq;Q?h&6t9Qh15X~ zt=e~7WTN4(aG$!R4HX}w+YlO?1J<(rSi;-Xq-f#T3ZKaQvUqlogte)Imu#UIP}{e1 zbYRq?AI}K(sDdTayWtNr*a94t)}nC1`AJsIv}cGUsLYFsF@Lz#z4|yv+siuT);L-I zqAt&MG$c2Ur&w-p%f+0M5#HxvfH6V{1ea)w~PwX4Fi_1zma0 z12H!Sy+Rj^kt(>Xcw;xSPXLM$8;^*jlE^=hQzNY?w1)3+dPW$%^*wLbotpdlvBmP} zM=-Pc$JQ!5OAm7DvFNJ`dQh&GcEJROc94Vg$13f+cWFiaKbi^09lt)OMCC0W4S<1> z*l}8ohz5Q%$Q)Ca?#CE7?uUl%r0A!UN0#wl(sZg`7x#1J?a7G&jn3P?J*Nb7!#S$I zA;+yliCd-$^`H{cyiXGQ!KNvybeCqFWA@qA#v~-_g4j-{o`iWnwP*@x^8&KatkEuG z*TiSmVW~nDgmU)Yi3iO4iOU zbm>w21`=V{QC-bRtJr}+w{ynct(l?PsjNOEU8x%MKEd-_pw^8vKFxaoKR>mNnl!b~ z#x?)4Q?ZWOLI?@cYo3elg^E(tE%zR6_*wv9LA$y+4=LVA*}kz{H?bT-by8xQVDS zZGhEyJ%6?%>QD022o>g%XswpSi#gM?sCe%)fG}csg0-PWed9g1T7EvidAbDCM$z;d z_7P3pAhnyjDR}Fu@#LDfI|7ClXUC39_Q}H0QM$`!_rZDi;?V}SZ^6`f^fUhDJ%|V~ zGt-fD3EUPaWW)3_oWGS6g1Vb7)I!?t<}Q1cVD4|Kb}3|7XBkSpK5dYth#bw60+%nl zsBUZEvyzEG3Idv0Z@(kqtv`Imr(bR4O$=M@2;`Ul{hRWnE#%fK;!EH=foPm=M*FIb zU|g;i56xSv9>9!hPk-(BgK;?{#1B3yD;fD3O1WU80r{$pH<8!&x^27MTN!v>Rm2_q zSplfvou)DqaKNatGX&jkBIr`;+1fEb>-`=Ppns^ItK09}8pa-(`6{~_5!B%zQkP_!@*IBsk){@hQmxgJoq0f|?^EaCCX@p;ckn04%s zC|9H#nH92qaQD1jPkTy8Ic#LHZviRKE~VH{B3GmcgyTmOEYF~oRHD|70$bqCdf*lIW5$Uhe@eB?Z(45LB)4o z+9&Sux}_1t*74PCxP7Zt$gv7>!avM$uNDVvZg_N{Z2rN4b5WHjKMNqk+?*1 zwI7IhK1-(#@|`68+$J)Zdjk>fj;v=z074z26v64jK+|t1e;$2u7xuPo%{u3z)VmL2!LU z$4JrNaj2wh(z=%to8~=g{&o;TLcYBYxAGzMS$g2*2vu>d2P3yza#Oco@akG9%EqqdKx5GHOsD)C^I=}zYkz=G1P#qx zK+s;?H@R1SsJCz+da>kW8@O0C@rWgYq~(j+N69}HYLp<_V^`&JW||I{Gub=O8cVt} zfgI&z?EEjNYk`qdco|0I4U7d1cwrn-0n^yXiUQni!rjd)@a%VP*v$M@l2N~PjFTU& z5HFBV(HqZ&4O6`(+y;-EiqD+bM+)g4YRExLWTwAKiKG`;{^Sj6xbFoGC_GbinzMUR zP+H+djmZ0rWK9qNGpJ)j28OUUf7R~qrN=qFlep*%YCLicaQPFqT*7CT-N={1o9&g# zj7OWl@=G#fWBnE^XvKz_T&xQn{L8BqI*SXtC)4vx8I*qZeJsi3Eh>6*-sw|8iiy>1 z05$IgB6G|-@cr2Td#I*m>_U}4;@$Ahw6VddyC%Y*M^Od*jSn8{#Mbv|et+6sK`8Gb ziZ``bw|z6f-j#Y?EN_1>#8Nzys)gB6M)`2N+XObB+EZ#lGSj&F+1pRgwX+ zo!g4bjDA}g>O=7`_fbr36$AZvIN8GP*3TGuj!C^Yp}$hLZ#$)|<0!czxK1DNi~?BnCJaIr%SepSRT;4~QFQx)}6*+pznn(+C@s;XsRCGoR3U%$>e^m240OzdK z{Rk9sOA5f+s&M2k!;;Z?-Q^>DCzpBldX++>BVv-_81g4&^=XIE(4DBKw_9w4Z*d3J zQ``z4W>JOnU(R~GRqPB+WTk|}UDaNFaEy$xqBjLL<{PPhk?wP9X2~dp_zMRd3Zi#V zqA~2+pL$2>VmxU_b>J&FlQJi#%2d@P!{c6y^=hh}X+K`s^pD##y%$qISnW6@_6v}_ zcI;&{V<4iH&bwYi)Q}h_e7c*p_rKYCx1tUA?X|c6cB=D(KzUv(s{Z&#p0{gEhxCNV z?w{R~G(w(y2S`Q`fC2DC?x$eM8Rji8n)iH+seg*Ck)SO{Z}HGa?Q=M}|u+1||NAOYGV=86S6{!*^psO2(Go4FGS0ugo^KJAaoR=4$N+V{;AZCs_Q0zq~TX zF4lhjBP#bNGi=sRtOE4@Rx#xB2FgqX*Ep|#%yQ84Jg>K5^6&cqWU}xnZnhU96_&u; zPbC>5?@)5(DWCNxdv?cL54Ei9UZ~XLF&~}DfVHqA;#t^4WWevR$|84voa;1mC6gpu zqnA+*W$@peFuXkVuTxow>}|v!;{!F~(|)k*^H}zaHi`1zk)KQgBqEF}>a$)%%-b(Y zM&K&|^p>*nXh?VKhUBe9`U#Z>wM%H+z%Z}AUB|7t*d>gD7xBDM_eoqvM1@>m77!+k zV?D03ZhrGjZ+!)`sQVJoIrapH`MDeD$hk$oK%XvR+MP`0P!a$z2YPcXc?zCUy9^-gg&TMs4orK6j6t?B-Aix*8R>w{gry|pWN zN1-h%m=jnJC$o9xJ)CRIwq#iON_$gOLLJI2+-^49J_I&5W2S1=*$tr=-`n+i5>kf* zB8FKsJ3dgc>DfzEPW0GKcAw#z*`2iUg1a1_0^!!dx0>{&o0Y2{HP?sF;+x>VPtlN} zJtnNzzW){-?XQ|lLT4>M%x9hc;T1;3mXOE-w6X`;h5jAIPQD_W3LrX2(1U!|D)V(~ z^5l*<=0JiDjpLZ-W}DPo-Uh1&%^<;Sr_P7T+a7ykjQhs_TstiA^KA>zv~a|OrAvi% zdrwg8+;I9vZ>ir1QJNiCj!@nfb~j-8SKs&COt<=M8d-JIInEN!akMXTZle5rMRSf9 zEV3V*KOOHhjELVu@;f8H!e2@oDW^Y*<`QRf5NcYJ{WufJsV&Jpoyv}{q&g##$e`{gd?)EK;3P$isT9R1!t<3^X zI;ZSc22#)KG?J&0Ys{gvansH*fp1&EIT{Or#aulZuea7|AL)v8-Lu?L<;3D`uxXcm zu^M?m+^yK3%bY~;Kic;2)c*j#mqJf9f_*Q9UIH!nnEOQW8c#4}`@^F~(+O&OjY*OD z4l|r6t*}w+*u%w#fg((wp4xBCWz_buD0iw~;V340jxzdsp6ck5qnR--i;Q(jueMC7 zarn#d?REK-wAE&A)~oK+d1;zS8QXSr_}GE#v{aS+%L1T|BB`X&kV#6d<|^^(J|BsauK{Dy760>{dxBuCHin9hTgC`r*)mV^Whg&-ZM#cHUEZ*%|<>Nmt*J5 zs``l}>tZvEJ=@ZRocOA;^#zBvwa`KDpXm2QeOifi4R-K+sl0En&sdTj@os_xm?V#$pD3bAF+9}v=F&r~<)rHK~1yN_NJaeJmo_Y&QUD?g4&q{g*f zthL20%$J$u=n$G~qrbtB7H2Lp_7%Wo{^ZvTf(As*Mir6lLgsExd*&E(zhwbSUByNz zyR=S^s{|L9(rL%h$eznHE?ZWBE-vzOSKNF0)_R84@Oigti;@X3^E z9n;vvI3RKtN!kez*HS;I`ypD%9WgjYD7m+J6!W)jS-z93MI~K^%YQvr|GWPOJ=O&I literal 0 HcmV?d00001 From 643ec9db3806f4ffe448762729dd31a49408541d Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 26 Jun 2025 18:57:34 +0000 Subject: [PATCH 056/155] implement geolocation api to fix CORS issue --- meshinfo_web.py | 26 ++++++++++++++++++++++++++ templates/map.html.j2 | 36 ++++++++++++++++++++++++++++-------- utils.py | 30 ++++++++++++++++++++++++++---- 3 files changed, 80 insertions(+), 12 deletions(-) diff --git a/meshinfo_web.py b/meshinfo_web.py index d29f4ded..dcbb54ba 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -2100,6 +2100,32 @@ def debug_clear_nodes(): 'message': 'Nodes singleton cleared. Check logs for details.' }) +@app.route('/api/geocode') +def api_geocode(): + """API endpoint for reverse geocoding to avoid CORS issues.""" + try: + lat = request.args.get('lat', type=float) + lon = request.args.get('lon', type=float) + + if lat is None or lon is None: + return jsonify({'error': 'Missing lat or lon parameters'}), 400 + + # Use the existing geocoding function from utils + geocoded = utils.geocode_position( + config.get('geocoding', 'apikey', fallback=''), + lat, + lon + ) + + if geocoded: + return jsonify(geocoded) + else: + return jsonify({'error': 'Geocoding failed'}), 500 + + except Exception as e: + logging.error(f"Geocoding error: {e}") + return jsonify({'error': 'Internal server error'}), 500 + def run(): # Enable Waitress logging config = configparser.ConfigParser() diff --git a/templates/map.html.j2 b/templates/map.html.j2 index 0d9dc409..183e9077 100644 --- a/templates/map.html.j2 +++ b/templates/map.html.j2 @@ -619,13 +619,19 @@ applyFilters(); async function reverseGeocode(lon, lat) { - return fetch('https://nominatim.openstreetmap.org/reverse?format=json&lon=' + lon + '&lat=' + lat) - .then(function(response) { - return response.json(); - }).then(function(json) { - console.log(json); - return json; - }); + try { + const response = await fetch('/api/geocode?lon=' + lon + '&lat=' + lat); + if (!response.ok) { + console.warn('Geocoding API returned error:', response.status); + return null; + } + const json = await response.json(); + console.log('Geocoding result:', json); + return json; + } catch (error) { + console.warn('Geocoding request failed:', error); + return null; + } } var container = document.getElementById('nodeDetail'); @@ -710,7 +716,21 @@ var properties = feature.getProperties(); var node = properties.node; var address = await reverseGeocode(node.position[0], node.position[1]); - var display_name = [address.address.town, address.address.city, address.address.county, address.address.state, address.address.country].filter(Boolean).join(', '); + var display_name = 'Unknown location'; + + if (address && address.address) { + // Handle Nominatim format + var location_parts = [address.address.town, address.address.city, address.address.county, address.address.state, address.address.country].filter(Boolean); + if (location_parts.length > 0) { + display_name = location_parts.join(', '); + } else if (address.display_name) { + // Fallback to display_name if address components are not available + display_name = address.display_name; + } + } else if (address && address.display_name) { + // Handle other geocoding service formats + display_name = address.display_name; + } // Get all connected node IDs const connectedNodeIds = new Set(); diff --git a/utils.py b/utils.py index 90a5ad2a..833a82bc 100644 --- a/utils.py +++ b/utils.py @@ -78,10 +78,32 @@ def geocode_position(api_key: str, latitude: float, longitude: float): """Retrieve geolocation data using an API.""" if latitude is None or longitude is None: return None - url = f"https://geocode.maps.co/reverse" + \ - f"?lat={latitude}&lon={longitude}&api_key={api_key}" - response = requests.get(url) - return response.json() if response.status_code == 200 else None + + # Try the paid service first if API key is provided + if api_key and api_key != 'YOUR_KEY_HERE': + try: + url = f"https://geocode.maps.co/reverse" + \ + f"?lat={latitude}&lon={longitude}&api_key={api_key}" + response = requests.get(url, timeout=5) + if response.status_code == 200: + return response.json() + except Exception as e: + logging.warning(f"Paid geocoding service failed: {e}") + + # Fallback to Nominatim (free, no API key required) + try: + url = f"https://nominatim.openstreetmap.org/reverse" + \ + f"?format=json&lat={latitude}&lon={longitude}&zoom=10" + headers = { + 'User-Agent': 'MeshInfo/1.0 (https://github.com/meshinfo-lite)' + } + response = requests.get(url, headers=headers, timeout=5) + if response.status_code == 200: + return response.json() + except Exception as e: + logging.warning(f"Nominatim geocoding service failed: {e}") + + return None def latlon_to_grid(lat, lon): From d4c108f3c88959e7f5dd492c17d838841ada9143 Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 26 Jun 2025 19:09:36 +0000 Subject: [PATCH 057/155] fix maps to ensure all nodes display (no more UNK) --- templates/map.html.j2 | 54 +++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/templates/map.html.j2 b/templates/map.html.j2 index 183e9077..9d06aeb5 100644 --- a/templates/map.html.j2 +++ b/templates/map.html.j2 @@ -305,36 +305,30 @@ var nodes = {}; var neighborLayers = []; {% for id, node in nodes.items() %} - {% if node.position is defined and node.position and node.position.longitude_i and node.position.latitude_i %} - {% set lat = node.position.latitude_i / 10000000 %} - {% set lon = node.position.longitude_i / 10000000 %} - {% if -90 <= lat <= 90 and -180 <= lon <= 180 %} - nodes['{{ id }}'] = { - id: '{{ id }}', - short_name: '{{ node.short_name }}', - long_name: '{{ node.long_name | e }}', - last_seen: '{{ time_ago(node.ts_seen) }}', - ts_seen: {{ node.ts_seen }}, - ts_uplink: {% if node.ts_uplink is defined and node.ts_uplink is not none %}{{ node.ts_uplink }}{% else %}null{% endif %}, - position: [{{ lon }}, {{ lat }}], - online: {% if node.active %}true{% else %}false{% endif %}, - channel: {% if node.channel is not none %}{{ node.channel }}{% else %}null{% endif %}, - zero_hop_data: {{ zero_hop_data.get(id, {'heard': [], 'heard_by': []}) | tojson | safe }} - }; - {% if node.neighbors %} - nodes['{{ id }}'].neighbors = [ - {% for neighbor in node.neighbors %} - { - id: '{{ utils.convert_node_id_from_int_to_hex(neighbor.neighbor_id) }}', - snr: '{{ neighbor.snr }}', - distance: '{{ neighbor.distance }}', - }, - {% endfor %} - ]; - {% else %} - nodes['{{ id }}'].neighbors = []; - {% endif %} - {% endif %} + nodes['{{ id }}'] = { + id: '{{ id }}', + short_name: '{{ node.short_name }}', + long_name: '{{ node.long_name | e }}', + last_seen: '{{ time_ago(node.ts_seen) }}', + ts_seen: {{ node.ts_seen }}, + ts_uplink: {% if node.ts_uplink is defined and node.ts_uplink is not none %}{{ node.ts_uplink }}{% else %}null{% endif %}, + position: [{% if node.position and node.position.longitude_i and node.position.latitude_i %}{{ node.position.longitude_i / 10000000 }}, {{ node.position.latitude_i / 10000000 }}{% else %}null, null{% endif %}], + online: {% if node.active %}true{% else %}false{% endif %}, + channel: {% if node.channel is not none %}{{ node.channel }}{% else %}null{% endif %}, + zero_hop_data: {{ zero_hop_data.get(id, {'heard': [], 'heard_by': []}) | tojson | safe }} + }; + {% if node.neighbors %} + nodes['{{ id }}'].neighbors = [ + {% for neighbor in node.neighbors %} + { + id: '{{ utils.convert_node_id_from_int_to_hex(neighbor.neighbor_id) }}', + snr: '{{ neighbor.snr }}', + distance: '{{ neighbor.distance }}', + }, + {% endfor %} + ]; + {% else %} + nodes['{{ id }}'].neighbors = []; {% endif %} {% endfor %} From 5380fd3e85af82e3d189e15792ccc89c764282d2 Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 26 Jun 2025 19:57:01 +0000 Subject: [PATCH 058/155] fixed location plots for nodes with no location --- templates/map.html.j2 | 83 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 13 deletions(-) diff --git a/templates/map.html.j2 b/templates/map.html.j2 index 9d06aeb5..5128dbc8 100644 --- a/templates/map.html.j2 +++ b/templates/map.html.j2 @@ -337,7 +337,17 @@ const positionMap = new Map(); Object.values(nodes).forEach(function(node) { - if (node.position) { + // Only add features for nodes with valid lat/lon + if ( + node.position && + Array.isArray(node.position) && + node.position[0] !== null && + node.position[1] !== null && + !isNaN(node.position[0]) && + !isNaN(node.position[1]) && + node.position[0] >= -180 && node.position[0] <= 180 && + node.position[1] >= -90 && node.position[1] <= 90 + ) { const posKey = node.position.join(','); const offset = positionMap.get(posKey) || 0; @@ -387,7 +397,7 @@ image: new Circle({ radius: 6, fill: new Fill({ - color: 'rgba(33, 150, 243, 1)' // Bright blue for active MQTT + color: 'rgba(33, 150, 243, 1)' }), stroke: new Stroke({ color: 'white', @@ -399,10 +409,8 @@ // MQTT uplink is old, fall back to regular node status coloring const offlineAge = parseInt(settings.nodesOfflineAge); if (settings.nodesOfflineAge === 'never' || now - node.ts_seen <= offlineAge) { - // Node has been seen within the offline age threshold - show as active style = onlineStyle; } else { - // Node hasn't been seen within the offline age threshold - show as offline style = offlineStyle; } } @@ -410,10 +418,8 @@ // No MQTT uplink info - check offline age const offlineAge = parseInt(settings.nodesOfflineAge); if (settings.nodesOfflineAge === 'never' || now - node.ts_seen <= offlineAge) { - // Node has been seen within the offline age threshold - show as active style = onlineStyle; } else { - // Node hasn't been seen within the offline age threshold - show as offline style = offlineStyle; } } @@ -421,7 +427,6 @@ // Apply channel-specific styling if needed if (settings.channelFilter !== 'all') { const channelId = parseInt(settings.channelFilter); - // Only apply channel-specific styling if node matches the selected channel if (node.channel === channelId) { const hasActiveUplink = node.ts_uplink !== null && (now - node.ts_uplink) <= parseInt(settings.nodesDisconnectedAge); style = new Style({ @@ -437,15 +442,13 @@ }) }); } else { - visible = false; // Hide nodes that don't match the selected channel + visible = false; } } } feature.setStyle(visible ? style : null); features.push(feature); - - // Increment the count for this position positionMap.set(posKey, offset + 1); } }); @@ -889,7 +892,25 @@ if (node.neighbors) { node.neighbors.forEach(function(neighbor) { var nnode = nodes[neighbor.id]; - if (!nnode || !nnode.position) return; + if ( + !nnode || + !nnode.position || + !Array.isArray(nnode.position) || + nnode.position[0] === null || + nnode.position[1] === null || + isNaN(nnode.position[0]) || + isNaN(nnode.position[1]) || + nnode.position[0] < -180 || nnode.position[0] > 180 || + nnode.position[1] < -90 || nnode.position[1] > 90 || + !node.position || + !Array.isArray(node.position) || + node.position[0] === null || + node.position[1] === null || + isNaN(node.position[0]) || + isNaN(node.position[1]) || + node.position[0] < -180 || node.position[0] > 180 || + node.position[1] < -90 || node.position[1] > 90 + ) return; const featureLine = new Feature({ geometry: new LineString([ @@ -921,7 +942,25 @@ // Draw lines for heard nodes node.zero_hop_data.heard.forEach(function(heard) { var nnode = nodes[heard.node_id]; - if (!nnode || !nnode.position) return; + if ( + !nnode || + !nnode.position || + !Array.isArray(nnode.position) || + nnode.position[0] === null || + nnode.position[1] === null || + isNaN(nnode.position[0]) || + isNaN(nnode.position[1]) || + nnode.position[0] < -180 || nnode.position[0] > 180 || + nnode.position[1] < -90 || nnode.position[1] > 90 || + !node.position || + !Array.isArray(node.position) || + node.position[0] === null || + node.position[1] === null || + isNaN(node.position[0]) || + isNaN(node.position[1]) || + node.position[0] < -180 || node.position[0] > 180 || + node.position[1] < -90 || node.position[1] > 90 + ) return; const featureLine = new Feature({ geometry: new LineString([ @@ -952,7 +991,25 @@ // Draw lines for heard-by nodes (if not already drawn) node.zero_hop_data.heard_by.forEach(function(heard) { var nnode = nodes[heard.node_id]; - if (!nnode || !nnode.position) return; + if ( + !nnode || + !nnode.position || + !Array.isArray(nnode.position) || + nnode.position[0] === null || + nnode.position[1] === null || + isNaN(nnode.position[0]) || + isNaN(nnode.position[1]) || + nnode.position[0] < -180 || nnode.position[0] > 180 || + nnode.position[1] < -90 || nnode.position[1] > 90 || + !node.position || + !Array.isArray(node.position) || + node.position[0] === null || + node.position[1] === null || + isNaN(node.position[0]) || + isNaN(node.position[1]) || + node.position[0] < -180 || node.position[0] > 180 || + node.position[1] < -90 || node.position[1] > 90 + ) return; // Skip if we already drew a line for this connection if (node.zero_hop_data.heard.some(h => h.node_id === heard.node_id)) return; From b9797165af7c314f4d68b4a0ec5df348f25388d7 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 27 Jun 2025 01:48:47 +0000 Subject: [PATCH 059/155] Enhance chat data handling and distance calculations. Optimized chat data retrieval with improved pagination and reception data processing. Added client-side distance calculation for node positions, including a new API endpoint for batch position retrieval. Updated templates to display distance information in message popovers, improving user experience. --- meshinfo_web.py | 177 +++++++++++++++++++++++++++++++++++-- templates/chat2.html.j2 | 191 ++++++++++++++++++++++++++++++---------- 2 files changed, 314 insertions(+), 54 deletions(-) diff --git a/meshinfo_web.py b/meshinfo_web.py index dcbb54ba..937ec9d3 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -1822,17 +1822,92 @@ def api_telemetry(node_id): @cache.memoize(timeout=60) def get_cached_chat_data(page=1, per_page=50): - """Cache the chat data.""" + """Cache the chat data with optimized query.""" md = get_meshdata() if not md: return None - chat_data = md.get_chat(page=page, per_page=per_page) - # Add pagination info - chat_data['start_item'] = (page - 1) * per_page + 1 if chat_data['total'] > 0 else 0 - chat_data['end_item'] = min(page * per_page, chat_data['total']) + # Get total count first (this is fast) + cur = md.db.cursor() + cur.execute("SELECT COUNT(DISTINCT t.message_id) FROM text t") + total = cur.fetchone()[0] + cur.close() - return chat_data + # Get paginated chat messages (without reception data) + offset = (page - 1) * per_page + cur = md.db.cursor(dictionary=True) + cur.execute(""" + SELECT t.* FROM text t + ORDER BY t.ts_created DESC + LIMIT %s OFFSET %s + """, (per_page, offset)) + messages = cur.fetchall() + cur.close() + + # Get reception data for these messages in a separate query + if messages: + message_ids = [msg['message_id'] for msg in messages] + placeholders = ','.join(['%s'] * len(message_ids)) + cur = md.db.cursor(dictionary=True) + cur.execute(f""" + SELECT message_id, received_by_id, rx_snr, rx_rssi, hop_limit, hop_start + FROM message_reception + WHERE message_id IN ({placeholders}) + """, message_ids) + receptions = cur.fetchall() + cur.close() + + # Group receptions by message_id + receptions_by_message = {} + for reception in receptions: + msg_id = reception['message_id'] + if msg_id not in receptions_by_message: + receptions_by_message[msg_id] = [] + receptions_by_message[msg_id].append({ + "node_id": reception['received_by_id'], + "rx_snr": float(reception['rx_snr']) if reception['rx_snr'] is not None else 0, + "rx_rssi": int(reception['rx_rssi']) if reception['rx_rssi'] is not None else 0, + "hop_limit": int(reception['hop_limit']) if reception['hop_limit'] is not None else None, + "hop_start": int(reception['hop_start']) if reception['hop_start'] is not None else None + }) + else: + receptions_by_message = {} + + # Process messages + chats = [] + prev_key = "" + for row in messages: + record = {} + for key, value in row.items(): + if isinstance(value, datetime.datetime): + record[key] = value.timestamp() + else: + record[key] = value + + # Add reception data + record["receptions"] = receptions_by_message.get(record['message_id'], []) + + # Convert IDs to hex + record["from"] = utils.convert_node_id_from_int_to_hex(record["from_id"]) + record["to"] = utils.convert_node_id_from_int_to_hex(record["to_id"]) + + # Deduplicate messages + msg_key = f"{record['from']}{record['to']}{record['text']}{record['message_id']}" + if msg_key != prev_key: + chats.append(record) + prev_key = msg_key + + return { + "items": chats, + "total": total, + "page": page, + "per_page": per_page, + "pages": (total + per_page - 1) // per_page, + "has_prev": page > 1, + "has_next": page * per_page < total, + "prev_num": page - 1, + "next_num": page + 1 + } def get_node_page_data(node_hex): """Fetch and process all data for the node page to prevent memory leaks.""" @@ -1951,6 +2026,24 @@ def chat(): debug=False, ) +@cache.memoize(timeout=300) # Cache for 5 minutes +def calculate_node_distance(node1_hex, node2_hex): + """Calculate distance between two nodes, cached to avoid repeated calculations.""" + nodes = get_cached_nodes() + if not nodes: + return None + + node1 = nodes.get(node1_hex) + node2 = nodes.get(node2_hex) + + if not node1 or not node2: + return None + + if not node1.get("position") or not node2.get("position"): + return None + + return utils.calculate_distance_between_nodes(node1, node2) + @app.route('/chat.html') def chat2(): page = request.args.get('page', 1, type=int) @@ -1965,11 +2058,42 @@ def chat2(): if not chat_data: abort(503, description="Database connection unavailable") + # Pre-process nodes to reduce template complexity + # Only include nodes that are actually used in the chat messages + used_node_ids = set() + for message in chat_data["items"]: + used_node_ids.add(message["from"]) + if message["to"] != "ffffffff": + used_node_ids.add(message["to"]) + for reception in message.get("receptions", []): + node_id = utils.convert_node_id_from_int_to_hex(reception["node_id"]) + used_node_ids.add(node_id) + + # Create simplified nodes dict with only needed data + simplified_nodes = {} + for node_id in used_node_ids: + if node_id in nodes: + node = nodes[node_id] + simplified_nodes[node_id] = { + 'long_name': node.get('long_name', ''), + 'short_name': node.get('short_name', ''), + 'hw_model': node.get('hw_model'), + 'hw_model_name': meshtastic_support.get_hardware_model_name(node.get('hw_model')) if node.get('hw_model') else None, + 'role': node.get('role'), + 'role_name': utils.get_role_name(node.get('role')) if node.get('role') is not None else None, + 'firmware_version': node.get('firmware_version'), + 'owner_username': node.get('owner_username'), + 'owner': node.get('owner'), + 'position': node.get('position'), + 'telemetry': node.get('telemetry'), + 'ts_seen': node.get('ts_seen') + } + return render_template( "chat2.html.j2", auth=auth(), config=config, - nodes=nodes, + nodes=simplified_nodes, chat=chat_data["items"], pagination=chat_data, utils=utils, @@ -2126,6 +2250,45 @@ def api_geocode(): logging.error(f"Geocoding error: {e}") return jsonify({'error': 'Internal server error'}), 500 +@cache.memoize(timeout=300) # Cache for 5 minutes +def get_node_positions_batch(node_ids): + """Get position data for multiple nodes efficiently.""" + nodes = get_cached_nodes() + if not nodes: + return {} + + positions = {} + for node_id in node_ids: + if node_id in nodes: + node = nodes[node_id] + if node.get('position') and node['position'].get('latitude') and node['position'].get('longitude'): + positions[node_id] = { + 'latitude': node['position']['latitude'], + 'longitude': node['position']['longitude'] + } + + return positions + +@app.route('/api/node-positions') +def api_node_positions(): + """API endpoint to get position data for specific nodes for client-side distance calculations.""" + try: + # Get list of node IDs from query parameter + node_ids = request.args.get('nodes', '').split(',') + node_ids = [nid.strip() for nid in node_ids if nid.strip()] + + if not node_ids: + return jsonify({'positions': {}}) + + # Use the cached batch function + positions = get_node_positions_batch(tuple(node_ids)) # Convert to tuple for caching + + return jsonify({'positions': positions}) + + except Exception as e: + logging.error(f"Error fetching node positions: {e}") + return jsonify({'error': 'Internal server error'}), 500 + def run(): # Enable Waitress logging config = configparser.ConfigParser() diff --git a/templates/chat2.html.j2 b/templates/chat2.html.j2 index 3cc48569..37fc62f2 100644 --- a/templates/chat2.html.j2 +++ b/templates/chat2.html.j2 @@ -43,6 +43,117 @@ window.addEventListener('load', function() { initializePopovers(); } }); + +// Client-side distance calculation +function calculateDistance(lat1, lon1, lat2, lon2) { + const R = 6371; // Earth's radius in kilometers + const dLat = (lat2 - lat1) * Math.PI / 180; + const dLon = (lon2 - lon1) * Math.PI / 180; + const a = Math.sin(dLat/2) * Math.sin(dLat/2) + + Math.cos(lat1 * Math.PI / 180) * Math.cos(lat2 * Math.PI / 180) * + Math.sin(dLon/2) * Math.sin(dLon/2); + const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1-a)); + return R * c; +} + +// Load position data and update tooltips with distances +async function loadDistances() { + try { + console.log('Starting distance calculation...'); + + // Collect all unique node IDs from the current page + const nodeIds = new Set(); + + // Get sender IDs from message containers + document.querySelectorAll('.message-container[data-sender-id]').forEach(container => { + const senderId = container.getAttribute('data-sender-id'); + if (senderId) nodeIds.add(senderId); + }); + + // Get receiver IDs from reception badges + document.querySelectorAll('.reception-badge[data-receiver-id]').forEach(badge => { + const receiverId = badge.getAttribute('data-receiver-id'); + if (receiverId) nodeIds.add(receiverId); + }); + + console.log('Found node IDs:', Array.from(nodeIds)); + + if (nodeIds.size === 0) { + console.warn('No node IDs found for distance calculation'); + return; + } + + // Convert to array and join for API call + const nodeIdsArray = Array.from(nodeIds); + const response = await fetch(`/api/node-positions?nodes=${nodeIdsArray.join(',')}`); + if (!response.ok) { + console.warn('Failed to load node positions for distance calculation'); + return; + } + + const data = await response.json(); + const positions = data.positions; + console.log('Received positions:', positions); + + // Update all receiver popovers with distance information + let updatedCount = 0; + document.querySelectorAll('.reception-badge[data-toggle="popover"]').forEach(badge => { + try { + const senderId = badge.closest('.message-container').getAttribute('data-sender-id'); + const receiverId = badge.getAttribute('data-receiver-id'); + + console.log(`Checking ${senderId} -> ${receiverId}`); + + if (senderId && receiverId && positions[senderId] && positions[receiverId]) { + const senderPos = positions[senderId]; + const receiverPos = positions[receiverId]; + const distance = calculateDistance( + senderPos.latitude, senderPos.longitude, + receiverPos.latitude, receiverPos.longitude + ); + + console.log(`Distance: ${distance.toFixed(2)} km`); + + // Get current content from data-content attribute + const currentContent = badge.getAttribute('data-content'); + if (currentContent && !currentContent.includes('Distance:')) { + const newContent = currentContent.replace('', + `
Distance: ${distance.toFixed(2)} km`); + + // Update the data-content attribute + badge.setAttribute('data-content', newContent); + + // Destroy and recreate the popover to pick up the new content + const $badge = $(badge); + $badge.popover('dispose'); + $badge.popover({ + html: true, + trigger: 'hover', + placement: 'bottom', + container: 'body', + delay: { show: 50, hide: 100 } + }); + + updatedCount++; + } + } + } catch (error) { + console.warn('Error updating popover for badge:', error); + } + }); + + console.log(`Updated ${updatedCount} popovers with distance information`); + + } catch (error) { + console.warn('Error loading distances:', error); + } +} + +// Load distances after page is fully loaded +window.addEventListener('load', function() { + // Longer delay to ensure popovers are fully initialized + setTimeout(loadDistances, 500); +}); {% endblock %} @@ -80,7 +191,7 @@ window.addEventListener('load', function() {
{% for message in chat %} -
+
{# Make header a flex container #} {# Group 1: Sender and optional Recipient #} @@ -88,79 +199,64 @@ window.addEventListener('load', function() { {# Sender Span (Always Shown) #} {% if message.from in nodes %} + {% set sender = nodes[message.from] %}
+ {{ sender.long_name|replace('"', '"') }}
{# Hardware Info #} - {% if nodes[message.from].hw_model %} - HW: {{ meshtastic_support.get_hardware_model_name(nodes[message.from].hw_model) }}
+ {% if sender.hw_model %} + HW: {{ sender.hw_model_name if sender.hw_model_name else 'Unknown' }}
{% endif %} - {% if nodes[message.from].firmware_version %} - FW: {{ nodes[message.from].firmware_version }}
+ {% if sender.firmware_version %} + FW: {{ sender.firmware_version }}
{% endif %} - {# --- Start Role Display --- #} - {% if nodes[message.from].role is not none %} - Role: - {% set role_val = nodes[message.from].role %} - {% if role_val == 0 %}Client - {% elif role_val == 1 %}Client Mute - {% elif role_val == 2 %}Router - {% elif role_val == 3 %}Router Client - {% elif role_val == 4 %}Repeater - {% elif role_val == 5 %}Tracker - {% elif role_val == 6 %}Sensor - {% elif role_val == 7 %}ATAK - {% elif role_val == 8 %}Client Hidden - {% elif role_val == 9 %}Lost and Found - {% elif role_val == 10 %}ATAK Tracker - {% elif role_val == 11 %}Router Late - {% else %}Unknown ({{ role_val }}) - {% endif %}
+ {# Role Display #} + {% if sender.role is not none %} + Role: {{ sender.role_name }}
{% endif %} - {# --- End Role Display --- #} {# Owner #} - {% if nodes[message.from].owner_username %} - Owner: {{ nodes[message.from].owner_username }}
+ {% if sender.owner_username %} + Owner: {{ sender.owner_username }}
{% endif %} {# Location #} - {% if nodes[message.from].position %} - {% if nodes[message.from].position.geocoded %} - Loc: {{ nodes[message.from].position.geocoded }}
- {% elif nodes[message.from].position.latitude is not none and nodes[message.from].position.longitude is not none %} - Lat: {{ '%.5f'|format(nodes[message.from].position.latitude) }}, Lon: {{ '%.5f'|format(nodes[message.from].position.longitude) }} - {% if nodes[message.from].position.altitude is not none %} - Alt: {{ nodes[message.from].position.altitude }}m + {% if sender.position %} + {% if sender.position.geocoded %} + Loc: {{ sender.position.geocoded }}
+ {% elif sender.position.latitude is not none and sender.position.longitude is not none %} + Lat: {{ '%.5f'|format(sender.position.latitude) }}, Lon: {{ '%.5f'|format(sender.position.longitude) }} + {% if sender.position.altitude is not none %} + Alt: {{ sender.position.altitude }}m {% endif %}
{% endif %} {% endif %} {# Telemetry Status #} - {% if nodes[message.from].telemetry %} - {% if nodes[message.from].telemetry.battery_level is not none %} - Batt: {{ nodes[message.from].telemetry.battery_level }}% - {% if nodes[message.from].telemetry.voltage is not none %} - ({{ '%.2f'|format(nodes[message.from].telemetry.voltage) }}V) + {% if sender.telemetry %} + {% if sender.telemetry.battery_level is not none %} + Batt: {{ sender.telemetry.battery_level }}% + {% if sender.telemetry.voltage is not none %} + ({{ '%.2f'|format(sender.telemetry.voltage) }}V) {% endif %}
- {% elif nodes[message.from].telemetry.voltage is not none %} - Voltage: {{ '%.2f'|format(nodes[message.from].telemetry.voltage) }}V
+ {% elif sender.telemetry.voltage is not none %} + Voltage: {{ '%.2f'|format(sender.telemetry.voltage) }}V
{% endif %} {# Optional: Environmentals #} - {% if nodes[message.from].telemetry.temperature is not none %} Temp: {{ '%.1f'|format(nodes[message.from].telemetry.temperature) }}°C {% endif %} - {% if nodes[message.from].telemetry.relative_humidity is not none %} RH: {{ '%.1f'|format(nodes[message.from].telemetry.relative_humidity) }}% {% endif %} - {% if nodes[message.from].telemetry.barometric_pressure is not none %} Pres: {{ '%.1f'|format(nodes[message.from].telemetry.barometric_pressure / 100) }}hPa {% endif %} {# Assuming pressure is in Pa #} - {% if nodes[message.from].telemetry.temperature is not none or nodes[message.from].telemetry.relative_humidity is not none or nodes[message.from].telemetry.barometric_pressure is not none %}
{% endif %} + {% if sender.telemetry.temperature is not none %} Temp: {{ '%.1f'|format(sender.telemetry.temperature) }}°C {% endif %} + {% if sender.telemetry.relative_humidity is not none %} RH: {{ '%.1f'|format(sender.telemetry.relative_humidity) }}% {% endif %} + {% if sender.telemetry.barometric_pressure is not none %} Pres: {{ '%.1f'|format(sender.telemetry.barometric_pressure / 100) }}hPa {% endif %} + {% if sender.telemetry.temperature is not none or sender.telemetry.relative_humidity is not none or sender.telemetry.barometric_pressure is not none %}
{% endif %} {% endif %} {# Last Seen #} - Last Seen: {{ time_ago(nodes[message.from].ts_seen) }} + Last Seen: {{ time_ago(sender.ts_seen) }}
"> - {{ nodes[message.from].long_name|replace('"', '"') }} ({{ nodes[message.from].short_name|replace('"', '"') }}) + {{ sender.long_name|replace('"', '"') }} ({{ sender.short_name|replace('"', '"') }}) {% else %} {{ message.from }} @@ -284,6 +380,7 @@ window.addEventListener('load', function() {
SNR: {{ '%.1f'|format(reception.rx_snr) }}dB
From dde5f36c3cb8b603647229a6f985f2c8989b3129 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 27 Jun 2025 02:58:44 +0000 Subject: [PATCH 060/155] Refactor distance calculation logic in chat template. Updated loadDistances function to target specific message containers, improving performance and user experience. Added hover-based prefetching for mobile view and batch loading of distances, ensuring efficient data handling. Enhanced popover updates for real-time distance display. --- templates/chat2.html.j2 | 191 ++++++++++++++++++++++++++++++---------- 1 file changed, 144 insertions(+), 47 deletions(-) diff --git a/templates/chat2.html.j2 b/templates/chat2.html.j2 index 37fc62f2..9ce9775e 100644 --- a/templates/chat2.html.j2 +++ b/templates/chat2.html.j2 @@ -57,33 +57,26 @@ function calculateDistance(lat1, lon1, lat2, lon2) { } // Load position data and update tooltips with distances -async function loadDistances() { +async function loadDistancesForMessage(messageContainer) { try { - console.log('Starting distance calculation...'); + // Get sender and receiver IDs for this specific message + const senderId = messageContainer.getAttribute('data-sender-id'); + const receiverBadges = messageContainer.querySelectorAll('.reception-badge[data-receiver-id]'); - // Collect all unique node IDs from the current page - const nodeIds = new Set(); - - // Get sender IDs from message containers - document.querySelectorAll('.message-container[data-sender-id]').forEach(container => { - const senderId = container.getAttribute('data-sender-id'); - if (senderId) nodeIds.add(senderId); - }); + if (!senderId || receiverBadges.length === 0) { + return; // No need to calculate distances + } - // Get receiver IDs from reception badges - document.querySelectorAll('.reception-badge[data-receiver-id]').forEach(badge => { + // Collect node IDs for this message only + const nodeIds = new Set([senderId]); + receiverBadges.forEach(badge => { const receiverId = badge.getAttribute('data-receiver-id'); if (receiverId) nodeIds.add(receiverId); }); - console.log('Found node IDs:', Array.from(nodeIds)); + console.log(`Loading positions for message: ${Array.from(nodeIds)}`); - if (nodeIds.size === 0) { - console.warn('No node IDs found for distance calculation'); - return; - } - - // Convert to array and join for API call + // Fetch positions for this message's nodes const nodeIdsArray = Array.from(nodeIds); const response = await fetch(`/api/node-positions?nodes=${nodeIdsArray.join(',')}`); if (!response.ok) { @@ -93,17 +86,13 @@ async function loadDistances() { const data = await response.json(); const positions = data.positions; - console.log('Received positions:', positions); - // Update all receiver popovers with distance information + // Update receiver popovers for this message only let updatedCount = 0; - document.querySelectorAll('.reception-badge[data-toggle="popover"]').forEach(badge => { + receiverBadges.forEach(badge => { try { - const senderId = badge.closest('.message-container').getAttribute('data-sender-id'); const receiverId = badge.getAttribute('data-receiver-id'); - console.log(`Checking ${senderId} -> ${receiverId}`); - if (senderId && receiverId && positions[senderId] && positions[receiverId]) { const senderPos = positions[senderId]; const receiverPos = positions[receiverId]; @@ -112,47 +101,155 @@ async function loadDistances() { receiverPos.latitude, receiverPos.longitude ); - console.log(`Distance: ${distance.toFixed(2)} km`); + console.log(`Distance ${senderId} -> ${receiverId}: ${distance.toFixed(2)} km`); - // Get current content from data-content attribute + // Update desktop popover const currentContent = badge.getAttribute('data-content'); if (currentContent && !currentContent.includes('Distance:')) { const newContent = currentContent.replace('
', `
Distance: ${distance.toFixed(2)} km
`); - - // Update the data-content attribute - badge.setAttribute('data-content', newContent); - - // Destroy and recreate the popover to pick up the new content const $badge = $(badge); - $badge.popover('dispose'); - $badge.popover({ - html: true, - trigger: 'hover', - placement: 'bottom', - container: 'body', - delay: { show: 50, hide: 100 } - }); - + const popover = $badge.data('bs.popover'); + badge.setAttribute('data-content', newContent); + + if (popover && $badge.attr('aria-describedby')) { + // If popover is visible, update its content live + const popoverId = $badge.attr('aria-describedby'); + const popoverElem = document.getElementById(popoverId); + if (popoverElem) { + const popoverBody = popoverElem.querySelector('.popover-body'); + if (popoverBody) { + popoverBody.innerHTML = newContent; + } + } + } else if (!popover) { + // If not initialized, initialize it + $badge.popover({ + html: true, + trigger: 'hover', + placement: 'bottom', + container: 'body', + delay: { show: 50, hide: 100 } + }); + } updatedCount++; } + + // Update mobile view + const mobileRow = messageContainer.querySelector(`.mobile-reception-row [href="/node_${receiverId}.html"]`)?.closest('.mobile-reception-row'); + if (mobileRow && !mobileRow.querySelector('.distance-metric')) { + const metricsDiv = mobileRow.querySelector('.mobile-metrics'); + if (metricsDiv) { + const distanceDiv = document.createElement('div'); + distanceDiv.className = 'metric distance-metric'; + distanceDiv.innerHTML = `Distance: ${distance.toFixed(2)} km`; + metricsDiv.appendChild(distanceDiv); + } + } } } catch (error) { console.warn('Error updating popover for badge:', error); } }); - console.log(`Updated ${updatedCount} popovers with distance information`); + console.log(`Updated ${updatedCount} popovers with distance information for this message`); } catch (error) { - console.warn('Error loading distances:', error); + console.warn('Error loading distances for message:', error); } } -// Load distances after page is fully loaded +// Set up hover-based prefetch +function setupHoverPrefetch() { + let hoverTimer = null; + const processedMessages = new Set(); // Track which messages we've already processed + + document.querySelectorAll('.message-container').forEach(container => { + container.addEventListener('mouseenter', function() { + const messageId = this.getAttribute('data-message-id') || this.querySelector('.message-content')?.textContent?.slice(0, 50); + + // Skip if we've already processed this message + if (processedMessages.has(messageId)) { + return; + } + + // Clear any existing timer + if (hoverTimer) { + clearTimeout(hoverTimer); + } + + // Set a small delay to avoid triggering on accidental hovers + hoverTimer = setTimeout(() => { + loadDistancesForMessage(this); + processedMessages.add(messageId); + }, 200); // 200ms delay + }); + + container.addEventListener('mouseleave', function() { + // Clear the timer if user moves away before delay + if (hoverTimer) { + clearTimeout(hoverTimer); + hoverTimer = null; + } + }); + }); +} + +// Batch load and inject distances for mobile view +async function batchLoadDistancesForMobile() { + // Only run on mobile + if (window.innerWidth >= 768) return; + // Collect all unique node IDs from the current page + const nodeIds = new Set(); + document.querySelectorAll('.message-container[data-sender-id]').forEach(container => { + const senderId = container.getAttribute('data-sender-id'); + if (senderId) nodeIds.add(senderId); + }); + document.querySelectorAll('.reception-badge[data-receiver-id]').forEach(badge => { + const receiverId = badge.getAttribute('data-receiver-id'); + if (receiverId) nodeIds.add(receiverId); + }); + if (nodeIds.size === 0) return; + const nodeIdsArray = Array.from(nodeIds); + const response = await fetch(`/api/node-positions?nodes=${nodeIdsArray.join(',')}`); + if (!response.ok) return; + const data = await response.json(); + const positions = data.positions; + // For each message, inject distances into mobile view + document.querySelectorAll('.message-container').forEach(messageContainer => { + const senderId = messageContainer.getAttribute('data-sender-id'); + messageContainer.querySelectorAll('.mobile-reception-row [href^="/node_"]').forEach(link => { + const receiverId = link.getAttribute('href').replace('/node_', '').replace('.html', ''); + const mobileRow = link.closest('.mobile-reception-row'); + if (mobileRow && !mobileRow.querySelector('.distance-metric')) { + if (senderId && receiverId && positions[senderId] && positions[receiverId]) { + const senderPos = positions[senderId]; + const receiverPos = positions[receiverId]; + const distance = calculateDistance( + senderPos.latitude, senderPos.longitude, + receiverPos.latitude, receiverPos.longitude + ); + const metricsDiv = mobileRow.querySelector('.mobile-metrics'); + if (metricsDiv) { + const distanceDiv = document.createElement('div'); + distanceDiv.className = 'metric distance-metric'; + distanceDiv.innerHTML = `Distance: ${distance.toFixed(2)} km`; + metricsDiv.appendChild(distanceDiv); + } + } + } + }); + }); +} + +// Initialize hover prefetch after page loads window.addEventListener('load', function() { - // Longer delay to ensure popovers are fully initialized - setTimeout(loadDistances, 500); + // Small delay to ensure DOM is ready + if (window.innerWidth >= 768) { + setTimeout(setupHoverPrefetch, 100); + } else { + setTimeout(batchLoadDistancesForMobile, 200); + } }); {% endblock %} @@ -191,7 +288,7 @@ window.addEventListener('load', function() {
{% for message in chat %} -
+
{# Make header a flex container #} {# Group 1: Sender and optional Recipient #} From d166e288bd132240db503f045b2c0af90791e112 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 27 Jun 2025 04:37:27 +0000 Subject: [PATCH 061/155] display time differential between message sent and message reception in chat --- meshinfo_web.py | 5 +++-- templates/chat2.html.j2 | 12 +++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/meshinfo_web.py b/meshinfo_web.py index 937ec9d3..602400fb 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -1850,7 +1850,7 @@ def get_cached_chat_data(page=1, per_page=50): placeholders = ','.join(['%s'] * len(message_ids)) cur = md.db.cursor(dictionary=True) cur.execute(f""" - SELECT message_id, received_by_id, rx_snr, rx_rssi, hop_limit, hop_start + SELECT message_id, received_by_id, rx_snr, rx_rssi, hop_limit, hop_start, rx_time FROM message_reception WHERE message_id IN ({placeholders}) """, message_ids) @@ -1868,7 +1868,8 @@ def get_cached_chat_data(page=1, per_page=50): "rx_snr": float(reception['rx_snr']) if reception['rx_snr'] is not None else 0, "rx_rssi": int(reception['rx_rssi']) if reception['rx_rssi'] is not None else 0, "hop_limit": int(reception['hop_limit']) if reception['hop_limit'] is not None else None, - "hop_start": int(reception['hop_start']) if reception['hop_start'] is not None else None + "hop_start": int(reception['hop_start']) if reception['hop_start'] is not None else None, + "rx_time": reception['rx_time'].timestamp() if isinstance(reception['rx_time'], datetime.datetime) else reception['rx_time'] }) else: receptions_by_message = {} diff --git a/templates/chat2.html.j2 b/templates/chat2.html.j2 index 9ce9775e..928877a5 100644 --- a/templates/chat2.html.j2 +++ b/templates/chat2.html.j2 @@ -485,6 +485,9 @@ window.addEventListener('load', function() { {% if reception.hop_start is not none and reception.hop_limit is not none %}
Hops: {{ reception.hop_start - reception.hop_limit }} of {{ reception.hop_start }} {% endif %} + {% if reception.rx_time and message.ts_created %} +
Reception: +{{ format_duration(reception.rx_time - message.ts_created) }} after sent + {% endif %}
"> {{ node.short_name|replace('"', '"') }} @@ -548,4 +551,11 @@ window.addEventListener('load', function() {
-{% endblock %} \ No newline at end of file +{% endblock %} + +{% macro format_duration(seconds) %} + {%- set s = seconds|int %} + {%- set m = s // 60 %} + {%- set s = s % 60 %} + {%- if m > 0 %}{{ m }}m {% endif %}{{ s }}s +{% endmacro %} \ No newline at end of file From 890b0ec8338a0f3f2c011d50ae1f9c9815d49cc6 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 27 Jun 2025 19:40:27 +0000 Subject: [PATCH 062/155] Add relay node processing and update reception storage logic. Enhanced message reception handling to include relay node information, updated database schema to accommodate relay edges, and introduced a new method for retrieving relay network data. Updated templates to display relay node details in message paths and maps. --- meshdata.py | 223 ++++++++- meshinfo_web.py | 66 ++- migrations/__init__.py | 4 + migrations/add_relay_edges_table.py | 25 + migrations/add_relay_node_to_reception.py | 41 ++ templates/graph4.html.j2 | 8 +- templates/message-paths.html.j2 | 569 ++++++++++++++++++++++ templates/message_map.html.j2 | 22 +- 8 files changed, 941 insertions(+), 17 deletions(-) create mode 100644 migrations/add_relay_edges_table.py create mode 100644 migrations/add_relay_node_to_reception.py create mode 100644 templates/message-paths.html.j2 diff --git a/meshdata.py b/meshdata.py index 1566ad14..9141977d 100644 --- a/meshdata.py +++ b/meshdata.py @@ -1586,6 +1586,12 @@ def store(self, data, topic): rx_snr = data.get("rx_snr") rx_rssi = data.get("rx_rssi") + # Process relay_node if present (firmware 2.5.x+) + relay_node = None + if "relay_node" in data and data["relay_node"] is not None: + # Convert relay_node to hex format (last 2 bytes of node ID) + relay_node = f"{data['relay_node']:04x}" # Convert to 4-char hex string (last 2 bytes) + # Store reception information if this is a received message with SNR/RSSI data if message_id and rx_snr is not None and rx_rssi is not None: received_by = None @@ -1596,7 +1602,7 @@ def store(self, data, topic): if received_by and received_by != data.get("from"): # Don't store reception by sender self.store_reception(message_id, data["from"], received_by, rx_snr, rx_rssi, - data.get("rx_time"), hop_limit, hop_start) + data.get("rx_time"), hop_limit, hop_start, relay_node) # Continue with the regular message type processing tp = data["type"] @@ -1632,17 +1638,18 @@ def store(self, data, topic): else: logging.warning(f"store: Unknown message type '{tp}' from node {data.get('from')}") - def store_reception(self, message_id, from_id, received_by_id, rx_snr, rx_rssi, rx_time, hop_limit, hop_start): + def store_reception(self, message_id, from_id, received_by_id, rx_snr, rx_rssi, rx_time, hop_limit, hop_start, relay_node=None): """Store reception information for any message type with hop data.""" sql = """INSERT INTO message_reception - (message_id, from_id, received_by_id, rx_snr, rx_rssi, rx_time, hop_limit, hop_start) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + (message_id, from_id, received_by_id, rx_snr, rx_rssi, rx_time, hop_limit, hop_start, relay_node) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) ON DUPLICATE KEY UPDATE rx_snr = VALUES(rx_snr), rx_rssi = VALUES(rx_rssi), rx_time = VALUES(rx_time), hop_limit = VALUES(hop_limit), - hop_start = VALUES(hop_start)""" + hop_start = VALUES(hop_start), + relay_node = VALUES(relay_node)""" params = ( message_id, from_id, @@ -1651,12 +1658,34 @@ def store_reception(self, message_id, from_id, received_by_id, rx_snr, rx_rssi, rx_rssi, rx_time, hop_limit, - hop_start + hop_start, + relay_node ) cur = self.db.cursor() cur.execute(sql, params) - cur.close() self.db.commit() + cur.close() + + # --- Relay Edges Table Update --- + # Only update if relay_node is present and not None + if relay_node: + try: + relay_suffix = relay_node[-2:] # Only last two hex digits + from_hex = format(from_id, '08x') + to_hex = format(received_by_id, '08x') + edge_sql = """ + INSERT INTO relay_edges (from_node, relay_suffix, to_node, first_seen, last_seen, count) + VALUES (%s, %s, %s, NOW(), NOW(), 1) + ON DUPLICATE KEY UPDATE last_seen = NOW(), count = count + 1 + """ + edge_params = (from_hex, relay_suffix, to_hex) + cur = self.db.cursor() + cur.execute(edge_sql, edge_params) + self.db.commit() + cur.close() + except Exception as e: + import logging + logging.error(f"Failed to update relay_edges: {e}") def setup_database(self): creates = [ @@ -2357,7 +2386,7 @@ def get_reception_details_batch(self, message_id, receiver_ids): # Handle single receiver case if len(receiver_ids) == 1: query = """ - SELECT received_by_id, rx_snr, rx_rssi, rx_time, hop_limit, hop_start + SELECT received_by_id, rx_snr, rx_rssi, rx_time, hop_limit, hop_start, relay_node FROM message_reception WHERE message_id = %s AND received_by_id = %s """ @@ -2365,7 +2394,7 @@ def get_reception_details_batch(self, message_id, receiver_ids): else: placeholders = ','.join(['%s'] * len(receiver_ids)) query = f""" - SELECT received_by_id, rx_snr, rx_rssi, rx_time, hop_limit, hop_start + SELECT received_by_id, rx_snr, rx_rssi, rx_time, hop_limit, hop_start, relay_node FROM message_reception WHERE message_id = %s AND received_by_id IN ({placeholders}) """ @@ -2399,6 +2428,182 @@ def get_heard_by_from_neighbors(self, node_id): cur.close() return heard_by + def get_relay_network_data(self, days=30): + """Infer relay network data from message_reception table, not relay_edges.""" + cursor = None + try: + # Ensure database connection is valid + if not self.db or not self.db.is_connected(): + self.connect_db() + + # 1. Load all relevant message_reception rows + cursor = self.db.cursor(dictionary=True) + if days > 0: + cutoff = int(time.time()) - days * 86400 + cursor.execute(""" + SELECT message_id, from_id, received_by_id, relay_node, rx_time, hop_limit, hop_start + FROM message_reception + WHERE rx_time > %s + """, (cutoff,)) + else: + cursor.execute(""" + SELECT message_id, from_id, received_by_id, relay_node, rx_time, hop_limit, hop_start + FROM message_reception + """) + receptions = cursor.fetchall() + + # 2. Load all nodes for lookup + nodes_dict = self.get_nodes() + node_ids_set = set(nodes_dict.keys()) + node_id_ints = {int(k, 16): k for k in node_ids_set} + + # 3. Build relay edges and virtual relay nodes + edge_map = {} # (from, to) -> {count, relay_suffix, first_seen, last_seen} + virtual_nodes = {} # relay_xx -> {id, relay_suffix} + endpoint_nodes = set() # Only nodes that are endpoints of at least one edge + for rec in receptions: + sender = rec['from_id'] + receiver = rec['received_by_id'] + relay_node = rec['relay_node'] + rx_time = rec['rx_time'] + hop_limit = rec['hop_limit'] + hop_start = rec['hop_start'] + sender_hex = format(sender, '08x') + receiver_hex = format(receiver, '08x') + # Determine edge type + if relay_node: + relay_suffix = relay_node[-2:].lower() + # Try to match relay_node to a known node + relay_match = None + for node_hex in node_ids_set: + if node_hex[-2:] == relay_suffix: + relay_match = node_hex if not relay_match else None # Only if unique + if relay_match: + from_node = relay_match + else: + from_node = f"relay_{relay_suffix}" + if from_node not in virtual_nodes: + virtual_nodes[from_node] = {'id': from_node, 'relay_suffix': relay_suffix} + to_node = receiver_hex + elif (hop_limit is None and hop_start is None) or (hop_start is not None and hop_limit is not None and hop_start - hop_limit == 0): + # Zero-hop: direct sender->receiver + from_node = sender_hex + to_node = receiver_hex + else: + continue # Ignore receptions that don't fit the above + # Edge key + edge_key = (from_node, to_node) + if edge_key not in edge_map: + edge_map[edge_key] = { + 'count': 0, + 'relay_suffix': relay_node[-2:].lower() if relay_node else None, + 'first_seen': rx_time, + 'last_seen': rx_time + } + edge_map[edge_key]['count'] += 1 + if rx_time: + if edge_map[edge_key]['first_seen'] is None or rx_time < edge_map[edge_key]['first_seen']: + edge_map[edge_key]['first_seen'] = rx_time + if edge_map[edge_key]['last_seen'] is None or rx_time > edge_map[edge_key]['last_seen']: + edge_map[edge_key]['last_seen'] = rx_time + # Track endpoints + endpoint_nodes.add(from_node) + endpoint_nodes.add(to_node) + + # 4. Build node list (real + virtual), but only those that are endpoints + nodes = [] + node_stats = {} + for node_hex in endpoint_nodes: + node_stats[node_hex] = {'message_count': 0, 'relay_count': 0} + # Count stats + for (from_node, to_node), edge in edge_map.items(): + node_stats[from_node]['message_count'] += edge['count'] + node_stats[to_node]['relay_count'] += edge['count'] + # Real nodes + for node_hex in endpoint_nodes: + if node_hex in nodes_dict: + node = nodes_dict[node_hex] + # Use graph_icon utility for robust icon path + node_name_for_icon = node.get('long_name', node.get('short_name', '')) + icon_url = utils.graph_icon(node_name_for_icon) + nodes.append({ + 'id': node_hex, + 'long_name': node.get('long_name') or f"Node {node_hex}", + 'short_name': node.get('short_name') or f"Node {node_hex}", + 'hw_model': node.get('hw_model') or 'Unknown', + 'firmware_version': node.get('firmware_version'), + 'role': node.get('role'), + 'owner': node.get('owner'), + 'last_seen': node.get('ts_seen'), + 'message_count': node_stats[node_hex]['message_count'], + 'relay_count': node_stats[node_hex]['relay_count'], + 'icon_url': icon_url + }) + # Virtual relay nodes + for node_hex in endpoint_nodes: + if node_hex.startswith('relay_'): + v = virtual_nodes[node_hex] + nodes.append({ + 'id': v['id'], + 'long_name': f"Relay {v['relay_suffix']}", + 'short_name': f"relay_{v['relay_suffix']}", + 'hw_model': 'Virtual', + 'firmware_version': None, + 'role': None, + 'owner': None, + 'last_seen': None, + 'message_count': node_stats[v['id']]['message_count'], + 'relay_count': node_stats[v['id']]['relay_count'], + 'icon_url': None + }) + # 5. Build edge list + edges = [] + for (from_node, to_node), edge in edge_map.items(): + edges.append({ + 'id': f"{from_node}-{to_node}", + 'from_node': from_node, + 'to_node': to_node, + 'relay_suffix': edge['relay_suffix'], + 'message_count': edge['count'], + 'first_seen': datetime.datetime.fromtimestamp(edge['first_seen']).strftime('%Y-%m-%d %H:%M:%S') if edge['first_seen'] else None, + 'last_seen': datetime.datetime.fromtimestamp(edge['last_seen']).strftime('%Y-%m-%d %H:%M:%S') if edge['last_seen'] else None + }) + # 6. Stats + total_nodes = len(nodes) + total_edges = len(edges) + total_messages = sum(e['message_count'] for e in edges) + avg_hops = total_edges / total_nodes if total_nodes > 0 else 0 + stats = { + 'total_nodes': total_nodes, + 'total_edges': total_edges, + 'total_messages': total_messages, + 'avg_hops': avg_hops + } + return { + 'nodes': nodes, + 'edges': edges, + 'stats': stats + } + except Exception as e: + logging.error(f"Error in get_relay_network_data: {e}") + # Return empty data structure on error + return { + 'nodes': [], + 'edges': [], + 'stats': { + 'total_nodes': 0, + 'total_edges': 0, + 'total_messages': 0, + 'avg_hops': 0 + } + } + finally: + if cursor: + try: + cursor.close() + except Exception as e: + logging.error(f"Error closing cursor: {e}") + def create_database(): config = configparser.ConfigParser() diff --git a/meshinfo_web.py b/meshinfo_web.py index 602400fb..84da4955 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -663,7 +663,7 @@ def latlon_to_xy(lon, lat): 'ts_created': message_time, 'receiver_ids': receiver_ids_list } - + return { 'nodes': nodes, 'message': message, @@ -696,7 +696,8 @@ def message_map(): convex_hull_area_km2=data['convex_hull_area_km2'], utils=utils, datetime=datetime.datetime, - timestamp=datetime.datetime.now() + timestamp=datetime.datetime.now(), + find_relay_node_by_suffix=find_relay_node_by_suffix ) @app.route('/traceroute_map.html') @@ -2319,6 +2320,67 @@ def clear_nodes_singleton(): else: logging.info("Nodes singleton was already None") +@cache.memoize(timeout=300) # Cache for 5 minutes +def find_relay_node_by_suffix(relay_suffix, nodes, receiver_ids=None, sender_id=None): + """Find the actual relay node by matching the last 2 hex chars against known node IDs.""" + if not relay_suffix or len(relay_suffix) < 2: + return None + relay_suffix = relay_suffix.lower()[-2:] # Only last two hex digits + + # 1. First, check if the sender itself matches the relay suffix + if sender_id: + sender_id_hex = utils.convert_node_id_from_int_to_hex(sender_id) + if sender_id_hex.lower()[-2:] == relay_suffix: + return sender_id_hex + + # 2. Check if any of the message receivers match the relay suffix + if receiver_ids: + for receiver_id in receiver_ids: + receiver_id_hex = utils.convert_node_id_from_int_to_hex(receiver_id) + if receiver_id_hex.lower()[-2:] == relay_suffix: + return receiver_id_hex + + # 3. Check active nodes (nodes that have been seen recently) + active_nodes = [] + for node_id_hex, node_data in nodes.items(): + if len(node_id_hex) == 8 and node_id_hex.lower()[-2:] == relay_suffix: + if (node_data.get('telemetry') or + node_data.get('position') or + node_data.get('ts_seen')): + active_nodes.append(node_id_hex) + if len(active_nodes) == 1: + return active_nodes[0] + + # 4. If no active nodes or multiple active nodes, check all known nodes + matching_nodes = [] + for node_id_hex, node_data in nodes.items(): + if len(node_id_hex) == 8 and node_id_hex.lower()[-2:] == relay_suffix: + matching_nodes.append(node_id_hex) + if len(matching_nodes) == 1: + return matching_nodes[0] + return None + +@app.route('/message-paths.html') +def message_paths(): + days = float(request.args.get('days', 0.167)) # Default to 4 hours if not provided + + md = get_meshdata() + if not md: + abort(503, description="Database connection unavailable") + + # Get relay network data + relay_data = md.get_relay_network_data(days) + + return render_template( + "message-paths.html.j2", + auth=auth(), + config=config, + relay_data=relay_data, + stats=relay_data['stats'], + utils=utils, + datetime=datetime.datetime, + timestamp=datetime.datetime.now() + ) if __name__ == '__main__': config = configparser.ConfigParser() diff --git a/migrations/__init__.py b/migrations/__init__.py index fb5ef987..519ec8c9 100644 --- a/migrations/__init__.py +++ b/migrations/__init__.py @@ -7,6 +7,8 @@ from .add_traceroute_id import migrate as add_traceroute_id from .add_positionlog_log_id import migrate as add_positionlog_log_id from .add_message_map_indexes import migrate as add_message_map_indexes +from .add_relay_node_to_reception import migrate as add_relay_node_to_reception +from .add_relay_edges_table import migrate as add_relay_edges_table # List of migrations to run in order MIGRATIONS = [ @@ -18,4 +20,6 @@ add_traceroute_id, add_positionlog_log_id, add_message_map_indexes, + add_relay_node_to_reception, + add_relay_edges_table, ] diff --git a/migrations/add_relay_edges_table.py b/migrations/add_relay_edges_table.py new file mode 100644 index 00000000..6bd80213 --- /dev/null +++ b/migrations/add_relay_edges_table.py @@ -0,0 +1,25 @@ +import logging + +def migrate(db): + """ + Create relay_edges table for inferred relay network paths. + """ + try: + cursor = db.cursor() + cursor.execute(""" + CREATE TABLE IF NOT EXISTS relay_edges ( + from_node VARCHAR(8) NOT NULL, + relay_suffix VARCHAR(2) NOT NULL, + to_node VARCHAR(8) NOT NULL, + first_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + last_seen TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + count INT DEFAULT 1, + PRIMARY KEY (from_node, relay_suffix, to_node) + ) + """) + db.commit() + logging.info("relay_edges table created or already exists.") + except Exception as e: + logging.error(f"Error creating relay_edges table: {e}") + db.rollback() + raise \ No newline at end of file diff --git a/migrations/add_relay_node_to_reception.py b/migrations/add_relay_node_to_reception.py new file mode 100644 index 00000000..81af70e2 --- /dev/null +++ b/migrations/add_relay_node_to_reception.py @@ -0,0 +1,41 @@ +import logging +import configparser + +def migrate(db): + """ + Migrate database to add relay_node column to message_reception table + """ + try: + cursor = db.cursor() + + # Ensure we're in the correct database + cursor.execute("SELECT DATABASE()") + current_db = cursor.fetchone()[0] + if not current_db: + raise Exception("No database selected") + + # Check if relay_node column exists in message_reception table + cursor.execute(""" + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_NAME = 'message_reception' + AND COLUMN_NAME = 'relay_node' + """) + has_relay_node = cursor.fetchone()[0] > 0 + + if not has_relay_node: + logging.info("Adding relay_node column to message_reception table...") + cursor.execute(""" + ALTER TABLE message_reception + ADD COLUMN relay_node VARCHAR(4) DEFAULT NULL, + ADD INDEX idx_message_reception_relay_node (relay_node) + """) + db.commit() + logging.info("Added relay_node column successfully") + else: + logging.info("relay_node column already exists in message_reception table") + + except Exception as e: + logging.error(f"Error during relay_node migration: {e}") + db.rollback() + raise \ No newline at end of file diff --git a/templates/graph4.html.j2 b/templates/graph4.html.j2 index be0a543d..1e514c93 100644 --- a/templates/graph4.html.j2 +++ b/templates/graph4.html.j2 @@ -140,7 +140,7 @@ name: 'cola', animate: true, refresh: 1, - maxSimulationTime: 15000, + maxSimulationTime: 500, nodeDimensionsIncludeLabels: true, fit: true, padding: 50, @@ -167,9 +167,9 @@ gridSnapIterations: 100, // Faster initial layout - initialUnconstrainedIterations: 100, - initialUserConstraintIterations: 100, - initialAllConstraintsIterations: 100, + initialUnconstrainedIterations: 1000, + initialUserConstraintIterations: 1000, + initialAllConstraintsIterations: 1000, groups: function() { // Create groups of nodes based on their connectivity diff --git a/templates/message-paths.html.j2 b/templates/message-paths.html.j2 new file mode 100644 index 00000000..f8c6a613 --- /dev/null +++ b/templates/message-paths.html.j2 @@ -0,0 +1,569 @@ +{% set this_page = "message-paths" %} +{% extends "layout.html.j2" %} + +{% block title %}Message Paths | MeshInfo{% endblock %} +{% block content %} +
+
📡 Message Paths
+

+ Visualizing inferred relay paths based on message propagation data. + Each edge represents a relay relationship where messages passed through intermediate nodes. +

+ +
+
+
+ + + + + +
+
+
+
+ + +
+
+ + +
+
+
+ +
+
+
+
+
+
+
{{ stats.total_nodes }}
+ Total Nodes +
+
+
{{ stats.total_edges }}
+ Relay Paths +
+
+
{{ stats.total_messages }}
+ Messages Relayed +
+
+
{{ "%.1f"|format(stats.avg_hops) }}
+ Avg Hops +
+
+
+
+
+
+
+ +
+ + + + + + + + + + + + + +{% endblock %} \ No newline at end of file diff --git a/templates/message_map.html.j2 b/templates/message_map.html.j2 index ca58ba2e..eab97236 100644 --- a/templates/message_map.html.j2 +++ b/templates/message_map.html.j2 @@ -185,7 +185,9 @@ type: 'receiver', lat: lat, // Use JS var lon: lon, // Use JS var - positionTime: {{ receiver_pos.position_time }} + positionTime: {{ receiver_pos.position_time }}, + relayNode: {% if details and details.relay_node %}'{{ details.relay_node }}'{% else %}null{% endif %}, + actualRelayNode: {% if details and details.relay_node %}{% set actual_relay = find_relay_node_by_suffix(details.relay_node[-2:], nodes, [receiver_id], message.from_id) %}{% if actual_relay %}'{{ actual_relay }}'{% else %}null{% endif %}{% else %}null{% endif %} }; console.log(" Created nodeData object:", nodeData); @@ -314,7 +316,9 @@ type: 'receiver', lat: lat, lon: lon, - positionTime: {{ receiver_pos.position_time }} + positionTime: {{ receiver_pos.position_time }}, + relayNode: {% if details and details.relay_node %}'{{ details.relay_node }}'{% else %}null{% endif %}, + actualRelayNode: {% if details and details.relay_node %}{% set actual_relay = find_relay_node_by_suffix(details.relay_node[-2:], nodes, [receiver_id], message.from_id) %}{% if actual_relay %}'{{ actual_relay }}'{% else %}null{% endif %}{% else %}null{% endif %} }; // Create receiver style with hop count text @@ -488,6 +492,20 @@ html += `Hops: ${node.hops}
`; } + // Add relay information for receivers + if (node.relayNode) { + if (node.actualRelayNode) { + // Show relay node's long_name as a link + let relayId = node.actualRelayNode; + let relayName = nodes[relayId] ? nodes[relayId].long_name : relayId; + html += `Relay Node: ${relayName}
`; + } else { + // Show only the last two hex digits + let relaySuffix = node.relayNode.slice(-2); + html += `Relay Suffix: ${relaySuffix} (unconfirmed)
`; + } + } + // Calculate distance using stored coordinates if (node.type === 'receiver') { var senderFeature = features[0]; From dc3b75e866e0c1761a450f376d695088fcb0bcbf Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 27 Jun 2025 19:57:23 +0000 Subject: [PATCH 063/155] Update MQTT server address in index template from mqtt.pugetmesh.org to meshtastic.pugetmesh.org. --- templates/index.html.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/index.html.j2 b/templates/index.html.j2 index c6b7e2a5..bed5cb0e 100644 --- a/templates/index.html.j2 +++ b/templates/index.html.j2 @@ -67,7 +67,7 @@
  • - Address: mqtt.pugetmesh.org + Address: meshtastic.pugetmesh.org
  • Username: meshdev From 8e480a9d5e40a51a4d9d446d3f39b9cb7b119855 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 27 Jun 2025 20:08:42 +0000 Subject: [PATCH 064/155] Add nodes data injection to message map template for enhanced map rendering. --- templates/message_map.html.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/message_map.html.j2 b/templates/message_map.html.j2 index eab97236..2ec2738e 100644 --- a/templates/message_map.html.j2 +++ b/templates/message_map.html.j2 @@ -74,6 +74,8 @@ import { isEmpty } from 'ol/extent.js'; import Polygon from 'ol/geom/Polygon.js'; + var nodes = {{ nodes|tojson|safe }}; + const map = new Map({ layers: [ new TileLayer({ From 3da3c525f6a4831fde981631e3a57758a536dd54 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 28 Jun 2025 00:26:22 +0000 Subject: [PATCH 065/155] Enhance node data structure and tooltip information in meshdata and message-paths templates. Added aggregation of first and last seen timestamps for nodes and relays, improved tooltip display for node and edge interactions, and introduced a time-ago formatting function for better user experience. --- meshdata.py | 59 +++++++++++++++++++----- templates/message-paths.html.j2 | 82 ++++++++++++++++++--------------- 2 files changed, 93 insertions(+), 48 deletions(-) diff --git a/meshdata.py b/meshdata.py index 9141977d..a7d72eea 100644 --- a/meshdata.py +++ b/meshdata.py @@ -2483,7 +2483,16 @@ def get_relay_network_data(self, days=30): else: from_node = f"relay_{relay_suffix}" if from_node not in virtual_nodes: - virtual_nodes[from_node] = {'id': from_node, 'relay_suffix': relay_suffix} + virtual_nodes[from_node] = { + 'id': from_node, + 'relay_suffix': relay_suffix, + 'short_name': from_node, + 'long_name': f"Relay {relay_suffix}", + 'hw_model': 'Virtual', + 'firmware_version': None, + 'role': None, + 'owner': None + } to_node = receiver_hex elif (hop_limit is None and hop_start is None) or (hop_start is not None and hop_limit is not None and hop_start - hop_limit == 0): # Zero-hop: direct sender->receiver @@ -2510,6 +2519,18 @@ def get_relay_network_data(self, days=30): endpoint_nodes.add(from_node) endpoint_nodes.add(to_node) + # Aggregate first_seen and last_seen for each node from edges + node_first_seen = {} + node_last_seen = {} + for (from_node, to_node), edge in edge_map.items(): + for node in [from_node, to_node]: + if edge['first_seen']: + if node not in node_first_seen or edge['first_seen'] < node_first_seen[node]: + node_first_seen[node] = edge['first_seen'] + if edge['last_seen']: + if node not in node_last_seen or edge['last_seen'] > node_last_seen[node]: + node_last_seen[node] = edge['last_seen'] + # 4. Build node list (real + virtual), but only those that are endpoints nodes = [] node_stats = {} @@ -2535,6 +2556,8 @@ def get_relay_network_data(self, days=30): 'role': node.get('role'), 'owner': node.get('owner'), 'last_seen': node.get('ts_seen'), + 'first_seen': node_first_seen.get(node_hex), + 'last_relay': node_last_seen.get(node_hex), 'message_count': node_stats[node_hex]['message_count'], 'relay_count': node_stats[node_hex]['relay_count'], 'icon_url': icon_url @@ -2543,17 +2566,31 @@ def get_relay_network_data(self, days=30): for node_hex in endpoint_nodes: if node_hex.startswith('relay_'): v = virtual_nodes[node_hex] + # Aggregate from edges where this relay is the from_node + relay_edges = [ + edge for (from_node, _), edge in edge_map.items() + if from_node == v['id'] + ] + if relay_edges: + # Edge fields: 'first_seen', 'last_seen' are timestamps, 'count' is message count + first_seen = min(e['first_seen'] for e in relay_edges if e['first_seen'] is not None) + last_seen = max(e['last_seen'] for e in relay_edges if e['last_seen'] is not None) + relay_count = sum(e['count'] for e in relay_edges if e.get('count')) + else: + first_seen = None + last_seen = None + relay_count = 0 nodes.append({ 'id': v['id'], - 'long_name': f"Relay {v['relay_suffix']}", - 'short_name': f"relay_{v['relay_suffix']}", - 'hw_model': 'Virtual', - 'firmware_version': None, - 'role': None, - 'owner': None, - 'last_seen': None, + 'short_name': v['short_name'], + 'long_name': v['long_name'], + 'hw_model': v['hw_model'], + 'hw_model_name': get_hardware_model_name(v['hw_model']) if v['hw_model'] is not None else None, + 'last_seen': last_seen, + 'first_seen': first_seen, + 'last_relay': last_seen, 'message_count': node_stats[v['id']]['message_count'], - 'relay_count': node_stats[v['id']]['relay_count'], + 'relay_count': relay_count, 'icon_url': None }) # 5. Build edge list @@ -2565,8 +2602,8 @@ def get_relay_network_data(self, days=30): 'to_node': to_node, 'relay_suffix': edge['relay_suffix'], 'message_count': edge['count'], - 'first_seen': datetime.datetime.fromtimestamp(edge['first_seen']).strftime('%Y-%m-%d %H:%M:%S') if edge['first_seen'] else None, - 'last_seen': datetime.datetime.fromtimestamp(edge['last_seen']).strftime('%Y-%m-%d %H:%M:%S') if edge['last_seen'] else None + 'first_seen': edge['first_seen'], # <-- raw timestamp + 'last_seen': edge['last_seen'] # <-- raw timestamp }) # 6. Stats total_nodes = len(nodes) diff --git a/templates/message-paths.html.j2 b/templates/message-paths.html.j2 index f8c6a613..af6f00b6 100644 --- a/templates/message-paths.html.j2 +++ b/templates/message-paths.html.j2 @@ -122,11 +122,35 @@ element.isNode() ? element.renderedPosition() : true; } - // Helper function to format timestamp - function formatTimestamp(ts) { - if (!ts || isNaN(ts)) return 'Never'; - var d = new Date(ts * 1000); - return d.toLocaleString(); + // Helper function to format timestamp as 'time ago' + function timeAgo(ts) { + if (!ts || ts === 'None' || ts === 'null') return 'Never'; + let date; + if (typeof ts === 'number') { + date = new Date(ts * 1000); + } else if (typeof ts === 'string' && !isNaN(Number(ts))) { + date = new Date(Number(ts) * 1000); + } else { + return 'Never'; + } + if (!date || isNaN(date.getTime())) return 'Never'; + const now = new Date(); + let seconds = Math.floor((now - date) / 1000); + const future = seconds < 0; + seconds = Math.abs(seconds); + // Clamp: If more than 5 minutes in the future, treat as 'just now' + if (future && seconds > 300) return 'just now'; + if (seconds < 60) return future ? `in ${seconds} seconds` : `${seconds} seconds ago`; + const minutes = Math.floor(seconds / 60); + if (minutes < 60) return future ? `in ${minutes} minutes` : `${minutes} minutes ago`; + const hours = Math.floor(minutes / 60); + if (hours < 24) return future ? `in ${hours} hours` : `${hours} hours ago`; + const days = Math.floor(hours / 24); + if (days < 30) return future ? `in ${days} days` : `${days} days ago`; + const months = Math.floor(days / 30); + if (months < 12) return future ? `in ${months} months` : `${months} months ago`; + const years = Math.floor(months / 12); + return future ? `in ${years} years` : `${years} years ago`; } // Store the original data @@ -151,7 +175,9 @@ short: node.short_name || node.id, long_name: node.long_name || 'Unknown', hw_model: node.hw_model || 'Unknown', - last_seen: node.last_seen || 'Never', + last_seen: node.last_seen, + last_relay: node.last_relay || node.last_seen, + first_seen: node.first_seen, message_count: node.message_count || 0, relay_count: node.relay_count || 0 }, node.icon_url ? { image: node.icon_url } : {}) @@ -352,9 +378,10 @@ // Handle node clicks cy.on('tap', 'node', function(e) { - e.preventDefault(); var nodeId = e.target.id(); - window.location.href = "node_" + nodeId.replace("!", "") + ".html"; + if (!nodeId.startsWith('relay_')) { + window.location.href = "node_" + nodeId.replace("!", "") + ".html"; + } }); // Add panzoom controls if available @@ -393,51 +420,47 @@ // Node tooltip handler cy.on('mouseover', 'node', function(e) { const node = e.target; - // Defensive: destroy any existing tooltip destroyCurrentTippy(); - // Defensive: check node is still valid and visible if (!isElementValid(node)) { console.warn('Node not valid for tooltip:', node.id()); return; } - // Highlight connected elements const connectedEdges = node.connectedEdges(); const connectedNodes = node.neighborhood().nodes(); - // Add highlighting classes node.addClass('highlighted-node'); connectedEdges.addClass('highlighted'); connectedNodes.addClass('highlighted-node'); - // Fade other elements cy.elements().removeClass('faded'); cy.elements().difference(connectedEdges.union(connectedNodes).union(node)).addClass('faded'); - try { const nodeData = node.data(); + const isVirtualRelay = nodeData.id && nodeData.id.startsWith('relay_'); + const nodeNameHtml = isVirtualRelay + ? `${nodeData.long_name || nodeData.id}` + : `${nodeData.long_name || nodeData.id}`; const content = `
    - ${nodeData.long_name || 'Unknown'}
    + ${nodeNameHtml}
    ${nodeData.id}

    - Hardware: ${nodeData.hw_model || 'Unknown'}
    - Last Seen: ${formatTimestamp(nodeData.last_seen)}
    + Hardware: ${nodeData.hw_model_name || nodeData.hw_model || 'Unknown'}
    + First Relay: ${timeAgo(nodeData.first_seen)}
    + Last Relay: ${timeAgo(nodeData.last_relay || nodeData.last_seen)}
    Messages Sent: ${nodeData.message_count || 0}
    Relay Count: ${nodeData.relay_count || 0}
    Connections: ${node.neighborhood().length - 1}
    `; - // Create dummy element for tippy const dummyDomEle = document.createElement('div'); dummyDomEle.innerHTML = content; document.body.appendChild(dummyDomEle); - // Get reference for positioning const ref = node.popperRef(); - // Create tippy instance currentTippy = tippy(dummyDomEle, { getReferenceClientRect: ref.getBoundingClientRect, @@ -451,10 +474,8 @@ theme: 'light-border', appendTo: document.body }); - // Show the tooltip currentTippy.show(); - } catch (error) { console.error('Error creating node tooltip:', error); destroyCurrentTippy(); @@ -464,59 +485,48 @@ // Edge tooltip handler cy.on('mouseover', 'edge', function(e) { const edge = e.target; - // Defensive: destroy any existing tooltip destroyCurrentTippy(); - // Defensive: check edge is still valid and visible if (!isElementValid(edge)) { console.warn('Edge not valid for tooltip:', edge.id()); return; } - // Highlight connected elements const sourceNode = edge.source(); const targetNode = edge.target(); - // Add highlighting classes edge.addClass('highlighted'); sourceNode.addClass('highlighted-node'); targetNode.addClass('highlighted-node'); - // Fade other elements cy.elements().removeClass('faded'); cy.elements().difference(cy.collection([edge, sourceNode, targetNode])).addClass('faded'); - try { const edgeData = edge.data(); const sourceNode = edge.source(); const targetNode = edge.target(); - let relayInfo = ''; if (edgeData.relay_suffix && edgeData.relay_suffix !== 'null' && edgeData.relay_suffix !== null && edgeData.relay_suffix !== undefined && edgeData.relay_suffix !== '') { relayInfo = `
    Relay: ${edgeData.relay_suffix}`; } else { relayInfo = '
    Type: Direct'; } - const content = `
    ${sourceNode.data('short') || sourceNode.id()} → ${targetNode.data('short') || targetNode.id()}
    ${edgeData.id}

    Messages: ${edgeData.message_count || 0}${relayInfo}
    - First Seen: ${formatTimestamp(edgeData.first_seen)}
    - Last Seen: ${formatTimestamp(edgeData.last_seen)} + First Seen: ${timeAgo(edgeData.first_seen)}
    + Last Seen: ${timeAgo(edgeData.last_seen)}
    `; - // Create dummy element for tippy const dummyDomEle = document.createElement('div'); dummyDomEle.innerHTML = content; document.body.appendChild(dummyDomEle); - // Get reference for positioning (use edge midpoint) const ref = edge.popperRef(); - // Create tippy instance currentTippy = tippy(dummyDomEle, { getReferenceClientRect: ref.getBoundingClientRect, @@ -530,10 +540,8 @@ theme: 'light-border', appendTo: document.body }); - // Show the tooltip currentTippy.show(); - } catch (error) { console.error('Error creating edge tooltip:', error); destroyCurrentTippy(); From b124f34ef982e2b943043759ccf310b1fc17c0aa Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 28 Jun 2025 02:25:02 +0000 Subject: [PATCH 066/155] Refactor relay node matching logic and enhance message map template. Improved the find_relay_node_by_suffix function to prioritize zero-hop neighbors and physical proximity, while updating the message map to include zero-hop link data for better relay node inference. Adjusted template to dynamically highlight active time filters based on user selection. --- meshdata.py | 233 +++++++++++++++++++------------- meshinfo_web.py | 203 ++++++++++++++++++++++------ templates/message-paths.html.j2 | 11 +- 3 files changed, 309 insertions(+), 138 deletions(-) diff --git a/meshdata.py b/meshdata.py index a7d72eea..6ed132b6 100644 --- a/meshdata.py +++ b/meshdata.py @@ -9,6 +9,7 @@ from timezone_utils import time_ago # Import time_ago from timezone_utils from meshtastic_support import get_hardware_model_name # Import get_hardware_model_name from meshtastic_support import types +from collections import defaultdict, deque class CustomJSONEncoder(json.JSONEncoder): @@ -2432,11 +2433,8 @@ def get_relay_network_data(self, days=30): """Infer relay network data from message_reception table, not relay_edges.""" cursor = None try: - # Ensure database connection is valid if not self.db or not self.db.is_connected(): self.connect_db() - - # 1. Load all relevant message_reception rows cursor = self.db.cursor(dictionary=True) if days > 0: cutoff = int(time.time()) - days * 86400 @@ -2451,16 +2449,30 @@ def get_relay_network_data(self, days=30): FROM message_reception """) receptions = cursor.fetchall() - - # 2. Load all nodes for lookup nodes_dict = self.get_nodes() node_ids_set = set(nodes_dict.keys()) node_id_ints = {int(k, 16): k for k in node_ids_set} - # 3. Build relay edges and virtual relay nodes - edge_map = {} # (from, to) -> {count, relay_suffix, first_seen, last_seen} - virtual_nodes = {} # relay_xx -> {id, relay_suffix} - endpoint_nodes = set() # Only nodes that are endpoints of at least one edge + # 1. Build zero-hop (direct) network + zero_hop_edges = [] + for rec in receptions: + hop_limit = rec['hop_limit'] + hop_start = rec['hop_start'] + sender = rec['from_id'] + receiver = rec['received_by_id'] + sender_hex = format(sender, '08x') + receiver_hex = format(receiver, '08x') + if (hop_limit is None and hop_start is None) or (hop_start is not None and hop_limit is not None and hop_start - hop_limit == 0): + zero_hop_edges.append((sender_hex, receiver_hex)) + + # Build zero-hop adjacency for real nodes + zero_hop_graph = defaultdict(set) + for a, b in zero_hop_edges: + zero_hop_graph[a].add(b) + zero_hop_graph[b].add(a) + + # 2. Group relay receptions by suffix + relay_suffix_edges = defaultdict(list) for rec in receptions: sender = rec['from_id'] receiver = rec['received_by_id'] @@ -2470,56 +2482,115 @@ def get_relay_network_data(self, days=30): hop_start = rec['hop_start'] sender_hex = format(sender, '08x') receiver_hex = format(receiver, '08x') - # Determine edge type if relay_node: relay_suffix = relay_node[-2:].lower() - # Try to match relay_node to a known node - relay_match = None - for node_hex in node_ids_set: - if node_hex[-2:] == relay_suffix: - relay_match = node_hex if not relay_match else None # Only if unique - if relay_match: - from_node = relay_match - else: - from_node = f"relay_{relay_suffix}" - if from_node not in virtual_nodes: - virtual_nodes[from_node] = { - 'id': from_node, - 'relay_suffix': relay_suffix, - 'short_name': from_node, - 'long_name': f"Relay {relay_suffix}", - 'hw_model': 'Virtual', - 'firmware_version': None, - 'role': None, - 'owner': None - } - to_node = receiver_hex + relay_suffix_edges[relay_suffix].append((sender_hex, receiver_hex, rx_time, relay_node)) elif (hop_limit is None and hop_start is None) or (hop_start is not None and hop_limit is not None and hop_start - hop_limit == 0): - # Zero-hop: direct sender->receiver - from_node = sender_hex - to_node = receiver_hex - else: - continue # Ignore receptions that don't fit the above - # Edge key - edge_key = (from_node, to_node) - if edge_key not in edge_map: - edge_map[edge_key] = { - 'count': 0, - 'relay_suffix': relay_node[-2:].lower() if relay_node else None, - 'first_seen': rx_time, - 'last_seen': rx_time - } - edge_map[edge_key]['count'] += 1 - if rx_time: - if edge_map[edge_key]['first_seen'] is None or rx_time < edge_map[edge_key]['first_seen']: - edge_map[edge_key]['first_seen'] = rx_time - if edge_map[edge_key]['last_seen'] is None or rx_time > edge_map[edge_key]['last_seen']: - edge_map[edge_key]['last_seen'] = rx_time - # Track endpoints - endpoint_nodes.add(from_node) - endpoint_nodes.add(to_node) - - # Aggregate first_seen and last_seen for each node from edges + relay_suffix_edges[None].append((sender_hex, receiver_hex, rx_time, None)) + + # 3. Find connected components for each relay suffix + relay_suffix_components = {} + relay_suffix_node_to_component = {} + for suffix, edges in relay_suffix_edges.items(): + graph = defaultdict(set) + for from_hex, to_hex, _, _ in edges: + graph[from_hex].add(to_hex) + graph[to_hex].add(from_hex) + visited = set() + components = [] + for node in graph: + if node in visited: + continue + queue = deque([node]) + comp = set() + while queue: + n = queue.popleft() + if n in visited: + continue + visited.add(n) + comp.add(n) + for neighbor in graph[n]: + if neighbor not in visited: + queue.append(neighbor) + if comp: + components.append(comp) + for idx, comp in enumerate(components): + relay_suffix_components[(suffix, idx)] = comp + for n in comp: + relay_suffix_node_to_component[(suffix, n)] = idx + + edge_map = {} + virtual_nodes = {} + endpoint_nodes = set() + # 4. Component-local consolidation + for suffix, edges in relay_suffix_edges.items(): + # Group edges by component + comp_edges = defaultdict(list) + for from_hex, to_hex, rx_time, relay_node in edges: + comp_idx = relay_suffix_node_to_component.get((suffix, from_hex), 0) + comp_edges[comp_idx].append((from_hex, to_hex, rx_time, relay_node)) + for comp_idx, comp_edge_list in comp_edges.items(): + comp_nodes = relay_suffix_components[(suffix, comp_idx)] + # Find all real nodes in the component with this suffix + real_nodes_with_suffix = [n for n in comp_nodes if n in node_ids_set and n[-2:] == (suffix if suffix is not None else '')] + if len(real_nodes_with_suffix) == 1 and suffix is not None: + # Consolidate: use the real node for all edges and endpoints in this component + unique_real_node = real_nodes_with_suffix[0] + for from_hex, to_hex, rx_time, relay_node in comp_edge_list: + edge_from = unique_real_node + edge_to = to_hex if to_hex != from_hex else unique_real_node + edge_key = (edge_from, edge_to) + if edge_key not in edge_map: + edge_map[edge_key] = { + 'count': 0, + 'relay_suffix': suffix, + 'first_seen': rx_time, + 'last_seen': rx_time, + 'virtual_id': None + } + edge_map[edge_key]['count'] += 1 + if rx_time: + if edge_map[edge_key]['first_seen'] is None or rx_time < edge_map[edge_key]['first_seen']: + edge_map[edge_key]['first_seen'] = rx_time + if edge_map[edge_key]['last_seen'] is None or rx_time > edge_map[edge_key]['last_seen']: + edge_map[edge_key]['last_seen'] = rx_time + endpoint_nodes.add(edge_from) + endpoint_nodes.add(edge_to) + else: + # Use a virtual node for this component + virtual_id = f"relay_{suffix}_{comp_idx+1}" if suffix is not None else None + for from_hex, to_hex, rx_time, relay_node in comp_edge_list: + edge_from = virtual_id if virtual_id else from_hex + if virtual_id and virtual_id not in virtual_nodes: + virtual_nodes[virtual_id] = { + 'id': virtual_id, + 'relay_suffix': suffix, + 'short_name': virtual_id, + 'long_name': f"Relay {suffix} ({comp_idx+1})" if suffix is not None else 'Relay', + 'hw_model': 'Virtual', + 'firmware_version': None, + 'role': None, + 'owner': None + } + edge_to = to_hex + edge_key = (edge_from, edge_to) + if edge_key not in edge_map: + edge_map[edge_key] = { + 'count': 0, + 'relay_suffix': suffix, + 'first_seen': rx_time, + 'last_seen': rx_time, + 'virtual_id': virtual_id if suffix is not None else None + } + edge_map[edge_key]['count'] += 1 + if rx_time: + if edge_map[edge_key]['first_seen'] is None or rx_time < edge_map[edge_key]['first_seen']: + edge_map[edge_key]['first_seen'] = rx_time + if edge_map[edge_key]['last_seen'] is None or rx_time > edge_map[edge_key]['last_seen']: + edge_map[edge_key]['last_seen'] = rx_time + endpoint_nodes.add(edge_from) + endpoint_nodes.add(edge_to) + node_first_seen = {} node_last_seen = {} for (from_node, to_node), edge in edge_map.items(): @@ -2531,21 +2602,17 @@ def get_relay_network_data(self, days=30): if node not in node_last_seen or edge['last_seen'] > node_last_seen[node]: node_last_seen[node] = edge['last_seen'] - # 4. Build node list (real + virtual), but only those that are endpoints nodes = [] node_stats = {} for node_hex in endpoint_nodes: node_stats[node_hex] = {'message_count': 0, 'relay_count': 0} - # Count stats for (from_node, to_node), edge in edge_map.items(): node_stats[from_node]['message_count'] += edge['count'] node_stats[to_node]['relay_count'] += edge['count'] - # Real nodes for node_hex in endpoint_nodes: if node_hex in nodes_dict: node = nodes_dict[node_hex] - # Use graph_icon utility for robust icon path - node_name_for_icon = node.get('long_name', node.get('short_name', '')) + node_name_for_icon = node.get('long_name') or node.get('short_name', '') icon_url = utils.graph_icon(node_name_for_icon) nodes.append({ 'id': node_hex, @@ -2562,38 +2629,21 @@ def get_relay_network_data(self, days=30): 'relay_count': node_stats[node_hex]['relay_count'], 'icon_url': icon_url }) - # Virtual relay nodes for node_hex in endpoint_nodes: - if node_hex.startswith('relay_'): - v = virtual_nodes[node_hex] - # Aggregate from edges where this relay is the from_node - relay_edges = [ - edge for (from_node, _), edge in edge_map.items() - if from_node == v['id'] - ] - if relay_edges: - # Edge fields: 'first_seen', 'last_seen' are timestamps, 'count' is message count - first_seen = min(e['first_seen'] for e in relay_edges if e['first_seen'] is not None) - last_seen = max(e['last_seen'] for e in relay_edges if e['last_seen'] is not None) - relay_count = sum(e['count'] for e in relay_edges if e.get('count')) - else: - first_seen = None - last_seen = None - relay_count = 0 + if node_hex.startswith('relay_') and node_hex in virtual_nodes: nodes.append({ - 'id': v['id'], - 'short_name': v['short_name'], - 'long_name': v['long_name'], - 'hw_model': v['hw_model'], - 'hw_model_name': get_hardware_model_name(v['hw_model']) if v['hw_model'] is not None else None, - 'last_seen': last_seen, - 'first_seen': first_seen, - 'last_relay': last_seen, - 'message_count': node_stats[v['id']]['message_count'], - 'relay_count': relay_count, + 'id': virtual_nodes[node_hex]['id'], + 'short_name': virtual_nodes[node_hex]['short_name'], + 'long_name': virtual_nodes[node_hex]['long_name'], + 'hw_model': virtual_nodes[node_hex]['hw_model'], + 'hw_model_name': get_hardware_model_name(virtual_nodes[node_hex]['hw_model']) if virtual_nodes[node_hex]['hw_model'] is not None else None, + 'last_seen': node_last_seen.get(node_hex), + 'first_seen': node_first_seen.get(node_hex), + 'last_relay': node_last_seen.get(node_hex), + 'message_count': node_stats[node_hex]['message_count'], + 'relay_count': node_stats[node_hex]['relay_count'], 'icon_url': None }) - # 5. Build edge list edges = [] for (from_node, to_node), edge in edge_map.items(): edges.append({ @@ -2602,10 +2652,9 @@ def get_relay_network_data(self, days=30): 'to_node': to_node, 'relay_suffix': edge['relay_suffix'], 'message_count': edge['count'], - 'first_seen': edge['first_seen'], # <-- raw timestamp - 'last_seen': edge['last_seen'] # <-- raw timestamp + 'first_seen': edge['first_seen'], + 'last_seen': edge['last_seen'] }) - # 6. Stats total_nodes = len(nodes) total_edges = len(edges) total_messages = sum(e['message_count'] for e in edges) diff --git a/meshinfo_web.py b/meshinfo_web.py index 84da4955..f9ab8f1a 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -684,20 +684,36 @@ def message_map(): if not data: return redirect(url_for('chat')) + # --- Provide zero_hop_links and position data for relay node inference --- + md = get_meshdata() + nodes = data['nodes'] + sender_id = data['message']['from_id'] + receiver_ids = data['message']['receiver_ids'] + # Get zero-hop links for the last 1 day (or configurable) + zero_hop_timeout = 86400 + cutoff_time = int(time.time()) - zero_hop_timeout + zero_hop_links, _ = md.get_zero_hop_links(cutoff_time) + # Get sender and receiver positions at message time + message_time = data['message']['ts_created'] + sender_pos = data['sender_position'] + receiver_positions = data['receiver_positions'] + # Pass the relay matcher and context to the template return render_template( "message_map.html.j2", auth=auth(), config=config, - nodes=data['nodes'], + nodes=nodes, message=data['message'], - sender_position=data['sender_position'], - receiver_positions=data['receiver_positions'], + sender_position=sender_pos, + receiver_positions=receiver_positions, receiver_details=data['receiver_details'], convex_hull_area_km2=data['convex_hull_area_km2'], utils=utils, datetime=datetime.datetime, timestamp=datetime.datetime.now(), - find_relay_node_by_suffix=find_relay_node_by_suffix + find_relay_node_by_suffix=lambda relay_suffix, nodes, receiver_ids=None, sender_id=None: find_relay_node_by_suffix( + relay_suffix, nodes, receiver_ids, sender_id, zero_hop_links=zero_hop_links, sender_pos=sender_pos, receiver_pos=None + ) ) @app.route('/traceroute_map.html') @@ -2320,45 +2336,150 @@ def clear_nodes_singleton(): else: logging.info("Nodes singleton was already None") -@cache.memoize(timeout=300) # Cache for 5 minutes -def find_relay_node_by_suffix(relay_suffix, nodes, receiver_ids=None, sender_id=None): - """Find the actual relay node by matching the last 2 hex chars against known node IDs.""" - if not relay_suffix or len(relay_suffix) < 2: - return None - relay_suffix = relay_suffix.lower()[-2:] # Only last two hex digits - - # 1. First, check if the sender itself matches the relay suffix - if sender_id: - sender_id_hex = utils.convert_node_id_from_int_to_hex(sender_id) - if sender_id_hex.lower()[-2:] == relay_suffix: - return sender_id_hex - - # 2. Check if any of the message receivers match the relay suffix - if receiver_ids: - for receiver_id in receiver_ids: - receiver_id_hex = utils.convert_node_id_from_int_to_hex(receiver_id) - if receiver_id_hex.lower()[-2:] == relay_suffix: - return receiver_id_hex - - # 3. Check active nodes (nodes that have been seen recently) - active_nodes = [] - for node_id_hex, node_data in nodes.items(): - if len(node_id_hex) == 8 and node_id_hex.lower()[-2:] == relay_suffix: - if (node_data.get('telemetry') or - node_data.get('position') or - node_data.get('ts_seen')): - active_nodes.append(node_id_hex) - if len(active_nodes) == 1: - return active_nodes[0] - - # 4. If no active nodes or multiple active nodes, check all known nodes - matching_nodes = [] +def find_relay_node_by_suffix(relay_suffix, nodes, receiver_ids=None, sender_id=None, zero_hop_links=None, sender_pos=None, receiver_pos=None, debug=False): + """ + Improved relay node matcher: prefer zero-hop/extended neighbors, then select the physically closest candidate to the sender (or receiver), using scoring only as a tiebreaker. + """ + import time + relay_suffix = relay_suffix.lower()[-2:] + candidates = [] for node_id_hex, node_data in nodes.items(): if len(node_id_hex) == 8 and node_id_hex.lower()[-2:] == relay_suffix: - matching_nodes.append(node_id_hex) - if len(matching_nodes) == 1: - return matching_nodes[0] - return None + candidates.append((node_id_hex, node_data)) + + if not candidates: + if debug: + print(f"[RelayMatch] No candidates for suffix {relay_suffix}") + return None + if len(candidates) == 1: + if debug: + print(f"[RelayMatch] Only one candidate for suffix {relay_suffix}: {candidates[0][0]}") + return candidates[0][0] + + # --- Zero-hop filter: only consider zero-hop neighbors if any exist --- + zero_hop_candidates = [] + if zero_hop_links: + for node_id_hex, node_data in candidates: + is_zero_hop = False + if sender_id and node_id_hex in zero_hop_links.get(sender_id, {}).get('heard', {}): + is_zero_hop = True + if receiver_ids: + for rid in receiver_ids: + if node_id_hex in zero_hop_links.get(rid, {}).get('heard', {}): + is_zero_hop = True + if is_zero_hop: + zero_hop_candidates.append((node_id_hex, node_data)) + if zero_hop_candidates: + if debug: + print(f"[RelayMatch] Restricting to zero-hop candidates: {[c[0] for c in zero_hop_candidates]}") + candidates = zero_hop_candidates + else: + # --- Extended neighbor filter: only consider candidates that have ever been heard by or heard from sender/receivers --- + extended_candidates = [] + if zero_hop_links: + local_set = set() + if sender_id and sender_id in zero_hop_links: + local_set.update(zero_hop_links[sender_id].get('heard', {}).keys()) + local_set.update(zero_hop_links[sender_id].get('heard_by', {}).keys()) + if receiver_ids: + for rid in receiver_ids: + if rid in zero_hop_links: + local_set.update(zero_hop_links[rid].get('heard', {}).keys()) + local_set.update(zero_hop_links[rid].get('heard_by', {}).keys()) + local_set_hex = set() + for n in local_set: + try: + if isinstance(n, int): + local_set_hex.add(utils.convert_node_id_from_int_to_hex(n)) + elif isinstance(n, str) and len(n) == 8: + local_set_hex.add(n) + except Exception: + continue + for node_id_hex, node_data in candidates: + if node_id_hex in local_set_hex: + extended_candidates.append((node_id_hex, node_data)) + if extended_candidates: + if debug: + print(f"[RelayMatch] Restricting to extended neighbor candidates: {[c[0] for c in extended_candidates]}") + candidates = extended_candidates + else: + if debug: + print(f"[RelayMatch] No local/extended candidates, using all: {[c[0] for c in candidates]}") + + # --- Distance-first selection among remaining candidates --- + def get_distance(node_data, ref_pos): + npos = node_data.get('position') + if not npos or not ref_pos: + return float('inf') + nlat = npos.get('latitude') if isinstance(npos, dict) else getattr(npos, 'latitude', None) + nlon = npos.get('longitude') if isinstance(npos, dict) else getattr(npos, 'longitude', None) + if nlat is None or nlon is None: + return float('inf') + return utils.distance_between_two_points(ref_pos['lat'], ref_pos['lon'], nlat, nlon) + + ref_pos = sender_pos if sender_pos else receiver_pos + if ref_pos: + # Compute distances + distances = [(node_id_hex, node_data, get_distance(node_data, ref_pos)) for node_id_hex, node_data in candidates] + min_dist = min(d[2] for d in distances) + closest = [d for d in distances if abs(d[2] - min_dist) < 1e-3] # Allow for float rounding + if debug: + print(f"[RelayMatch] Closest candidates by distance: {[(c[0], c[2]) for c in closest]}") + if len(closest) == 1: + return closest[0][0] + # If tie, fall back to scoring among closest + candidates = [(c[0], c[1]) for c in closest] + + # --- Scoring system as tiebreaker --- + scores = {} + now = time.time() + for node_id_hex, node_data in candidates: + score = 0 + reasons = [] + if zero_hop_links: + if sender_id and node_id_hex in zero_hop_links.get(sender_id, {}).get('heard', {}): + score += 100 + reasons.append('zero-hop-sender') + if receiver_ids: + for rid in receiver_ids: + if node_id_hex in zero_hop_links.get(rid, {}).get('heard', {}): + score += 100 + reasons.append(f'zero-hop-receiver-{rid}') + proximity_score = 0 + pos_fresh = False + if sender_pos and node_data.get('position'): + npos = node_data['position'] + nlat = npos.get('latitude') if isinstance(npos, dict) else getattr(npos, 'latitude', None) + nlon = npos.get('longitude') if isinstance(npos, dict) else getattr(npos, 'longitude', None) + ntime = npos.get('position_time') if isinstance(npos, dict) else getattr(npos, 'position_time', None) + if nlat is not None and nlon is not None and ntime is not None: + if now - ntime > 21600: + score -= 50 + reasons.append('stale-position') + else: + pos_fresh = True + dist = utils.distance_between_two_points(sender_pos['lat'], sender_pos['lon'], nlat, nlon) + proximity_score = max(0, 100 - dist * 2) + score += proximity_score + reasons.append(f'proximity:{dist:.1f}km(+{proximity_score:.1f})') + else: + score -= 100 + reasons.append('missing-position') + if node_data.get('ts_seen') and (now - node_data['ts_seen'] < 3600): + score += 10 + reasons.append('recently-seen') + if node_data.get('role') not in [1, 8]: + score += 5 + reasons.append('relay-capable') + scores[node_id_hex] = (score, reasons) + if debug: + print(f"[RelayMatch] Candidates for suffix {relay_suffix}:") + for nid, (score, reasons) in scores.items(): + print(f" {nid}: score={score}, reasons={reasons}") + best = max(scores.items(), key=lambda x: x[1][0]) + if debug: + print(f"[RelayMatch] Selected {best[0]} for suffix {relay_suffix} (score={best[1][0]})") + return best[0] @app.route('/message-paths.html') def message_paths(): diff --git a/templates/message-paths.html.j2 b/templates/message-paths.html.j2 index af6f00b6..d6565bf2 100644 --- a/templates/message-paths.html.j2 +++ b/templates/message-paths.html.j2 @@ -12,12 +12,13 @@
    + {% set days_val = request.args.get('days', '0.167') %}
    - - - - - + + + + +
    From a457ede5f81de4760b45de2c047fd2ed8e841a56 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 28 Jun 2025 02:30:00 +0000 Subject: [PATCH 067/155] Fix distance calculation in find_relay_node_by_suffix function to use 'latitude' and 'longitude' keys instead of 'lat' and 'lon'. Added checks for missing coordinates to improve reliability of proximity scoring. --- meshinfo_web.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/meshinfo_web.py b/meshinfo_web.py index f9ab8f1a..0b05cbb7 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -2415,7 +2415,12 @@ def get_distance(node_data, ref_pos): nlon = npos.get('longitude') if isinstance(npos, dict) else getattr(npos, 'longitude', None) if nlat is None or nlon is None: return float('inf') - return utils.distance_between_two_points(ref_pos['lat'], ref_pos['lon'], nlat, nlon) + # Fix: Use 'latitude' and 'longitude' keys, not 'lat' and 'lon' + ref_lat = ref_pos.get('latitude') if isinstance(ref_pos, dict) else getattr(ref_pos, 'latitude', None) + ref_lon = ref_pos.get('longitude') if isinstance(ref_pos, dict) else getattr(ref_pos, 'longitude', None) + if ref_lat is None or ref_lon is None: + return float('inf') + return utils.distance_between_two_points(ref_lat, ref_lon, nlat, nlon) ref_pos = sender_pos if sender_pos else receiver_pos if ref_pos: @@ -2458,10 +2463,17 @@ def get_distance(node_data, ref_pos): reasons.append('stale-position') else: pos_fresh = True - dist = utils.distance_between_two_points(sender_pos['lat'], sender_pos['lon'], nlat, nlon) - proximity_score = max(0, 100 - dist * 2) - score += proximity_score - reasons.append(f'proximity:{dist:.1f}km(+{proximity_score:.1f})') + # Fix: Use 'latitude' and 'longitude' keys, not 'lat' and 'lon' + sender_lat = sender_pos.get('latitude') if isinstance(sender_pos, dict) else getattr(sender_pos, 'latitude', None) + sender_lon = sender_pos.get('longitude') if isinstance(sender_pos, dict) else getattr(sender_pos, 'longitude', None) + if sender_lat is not None and sender_lon is not None: + dist = utils.distance_between_two_points(sender_lat, sender_lon, nlat, nlon) + proximity_score = max(0, 100 - dist * 2) + score += proximity_score + reasons.append(f'proximity:{dist:.1f}km(+{proximity_score:.1f})') + else: + score -= 50 + reasons.append('missing-sender-position') else: score -= 100 reasons.append('missing-position') From 93b54d9827db1c340313cae5969730ec09ab263b Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 28 Jun 2025 02:41:18 +0000 Subject: [PATCH 068/155] Update message map template to conditionally display relay information based on Shift key press. Added hint for users to hold Shift for relay details when not shown. --- templates/message_map.html.j2 | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/templates/message_map.html.j2 b/templates/message_map.html.j2 index 2ec2738e..9829dbc0 100644 --- a/templates/message_map.html.j2 +++ b/templates/message_map.html.j2 @@ -474,6 +474,7 @@ if (feature && feature.get('node')) { var node = feature.get('node'); var coordinates = feature.getGeometry().getCoordinates(); + var showRelayInfo = evt.originalEvent && evt.originalEvent.shiftKey; var html = '
    '; @@ -494,8 +495,8 @@ html += `Hops: ${node.hops}
    `; } - // Add relay information for receivers - if (node.relayNode) { + // Add relay information for receivers only when Shift is held + if (node.relayNode && showRelayInfo) { if (node.actualRelayNode) { // Show relay node's long_name as a link let relayId = node.actualRelayNode; @@ -529,6 +530,11 @@ html += `Position Updated: ${formatTimestamp(node.positionTime)}
    `; } + // Add hint about relay information if available but not shown + if (node.relayNode && !showRelayInfo) { + html += 'Hold Shift for relay details
    '; + } + html += '
    '; content.innerHTML = html; From 265b793b70733b31891d72b6519ba0c85923ec37 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 28 Jun 2025 02:45:11 +0000 Subject: [PATCH 069/155] Remove hint for relay information in message map template. This change simplifies the user interface by eliminating the suggestion to hold Shift for relay details when not displayed. --- templates/message_map.html.j2 | 5 ----- 1 file changed, 5 deletions(-) diff --git a/templates/message_map.html.j2 b/templates/message_map.html.j2 index 9829dbc0..45e52515 100644 --- a/templates/message_map.html.j2 +++ b/templates/message_map.html.j2 @@ -530,11 +530,6 @@ html += `Position Updated: ${formatTimestamp(node.positionTime)}
    `; } - // Add hint about relay information if available but not shown - if (node.relayNode && !showRelayInfo) { - html += 'Hold Shift for relay details
    '; - } - html += '
    '; content.innerHTML = html; From 0bd90ff280debd5d5d60a3ab5e20cec621710590 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 28 Jun 2025 03:16:27 +0000 Subject: [PATCH 070/155] Implement configurable log retention and enhance log filtering in templates. Added a configurable retention count for meshlog entries, defaulting to 1000, and updated the logs template to support node filtering. Improved handling of datetime objects in scoring logic for relay nodes. --- meshdata.py | 8 +++++++- meshinfo_web.py | 19 ++++++++++++++++--- templates/logs.html.j2 | 12 +++++++++++- templates/node.html.j2 | 3 ++- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/meshdata.py b/meshdata.py index 6ed132b6..dcfd16e9 100644 --- a/meshdata.py +++ b/meshdata.py @@ -1524,7 +1524,13 @@ def log_data(self, topic, data): cur = self.db.cursor() cur.execute(f"SELECT COUNT(*) FROM meshlog") count = cur.fetchone()[0] - if count >= 1000: + + # Get configurable retention count, default to 1000 if not set + retention_count = self.config.getint("server", "log_retention_count", fallback=1000) + + if count >= retention_count: + if self.debug: + logging.debug(f"Log retention limit reached ({count} >= {retention_count}), removing oldest log entry") cur.execute(f"DELETE FROM meshlog ORDER BY ts_created ASC LIMIT 1") self.db.commit() diff --git a/meshinfo_web.py b/meshinfo_web.py index 0b05cbb7..b6ba641d 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -1111,12 +1111,17 @@ def logs(): md = get_meshdata() if not md: # Check if MeshData failed to initialize abort(503, description="Database connection unavailable") + + # Get node filter from query parameter + node_filter = request.args.get('node') + logs = md.get_logs() return render_template( "logs.html.j2", auth=auth(), config=config, logs=logs, + node_filter=node_filter, # Pass the node filter to template utils=utils, datetime=datetime.datetime, timestamp=datetime.datetime.now(), @@ -2458,6 +2463,9 @@ def get_distance(node_data, ref_pos): nlon = npos.get('longitude') if isinstance(npos, dict) else getattr(npos, 'longitude', None) ntime = npos.get('position_time') if isinstance(npos, dict) else getattr(npos, 'position_time', None) if nlat is not None and nlon is not None and ntime is not None: + # Convert datetime to timestamp if needed + if isinstance(ntime, datetime.datetime): + ntime = ntime.timestamp() if now - ntime > 21600: score -= 50 reasons.append('stale-position') @@ -2477,9 +2485,14 @@ def get_distance(node_data, ref_pos): else: score -= 100 reasons.append('missing-position') - if node_data.get('ts_seen') and (now - node_data['ts_seen'] < 3600): - score += 10 - reasons.append('recently-seen') + ts_seen = node_data.get('ts_seen') + if ts_seen: + # Convert datetime to timestamp if needed + if isinstance(ts_seen, datetime.datetime): + ts_seen = ts_seen.timestamp() + if now - ts_seen < 3600: + score += 10 + reasons.append('recently-seen') if node_data.get('role') not in [1, 8]: score += 5 reasons.append('relay-capable') diff --git a/templates/logs.html.j2 b/templates/logs.html.j2 index 014ca7db..96bb65ce 100644 --- a/templates/logs.html.j2 +++ b/templates/logs.html.j2 @@ -33,7 +33,7 @@
    - +
    @@ -290,6 +290,16 @@ document.addEventListener('DOMContentLoaded', function() { applyFilters(); }); + // Auto-apply filter if node parameter is provided + {% if node_filter %} + // If a node filter was provided via URL, apply it automatically + console.log('Auto-applying node filter for: {{ node_filter }}'); + // Trigger the input event to ensure the filter is applied + setTimeout(() => { + senderFilter.dispatchEvent(new Event('input', { bubbles: true })); + }, 100); // Small delay to ensure DOM is ready + {% endif %} + // Initial filter application applyFilters(); diff --git a/templates/node.html.j2 b/templates/node.html.j2 index 5fae6cb1..6c19401a 100644 --- a/templates/node.html.j2 +++ b/templates/node.html.j2 @@ -204,7 +204,8 @@ target="_blank">Armooo's MeshView
    --> PugetMesh Map
    - MeshMap + MeshMap
    + Logs
    From edc5e0deae7e1338417476ff04b25fac8416afc0 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 28 Jun 2025 04:27:56 +0000 Subject: [PATCH 071/155] Add hardware model statistics API and update templates for filtering and visualization. Introduced a new endpoint to fetch most and least common hardware models, updated node templates to support filtering by hardware model, and enhanced metrics page with charts for visualizing hardware model distribution. --- meshinfo_web.py | 83 +++++++++++- templates/allnodes.html.j2 | 29 +++++ templates/metrics.html.j2 | 241 ++++++++++++++++++++++++++++++++++- templates/node_table.html.j2 | 2 + templates/nodes.html.j2 | 8 ++ 5 files changed, 357 insertions(+), 6 deletions(-) create mode 100644 templates/allnodes.html.j2 diff --git a/meshinfo_web.py b/meshinfo_web.py index b6ba641d..35576110 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -2152,10 +2152,13 @@ def nodes(): nodes = get_cached_nodes() if not nodes: abort(503, description="Database connection unavailable") - latest = get_cached_latest_node() logging.info(f"/nodes.html: Loaded {len(nodes)} nodes.") + # Get hardware model filter from query parameters + hw_model_filter = request.args.get('hw_model') + hw_name_filter = request.args.get('hw_name') + return render_template( "nodes.html.j2", auth=auth(), @@ -2163,11 +2166,13 @@ def nodes(): nodes=nodes, show_inactive=False, latest=latest, + hw_model_filter=hw_model_filter, + hw_name_filter=hw_name_filter, hardware=meshtastic_support.HardwareModel, meshtastic_support=meshtastic_support, utils=utils, datetime=datetime.datetime, - timestamp=datetime.datetime.now(), + timestamp=datetime.datetime.now() ) @app.route('/allnodes.html') @@ -2176,21 +2181,27 @@ def allnodes(): nodes = get_cached_nodes() if not nodes: abort(503, description="Database connection unavailable") - latest = get_cached_latest_node() + logging.info(f"/allnodes.html: Loaded {len(nodes)} nodes.") + + # Get hardware model filter from query parameters + hw_model_filter = request.args.get('hw_model') + hw_name_filter = request.args.get('hw_name') return render_template( - "nodes.html.j2", + "allnodes.html.j2", auth=auth(), config=config, nodes=nodes, show_inactive=True, latest=latest, + hw_model_filter=hw_model_filter, + hw_name_filter=hw_name_filter, hardware=meshtastic_support.HardwareModel, meshtastic_support=meshtastic_support, utils=utils, datetime=datetime.datetime, - timestamp=datetime.datetime.now(), + timestamp=datetime.datetime.now() ) @app.route('/api/debug/memory') @@ -2528,6 +2539,68 @@ def message_paths(): timestamp=datetime.datetime.now() ) +@app.route('/api/hardware-models') +def get_hardware_models(): + """Get hardware model statistics for the most and least common models.""" + try: + md = get_meshdata() + if not md: + return jsonify({'error': 'Database connection unavailable'}), 503 + + # Get hardware model statistics + cur = md.db.cursor(dictionary=True) + + # Query to get hardware model counts with model names + sql = """ + SELECT + hw_model, + COUNT(*) as node_count, + GROUP_CONCAT(DISTINCT short_name ORDER BY short_name SEPARATOR ', ') as sample_names + FROM nodeinfo + WHERE hw_model IS NOT NULL + GROUP BY hw_model + ORDER BY node_count DESC + """ + + cur.execute(sql) + results = cur.fetchall() + cur.close() + + # Process results and get hardware model names + hardware_stats = [] + for row in results: + hw_model_id = row['hw_model'] + hw_model_name = meshtastic_support.get_hardware_model_name(hw_model_id) + + # Get a sample node for icon + sample_node = row['sample_names'].split(', ')[0] if row['sample_names'] else f"Model {hw_model_id}" + + hardware_stats.append({ + 'model_id': hw_model_id, + 'model_name': hw_model_name or f"Unknown Model {hw_model_id}", + 'node_count': row['node_count'], + 'sample_names': row['sample_names'], + 'icon_url': utils.graph_icon(sample_node) + }) + + # Get top 15 most common + most_common = hardware_stats[:15] + + # Get bottom 15 least common (but only if we have more than 15 total models) + # Sort in ascending order (lowest count first) + least_common = hardware_stats[-15:] if len(hardware_stats) > 15 else hardware_stats + least_common = sorted(least_common, key=lambda x: x['node_count']) + + return jsonify({ + 'most_common': most_common, + 'least_common': least_common, + 'total_models': len(hardware_stats) + }) + + except Exception as e: + logging.error(f"Error fetching hardware models: {e}") + return jsonify({'error': 'Failed to fetch hardware model data'}), 500 + if __name__ == '__main__': config = configparser.ConfigParser() config.read('config.ini') diff --git a/templates/allnodes.html.j2 b/templates/allnodes.html.j2 new file mode 100644 index 00000000..a3d401fe --- /dev/null +++ b/templates/allnodes.html.j2 @@ -0,0 +1,29 @@ +{% set this_page = "allnodes" %} +{% extends "layout.html.j2" %} + +{% block title %}All Nodes | MeshInfo{% endblock %} + +{% block content %} +{% if latest and latest.id is not none %} +{% set lnodeid = utils.convert_node_id_from_int_to_hex(latest.id) %} +{% if lnodeid and lnodeid in nodes %} +{% set lnode = nodes[lnodeid] %} +
    + 📌 Welcome to our newest node, {{ lnode.long_name }} ({{ lnode.short_name }}).👋 +
    +{% endif %} +{% endif %} + +{% if hw_model_filter and hw_name_filter %} +
    + 🔧 Showing nodes with hardware model: {{ hw_name_filter }} + Clear Filter +
    +{% endif %} + +
    +
    All Nodes (Including Offline)
    + {% include 'node_search.html.j2' %} + {% include 'node_table.html.j2' %} +
    +{% endblock %} \ No newline at end of file diff --git a/templates/metrics.html.j2 b/templates/metrics.html.j2 index 64e13cd6..92046b4d 100644 --- a/templates/metrics.html.j2 +++ b/templates/metrics.html.j2 @@ -222,6 +222,16 @@ Mesh Metrics

    Signal Strength (SNR)

    + +
    +

    Most Common Hardware Models

    + +
    + +
    +

    Least Common Hardware Models

    + +
@@ -276,6 +286,8 @@ Mesh Metrics let batteryLevelChart; let temperatureChart; let snrChart; + let mostCommonHardwareChart; + let leastCommonHardwareChart; // Chart colors const chartColors = { @@ -284,7 +296,9 @@ Mesh Metrics channelUtil: '#FF9800', batteryLevel: '#9C27B0', temperature: '#F44336', - snr: '#009688' + snr: '#009688', + mostCommonHardware: '#4CAF50', + leastCommonHardware: '#FF5722' }; // Initialize charts when the page loads @@ -292,6 +306,7 @@ Mesh Metrics initializeCharts(); refreshData(); updateChattiestNodes(); + updateHardwareModels(); // Set up auto-refresh every 5 minutes setInterval(refreshData, 5 * 60 * 1000); @@ -528,11 +543,172 @@ Mesh Metrics } } }); + + // Most Common Hardware Models Chart + mostCommonHardwareChart = new Chart(document.getElementById('mostCommonHardwareChart'), { + type: 'bar', + data: { + labels: [], + datasets: [{ + label: 'Number of Nodes', + data: [], + backgroundColor: chartColors.mostCommonHardware + '90', + borderColor: chartColors.mostCommonHardware, + borderWidth: 1, + borderRadius: 2, + barPercentage: 0.8, + categoryPercentage: 0.9 + }] + }, + options: { + ...commonOptions, + indexAxis: 'y', // Horizontal bar chart + onClick: function(event, elements) { + if (elements.length > 0) { + const index = elements[0].index; + const modelName = this.data.labels[index]; + const modelId = this.data.datasets[0].modelIds[index]; + window.open(`/allnodes.html?hw_model=${encodeURIComponent(modelId)}&hw_name=${encodeURIComponent(modelName)}`, '_blank'); + } + }, + onHover: function(event, elements) { + event.native.target.style.cursor = elements.length > 0 ? 'pointer' : 'default'; + }, + scales: { + x: { + ...commonOptions.scales.x, + title: { + display: true, + text: 'Number of Nodes' + } + }, + y: { + ...commonOptions.scales.y, + title: { + display: false + }, + ticks: { + ...commonOptions.scales.y.ticks, + callback: function(value, index) { + const label = this.getLabelForValue(value); + // Truncate long labels but ensure they're readable + if (label.length > 25) { + return label.substring(0, 22) + '...'; + } + return label; + } + } + } + }, + plugins: { + ...commonOptions.plugins, + legend: { + display: false + }, + tooltip: { + callbacks: { + title: function(context) { + // Show full name in tooltip + return context[0].label; + }, + afterBody: function(context) { + return 'Click to view nodes with this hardware model'; + } + } + } + } + } + }); + + // Least Common Hardware Models Chart + leastCommonHardwareChart = new Chart(document.getElementById('leastCommonHardwareChart'), { + type: 'bar', + data: { + labels: [], + datasets: [{ + label: 'Number of Nodes', + data: [], + backgroundColor: chartColors.leastCommonHardware + '90', + borderColor: chartColors.leastCommonHardware, + borderWidth: 1, + borderRadius: 2, + barPercentage: 0.8, + categoryPercentage: 0.9 + }] + }, + options: { + ...commonOptions, + indexAxis: 'y', // Horizontal bar chart + onClick: function(event, elements) { + if (elements.length > 0) { + const index = elements[0].index; + const modelName = this.data.labels[index]; + const modelId = this.data.datasets[0].modelIds[index]; + window.open(`/allnodes.html?hw_model=${encodeURIComponent(modelId)}&hw_name=${encodeURIComponent(modelName)}`, '_blank'); + } + }, + onHover: function(event, elements) { + event.native.target.style.cursor = elements.length > 0 ? 'pointer' : 'default'; + }, + scales: { + x: { + ...commonOptions.scales.x, + title: { + display: true, + text: 'Number of Nodes' + }, + ticks: { + ...commonOptions.scales.x.ticks, + stepSize: 1, + callback: function(value) { + // Only show whole numbers + return Math.floor(value) === value ? value : ''; + } + } + }, + y: { + ...commonOptions.scales.y, + title: { + display: false + }, + ticks: { + ...commonOptions.scales.y.ticks, + callback: function(value, index) { + const label = this.getLabelForValue(value); + // Truncate long labels but ensure they're readable + if (label.length > 25) { + return label.substring(0, 22) + '...'; + } + return label; + } + } + } + }, + plugins: { + ...commonOptions.plugins, + legend: { + display: false + }, + tooltip: { + callbacks: { + title: function(context) { + // Show full name in tooltip + return context[0].label; + }, + afterBody: function(context) { + return 'Click to view nodes with this hardware model'; + } + } + } + } + } + }); } // Refresh all data function refreshData() { updateMetrics(); + updateHardwareModels(); } // Update metrics based on selected time range @@ -581,6 +757,69 @@ Mesh Metrics }); } + // Update hardware models data + function updateHardwareModels() { + // Show loading state for hardware charts + document.querySelectorAll('#mostCommonHardwareChart, #leastCommonHardwareChart').forEach(canvas => { + canvas.parentElement.style.opacity = '0.7'; + }); + + // Fetch data from API + fetch('/api/hardware-models') + .then(response => { + if (!response.ok) { + throw new Error('Network response was not ok'); + } + return response.json(); + }) + .then(data => { + if (data.error) { + console.error('Error fetching hardware models:', data.error); + return; + } + + // Update most common hardware chart + updateHardwareChart(mostCommonHardwareChart, data.most_common, 'Most Common Hardware Models'); + + // Update least common hardware chart + updateHardwareChart(leastCommonHardwareChart, data.least_common, 'Least Common Hardware Models'); + + // Reset loading state + document.querySelectorAll('#mostCommonHardwareChart, #leastCommonHardwareChart').forEach(canvas => { + canvas.parentElement.style.opacity = '1'; + }); + }) + .catch(error => { + console.error('Error fetching hardware models data:', error); + // Reset loading state + document.querySelectorAll('#mostCommonHardwareChart, #leastCommonHardwareChart').forEach(canvas => { + canvas.parentElement.style.opacity = '1'; + }); + }); + } + + // Update a hardware chart with new data + function updateHardwareChart(chart, data, title) { + if (!data || !Array.isArray(data)) { + console.error('Invalid data format for hardware chart update'); + return; + } + + // Prepare labels and data + const labels = data.map(item => item.model_name); + const values = data.map(item => item.node_count); + const modelIds = data.map(item => item.model_id); + + // Update chart data + chart.data.labels = labels; + chart.data.datasets[0].data = values; + chart.data.datasets[0].label = 'Number of Nodes'; + chart.data.datasets[0].modelIds = modelIds; // Store model IDs for click handlers + + // Update chart + chart.update('none'); + } + // Update a chart with new data function updateChart(chart, data, label, yAxisLabel) { if (!data || !data.labels || !data.data) { diff --git a/templates/node_table.html.j2 b/templates/node_table.html.j2 index 94703d10..8a620ed0 100644 --- a/templates/node_table.html.j2 +++ b/templates/node_table.html.j2 @@ -44,6 +44,7 @@ {% for id, node in nodes.items()|sort(attribute='1.short_name') %} {% if node.active or show_inactive %} + {% if not hw_model_filter or node.hw_model == hw_model_filter|int %} Avatar {% endif %} + {% endif %} {% endfor %} diff --git a/templates/nodes.html.j2 b/templates/nodes.html.j2 index 375cc42e..5a96af48 100644 --- a/templates/nodes.html.j2 +++ b/templates/nodes.html.j2 @@ -13,6 +13,14 @@
{% endif %} {% endif %} + +{% if hw_model_filter and hw_name_filter %} +
+ 🔧 Showing nodes with hardware model: {{ hw_name_filter }} + Clear Filter +
+{% endif %} +
{{ this_page.title() }}
{% include 'node_search.html.j2' %} From e9c921563615e5980bbf648473ee7a9ff0a841b4 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 28 Jun 2025 17:31:26 +0000 Subject: [PATCH 072/155] Refactor memory management and optimize hardware model data handling. Updated memory watchdog to clear nodes singleton and force garbage collection at critical levels. Enhanced get_hardware_models function to use tuples for reduced memory overhead and added a caching layer for improved performance. --- custom.cnf | 6 ++--- meshdata.py | 20 +++++++++++++++- meshinfo_web.py | 64 ++++++++++++++++++++++++++++++++++++------------- 3 files changed, 69 insertions(+), 21 deletions(-) diff --git a/custom.cnf b/custom.cnf index 50e697f7..80052157 100644 --- a/custom.cnf +++ b/custom.cnf @@ -1,6 +1,6 @@ [mysqld] aria_pagecache_buffer_size = 32M -aria_sort_buffer_size = 64M -innodb_buffer_pool_size = 128M +aria_sort_buffer_size = 32M +innodb_buffer_pool_size = 32M key_buffer_size = 32M -myisam_sort_buffer_size = 32M \ No newline at end of file +myisam_sort_buffer_size = 32M diff --git a/meshdata.py b/meshdata.py index dcfd16e9..eee300b9 100644 --- a/meshdata.py +++ b/meshdata.py @@ -2232,7 +2232,25 @@ def get_neighbors_data(self, view_type='neighbor_info', days=1, zero_hop_timeout continue # Skip inactive nodes node_id_int = utils.convert_node_id_from_hex_to_int(node_id_hex) - final_node_data = dict(node_base_data) # Start with base data + + # Only copy the fields we need instead of the entire dict + final_node_data = { + 'id': node_base_data.get('id'), + 'long_name': node_base_data.get('long_name'), + 'short_name': node_base_data.get('short_name'), + 'hw_model': node_base_data.get('hw_model'), + 'role': node_base_data.get('role'), + 'firmware_version': node_base_data.get('firmware_version'), + 'owner': node_base_data.get('owner'), + 'ts_seen': node_base_data.get('ts_seen'), + 'ts_created': node_base_data.get('ts_created'), + 'ts_updated': node_base_data.get('ts_updated'), + 'active': node_base_data.get('active'), + 'last_seen': node_base_data.get('last_seen'), + 'channel': node_base_data.get('channel'), + 'position': node_base_data.get('position'), + 'telemetry': node_base_data.get('telemetry') + } # Initialize lists final_node_data['neighbors'] = [] diff --git a/meshinfo_web.py b/meshinfo_web.py index 35576110..ac924c41 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -473,6 +473,8 @@ def memory_watchdog(): log_cache_stats() with app.app_context(): cache.clear() + # Also clear the nodes singleton to free memory + clear_nodes_singleton() gc.collect() logging.info("Cache stats after high memory cleanup:") log_cache_stats() @@ -483,6 +485,9 @@ def memory_watchdog(): log_detailed_memory_analysis() logging.info("Cache stats at critical memory level:") log_cache_stats() + # Force clear nodes singleton at critical levels + clear_nodes_singleton() + gc.collect() except Exception as e: logging.error(f"Error in memory watchdog: {e}") @@ -2239,6 +2244,10 @@ def debug_cleanup(): cleanup_cache() + # Also clear nodes singleton and force garbage collection + clear_nodes_singleton() + gc.collect() + return jsonify({ 'status': 'success', 'message': 'Cache cleanup completed. Check logs for details.' @@ -2539,13 +2548,13 @@ def message_paths(): timestamp=datetime.datetime.now() ) -@app.route('/api/hardware-models') -def get_hardware_models(): +@cache.memoize(timeout=300) # Cache for 5 minutes +def get_cached_hardware_models(): """Get hardware model statistics for the most and least common models.""" try: md = get_meshdata() if not md: - return jsonify({'error': 'Database connection unavailable'}), 503 + return {'error': 'Database connection unavailable'} # Get hardware model statistics cur = md.db.cursor(dictionary=True) @@ -2566,7 +2575,7 @@ def get_hardware_models(): results = cur.fetchall() cur.close() - # Process results and get hardware model names + # Process results and get hardware model names - use tuples to reduce memory hardware_stats = [] for row in results: hw_model_id = row['hw_model'] @@ -2575,13 +2584,14 @@ def get_hardware_models(): # Get a sample node for icon sample_node = row['sample_names'].split(', ')[0] if row['sample_names'] else f"Model {hw_model_id}" - hardware_stats.append({ - 'model_id': hw_model_id, - 'model_name': hw_model_name or f"Unknown Model {hw_model_id}", - 'node_count': row['node_count'], - 'sample_names': row['sample_names'], - 'icon_url': utils.graph_icon(sample_node) - }) + # Use tuple instead of dict to reduce memory overhead + hardware_stats.append(( + hw_model_id, + hw_model_name or f"Unknown Model {hw_model_id}", + row['node_count'], + row['sample_names'], + utils.graph_icon(sample_node) + )) # Get top 15 most common most_common = hardware_stats[:15] @@ -2589,17 +2599,37 @@ def get_hardware_models(): # Get bottom 15 least common (but only if we have more than 15 total models) # Sort in ascending order (lowest count first) least_common = hardware_stats[-15:] if len(hardware_stats) > 15 else hardware_stats - least_common = sorted(least_common, key=lambda x: x['node_count']) + least_common = sorted(least_common, key=lambda x: x[2]) # Sort by node_count (index 2) + + # Convert tuples to dicts only for JSON serialization + def tuple_to_dict(hw_tuple): + return { + 'model_id': hw_tuple[0], + 'model_name': hw_tuple[1], + 'node_count': hw_tuple[2], + 'sample_names': hw_tuple[3], + 'icon_url': hw_tuple[4] + } - return jsonify({ - 'most_common': most_common, - 'least_common': least_common, + return { + 'most_common': [tuple_to_dict(hw) for hw in most_common], + 'least_common': [tuple_to_dict(hw) for hw in least_common], 'total_models': len(hardware_stats) - }) + } except Exception as e: logging.error(f"Error fetching hardware models: {e}") - return jsonify({'error': 'Failed to fetch hardware model data'}), 500 + return {'error': 'Failed to fetch hardware model data'} + +@app.route('/api/hardware-models') +def get_hardware_models(): + """Get hardware model statistics for the most and least common models.""" + result = get_cached_hardware_models() + + if 'error' in result: + return jsonify(result), 503 if result['error'] == 'Database connection unavailable' else 500 + + return jsonify(result) if __name__ == '__main__': config = configparser.ConfigParser() From c84aabe957f8cd993625d1cbe749130cd61fee5c Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 29 Jun 2025 04:48:09 +0000 Subject: [PATCH 073/155] update build process to include amd64 and arm64 --- .github/workflows/deploy.yaml | 23 ++++++++++++++++++++++- meshinfo_web.py | 35 ++++++++++++++++++++++++++++------- scripts/docker-build.sh | 22 +++++++++++++--------- scripts/release.sh | 2 +- 4 files changed, 64 insertions(+), 18 deletions(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 6f197e24..3190787e 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -7,10 +7,29 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 + with: + fetch-depth: 0 # Fetch all history for proper versioning - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + - name: Calculate version + id: version + run: | + # Get the latest git tag + LATEST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0") + echo "Latest tag: $LATEST_TAG" + + # Get current commit hash + COMMIT_HASH=$(git rev-parse --short HEAD) + echo "Commit hash: $COMMIT_HASH" + + # Create version string + VERSION="${LATEST_TAG}-${COMMIT_HASH}" + echo "version=${VERSION}" >> $GITHUB_OUTPUT + echo "latest_tag=${LATEST_TAG}" >> $GITHUB_OUTPUT + echo "commit_hash=${COMMIT_HASH}" >> $GITHUB_OUTPUT + - name: Log in to GitHub Container Registry uses: docker/login-action@v2 with: @@ -22,7 +41,9 @@ jobs: uses: docker/build-push-action@v4 with: context: . + platforms: linux/amd64,linux/arm64 push: true tags: | ghcr.io/agessaman/meshinfo-lite:latest - ghcr.io/agessaman/meshinfo-lite:${{ github.sha }} + ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.version }} + ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.latest_tag }} diff --git a/meshinfo_web.py b/meshinfo_web.py index ac924c41..427db458 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -147,6 +147,9 @@ def cleanup_cache(): # Clear the nodes singleton clear_nodes_singleton() + # Clear nodes-related cache entries + clear_nodes_related_cache() + # Clear the cache with app.app_context(): cache.clear() @@ -475,6 +478,8 @@ def memory_watchdog(): cache.clear() # Also clear the nodes singleton to free memory clear_nodes_singleton() + # Clear nodes-related cache entries + clear_nodes_related_cache() gc.collect() logging.info("Cache stats after high memory cleanup:") log_cache_stats() @@ -487,6 +492,7 @@ def memory_watchdog(): log_cache_stats() # Force clear nodes singleton at critical levels clear_nodes_singleton() + clear_nodes_related_cache() gc.collect() except Exception as e: @@ -571,8 +577,11 @@ def get_cached_nodes(): return nodes_data @cache.memoize(timeout=60) -def get_cached_active_nodes(nodes): +def get_cached_active_nodes(): """Cache the active nodes calculation.""" + nodes = get_cached_nodes() + if not nodes: + return {} return utils.active_nodes(nodes) @cache.memoize(timeout=60) @@ -590,9 +599,6 @@ def get_cached_message_map_data(message_id): if not md: return None - # Get current node info for names using singleton - nodes = get_cached_nodes() - # Get message and basic reception data cursor = md.db.cursor(dictionary=True) cursor.execute(""" @@ -670,7 +676,6 @@ def latlon_to_xy(lon, lat): } return { - 'nodes': nodes, 'message': message, 'sender_position': sender_position, 'receiver_positions': receiver_positions, @@ -689,9 +694,11 @@ def message_map(): if not data: return redirect(url_for('chat')) + # Get nodes separately to avoid caching multiple copies + nodes = get_cached_nodes() + # --- Provide zero_hop_links and position data for relay node inference --- md = get_meshdata() - nodes = data['nodes'] sender_id = data['message']['from_id'] receiver_ids = data['message']['receiver_ids'] # Get zero-hop links for the last 1 day (or configurable) @@ -2138,7 +2145,7 @@ def serve_index(success_message=None, error_message=None): if not nodes: abort(503, description="Database connection unavailable") - active_nodes = get_cached_active_nodes(nodes) + active_nodes = get_cached_active_nodes() return render_template( "index.html.j2", @@ -2246,6 +2253,7 @@ def debug_cleanup(): # Also clear nodes singleton and force garbage collection clear_nodes_singleton() + clear_nodes_related_cache() gc.collect() return jsonify({ @@ -2260,6 +2268,7 @@ def debug_clear_nodes(): abort(401) clear_nodes_singleton() + clear_nodes_related_cache() gc.collect() return jsonify({ @@ -2361,6 +2370,18 @@ def clear_nodes_singleton(): else: logging.info("Nodes singleton was already None") +def clear_nodes_related_cache(): + """Clear cache entries that might be holding onto nodes dictionaries.""" + try: + # Clear specific cached functions that might hold nodes data + cache.delete_memoized(get_cached_active_nodes) + cache.delete_memoized(get_cached_message_map_data) + cache.delete_memoized(get_cached_graph_data) + cache.delete_memoized(get_cached_neighbors_data) + logging.info("Cleared nodes-related cache entries") + except Exception as e: + logging.error(f"Error clearing nodes-related cache: {e}") + def find_relay_node_by_suffix(relay_suffix, nodes, receiver_ids=None, sender_id=None, zero_hop_links=None, sender_pos=None, receiver_pos=None, debug=False): """ Improved relay node matcher: prefer zero-hop/extended neighbors, then select the physically closest candidate to the sender (or receiver), using scoring only as a tiebreaker. diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index 76cbb358..2e6bca89 100644 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -2,15 +2,19 @@ # build -# set version from args if not, exit -#if [ -z "$1" ] -# then -# echo "No version supplied (e.g. 1.0.0)" -# exit 1 -#fi +# Get version from git if not provided +if [ -z "$1" ]; then + # Get the latest git tag + LATEST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0") + # Get current commit hash + COMMIT_HASH=$(git rev-parse --short HEAD) + VERSION="${LATEST_TAG}-${COMMIT_HASH}" + echo "Using git-based version: $VERSION" +else + VERSION=$1 + echo "Using provided version: $VERSION" +fi -REPO=dadecoza/meshinfo -# VERSION=$1 -VERSION=latest +REPO=agessaman/meshinfo docker build -t $REPO:$VERSION --platform=linux/amd64 . diff --git a/scripts/release.sh b/scripts/release.sh index b0743030..11af03ee 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -9,7 +9,7 @@ if [ -z "$1" ] exit 1 fi -REPO=dadecoza/meshinfo +REPO=agessaman/meshinfo VERSION=$1 # echo $CR_PAT | docker login ghcr.io -u USERNAME --password-stdin git tag -a $VERSION -m "Version $VERSION" && git push --tags && \ From 7817b1bf695060bd19d7d054489096f6b0f31724 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 29 Jun 2025 04:55:08 +0000 Subject: [PATCH 074/155] add gdal and geos dependencies --- Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index d02bb4a7..fe9b3462 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,11 @@ RUN apt-get update && \ fonts-symbola \ fontconfig \ freetype2-demos \ + libgdal-dev \ + gdal-bin \ + libgeos-dev \ + libproj-dev \ + proj-bin \ && apt-get clean && \ rm -rf /var/lib/apt/lists/* From 5d47a7eb2cdb66589da1df38dab1bea989a8ba88 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 29 Jun 2025 05:11:11 +0000 Subject: [PATCH 075/155] optimize pip install --- Dockerfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index fe9b3462..80996dc7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,10 @@ ENV PYTHONUNBUFFERED=1 \ # Set standard locations PATH="/app/.local/bin:${PATH}" \ # Consistent port for the app - APP_PORT=8000 + APP_PORT=8000 \ + # Optimize pip for faster builds + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 RUN groupadd --system app && \ useradd --system --gid app --home-dir /app --create-home app @@ -38,7 +41,8 @@ RUN fc-cache -fv COPY requirements.txt banner run.sh ./ -RUN pip install --upgrade pip +# Upgrade pip and install all packages with optimizations +RUN pip install --upgrade pip setuptools wheel RUN su app -c "pip install --no-cache-dir --user -r requirements.txt" COPY --chown=app:app banner run.sh ./ From 69a6c658ccba995d8d44e09cd537cacab0e22377 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 29 Jun 2025 05:25:03 +0000 Subject: [PATCH 076/155] Add architecture-specific optimizations for rasterio in Dockerfile. Implemented conditional installation of build tools for ARM64 to enhance compilation speed. --- Dockerfile | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 80996dc7..bc1abe37 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,6 +43,21 @@ COPY requirements.txt banner run.sh ./ # Upgrade pip and install all packages with optimizations RUN pip install --upgrade pip setuptools wheel + +# Architecture-specific optimizations for rasterio +ARG TARGETPLATFORM +ENV GDAL_CONFIG=/usr/bin/gdal-config + +# For ARM64, install build tools to speed up compilation +RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + python3-dev \ + && apt-get clean && \ + rm -rf /var/lib/apt/lists/*; \ + fi + RUN su app -c "pip install --no-cache-dir --user -r requirements.txt" COPY --chown=app:app banner run.sh ./ @@ -59,4 +74,4 @@ USER app EXPOSE ${APP_PORT} -CMD ["./run.sh"] +CMD ["./run.sh"] \ No newline at end of file From 2af96ecb42cbe315dc4c982070a9b2cd9162b78c Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 29 Jun 2025 05:42:38 +0000 Subject: [PATCH 077/155] Enhance Dockerfile for ARM64 support by adding conditional rasterio installation via conda. Updated pip installation process to exclude rasterio from requirements for ARM64, improving build efficiency. --- Dockerfile | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index bc1abe37..398149f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,26 +39,34 @@ RUN apt-get update && \ RUN fc-cache -fv +# Architecture-specific rasterio installation +ARG TARGETPLATFORM +ENV GDAL_CONFIG=/usr/bin/gdal-config + +# For ARM64: Install conda/mamba and use conda-forge for rasterio +RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ + curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh && \ + bash Miniforge3-Linux-aarch64.sh -b -p /opt/conda && \ + rm Miniforge3-Linux-aarch64.sh && \ + /opt/conda/bin/mamba install -c conda-forge rasterio -y; \ + fi + +# Update PATH to include conda if installed +ENV PATH="/opt/conda/bin:${PATH}" + COPY requirements.txt banner run.sh ./ -# Upgrade pip and install all packages with optimizations +# Upgrade pip and install packages RUN pip install --upgrade pip setuptools wheel -# Architecture-specific optimizations for rasterio -ARG TARGETPLATFORM -ENV GDAL_CONFIG=/usr/bin/gdal-config - -# For ARM64, install build tools to speed up compilation +# Install requirements, excluding rasterio for ARM64 (already installed via conda) RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - build-essential \ - python3-dev \ - && apt-get clean && \ - rm -rf /var/lib/apt/lists/*; \ + grep -v "^rasterio" requirements.txt > requirements_filtered.txt || echo "" > requirements_filtered.txt; \ + else \ + cp requirements.txt requirements_filtered.txt; \ fi -RUN su app -c "pip install --no-cache-dir --user -r requirements.txt" +RUN su app -c "pip install --no-cache-dir --user -r requirements_filtered.txt" COPY --chown=app:app banner run.sh ./ COPY --chown=app:app *.py ./ From 494e74c6a1d5abf15eaf599337ecca2241ef5641 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 29 Jun 2025 05:48:52 +0000 Subject: [PATCH 078/155] Update Dockerfile to include conditional installation of curl for ARM64 platform. This enhancement improves compatibility and optimizes the build process for different architectures. --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 398149f2..d7033971 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,7 @@ RUN groupadd --system app && \ WORKDIR /app # Install system dependencies +ARG TARGETPLATFORM RUN apt-get update && \ apt-get install -y --no-install-recommends \ libexpat1 \ @@ -34,13 +35,13 @@ RUN apt-get update && \ libgeos-dev \ libproj-dev \ proj-bin \ + $([ "$TARGETPLATFORM" = "linux/arm64" ] && echo "curl") \ && apt-get clean && \ rm -rf /var/lib/apt/lists/* RUN fc-cache -fv # Architecture-specific rasterio installation -ARG TARGETPLATFORM ENV GDAL_CONFIG=/usr/bin/gdal-config # For ARM64: Install conda/mamba and use conda-forge for rasterio From c693730e76a34a430139b8867ae781e1b8804c62 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 29 Jun 2025 06:01:20 +0000 Subject: [PATCH 079/155] Refactor GitHub Actions workflow to create a multi-architecture Docker manifest. Updated job name and removed unnecessary steps, focusing on manifest creation and pushing for different architectures. --- .github/workflows/deploy-amd64.yaml | 49 +++++++++++++++++++++++++++++ .github/workflows/deploy-arm64.yaml | 49 +++++++++++++++++++++++++++++ .github/workflows/deploy.yaml | 44 ++++++++++++++------------ 3 files changed, 122 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/deploy-amd64.yaml create mode 100644 .github/workflows/deploy-arm64.yaml diff --git a/.github/workflows/deploy-amd64.yaml b/.github/workflows/deploy-amd64.yaml new file mode 100644 index 00000000..404cf965 --- /dev/null +++ b/.github/workflows/deploy-amd64.yaml @@ -0,0 +1,49 @@ +name: Build and Push AMD64 +on: + workflow_dispatch: # Only triggered manually +jobs: + build-and-push-amd64: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Fetch all history for proper versioning + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Calculate version + id: version + run: | + # Get the latest git tag + LATEST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0") + echo "Latest tag: $LATEST_TAG" + + # Get current commit hash + COMMIT_HASH=$(git rev-parse --short HEAD) + echo "Commit hash: $COMMIT_HASH" + + # Create version string + VERSION="${LATEST_TAG}-${COMMIT_HASH}" + echo "version=${VERSION}" >> $GITHUB_OUTPUT + echo "latest_tag=${LATEST_TAG}" >> $GITHUB_OUTPUT + echo "commit_hash=${COMMIT_HASH}" >> $GITHUB_OUTPUT + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push AMD64 Docker image + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/amd64 + push: true + tags: | + ghcr.io/agessaman/meshinfo-lite:latest-amd64 + ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.version }}-amd64 + ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.latest_tag }}-amd64 \ No newline at end of file diff --git a/.github/workflows/deploy-arm64.yaml b/.github/workflows/deploy-arm64.yaml new file mode 100644 index 00000000..2bed5afd --- /dev/null +++ b/.github/workflows/deploy-arm64.yaml @@ -0,0 +1,49 @@ +name: Build and Push ARM64 +on: + workflow_dispatch: # Only triggered manually +jobs: + build-and-push-arm64: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Fetch all history for proper versioning + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Calculate version + id: version + run: | + # Get the latest git tag + LATEST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0") + echo "Latest tag: $LATEST_TAG" + + # Get current commit hash + COMMIT_HASH=$(git rev-parse --short HEAD) + echo "Commit hash: $COMMIT_HASH" + + # Create version string + VERSION="${LATEST_TAG}-${COMMIT_HASH}" + echo "version=${VERSION}" >> $GITHUB_OUTPUT + echo "latest_tag=${LATEST_TAG}" >> $GITHUB_OUTPUT + echo "commit_hash=${COMMIT_HASH}" >> $GITHUB_OUTPUT + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push ARM64 Docker image + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/arm64 + push: true + tags: | + ghcr.io/agessaman/meshinfo-lite:latest-arm64 + ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.version }}-arm64 + ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.latest_tag }}-arm64 \ No newline at end of file diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 3190787e..205d53dd 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -1,18 +1,11 @@ -name: Manual Build and Push +name: Create Multi-Arch Manifest on: workflow_dispatch: # Only triggered manually jobs: - build-and-push: + create-manifest: runs-on: ubuntu-latest + needs: [] # No dependencies - can run independently steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 # Fetch all history for proper versioning - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Calculate version id: version run: | @@ -37,13 +30,24 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push Docker image - uses: docker/build-push-action@v4 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: | - ghcr.io/agessaman/meshinfo-lite:latest - ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.version }} - ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.latest_tag }} + - name: Create and push multi-arch manifest + run: | + # Create manifest for latest + docker manifest create ghcr.io/agessaman/meshinfo-lite:latest \ + ghcr.io/agessaman/meshinfo-lite:latest-amd64 \ + ghcr.io/agessaman/meshinfo-lite:latest-arm64 + + # Create manifest for version + docker manifest create ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.version }} \ + ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.version }}-amd64 \ + ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.version }}-arm64 + + # Create manifest for tag + docker manifest create ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.latest_tag }} \ + ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.latest_tag }}-amd64 \ + ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.latest_tag }}-arm64 + + # Push manifests + docker manifest push ghcr.io/agessaman/meshinfo-lite:latest + docker manifest push ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.version }} + docker manifest push ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.latest_tag }} From 1818bf58dc1898dc89103932d9ef1d0be5456048 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 29 Jun 2025 06:16:36 +0000 Subject: [PATCH 080/155] Add MEDIUM_SLOW channel to Channel and ShortChannel enums, and update migration list to include message reception timestamp creation. --- meshtastic_support.py | 2 + migrations/__init__.py | 2 + .../add_message_reception_ts_created.py | 45 +++++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 migrations/add_message_reception_ts_created.py diff --git a/meshtastic_support.py b/meshtastic_support.py index 0903351d..624b28d5 100644 --- a/meshtastic_support.py +++ b/meshtastic_support.py @@ -160,6 +160,7 @@ class Channel(Enum): MEDIUM_FAST = 31 SHORT_FAST = 112 LONG_MODERATE = 88 + MEDIUM_SLOW = 24 # Additional channels will be added as they are discovered class ShortChannel(Enum): @@ -171,6 +172,7 @@ class ShortChannel(Enum): MF = 31 SF = 112 LM = 88 + MS = 24 # Additional channels will be added as they are discovered def get_channel_name(channel_value, use_short_names=False): diff --git a/migrations/__init__.py b/migrations/__init__.py index 519ec8c9..db850630 100644 --- a/migrations/__init__.py +++ b/migrations/__init__.py @@ -9,6 +9,7 @@ from .add_message_map_indexes import migrate as add_message_map_indexes from .add_relay_node_to_reception import migrate as add_relay_node_to_reception from .add_relay_edges_table import migrate as add_relay_edges_table +from .add_message_reception_ts_created import migrate as add_message_reception_ts_created # List of migrations to run in order MIGRATIONS = [ @@ -22,4 +23,5 @@ add_message_map_indexes, add_relay_node_to_reception, add_relay_edges_table, + add_message_reception_ts_created, ] diff --git a/migrations/add_message_reception_ts_created.py b/migrations/add_message_reception_ts_created.py new file mode 100644 index 00000000..ad033cef --- /dev/null +++ b/migrations/add_message_reception_ts_created.py @@ -0,0 +1,45 @@ +import logging + +def migrate(db): + """ + Add ts_created column to message_reception table for compatibility + """ + try: + cursor = db.cursor() + + # Check if ts_created column exists in message_reception table + cursor.execute(""" + SELECT COUNT(*) + FROM information_schema.COLUMNS + WHERE TABLE_NAME = 'message_reception' + AND COLUMN_NAME = 'ts_created' + """) + has_ts_created = cursor.fetchone()[0] > 0 + + if not has_ts_created: + logging.info("Adding ts_created column to message_reception table...") + cursor.execute(""" + ALTER TABLE message_reception + ADD COLUMN ts_created TIMESTAMP DEFAULT CURRENT_TIMESTAMP + COMMENT 'Timestamp when the reception was recorded' + """) + + # Update existing records to use rx_time converted to timestamp + logging.info("Updating existing records with rx_time converted to ts_created...") + cursor.execute(""" + UPDATE message_reception + SET ts_created = FROM_UNIXTIME(rx_time) + WHERE rx_time IS NOT NULL AND ts_created IS NULL + """) + + db.commit() + logging.info("Added ts_created column successfully") + else: + logging.info("ts_created column already exists in message_reception table") + + except Exception as e: + logging.error(f"Error during ts_created migration: {e}") + db.rollback() + raise + finally: + cursor.close() \ No newline at end of file From 8e7a30886cccce1bcbb1e321aa4a4eb32380cafe Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 29 Jun 2025 06:32:35 +0000 Subject: [PATCH 081/155] Update GitHub Actions workflow to include repository checkout step with full history for accurate versioning. --- .github/workflows/deploy.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 205d53dd..f4871ca9 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -6,6 +6,11 @@ jobs: runs-on: ubuntu-latest needs: [] # No dependencies - can run independently steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 # Fetch all history for proper versioning + - name: Calculate version id: version run: | From e27f4b8ce6b6bde259f14885ee2706a8995e058a Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 29 Jun 2025 17:45:18 +0000 Subject: [PATCH 082/155] Add caching for Docker layers in GitHub Actions workflow to improve build efficiency for ARM64 architecture. --- .github/workflows/deploy-arm64.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-arm64.yaml b/.github/workflows/deploy-arm64.yaml index 2bed5afd..1ef24bc4 100644 --- a/.github/workflows/deploy-arm64.yaml +++ b/.github/workflows/deploy-arm64.yaml @@ -13,6 +13,14 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + - name: Cache Docker layers + uses: actions/cache@v4 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-arm64-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-arm64- + - name: Calculate version id: version run: | @@ -46,4 +54,6 @@ jobs: tags: | ghcr.io/agessaman/meshinfo-lite:latest-arm64 ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.version }}-arm64 - ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.latest_tag }}-arm64 \ No newline at end of file + ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.latest_tag }}-arm64 + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache,mode=max \ No newline at end of file From 1f4010a5c5b297824f169dcad2638f5b03cd6641 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 29 Jun 2025 18:08:06 +0000 Subject: [PATCH 083/155] Update GitHub Actions workflow to disable provenance for Docker image builds, streamlining the deployment process for amd64 architecture. --- .github/workflows/deploy-amd64.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-amd64.yaml b/.github/workflows/deploy-amd64.yaml index 404cf965..96428eef 100644 --- a/.github/workflows/deploy-amd64.yaml +++ b/.github/workflows/deploy-amd64.yaml @@ -43,6 +43,7 @@ jobs: context: . platforms: linux/amd64 push: true + provenance: false tags: | ghcr.io/agessaman/meshinfo-lite:latest-amd64 ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.version }}-amd64 From f005035fd3a97ba0269cb615d65b0ded7c189fa0 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 29 Jun 2025 18:12:13 +0000 Subject: [PATCH 084/155] Update GitHub Actions workflow to disable provenance for Docker image builds on ARM64, enhancing deployment efficiency. --- .github/workflows/deploy-arm64.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-arm64.yaml b/.github/workflows/deploy-arm64.yaml index 1ef24bc4..f949fb66 100644 --- a/.github/workflows/deploy-arm64.yaml +++ b/.github/workflows/deploy-arm64.yaml @@ -51,6 +51,7 @@ jobs: context: . platforms: linux/arm64 push: true + provenance: false tags: | ghcr.io/agessaman/meshinfo-lite:latest-arm64 ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.version }}-arm64 From 1d5ccbc231f14e5dcd384a8717eba9707df7a39f Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 29 Jun 2025 19:49:38 +0000 Subject: [PATCH 085/155] Update configuration and templates for customizable mesh settings, including region, contact links, and theme colors. Replace hardcoded values with dynamic configuration options in HTML templates. Allow logo overwriting through docker compose mount. --- config.ini.sample | 43 +++++++++++++++++++++-------- templates/index.html.j2 | 47 ++++++++++++++++++++------------ templates/layout.html.j2 | 26 ++++++++++++++---- www/images/{ => logos}/logo.png | Bin 4 files changed, 81 insertions(+), 35 deletions(-) rename www/images/{ => logos}/logo.png (100%) diff --git a/config.ini.sample b/config.ini.sample index 0f06cfe4..dce2db06 100644 --- a/config.ini.sample +++ b/config.ini.sample @@ -1,17 +1,22 @@ [mesh] -name=ZA Mesh -short_name=ZA Mesh -description=Serving Meshtastic to South Africa. -contact=dade@dade.co.za -url=https://mesh.zr1rf.za.net -latitude=-33.918861 -longitude=18.423300 +name=Your Mesh Name +short_name=YourMesh +region=Your Mesh Region +description=Serving Meshtastic to your region. +contact=https://yourmesh.org +url=https://yourmesh.org +# Optional: link to your mesh's configuration guide +config_url=https://yourmesh.org/config-instructions +# Optional: link to your mesh's Discord or community chat +discord_url=https://discord.gg/yourdiscord +latitude=0.0 +longitude=0.0 channel_key=1PG7OiApB1nwvP+rz05pAQ== [mqtt] -broker=mqtt.zr1rf.za.net +broker=mqtt.yourmesh.org port=1883 -topic=msh/ZA/# +topic=msh/US/# username=meshdev password=large4cats @@ -36,7 +41,7 @@ metrics_average_interval=7200 [channels] # Comma-separated list of channel names to ignore -#ignored_channels=MVR_Commo +#ignored_channels=PKI [geocoding] enabled=false @@ -44,7 +49,7 @@ apikey=YOUR_KEY_HERE [registrations] enabled=true -jwt_secret=6219c1c2364499639ce9d16c33863c4f6fedb7a4a1a0f29524c20d95cb00e5f5 +jwt_secret=YOUR_JWT_SECRET [smtp] email=YOUR_EMAIL_ADDRESS @@ -55,4 +60,18 @@ port=SMTP_PORT [los] enabled=false max_distance=10000 -cache_duration=43200 \ No newline at end of file +cache_duration=43200 + +[tools] +meshmap_label=MeshMap +meshmap=https://meshmap.net/ +pugetmesh_map_label=PugetMesh Map +pugetmesh_map=https://mqtt.davekeogh.com/?lat=47.60292227835496&lng=237.49420166015628&zoom=10 +meshsense_label=MeshSense Map +meshsense=https://meshsense.affirmatech.com/ + +[theme] +# Header background color (navbar) +header_color=#9fdef9 +# Accent color (login/logout button, etc.) +accent_color=#17a2b8 \ No newline at end of file diff --git a/templates/index.html.j2 b/templates/index.html.j2 index bed5cb0e..89ac6fcb 100644 --- a/templates/index.html.j2 +++ b/templates/index.html.j2 @@ -3,13 +3,17 @@ {% block title %}{{ config["mesh"]["name"] }}{% endblock %} {% block content %} +{% set mesh_name = config['mesh'].get('name', 'Mesh Network') or 'Mesh Network' %} +{% set mesh_region = config['mesh'].get('region', 'your region') or 'your region' %} +{% set mesh_short_name = config['mesh'].get('short_name', 'Mesh') or 'Mesh' %} +
- logo -

{{ config["mesh"]["name"] }}

+ logo +

{{ mesh_name }}

- This site provides information on {{ active_nodes|length }} active nodes in the Puget Sound Meshtastic network. + This site provides information on {{ active_nodes|length }} active nodes in the {{ mesh_region }} region ({{ mesh_name }} network).

Last updated: {{ format_timestamp(timestamp.timestamp()) }}

@@ -23,7 +27,7 @@

- MeshInfo is a web-based tool which collects and visualizes data from Meshtastic nodes across the Puget Sound region. It provides visibility into our PugetMesh community mesh network, helping both new and experienced members understand network coverage, identify growth opportunities, and troubleshoot connectivity issues. + MeshInfo is a web-based tool which collects and visualizes data from Meshtastic nodes across the {{ mesh_region }} region. It provides visibility into your community mesh network, helping both new and experienced members understand network coverage, identify growth opportunities, and troubleshoot connectivity issues.

Through MeshInfo, you can view node locations on maps, check detailed node information including telemetry data, visualize node connections, and search for specific nodes by ID or name. @@ -61,19 +65,19 @@

Note: Your node will still appear in MeshInfo without location data, but distance and direction information to other nodes won't be available.

-

PugetMesh MQTT Connection (recommended):

-

Important: Users who don't connect directly to the PugetMesh MQTT server won't be sharing information with other users in the region. Connecting your node to our MQTT server allows your data to contribute to the regional mesh network and helps everyone better understand network coverage.

-

To connect your node directly to the PugetMesh MQTT server, configure these settings in Settings > Modules > MQTT:

+

Mesh MQTT Connection (recommended):

+

Important: Users who don't connect directly to the MQTT server won't be sharing information with other users in the region. Connecting your node to the MQTT server allows your data to contribute to the regional mesh network and helps everyone better understand network coverage.

+

To connect your node directly to the {{ mesh_short_name }} MQTT server, configure these settings in Settings > Modules > MQTT:

  • - Address: meshtastic.pugetmesh.org + Address: {{ config['mqtt']['broker'] }}
  • - Username: meshdev + Username: {{ config['mqtt']['username'] }}
  • - Password: large4cats + Password: {{ config['mqtt']['password'] }}
  • Encryption Enabled: Yes @@ -85,7 +89,7 @@ TLS Enabled: No
  • - Root topic: msh/US + Root topic: {{ config['mqtt']['topic'].rstrip('/#') }}
@@ -108,9 +112,11 @@

- For complete setup instructions, visit the PugetMesh configuration page: + For complete setup instructions, visit the {{ mesh_short_name }} configuration page:

- View Full Configuration Guide + {% if config['mesh'].get('config_url') %} + View Full Configuration Guide + {% endif %}
@@ -120,15 +126,22 @@
-

Join the PugetMesh Community

+

Join the {{ mesh_short_name }} Community

- PugetMesh is a volunteer-driven group supporting mesh networks in the Puget Sound region. We focus on building a resilient off-grid communication network using Meshtastic technology. + {{ mesh_short_name }} is a volunteer-driven group supporting mesh networks in your region. We focus on building a resilient off-grid communication network using Meshtastic technology.

- Join Our Discord - Visit PugetMesh.org + {% if config['mesh'].get('discord_url') %} + + + + + Join Our Discord + + {% endif %} + Visit {{ mesh_short_name }} Website
diff --git a/templates/layout.html.j2 b/templates/layout.html.j2 index 90444a4e..f23e35d8 100644 --- a/templates/layout.html.j2 +++ b/templates/layout.html.j2 @@ -24,6 +24,20 @@ + + {% block head %}{% endblock %} @@ -51,10 +65,10 @@ Tools
- - PugetMesh Map - MeshMap - MeshSense Map + {% for key, value in config['tools'].items() if not key.endswith('_label') %} + {% set label = config['tools'].get(key + '_label', key.replace('_', ' ').title()) %} + {{ label }} + {% endfor %}
  • @@ -62,9 +76,9 @@
    • {% if not auth %} - Login + Login {% else %} - Logout + Logout {% endif %}
    diff --git a/www/images/logo.png b/www/images/logos/logo.png similarity index 100% rename from www/images/logo.png rename to www/images/logos/logo.png From 86b6f8cd93bcad8e071f298773d5c82eaa5da075 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 29 Jun 2025 19:59:07 +0000 Subject: [PATCH 086/155] Add conditional rendering for #MeshtasticMonday link in layout template based on the current weekday. --- templates/layout.html.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/layout.html.j2 b/templates/layout.html.j2 index f23e35d8..dc4f6e53 100644 --- a/templates/layout.html.j2 +++ b/templates/layout.html.j2 @@ -71,9 +71,11 @@ {% endfor %}
    + {% if datetime.datetime.now().weekday() == 0 %}
  • #MeshtasticMonday
    • + {% endif %} {% if not auth %} Login From 9648d34a2cff59c210e4ef5c6a43616b73a816e3 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 29 Jun 2025 20:01:58 +0000 Subject: [PATCH 087/155] Update configuration and layout template to include page background color setting --- config.ini.sample | 4 +++- templates/layout.html.j2 | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/config.ini.sample b/config.ini.sample index dce2db06..7868d852 100644 --- a/config.ini.sample +++ b/config.ini.sample @@ -74,4 +74,6 @@ meshsense=https://meshsense.affirmatech.com/ # Header background color (navbar) header_color=#9fdef9 # Accent color (login/logout button, etc.) -accent_color=#17a2b8 \ No newline at end of file +accent_color=#17a2b8 +# Page background color +page_background_color=#ffffff \ No newline at end of file diff --git a/templates/layout.html.j2 b/templates/layout.html.j2 index dc4f6e53..94d5e589 100644 --- a/templates/layout.html.j2 +++ b/templates/layout.html.j2 @@ -36,6 +36,9 @@ .btn-accent:hover, .btn-info:hover { filter: brightness(0.9); } + body { + background-color: {{ config['theme'].get('page_background_color', '#f8f9fa') }} !important; + } {% block head %}{% endblock %} From f7246ff35f30537068dfd6e6ad56c35cef1aaeea Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 29 Jun 2025 22:16:39 +0000 Subject: [PATCH 088/155] Enhance templates and logic by adding datetime to Jinja globals, improving date handling in layout, and ensuring safe JSON output for node names in message map. --- meshinfo_web.py | 3 +++ templates/layout.html.j2 | 2 +- templates/message_map.html.j2 | 12 ++++++------ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/meshinfo_web.py b/meshinfo_web.py index 427db458..360bc36b 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -171,6 +171,7 @@ def cleanup_cache(): app.jinja_env.globals.update(time_ago=time_ago) app.jinja_env.globals.update(min=min) app.jinja_env.globals.update(max=max) +app.jinja_env.globals.update(datetime=datetime) # Add template filters @app.template_filter('safe_hw_model') @@ -2542,6 +2543,8 @@ def get_distance(node_data, ref_pos): print(f"[RelayMatch] Candidates for suffix {relay_suffix}:") for nid, (score, reasons) in scores.items(): print(f" {nid}: score={score}, reasons={reasons}") + if not scores: + return None best = max(scores.items(), key=lambda x: x[1][0]) if debug: print(f"[RelayMatch] Selected {best[0]} for suffix {relay_suffix} (score={best[1][0]})") diff --git a/templates/layout.html.j2 b/templates/layout.html.j2 index 94d5e589..3e98d895 100644 --- a/templates/layout.html.j2 +++ b/templates/layout.html.j2 @@ -74,7 +74,7 @@ {% endfor %}
    - {% if datetime.datetime.now().weekday() == 0 %} + {% if datetime.now().weekday() == 0 %}
  • #MeshtasticMonday
    • diff --git a/templates/message_map.html.j2 b/templates/message_map.html.j2 index 45e52515..3d7f2bd2 100644 --- a/templates/message_map.html.j2 +++ b/templates/message_map.html.j2 @@ -146,8 +146,8 @@ node: { id: '{{ from_id_hex }}', {# Get names from the main nodes dict #} - name: '{{ nodes[from_id_hex].short_name if from_id_hex in nodes else "???" }}', - longName: '{{ nodes[from_id_hex].long_name if from_id_hex in nodes else "Unknown Sender" }}', + name: {{ nodes[from_id_hex].short_name|tojson if from_id_hex in nodes else '"???"' }}, + longName: {{ nodes[from_id_hex].long_name|tojson if from_id_hex in nodes else '"Unknown Sender"' }}, type: 'sender', lat: {{ sender_position.latitude_i / 10000000 }}, lon: {{ sender_position.longitude_i / 10000000 }}, @@ -180,8 +180,8 @@ {% set details = receiver_details.get(receiver_id) %} var nodeData = { id: '{{ rid_hex }}', - name: '{{ nodes[rid_hex].short_name if rid_hex in nodes else "???" }}', - longName: '{{ nodes[rid_hex].long_name if rid_hex in nodes else "Unknown Receiver" }}', + name: {{ nodes[rid_hex].short_name|tojson if rid_hex in nodes else '"???"' }}, + longName: {{ nodes[rid_hex].long_name|tojson if rid_hex in nodes else '"Unknown Receiver"' }}, snr: {{ details.rx_snr if details else 'null' }}, hops: {% if details and details.hop_start is not none and details.hop_limit is not none %}{{ details.hop_start - details.hop_limit }}{% else %}0{% endif %}, type: 'receiver', @@ -311,8 +311,8 @@ var nodeData = { id: '{{ rid_hex }}', - name: '{{ nodes[rid_hex].short_name if rid_hex in nodes else "???" }}', - longName: '{{ nodes[rid_hex].long_name if rid_hex in nodes else "Unknown Receiver" }}', + name: {{ nodes[rid_hex].short_name|tojson if rid_hex in nodes else '"???"' }}, + longName: {{ nodes[rid_hex].long_name|tojson if rid_hex in nodes else '"Unknown Receiver"' }}, snr: {{ details.rx_snr if details else 'null' }}, hops: hops, type: 'receiver', From 0f52917ce6aa0e7341d707cb41efca9fb28650a2 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 29 Jun 2025 22:38:58 +0000 Subject: [PATCH 089/155] Fix datetime reference in layout template to ensure correct weekday evaluation for conditional rendering of #MeshtasticMonday link. --- templates/layout.html.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/layout.html.j2 b/templates/layout.html.j2 index 3e98d895..94d5e589 100644 --- a/templates/layout.html.j2 +++ b/templates/layout.html.j2 @@ -74,7 +74,7 @@ {% endfor %}
    - {% if datetime.now().weekday() == 0 %} + {% if datetime.datetime.now().weekday() == 0 %}
  • #MeshtasticMonday
    • From 2dbb62cf5487be367dbf83982895763d4715af85 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 29 Jun 2025 22:45:08 +0000 Subject: [PATCH 090/155] Refactor datetime reference in Jinja globals to use datetime.datetime for consistency and correct weekday evaluation in layout template. --- meshinfo_web.py | 2 +- templates/layout.html.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/meshinfo_web.py b/meshinfo_web.py index 360bc36b..d94c4ce9 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -171,7 +171,7 @@ def cleanup_cache(): app.jinja_env.globals.update(time_ago=time_ago) app.jinja_env.globals.update(min=min) app.jinja_env.globals.update(max=max) -app.jinja_env.globals.update(datetime=datetime) +app.jinja_env.globals.update(datetime=datetime.datetime) # Add template filters @app.template_filter('safe_hw_model') diff --git a/templates/layout.html.j2 b/templates/layout.html.j2 index 94d5e589..3e98d895 100644 --- a/templates/layout.html.j2 +++ b/templates/layout.html.j2 @@ -74,7 +74,7 @@ {% endfor %} - {% if datetime.datetime.now().weekday() == 0 %} + {% if datetime.now().weekday() == 0 %}
  • #MeshtasticMonday
    • From e693c3d4ce2abcdfc786db5e3c76c50ed0a3be96 Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 30 Jun 2025 02:21:00 +0000 Subject: [PATCH 091/155] Enhance configuration and templates by adding new theme color options, updating layout styles, and centering the JSON download link for improved UI consistency. --- config.ini.sample | 22 +- generate_css.py | 40 ++ run.sh | 1 + templates/layout.html.j2 | 17 - templates/neighbors.html.j2 | 4 +- www/css/meshinfo.css.template | 687 ++++++++++++++++++++++++++++++++++ 6 files changed, 752 insertions(+), 19 deletions(-) create mode 100644 generate_css.py create mode 100644 www/css/meshinfo.css.template diff --git a/config.ini.sample b/config.ini.sample index 7868d852..508956cd 100644 --- a/config.ini.sample +++ b/config.ini.sample @@ -73,7 +73,27 @@ meshsense=https://meshsense.affirmatech.com/ [theme] # Header background color (navbar) header_color=#9fdef9 +# Header brand/title color +header_brand_color=#000 +# Header navigation link color +header_link_color=#555 +# Header active navigation link color +header_link_active_color=#000 # Accent color (login/logout button, etc.) accent_color=#17a2b8 # Page background color -page_background_color=#ffffff \ No newline at end of file +page_background_color=#ffffff +# Table header background color +table_header_color=#D7F9FF +# Table border color +table_border_color=#dee2e6 +# Table alternating row background color +table_alternating_row_color=#f0f0f0 +# Link color +link_color=#007bff +# Link hover color +link_color_hover=#0056b3 +# Control color (buttons, form controls) +control_color=#17a2b8 +# Control hover color +control_color_hover=#1396a5 \ No newline at end of file diff --git a/generate_css.py b/generate_css.py new file mode 100644 index 00000000..d16dc3e1 --- /dev/null +++ b/generate_css.py @@ -0,0 +1,40 @@ +import configparser +import os +import logging + +CONFIG_PATH = 'config.ini' +TEMPLATE_PATH = 'www/css/meshinfo.css.template' +OUTPUT_PATH = 'www/css/meshinfo.css' + +# Default theme values +DEFAULTS = { + 'header_color': '#9fdef9', + 'table_header_color': '#D7F9FF', + 'table_alternating_row_color': '#f0f0f0', + 'accent_color': '#17a2b8', + 'page_background_color': '#ffffff', + 'table_border_color': '#dee2e6', + 'link_color': '#007bff', + 'link_color_hover': '#0056b3', + 'control_color': '#17a2b8', + 'control_color_hover': '#1396a5', + 'header_link_color': '#555', + 'header_link_active_color': '#000', + 'header_brand_color': '#000', +} + +config = configparser.ConfigParser() +config.read(CONFIG_PATH) +theme = dict(DEFAULTS) +theme.update(config['theme'] if 'theme' in config else {}) + +with open(TEMPLATE_PATH) as f: + css = f.read() + +for key, value in theme.items(): + css = css.replace('{{' + key + '}}', value) + +with open(OUTPUT_PATH, 'w') as f: + f.write(css) + +logging.info(f"Generated {OUTPUT_PATH} from {TEMPLATE_PATH} using theme from {CONFIG_PATH}") \ No newline at end of file diff --git a/run.sh b/run.sh index 1fa797ce..7e55276c 100755 --- a/run.sh +++ b/run.sh @@ -1,3 +1,4 @@ #!/usr/bin/env bash +python generate_css.py python main.py diff --git a/templates/layout.html.j2 b/templates/layout.html.j2 index 3e98d895..5027163e 100644 --- a/templates/layout.html.j2 +++ b/templates/layout.html.j2 @@ -24,23 +24,6 @@ - - {% block head %}{% endblock %} diff --git a/templates/neighbors.html.j2 b/templates/neighbors.html.j2 index 5f565601..fd3d2baf 100644 --- a/templates/neighbors.html.j2 +++ b/templates/neighbors.html.j2 @@ -194,6 +194,8 @@



    - Download JSON +
    + Download JSON +
    {% endblock %} \ No newline at end of file diff --git a/www/css/meshinfo.css.template b/www/css/meshinfo.css.template new file mode 100644 index 00000000..177a94d5 --- /dev/null +++ b/www/css/meshinfo.css.template @@ -0,0 +1,687 @@ +body { + font-family: monospace; + font-size: 12px; + background-color: {{page_background_color}} !important; +} + +th { + background-color: {{table_header_color}}; + text-align: center; +} + +nav { + background-color: {{header_color}}; +} + +.table-striped>tbody>tr:nth-child(2n+1)>td, +.table-striped>tbody>tr:nth-child(2n+1)>th { + background-color: {{table_alternating_row_color}}; +} + +/* Main table striping - consolidated rule for odd rows only */ +.table-striped>tbody>tr:nth-of-type(odd) { + background-color: {{table_alternating_row_color}} !important; +} + +/* Make inner tables completely transparent */ +.neighbor-table { + border-style: hidden !important; + background: transparent !important; +} + +.neighbor-table tr { + background: transparent !important; + border-style: hidden !important; +} + +.neighbor-table td { + border-style: hidden !important; + padding-right: 8px; + background: transparent !important; +} + +/* Badge styling */ +.badge.bg-good-snr { + background-color: #2d8659; + /* Dark green */ + color: white; +} + +.badge.bg-neighbor { + background-color: #2b5797; + /* Dark blue */ + color: white; +} + +.badge.bg-message { + background-color: #505050; + /* Dark gray */ + color: white; +} + +/* Additional badge styles */ +.badge.bg-purple { + background-color: #6f42c1; + color: white; +} + +.badge.bg-success { + background-color: #28a745; + color: white; +} + +/* Layout classes */ +.neighbor-table td.node-id { + min-width: 85px; + /* Increased from 60px to accommodate 4 letters + badge */ + white-space: nowrap; + /* Prevent wrapping */ + padding-right: 8px; +} + +.neighbor-table td.snr-value, +.neighbor-table td.distance, +.neighbor-table td.message-count { + white-space: nowrap; +} + +.neighbor-table td.node-id .badge { + margin-right: 4px; + /* Add consistent spacing between badge and name */ + display: inline-block; + /* Ensure badge and text stay together */ +} + +/* SNR badge styling */ +.badge.snr-good { + background-color: #C8E6C9; + /* Muted light green */ + color: black; +} + +.badge.snr-adequate { + background-color: #FFF9C4; + /* Muted light yellow */ + color: black; +} + +.badge.snr-poor { + background-color: #FFE0B2; + /* Muted light orange */ + color: black; +} + +.badge.snr-very-poor { + background-color: #FFCCBC; + /* Muted light red-orange */ + color: black; +} + +/* Additional Styles for Chat2 */ +.chat-container { + max-width: 1200px; + margin: 0 auto; + padding: 20px; +} + +.chat-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 20px; + padding: 10px 0; + border-bottom: 1px solid #e0e0e0; +} + +.chat-messages { + display: flex; + flex-direction: column; + gap: 16px; +} + +.message-container { + width: 100%; +} + +.message-bubble { + background: {{table_alternating_row_color}}; + border-radius: 12px; + padding: 12px 16px; + position: relative; +} + +.message-header { + display: flex; + align-items: center; + gap: 12px; + margin-bottom: 8px; + font-size: 0.9em; +} + +.message-sender { + font-weight: 500; + flex-grow: 1; +} + +.map-link, +.map-link-placeholder { + width: 32px; + /* Set a fixed width */ + text-align: center; +} + +.node-link:hover { + text-decoration: underline; +} + +.message-timestamp { + color: #666; + font-size: 0.85em; + white-space: nowrap; +} + +.map-link { + color: #2b5797; + font-size: 1.1em; + padding: 4px 8px; + border-radius: 4px; + transition: background-color 0.2s; + display: inline-block; +} + +.map-link:hover { + background-color: rgba(0, 0, 0, 0.05); + text-decoration: none; +} + +.map-link-placeholder { + display: inline-block; +} + +.message-content { + word-wrap: break-word; + font-size: 1.1em; + margin: 8px 0; +} + +.message-receptions { + display: flex; + flex-wrap: wrap; + gap: 8px; + margin-top: 8px; + padding-top: 8px; + border-top: 1px solid rgba(0, 0, 0, 0.1); +} + +.reception-badge { + display: inline-flex; + align-items: center; + background: #f8f9fa; + border: 1px solid {{table_border_color}}; + border-radius: 12px; + padding: 4px 10px; + font-size: 0.85em; + text-decoration: none; + color: inherit; + transition: background-color 0.2s; + margin: 2px; +} + +.reception-badge .snr-indicator { + margin-right: 6px; +} + +.reception-badge:hover { + background-color: #e9ecef; + text-decoration: none; + color: inherit; +} + +.snr-indicator { + width: 8px; + height: 8px; + border-radius: 50%; +} + +.snr-good .snr-indicator { + background-color: #2d8659; +} + +.snr-adequate .snr-indicator { + background-color: #ffd700; +} + +.snr-poor .snr-indicator { + background-color: #ff9800; +} + +.snr-very-poor .snr-indicator { + background-color: #f44336; +} + +.receiver-popover { + font-size: 0.9em; + line-height: 1.4; +} + +.load-more { + display: flex; + justify-content: center; + margin: 20px 0; +} + +.load-more button { + background: transparent; + border: 1px solid {{header_color}}; + color: #2b5797; + padding: 8px 16px; + border-radius: 20px; + cursor: pointer; + transition: all 0.2s ease; +} + +.load-more button:hover { + background: {{header_color}}; +} + +.load-more button:disabled { + opacity: 0.6; + cursor: not-allowed; +} + +@media (max-width: 768px) { + .chat-container { + padding: 10px; + } + + .message-header { + flex-wrap: wrap; + } + + .message-receptions { + gap: 4px; + } + + .reception-badge { + font-size: 0.8em; + } +} + +.message-channel { + color: #666; + font-size: 0.85em; + padding: 2px 6px; + background: rgba(0, 0, 0, 0.05); + border-radius: 4px; + white-space: nowrap; + margin-right: 0.5em; +} + +.btn-group .btn { + padding: 0.25rem 0.5rem; +} + +.btn-group .btn i { + font-size: 1rem; +} + +/* Mobile Reception Styles */ +.mobile-receptions { + display: none; + flex-direction: column; + gap: 8px; + width: 100%; +} + +.mobile-reception-row { + background: #ffffff; + border: 1px solid #e0e0e0; + border-radius: 8px; + padding: 8px 12px; +} + +.mobile-node-name { + font-weight: 500; + margin-bottom: 4px; +} + +.mobile-node-name a { + color: inherit; + text-decoration: none; +} + +.mobile-metrics { + display: flex; + flex-wrap: wrap; + gap: 8px; + font-size: 0.9em; +} + +.mobile-metrics .metric { + display: flex; + align-items: center; + gap: 4px; + padding: 2px 6px; + border-radius: 4px; + background: #f5f5f5; +} + +.mobile-metrics .snr .snr-indicator { + display: inline-block; + width: 8px; + height: 8px; + border-radius: 50%; + margin-right: 4px; +} + +/* Desktop/Mobile visibility */ +.desktop-receptions { + display: flex; +} + +@media (max-width: 768px) { + .desktop-receptions { + display: none !important; + } + + .mobile-receptions { + display: flex !important; + } + + .message-receptions { + margin-top: 12px; + } + + /* Hide popovers on mobile */ + .popover { + display: none !important; + } +} + +@media (min-width: 769px) { + .desktop-receptions { + display: flex !important; + } + + .mobile-receptions { + display: none !important; + } + + /* Ensure popovers display properly on desktop */ + .popover { + max-width: 276px; + font-family: inherit; + z-index: 1060; + display: block; + } +} + +/* Map Settings Panel Styles */ +#settings { + max-width: 350px; + font-size: 0.9em; + transition: all 0.3s ease; +} + +#settings h5 { + font-size: 1.1em; + font-weight: 500; +} + +#settings .form-label { + font-weight: 500; + font-size: 0.9em; + margin-bottom: 0.25rem; +} + +#settings .form-select-sm, +#settings .form-control-sm { + font-size: 0.85em; + padding: 0.25rem 0.5rem; +} + +#settings .text-muted { + font-size: 0.8em; + opacity: 0.8; +} + +#settings .mb-3:last-child { + margin-bottom: 0 !important; +} + +/* Make settings panel collapsible on mobile */ +@media (max-width: 768px) { + #settings { + max-height: 60vh; + overflow-y: auto; + max-width: calc(100% - 40px); + margin: 20px; + } +} + +/* Add smooth transitions for filter changes */ +.ol-layer { + transition: opacity 0.3s ease; +} + +/* Improve visibility of map controls */ +.ol-control button { + background-color: rgba(255, 255, 255, 0.9) !important; + color: #333 !important; +} + +.ol-control button:hover { + background-color: rgba(255, 255, 255, 1) !important; +} + +/* Settings Toggle Button */ +.settings-toggle { + position: fixed; + top: 70px; /* Below navbar */ + right: 20px; + z-index: 1001; + width: 40px; + height: 40px; + border-radius: 50%; + background: white; + border: 1px solid {{table_border_color}}; + box-shadow: 0 2px 4px rgba(0,0,0,0.1); + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; + transition: all 0.2s ease; +} + +.settings-toggle:hover { + background: #f8f9fa; + transform: rotate(30deg); +} + +.settings-toggle i { + font-size: 1.2em; + color: #333; +} + +/* Settings Panel */ +.settings-panel { + position: fixed; + top: 120px; /* Below settings toggle */ + right: 20px; + z-index: 1000; + background-color: white; + border-radius: 8px; + box-shadow: 0 2px 4px rgba(0,0,0,0.1); + padding: 1rem; + max-width: 350px; + max-height: calc(100vh - 140px); + overflow-y: auto; + font-size: 0.9em; +} + +/* Node Info Panel */ +.node-info-panel { + position: fixed; + top: 180px; /* Increased from 200px to move it down 70px */ + right: 20px; + z-index: 999; + background-color: white; + border-radius: 8px; + box-shadow: 0 2px 4px rgba(0,0,0,0.1); + padding: 1rem; + max-width: 350px; + max-height: calc(100vh - 140px); + overflow-y: auto; +} + +/* Make settings and node info panels responsive */ +@media (max-width: 768px) { + .settings-panel, + .node-info-panel { + max-width: calc(100% - 40px); + margin: 20px; + } + + .settings-panel { + top: 70px; + max-height: calc(100vh - 90px); + } + + .node-info-panel { + top: auto; + bottom: 20px; + max-height: 60vh; + } +} + +/* Improve map controls visibility */ +.ol-control { + background: none; +} + +.ol-control button { + background-color: white !important; + color: #333 !important; + border: 1px solid {{table_border_color}} !important; + border-radius: 4px !important; + margin: 2px !important; + transition: all 0.2s ease; +} + +.ol-control button:hover { + background-color: #f8f9fa !important; + color: #000 !important; +} + +/* Make OpenStreetMap attribution less prominent on all OpenLayers maps */ +.ol-attribution, .ol-attribution ul { + font-size: 0.75em !important; + opacity: 0.7; +} + +/* --- THEME AND CUSTOM STYLES FROM layout.html.j2 --- */ + +.navbar { + background-color: {{header_color}} !important; + color: {{header_link_color}} !important; +} + +.navbar-brand { + color: {{header_brand_color}} !important; +} + +.navbar .navbar-brand:hover, .navbar .navbar-brand:focus { + color: {{header_link_color}} !important; + text-decoration: none !important; +} + +.navbar .nav-link, .navbar .dropdown-item { + color: {{header_link_color}} !important; + text-decoration: none !important; +} +.navbar .nav-link.active, .navbar .active > .nav-link, .navbar .nav-link:active { + color: {{header_link_active_color}} !important; + text-decoration: none !important; +} +.navbar .nav-link:hover, .navbar .nav-link:focus, .navbar .dropdown-item:hover, .navbar .dropdown-item:focus { + color: {{header_link_active_color}} !important; + text-decoration: none !important; +} + +a { + color: {{link_color}} !important; +} +a:hover, a:focus { + color: {{link_color_hover}} !important; + text-decoration: underline; +} + +.btn, .form-control, .page-link { + border-color: {{control_color}} !important; + color: #fff !important; +} +.btn-primary, .btn-accent, .btn-info { + background-color: {{control_color}} !important; + border-color: {{control_color}} !important; + color: #fff !important; +} +.btn-primary:hover, .btn-accent:hover, .btn-info:hover, +.btn-primary:focus, .btn-accent:focus, .btn-info:focus, +.btn-primary:active, .btn-accent:active, .btn-info:active { + background-color: {{control_color_hover}} !important; + border-color: {{control_color_hover}} !important; + color: #fff !important; + filter: brightness(0.95); +} +.btn-outline-primary { + color: {{control_color}} !important; + border-color: {{control_color}} !important; + background-color: #fff !important; +} +.btn-outline-primary:hover, +.btn-outline-primary:focus, +.btn-outline-primary:active, +.btn-outline-primary.active { + color: #fff !important; + background-color: {{control_color_hover}} !important; + border-color: {{control_color_hover}} !important; +} + +/* Prevent global link color from affecting chat node links */ +.chat-messages a, +.message-container a { + color: #000 !important; +} + +/* Table theming */ +.table thead th { + background-color: {{table_header_color}} !important; +} +.table { + border-color: {{table_border_color}} !important; +} +.table td, .table th { + border-color: {{table_border_color}} !important; +} + +/* Pagination link and icon theming */ +.page-link, .btn-outline-secondary { + color: {{control_color}} !important; + border-color: {{control_color}} !important; + background-color: #fff !important; +} +.page-link:hover, .page-link:focus, .btn-outline-secondary:hover, .btn-outline-secondary:focus { + color: #fff !important; + background-color: {{control_color_hover}} !important; + border-color: {{control_color_hover}} !important; +} +.page-link .bi, .btn-outline-secondary .bi { + color: inherit !important; +} +.page-link:disabled, .btn-outline-secondary:disabled { + color: #ccc !important; + background-color: #f8f9fa !important; + border-color: #dee2e6 !important; +} + +/* --- END MIGRATED STYLES --- */ \ No newline at end of file From 21d86a0bc0d9113c4019eff526ab77bd9e8f6009 Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 30 Jun 2025 02:48:18 +0000 Subject: [PATCH 092/155] Update templates and configuration to replace logo image format, enhance text clarity, and improve layout structure for better user experience. --- docker-compose.yml.sample | 1 + templates/index.html.j2 | 20 +++++++++++--------- templates/layout.html.j2 | 3 ++- www/images/logos/logo.webp | Bin 0 -> 16176 bytes 4 files changed, 14 insertions(+), 10 deletions(-) create mode 100644 www/images/logos/logo.webp diff --git a/docker-compose.yml.sample b/docker-compose.yml.sample index 9e9111cb..b0e09df7 100644 --- a/docker-compose.yml.sample +++ b/docker-compose.yml.sample @@ -21,6 +21,7 @@ services: volumes: - ./config.ini:/app/config.ini - ./srtm_data:/app/srtm_data + # - ./logo.webp:/app/www/images/logos/logo.webp environment: - PYTHONUNBUFFERED=1 ports: diff --git a/templates/index.html.j2 b/templates/index.html.j2 index 89ac6fcb..feb4e83e 100644 --- a/templates/index.html.j2 +++ b/templates/index.html.j2 @@ -10,10 +10,10 @@
    - logo + logo

    {{ mesh_name }}

    - This site provides information on {{ active_nodes|length }} active nodes in the {{ mesh_region }} region ({{ mesh_name }} network). + This site provides information on {{ active_nodes|length }} active nodes in the {{ mesh_region }} region ({{ mesh_short_name }}) network.

    Last updated: {{ format_timestamp(timestamp.timestamp()) }}

    @@ -57,7 +57,7 @@ Enable Position Sharing (optional but recommended): Go to Settings > Channels > LongFast and set "Position Enabled" to TRUE. Set "Uplink Enabled" to TRUE. This allows your node's position and recieved messages to be shared with other nodes and the MQTT server.
  • - Set Position Precision: Set position precision slider to 1194ft (364m). This is the most accurate setting that will still show up on maps. You can use higher precision (lower distance values), but they will be rounded to this level for privacy reasons. Settings with lower precision (larger distances) will appear as configured. + Set Position Precision: Set the position precision slider to a value you are comfortable with sharing. The map and node information pages will represent your position with the accuracy that you select for public channels and MapInfo packets.
  • Configure Your Position: Either use GPS (if your device has one) or set a fixed position via Settings > Position. @@ -65,9 +65,9 @@

    Note: Your node will still appear in MeshInfo without location data, but distance and direction information to other nodes won't be available.

    -

    Mesh MQTT Connection (recommended):

    -

    Important: Users who don't connect directly to the MQTT server won't be sharing information with other users in the region. Connecting your node to the MQTT server allows your data to contribute to the regional mesh network and helps everyone better understand network coverage.

    -

    To connect your node directly to the {{ mesh_short_name }} MQTT server, configure these settings in Settings > Modules > MQTT:

    +

    MQTT Connection (recommended):

    +

    Important: Users who connect to the MQTT can share what their node is hearing on the LoRa network. Connecting your node to the MQTT server allows your data to contribute to analysis of the regional mesh network, and helps everyone better understand network coverage.

    +

    To connect your node to the {{ mesh_short_name }} MQTT server, configure these settings in Settings > Modules > MQTT:

    • @@ -110,12 +110,14 @@ Benefits: When enabled across multiple nodes, this helps build a comprehensive graph of the mesh network, showing how nodes are connected and the quality of links between them (displayed as SNR values).
    - + {% if config['mesh'].get('config_url') %}

    For complete setup instructions, visit the {{ mesh_short_name }} configuration page:

    - {% if config['mesh'].get('config_url') %} - View Full Configuration Guide + +
    + View Full Configuration Guide +
    {% endif %}
    • diff --git a/templates/layout.html.j2 b/templates/layout.html.j2 index 5027163e..8650161c 100644 --- a/templates/layout.html.j2 +++ b/templates/layout.html.j2 @@ -57,7 +57,8 @@ {% endfor %} - {% if datetime.now().weekday() == 0 %} + {% set current_local_time = convert_to_local(datetime.now().timestamp()) %} + {% if current_local_time.weekday() == 0 %}
  • #MeshtasticMonday
    • diff --git a/www/images/logos/logo.webp b/www/images/logos/logo.webp new file mode 100644 index 0000000000000000000000000000000000000000..621d00016b9f361f21c428a59fea4706818fb212 GIT binary patch literal 16176 zcmV-0KhMBYNk&E}KL7w%MM6+kP&il$0000G0002T008s=06|PpNcsQ(00DqoZQJtX ziMzWCoeTmMBT(rIFaaAlRBRFHY3V3k2k$&V5_AF)PNmjpcT>@2<4az4+Oc= zTG~T15yH0r|KqRnwE08hOW?2aG}+Z!WVfypbXVg`;IHyD+0|NPx2_X(SK~|Iuktk6 z)mmh?t`lsD)*`!F>)-P{-71}184kUgQfpY+tO2F3&fTdRju#W%$>({d1eH}Ra zfOF%pIYIllYYu78&7tk4DT#Kqm&J6N*Nq{~gQ3k1sWbpqP&gn6J^%o4mjImsD#!r! z06t9~ibNuzp%;$slqdwmv$uIq*MY`0{yNYAWhNoE&Tlmc(3hKy;{5hkmY3ANGxFKs zC({2PywChU;~${^UH=pP>-z`)PxD^>zqEJT^FQ4`#Q$&g7x=f_FYdp!Kj6MVzZL#R{hRv-?O*o~-G85tmj4O;o4GSZ{;mFV{BP|~ zz!$0?G5+uUqx?_&KkmQ0KFHs?{$c-P_z&{W=^x#{w*KdTcmMzR0sM3Mhxh;Pf4INl zfByg3`uqE@`QH}*s(*0*PyR3V_xvZ=7xFLXf8)Qnf0zHS{xkpo{y)UO+<)1Bc>g{B zr~K#t|99V)e_#Jw{oDMv{!jED|Nj9$kAFS?IsL!p>s%qmblo(yHX4=koU) zm&vBCl)Y)KRuWTTn8U3+1>E6~i4RmLstQBs6pT$~w641PnlHU^c%6D{E`n23ac5(5 zY+Bik(pPhb!oXBpsBa}qrIVkW6JcJ;ylI;f7wh~1UW7-e-$YPg#$wO2vl(%6&f$Fx z@*WG(L|aR!T!RRaPu7Mrj`iDih0DaXopkfmqkyRl{}Z11!GlWNvK-~?{9V2}%6WXD z*pgzOtO{SIt2jgfSbgs9!HfuUm_p|=3@V&zvJdS{ui0b66QSyW2LTw9w`XCj;SPtECP}MFJ}t+j zA964=!4;pxqL}Wv9iH#&EhK4?L%|UkC}52uF4zX1tz+VXAH$EQo7(K8!7B8SRac7o zMV?(rL|giT(6hRYEm*|s5E3-&rzH5RWZN{#C&d1-gHn|zo*OeLQK_Y#(E;HzfFu04 z==)M}A%ktr47APY%`NbL#^3pOmsEXWn!he*dff6isaK-5()(EDcQkHmj@S}XP>&v_%+b!5Lb$$%&Q{uyar-42cOoV5xUG?C%|_{2js*G>T*jd|4IXQ+i5p84IB`54y_1)-gXh95;JE=Ovc{ zIw|zsy}9nHrW)s{kun+EzqmJ8Lc$>+0<%sNvl(dzPRg__ffsd@DKSTGIu@ z1UFFQTPLeuCXr+z(Waao36QWv`SGUGgJEt46F&v6!AXXtK)kQ<=2eyk^`ylc;2(EQ zC|~@)W^^4d;Pv#!Bpqc$8g2PGYV%SDk<(OLBJ`qsU2; zZ(gUmVpe61&|5|TQ`P2wx2}8-tXT2f8B@bU0HFYG84n$cR186s{*^huI{~Gf`?v?^ zzl1TCwHKy%Sn2+U8=VjBo3V;G-qZo@1ncEPe$am)liT0k&A3nRoreScb>*Rqc7AH| zQ)DC&ODeb4@UtY>?Bkm_WuEN7{EfZ0Y%vBxux&i&e1;7twWbS4oBp?R*3c`Wi%;`< zMVLtm(C<9FjM#qn&>%;5y8ZqzEU{9q;Z#DWHYg$$jL|dobkYOj?-Vk&_T8|=85Qtk zo8S|3iSRK0Y(4oUB7X+L5C^d|OH5OrsGqYgfB0nT`23|*mHX`kc5~B0t%FF#b>ll* zZ}V(R`9h*vNbJqw63Bc6OLB>WO1OqSF)KU=?DR4eIeIwkcrULcyHul}XNwgrQyBlzUIigOItQ1TJn+jR|0~8mB z9HTLgB7B{c^Z5jswovhxB$p&Z%ZK5djp&qH&LSpAv~OD?+#kiXIDl8`1SjUcn;U(X4lXNW11xBm(~_@%r@!|IK_VHxeF@I zpj{}y9F90CcsAS*V?LXA*tlW$Q*EQ&bv!vOg(SMTL}#pT9Iuu@$X>eRR42;5P5i?; zJ*?K-aJLOhD|ty1i+NQ?-Y#e}8{|@!XhyIaTm&CWGv8qLJ=97!IT=S=JJT%{otP>F zS$0&M`CollKdVPVVe$5m{RjtjEG09Cny4^e9F4EoARMSE zlRv28tg=jy;*m+eM>*Pynj%*8N0A9+!?WeSF1@S|V&YyG*2WuHUsjO+5@zdEQivk$ zzKF6lci1|`x)iz380gB|;JYXgQ1aZc^JV#zgaLLMq)x~HdtFo$*8 zu~7o+g~A`}F4IREG2Jb9b|>EBQ1yzJV;p>!@_vyd0dONDUmBy_)}9|PDG;aTX*~yw zHVP}M_zhRUS!KS{7J`Qm>3gP6+#^#SH4&11+ilKg_rbn%Tq;w)x|;)Jc-XzcG38J} zc;D|!O3n25Jay|?B?)D8|4F%eT6&~;d=n%2=9~+6h*@-3Fu8$ea zE4rKBrAid@I3I_-B?+&-2f|p7<{ksH^Tj=s4Z@CT)1`<%J?FNd)_GO3ZYmL$`I0Ps zqy^@#@Q~FlB`=CLE@p1J?OC;3x}2-6kftlHv5aSfWZ9XQT36z(v->}&EpZ%5AxzN^ zt%AIg#C)KshQvFfV@f(rYzVNR?A2;FmC#70!6-C}K$J!Jb!8R=ej4FBU7sw`F|~O_ z4{7Bt09)F{Aelm%bJ$b+dARIhNNXuho$f&t1zr2-1V6V$MiPY8u9LNU7_+w@v0CY! z?IG_)}w!q0h@xW0ogCD;wqZIbq20Bi#}T#=q6`;_u;vC*yDSH+q0Wi zSs5oq;~nrrP<-HwifE`o-lRsr%)p~NZx4s1%f)~iR$O(d8Xr6Io-*Gga>eTtjZO&M$Hr zK)mq#yk{6%L$Aeu)Y!{(NoUh?VngD$lr0g)KIaeBL42Qz7cr4~&5}c;gD^zKu$FT-qFwsD zrs)Bns(b2d`ZSJProXJ~PrSJ_fRI3JKy-`a_B&bV0s@9j;hqRkF&=Gx*Ph_-4=xbsNa?#^> zW~s`rf;qgm)uoTtK>tGWzFAKHOxbW*85&W2i_soeA>ajH|F0xW11DRAduClM%jH|nQ>onVQ#++gb1<)+nHem?ka zv-HV7+hm+HVdmY#&RTW6dGVriAM^|i;EH0`pXBU$>9wr=e6O^DBQ!F)o(lMitx8l1 zGPg2ZcB2NchP`ZKuOO+ci!Jvv2G)%>tThmdQ8t_gjm1kiBkRcmwdehC&5*0O=4kCs zVvq8|U5dYFvgbfx^P5a|!#@3nMuMEqe-lW`sR2)Bv>kbt-D>l$YTFLtoDK4F5rKV9 zqPoH`X!Y4T0&N-*1W!rZdiC%5li;vdW$euSzyrHU+_T|A#oX^G(5fzWeG{VEC5R`+ z*g*DDIQpQLJUy*`ZdJso4hG}UFNC&CO}~778DGX%7q6d*!fTzO33`?(?tM?nF44Iv z!EN<_eWr$TD1lsKmOb5OoGgbyGq@3zLFsf)m{$h>v>pdw^27qbc=*gqEcDch&i$F& z00n89TDn6H81#HzfH?{!AmST5)^+ysXhxcN(mtXlKO73!7T0*#8MbuEsYEgPTTSO_ z9b+U#KlL67Wv6`oHIWRclKA)~u^ApSD>kL+33WB^XmWK!DLJ><J>P9teCj5 zZqDc5d$V7#CYMP=9OFnLjv++v=2s_3uffuGDwh1J$%X~O%lxHYs znzP$QqJo3sVkMU>z2omuK*Je&;6-0G2XXvQm%2n`Dp*hq&eHDAmL+CJ+58K$Kj3!F zleMx@6%#fnIE5RP_t*8FT7BmbN9Br&X@_cb=Cell+BQgVgJy`wyVwz1{WzXzlqQl< zmzX-gP662s+rFV_GV327WYkJHY@%z52%0;Dq9vA&+ecrGk|wDKOY3szK+05!M}?Z^ zIm1=euqQxqET6g|o&Z6s-HCqeT2t-5ny$dnoJa`D+-DscbWW%A8ToDlx&(fu832X+_&8^1S-^$3lT>+?Bfu~41q>%Iport|Kz zHcp-*^3)lYIJxaFqL$GNu@7|6PPr;H!|u8{{V`wt1Ct&9{Pb=}l-A+H^P~R+@Rh6x zkFzn&eUzf-AkJ5KWW0i&`)Qn zg3+Z+ZO~3m$FeF%!=~-MQbe+pIHFgSft%*GN+|Xah}3-XuQqkfDLp#bf{er(^;k;D z`io_Z+gj?YH?DnS1Z|G-2Nf+XVVyLV2P}9@W?rxl#k-BGkr6`+n$|HiSl$GS(DN6j z*7Kolhf+o#xn*NboD5g_VVhZLybaXa=lW3rC+CyiYxN4DVzcVCt`>_XRk_hfi}`9T zwJkcGjkh_~*lSt?3(Qu}MdV1%Rz55Z%n#J=r5)KV0%^i)ICO8@w}&yWc;SCbjqU;2)q$L zy}+f)qj-1WptM0fpK`ZtN>ugNBd6DHX+hf~*bT*A%`3R@-c6*HjeO^V&ePzKOT-tu z9C@UfvyP47)|+e#=rKZDOsyO}5uJnc_U%F{G+5Oj#6karpFE8-s!O(hV7j(2`4yjI%s6{$$~c@vH^+ z*PNN!()Y1B5^LUE+Y3d8(yPN0qdP9`2lQOBtF&M@#2O%Uia(kK$uJUN-PRex!E zwjso)p1z1uL)|EVsH#Vf6aeDzhyfRnfsxJSQyW(b;R8^v5WFZ8Y*DSgDkVA>$24Oy0xEQPPKu(`?6Ojem*h{z>ZeUk@rx9MXh6T zM0105lh9HwPrQDN|EPuSeFj2^S%$W{m^7Wl!MoHbSQ(n#jQ~FK*xZ!Daamd7`|lPi zKvoi(qI|sF9Ij#$tk-#HUba%UQ}8(t;A1g-!?D;*d%L8;kCDiCA%Kp}!9U&^-D~OG z(CuqbDVB+kAuw*?{aFmJM0jHgVU8rV2EM=g#y=HMOb7(7bIsGwxA-eG$E)bWaKJ_s zMIfMKvXXIU1_$2?CPuoF(KaR)t6!aqgfS}URc7kUZ~bH7X#3OtY*tdwo>yWC+q7eue<(1WZS+)taI`A)KmRPE}&)+jOtOl6ZV)&%cDT zxu(w^@&M8gu=6t5hUKU<&QDJCK+J^p&XA+hvX@9XQ#TC4_UTSc9cv70FohwOt&J;$&rw%0Dx^#bdHbv#b)5Bd zCG3*{+~%(@=%qimi~(CTruA99#(Q^qp-WQee92lPC-nCN`TwTP*^w;A^`UGr18}}S zTu@{x8gqnX&xQ@CQjHRGLPGIHwdGg{LUbH$#j6LsRPGJ-u?|NAgUKzjI8&pTge}IP z<|@V$L2gfP&8WGGQ%)%TM=}an#R)V5Nct4Iajmanthsnr_2LSe-^AL!LNff}FzMa= z{D&0{;r~il#&`e+Z8MP+o}I3?r1lw>jV%k8{^pAK<8)sPwlecUw^5)iR$C&GRMm3i zuSGZ6RH4bE zI{gMzc2ygv3yHuNj<6gjL9Kk)y@=+a8}>sAbLIg?*soiW>sl_YZF7u0Ta#ijfy5*7 zo}7IStGKCRP_(HKg*U}+4oPNYZTZV@`tnLEEDVM@bTil{pYF2LU*S)N7P{&xrT5?* zQsB$^U0aq;n6HRU4n@QP-k;=#W@M5=3qMnu@;MiQ2WUPtjWE7LQFm8@anx;$b8hYl zIP~=$%fI&iYM%yI{9nuaR?odyH`W~ZC6{Sa$%DzkzIjubkC>%R1;+4$>SDNafI!eY&A|L<*>qEs-wo~m^pu|JJBF_V4tpL&$ZfLnV_qY zuTFG1+NEFBrDGHUn!gJFx4`r?5$BH&2b<}Pz7F@K^yt&vFNOE%fw{cjWd9wUjd`Z? z8^=ujHb^_{8tnPpp0$ZkrKUSGfi+*|XU*0Fp$w&x;7~D6Hjwo1%$>Q{E}b?lSv|8- zdYz1`jVZ8+fxJUjQ=8Lt>RzXZ!Av66hK0it>__>m@4+SGFxJB8GDIlH2<)d<*Y%J4 ziDnl)<1Op99vvzo4UZXaco0CT0VotD(&$7T44EEjvW<U>az@|=OZWr5|gdumI$L{BO)K?<23x&s;H8KXPCrA z{%E_l5+|+RYar*?0Equ_^)MF3AL5;H1h9?L*a4dQg=HQplnc?bmkHaU_1JstMH?GZ z8~nhJ#wqx{$bb$PFbnbNLSuCx#q%xY`RTNUWP{LG1|rmugp=6dRlcKNxBg3BW&>8s zw=lx)Eyb58^4>U?3x;y&6~xI(=VnN!AM*(v@bL6fj8(KE^B&Wu4FCv#q{Us^mdLfK}k7F@I^hA z-hn0Z-@q%x@I_%om0FQK)C0Shn|!8096><(H6Rhi>GZjvki6Kiy%oxfa2)wQMp`Xq zj-bg1Y=i^-<=U;S73qZ69jpf8HI#a98Fjk)WjWt;CVEpTmmITy)tba?KW!|ASF7O$ zO!P|jl0)P$ku0Jsd^{Xis|A0$uQxZ>ibe`}yWz@$dgu+;p*Us(-C4gU5|B#<^J_0xR( zUz6EygF4cSKiCs_5KN2btaEuZ({@~hG_41Dhh|Drr6Hw)gd)N7&alw&^3@OwnBWgm z)^V;xLbgR z|Kuvt-?!G!J>s5QKl9#=qU^@B^w(DLkwuxA{KdfP$#3*>Qekz1G^+n=+%!+J(AN#` z*EVnA)yA=?g9gk?GCEDhnE1GfZ!;NjZe5NOU2|!QF3T&T6y=$RK^v%$ensQF9*VSG zDFd1@>*b2N;LG#iaOq^GO6GsBIG7J1K<}Cu!lxawL8V7MyyjUx5{GVQkR{rzCeXvrdZ?F7x)5JcsJL(6 zX;3@1^}dpkY>2hzT3a$R!JobxG_1k~Nv|Zj{2^z4sCvN*smWO6uTf-@;fO`|S;&X^ zVBD#bFIv@Nhttk@M8C+SJUjFH0~x^Uc^_lIt1E#H{l2hSoG`R1@j+4h9;m8D2InDO zB#K=^?PKqlISLvrWu-6C(8z0!w6njddH@Ck)0%qA|^-aZr;2B&C zC+i*AJTezCNu}XDHtrYip5g?GVzwBGS^McE9Z$E2YFa7Daa3_Lmw932#+ z0~v~zO0BY(d*!vO(6Dk-NP^L8n-wd_wmnvZrw^xL-4Iy7N_^6d?#CL*`261o_E|iN z;HWV{FGJfF}0`ewwgRt4?u0CcM5x%2M{Tb%oX zZzF@^hP`hks0n)}5eQQ)K=S19>WEyLx^0D_e?DCUX*RmEguGP6m^#)KePCKnSAlOx zI&uX2VP|f9a9RY;cCniCO5-`5dth=lL(tcY8JmTo zi6FyEAObPh>vy#Wa2&J=lKe4zknmP^r^TTZ-2L&gSe$+5`7EuZ9*}F=-)8F7&-Z3k z(`)Z?b6uncSBffCPkQDT*;wb=H-yKS@m9@Bi7fP3{=pZroIKTzN3FJ( zMr+;IriBG;H^T@Pds)*7Kw$2j>VyvjkJ)Zbybdl=mST_3M+!i^KPR*srG*9kxYWIA zsc;*2p1F7|W5s!VJb0;bW)WfGFfD|{sV~Ak$PO9NT&{w|ZKI~0XuB{HkpExyat%4n zmfOFd#rFCZ79-dcs$#y(S=l^T29SL9xE}l44(%$0`rS>L+SpTG7(y!|KGt+pHabDs zT?y^^8exLgfJ)#~+J<m$i*Q6b1ZW+R+URE%}koGm-cQq5f6gMukha2Y2`O5ZJ(P|cT+~!l{3>y3N|tYu_@*FmVF}Lwji2WG{=7*-7~SNLrgpBg zu6Egao0KgUjDYyf&xkLW&{U1>G-F3ARxF7CtCQy%p%|tJF&Oh~EK}+sPlFytQ>;?w z;qR5Q)&sj+cX3JGeNqCf$-{S>@)-?P(qM*Y5uAj-9wstMj#5N3iB5Go$+OxU#975 zI6()f)ee+XSWRDSc3yS&tNZ!&x2Y0gBbefc7fZc)sp9%{8)E4M5x9qhL>zRUCjS1v zlO*ypvSkR1hJJ12Lf$Qw3v+>Li#nQ;^=_jXfB zfU~y>tVpka3r7`Riq$BOxC~US%ufPIr^z*SkcJ`Sdu=77+d4^*P5Rav_}R<4W##bn zbhd(S!B2|`@k>`n)^xSKl~(lRdo{Sn{Mb`I>Jj-nVXr_3BJfBXVHhsF>5p5F820MOo=V~B(VgJrCpvQ%cOr%CIx37yaYT2@EkUbe^23j|?-$mReB zOKC|h5IQbZ$_Dru_=nvYxpX1@>#Aax&%umPJ(fom>tfaN3_dyQ8Uh9m-U zRtEB&Ajy7tIs*g~-k0jG`j4EKlpa00X&j}*~Sn$3WeQs>~)wUITyJ%;TDS9|mE_Y4Vc09baGnB51!S6~AIHF*5(pE*WBPgJ5zD$$-* zn{JH0JRc$h`ie*oh2o)${t`OpgTo7do_ZB56q7%KpHKYF{R^tk;!!%oPiisl zMH5?E0+wIjv46&7(gm{6pLnP(x4UoTX~1aw^2=K);(&HV+9P0WL4naT~ zVGIHts29-YsgYD|sgV!F?`Cud=h*zI!09~>CRI{<7F*)7=fR9${X3oRgOqf4D(o%I zMD=^p_!�J~hbgcoT1O#A)Ur)Rr4w-O&a!;=7i#C5nATFMBxr7;F{yymZEce>oR{ zV#u2Dy|XAe*e>~=IMUj44?YMntwp>SFk8^g7)=wKW+==~(#lb7~O2IjluQ@+9P3Gy?FL@$%At?p2_#cQZwD7$D>ZEtf*Tg9V7Bkd=#gQ zwY+KkQ%AN|zhv=WpWcM}Acp$Ay*jPRP2MMvAPVYclYhAz+2O*tZ`I2_wjoJwZHSCct#ar*^NJRpkjSN>*cW6%T%mE4>o$$>hCN3#>S zZWAMEI4EER-|-1`88uD@yrCs3ktah<$ZG{6KjUuhTk*YJ zvy=h=`KSA9c??4Xn3a6?~#K>msdnT|p9?pqr3&Uy;5R3K&HT zXaq?~HSrkK^BwGqByWn|Vc@&YrCTjQc&gSmSRF^r=309MJxE{z79oeW+I{sX-a9no3ZyM(qn;l@)*nAn#5$=HKq#Mw&(SvQopCKa?!(l2pIZ*b#z}Cl4vHDejSIVo_{tA#zPGVLf7C z_Ew|fIP0rxgUY3(vxu{QaP-|+@_xAydh>BjM12=0|L=L2H-h&-ip(gs0KX+mz{MOP z5n^S9D=!qS#W8;z@~xo_VE9+)QQ)Kf^^>$c%*|Cu2O<;THKoy|Dln-QB}?We;i2Gj z4(AoIEv{4Ni4J>&6DJV!IN593k|b(xHq<@kN_pFbs(-256c3j{sgH&wcPNrgHYvXa$45p~`F%NJeAwGD~1Xw;?=u;kv0a z2s8OHwkVH?C7@JCi-JM>8j6!R532m=RGYpO$8W#7Zd#`yU(o<4k9rD=1*mLp(tJP_ z#`#XW_j#$NnU#dAc>6)EL=hCn*(Y4rQy{i;9wA^|ln^{OpIcgPgeQrnPMc-$ren!d z&Z1&hB_C#CiTawCO_15rc?F?SBUrfZb=6~K?|fLsQ#J^;Ec!;@$Ve7}Ia)pISQpyPNJ%%$jPbIP zruQL3`mX9TN9>kVlJfDp$l+0CKKn%Rys9#Lw!j)Yk36#fDE%ZOLZ2zZ$WQOv7?fzu z=|@m27}kNXmZn~aIEu+|An!O@K(2Dy8rZ5V(}+UpNwerG#TK;bNJw1glg3HYz)~zA z3>{9>*vSHS29Hofg`_}aDiF7jv`(#E_iGjDv?r={Pj0kXpIVJ^K@DBbIg!wERVo3D z5@l1Rh^pwt%3yZ+8nBh}1P z{Hmuy8Dlcssy*v@jyW;!-oX1aE0}ouZ_?<;xBf3Xa>(ct5o(ZF2ePaxNz_%}>WId;bp)o|x6PecA3yRSqZjw%* zKO5wb!)F^vfCOXylyUZ)d+D@(ZokXm-Hj0ZvGHknHYqs=CIpH!_F;zxW(Yy9IitCv zk{VrUG-%#rqTSV#badLdIw-qrF$WN!WE$Ds$ProNLxEXjjCV7EDw3T^>GEq)4nPhG-8@GO(~#qJG46=~ zHu8zp;DQQK97cff;0OXr(%57QHb-VAP!cHjF=)nfr;;4YZRGjiWn_9Ld+G?)R&dXL z$DZ*mbfSLFsqJ2>7Rx4ruiFoK@6tqc+&aS*_o@+nh{#=(2C0(_4_@KrL`Jg+g>U`%>L>Af)`HJgX)IIUH~~CdRz+OCN@1nGTwkuQs8Z$F{W^+zvFm9qkPu?;|s|S zyM2%};Op@v#;*pwlVkRau^bRLDYfs&fVsd~al#gG!23wLy|0+l8F~c+d-f6*Y{v(= zuzu97Bu-yr5>el;c|=IUF`EyarXYz4sNF5}Z_{HSu@`TlaCSbfy?S{5$2-0pI#76m z6CH6z(czWIoR17j(#cDo3LlO;-|`=zbA6qslRKc!xsy2*ScyYI_xZ6FZv1d86Kcl_ z-2e)J0`|2%HX7@Lpa&Y{gU6f`%(nbDlL>BR({Yr%hVlFm1ocR{IQnWPjfkMI`NZTY z;E|<6BG_@D7N!&a#G2l?Le5zax|b-6PAf$^t?SVo8s2G8 zOLc%{1YzpLol&A~g3#EAso2_pk)D7g+eG+@$kl4*G8PDNlf|y_DXCr`nMmfV+VkWs zqLW{G!G;MTep#9aNKrKDT{XE3VQSn>T!M_aBzRT*C}WRu%0aXF64r|bD+U>gC9?!? zN9Yugz4E(N55jsFfrZTV;jHv}) z-YjNAo7!lXuARk9;)Ene45(r#C>u|XMkOZW`*ia!MtD03*Fhl)fjln7Tz8biPY5{?>-m1)}w@8TGG-f;@szmHRY- z7WZJ^7P*>gn4S3iZln(x87g=iFqaqDk2P3fOv7Y$%dh|z7QIk?ub8Cgqk|C-neIuX zJ&^*jOg3m6!qr(F3o1h>(DqI_#jwp2ah+en0h?&zxIZXaH!=qL5y5KGfv86O9^2f4 z52-E);!P%NWO&jE@jbvYjFXiA$pgpatpuV##29U=V@oi(Rfs((D(hL#>*MTG;KsB& zFeS}_k*i|^3uO^ZN_ zT6bl8a48@O>*aaYPpo}(mlPAz{d^eZB-FB^zE&smmdh3ypnv^qkH>rbs$h?F0CvPl zHVIzvu`;K6O;19?%qQ_j@WVqh^=&BU#k=ERd>>`om@8S?GT!oCW`pyF*=8|)vOn73 zV~$6DJ`Zq=91KDIG}Hq+;dIBA7cG?I`|+!e3>dtNKRVC45XDvhvW;x|>`k=aHyYOY%MlmS?@`pXGp3`E4E4dtBmFx0Zs=8J$?I$caNq1lYv z!jznXY39&m9?UnZ?)@d4Cv|n6w%#~inJOWs*Br5eBVI=&e#@|r`G;^!87C2CLMH51 zw8JB}S=P&4VqNp)TmGid_|$_)@WhYW*UGaopYRL|T%+(dNDm=J>nf8pq3fc38CM?H zrxqpmJ68y9Gc={;(8?AvdmyGLLzdJfnLz^IT-C&W#Elmdbc%5Z0~bBu4C%8+^AtWI zlbS4@1_49nlD(T0%Koiw_s)u5NmO3QHQrCxBvl1Z5Z(ykqJ zI>oufisy?f>EXd8j)jgGh|N4*mf6vN())jmb{h(8FhcYRwH=s(4gi=vEjk^ns*uEQ5zg4KRXF0MrA8L49+f9J_ zvMBOAF#0sN@FtwFwNe$az%L)%Oqa>m(%x;_Ed)5_`Go3h6T-yedX5-)koBK^jO&<= z87whb^#&7`F`jC(wiNlPjv$Zf&9)EoJ$iDk+7g?Bl^EoU=cVX!EVCNPU+Pc7UBxVkO<3MOi2>+sKm`orJ5q8=c> ztI)Db^2Fy`7Z6)G!SGD&x2>Z@P99(`IiZ^kXZm0&Mq7QR+6`gEuV!1hAW=Pq=rB6m zj#-Dq&#)AWkyOoi38^LrfFJf}O%MEs#iPbb2Ax5y2iqYI1*E`wP4|$~ZcJ9S_<33F zFKRRv0SNSv5S}-h=UmuHe2&@f>tuBx001%JS&a$r4NG#1V)By(g^~7_gsD{R4w<&2 z5YSUvJ*!S4UeG%g)oQYyn(mLU%K$KEE5oQka2q9qN|Ca-)#;QlAx!Ax)QCt)d zP)lTFHsfH0#taNpLw7E2i1n6~xx};kr@s>#W>M~)hIHS*WqciG$O)5Ms_D!q4E^es zIa&U)0URfBx}T;}Q}u?0+YP&1LP*%kX}KRrJ$&bO=GO|W)zR<*YEj-3o!ASuMBnU^ zydEdSqtigut`3SdT-YBOZb{Y+BM?q#YliIYSHW0QT~i3{3$ne2{kL%3DP(&0l^FLIE{Xt(Kf)6?qkVl^@p zTb21oIh$K@E8#>-@`;x#ZutJob;m3^;q%r{r1x@k&gI)luH9Jx@r|S zE~}}kg1t5dx#iaU_0j~mfYP8xryaVNGkYX`ipp*WSQD-e%V(^f*InP7b{>72R$8T_ zV2*3AE!%=JxO*lHXV6yk?zkO$Tm(B@vZ;!V&a`GA7~8|C?_L$$|M(1d`O zgaJ@6#~9Kw8fNH)I1oFcm;Ak_I@Ukpu(9_pMSZzd42MIM$3nWM^)}v-qyDUuBVY-%lM7=--be-{wa8eLS%MPLLv*{CAQqWU_xg; z|Ifqb-k~~`% zNai%=vV#a_w&o?h6Ed<~CkfP||1=M(!Y8+<%#)GPYj2xt(@=u5ZVVMg8g_McihVIj zx!4%kktSCYod2wm}$F#a9_<2SlXWn<7WU8KNmOv+uqV^aTzh+!_qD_Er>()t55^LS99hmOotfZ^+4?%@wtvwG*prP&R5|NR#eCc^+UIhu|fOG9Cs_?F8@aXmF*%Vl*6x6_Id(O|dtfk7clB zFtJEXK%b8StnR=V z9Xg6twwkLpFTTp`E5nwm5gve8VO7iV6@HAO$1H-_`|27UJR)beXvWW}c|p4Z7HF@7 zJYVyLjshZ*S+uPz^KXfhGBmf9AOOb#vYd*i@Hiawzs^O33FEdoW%di0gN;dsED zJIk-{rYz}Ji3Ru9jsgVlECC*;p!KiTu|ReafHc^rTTHSrPBR+}tV<bSt8Cl<$T~=I?T$E(N~Z znCDaUgo7o!eib6h)D{Jp#6Vzv-k`nXG*H?|aeQ1Ha#baFCQn24SJAX#0HRw>?j9~@ z4O0Q$f}(uiiE#ib#7E~(MnsTI$jO$KsYmj%wlj(CK_3R;S51K{xQ)5=6a))>XL!}~ zAIuu+$qL^C3~eF6(>r^_lio@C_ej2fsd5BKV=4e_nC%3zP{2nfN0a8xe~J&7Xk1gq zW?K^`uE2{xvj_=P?m;XW5=l6Bfi)5|z)c;*w88D1{7`20(*ocvHu5@>quKo`;LWGB z{)_2mJT>>p8gNp#NJ;_5#;p{IV^rBMrR~T$*1pjlb7Q;mDwuCqt#;Gh#|YKtGQXEe zZ{Q2-b(n}~SuD8g{f7FL_pq?@$X`P5a?lV5R5=R-w~5d2ib`}%X(nxK#D8~l@m?#2pjfT4dpdIy`t-%fR^Wj4~KazG-XMPyNmAOEqV*uU~(-R~JaZA*4DO${^PF}DNyCs32TtiQ0`CKP z`=z*b3Fd<{QUVCsB9y~3nw(|bC-{~L02L(zSJ$CLP~6Dx=8uI6?L@`gKs8ww4HfkR z?erH46goJDL@b2n@dfX#Pu0>{Ua)?|g0^lR-hct%={`1npa31XwPdKfKRSQrxT=k- Op<#H1RJ1Sv00030oQCQE literal 0 HcmV?d00001 From 890c62464f41cfe6c8cb05594e3db0e626f348f6 Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 30 Jun 2025 03:07:56 +0000 Subject: [PATCH 093/155] Add chat box styling options to configuration and templates - Introduced new configuration options for chat box background and border colors in config.ini.sample and generate_css.py. - Updated meshinfo.css.template to utilize the new chat box colors, allowing for customizable chat box appearance. --- config.ini.sample | 6 +++++- generate_css.py | 2 ++ www/css/meshinfo.css.template | 5 ++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/config.ini.sample b/config.ini.sample index 508956cd..15d8321e 100644 --- a/config.ini.sample +++ b/config.ini.sample @@ -96,4 +96,8 @@ link_color_hover=#0056b3 # Control color (buttons, form controls) control_color=#17a2b8 # Control hover color -control_color_hover=#1396a5 \ No newline at end of file +control_color_hover=#1396a5 +# Chat box background color +chat_box_background_color=#f8f9fa +# Chat box border color (leave empty for no border) +chat_box_border_color= \ No newline at end of file diff --git a/generate_css.py b/generate_css.py index d16dc3e1..9485a174 100644 --- a/generate_css.py +++ b/generate_css.py @@ -21,6 +21,8 @@ 'header_link_color': '#555', 'header_link_active_color': '#000', 'header_brand_color': '#000', + 'chat_box_background_color': '#f8f9fa', + 'chat_box_border_color': '', } config = configparser.ConfigParser() diff --git a/www/css/meshinfo.css.template b/www/css/meshinfo.css.template index 177a94d5..2886b1d2 100644 --- a/www/css/meshinfo.css.template +++ b/www/css/meshinfo.css.template @@ -144,10 +144,13 @@ nav { } .message-bubble { - background: {{table_alternating_row_color}}; + background: {{chat_box_background_color}}; border-radius: 12px; padding: 12px 16px; position: relative; + {% if chat_box_border_color %} + border: 1px solid {{chat_box_border_color}}; + {% endif %} } .message-header { From 9a96ac5025464f925ebca0d34afbb05b3a2d1476 Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 30 Jun 2025 03:30:00 +0000 Subject: [PATCH 094/155] Update configuration and templates to enhance UI with new background color options - Changed chat box background color in config.ini.sample for improved aesthetics. - Added a new banner background color option in the configuration. - Updated templates to apply a new class for the newest node welcome message, ensuring consistent styling. - Modified CSS to include styles for the new class and banner background. --- config.ini.sample | 6 ++++-- templates/allnodes.html.j2 | 2 +- templates/nodes.html.j2 | 2 +- www/css/meshinfo.css.template | 14 +++++++++----- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/config.ini.sample b/config.ini.sample index 15d8321e..b9a74cb7 100644 --- a/config.ini.sample +++ b/config.ini.sample @@ -98,6 +98,8 @@ control_color=#17a2b8 # Control hover color control_color_hover=#1396a5 # Chat box background color -chat_box_background_color=#f8f9fa +chat_box_background_color=#f0f0f0 # Chat box border color (leave empty for no border) -chat_box_border_color= \ No newline at end of file +chat_box_border_color= +# Banner background color (e.g. for newest node welcome in nodes page) +banner_background_color=#F9F9D7 \ No newline at end of file diff --git a/templates/allnodes.html.j2 b/templates/allnodes.html.j2 index a3d401fe..20ccb86c 100644 --- a/templates/allnodes.html.j2 +++ b/templates/allnodes.html.j2 @@ -8,7 +8,7 @@ {% set lnodeid = utils.convert_node_id_from_int_to_hex(latest.id) %} {% if lnodeid and lnodeid in nodes %} {% set lnode = nodes[lnodeid] %} -
    +
    📌 Welcome to our newest node, {{ lnode.long_name }} ({{ lnode.short_name }}).👋
    {% endif %} diff --git a/templates/nodes.html.j2 b/templates/nodes.html.j2 index 5a96af48..459e1901 100644 --- a/templates/nodes.html.j2 +++ b/templates/nodes.html.j2 @@ -8,7 +8,7 @@ {% set lnodeid = utils.convert_node_id_from_int_to_hex(latest.id) %} {% if lnodeid and lnodeid in nodes %} {% set lnode = nodes[lnodeid] %} -
    +
    📌 Welcome to our newest node, {{ lnode.long_name }} ({{ lnode.short_name }}).👋
    {% endif %} diff --git a/www/css/meshinfo.css.template b/www/css/meshinfo.css.template index 2886b1d2..3b36b66f 100644 --- a/www/css/meshinfo.css.template +++ b/www/css/meshinfo.css.template @@ -148,7 +148,7 @@ nav { border-radius: 12px; padding: 12px 16px; position: relative; - {% if chat_box_border_color %} + {% if chat_box_border_color and chat_box_border_color.strip() and chat_box_border_color != 'none' and chat_box_border_color != 'transparent' and chat_box_border_color != '' %} border: 1px solid {{chat_box_border_color}}; {% endif %} } @@ -650,9 +650,9 @@ a:hover, a:focus { border-color: {{control_color_hover}} !important; } -/* Prevent global link color from affecting chat node links */ -.chat-messages a, -.message-container a { +/* Prevent global link color from affecting only reception badge links in chat */ +.reception-badge, +.mobile-node-name a { color: #000 !important; } @@ -687,4 +687,8 @@ a:hover, a:focus { border-color: #dee2e6 !important; } -/* --- END MIGRATED STYLES --- */ \ No newline at end of file +/* --- END MIGRATED STYLES --- */ + +.newest-node-welcome, .banner { + background: {{banner_background_color}} !important; +} \ No newline at end of file From c63a76527004e7f2672859c547d8df41488d40da Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 30 Jun 2025 03:38:24 +0000 Subject: [PATCH 095/155] Enhance GitHub Actions workflow for multi-architecture Docker image builds - Updated the deploy.yaml to include separate jobs for building and pushing AMD64 and ARM64 Docker images. - Added caching for Docker layers to optimize build times for ARM64 images. - Modified the create-manifest job to depend on the new build jobs, ensuring proper execution order. - Improved version calculation logic for tagging images with the latest version and commit hash. --- .github/workflows/deploy.yaml | 110 ++++++++++++++++++++++++++++------ 1 file changed, 91 insertions(+), 19 deletions(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index f4871ca9..aa6c7a99 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -1,58 +1,130 @@ -name: Create Multi-Arch Manifest +name: Build, Push, and Publish Multi-Arch Manifest + on: - workflow_dispatch: # Only triggered manually + workflow_dispatch: + jobs: + build-and-push-amd64: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Calculate version + id: version + run: | + LATEST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0") + echo "Latest tag: $LATEST_TAG" + COMMIT_HASH=$(git rev-parse --short HEAD) + echo "Commit hash: $COMMIT_HASH" + VERSION="${LATEST_TAG}-${COMMIT_HASH}" + echo "version=${VERSION}" >> $GITHUB_OUTPUT + echo "latest_tag=${LATEST_TAG}" >> $GITHUB_OUTPUT + echo "commit_hash=${COMMIT_HASH}" >> $GITHUB_OUTPUT + - name: Log in to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push AMD64 Docker image + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/amd64 + push: true + provenance: false + tags: | + ghcr.io/agessaman/meshinfo-lite:latest-amd64 + ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.version }}-amd64 + ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.latest_tag }}-amd64 + + build-and-push-arm64: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Cache Docker layers + uses: actions/cache@v4 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-arm64-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-arm64- + - name: Calculate version + id: version + run: | + LATEST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0") + echo "Latest tag: $LATEST_TAG" + COMMIT_HASH=$(git rev-parse --short HEAD) + echo "Commit hash: $COMMIT_HASH" + VERSION="${LATEST_TAG}-${COMMIT_HASH}" + echo "version=${VERSION}" >> $GITHUB_OUTPUT + echo "latest_tag=${LATEST_TAG}" >> $GITHUB_OUTPUT + echo "commit_hash=${COMMIT_HASH}" >> $GITHUB_OUTPUT + - name: Log in to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push ARM64 Docker image + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/arm64 + push: true + provenance: false + tags: | + ghcr.io/agessaman/meshinfo-lite:latest-arm64 + ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.version }}-arm64 + ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.latest_tag }}-arm64 + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache,mode=max + create-manifest: runs-on: ubuntu-latest - needs: [] # No dependencies - can run independently + needs: [build-and-push-amd64, build-and-push-arm64] steps: - name: Checkout repository uses: actions/checkout@v3 with: - fetch-depth: 0 # Fetch all history for proper versioning - + fetch-depth: 0 - name: Calculate version id: version run: | - # Get the latest git tag LATEST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0") echo "Latest tag: $LATEST_TAG" - - # Get current commit hash COMMIT_HASH=$(git rev-parse --short HEAD) echo "Commit hash: $COMMIT_HASH" - - # Create version string VERSION="${LATEST_TAG}-${COMMIT_HASH}" echo "version=${VERSION}" >> $GITHUB_OUTPUT echo "latest_tag=${LATEST_TAG}" >> $GITHUB_OUTPUT echo "commit_hash=${COMMIT_HASH}" >> $GITHUB_OUTPUT - - name: Log in to GitHub Container Registry uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Create and push multi-arch manifest run: | - # Create manifest for latest docker manifest create ghcr.io/agessaman/meshinfo-lite:latest \ ghcr.io/agessaman/meshinfo-lite:latest-amd64 \ ghcr.io/agessaman/meshinfo-lite:latest-arm64 - - # Create manifest for version docker manifest create ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.version }} \ ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.version }}-amd64 \ ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.version }}-arm64 - - # Create manifest for tag docker manifest create ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.latest_tag }} \ ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.latest_tag }}-amd64 \ ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.latest_tag }}-arm64 - - # Push manifests docker manifest push ghcr.io/agessaman/meshinfo-lite:latest docker manifest push ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.version }} docker manifest push ghcr.io/agessaman/meshinfo-lite:${{ steps.version.outputs.latest_tag }} From b36bd9fcd621ae0f398378494890c1fe2f446df6 Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 30 Jun 2025 04:11:34 +0000 Subject: [PATCH 096/155] Add channel color and update templates for dynamic channel options - Added a new color mapping for MediumSlow (24) in the channel color utility. - Updated the logs.html.j2 template to dynamically generate channel options from the Channel list. - Modified the map.html.j2 template to utilize the new channel color and display the channel name dynamically in the node information. --- templates/logs.html.j2 | 9 +++------ templates/map.html.j2 | 8 +++----- utils.py | 1 + 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/templates/logs.html.j2 b/templates/logs.html.j2 index 96bb65ce..35791117 100644 --- a/templates/logs.html.j2 +++ b/templates/logs.html.j2 @@ -52,12 +52,9 @@
    diff --git a/templates/map.html.j2 b/templates/map.html.j2 index 5128dbc8..1a1d6166 100644 --- a/templates/map.html.j2 +++ b/templates/map.html.j2 @@ -170,6 +170,7 @@ // Add channel color utility function const channelColors = { 8: '#4CAF50', // Green for LongFast + 24: '#9C27B0', // Purple for MediumSlow 31: '#2196F3', // Blue for MediumFast 112: '#FF9800', // Orange for ShortFast }; @@ -315,6 +316,7 @@ position: [{% if node.position and node.position.longitude_i and node.position.latitude_i %}{{ node.position.longitude_i / 10000000 }}, {{ node.position.latitude_i / 10000000 }}{% else %}null, null{% endif %}], online: {% if node.active %}true{% else %}false{% endif %}, channel: {% if node.channel is not none %}{{ node.channel }}{% else %}null{% endif %}, + channel_name: {% if node.channel is not none %}'{{ utils.get_channel_name(node.channel) }}'{% else %}'LongFast (8) [default]'{% endif %}, zero_hop_data: {{ zero_hop_data.get(id, {'heard': [], 'heard_by': []}) | tojson | safe }} }; {% if node.neighbors %} @@ -803,11 +805,7 @@ + '
    Position: ' + node.position + '
    ' + '
    Location: ' + display_name + '
    ' + '
    Channel: ' - + (node.channel === null ? 'LongFast (8) [default]' : - node.channel === 8 ? 'LongFast (8)' : - node.channel === 31 ? 'MediumFast (31)' : - node.channel === 112 ? 'ShortFast (112)' : - 'Channel ' + node.channel) + + node.channel_name + '
    ' + '
    '; diff --git a/utils.py b/utils.py index 833a82bc..2ba39dc8 100644 --- a/utils.py +++ b/utils.py @@ -312,6 +312,7 @@ def get_channel_color(channel_value): # Define a set of visually pleasing colors for known channels channel_colors = { 8: "#4CAF50", # Green for LongFast + 24: "#9C27B0", # Purple for MediumSlow 31: "#2196F3", # Blue for MediumFast 112: "#FF9800", # Orange for ShortFast # Add more channels as they are discovered From 96034587fabe142b80e4cf2fa0ae8df518aa9231 Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 30 Jun 2025 04:36:52 +0000 Subject: [PATCH 097/155] Add themed Bootstrap popover styles and update badge colors in CSS template - Introduced new styles for Bootstrap popovers, including background, border, and text colors, to enhance UI consistency. - Updated badge styles for improved contrast and readability, ensuring better visibility in the interface. --- www/css/meshinfo.css.template | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/www/css/meshinfo.css.template b/www/css/meshinfo.css.template index 3b36b66f..f72fd089 100644 --- a/www/css/meshinfo.css.template +++ b/www/css/meshinfo.css.template @@ -691,4 +691,43 @@ a:hover, a:focus { .newest-node-welcome, .banner { background: {{banner_background_color}} !important; +} + +/* Themed Bootstrap Popover Styles */ +.popover { + background-color: {{page_background_color}} !important; + border: 1px solid {{table_border_color}} !important; + color: {{header_link_color}} !important; + box-shadow: 0 2px 8px rgba(0,0,0,0.10); + border-radius: 8px; + font-family: inherit; + font-size: 0.98em; + z-index: 1060; +} +.popover-header { + background-color: {{table_header_color}} !important; + color: {{header_link_color}} !important; + border-bottom: 1px solid {{table_border_color}} !important; + font-weight: 600; + border-top-left-radius: 8px; + border-top-right-radius: 8px; +} +.popover-body { + background-color: {{page_background_color}} !important; + color: {{header_link_color}} !important; + border-bottom-left-radius: 8px; + border-bottom-right-radius: 8px; +} +.receiver-popover { + font-size: 0.98em; + line-height: 1.5; + color: {{header_link_color}} !important; + font-family: inherit; +} + +.badge.bg-secondary { + background-color: #e0e0e0 !important; /* Light gray for high contrast */ + color: #222 !important; /* Dark text for readability */ + border: 1px solid {{table_border_color}} !important; + font-weight: 600; } \ No newline at end of file From 941717c592a5b2027f781d655efd94ce0ef39c6f Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 30 Jun 2025 06:37:10 +0000 Subject: [PATCH 098/155] Implement environmental telemetry feature and optimize telemetry cleanup logic - Added a new API endpoint to retrieve environmental telemetry data for nodes, allowing users to access temperature, humidity, pressure, and other metrics over specified time ranges. - Enhanced the telemetry cleanup logic to reduce database load by implementing a counter-based cleanup strategy, running retention policies every 50 inserts or based on configured retention days. - Updated the node detail template to include a new section for displaying environmental telemetry with dynamic chart rendering based on user-selected time ranges. - Introduced CSS styles for the environmental telemetry dropdown and improved layout for better user experience. --- meshdata.py | 72 +- meshinfo_web.py | 9 + templates/node.html.j2 | 1194 ++++++++++++++++++++------------- www/css/meshinfo.css.template | 40 ++ 4 files changed, 830 insertions(+), 485 deletions(-) diff --git a/meshdata.py b/meshdata.py index eee300b9..1867a7f9 100644 --- a/meshdata.py +++ b/meshdata.py @@ -1281,25 +1281,31 @@ def store_telemetry(self, data): # Use class-level config that's already loaded retention_days = self.config.getint("server", "telemetry_retention_days", fallback=None) if self.config else None - # Only check for cleanup every N records (using modulo on auto-increment ID or timestamp) - cur = self.db.cursor() - cur.execute("SELECT COUNT(*) FROM telemetry WHERE ts_created > DATE_SUB(NOW(), INTERVAL 1 HOUR)") - recent_count = cur.fetchone()[0] + # Use a simple counter to reduce cleanup frequency while still honoring retention + if not hasattr(self, '_telemetry_insert_count'): + self._telemetry_insert_count = 0 + self._telemetry_insert_count += 1 + + # Check for cleanup - run retention policy every 50 inserts or if retention_days is set + should_cleanup = (retention_days is not None) or (self._telemetry_insert_count % 50 == 0) - # Only run cleanup if we have accumulated enough recent records - if recent_count > 200: # Adjust threshold as needed + if should_cleanup: + cur = self.db.cursor() if retention_days: + # Always honor retention_days policy cur.execute("""DELETE FROM telemetry WHERE ts_created < DATE_SUB(NOW(), INTERVAL %s DAY)""", (retention_days,)) else: - # More efficient count-based cleanup - cur.execute("""DELETE FROM telemetry WHERE ts_created <= - (SELECT ts_created FROM - (SELECT ts_created FROM telemetry ORDER BY ts_created DESC LIMIT 1 OFFSET 20000) t)""") - - cur.close() - self.db.commit() + # Fallback to count-based cleanup only if no retention_days set + cur.execute("SELECT COUNT(*) FROM telemetry WHERE ts_created > DATE_SUB(NOW(), INTERVAL 1 HOUR)") + recent_count = cur.fetchone()[0] + if recent_count > 200: # Only run count-based cleanup if we have enough recent activity + cur.execute("""DELETE FROM telemetry WHERE ts_created <= + (SELECT ts_created FROM + (SELECT ts_created FROM telemetry ORDER BY ts_created DESC LIMIT 1 OFFSET 20000) t)""") + cur.close() + self.db.commit() node_id = self.verify_node(data["from"]) payload = dict(data["decoded"]["json_payload"]) @@ -2356,6 +2362,46 @@ def get_telemetry_for_node(self, node_id): cur.close() return telemetry + def get_environmental_telemetry_for_node(self, node_id, days=1): + """ + Return environmental telemetry for the given node, ordered by timestamp ascending. + Each record is a dict with keys: temperature, barometric_pressure, relative_humidity, gas_resistance, voltage, current, ts_created. + + Args: + node_id: The node ID to get telemetry for + days: Number of days to look back (default 1 for 24 hours) + """ + telemetry = [] + sql = """ + SELECT + temperature, + barometric_pressure, + relative_humidity, + gas_resistance, + voltage, + current, + UNIX_TIMESTAMP(ts_created) as ts_created + FROM telemetry + WHERE id = %s AND ts_created >= NOW() - INTERVAL %s DAY + ORDER BY ts_created ASC + """ + params = (node_id, days) + cur = self.db.cursor() + cur.execute(sql, params) + rows = cur.fetchall() + column_names = [desc[0] for desc in cur.description] + for row in rows: + record = {} + for i in range(len(row)): + value = row[i] + if isinstance(value, datetime.datetime): + record[column_names[i]] = int(value.timestamp()) + else: + record[column_names[i]] = value + telemetry.append(record) + cur.close() + return telemetry + def get_positions_at_time(self, node_ids, timestamp): """Get the closest position for each node using a single reused cursor.""" if not node_ids: diff --git a/meshinfo_web.py b/meshinfo_web.py index d94c4ce9..cc133069 100644 --- a/meshinfo_web.py +++ b/meshinfo_web.py @@ -1855,6 +1855,15 @@ def api_telemetry(node_id): telemetry = md.get_telemetry_for_node(node_id) return jsonify(telemetry) +@app.route('/api/environmental-telemetry/') +def api_environmental_telemetry(node_id): + md = get_meshdata() + days = request.args.get('days', 1, type=int) + # Limit days to reasonable range (1-30 days) + days = max(1, min(30, days)) + telemetry = md.get_environmental_telemetry_for_node(node_id, days) + return jsonify(telemetry) + @cache.memoize(timeout=60) def get_cached_chat_data(page=1, per_page=50): """Cache the chat data with optimized query.""" diff --git a/templates/node.html.j2 b/templates/node.html.j2 index 6c19401a..3fc38290 100644 --- a/templates/node.html.j2 +++ b/templates/node.html.j2 @@ -17,7 +17,7 @@ Avatar - +
    {{ node.short_name }} - {{ node.long_name }}
    @@ -67,491 +67,453 @@
    -
    - {% if node.position and node.position.latitude_i and node.position.longitude_i %} -
    - {% endif %} - {% if node.telemetry %} -
    - -
    - - {% endif %} -
    Elsewhere
    -
    - - PugetMesh Map
    - MeshMap
    - Logs + + {% endif %} +
    +
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - {% if node.position %} - - - - - - - - - - - - - - - - - {% endif %} - - - - - - - - - - - - - - - - - - - - - - - - - -
    Details
    ID (hex) - {{ utils.convert_node_id_from_int_to_hex(node.id) }} -
    ID (int) - {{ node.id }} -
    Hardware - {% if node.hw_model is not none and meshtastic_support.HardwareModel(node.hw_model) in meshtastic_support.HARDWARE_PHOTOS %} - {{ meshtastic_support.get_hardware_model_name(node.hw_model) }} - {% endif %} - {{ meshtastic_support.get_hardware_model_name(node.hw_model) }} -
    Firmware version - {% if node.firmware_version %} - {{ node.firmware_version }} - {% else %} - Unknown - {% endif %} - - - -
    Role - {% if node.role is not none %} - {% if node.role == 0 %} - Client - {% elif node.role == 1 %} - Client Mute - {% elif node.role == 2 %} - Router - {% elif node.role == 3 %} - Router Client - {% elif node.role == 4 %} - Repeater - {% elif node.role == 5 %} - Tracker - {% elif node.role == 6 %} - Sensor - {% elif node.role == 7 %} - ATAK - {% elif node.role == 8 %} - Client Hidden - {% elif node.role == 9 %} - Lost and Found - {% elif node.role == 10 %} - ATAK Tracker - {% elif node.role == 11 %} - Router Late - {% else %} - Unknown ({{ node.role }}) - {% endif %} - {% else %} - Unknown - {% endif %} -
    Position - {% if node.position and node.position.latitude_i and node.position.longitude_i %} - {{ node.position.longitude_i / 1e7 }}, {{ node.position.latitude_i / 1e7 }} - {% else %} - Unknown - {% endif %} -
    Grid Square - {% if node.position and node.position.latitude_i and node.position.longitude_i %} - {% set grid = utils.latlon_to_grid(node.position.latitude_i / 1e7, node.position.longitude_i / 1e7) %} - {{ grid }} - {% else %} - Unknown - {% endif %} -
    Location - {% if node.position and node.position.geocoded %} - {{ node.position.geocoded }} - {% else %} - Unknown - {% endif %} -
    Altitude - {% if node.position and node.position.altitude %} - {{ node.position.altitude }} m - {% else %} - Unknown - {% endif %} -
    Status - {% if node.active %} - Online - {% else %} - Offline - {% endif %} -
    First Seen - {% if node.ts_created %} - {{ format_timestamp(node.ts_created) }} - {% else %} - Unknown - {% endif %} -
    Last Seen - {% if node.ts_seen %} - {{ format_timestamp(node.ts_seen) }} ({{ time_ago(node.ts_seen) }}) - {% else %} - Unknown - {% endif %} -
    Owner - {% if node.owner_username %} - {{ node.owner_username }} - {% else %} - Unknown - {% endif %} -
    Channel - {% if node.channel %} - {{ utils.get_channel_name(node.channel, use_short_names=False) }} - {% else %} - Unknown - {% endif %} -
    Updated via - {% if node.updated_via %} - {% set vid = utils.convert_node_id_from_int_to_hex(node.updated_via) %} - {% if node.updated_via == node.id %} - Self - {% elif vid in linked_nodes_details %} - {{ linked_nodes_details[vid]["long_name"] }} - {% else %} - {{ vid }} - {% endif %} - {% else %} - Unknown - {% endif %} -
    - - - - - - - - - - - - - - {% set processed_nodes = [] %} - - - {% for neighbor in node.neighbors %} - {% set nid = utils.convert_node_id_from_int_to_hex(neighbor.neighbor_id) %} - {% set nnode = linked_nodes_details[nid] if nid in linked_nodes_details else None %} - {% if nid not in processed_nodes %} - {% set processed_nodes = processed_nodes + [nid] %} - - - - - - + +
    +
    +
    Heard (zero hop)
    NodeSignalDistanceSource
    - {% if nnode %} - {{ nnode.short_name }} - {% else %} - UNK - {% endif %} - - SNR: {{ neighbor.snr }} - - {% if nid in linked_nodes_details %} - {% if linked_nodes_details[nid].position and node.position %} - {% if linked_nodes_details[nid].position.latitude_i is not none and linked_nodes_details[nid].position.longitude_i is not none and node.position.latitude_i is not none and node.position.longitude_i is not none %} - {% set dist = utils.calculate_distance_between_nodes(linked_nodes_details[nid], node) %} - {% if dist %} - - {{ dist|round(2) }} km 🏔️ - - {% endif %} - {% endif %} - {% endif %} - {% endif %} - N
    + + + + + + + + + + + + + + + + + + + + + + + + + + + {% if node.position %} + + + + + + + + + + + + + + + + {% endif %} - {% endfor %} - - - {% if zero_hop_heard %} - {% for heard in zero_hop_heard %} - {% set nid = utils.convert_node_id_from_int_to_hex(heard.from_id) %} + + + + + + + + + + + + + + + + + + + + + + + + + +
    Details
    ID (hex) + {{ utils.convert_node_id_from_int_to_hex(node.id) }} +
    ID (int) + {{ node.id }} +
    Hardware + {% if node.hw_model is not none and meshtastic_support.HardwareModel(node.hw_model) in meshtastic_support.HARDWARE_PHOTOS %} + {{ meshtastic_support.get_hardware_model_name(node.hw_model) }} + {% endif %} + {{ meshtastic_support.get_hardware_model_name(node.hw_model) }} +
    Firmware version + {% if node.firmware_version %} + {{ node.firmware_version }} + {% else %} + Unknown + {% endif %} + + + +
    Role + {% if node.role is not none %} + {% if node.role == 0 %} + Client + {% elif node.role == 1 %} + Client Mute + {% elif node.role == 2 %} + Router + {% elif node.role == 3 %} + Router Client + {% elif node.role == 4 %} + Repeater + {% elif node.role == 5 %} + Tracker + {% elif node.role == 6 %} + Sensor + {% elif node.role == 7 %} + ATAK + {% elif node.role == 8 %} + Client Hidden + {% elif node.role == 9 %} + Lost and Found + {% elif node.role == 10 %} + ATAK Tracker + {% elif node.role == 11 %} + Router Late + {% else %} + Unknown ({{ node.role }}) + {% endif %} + {% else %} + Unknown + {% endif %} +
    Position + {% if node.position and node.position.latitude_i and node.position.longitude_i %} + {{ node.position.longitude_i / 1e7 }}, {{ node.position.latitude_i / 1e7 }} + {% else %} + Unknown + {% endif %} +
    Grid Square + {% if node.position and node.position.latitude_i and node.position.longitude_i %} + {% set grid = utils.latlon_to_grid(node.position.latitude_i / 1e7, node.position.longitude_i / 1e7) %} + {{ grid }} + {% else %} + Unknown + {% endif %} +
    Location + {% if node.position and node.position.geocoded %} + {{ node.position.geocoded }} + {% else %} + Unknown + {% endif %} +
    Altitude + {% if node.position and node.position.altitude %} + {{ node.position.altitude }} m + {% else %} + Unknown + {% endif %} +
    Status + {% if node.active %} + Online + {% else %} + Offline + {% endif %} +
    First Seen + {% if node.ts_created %} + {{ format_timestamp(node.ts_created) }} + {% else %} + Unknown + {% endif %} +
    Last Seen + {% if node.ts_seen %} + {{ format_timestamp(node.ts_seen) }} ({{ time_ago(node.ts_seen) }}) + {% else %} + Unknown + {% endif %} +
    Owner + {% if node.owner_username %} + {{ node.owner_username }} + {% else %} + Unknown + {% endif %} +
    Channel + {% if node.channel %} + {{ utils.get_channel_name(node.channel, use_short_names=False) }} + {% else %} + Unknown + {% endif %} +
    Updated via + {% if node.updated_via %} + {% set vid = utils.convert_node_id_from_int_to_hex(node.updated_via) %} + {% if node.updated_via == node.id %} + Self + {% elif vid in linked_nodes_details %} + {{ linked_nodes_details[vid]["long_name"] }} + {% else %} + {{ vid }} + {% endif %} + {% else %} + Unknown + {% endif %} +
    + + + + + + + + + + + + + + {% set processed_nodes = [] %} + + + {% for neighbor in node.neighbors %} + {% set nid = utils.convert_node_id_from_int_to_hex(neighbor.neighbor_id) %} + {% set nnode = linked_nodes_details[nid] if nid in linked_nodes_details else None %} {% if nid not in processed_nodes %} {% set processed_nodes = processed_nodes + [nid] %} - + {% endif %} {% endfor %} - {% endif %} - -
    Heard (zero hop)
    NodeSignalDistanceSource
    - {% if nid in linked_nodes_details %} - {{ linked_nodes_details[nid].short_name }} + {% if nnode %} + {{ nnode.short_name }} {% else %} UNK {% endif %} - SNR: {{ heard.best_snr }} max / {{ heard.avg_snr|round(1) }} avg + SNR: {{ neighbor.snr }} {% if nid in linked_nodes_details %} - {% set dist = utils.calculate_distance_between_nodes(linked_nodes_details[nid], node) %} - {% if dist %} - {% if linked_nodes_details[nid].position and linked_nodes_details[nid].position.latitude_i and linked_nodes_details[nid].position.longitude_i and node.position and node.position.latitude_i and node.position.longitude_i %} - - {{ dist|round(2) }} km 🏔️ - - {% else %} - {{ dist|round(2) }} km + {% if linked_nodes_details[nid].position and node.position %} + {% if linked_nodes_details[nid].position.latitude_i is not none and linked_nodes_details[nid].position.longitude_i is not none and node.position.latitude_i is not none and node.position.longitude_i is not none %} + {% set dist = utils.calculate_distance_between_nodes(linked_nodes_details[nid], node) %} + {% if dist %} + + {{ dist|round(2) }} km 🏔️ + + {% endif %} + {% endif %} {% endif %} {% endif %} - {% endif %} M ({{ heard.count }}) {{ time_ago(heard.last_rx_time) }}N
    - - - - - - - - - - - - - - - {% set processed_nodes = [] %} - - - {% for neighbor in neighbor_heard_by %} - {% set nid = utils.convert_node_id_from_int_to_hex(neighbor.id) %} - {% if nid not in processed_nodes %} - {% set processed_nodes = processed_nodes + [nid] %} - - - - + + + - - + {% endif %} + + + + {% endif %} + {% endfor %} {% endif %} - {% endfor %} - - - {% if zero_hop_heard_by %} - {% for heard in zero_hop_heard_by %} - {% set nid = utils.convert_node_id_from_int_to_hex(heard.received_by_id) %} + +
    Heard By (zero hop)
    NodeSignalDistanceSource
    - {% if nid in linked_nodes_details %} - {{ linked_nodes_details[nid].short_name }} - {% else %} - UNK - {% endif %} - - SNR: {{ neighbor.snr }} - - {% if nid in linked_nodes_details %} - {% if linked_nodes_details[nid].position and node.position %} - {% if linked_nodes_details[nid].position.latitude_i is not none and linked_nodes_details[nid].position.longitude_i is not none and node.position.latitude_i is not none and node.position.longitude_i is not none %} - {% set dist = utils.calculate_distance_between_nodes(linked_nodes_details[nid], node) %} - {% if dist %} + + + {% if zero_hop_heard %} + {% for heard in zero_hop_heard %} + {% set nid = utils.convert_node_id_from_int_to_hex(heard.from_id) %} + {% if nid not in processed_nodes %} + {% set processed_nodes = processed_nodes + [nid] %} +
    + {% if nid in linked_nodes_details %} + {{ linked_nodes_details[nid].short_name }} + {% else %} + UNK + {% endif %} + + SNR: {{ heard.best_snr }} max / {{ heard.avg_snr|round(1) }} avg + + {% if nid in linked_nodes_details %} + {% set dist = utils.calculate_distance_between_nodes(linked_nodes_details[nid], node) %} + {% if dist %} + {% if linked_nodes_details[nid].position and linked_nodes_details[nid].position.latitude_i and linked_nodes_details[nid].position.longitude_i and node.position and node.position.latitude_i and node.position.longitude_i %} {{ dist|round(2) }} km 🏔️ + {% else %} + {{ dist|round(2) }} km {% endif %} {% endif %} - {% endif %} - {% endif %} - N
    M ({{ heard.count }}) {{ time_ago(heard.last_rx_time) }}
    + + + + + + + + + + + + + + + {% set processed_nodes = [] %} + + + {% for neighbor in neighbor_heard_by %} + {% set nid = utils.convert_node_id_from_int_to_hex(neighbor.id) %} {% if nid not in processed_nodes %} {% set processed_nodes = processed_nodes + [nid] %} @@ -563,41 +525,329 @@ {% endif %} - + {% endif %} {% endfor %} - {% endif %} - -
    Heard By (zero hop)
    NodeSignalDistanceSource
    - SNR: {{ heard.best_snr }} max / {{ heard.avg_snr|round(1) }} avg + SNR: {{ neighbor.snr }} {% if nid in linked_nodes_details %} - {% set dist = utils.calculate_distance_between_nodes(linked_nodes_details[nid], node) %} - {% if dist %} - {% if linked_nodes_details[nid].position and linked_nodes_details[nid].position.latitude_i and linked_nodes_details[nid].position.longitude_i and node.position and node.position.latitude_i and node.position.longitude_i %} - - {{ dist|round(2) }} km 🏔️ - - {% else %} - {{ dist|round(2) }} km + {% if linked_nodes_details[nid].position and node.position %} + {% if linked_nodes_details[nid].position.latitude_i is not none and linked_nodes_details[nid].position.longitude_i is not none and node.position.latitude_i is not none and node.position.longitude_i is not none %} + {% set dist = utils.calculate_distance_between_nodes(linked_nodes_details[nid], node) %} + {% if dist %} + + {{ dist|round(2) }} km 🏔️ + + {% endif %} + {% endif %} {% endif %} {% endif %} - {% endif %} M ({{ heard.count }}) {{ time_ago(heard.last_rx_time) }}N
    + + + {% if zero_hop_heard_by %} + {% for heard in zero_hop_heard_by %} + {% set nid = utils.convert_node_id_from_int_to_hex(heard.received_by_id) %} + {% if nid not in processed_nodes %} + {% set processed_nodes = processed_nodes + [nid] %} + + + {% if nid in linked_nodes_details %} + {{ linked_nodes_details[nid].short_name }} + {% else %} + UNK + {% endif %} + + + SNR: {{ heard.best_snr }} max / {{ heard.avg_snr|round(1) }} avg + + + {% if nid in linked_nodes_details %} + {% set dist = utils.calculate_distance_between_nodes(linked_nodes_details[nid], node) %} + {% if dist %} + {% if linked_nodes_details[nid].position and linked_nodes_details[nid].position.latitude_i and linked_nodes_details[nid].position.longitude_i and node.position and node.position.latitude_i and node.position.longitude_i %} + + {{ dist|round(2) }} km 🏔️ + + {% else %} + {{ dist|round(2) }} km + {% endif %} + {% endif %} + {% endif %} + + M ({{ heard.count }}) {{ time_ago(heard.last_rx_time) }} + + {% endif %} + {% endfor %} + {% endif %} + + +
    - {% if los_profiles %} -
    Line-of-sight with nodes in {{ max_distance | round(2) }} km radius
    -
    - {% for id, data in los_profiles.items()|sort(attribute='1.distance') %} -
    - + + {% if node.telemetry %} +
    +
    +
    +
    Environmental Telemetry
    +
    +
    + + +
    + +
    + +
    + +
    +
    + +
    +
    +
    - {% endfor %}
    {% endif %}
    diff --git a/www/css/meshinfo.css.template b/www/css/meshinfo.css.template index f72fd089..ef005196 100644 --- a/www/css/meshinfo.css.template +++ b/www/css/meshinfo.css.template @@ -730,4 +730,44 @@ a:hover, a:focus { color: #222 !important; /* Dark text for readability */ border: 1px solid {{table_border_color}} !important; font-weight: 600; +} + +/* Environmental Telemetry Dropdown Theming */ +#environmentalDropdown { + border: 2px solid {{control_color}} !important; + border-radius: 0.375rem !important; + color: {{control_color}} !important; + font-family: monospace, monospace; + font-weight: 600; + background: #fff !important; + height: 2.25rem !important; + min-width: 120px; + margin-left: 1rem; + margin-top: 0.15rem; + margin-bottom: 0.15rem; + display: inline-block; + vertical-align: middle; + box-shadow: none; + transition: border-color 0.2s; +} +#environmentalDropdown:focus, #environmentalDropdown:hover { + border-color: {{control_color_hover}} !important; + outline: none !important; + box-shadow: 0 0 0 0.1rem rgba(0,123,255,.15); +} +@media (max-width: 767.98px) { + #environmentalDropdown { + margin-left: 0.5rem; + margin-top: 0.5rem; + margin-bottom: 0.5rem; + width: 100%; + min-width: 0; + } +} + +/* Ensure gap between time range and telemetry selector at desktop widths */ +@media (min-width: 768px) { + #environmentalTelemetryContainer .btn-group + #environmentalTabs { + margin-left: 2rem !important; + } } \ No newline at end of file From 14a0516c597fa8c97df0ed245df08b73399f0580 Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 30 Jun 2025 15:37:12 +0000 Subject: [PATCH 099/155] Update telemetry validation and enhance environmental telemetry display - Modified voltage validation logic to ensure only positive values are accepted. - Updated the environmental telemetry container to initially hide and dynamically display based on data availability. - Improved non-null count logic for voltage values in telemetry data to only consider positive readings. - Added new CSS styles for filter inputs to enhance user interface consistency. --- meshdata.py | 2 +- templates/node.html.j2 | 13 ++++++--- www/css/meshinfo.css.template | 54 +++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 5 deletions(-) diff --git a/meshdata.py b/meshdata.py index 1867a7f9..97f9f01a 100644 --- a/meshdata.py +++ b/meshdata.py @@ -1343,7 +1343,7 @@ def validate_telemetry_value(value, field_name=None): elif field_name == 'voltage': # Voltage should be positive and reasonable # Allow up to 100V to accommodate various power systems - if float_val < 0 or float_val > 100: + if float_val <= 0 or float_val > 100: return None elif field_name == 'temperature': # Temperature should be within reasonable sensor range diff --git a/templates/node.html.j2 b/templates/node.html.j2 index 3fc38290..c06732df 100644 --- a/templates/node.html.j2 +++ b/templates/node.html.j2 @@ -595,7 +595,7 @@ {% if node.telemetry %}
    -
    +