From 292b2421d3db11f364accf68544f438677fc78d1 Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Sat, 11 Apr 2026 13:25:22 -0600 Subject: [PATCH 01/19] perf: viewport-aware downsampling and 60fps render throttling for minimap - Add viewport-aware MinMax downsampling for the main plot so OxyPlot never renders more than ~4000 points per channel regardless of dataset size - Throttle minimap interaction renders at 60fps via DispatcherTimer instead of invalidating both plots on every mouse move - Break the feedback loop between minimap drag and main axis AxisChanged event using a guard flag (IsSyncingFromMinimap) - Add binary search (FindVisibleRange) and sub-range Downsample overload to MinMaxDownsampler for O(log n) viewport extraction - Replace 1M hard point cap with 50M limit using SQL-level Take() to avoid materializing entire result sets into memory - Add composite DB index on (LoggingSessionID, TimestampTicks) for faster session queries - Use LineSeries.Tag for key lookup instead of fragile title-string parsing - Remove unused _sessionPoints dictionary - Add 11 unit tests for MinMaxDownsampler (binary search, sub-range, perf) Co-Authored-By: Claude Opus 4.6 --- .../Helpers/MinMaxDownsamplerTests.cs | 207 ++++++++++++++++++ Daqifi.Desktop/Helpers/MinMaxDownsampler.cs | 132 +++++++++-- Daqifi.Desktop/Loggers/DatabaseLogger.cs | 144 +++++++++--- Daqifi.Desktop/Loggers/LoggingContext.cs | 11 +- .../View/MinimapInteractionController.cs | 51 ++++- 5 files changed, 489 insertions(+), 56 deletions(-) create mode 100644 Daqifi.Desktop.Test/Helpers/MinMaxDownsamplerTests.cs diff --git a/Daqifi.Desktop.Test/Helpers/MinMaxDownsamplerTests.cs b/Daqifi.Desktop.Test/Helpers/MinMaxDownsamplerTests.cs new file mode 100644 index 00000000..d9fcdb62 --- /dev/null +++ b/Daqifi.Desktop.Test/Helpers/MinMaxDownsamplerTests.cs @@ -0,0 +1,207 @@ +using Daqifi.Desktop.Helpers; +using OxyPlot; + +namespace Daqifi.Desktop.Test.Helpers; + +[TestClass] +public class MinMaxDownsamplerTests +{ + #region Downsample Tests + + [TestMethod] + public void Downsample_EmptyList_ReturnsEmpty() + { + var result = MinMaxDownsampler.Downsample(new List(), 10); + Assert.AreEqual(0, result.Count); + } + + [TestMethod] + public void Downsample_FewPointsBelowThreshold_ReturnsAll() + { + var points = new List + { + new(0, 1), new(1, 2), new(2, 3) + }; + var result = MinMaxDownsampler.Downsample(points, 10); + Assert.AreEqual(3, result.Count); + } + + [TestMethod] + public void Downsample_LargeDataset_ProducesCorrectSize() + { + var points = new List(); + for (var i = 0; i < 10000; i++) + { + points.Add(new DataPoint(i, Math.Sin(i * 0.01))); + } + + var result = MinMaxDownsampler.Downsample(points, 100); + Assert.IsTrue(result.Count <= 200); + Assert.IsTrue(result.Count > 0); + } + + [TestMethod] + public void Downsample_PreservesMinMax() + { + var points = new List(); + for (var i = 0; i < 10000; i++) + { + points.Add(new DataPoint(i, Math.Sin(i * 0.01))); + } + + var result = MinMaxDownsampler.Downsample(points, 100); + var resultMax = result.Max(p => p.Y); + var resultMin = result.Min(p => p.Y); + var sourceMax = points.Max(p => p.Y); + var sourceMin = points.Min(p => p.Y); + + Assert.IsTrue(Math.Abs(resultMax - sourceMax) < 0.05, "Should preserve approximate max"); + Assert.IsTrue(Math.Abs(resultMin - sourceMin) < 0.05, "Should preserve approximate min"); + } + + #endregion + + #region Sub-range Downsample Tests + + [TestMethod] + public void Downsample_SubRange_OperatesOnCorrectRange() + { + var points = new List(); + for (var i = 0; i < 1000; i++) + { + points.Add(new DataPoint(i, i < 500 ? 0 : 100)); + } + + // Downsample only the second half (where values are 100) + var result = MinMaxDownsampler.Downsample(points, 500, 1000, 50); + Assert.IsTrue(result.All(p => p.Y == 100), "All downsampled points from second half should be 100"); + } + + [TestMethod] + public void Downsample_SubRange_EmptyRange_ReturnsEmpty() + { + var points = new List { new(0, 1), new(1, 2) }; + var result = MinMaxDownsampler.Downsample(points, 0, 0, 10); + Assert.AreEqual(0, result.Count); + } + + [TestMethod] + public void Downsample_SubRange_SmallRange_ReturnsAll() + { + var points = new List(); + for (var i = 0; i < 100; i++) + { + points.Add(new DataPoint(i, i)); + } + + // Sub-range of 5 points with bucket count of 10 — should return all 5 + var result = MinMaxDownsampler.Downsample(points, 10, 15, 10); + Assert.AreEqual(5, result.Count); + } + + #endregion + + #region FindVisibleRange Tests + + [TestMethod] + public void FindVisibleRange_EmptyList_ReturnsZeroRange() + { + var (start, end) = MinMaxDownsampler.FindVisibleRange(new List(), 0, 10); + Assert.AreEqual(0, start); + Assert.AreEqual(0, end); + } + + [TestMethod] + public void FindVisibleRange_AllVisible_ReturnsFullRange() + { + var points = new List + { + new(0, 0), new(1, 1), new(2, 2), new(3, 3), new(4, 4) + }; + + var (start, end) = MinMaxDownsampler.FindVisibleRange(points, -1, 5); + Assert.AreEqual(0, start); + Assert.AreEqual(5, end); + } + + [TestMethod] + public void FindVisibleRange_MiddleSection_ReturnsPaddedRange() + { + var points = new List(); + for (var i = 0; i < 100; i++) + { + points.Add(new DataPoint(i, i)); + } + + var (start, end) = MinMaxDownsampler.FindVisibleRange(points, 30, 60); + + // Should include padding: one point before 30 and one point after 60 + Assert.IsTrue(start <= 29, $"Start ({start}) should be at or before index 29"); + Assert.IsTrue(end >= 62, $"End ({end}) should be at or after index 62"); + } + + [TestMethod] + public void FindVisibleRange_NoPointsInRange_IncludesAdjacentPoints() + { + var points = new List + { + new(0, 0), new(1, 1), new(10, 10), new(11, 11) + }; + + // Range 5-8 has no points; binary search lands between index 1 (X=1) and 2 (X=10) + // With padding: start backs up 1 from index 2 → 1, end advances 1 from index 2 → 3 + var (start, end) = MinMaxDownsampler.FindVisibleRange(points, 5, 8); + Assert.AreEqual(1, start, "Should include point at X=1 (one before gap)"); + Assert.AreEqual(3, end, "Should include point at X=10 (one after gap)"); + } + + [TestMethod] + public void FindVisibleRange_AtBoundaries_ClampsCorrectly() + { + var points = new List + { + new(0, 0), new(1, 1), new(2, 2) + }; + + // Range starts before data + var (start, _) = MinMaxDownsampler.FindVisibleRange(points, -10, 1); + Assert.AreEqual(0, start, "Start should be clamped to 0"); + + // Range ends after data + var (_, end) = MinMaxDownsampler.FindVisibleRange(points, 1, 100); + Assert.AreEqual(3, end, "End should be clamped to list length"); + } + + [TestMethod] + public void FindVisibleRange_SinglePoint_ReturnsIt() + { + var points = new List { new(5, 10) }; + + var (start, end) = MinMaxDownsampler.FindVisibleRange(points, 0, 10); + Assert.AreEqual(0, start); + Assert.AreEqual(1, end); + } + + [TestMethod] + public void FindVisibleRange_LargeDataset_Performance() + { + var points = new List(); + for (var i = 0; i < 1_000_000; i++) + { + points.Add(new DataPoint(i, Math.Sin(i * 0.001))); + } + + var sw = System.Diagnostics.Stopwatch.StartNew(); + for (var i = 0; i < 1000; i++) + { + MinMaxDownsampler.FindVisibleRange(points, 400000, 600000); + } + sw.Stop(); + + // 1000 binary searches on 1M points should be well under 100ms + Assert.IsTrue(sw.ElapsedMilliseconds < 100, + $"1000 binary searches took {sw.ElapsedMilliseconds}ms, expected < 100ms"); + } + + #endregion +} diff --git a/Daqifi.Desktop/Helpers/MinMaxDownsampler.cs b/Daqifi.Desktop/Helpers/MinMaxDownsampler.cs index c50733dc..499303e4 100644 --- a/Daqifi.Desktop/Helpers/MinMaxDownsampler.cs +++ b/Daqifi.Desktop/Helpers/MinMaxDownsampler.cs @@ -23,24 +23,52 @@ public static List Downsample(IReadOnlyList points, int bu return []; } - if (points.Count <= bucketCount * 2) + return Downsample(points, 0, points.Count, bucketCount); + } + + /// + /// Downsamples a sub-range of a sorted list using min/max aggregation per bucket. + /// Operates on indices [, ) + /// without copying the source list. + /// + /// Time-sorted data points. + /// Inclusive start index of the sub-range. + /// Exclusive end index of the sub-range. + /// Number of buckets to divide the time range into. + /// A downsampled list of data points preserving the visual envelope. + public static List Downsample(IReadOnlyList points, int startIndex, int endIndex, int bucketCount) + { + ArgumentNullException.ThrowIfNull(points); + + var count = endIndex - startIndex; + if (count <= 0 || bucketCount <= 0) { - return new List(points); + return []; } - var result = new List(bucketCount * 2); + if (count <= bucketCount * 2) + { + var result = new List(count); + for (var i = startIndex; i < endIndex; i++) + { + result.Add(points[i]); + } + return result; + } - var xMin = points[0].X; - var xMax = points[points.Count - 1].X; + var output = new List(bucketCount * 2); + + var xMin = points[startIndex].X; + var xMax = points[endIndex - 1].X; var xRange = xMax - xMin; if (xRange <= 0) { - return [points[0]]; + return [points[startIndex]]; } var bucketWidth = xRange / bucketCount; - var pointIndex = 0; + var pointIndex = startIndex; for (var bucket = 0; bucket < bucketCount; bucket++) { @@ -54,7 +82,7 @@ public static List Downsample(IReadOnlyList points, int bu var hasPoints = false; var isLastBucket = bucket == bucketCount - 1; - while (pointIndex < points.Count && (isLastBucket || points[pointIndex].X < bucketEnd)) + while (pointIndex < endIndex && (isLastBucket || points[pointIndex].X < bucketEnd)) { var p = points[pointIndex]; hasPoints = true; @@ -82,22 +110,100 @@ public static List Downsample(IReadOnlyList points, int bu // Emit min and max in X-order to preserve visual continuity if (minYX <= maxYX) { - result.Add(new DataPoint(minYX, minY)); + output.Add(new DataPoint(minYX, minY)); if (Math.Abs(minY - maxY) > double.Epsilon) { - result.Add(new DataPoint(maxYX, maxY)); + output.Add(new DataPoint(maxYX, maxY)); } } else { - result.Add(new DataPoint(maxYX, maxY)); + output.Add(new DataPoint(maxYX, maxY)); if (Math.Abs(minY - maxY) > double.Epsilon) { - result.Add(new DataPoint(minYX, minY)); + output.Add(new DataPoint(minYX, minY)); } } } - return result; + return output; + } + + /// + /// Finds the index range [startIndex, endIndex) of points whose X values + /// fall within [xMin, xMax], with one-point padding on each side for visual continuity. + /// Uses binary search for O(log n) performance on sorted data. + /// + /// Time-sorted data points. + /// Minimum visible X value. + /// Maximum visible X value. + /// Tuple of (inclusive start index, exclusive end index). + public static (int startIndex, int endIndex) FindVisibleRange( + IReadOnlyList sortedPoints, double xMin, double xMax) + { + if (sortedPoints.Count == 0) + { + return (0, 0); + } + + // Binary search for first index where X >= xMin, then back up 1 for continuity + var start = BinarySearchLower(sortedPoints, xMin); + if (start > 0) + { + start--; + } + + // Binary search for first index where X > xMax, then add 1 for continuity + var end = BinarySearchUpper(sortedPoints, xMax); + if (end < sortedPoints.Count) + { + end++; + } + + return (start, end); + } + + /// + /// Returns the index of the first element whose X is >= value. + /// + private static int BinarySearchLower(IReadOnlyList points, double value) + { + var lo = 0; + var hi = points.Count; + while (lo < hi) + { + var mid = lo + (hi - lo) / 2; + if (points[mid].X < value) + { + lo = mid + 1; + } + else + { + hi = mid; + } + } + return lo; + } + + /// + /// Returns the index of the first element whose X is > value. + /// + private static int BinarySearchUpper(IReadOnlyList points, double value) + { + var lo = 0; + var hi = points.Count; + while (lo < hi) + { + var mid = lo + (hi - lo) / 2; + if (points[mid].X <= value) + { + lo = mid + 1; + } + else + { + hi = mid; + } + } + return lo; } } diff --git a/Daqifi.Desktop/Loggers/DatabaseLogger.cs b/Daqifi.Desktop/Loggers/DatabaseLogger.cs index f960687a..24c1a1dc 100644 --- a/Daqifi.Desktop/Loggers/DatabaseLogger.cs +++ b/Daqifi.Desktop/Loggers/DatabaseLogger.cs @@ -103,6 +103,8 @@ public partial class DatabaseLogger : ObservableObject, ILogger { #region Constants private const int MINIMAP_BUCKET_COUNT = 800; + private const int MAIN_PLOT_BUCKET_COUNT = 2000; + private const int MAX_IN_MEMORY_POINTS = 50_000_000; #endregion #region Private Data @@ -110,7 +112,6 @@ public partial class DatabaseLogger : ObservableObject, ILogger public ObservableCollection DeviceLegendGroups { get; } = new(); private readonly Dictionary<(string deviceSerial, string channelName), List> _allSessionPoints = new(); private readonly BlockingCollection _buffer = new(); - private readonly Dictionary<(string deviceSerial, string channelName), List> _sessionPoints = new(); private readonly Dictionary<(string deviceSerial, string channelName), LineSeries> _minimapSeries = new(); private DateTime? _firstTime; @@ -121,6 +122,9 @@ public partial class DatabaseLogger : ObservableObject, ILogger private RectangleAnnotation _minimapDimLeft; private RectangleAnnotation _minimapDimRight; private MinimapInteractionController _minimapInteraction; + internal bool IsSyncingFromMinimap; + private double _lastViewportMin = double.NaN; + private double _lastViewportMax = double.NaN; [ObservableProperty] private PlotModel _plotModel; @@ -325,7 +329,8 @@ private void InitializeMinimapPlotModel() MinimapPlotModel, _minimapSelectionRect, _minimapDimLeft, - _minimapDimRight); + _minimapDimRight, + this); } #endregion @@ -397,7 +402,8 @@ public void ClearPlot() Application.Current.Dispatcher.Invoke(() => { _firstTime = null; - _sessionPoints.Clear(); + _lastViewportMin = double.NaN; + _lastViewportMax = double.NaN; _allSessionPoints.Clear(); _minimapSeries.Clear(); PlotModel.Series.Clear(); @@ -432,20 +438,23 @@ public void DisplayLoggingSession(LoggingSession session) { context.ChangeTracker.AutoDetectChangesEnabled = false; - var dbSamples = context.Samples.AsNoTracking() - .Where(s => s.LoggingSessionID == session.ID) - .OrderBy(s => s.TimestampTicks) - .Select(s => new { s.ChannelName, s.DeviceSerialNo, s.Type, s.Color, s.TimestampTicks, s.Value }) - .ToList(); // Bring data into memory + var baseQuery = context.Samples.AsNoTracking() + .Where(s => s.LoggingSessionID == session.ID); - var samplesCount = dbSamples.Count; - const int dataPointsToShow = 1000000; + var totalSamplesCount = baseQuery.Count(); - if (samplesCount > dataPointsToShow) + if (totalSamplesCount > MAX_IN_MEMORY_POINTS) { - subtitle = $"\nOnly showing {dataPointsToShow:n0} out of {samplesCount:n0} data points"; + subtitle = $"\nShowing first {MAX_IN_MEMORY_POINTS:n0} of {totalSamplesCount:n0} data points"; } + // Only materialize up to the limit to avoid excessive memory usage + var dbSamples = baseQuery + .OrderBy(s => s.TimestampTicks) + .Select(s => new { s.ChannelName, s.DeviceSerialNo, s.Type, s.Color, s.TimestampTicks, s.Value }) + .Take(MAX_IN_MEMORY_POINTS) + .ToList(); + var channelInfoList = dbSamples .Select(s => new { s.ChannelName, s.DeviceSerialNo, s.Type, s.Color }) .Distinct() @@ -459,9 +468,6 @@ public void DisplayLoggingSession(LoggingSession session) tempLegendItemsList.Add(legendItem); } - // This part still needs to be careful about _allSessionPoints access if it's used by UI directly - // For now, _allSessionPoints is used to populate series ItemsSource later on UI thread - var dataSampleCount = 0; foreach (var sample in dbSamples) { var key = (sample.DeviceSerialNo, sample.ChannelName); @@ -472,12 +478,6 @@ public void DisplayLoggingSession(LoggingSession session) { points.Add(new DataPoint(deltaTime, sample.Value)); } - - dataSampleCount++; - if (dataSampleCount >= dataPointsToShow) - { - break; - } } } @@ -489,10 +489,7 @@ public void DisplayLoggingSession(LoggingSession session) { var downsampled = MinMaxDownsampler.Downsample(kvp.Value, MINIMAP_BUCKET_COUNT); var matchingSeries = tempSeriesList.FirstOrDefault(s => - { - var parts = s.Title.Split([" : ("], StringSplitOptions.None); - return parts.Length == 2 && parts[0] == kvp.Key.channelName && parts[1].TrimEnd(')') == kvp.Key.deviceSerial; - }); + s.Tag is (string ds, string cn) && ds == kvp.Key.deviceSerial && cn == kvp.Key.channelName); minimapSeriesData.Add((kvp.Key.channelName, kvp.Key.deviceSerial, matchingSeries?.Color ?? OxyColors.Gray, downsampled)); } } @@ -525,12 +522,18 @@ public void DisplayLoggingSession(LoggingSession session) foreach (var series in tempSeriesList) { PlotModel.Series.Add(series); - // Assign data to series (ItemsSource) - // The key for _allSessionPoints must match how it was populated - var key = (series.Title.Split([" : ("], StringSplitOptions.None)[1].TrimEnd(')'), series.Title.Split([" : ("], StringSplitOptions.None)[0]); - if (_allSessionPoints.TryGetValue(key, out var points)) + if (series.Tag is (string deviceSerial, string channelName) + && _allSessionPoints.TryGetValue((deviceSerial, channelName), out var points)) { - series.ItemsSource = points; + // Set initial ItemsSource to downsampled full range + if (points.Count > MAIN_PLOT_BUCKET_COUNT * 2) + { + series.ItemsSource = MinMaxDownsampler.Downsample(points, MAIN_PLOT_BUCKET_COUNT); + } + else + { + series.ItemsSource = points; + } } } @@ -657,15 +660,14 @@ public void ResumeConsumer() private (LineSeries series, LoggedSeriesLegendItem legendItem) AddChannelSeries(string channelName, string deviceSerialNo, ChannelType type, string color) { var key = (DeviceSerialNo: deviceSerialNo, channelName); - _sessionPoints.Add(key, []); _allSessionPoints.Add(key, []); var newLineSeries = new LineSeries { Title = $"{channelName} : ({deviceSerialNo})", - ItemsSource = _sessionPoints.Last().Value, // This will be empty initially, data is added later + Tag = (deviceSerialNo, channelName), Color = OxyColor.Parse(color), - IsVisible = true // Default to visible + IsVisible = true }; var legendItem = new LoggedSeriesLegendItem( @@ -677,7 +679,6 @@ public void ResumeConsumer() newLineSeries, PlotModel, this); - // LegendItems.Add(legendItem); // Removed: To be added in DisplayLoggingSession on UI thread newLineSeries.YAxisKey = type switch { @@ -686,14 +687,19 @@ public void ResumeConsumer() _ => newLineSeries.YAxisKey }; - // PlotModel.Series.Add(newLineSeries); // Removed: To be added in DisplayLoggingSession on UI thread - // OnPropertyChanged("PlotModel"); // Removed: To be called in DisplayLoggingSession on UI thread return (newLineSeries, legendItem); } #region Minimap Synchronization private void OnMainTimeAxisChanged(object? sender, AxisChangedEventArgs e) { + if (IsSyncingFromMinimap) + { + return; + } + + UpdateMainPlotViewport(); + var timeAxis = PlotModel.Axes.FirstOrDefault(a => a.Key == "Time"); if (timeAxis == null) { @@ -707,6 +713,67 @@ private void OnMainTimeAxisChanged(object? sender, AxisChangedEventArgs e) MinimapPlotModel.InvalidatePlot(false); } + /// + /// Re-downsamples each main plot series for the currently visible time range. + /// Skips the update if the viewport hasn't changed since the last call. + /// + private void UpdateMainPlotViewport() + { + var timeAxis = PlotModel.Axes.FirstOrDefault(a => a.Key == "Time"); + if (timeAxis == null) + { + return; + } + + var visibleMin = timeAxis.ActualMinimum; + var visibleMax = timeAxis.ActualMaximum; + + // Skip if viewport hasn't changed + if (visibleMin == _lastViewportMin && visibleMax == _lastViewportMax) + { + return; + } + + _lastViewportMin = visibleMin; + _lastViewportMax = visibleMax; + + foreach (var series in PlotModel.Series.OfType()) + { + if (series.Tag is not (string deviceSerial, string channelName)) + { + continue; + } + + var key = (deviceSerial, channelName); + if (!_allSessionPoints.TryGetValue(key, out var allPoints) || allPoints.Count == 0) + { + continue; + } + + var (startIdx, endIdx) = MinMaxDownsampler.FindVisibleRange(allPoints, visibleMin, visibleMax); + var visibleCount = endIdx - startIdx; + + if (visibleCount <= MAIN_PLOT_BUCKET_COUNT * 2) + { + // Few enough points to render directly + series.ItemsSource = allPoints.GetRange(startIdx, visibleCount); + } + else + { + series.ItemsSource = MinMaxDownsampler.Downsample(allPoints, startIdx, endIdx, MAIN_PLOT_BUCKET_COUNT); + } + } + } + + /// + /// Called by the minimap interaction controller to update the main plot's + /// viewport downsampling after a minimap-driven zoom/pan. + /// + public void OnMinimapViewportChanged() + { + UpdateMainPlotViewport(); + } + /// /// Updates the visibility of a minimap series to match its main plot counterpart. /// @@ -743,6 +810,9 @@ private void SaveGraph() private void ResetZoom() { PlotModel.ResetAllAxes(); + _lastViewportMin = double.NaN; + _lastViewportMax = double.NaN; + UpdateMainPlotViewport(); PlotModel.InvalidatePlot(true); } @@ -750,6 +820,7 @@ private void ResetZoom() private void ZoomOutX() { PlotModel.Axes[2].ZoomAtCenter(0.8); + UpdateMainPlotViewport(); PlotModel.InvalidatePlot(true); } @@ -757,6 +828,7 @@ private void ZoomOutX() private void ZoomInX() { PlotModel.Axes[2].ZoomAtCenter(1.25); + UpdateMainPlotViewport(); PlotModel.InvalidatePlot(true); } diff --git a/Daqifi.Desktop/Loggers/LoggingContext.cs b/Daqifi.Desktop/Loggers/LoggingContext.cs index 3ed7045f..981b62c8 100644 --- a/Daqifi.Desktop/Loggers/LoggingContext.cs +++ b/Daqifi.Desktop/Loggers/LoggingContext.cs @@ -1,4 +1,4 @@ -using Daqifi.Desktop.Channel; +using Daqifi.Desktop.Channel; using Microsoft.EntityFrameworkCore; namespace Daqifi.Desktop.Logger; @@ -18,7 +18,12 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) entity.Property(ls => ls.Name).IsRequired(); }); - modelBuilder.Entity().ToTable("Samples"); + modelBuilder.Entity(entity => + { + entity.ToTable("Samples"); + entity.HasIndex(s => new { s.LoggingSessionID, s.TimestampTicks }) + .HasDatabaseName("IX_Samples_SessionTime"); + }); modelBuilder.Entity(entity => { @@ -31,4 +36,4 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) public DbSet Sessions { get; set; } public DbSet Samples { get; set; } -} \ No newline at end of file +} diff --git a/Daqifi.Desktop/View/MinimapInteractionController.cs b/Daqifi.Desktop/View/MinimapInteractionController.cs index 9bcae8e4..a04a17f4 100644 --- a/Daqifi.Desktop/View/MinimapInteractionController.cs +++ b/Daqifi.Desktop/View/MinimapInteractionController.cs @@ -1,6 +1,8 @@ +using Daqifi.Desktop.Logger; using OxyPlot; using OxyPlot.Annotations; using OxyPlot.Axes; +using System.Windows.Threading; using Cursor = System.Windows.Input.Cursor; using Cursors = System.Windows.Input.Cursors; using Application = System.Windows.Application; @@ -12,7 +14,7 @@ namespace Daqifi.Desktop.View; /// Handles mouse interactions on the minimap PlotModel to enable drag/resize /// of the selection rectangle, synchronized with the main plot's time axis. /// Provides cursor feedback: resize arrows on edges, grab hand inside selection, -/// and pointer outside. +/// and pointer outside. Renders are throttled to 60fps via a DispatcherTimer. /// public class MinimapInteractionController : IDisposable { @@ -24,6 +26,7 @@ public class MinimapInteractionController : IDisposable private readonly RectangleAnnotation _dimRight; private readonly string _mainTimeAxisKey; private readonly string _minimapTimeAxisKey; + private readonly DatabaseLogger _databaseLogger; private enum DragMode { None, Pan, ResizeLeft, ResizeRight } private DragMode _dragMode = DragMode.None; @@ -33,6 +36,9 @@ private enum DragMode { None, Pan, ResizeLeft, ResizeRight } private Cursor _lastCursor; private const double EDGE_TOLERANCE_FRACTION = 0.02; + + private bool _isDirty; + private readonly DispatcherTimer _renderTimer; #endregion #region Constructor @@ -42,6 +48,7 @@ public MinimapInteractionController( RectangleAnnotation selectionRect, RectangleAnnotation dimLeft, RectangleAnnotation dimRight, + DatabaseLogger databaseLogger, string mainTimeAxisKey = "Time", string minimapTimeAxisKey = "MinimapTime") { @@ -50,12 +57,20 @@ public MinimapInteractionController( _selectionRect = selectionRect; _dimLeft = dimLeft; _dimRight = dimRight; + _databaseLogger = databaseLogger; _mainTimeAxisKey = mainTimeAxisKey; _minimapTimeAxisKey = minimapTimeAxisKey; _minimapPlotModel.MouseDown += OnMouseDown; _minimapPlotModel.MouseMove += OnMouseMove; _minimapPlotModel.MouseUp += OnMouseUp; + + _renderTimer = new DispatcherTimer(DispatcherPriority.Render) + { + Interval = TimeSpan.FromMilliseconds(16) + }; + _renderTimer.Tick += OnRenderTick; + _renderTimer.Start(); } #endregion @@ -261,6 +276,15 @@ private void OnMouseUp(object? sender, OxyMouseEventArgs e) { _dragMode = DragMode.None; + // Flush final render for accuracy + if (_isDirty) + { + _isDirty = false; + _databaseLogger.OnMinimapViewportChanged(); + _mainPlotModel.InvalidatePlot(false); + _minimapPlotModel.InvalidatePlot(false); + } + // Update cursor based on final position var minimapTimeAxis = GetMinimapTimeAxis(); if (minimapTimeAxis != null) @@ -274,6 +298,21 @@ private void OnMouseUp(object? sender, OxyMouseEventArgs e) } #endregion + #region Render Throttling + private void OnRenderTick(object? sender, EventArgs e) + { + if (!_isDirty) + { + return; + } + + _isDirty = false; + _databaseLogger.OnMinimapViewportChanged(); + _mainPlotModel.InvalidatePlot(false); + _minimapPlotModel.InvalidatePlot(false); + } + #endregion + #region Private Methods private void ApplyToMainPlot(double min, double max) { @@ -283,11 +322,13 @@ private void ApplyToMainPlot(double min, double max) return; } + _databaseLogger.IsSyncingFromMinimap = true; mainTimeAxis.Zoom(min, max); + _databaseLogger.IsSyncingFromMinimap = false; + _dimLeft.MaximumX = min; _dimRight.MinimumX = max; - _mainPlotModel.InvalidatePlot(false); - _minimapPlotModel.InvalidatePlot(false); + _isDirty = true; } private LinearAxis? GetMinimapTimeAxis() @@ -316,10 +357,12 @@ private double GetMinimapDataMin() #region IDisposable /// - /// Unsubscribes all event handlers from the minimap PlotModel. + /// Unsubscribes all event handlers from the minimap PlotModel and stops the render timer. /// public void Dispose() { + _renderTimer.Stop(); + _renderTimer.Tick -= OnRenderTick; _minimapPlotModel.MouseDown -= OnMouseDown; _minimapPlotModel.MouseMove -= OnMouseMove; _minimapPlotModel.MouseUp -= OnMouseUp; From 44d7b02947c1cfbf09aa5066bca9465d2850a4b8 Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Sat, 11 Apr 2026 13:32:08 -0600 Subject: [PATCH 02/19] fix: restore full-range data on ResetZoom to prevent data clipping After viewport downsampling, ItemsSource contains only the visible subset. ResetAllAxes() auto-ranges from this subset, causing the plot to appear clipped to the last zoomed range. Fix by restoring full-range downsampled data before resetting axes. Co-Authored-By: Claude Opus 4.6 --- Daqifi.Desktop/Loggers/DatabaseLogger.cs | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/Daqifi.Desktop/Loggers/DatabaseLogger.cs b/Daqifi.Desktop/Loggers/DatabaseLogger.cs index 24c1a1dc..9cfcaab8 100644 --- a/Daqifi.Desktop/Loggers/DatabaseLogger.cs +++ b/Daqifi.Desktop/Loggers/DatabaseLogger.cs @@ -809,10 +809,26 @@ private void SaveGraph() [RelayCommand] private void ResetZoom() { - PlotModel.ResetAllAxes(); + // Restore full-range downsampled data before resetting axes, otherwise + // OxyPlot auto-ranges to the current (viewport-subset) ItemsSource. + foreach (var series in PlotModel.Series.OfType()) + { + if (series.Tag is not (string deviceSerial, string channelName)) + { + continue; + } + + if (_allSessionPoints.TryGetValue((deviceSerial, channelName), out var allPoints) && allPoints.Count > 0) + { + series.ItemsSource = allPoints.Count > MAIN_PLOT_BUCKET_COUNT * 2 + ? MinMaxDownsampler.Downsample(allPoints, MAIN_PLOT_BUCKET_COUNT) + : allPoints; + } + } + _lastViewportMin = double.NaN; _lastViewportMax = double.NaN; - UpdateMainPlotViewport(); + PlotModel.ResetAllAxes(); PlotModel.InvalidatePlot(true); } From a567f906690366d3ad19d121f7ee7f976393fa50 Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Sat, 11 Apr 2026 13:51:53 -0600 Subject: [PATCH 03/19] fix: use explicit time axis range in ResetZoom instead of auto-range MinMax downsampling doesn't preserve exact first/last X values (it emits points at min/max Y positions within each bucket). When OxyPlot auto- ranges from downsampled data, the axis range is narrower than the actual data, causing a cascading clip effect through OnMainTimeAxisChanged. Fix by computing the full data range from _allSessionPoints and explicitly setting the time axis via Zoom(), while only auto-ranging the Y axes. Co-Authored-By: Claude Opus 4.6 --- Daqifi.Desktop/Loggers/DatabaseLogger.cs | 33 +++++++++++++++++------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/Daqifi.Desktop/Loggers/DatabaseLogger.cs b/Daqifi.Desktop/Loggers/DatabaseLogger.cs index 9cfcaab8..7d783c59 100644 --- a/Daqifi.Desktop/Loggers/DatabaseLogger.cs +++ b/Daqifi.Desktop/Loggers/DatabaseLogger.cs @@ -809,26 +809,39 @@ private void SaveGraph() [RelayCommand] private void ResetZoom() { - // Restore full-range downsampled data before resetting axes, otherwise - // OxyPlot auto-ranges to the current (viewport-subset) ItemsSource. - foreach (var series in PlotModel.Series.OfType()) + // Reset Y axes to auto-range for amplitude + foreach (var axis in PlotModel.Axes) { - if (series.Tag is not (string deviceSerial, string channelName)) + if (axis.Key != "Time") { - continue; + axis.Reset(); } + } - if (_allSessionPoints.TryGetValue((deviceSerial, channelName), out var allPoints) && allPoints.Count > 0) + // Compute the full data range from source data (not downsampled) and + // explicitly set the time axis rather than relying on auto-range, which + // would use the current ItemsSource extent (potentially narrowed by + // viewport downsampling). + var fullMin = double.MaxValue; + var fullMax = double.MinValue; + foreach (var kvp in _allSessionPoints) + { + if (kvp.Value.Count > 0) { - series.ItemsSource = allPoints.Count > MAIN_PLOT_BUCKET_COUNT * 2 - ? MinMaxDownsampler.Downsample(allPoints, MAIN_PLOT_BUCKET_COUNT) - : allPoints; + fullMin = Math.Min(fullMin, kvp.Value[0].X); + fullMax = Math.Max(fullMax, kvp.Value[^1].X); } } + var timeAxis = PlotModel.Axes.FirstOrDefault(a => a.Key == "Time"); + if (timeAxis != null && fullMin < fullMax) + { + timeAxis.Zoom(fullMin, fullMax); + } + _lastViewportMin = double.NaN; _lastViewportMax = double.NaN; - PlotModel.ResetAllAxes(); + UpdateMainPlotViewport(); PlotModel.InvalidatePlot(true); } From 65f70dfc2e968275ee28843891abfa1a31f6ef15 Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Sat, 11 Apr 2026 14:01:45 -0600 Subject: [PATCH 04/19] fix: force OxyPlot to re-read ItemsSource after viewport downsample update InvalidatePlot(false) only re-renders from OxyPlot's internal cache, ignoring changes to ItemsSource. After zoom button + minimap drag, the main plot showed missing data because the updated downsampled data was never picked up. Changed to InvalidatePlot(true) in both OnRenderTick and OnMouseUp to ensure OxyPlot reads the fresh ItemsSource. Co-Authored-By: Claude Opus 4.6 --- Daqifi.Desktop/View/MinimapInteractionController.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Daqifi.Desktop/View/MinimapInteractionController.cs b/Daqifi.Desktop/View/MinimapInteractionController.cs index a04a17f4..f252e2fe 100644 --- a/Daqifi.Desktop/View/MinimapInteractionController.cs +++ b/Daqifi.Desktop/View/MinimapInteractionController.cs @@ -281,7 +281,7 @@ private void OnMouseUp(object? sender, OxyMouseEventArgs e) { _isDirty = false; _databaseLogger.OnMinimapViewportChanged(); - _mainPlotModel.InvalidatePlot(false); + _mainPlotModel.InvalidatePlot(true); _minimapPlotModel.InvalidatePlot(false); } @@ -308,7 +308,7 @@ private void OnRenderTick(object? sender, EventArgs e) _isDirty = false; _databaseLogger.OnMinimapViewportChanged(); - _mainPlotModel.InvalidatePlot(false); + _mainPlotModel.InvalidatePlot(true); _minimapPlotModel.InvalidatePlot(false); } #endregion From 28dc2ccc6c232c3a118bd3378d1bca086c06d89e Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Sat, 11 Apr 2026 14:09:39 -0600 Subject: [PATCH 05/19] perf: eliminate GC pressure and throttle main plot viewport updates Two performance improvements for sustained 60fps during interaction: 1. Reuse cached List per series in UpdateMainPlotViewport() instead of allocating new lists every frame. With 16 channels at 60fps, this eliminates ~960 list allocations/sec that were causing GC micro-stutters. 2. Throttle viewport updates from main plot pan/zoom to 60fps via DispatcherTimer + dirty flag (matching the minimap's approach). Previously, OnMainTimeAxisChanged called UpdateMainPlotViewport() synchronously on every mouse move event (~120Hz). Co-Authored-By: Claude Opus 4.6 --- Daqifi.Desktop/Loggers/DatabaseLogger.cs | 73 +++++++++++++++++++++--- 1 file changed, 66 insertions(+), 7 deletions(-) diff --git a/Daqifi.Desktop/Loggers/DatabaseLogger.cs b/Daqifi.Desktop/Loggers/DatabaseLogger.cs index 7d783c59..c7232110 100644 --- a/Daqifi.Desktop/Loggers/DatabaseLogger.cs +++ b/Daqifi.Desktop/Loggers/DatabaseLogger.cs @@ -18,6 +18,7 @@ using EFCore.BulkExtensions; using CommunityToolkit.Mvvm.ComponentModel; using CommunityToolkit.Mvvm.Input; +using System.Windows.Threading; using Application = System.Windows.Application; using FontWeights = OxyPlot.FontWeights; @@ -111,6 +112,7 @@ public partial class DatabaseLogger : ObservableObject, ILogger public ObservableCollection LegendItems { get; } = new(); public ObservableCollection DeviceLegendGroups { get; } = new(); private readonly Dictionary<(string deviceSerial, string channelName), List> _allSessionPoints = new(); + private readonly Dictionary<(string deviceSerial, string channelName), List> _downsampledCache = new(); private readonly BlockingCollection _buffer = new(); private readonly Dictionary<(string deviceSerial, string channelName), LineSeries> _minimapSeries = new(); @@ -125,6 +127,8 @@ public partial class DatabaseLogger : ObservableObject, ILogger internal bool IsSyncingFromMinimap; private double _lastViewportMin = double.NaN; private double _lastViewportMax = double.NaN; + private bool _viewportDirty; + private DispatcherTimer _viewportThrottleTimer; [ObservableProperty] private PlotModel _plotModel; @@ -229,6 +233,14 @@ public DatabaseLogger(IDbContextFactory loggingContext) // Subscribe to main time axis changes for minimap sync timeAxis.AxisChanged += OnMainTimeAxisChanged; + // Throttle viewport updates from main plot interaction to 60fps + _viewportThrottleTimer = new DispatcherTimer(DispatcherPriority.Render) + { + Interval = TimeSpan.FromMilliseconds(16) + }; + _viewportThrottleTimer.Tick += OnViewportThrottleTick; + _viewportThrottleTimer.Start(); + // Initialize minimap PlotModel InitializeMinimapPlotModel(); @@ -405,6 +417,7 @@ public void ClearPlot() _lastViewportMin = double.NaN; _lastViewportMax = double.NaN; _allSessionPoints.Clear(); + _downsampledCache.Clear(); _minimapSeries.Clear(); PlotModel.Series.Clear(); LegendItems.Clear(); @@ -525,15 +538,25 @@ public void DisplayLoggingSession(LoggingSession session) if (series.Tag is (string deviceSerial, string channelName) && _allSessionPoints.TryGetValue((deviceSerial, channelName), out var points)) { - // Set initial ItemsSource to downsampled full range + // Set initial ItemsSource to downsampled full range using cached list + var key = (deviceSerial, channelName); + if (!_downsampledCache.TryGetValue(key, out var cached)) + { + cached = new List(MAIN_PLOT_BUCKET_COUNT * 2); + _downsampledCache[key] = cached; + } + + cached.Clear(); if (points.Count > MAIN_PLOT_BUCKET_COUNT * 2) { - series.ItemsSource = MinMaxDownsampler.Downsample(points, MAIN_PLOT_BUCKET_COUNT); + cached.AddRange(MinMaxDownsampler.Downsample(points, MAIN_PLOT_BUCKET_COUNT)); } else { - series.ItemsSource = points; + cached.AddRange(points); } + + series.ItemsSource = cached; } } @@ -698,7 +721,8 @@ private void OnMainTimeAxisChanged(object? sender, AxisChangedEventArgs e) return; } - UpdateMainPlotViewport(); + // Mark viewport dirty — the throttle timer will handle the actual update at 60fps + _viewportDirty = true; var timeAxis = PlotModel.Axes.FirstOrDefault(a => a.Key == "Time"); if (timeAxis == null) @@ -713,6 +737,22 @@ private void OnMainTimeAxisChanged(object? sender, AxisChangedEventArgs e) MinimapPlotModel.InvalidatePlot(false); } + /// + /// Throttled viewport update tick — processes dirty flag at 60fps to avoid + /// re-downsampling on every mouse move during main plot pan/zoom. + /// + private void OnViewportThrottleTick(object? sender, EventArgs e) + { + if (!_viewportDirty) + { + return; + } + + _viewportDirty = false; + UpdateMainPlotViewport(); + PlotModel.InvalidatePlot(true); + } + /// /// Re-downsamples each main plot series for the currently visible time range. /// Skips the update if the viewport hasn't changed since the last call. @@ -753,14 +793,33 @@ private void UpdateMainPlotViewport() var (startIdx, endIdx) = MinMaxDownsampler.FindVisibleRange(allPoints, visibleMin, visibleMax); var visibleCount = endIdx - startIdx; + // Reuse cached list to avoid GC pressure during interaction + if (!_downsampledCache.TryGetValue(key, out var cached)) + { + cached = new List(MAIN_PLOT_BUCKET_COUNT * 2); + _downsampledCache[key] = cached; + } + if (visibleCount <= MAIN_PLOT_BUCKET_COUNT * 2) { - // Few enough points to render directly - series.ItemsSource = allPoints.GetRange(startIdx, visibleCount); + // Few enough points to render directly — copy into cached list + cached.Clear(); + for (var i = startIdx; i < endIdx; i++) + { + cached.Add(allPoints[i]); + } } else { - series.ItemsSource = MinMaxDownsampler.Downsample(allPoints, startIdx, endIdx, MAIN_PLOT_BUCKET_COUNT); + var downsampled = MinMaxDownsampler.Downsample(allPoints, startIdx, endIdx, MAIN_PLOT_BUCKET_COUNT); + cached.Clear(); + cached.AddRange(downsampled); + } + + // Only set ItemsSource once per series — subsequent updates reuse the same list + if (series.ItemsSource != cached) + { + series.ItemsSource = cached; } } } From be35c1b68987a648027d859135a33cdb18cdc9b4 Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Sat, 11 Apr 2026 14:16:48 -0600 Subject: [PATCH 06/19] docs: document OxyPlot performance patterns and gotchas in CLAUDE.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records hard-won lessons from the minimap performance work: - Why viewport-aware downsampling over global LTTB decimation - InvalidatePlot(true) vs (false) and when each is required - MinMax downsampling X-boundary drift causing auto-range shrinkage - Minimap ↔ main plot feedback loop prevention pattern - GC pressure from per-frame list allocations - DispatcherTimer + dirty flag throttle pattern Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 3ee79e72..b6e68857 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -242,6 +242,7 @@ When working on: - **Firewall/Network**: Ensure admin privileges handled properly, verify ports match - **WiFi Discovery Issues**: Check network interface detection and port configuration - **Manual IP Connections**: Ensure TCP port matches what device discovery reports +- **Plot/Minimap changes**: Read the "Plot Rendering (OxyPlot)" section below — there are non-obvious gotchas with `InvalidatePlot`, auto-range, and feedback loops. Key files: `DatabaseLogger.cs`, `MinimapInteractionController.cs`, `MinMaxDownsampler.cs` - **New features**: Add unit tests with 80% coverage minimum ## Performance Considerations @@ -252,6 +253,22 @@ When working on: - Use dependency injection for testability - Cache expensive operations +### Plot Rendering (OxyPlot) + +The logged data viewer uses viewport-aware MinMax downsampling to handle large datasets (16+ channels × 1M+ points). Key architecture decisions and gotchas: + +**Viewport-aware downsampling over global decimation**: Global downsampling (e.g., LTTB to ~5000 points) conflicts with the minimap — when zoomed into a 1-minute slice of 24 hours, only ~3 points would be visible. Instead, we binary search the visible range and downsample only that slice to ~4000 points per channel. This gives full detail when zoomed in. See `MinMaxDownsampler.FindVisibleRange()` and `DatabaseLogger.UpdateMainPlotViewport()`. + +**OxyPlot `InvalidatePlot(true)` vs `InvalidatePlot(false)`**: `false` re-renders from OxyPlot's internal cached point arrays. `true` forces OxyPlot to re-read `ItemsSource` and rebuild those arrays. You MUST use `true` whenever you change a series' `ItemsSource` or mutate the underlying list — otherwise the plot renders stale data. This was the root cause of a bug where zoom + minimap drag showed missing data. + +**MinMax downsampling does NOT preserve original X boundaries**: Downsampled data emits points at min/max Y positions within each bucket, not at bucket edges. So the first and last X values of downsampled data may differ from the original data. This means `ResetAllAxes()` (which auto-ranges from current `ItemsSource`) can progressively shrink the visible range. Fix: explicitly compute the full time range from source data and use `axis.Zoom(min, max)` instead of auto-range. + +**Feedback loops between minimap and main plot**: The minimap syncs bidirectionally with the main plot's time axis. Without a guard flag (`IsSyncingFromMinimap`), dragging the minimap triggers `AxisChanged` on the main plot, which updates the minimap, creating a render loop. Always set the guard before programmatic axis changes. + +**GC pressure during interaction**: Allocating new `List` per channel per frame (~960/sec at 60fps with 16 channels) causes Gen0 GC micro-stutters. Reuse cached lists per series key — clear and refill instead of allocating new ones. See `_downsampledCache` in `DatabaseLogger`. + +**Throttle pattern**: Both minimap drag and main plot pan/zoom use a `DispatcherTimer` (16ms / 60fps) + dirty flag pattern. Mouse events set the flag; the timer tick does the actual work. This caps expensive operations (re-downsample + render) at 60Hz regardless of input event frequency. + ## Error Handling - Use try-catch at appropriate levels From 80eca475337efa2aa72a561c4723ba12f107c159 Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Sat, 11 Apr 2026 14:19:06 -0600 Subject: [PATCH 07/19] docs: add ADR directory with viewport-aware downsampling decision record ADR 001 documents why we chose viewport-aware MinMax downsampling over global LTTB (PR #457), pre-computed pyramids, GPU rendering, and on-demand DB queries. Records the trade-offs, consequences, and follow-up work for future contributors. Co-Authored-By: Claude Opus 4.6 --- docs/adr/001-viewport-aware-downsampling.md | 87 +++++++++++++++++++++ docs/adr/README.md | 21 +++++ 2 files changed, 108 insertions(+) create mode 100644 docs/adr/001-viewport-aware-downsampling.md create mode 100644 docs/adr/README.md diff --git a/docs/adr/001-viewport-aware-downsampling.md b/docs/adr/001-viewport-aware-downsampling.md new file mode 100644 index 00000000..c37e1e11 --- /dev/null +++ b/docs/adr/001-viewport-aware-downsampling.md @@ -0,0 +1,87 @@ +# ADR 001: Viewport-Aware Downsampling for Large Dataset Rendering + +**Status**: Accepted +**Date**: 2026-04 +**PR**: #467 (supersedes #457) + +## Context + +The DAQiFi Desktop application needs to display logged data sessions that can be very large — a typical worst case is 16 channels at 1000 Hz for 24 hours, producing ~1.38 billion data points. PR #458 added an overview minimap for navigating these sessions, but the main plot still rendered every data point, making the UI sluggish with large sessions. + +OxyPlot (our charting library) iterates every point in a series during render, even when zoomed into a tiny region. With 1M points per channel and 16 channels, that's 16M point iterations per frame — far too slow for interactive pan/zoom. + +We needed a downsampling strategy that: + +1. Keeps the main plot under ~4000 points per channel regardless of dataset size +2. Shows full detail when zoomed in (not a blurry approximation) +3. Works with the minimap's ability to select arbitrary time ranges +4. Maintains 60fps during drag/resize interactions + +## Decision + +Use **viewport-aware MinMax downsampling**: on every viewport change, binary search the source data for the visible time range, then downsample only that slice to ~4000 points using min/max aggregation per bucket. + +### How it works + +1. Each channel's full dataset is kept in memory as a sorted `List` (`_allSessionPoints`) +2. When the viewport changes (minimap drag, zoom, pan), `UpdateMainPlotViewport()`: + - Binary searches for the visible range indices — O(log n) via `MinMaxDownsampler.FindVisibleRange()` + - If the visible slice is small enough (< 4000 points), uses it directly + - Otherwise, downsamples via `MinMaxDownsampler.Downsample(points, startIdx, endIdx, 2000)` — divides into 2000 buckets, emits the min and max Y value per bucket (up to 4000 output points) +3. The downsampled data is written into a **reusable cached list** per series (not a new allocation) and set as the series' `ItemsSource` +4. Viewport updates are **throttled to 60fps** via DispatcherTimer + dirty flag, both for minimap-driven changes and main plot pan/zoom + +### Key files + +- `MinMaxDownsampler.cs` — binary search + min/max downsampling algorithm +- `DatabaseLogger.UpdateMainPlotViewport()` — viewport change handler +- `MinimapInteractionController.cs` — 60fps throttled minimap interaction + +## Alternatives Considered + +### 1. Global LTTB Decimation (PR #457) + +PR #457 applied Largest Triangle Three Buckets (LTTB) decimation globally to ~5000 points per channel at load time. + +**Rejected because**: Global decimation fundamentally conflicts with the minimap. If you decimate 24 hours of data to 5000 points and then zoom into a 1-minute window, only ~3 points would be visible — the plot would be empty or misleadingly sparse. The minimap's entire purpose is to let users zoom into arbitrary ranges, so the downsampling must be viewport-aware. + +LTTB also has higher computational cost per point than MinMax (triangle area calculations vs simple comparisons), which matters when re-downsampling at 60fps. + +### 2. Pre-computed Multi-Resolution Pyramid + +Build multiple resolution levels at load time (e.g., 1:1, 1:10, 1:100, 1:1000) and select the appropriate level based on zoom. + +**Rejected because**: Significant memory overhead (nearly 2x the raw data), complex invalidation if data changes, and overkill for our use case. The binary search + linear scan approach is fast enough (~1ms for 16 channels) that on-the-fly downsampling at 60fps is feasible without pre-computation. + +### 3. GPU-Accelerated Rendering + +Replace OxyPlot with a GPU-backed charting library (e.g., SciChart, LiveCharts2 with SkiaSharp). + +**Rejected because**: Major dependency change with significant migration cost. OxyPlot is well-integrated with our WPF MVVM architecture. Viewport-aware downsampling reduces the point count enough (~64K total) that OxyPlot renders comfortably within a 16ms frame budget. If we outgrow this approach, GPU rendering remains a future option. + +### 4. Virtual Scrolling / On-Demand DB Queries + +Only load the visible time range from SQLite on each viewport change, avoiding keeping all data in memory. + +**Rejected because**: SQLite query latency (~5-50ms depending on range size) is too high for 60fps interaction. Users would see visible lag during minimap drag. Keeping data in memory with a practical cap (50M points, ~800MB) is the right trade-off for interactive performance. The DB index we added (`IX_Samples_SessionTime`) supports this pattern if we ever need to implement paging for truly enormous datasets. + +## Consequences + +### Positive + +- **60fps interaction**: ~10-15ms per frame with 16 channels × 1M points +- **Full zoom fidelity**: Zooming into a 1-second window of a 24-hour session shows every data point +- **Low complexity**: The core algorithm is ~80 lines (binary search + min/max loop) +- **No external dependencies**: Pure C# implementation + +### Negative + +- **Memory usage**: Full dataset lives in memory. Capped at 50M points (~800MB). Sessions exceeding this are truncated with a UI warning. +- **MinMax doesn't preserve exact X boundaries**: Downsampled first/last X values may differ from source data. This required explicit time axis ranging in `ResetZoom` instead of relying on OxyPlot auto-range (see `InvalidatePlot` gotchas in CLAUDE.md). +- **List mutation pattern**: Reusing cached lists and calling `InvalidatePlot(true)` is non-obvious. `InvalidatePlot(false)` silently renders stale data. This is documented in CLAUDE.md but remains a footgun for future changes. + +### Follow-up Work + +- If datasets exceed the 50M point memory cap, consider hybrid approach: keep a coarse in-memory overview + on-demand DB queries for zoomed-in detail +- Monitor whether the `GetRange()` copy in the "few enough points" path becomes a bottleneck — could be replaced with a `ListSegment` wrapper if needed +- PR #457 should be closed as superseded diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 00000000..2859d2fe --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,21 @@ +# Architecture Decision Records (ADRs) + +This directory captures significant architectural decisions for the DAQiFi Desktop application. + +Each ADR documents the context, decision, alternatives considered, and consequences of a technical choice. They serve as a historical record so future contributors understand *why* something was built a certain way, not just *how*. + +## Format + +ADRs follow a lightweight template: + +- **Status**: Accepted, Superseded, or Deprecated +- **Context**: The problem or situation that prompted the decision +- **Decision**: What we chose to do +- **Alternatives Considered**: What else we evaluated and why we rejected it +- **Consequences**: Trade-offs, risks, and follow-up work + +## Index + +| # | Title | Status | Date | +|---|-------|--------|------| +| [001](001-viewport-aware-downsampling.md) | Viewport-aware downsampling for large dataset rendering | Accepted | 2026-04 | From 4f4898e53cef118050c8dcb352d51e1148c196df Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Sat, 11 Apr 2026 14:20:39 -0600 Subject: [PATCH 08/19] docs: trim CLAUDE.md plot section to concise gotchas with ADR pointer Reduces context window usage by replacing detailed explanations with bullet-point gotchas and linking to ADR 001 for the full rationale. Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index b6e68857..864fff09 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -255,19 +255,12 @@ When working on: ### Plot Rendering (OxyPlot) -The logged data viewer uses viewport-aware MinMax downsampling to handle large datasets (16+ channels × 1M+ points). Key architecture decisions and gotchas: +The logged data viewer uses viewport-aware downsampling for 60fps interaction with large datasets. See [ADR 001](docs/adr/001-viewport-aware-downsampling.md) for full context. Key gotchas when modifying plot code: -**Viewport-aware downsampling over global decimation**: Global downsampling (e.g., LTTB to ~5000 points) conflicts with the minimap — when zoomed into a 1-minute slice of 24 hours, only ~3 points would be visible. Instead, we binary search the visible range and downsample only that slice to ~4000 points per channel. This gives full detail when zoomed in. See `MinMaxDownsampler.FindVisibleRange()` and `DatabaseLogger.UpdateMainPlotViewport()`. - -**OxyPlot `InvalidatePlot(true)` vs `InvalidatePlot(false)`**: `false` re-renders from OxyPlot's internal cached point arrays. `true` forces OxyPlot to re-read `ItemsSource` and rebuild those arrays. You MUST use `true` whenever you change a series' `ItemsSource` or mutate the underlying list — otherwise the plot renders stale data. This was the root cause of a bug where zoom + minimap drag showed missing data. - -**MinMax downsampling does NOT preserve original X boundaries**: Downsampled data emits points at min/max Y positions within each bucket, not at bucket edges. So the first and last X values of downsampled data may differ from the original data. This means `ResetAllAxes()` (which auto-ranges from current `ItemsSource`) can progressively shrink the visible range. Fix: explicitly compute the full time range from source data and use `axis.Zoom(min, max)` instead of auto-range. - -**Feedback loops between minimap and main plot**: The minimap syncs bidirectionally with the main plot's time axis. Without a guard flag (`IsSyncingFromMinimap`), dragging the minimap triggers `AxisChanged` on the main plot, which updates the minimap, creating a render loop. Always set the guard before programmatic axis changes. - -**GC pressure during interaction**: Allocating new `List` per channel per frame (~960/sec at 60fps with 16 channels) causes Gen0 GC micro-stutters. Reuse cached lists per series key — clear and refill instead of allocating new ones. See `_downsampledCache` in `DatabaseLogger`. - -**Throttle pattern**: Both minimap drag and main plot pan/zoom use a `DispatcherTimer` (16ms / 60fps) + dirty flag pattern. Mouse events set the flag; the timer tick does the actual work. This caps expensive operations (re-downsample + render) at 60Hz regardless of input event frequency. +- **`InvalidatePlot(true)` vs `(false)`**: Use `true` whenever `ItemsSource` or its underlying list has changed — `false` renders stale cached data +- **Don't use `ResetAllAxes()` with downsampled data**: Auto-range reads from `ItemsSource`, which may have shifted X boundaries. Use explicit `axis.Zoom(min, max)` from source data +- **Guard flag for minimap sync**: Always set `IsSyncingFromMinimap` before programmatic axis changes to prevent feedback loops +- **Reuse cached lists**: Don't allocate new `List` per frame — use `_downsampledCache` ## Error Handling From 7a1a85a3e3882c27ff51959d7eae1a92b7cdc60f Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Sat, 11 Apr 2026 14:26:13 -0600 Subject: [PATCH 09/19] =?UTF-8?q?fix:=20address=20Qodo=20review=20?= =?UTF-8?q?=E2=80=94=20stream=20samples,=20reduce=20cap,=20add=20Dispose?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Stream samples via AsEnumerable() instead of ToList() to avoid materializing a large intermediate anonymous object list (OOM risk) 2. Reduce MAX_IN_MEMORY_POINTS from 50M to 10M (~160MB) as a more defensible default for desktop memory constraints 3. Separate channel metadata query from sample streaming 4. Add IDisposable to DatabaseLogger — stops viewport throttle timer, disposes minimap interaction controller, buffer, and consumer gate Co-Authored-By: Claude Opus 4.6 --- Daqifi.Desktop/Loggers/DatabaseLogger.cs | 40 +++++++++++++++++------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/Daqifi.Desktop/Loggers/DatabaseLogger.cs b/Daqifi.Desktop/Loggers/DatabaseLogger.cs index c7232110..170e4cba 100644 --- a/Daqifi.Desktop/Loggers/DatabaseLogger.cs +++ b/Daqifi.Desktop/Loggers/DatabaseLogger.cs @@ -100,12 +100,12 @@ public LoggedSeriesLegendItem( } } -public partial class DatabaseLogger : ObservableObject, ILogger +public partial class DatabaseLogger : ObservableObject, ILogger, IDisposable { #region Constants private const int MINIMAP_BUCKET_COUNT = 800; private const int MAIN_PLOT_BUCKET_COUNT = 2000; - private const int MAX_IN_MEMORY_POINTS = 50_000_000; + private const int MAX_IN_MEMORY_POINTS = 10_000_000; #endregion #region Private Data @@ -461,16 +461,11 @@ public void DisplayLoggingSession(LoggingSession session) subtitle = $"\nShowing first {MAX_IN_MEMORY_POINTS:n0} of {totalSamplesCount:n0} data points"; } - // Only materialize up to the limit to avoid excessive memory usage - var dbSamples = baseQuery - .OrderBy(s => s.TimestampTicks) - .Select(s => new { s.ChannelName, s.DeviceSerialNo, s.Type, s.Color, s.TimestampTicks, s.Value }) - .Take(MAX_IN_MEMORY_POINTS) - .ToList(); - - var channelInfoList = dbSamples + // Query channel metadata first (small result set) + var channelInfoList = baseQuery .Select(s => new { s.ChannelName, s.DeviceSerialNo, s.Type, s.Color }) .Distinct() + .ToList() .NaturalOrderBy(s => s.ChannelName) .ToList(); @@ -481,7 +476,14 @@ public void DisplayLoggingSession(LoggingSession session) tempLegendItemsList.Add(legendItem); } - foreach (var sample in dbSamples) + // Stream samples directly into per-channel lists to avoid + // materializing a massive intermediate list (addresses OOM risk) + var sampleCount = 0; + foreach (var sample in baseQuery + .OrderBy(s => s.TimestampTicks) + .Select(s => new { s.ChannelName, s.DeviceSerialNo, s.TimestampTicks, s.Value }) + .Take(MAX_IN_MEMORY_POINTS) + .AsEnumerable()) { var key = (sample.DeviceSerialNo, sample.ChannelName); if (_firstTime == null) { _firstTime = new DateTime(sample.TimestampTicks); } @@ -491,6 +493,8 @@ public void DisplayLoggingSession(LoggingSession session) { points.Add(new DataPoint(deltaTime, sample.Value)); } + + sampleCount++; } } @@ -934,4 +938,18 @@ private void ZoomInY() PlotModel.InvalidatePlot(true); } #endregion + + #region IDisposable + /// + /// Stops timers and unsubscribes event handlers to prevent leaks. + /// + public void Dispose() + { + _viewportThrottleTimer.Stop(); + _viewportThrottleTimer.Tick -= OnViewportThrottleTick; + _minimapInteraction?.Dispose(); + _buffer.Dispose(); + _consumerGate.Dispose(); + } + #endregion } \ No newline at end of file From a90e38f51ebf368f948c8ce419000259470fa0fb Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Sat, 11 Apr 2026 14:27:52 -0600 Subject: [PATCH 10/19] fix: address remaining Qodo review items (4-6) 4. Wrap IsSyncingFromMinimap flag set/unset in try/finally to ensure the flag is reset even if Axis.Zoom() throws 5. Relax perf test threshold from 100ms to 1000ms to prevent flaky failures on slow CI runners 6. Add ArgumentOutOfRangeException guards on startIndex/endIndex in MinMaxDownsampler.Downsample sub-range overload Co-Authored-By: Claude Opus 4.6 --- Daqifi.Desktop.Test/Helpers/MinMaxDownsamplerTests.cs | 7 ++++--- Daqifi.Desktop/Helpers/MinMaxDownsampler.cs | 2 ++ Daqifi.Desktop/View/MinimapInteractionController.cs | 10 ++++++++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/Daqifi.Desktop.Test/Helpers/MinMaxDownsamplerTests.cs b/Daqifi.Desktop.Test/Helpers/MinMaxDownsamplerTests.cs index d9fcdb62..332f203c 100644 --- a/Daqifi.Desktop.Test/Helpers/MinMaxDownsamplerTests.cs +++ b/Daqifi.Desktop.Test/Helpers/MinMaxDownsamplerTests.cs @@ -198,9 +198,10 @@ public void FindVisibleRange_LargeDataset_Performance() } sw.Stop(); - // 1000 binary searches on 1M points should be well under 100ms - Assert.IsTrue(sw.ElapsedMilliseconds < 100, - $"1000 binary searches took {sw.ElapsedMilliseconds}ms, expected < 100ms"); + // 1000 binary searches on 1M points should be well under 1 second, + // even on slow CI runners. Typical desktop: < 10ms. + Assert.IsTrue(sw.ElapsedMilliseconds < 1000, + $"1000 binary searches took {sw.ElapsedMilliseconds}ms, expected < 1000ms"); } #endregion diff --git a/Daqifi.Desktop/Helpers/MinMaxDownsampler.cs b/Daqifi.Desktop/Helpers/MinMaxDownsampler.cs index 499303e4..e6456efa 100644 --- a/Daqifi.Desktop/Helpers/MinMaxDownsampler.cs +++ b/Daqifi.Desktop/Helpers/MinMaxDownsampler.cs @@ -39,6 +39,8 @@ public static List Downsample(IReadOnlyList points, int bu public static List Downsample(IReadOnlyList points, int startIndex, int endIndex, int bucketCount) { ArgumentNullException.ThrowIfNull(points); + ArgumentOutOfRangeException.ThrowIfNegative(startIndex); + ArgumentOutOfRangeException.ThrowIfGreaterThan(endIndex, points.Count); var count = endIndex - startIndex; if (count <= 0 || bucketCount <= 0) diff --git a/Daqifi.Desktop/View/MinimapInteractionController.cs b/Daqifi.Desktop/View/MinimapInteractionController.cs index f252e2fe..d1e6910a 100644 --- a/Daqifi.Desktop/View/MinimapInteractionController.cs +++ b/Daqifi.Desktop/View/MinimapInteractionController.cs @@ -323,8 +323,14 @@ private void ApplyToMainPlot(double min, double max) } _databaseLogger.IsSyncingFromMinimap = true; - mainTimeAxis.Zoom(min, max); - _databaseLogger.IsSyncingFromMinimap = false; + try + { + mainTimeAxis.Zoom(min, max); + } + finally + { + _databaseLogger.IsSyncingFromMinimap = false; + } _dimLeft.MaximumX = min; _dimRight.MinimumX = max; From 8c5b8db273ffd544b55cac1f88946a88fa86c17b Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Sat, 11 Apr 2026 21:43:56 -0600 Subject: [PATCH 11/19] perf: two-phase progressive loading for large sessions Session 2 (18M samples, 32 channels) was taking ~30s to load because the entire dataset was read from SQLite before showing anything. Phase 1 (<1s): Get channel metadata from the first timestamp (6ms via composite index), load first 100K samples (16ms via index scan), and display immediately. The user sees data in under a second. Phase 2 (background): Stream remaining samples up to the 10M cap, then refresh the minimap and main plot with full-fidelity data. Key insight: SQLite's composite index on (LoggingSessionID, TimestampTicks) makes LIMIT queries nearly instant, but DISTINCT/COUNT/full scans over 18M rows are inherently slow (5-15s). By showing partial data first, perceived load time drops from 30s to <1s. Also extracted helper methods (PrepareMinimapData, SetupUiCollections, SetupMinimapSeries) to reduce duplication between phases. Co-Authored-By: Claude Opus 4.6 --- Daqifi.Desktop/Loggers/DatabaseLogger.cs | 290 ++++++++++++++++------- 1 file changed, 199 insertions(+), 91 deletions(-) diff --git a/Daqifi.Desktop/Loggers/DatabaseLogger.cs b/Daqifi.Desktop/Loggers/DatabaseLogger.cs index 170e4cba..916b0d88 100644 --- a/Daqifi.Desktop/Loggers/DatabaseLogger.cs +++ b/Daqifi.Desktop/Loggers/DatabaseLogger.cs @@ -106,6 +106,7 @@ public partial class DatabaseLogger : ObservableObject, ILogger, IDisposable private const int MINIMAP_BUCKET_COUNT = 800; private const int MAIN_PLOT_BUCKET_COUNT = 2000; private const int MAX_IN_MEMORY_POINTS = 10_000_000; + private const int INITIAL_LOAD_POINTS = 100_000; #endregion #region Private Data @@ -440,13 +441,15 @@ public void DisplayLoggingSession(LoggingSession session) // ClearPlot is already dispatcher-wrapped ClearPlot(); - // Data fetching and processing (can be on background thread) var sessionName = session.Name; var subtitle = string.Empty; - var tempSeriesList = new List(); var tempLegendItemsList = new List(); + int totalSamplesCount; + // ── Phase 1: Fast initial load (<1s) ────────────────────────── + // Get channel metadata from first timestamp (6ms via index) + // and load a small initial batch for immediate display using (var context = _loggingContext.CreateDbContext()) { context.ChangeTracker.AutoDetectChangesEnabled = false; @@ -454,17 +457,28 @@ public void DisplayLoggingSession(LoggingSession session) var baseQuery = context.Samples.AsNoTracking() .Where(s => s.LoggingSessionID == session.ID); - var totalSamplesCount = baseQuery.Count(); + // Get the first timestamp to extract channel info (instant via composite index) + var firstSample = baseQuery + .OrderBy(s => s.TimestampTicks) + .Select(s => new { s.TimestampTicks }) + .FirstOrDefault(); - if (totalSamplesCount > MAX_IN_MEMORY_POINTS) + if (firstSample == null) { - subtitle = $"\nShowing first {MAX_IN_MEMORY_POINTS:n0} of {totalSamplesCount:n0} data points"; + // Empty session + Application.Current.Dispatcher.Invoke(() => + { + PlotModel.Title = sessionName; + HasSessionData = false; + PlotModel.InvalidatePlot(true); + }); + return; } - // Query channel metadata first (small result set) + // Get all channels from the first timestamp (32 rows, instant) var channelInfoList = baseQuery + .Where(s => s.TimestampTicks == firstSample.TimestampTicks) .Select(s => new { s.ChannelName, s.DeviceSerialNo, s.Type, s.Color }) - .Distinct() .ToList() .NaturalOrderBy(s => s.ChannelName) .ToList(); @@ -476,13 +490,11 @@ public void DisplayLoggingSession(LoggingSession session) tempLegendItemsList.Add(legendItem); } - // Stream samples directly into per-channel lists to avoid - // materializing a massive intermediate list (addresses OOM risk) - var sampleCount = 0; + // Load initial batch for fast display (100K rows, ~16ms via index) foreach (var sample in baseQuery .OrderBy(s => s.TimestampTicks) .Select(s => new { s.ChannelName, s.DeviceSerialNo, s.TimestampTicks, s.Value }) - .Take(MAX_IN_MEMORY_POINTS) + .Take(INITIAL_LOAD_POINTS) .AsEnumerable()) { var key = (sample.DeviceSerialNo, sample.ChannelName); @@ -493,120 +505,216 @@ public void DisplayLoggingSession(LoggingSession session) { points.Add(new DataPoint(deltaTime, sample.Value)); } - - sampleCount++; } - } - // Prepare downsampled minimap data on the background thread - var minimapSeriesData = new List<(string channelName, string deviceSerial, OxyColor color, List downsampled)>(); - foreach (var kvp in _allSessionPoints) - { - if (kvp.Value.Count > 0) - { - var downsampled = MinMaxDownsampler.Downsample(kvp.Value, MINIMAP_BUCKET_COUNT); - var matchingSeries = tempSeriesList.FirstOrDefault(s => - s.Tag is (string ds, string cn) && ds == kvp.Key.deviceSerial && cn == kvp.Key.channelName); - minimapSeriesData.Add((kvp.Key.channelName, kvp.Key.deviceSerial, matchingSeries?.Color ?? OxyColors.Gray, downsampled)); - } + totalSamplesCount = baseQuery.Count(); } - // Update UI-bound collections and properties on the UI thread + // Show the initial data immediately + var initialMinimapData = PrepareMinimapData(tempSeriesList); Application.Current.Dispatcher.Invoke(() => { PlotModel.Title = sessionName; - PlotModel.Subtitle = subtitle; + PlotModel.Subtitle = totalSamplesCount > INITIAL_LOAD_POINTS + ? "\nLoading full dataset..." + : string.Empty; + + SetupUiCollections(tempSeriesList, tempLegendItemsList); + SetupMinimapSeries(initialMinimapData); + HasSessionData = tempSeriesList.Count > 0; + PlotModel.InvalidatePlot(true); + }); - foreach (var legendItem in tempLegendItemsList) + // ── Phase 2: Load remaining data in background ──────────────── + if (totalSamplesCount > INITIAL_LOAD_POINTS) + { + if (totalSamplesCount > MAX_IN_MEMORY_POINTS) { - LegendItems.Add(legendItem); + subtitle = $"\nShowing first {MAX_IN_MEMORY_POINTS:n0} of {totalSamplesCount:n0} data points"; } - // Build grouped legend by device - DeviceLegendGroups.Clear(); - var groupDict = new Dictionary(); - foreach (var legendItem in tempLegendItemsList) + // Clear phase 1 data and reload the full set + foreach (var kvp in _allSessionPoints) { - if (!groupDict.TryGetValue(legendItem.DeviceSerialNo, out var group)) - { - group = new DeviceLegendGroup(legendItem.DeviceSerialNo); - groupDict[legendItem.DeviceSerialNo] = group; - DeviceLegendGroups.Add(group); - } - group.Channels.Add(legendItem); + kvp.Value.Clear(); } + _firstTime = null; - foreach (var series in tempSeriesList) + using (var context = _loggingContext.CreateDbContext()) { - PlotModel.Series.Add(series); - if (series.Tag is (string deviceSerial, string channelName) - && _allSessionPoints.TryGetValue((deviceSerial, channelName), out var points)) + context.ChangeTracker.AutoDetectChangesEnabled = false; + + foreach (var sample in context.Samples.AsNoTracking() + .Where(s => s.LoggingSessionID == session.ID) + .OrderBy(s => s.TimestampTicks) + .Select(s => new { s.ChannelName, s.DeviceSerialNo, s.TimestampTicks, s.Value }) + .Take(MAX_IN_MEMORY_POINTS) + .AsEnumerable()) { - // Set initial ItemsSource to downsampled full range using cached list - var key = (deviceSerial, channelName); - if (!_downsampledCache.TryGetValue(key, out var cached)) - { - cached = new List(MAIN_PLOT_BUCKET_COUNT * 2); - _downsampledCache[key] = cached; - } + var key = (sample.DeviceSerialNo, sample.ChannelName); + if (_firstTime == null) { _firstTime = new DateTime(sample.TimestampTicks); } + var deltaTime = (sample.TimestampTicks - _firstTime.Value.Ticks) / 10000.0; - cached.Clear(); - if (points.Count > MAIN_PLOT_BUCKET_COUNT * 2) + if (_allSessionPoints.TryGetValue(key, out var points)) { - cached.AddRange(MinMaxDownsampler.Downsample(points, MAIN_PLOT_BUCKET_COUNT)); + points.Add(new DataPoint(deltaTime, sample.Value)); } - else - { - cached.AddRange(points); - } - - series.ItemsSource = cached; } } - // Populate minimap with downsampled series - MinimapPlotModel.Series.Clear(); - _minimapSeries.Clear(); - foreach (var (channelName, deviceSerial, color, downsampled) in minimapSeriesData) + // Refresh UI with full data + var fullMinimapData = PrepareMinimapData(tempSeriesList); + Application.Current.Dispatcher.Invoke(() => { - var minimapLine = new LineSeries + PlotModel.Subtitle = subtitle; + + // Update main plot series with full downsampled data + foreach (var series in PlotModel.Series.OfType()) { - Color = color, - StrokeThickness = 1, - ItemsSource = downsampled, - XAxisKey = "MinimapTime", - YAxisKey = "MinimapY" - }; - MinimapPlotModel.Series.Add(minimapLine); - _minimapSeries[(deviceSerial, channelName)] = minimapLine; - } + if (series.Tag is (string deviceSerial, string channelName) + && _allSessionPoints.TryGetValue((deviceSerial, channelName), out var points)) + { + var key = (deviceSerial, channelName); + if (!_downsampledCache.TryGetValue(key, out var cached)) + { + cached = new List(MAIN_PLOT_BUCKET_COUNT * 2); + _downsampledCache[key] = cached; + } + + cached.Clear(); + if (points.Count > MAIN_PLOT_BUCKET_COUNT * 2) + { + cached.AddRange(MinMaxDownsampler.Downsample(points, MAIN_PLOT_BUCKET_COUNT)); + } + else + { + cached.AddRange(points); + } + + series.ItemsSource = cached; + } + } - MinimapPlotModel.ResetAllAxes(); + // Refresh minimap with full data + SetupMinimapSeries(fullMinimapData); + _lastViewportMin = double.NaN; + _lastViewportMax = double.NaN; + PlotModel.InvalidatePlot(true); + }); + } + } + catch (Exception ex) + { + _appLogger.Error(ex, "Failed in DisplayLoggingSession"); + } + } - // Initialize selection rectangle to full data range - // Use data bounds directly since ActualMinimum/Maximum aren't set until render - if (minimapSeriesData.Count > 0) + /// + /// Prepares downsampled minimap series data from _allSessionPoints on the background thread. + /// + private List<(string channelName, string deviceSerial, OxyColor color, List downsampled)> + PrepareMinimapData(List seriesList) + { + var result = new List<(string channelName, string deviceSerial, OxyColor color, List downsampled)>(); + foreach (var kvp in _allSessionPoints) + { + if (kvp.Value.Count > 0) + { + var downsampled = MinMaxDownsampler.Downsample(kvp.Value, MINIMAP_BUCKET_COUNT); + var matchingSeries = seriesList.FirstOrDefault(s => + s.Tag is (string ds, string cn) && ds == kvp.Key.deviceSerial && cn == kvp.Key.channelName); + result.Add((kvp.Key.channelName, kvp.Key.deviceSerial, matchingSeries?.Color ?? OxyColors.Gray, downsampled)); + } + } + return result; + } + + /// + /// Sets up UI collections (legend items, device groups, series) on the UI thread. + /// + private void SetupUiCollections(List seriesList, List legendItems) + { + foreach (var legendItem in legendItems) + { + LegendItems.Add(legendItem); + } + + DeviceLegendGroups.Clear(); + var groupDict = new Dictionary(); + foreach (var legendItem in legendItems) + { + if (!groupDict.TryGetValue(legendItem.DeviceSerialNo, out var group)) + { + group = new DeviceLegendGroup(legendItem.DeviceSerialNo); + groupDict[legendItem.DeviceSerialNo] = group; + DeviceLegendGroups.Add(group); + } + group.Channels.Add(legendItem); + } + + foreach (var series in seriesList) + { + PlotModel.Series.Add(series); + if (series.Tag is (string deviceSerial, string channelName) + && _allSessionPoints.TryGetValue((deviceSerial, channelName), out var points)) + { + var key = (deviceSerial, channelName); + if (!_downsampledCache.TryGetValue(key, out var cached)) { - var dataMinX = minimapSeriesData.Where(d => d.downsampled.Count > 0).Min(d => d.downsampled[0].X); - var dataMaxX = minimapSeriesData.Where(d => d.downsampled.Count > 0).Max(d => d.downsampled[^1].X); - _minimapSelectionRect.MinimumX = dataMinX; - _minimapSelectionRect.MaximumX = dataMaxX; - _minimapDimLeft.MaximumX = dataMinX; - _minimapDimRight.MinimumX = dataMaxX; + cached = new List(MAIN_PLOT_BUCKET_COUNT * 2); + _downsampledCache[key] = cached; } - MinimapPlotModel.InvalidatePlot(true); + cached.Clear(); + if (points.Count > MAIN_PLOT_BUCKET_COUNT * 2) + { + cached.AddRange(MinMaxDownsampler.Downsample(points, MAIN_PLOT_BUCKET_COUNT)); + } + else + { + cached.AddRange(points); + } - HasSessionData = tempSeriesList.Count > 0; + series.ItemsSource = cached; + } + } + } - OnPropertyChanged("SessionPoints"); // If SessionPoints is still relevant - PlotModel.InvalidatePlot(true); - }); + /// + /// Populates minimap series and sets up the selection rectangle on the UI thread. + /// + private void SetupMinimapSeries( + List<(string channelName, string deviceSerial, OxyColor color, List downsampled)> minimapData) + { + MinimapPlotModel.Series.Clear(); + _minimapSeries.Clear(); + foreach (var (channelName, deviceSerial, color, downsampled) in minimapData) + { + var minimapLine = new LineSeries + { + Color = color, + StrokeThickness = 1, + ItemsSource = downsampled, + XAxisKey = "MinimapTime", + YAxisKey = "MinimapY" + }; + MinimapPlotModel.Series.Add(minimapLine); + _minimapSeries[(deviceSerial, channelName)] = minimapLine; } - catch (Exception ex) + + MinimapPlotModel.ResetAllAxes(); + + if (minimapData.Count > 0) { - _appLogger.Error(ex, "Failed in DisplayLoggingSession"); + var dataMinX = minimapData.Where(d => d.downsampled.Count > 0).Min(d => d.downsampled[0].X); + var dataMaxX = minimapData.Where(d => d.downsampled.Count > 0).Max(d => d.downsampled[^1].X); + _minimapSelectionRect.MinimumX = dataMinX; + _minimapSelectionRect.MaximumX = dataMaxX; + _minimapDimLeft.MaximumX = dataMinX; + _minimapDimRight.MinimumX = dataMaxX; } + + MinimapPlotModel.InvalidatePlot(true); } public void DeleteLoggingSession(LoggingSession session) From 700e52d487d4bd9117653d2e7dc0ec7c4b4c3a57 Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Sat, 11 Apr 2026 21:52:06 -0600 Subject: [PATCH 12/19] perf: replace full data scan with sampled index seeks for Phase 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2 was streaming all 10M+ rows sequentially (~30s). Now uses targeted index seeks: divides the time range into 3000 segments and seeks to each segment boundary via the composite index. Each seek reads one batch of interleaved channel data (~32 rows). Result: ~96K rows covering the full time range in ~1-3 seconds, regardless of total dataset size (18M, 100M, doesn't matter). Benchmark on 18M-sample session: - Before: ~30s (sequential scan of 10M rows through EF Core) - After: ~1-3s (3000 index seeks × 32 rows via raw ADO.NET) Uses raw ADO.NET with a prepared statement for minimal per-query overhead instead of EF Core materialization. Co-Authored-By: Claude Opus 4.6 --- Daqifi.Desktop/Loggers/DatabaseLogger.cs | 152 ++++++++++++++++++----- 1 file changed, 120 insertions(+), 32 deletions(-) diff --git a/Daqifi.Desktop/Loggers/DatabaseLogger.cs b/Daqifi.Desktop/Loggers/DatabaseLogger.cs index 916b0d88..a3a4a81e 100644 --- a/Daqifi.Desktop/Loggers/DatabaseLogger.cs +++ b/Daqifi.Desktop/Loggers/DatabaseLogger.cs @@ -105,8 +105,8 @@ public partial class DatabaseLogger : ObservableObject, ILogger, IDisposable #region Constants private const int MINIMAP_BUCKET_COUNT = 800; private const int MAIN_PLOT_BUCKET_COUNT = 2000; - private const int MAX_IN_MEMORY_POINTS = 10_000_000; private const int INITIAL_LOAD_POINTS = 100_000; + private const int SAMPLED_POINTS_PER_CHANNEL = 3000; #endregion #region Private Data @@ -525,48 +525,27 @@ public void DisplayLoggingSession(LoggingSession session) PlotModel.InvalidatePlot(true); }); - // ── Phase 2: Load remaining data in background ──────────────── + // ── Phase 2: Load sampled data covering full time range (~1-3s) ── + // Instead of streaming all 10M+ rows (30s), do N targeted index + // seeks spread across the time range. Each seek reads one batch + // of interleaved channel data at that timestamp position. + // Result: ~96K rows covering the full range in ~1-3 seconds. if (totalSamplesCount > INITIAL_LOAD_POINTS) { - if (totalSamplesCount > MAX_IN_MEMORY_POINTS) - { - subtitle = $"\nShowing first {MAX_IN_MEMORY_POINTS:n0} of {totalSamplesCount:n0} data points"; - } - - // Clear phase 1 data and reload the full set + // Clear phase 1 data and reload with sampled data foreach (var kvp in _allSessionPoints) { kvp.Value.Clear(); } _firstTime = null; - using (var context = _loggingContext.CreateDbContext()) - { - context.ChangeTracker.AutoDetectChangesEnabled = false; - - foreach (var sample in context.Samples.AsNoTracking() - .Where(s => s.LoggingSessionID == session.ID) - .OrderBy(s => s.TimestampTicks) - .Select(s => new { s.ChannelName, s.DeviceSerialNo, s.TimestampTicks, s.Value }) - .Take(MAX_IN_MEMORY_POINTS) - .AsEnumerable()) - { - var key = (sample.DeviceSerialNo, sample.ChannelName); - if (_firstTime == null) { _firstTime = new DateTime(sample.TimestampTicks); } - var deltaTime = (sample.TimestampTicks - _firstTime.Value.Ticks) / 10000.0; - - if (_allSessionPoints.TryGetValue(key, out var points)) - { - points.Add(new DataPoint(deltaTime, sample.Value)); - } - } - } + LoadSampledData(session.ID, tempSeriesList.Count); - // Refresh UI with full data + // Refresh UI with sampled full-range data var fullMinimapData = PrepareMinimapData(tempSeriesList); Application.Current.Dispatcher.Invoke(() => { - PlotModel.Subtitle = subtitle; + PlotModel.Subtitle = string.Empty; // Update main plot series with full downsampled data foreach (var series in PlotModel.Series.OfType()) @@ -595,7 +574,7 @@ public void DisplayLoggingSession(LoggingSession session) } } - // Refresh minimap with full data + // Refresh minimap with full-range data SetupMinimapSeries(fullMinimapData); _lastViewportMin = double.NaN; _lastViewportMax = double.NaN; @@ -629,6 +608,115 @@ public void DisplayLoggingSession(LoggingSession session) return result; } + /// + /// Loads a uniformly sampled subset of data covering the full time range + /// using targeted index seeks. Instead of reading all N million rows, + /// divides the time range into SAMPLED_POINTS_PER_CHANNEL segments and + /// seeks to each segment boundary via the composite index. Each seek + /// reads one batch of interleaved channel data (~channelCount rows). + /// Result: ~3000 points per channel in ~1-3 seconds regardless of total dataset size. + /// + private void LoadSampledData(int sessionId, int channelCount) + { + using var context = _loggingContext.CreateDbContext(); + var connection = context.Database.GetDbConnection(); + connection.Open(); + + // Get time bounds via index (instant) + long minTicks, maxTicks; + using (var boundsCmd = connection.CreateCommand()) + { + boundsCmd.CommandText = @" + SELECT MIN(TimestampTicks), MAX(TimestampTicks) + FROM Samples + WHERE LoggingSessionID = @id"; + var idParam = boundsCmd.CreateParameter(); + idParam.ParameterName = "@id"; + idParam.Value = sessionId; + boundsCmd.Parameters.Add(idParam); + + using var reader = boundsCmd.ExecuteReader(); + if (!reader.Read() || reader.IsDBNull(0)) + { + return; + } + + minTicks = reader.GetInt64(0); + maxTicks = reader.GetInt64(1); + } + + if (minTicks >= maxTicks) + { + return; + } + + _firstTime = new DateTime(minTicks); + var tickStep = (maxTicks - minTicks) / SAMPLED_POINTS_PER_CHANNEL; + // Read at least channelCount rows per seek to get one sample per channel + var batchSize = Math.Max(channelCount * 2, 100); + + // Prepared statement for repeated seeks + using var seekCmd = connection.CreateCommand(); + seekCmd.CommandText = @" + SELECT ChannelName, DeviceSerialNo, TimestampTicks, Value + FROM Samples + WHERE LoggingSessionID = @id AND TimestampTicks >= @t + ORDER BY TimestampTicks + LIMIT @limit"; + + var seekIdParam = seekCmd.CreateParameter(); + seekIdParam.ParameterName = "@id"; + seekIdParam.Value = sessionId; + seekCmd.Parameters.Add(seekIdParam); + + var seekTParam = seekCmd.CreateParameter(); + seekTParam.ParameterName = "@t"; + seekTParam.Value = minTicks; + seekCmd.Parameters.Add(seekTParam); + + var seekLimitParam = seekCmd.CreateParameter(); + seekLimitParam.ParameterName = "@limit"; + seekLimitParam.Value = batchSize; + seekCmd.Parameters.Add(seekLimitParam); + + seekCmd.Prepare(); + + // Track which timestamps we've already added to avoid duplicates + // from overlapping batches + var lastAddedTimestamp = new Dictionary<(string, string), long>(); + + for (var i = 0; i < SAMPLED_POINTS_PER_CHANNEL; i++) + { + var seekTimestamp = minTicks + i * tickStep; + seekTParam.Value = seekTimestamp; + + using var reader = seekCmd.ExecuteReader(); + while (reader.Read()) + { + var channelName = reader.GetString(0); + var deviceSerialNo = reader.GetString(1); + var timestampTicks = reader.GetInt64(2); + var value = reader.GetDouble(3); + + var key = (deviceSerialNo, channelName); + + // Skip duplicate timestamps from overlapping batches + if (lastAddedTimestamp.TryGetValue(key, out var lastT) && timestampTicks <= lastT) + { + continue; + } + + lastAddedTimestamp[key] = timestampTicks; + + var deltaTime = (timestampTicks - _firstTime.Value.Ticks) / 10000.0; + if (_allSessionPoints.TryGetValue(key, out var points)) + { + points.Add(new DataPoint(deltaTime, value)); + } + } + } + } + /// /// Sets up UI collections (legend items, device groups, series) on the UI thread. /// From 79262b37b7278da857070eedec958ae868b6121d Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Sat, 11 Apr 2026 21:59:47 -0600 Subject: [PATCH 13/19] feat: on-demand DB fetch for full-resolution data when zoomed in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the user zooms into a narrow time window, the sampled in-memory data (~3000 points/channel) becomes too sparse to show the true waveform. Now detects this condition and fetches full-resolution data directly from the database for just the visible window. How it works: - UpdateMainPlotViewport() checks if the visible range of sampled data has fewer points than MAIN_PLOT_BUCKET_COUNT (2000) - If so, FetchViewportDataFromDb() queries the DB using the composite index: WHERE TimestampTicks BETWEEN @min AND @max - The fetched data is downsampled if needed and displayed - Falls back to in-memory data when zoomed out (sufficient density) Performance: a 1-second window at 1000Hz × 32 channels = ~32K rows, which takes ~16ms to read via the composite index. Stays well within the 60fps frame budget. Co-Authored-By: Claude Opus 4.6 --- Daqifi.Desktop/Loggers/DatabaseLogger.cs | 170 ++++++++++++++++++++++- 1 file changed, 167 insertions(+), 3 deletions(-) diff --git a/Daqifi.Desktop/Loggers/DatabaseLogger.cs b/Daqifi.Desktop/Loggers/DatabaseLogger.cs index a3a4a81e..14bcb574 100644 --- a/Daqifi.Desktop/Loggers/DatabaseLogger.cs +++ b/Daqifi.Desktop/Loggers/DatabaseLogger.cs @@ -130,6 +130,7 @@ public partial class DatabaseLogger : ObservableObject, ILogger, IDisposable private double _lastViewportMax = double.NaN; private bool _viewportDirty; private DispatcherTimer _viewportThrottleTimer; + private int? _currentSessionId; [ObservableProperty] private PlotModel _plotModel; @@ -415,6 +416,7 @@ public void ClearPlot() Application.Current.Dispatcher.Invoke(() => { _firstTime = null; + _currentSessionId = null; _lastViewportMin = double.NaN; _lastViewportMax = double.NaN; _allSessionPoints.Clear(); @@ -440,6 +442,7 @@ public void DisplayLoggingSession(LoggingSession session) { // ClearPlot is already dispatcher-wrapped ClearPlot(); + _currentSessionId = session.ID; var sessionName = session.Name; var subtitle = string.Empty; @@ -955,6 +958,8 @@ private void OnViewportThrottleTick(object? sender, EventArgs e) /// /// Re-downsamples each main plot series for the currently visible time range. + /// When zoomed in far enough that the sampled in-memory data is too sparse, + /// fetches full-resolution data from the database for the visible window. /// Skips the update if the viewport hasn't changed since the last call. /// private void UpdateMainPlotViewport() @@ -977,6 +982,47 @@ private void UpdateMainPlotViewport() _lastViewportMin = visibleMin; _lastViewportMax = visibleMax; + // Check if sampled in-memory data is too sparse for this zoom level. + // If fewer sampled points are visible than our target density, fetch + // full-resolution data from the DB for this window. + var needsDbFetch = false; + if (_currentSessionId.HasValue && _firstTime.HasValue) + { + foreach (var kvp in _allSessionPoints) + { + if (kvp.Value.Count == 0) + { + continue; + } + + var (si, ei) = MinMaxDownsampler.FindVisibleRange(kvp.Value, visibleMin, visibleMax); + var sampledVisible = ei - si; + // Sampled data is sparse if we have fewer points than target AND + // the in-memory data is actually sampled (not the full dataset) + if (sampledVisible < MAIN_PLOT_BUCKET_COUNT && kvp.Value.Count >= SAMPLED_POINTS_PER_CHANNEL / 2) + { + needsDbFetch = true; + } + break; + } + } + + if (needsDbFetch) + { + FetchViewportDataFromDb(visibleMin, visibleMax); + } + else + { + UpdateSeriesFromMemory(visibleMin, visibleMax); + } + } + + /// + /// Updates series ItemsSource from the in-memory sampled data. + /// Used when the sampled data has sufficient density for the current viewport. + /// + private void UpdateSeriesFromMemory(double visibleMin, double visibleMax) + { foreach (var series in PlotModel.Series.OfType()) { if (series.Tag is not (string deviceSerial, string channelName)) @@ -993,7 +1039,6 @@ private void UpdateMainPlotViewport() var (startIdx, endIdx) = MinMaxDownsampler.FindVisibleRange(allPoints, visibleMin, visibleMax); var visibleCount = endIdx - startIdx; - // Reuse cached list to avoid GC pressure during interaction if (!_downsampledCache.TryGetValue(key, out var cached)) { cached = new List(MAIN_PLOT_BUCKET_COUNT * 2); @@ -1002,7 +1047,6 @@ private void UpdateMainPlotViewport() if (visibleCount <= MAIN_PLOT_BUCKET_COUNT * 2) { - // Few enough points to render directly — copy into cached list cached.Clear(); for (var i = startIdx; i < endIdx; i++) { @@ -1016,7 +1060,127 @@ private void UpdateMainPlotViewport() cached.AddRange(downsampled); } - // Only set ItemsSource once per series — subsequent updates reuse the same list + if (series.ItemsSource != cached) + { + series.ItemsSource = cached; + } + } + } + + /// + /// Fetches full-resolution data from the database for the visible time window, + /// then downsamples if needed. Uses the composite index for fast range queries. + /// Typically completes in ~1-20ms depending on the window size. + /// + private void FetchViewportDataFromDb(double visibleMin, double visibleMax) + { + if (!_currentSessionId.HasValue || !_firstTime.HasValue) + { + return; + } + + // Convert plot X (ms) back to DB ticks + var firstTimeTicks = _firstTime.Value.Ticks; + var minTicks = firstTimeTicks + (long)(visibleMin * 10000.0); + var maxTicks = firstTimeTicks + (long)(visibleMax * 10000.0); + + // Pad slightly to ensure edge continuity + var tickRange = maxTicks - minTicks; + var padding = Math.Max(tickRange / 100, 10000); + minTicks -= padding; + maxTicks += padding; + + // Build per-channel point lists from DB + var dbPoints = new Dictionary<(string, string), List>(); + foreach (var key in _allSessionPoints.Keys) + { + dbPoints[key] = new List(); + } + + try + { + using var context = _loggingContext.CreateDbContext(); + var connection = context.Database.GetDbConnection(); + connection.Open(); + + using var cmd = connection.CreateCommand(); + cmd.CommandText = @" + SELECT ChannelName, DeviceSerialNo, TimestampTicks, Value + FROM Samples + WHERE LoggingSessionID = @id + AND TimestampTicks >= @minT + AND TimestampTicks <= @maxT + ORDER BY TimestampTicks"; + + var idParam = cmd.CreateParameter(); + idParam.ParameterName = "@id"; + idParam.Value = _currentSessionId.Value; + cmd.Parameters.Add(idParam); + + var minParam = cmd.CreateParameter(); + minParam.ParameterName = "@minT"; + minParam.Value = minTicks; + cmd.Parameters.Add(minParam); + + var maxParam = cmd.CreateParameter(); + maxParam.ParameterName = "@maxT"; + maxParam.Value = maxTicks; + cmd.Parameters.Add(maxParam); + + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + { + var channelName = reader.GetString(0); + var deviceSerialNo = reader.GetString(1); + var timestampTicks = reader.GetInt64(2); + var value = reader.GetDouble(3); + + var key = (deviceSerialNo, channelName); + var deltaTime = (timestampTicks - firstTimeTicks) / 10000.0; + + if (dbPoints.TryGetValue(key, out var points)) + { + points.Add(new DataPoint(deltaTime, value)); + } + } + } + catch (Exception ex) + { + _appLogger.Error(ex, "Failed to fetch viewport data from DB"); + UpdateSeriesFromMemory(visibleMin, visibleMax); + return; + } + + // Update each series with the DB-fetched data, downsampled if needed + foreach (var series in PlotModel.Series.OfType()) + { + if (series.Tag is not (string deviceSerial, string channelName)) + { + continue; + } + + var key = (deviceSerial, channelName); + if (!dbPoints.TryGetValue(key, out var fetchedPoints) || fetchedPoints.Count == 0) + { + continue; + } + + if (!_downsampledCache.TryGetValue(key, out var cached)) + { + cached = new List(MAIN_PLOT_BUCKET_COUNT * 2); + _downsampledCache[key] = cached; + } + + cached.Clear(); + if (fetchedPoints.Count <= MAIN_PLOT_BUCKET_COUNT * 2) + { + cached.AddRange(fetchedPoints); + } + else + { + cached.AddRange(MinMaxDownsampler.Downsample(fetchedPoints, MAIN_PLOT_BUCKET_COUNT)); + } + if (series.ItemsSource != cached) { series.ItemsSource = cached; From 7000ef8aa27e5701ce85aa0bec94e9a67b166655 Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Sat, 11 Apr 2026 22:06:22 -0600 Subject: [PATCH 14/19] perf: smooth 60fps minimap drag with deferred high-fidelity DB fetch During minimap drag and main plot pan/zoom, use only in-memory sampled data for instant viewport updates. Full-resolution DB queries are deferred until interaction ends (mouse up) or settles (200ms idle), ensuring smooth 60fps responsiveness even on large datasets. Co-Authored-By: Claude Opus 4.6 --- Daqifi.Desktop/Loggers/DatabaseLogger.cs | 59 +++++++++++++++++-- .../View/MinimapInteractionController.cs | 10 +++- 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/Daqifi.Desktop/Loggers/DatabaseLogger.cs b/Daqifi.Desktop/Loggers/DatabaseLogger.cs index 14bcb574..ce797828 100644 --- a/Daqifi.Desktop/Loggers/DatabaseLogger.cs +++ b/Daqifi.Desktop/Loggers/DatabaseLogger.cs @@ -130,6 +130,7 @@ public partial class DatabaseLogger : ObservableObject, ILogger, IDisposable private double _lastViewportMax = double.NaN; private bool _viewportDirty; private DispatcherTimer _viewportThrottleTimer; + private DispatcherTimer _settleTimer; private int? _currentSessionId; [ObservableProperty] @@ -243,6 +244,14 @@ public DatabaseLogger(IDbContextFactory loggingContext) _viewportThrottleTimer.Tick += OnViewportThrottleTick; _viewportThrottleTimer.Start(); + // Settle timer: triggers high-fidelity DB fetch 200ms after the last + // viewport change (covers both main plot pan/zoom and minimap drag) + _settleTimer = new DispatcherTimer(DispatcherPriority.Background) + { + Interval = TimeSpan.FromMilliseconds(200) + }; + _settleTimer.Tick += OnSettleTick; + // Initialize minimap PlotModel InitializeMinimapPlotModel(); @@ -952,7 +961,24 @@ private void OnViewportThrottleTick(object? sender, EventArgs e) } _viewportDirty = false; - UpdateMainPlotViewport(); + UpdateMainPlotViewport(highFidelity: false); + PlotModel.InvalidatePlot(true); + + // Restart the settle timer — it will fire 200ms after the last change + _settleTimer.Stop(); + _settleTimer.Start(); + } + + /// + /// Fires 200ms after the last viewport change. Triggers a high-fidelity + /// DB fetch so zoomed-in views show full-resolution data once interaction settles. + /// + private void OnSettleTick(object? sender, EventArgs e) + { + _settleTimer.Stop(); + _lastViewportMin = double.NaN; + _lastViewportMax = double.NaN; + UpdateMainPlotViewport(highFidelity: true); PlotModel.InvalidatePlot(true); } @@ -962,7 +988,7 @@ private void OnViewportThrottleTick(object? sender, EventArgs e) /// fetches full-resolution data from the database for the visible window. /// Skips the update if the viewport hasn't changed since the last call. /// - private void UpdateMainPlotViewport() + private void UpdateMainPlotViewport(bool highFidelity = true) { var timeAxis = PlotModel.Axes.FirstOrDefault(a => a.Key == "Time"); if (timeAxis == null) @@ -982,6 +1008,15 @@ private void UpdateMainPlotViewport() _lastViewportMin = visibleMin; _lastViewportMax = visibleMax; + // During drag (highFidelity=false), always use fast in-memory data to + // maintain smooth 60fps. DB fetches only happen on settle (mouse up, + // zoom buttons) when highFidelity=true. + if (!highFidelity) + { + UpdateSeriesFromMemory(visibleMin, visibleMax); + return; + } + // Check if sampled in-memory data is too sparse for this zoom level. // If fewer sampled points are visible than our target density, fetch // full-resolution data from the DB for this window. @@ -1189,12 +1224,24 @@ FROM Samples } /// - /// Called by the minimap interaction controller to update the main plot's - /// viewport downsampling after a minimap-driven zoom/pan. + /// Called by the minimap interaction controller during drag to update the + /// main plot. Uses in-memory sampled data only (no DB queries) for smooth 60fps. /// public void OnMinimapViewportChanged() { - UpdateMainPlotViewport(); + UpdateMainPlotViewport(highFidelity: false); + } + + /// + /// Called when minimap interaction ends (mouse up). Fetches full-resolution + /// data from the database if the zoom level warrants it. + /// + public void OnMinimapInteractionEnded() + { + _settleTimer.Stop(); + _lastViewportMin = double.NaN; + _lastViewportMax = double.NaN; + UpdateMainPlotViewport(highFidelity: true); } /// @@ -1307,6 +1354,8 @@ public void Dispose() { _viewportThrottleTimer.Stop(); _viewportThrottleTimer.Tick -= OnViewportThrottleTick; + _settleTimer.Stop(); + _settleTimer.Tick -= OnSettleTick; _minimapInteraction?.Dispose(); _buffer.Dispose(); _consumerGate.Dispose(); diff --git a/Daqifi.Desktop/View/MinimapInteractionController.cs b/Daqifi.Desktop/View/MinimapInteractionController.cs index d1e6910a..4350e56a 100644 --- a/Daqifi.Desktop/View/MinimapInteractionController.cs +++ b/Daqifi.Desktop/View/MinimapInteractionController.cs @@ -276,14 +276,20 @@ private void OnMouseUp(object? sender, OxyMouseEventArgs e) { _dragMode = DragMode.None; - // Flush final render for accuracy + // Flush final render and trigger high-fidelity DB fetch if (_isDirty) { _isDirty = false; - _databaseLogger.OnMinimapViewportChanged(); + _databaseLogger.OnMinimapInteractionEnded(); _mainPlotModel.InvalidatePlot(true); _minimapPlotModel.InvalidatePlot(false); } + else + { + // Even if no pending render, trigger DB fetch for the final position + _databaseLogger.OnMinimapInteractionEnded(); + _mainPlotModel.InvalidatePlot(true); + } // Update cursor based on final position var minimapTimeAxis = GetMinimapTimeAxis(); From b6755ce6ce39184afda7fd94f72bdaf4a31e9212 Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Sat, 11 Apr 2026 22:09:49 -0600 Subject: [PATCH 15/19] =?UTF-8?q?fix:=20address=20review=20issues=20?= =?UTF-8?q?=E2=80=94=20sparse=20check,=20axis=20indices,=20mouse-up=20clea?= =?UTF-8?q?nup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Sparse check now examines ALL channels instead of just the first one in dictionary iteration order; breaks only when a sparse channel is found - Replace hardcoded PlotModel.Axes[0]/[2] with key-based lookups ("Time", "Analog") in zoom commands for robustness against axis reordering - Simplify duplicated if/else branches in MinimapInteractionController OnMouseUp — both paths were calling the same methods Co-Authored-By: Claude Opus 4.6 --- Daqifi.Desktop/Loggers/DatabaseLogger.cs | 29 ++++++++++++------- .../View/MinimapInteractionController.cs | 20 ++++--------- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/Daqifi.Desktop/Loggers/DatabaseLogger.cs b/Daqifi.Desktop/Loggers/DatabaseLogger.cs index ce797828..2a7639ca 100644 --- a/Daqifi.Desktop/Loggers/DatabaseLogger.cs +++ b/Daqifi.Desktop/Loggers/DatabaseLogger.cs @@ -1017,9 +1017,8 @@ private void UpdateMainPlotViewport(bool highFidelity = true) return; } - // Check if sampled in-memory data is too sparse for this zoom level. - // If fewer sampled points are visible than our target density, fetch - // full-resolution data from the DB for this window. + // Check if ANY channel's sampled in-memory data is too sparse for + // this zoom level. If so, fetch full-resolution data from the DB. var needsDbFetch = false; if (_currentSessionId.HasValue && _firstTime.HasValue) { @@ -1030,15 +1029,19 @@ private void UpdateMainPlotViewport(bool highFidelity = true) continue; } + // Only check channels that are actually sampled (not full datasets) + if (kvp.Value.Count < SAMPLED_POINTS_PER_CHANNEL / 2) + { + continue; + } + var (si, ei) = MinMaxDownsampler.FindVisibleRange(kvp.Value, visibleMin, visibleMax); var sampledVisible = ei - si; - // Sampled data is sparse if we have fewer points than target AND - // the in-memory data is actually sampled (not the full dataset) - if (sampledVisible < MAIN_PLOT_BUCKET_COUNT && kvp.Value.Count >= SAMPLED_POINTS_PER_CHANNEL / 2) + if (sampledVisible < MAIN_PLOT_BUCKET_COUNT) { needsDbFetch = true; + break; } - break; } } @@ -1318,7 +1321,8 @@ private void ResetZoom() [RelayCommand] private void ZoomOutX() { - PlotModel.Axes[2].ZoomAtCenter(0.8); + var timeAxis = PlotModel.Axes.FirstOrDefault(a => a.Key == "Time"); + timeAxis?.ZoomAtCenter(0.8); UpdateMainPlotViewport(); PlotModel.InvalidatePlot(true); } @@ -1326,7 +1330,8 @@ private void ZoomOutX() [RelayCommand] private void ZoomInX() { - PlotModel.Axes[2].ZoomAtCenter(1.25); + var timeAxis = PlotModel.Axes.FirstOrDefault(a => a.Key == "Time"); + timeAxis?.ZoomAtCenter(1.25); UpdateMainPlotViewport(); PlotModel.InvalidatePlot(true); } @@ -1334,14 +1339,16 @@ private void ZoomInX() [RelayCommand] private void ZoomOutY() { - PlotModel.Axes[0].ZoomAtCenter(0.8); + var analogAxis = PlotModel.Axes.FirstOrDefault(a => a.Key == "Analog"); + analogAxis?.ZoomAtCenter(0.8); PlotModel.InvalidatePlot(true); } [RelayCommand] private void ZoomInY() { - PlotModel.Axes[0].ZoomAtCenter(1.25); + var analogAxis = PlotModel.Axes.FirstOrDefault(a => a.Key == "Analog"); + analogAxis?.ZoomAtCenter(1.25); PlotModel.InvalidatePlot(true); } #endregion diff --git a/Daqifi.Desktop/View/MinimapInteractionController.cs b/Daqifi.Desktop/View/MinimapInteractionController.cs index 4350e56a..3038cedb 100644 --- a/Daqifi.Desktop/View/MinimapInteractionController.cs +++ b/Daqifi.Desktop/View/MinimapInteractionController.cs @@ -276,20 +276,12 @@ private void OnMouseUp(object? sender, OxyMouseEventArgs e) { _dragMode = DragMode.None; - // Flush final render and trigger high-fidelity DB fetch - if (_isDirty) - { - _isDirty = false; - _databaseLogger.OnMinimapInteractionEnded(); - _mainPlotModel.InvalidatePlot(true); - _minimapPlotModel.InvalidatePlot(false); - } - else - { - // Even if no pending render, trigger DB fetch for the final position - _databaseLogger.OnMinimapInteractionEnded(); - _mainPlotModel.InvalidatePlot(true); - } + // Flush any pending dirty state and trigger high-fidelity DB fetch + // for the final viewport position (regardless of dirty flag state) + _isDirty = false; + _databaseLogger.OnMinimapInteractionEnded(); + _mainPlotModel.InvalidatePlot(true); + _minimapPlotModel.InvalidatePlot(false); // Update cursor based on final position var minimapTimeAxis = GetMinimapTimeAxis(); From 243b144a01730949482f804fc15faa5fb160c4bb Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Sat, 11 Apr 2026 22:28:31 -0600 Subject: [PATCH 16/19] perf: async sampled-seek DB fetch with visual refining indicator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the synchronous range-scan query in FetchViewportDataFromDb with sampled index seeks (same technique as LoadSampledData), capping reads at ~4000 * channelCount rows regardless of window size. Run the DB work on a background thread with CancellationToken support so the UI stays responsive and cursors update normally during the fetch. Add a thin indeterminate ProgressBar (2px, brand blue) between the main plot and minimap that appears while IsRefiningData is true — a subtle visual cue that higher-fidelity data is loading without any text or modal overlay. Co-Authored-By: Claude Opus 4.6 --- Daqifi.Desktop/Loggers/DatabaseLogger.cs | 242 +++++++++++++++-------- Daqifi.Desktop/MainWindow.xaml | 23 ++- 2 files changed, 181 insertions(+), 84 deletions(-) diff --git a/Daqifi.Desktop/Loggers/DatabaseLogger.cs b/Daqifi.Desktop/Loggers/DatabaseLogger.cs index 2a7639ca..0d2c96d6 100644 --- a/Daqifi.Desktop/Loggers/DatabaseLogger.cs +++ b/Daqifi.Desktop/Loggers/DatabaseLogger.cs @@ -132,6 +132,7 @@ public partial class DatabaseLogger : ObservableObject, ILogger, IDisposable private DispatcherTimer _viewportThrottleTimer; private DispatcherTimer _settleTimer; private int? _currentSessionId; + private CancellationTokenSource _fetchCts; [ObservableProperty] private PlotModel _plotModel; @@ -154,6 +155,13 @@ public partial class DatabaseLogger : ObservableObject, ILogger, IDisposable /// [ObservableProperty] private bool _hasSessionData; + + /// + /// True while fetching high-fidelity data from the database in the background. + /// Bound to a subtle progress indicator in the UI. + /// + [ObservableProperty] + private bool _isRefiningData; #endregion #region Legend @@ -1106,9 +1114,10 @@ private void UpdateSeriesFromMemory(double visibleMin, double visibleMax) } /// - /// Fetches full-resolution data from the database for the visible time window, - /// then downsamples if needed. Uses the composite index for fast range queries. - /// Typically completes in ~1-20ms depending on the window size. + /// Fetches high-resolution data from the database for the visible time window + /// using sampled index seeks (same technique as LoadSampledData). Runs on a + /// background thread to keep the UI responsive. Cancels any in-flight fetch + /// when a new one starts. Results are marshaled back to the UI thread. /// private void FetchViewportDataFromDb(double visibleMin, double visibleMax) { @@ -1117,113 +1126,178 @@ private void FetchViewportDataFromDb(double visibleMin, double visibleMax) return; } - // Convert plot X (ms) back to DB ticks + // Cancel any in-flight fetch + _fetchCts?.Cancel(); + _fetchCts?.Dispose(); + _fetchCts = new CancellationTokenSource(); + var ct = _fetchCts.Token; + + var sessionId = _currentSessionId.Value; var firstTimeTicks = _firstTime.Value.Ticks; + var channelKeys = _allSessionPoints.Keys.ToList(); + + // Convert plot X (ms) back to DB ticks with padding var minTicks = firstTimeTicks + (long)(visibleMin * 10000.0); var maxTicks = firstTimeTicks + (long)(visibleMax * 10000.0); - - // Pad slightly to ensure edge continuity var tickRange = maxTicks - minTicks; var padding = Math.Max(tickRange / 100, 10000); minTicks -= padding; maxTicks += padding; - // Build per-channel point lists from DB - var dbPoints = new Dictionary<(string, string), List>(); - foreach (var key in _allSessionPoints.Keys) - { - dbPoints[key] = new List(); - } + IsRefiningData = true; - try + Task.Run(() => { - using var context = _loggingContext.CreateDbContext(); - var connection = context.Database.GetDbConnection(); - connection.Open(); + var dbPoints = new Dictionary<(string, string), List>(); + foreach (var key in channelKeys) + { + dbPoints[key] = new List(); + } - using var cmd = connection.CreateCommand(); - cmd.CommandText = @" - SELECT ChannelName, DeviceSerialNo, TimestampTicks, Value - FROM Samples - WHERE LoggingSessionID = @id - AND TimestampTicks >= @minT - AND TimestampTicks <= @maxT - ORDER BY TimestampTicks"; + try + { + ct.ThrowIfCancellationRequested(); + + using var context = _loggingContext.CreateDbContext(); + var connection = context.Database.GetDbConnection(); + connection.Open(); + + // Use sampled seeks: divide the visible window into + // MAIN_PLOT_BUCKET_COUNT segments and read a small batch + // at each position. This reads ~4000 * channelCount rows + // instead of potentially millions. + var seekCount = MAIN_PLOT_BUCKET_COUNT; + var seekTickStep = (maxTicks - minTicks) / seekCount; + var batchSize = Math.Max(channelKeys.Count * 2, 100); + + using var seekCmd = connection.CreateCommand(); + seekCmd.CommandText = @" + SELECT ChannelName, DeviceSerialNo, TimestampTicks, Value + FROM Samples + WHERE LoggingSessionID = @id + AND TimestampTicks >= @t + AND TimestampTicks <= @maxT + ORDER BY TimestampTicks + LIMIT @limit"; + + var idParam = seekCmd.CreateParameter(); + idParam.ParameterName = "@id"; + idParam.Value = sessionId; + seekCmd.Parameters.Add(idParam); + + var tParam = seekCmd.CreateParameter(); + tParam.ParameterName = "@t"; + tParam.Value = minTicks; + seekCmd.Parameters.Add(tParam); + + var maxTParam = seekCmd.CreateParameter(); + maxTParam.ParameterName = "@maxT"; + maxTParam.Value = maxTicks; + seekCmd.Parameters.Add(maxTParam); + + var limitParam = seekCmd.CreateParameter(); + limitParam.ParameterName = "@limit"; + limitParam.Value = batchSize; + seekCmd.Parameters.Add(limitParam); + + seekCmd.Prepare(); + + var lastAddedTimestamp = new Dictionary<(string, string), long>(); + + for (var i = 0; i < seekCount; i++) + { + ct.ThrowIfCancellationRequested(); - var idParam = cmd.CreateParameter(); - idParam.ParameterName = "@id"; - idParam.Value = _currentSessionId.Value; - cmd.Parameters.Add(idParam); + tParam.Value = minTicks + i * seekTickStep; - var minParam = cmd.CreateParameter(); - minParam.ParameterName = "@minT"; - minParam.Value = minTicks; - cmd.Parameters.Add(minParam); + using var reader = seekCmd.ExecuteReader(); + while (reader.Read()) + { + var channelName = reader.GetString(0); + var deviceSerialNo = reader.GetString(1); + var timestampTicks = reader.GetInt64(2); + var value = reader.GetDouble(3); - var maxParam = cmd.CreateParameter(); - maxParam.ParameterName = "@maxT"; - maxParam.Value = maxTicks; - cmd.Parameters.Add(maxParam); + var key = (deviceSerialNo, channelName); - using var reader = cmd.ExecuteReader(); - while (reader.Read()) - { - var channelName = reader.GetString(0); - var deviceSerialNo = reader.GetString(1); - var timestampTicks = reader.GetInt64(2); - var value = reader.GetDouble(3); + if (lastAddedTimestamp.TryGetValue(key, out var lastT) && timestampTicks <= lastT) + { + continue; + } - var key = (deviceSerialNo, channelName); - var deltaTime = (timestampTicks - firstTimeTicks) / 10000.0; + lastAddedTimestamp[key] = timestampTicks; + var deltaTime = (timestampTicks - firstTimeTicks) / 10000.0; - if (dbPoints.TryGetValue(key, out var points)) - { - points.Add(new DataPoint(deltaTime, value)); + if (dbPoints.TryGetValue(key, out var points)) + { + points.Add(new DataPoint(deltaTime, value)); + } + } } } - } - catch (Exception ex) - { - _appLogger.Error(ex, "Failed to fetch viewport data from DB"); - UpdateSeriesFromMemory(visibleMin, visibleMax); - return; - } - - // Update each series with the DB-fetched data, downsampled if needed - foreach (var series in PlotModel.Series.OfType()) - { - if (series.Tag is not (string deviceSerial, string channelName)) + catch (OperationCanceledException) { - continue; + return; } - - var key = (deviceSerial, channelName); - if (!dbPoints.TryGetValue(key, out var fetchedPoints) || fetchedPoints.Count == 0) + catch (Exception ex) { - continue; + _appLogger.Error(ex, "Failed to fetch viewport data from DB"); + Application.Current?.Dispatcher.Invoke(() => + { + IsRefiningData = false; + UpdateSeriesFromMemory(visibleMin, visibleMax); + PlotModel.InvalidatePlot(true); + }); + return; } - if (!_downsampledCache.TryGetValue(key, out var cached)) + // Marshal results back to UI thread + Application.Current?.Dispatcher.Invoke(() => { - cached = new List(MAIN_PLOT_BUCKET_COUNT * 2); - _downsampledCache[key] = cached; - } + if (ct.IsCancellationRequested) + { + return; + } - cached.Clear(); - if (fetchedPoints.Count <= MAIN_PLOT_BUCKET_COUNT * 2) - { - cached.AddRange(fetchedPoints); - } - else - { - cached.AddRange(MinMaxDownsampler.Downsample(fetchedPoints, MAIN_PLOT_BUCKET_COUNT)); - } + foreach (var series in PlotModel.Series.OfType()) + { + if (series.Tag is not (string deviceSerial, string channelName)) + { + continue; + } - if (series.ItemsSource != cached) - { - series.ItemsSource = cached; - } - } + var key = (deviceSerial, channelName); + if (!dbPoints.TryGetValue(key, out var fetchedPoints) || fetchedPoints.Count == 0) + { + continue; + } + + if (!_downsampledCache.TryGetValue(key, out var cached)) + { + cached = new List(MAIN_PLOT_BUCKET_COUNT * 2); + _downsampledCache[key] = cached; + } + + cached.Clear(); + if (fetchedPoints.Count <= MAIN_PLOT_BUCKET_COUNT * 2) + { + cached.AddRange(fetchedPoints); + } + else + { + cached.AddRange(MinMaxDownsampler.Downsample(fetchedPoints, MAIN_PLOT_BUCKET_COUNT)); + } + + if (series.ItemsSource != cached) + { + series.ItemsSource = cached; + } + } + + IsRefiningData = false; + PlotModel.InvalidatePlot(true); + }); + }, ct); } /// @@ -1363,6 +1437,8 @@ public void Dispose() _viewportThrottleTimer.Tick -= OnViewportThrottleTick; _settleTimer.Stop(); _settleTimer.Tick -= OnSettleTick; + _fetchCts?.Cancel(); + _fetchCts?.Dispose(); _minimapInteraction?.Dispose(); _buffer.Dispose(); _consumerGate.Dispose(); diff --git a/Daqifi.Desktop/MainWindow.xaml b/Daqifi.Desktop/MainWindow.xaml index 04451046..cba2ab5f 100644 --- a/Daqifi.Desktop/MainWindow.xaml +++ b/Daqifi.Desktop/MainWindow.xaml @@ -470,6 +470,7 @@ + @@ -565,8 +566,28 @@ + + + + + + + - From 8f70685c6800f5e970123c7b2005871e5a5443c2 Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Sat, 11 Apr 2026 22:33:12 -0600 Subject: [PATCH 17/19] fix: reset axes when switching sessions so new session shows full extent ClearPlot now resets all axes (time, analog, digital) so selecting a new logging session starts at the full data range instead of keeping the previous session's zoom level. Also cancels any in-flight DB fetch and stops the settle timer on session switch. Co-Authored-By: Claude Opus 4.6 --- Daqifi.Desktop/Loggers/DatabaseLogger.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Daqifi.Desktop/Loggers/DatabaseLogger.cs b/Daqifi.Desktop/Loggers/DatabaseLogger.cs index 0d2c96d6..ca5f730f 100644 --- a/Daqifi.Desktop/Loggers/DatabaseLogger.cs +++ b/Daqifi.Desktop/Loggers/DatabaseLogger.cs @@ -436,6 +436,11 @@ public void ClearPlot() _currentSessionId = null; _lastViewportMin = double.NaN; _lastViewportMax = double.NaN; + _fetchCts?.Cancel(); + _fetchCts?.Dispose(); + _fetchCts = null; + IsRefiningData = false; + _settleTimer.Stop(); _allSessionPoints.Clear(); _downsampledCache.Clear(); _minimapSeries.Clear(); @@ -444,6 +449,13 @@ public void ClearPlot() DeviceLegendGroups.Clear(); PlotModel.Title = string.Empty; PlotModel.Subtitle = string.Empty; + + // Reset all axes so the new session starts at full extent + foreach (var axis in PlotModel.Axes) + { + axis.Reset(); + } + PlotModel.InvalidatePlot(true); MinimapPlotModel.Series.Clear(); From f714fd789821813d635ee644c998aa4fcc27eb3e Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Sat, 11 Apr 2026 22:38:31 -0600 Subject: [PATCH 18/19] docs: update ADR and CLAUDE.md to reflect async fetch and progressive loading MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 001: Updated to document the hybrid approach — on-demand DB queries were partially adopted (async sampled seeks on settle) rather than fully rejected. Updated "How it works" to cover two-phase loading and drag/settle distinction. Added known issues (thread safety, consumer shutdown) to follow-up work. CLAUDE.md: Added three new gotchas — async DB fetch lifecycle, drag vs settle pattern, and session-switching axis reset. Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 5 ++- docs/adr/001-viewport-aware-downsampling.md | 43 +++++++++++++-------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 864fff09..75765a15 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -255,12 +255,15 @@ When working on: ### Plot Rendering (OxyPlot) -The logged data viewer uses viewport-aware downsampling for 60fps interaction with large datasets. See [ADR 001](docs/adr/001-viewport-aware-downsampling.md) for full context. Key gotchas when modifying plot code: +The logged data viewer uses viewport-aware downsampling with progressive loading for 60fps interaction with large datasets. See [ADR 001](docs/adr/001-viewport-aware-downsampling.md) for full context. Key gotchas when modifying plot code: - **`InvalidatePlot(true)` vs `(false)`**: Use `true` whenever `ItemsSource` or its underlying list has changed — `false` renders stale cached data - **Don't use `ResetAllAxes()` with downsampled data**: Auto-range reads from `ItemsSource`, which may have shifted X boundaries. Use explicit `axis.Zoom(min, max)` from source data - **Guard flag for minimap sync**: Always set `IsSyncingFromMinimap` before programmatic axis changes to prevent feedback loops - **Reuse cached lists**: Don't allocate new `List` per frame — use `_downsampledCache` +- **High-fidelity DB fetch is async**: `FetchViewportDataFromDb` runs on a background thread. Cancel in-flight fetches via `_fetchCts` before starting new ones. Results marshal back to UI via `Dispatcher.Invoke` +- **Drag vs settle pattern**: During continuous interaction (minimap drag, main plot pan), use only in-memory sampled data (`highFidelity: false`). DB fetches happen on mouse-up or after 200ms idle (`highFidelity: true`) +- **Session switching must reset axes**: `ClearPlot()` calls `axis.Reset()` on all axes. Without this, the new session inherits the previous session's zoom range ## Error Handling diff --git a/docs/adr/001-viewport-aware-downsampling.md b/docs/adr/001-viewport-aware-downsampling.md index c37e1e11..170cb46b 100644 --- a/docs/adr/001-viewport-aware-downsampling.md +++ b/docs/adr/001-viewport-aware-downsampling.md @@ -23,19 +23,21 @@ Use **viewport-aware MinMax downsampling**: on every viewport change, binary sea ### How it works -1. Each channel's full dataset is kept in memory as a sorted `List` (`_allSessionPoints`) -2. When the viewport changes (minimap drag, zoom, pan), `UpdateMainPlotViewport()`: - - Binary searches for the visible range indices — O(log n) via `MinMaxDownsampler.FindVisibleRange()` - - If the visible slice is small enough (< 4000 points), uses it directly - - Otherwise, downsamples via `MinMaxDownsampler.Downsample(points, startIdx, endIdx, 2000)` — divides into 2000 buckets, emits the min and max Y value per bucket (up to 4000 output points) -3. The downsampled data is written into a **reusable cached list** per series (not a new allocation) and set as the series' `ItemsSource` -4. Viewport updates are **throttled to 60fps** via DispatcherTimer + dirty flag, both for minimap-driven changes and main plot pan/zoom +**Session loading (two-phase progressive):** +1. **Phase 1** (<1s): Load first 100K samples via index scan for immediate display +2. **Phase 2** (~1-3s): Load a sampled overview via ~3000 targeted index seeks spread across the full time range. Each seek reads one batch of interleaved channel data. Result: ~3000 points/channel covering the full range + +**Viewport updates (drag vs settle):** +1. **During interaction** (minimap drag, pan/zoom): `UpdateMainPlotViewport(highFidelity: false)` uses only in-memory sampled data. Binary searches for the visible range — O(log n) via `MinMaxDownsampler.FindVisibleRange()` — then downsamples to ~4000 points per channel via min/max aggregation. Written into a **reusable cached list** per series (no allocation). +2. **On settle** (mouse-up or 200ms idle): `UpdateMainPlotViewport(highFidelity: true)` checks if the in-memory data is too sparse for the current zoom level. If so, fires an async background DB fetch using sampled index seeks within the visible window, then marshals results back to the UI thread. A `CancellationToken` ensures only the latest fetch completes. +3. Viewport updates are **throttled to 60fps** via DispatcherTimer + dirty flag, both for minimap-driven changes and main plot pan/zoom ### Key files - `MinMaxDownsampler.cs` — binary search + min/max downsampling algorithm -- `DatabaseLogger.UpdateMainPlotViewport()` — viewport change handler -- `MinimapInteractionController.cs` — 60fps throttled minimap interaction +- `DatabaseLogger.cs` — two-phase loading, viewport updates, async DB fetch (`UpdateMainPlotViewport`, `FetchViewportDataFromDb`, `LoadSampledData`) +- `MinimapInteractionController.cs` — 60fps throttled minimap interaction with drag/settle distinction +- `LoggingContext.cs` — composite DB index `IX_Samples_SessionTime` on `(LoggingSessionID, TimestampTicks)` ## Alternatives Considered @@ -59,29 +61,38 @@ Replace OxyPlot with a GPU-backed charting library (e.g., SciChart, LiveCharts2 **Rejected because**: Major dependency change with significant migration cost. OxyPlot is well-integrated with our WPF MVVM architecture. Viewport-aware downsampling reduces the point count enough (~64K total) that OxyPlot renders comfortably within a 16ms frame budget. If we outgrow this approach, GPU rendering remains a future option. -### 4. Virtual Scrolling / On-Demand DB Queries +### 4. Virtual Scrolling / On-Demand DB Queries (Partially Adopted) Only load the visible time range from SQLite on each viewport change, avoiding keeping all data in memory. -**Rejected because**: SQLite query latency (~5-50ms depending on range size) is too high for 60fps interaction. Users would see visible lag during minimap drag. Keeping data in memory with a practical cap (50M points, ~800MB) is the right trade-off for interactive performance. The DB index we added (`IX_Samples_SessionTime`) supports this pattern if we ever need to implement paging for truly enormous datasets. +**Initially rejected** for pure on-demand because SQLite query latency is too high for 60fps continuous interaction (minimap drag, pan). **However**, we adopted a hybrid approach: + +- **During drag**: use in-memory sampled data only (3000 points/channel, loaded at session start via sampled index seeks). This guarantees <1ms viewport updates for smooth 60fps. +- **On settle** (mouse-up or 200ms idle): fetch high-resolution data from SQLite for just the visible window via async sampled index seeks on a background thread. The composite index (`IX_Samples_SessionTime`) makes these seeks fast regardless of total dataset size. + +This gives the best of both worlds: instantaneous interaction with progressive refinement to full fidelity when the user stops moving. ## Consequences ### Positive -- **60fps interaction**: ~10-15ms per frame with 16 channels × 1M points -- **Full zoom fidelity**: Zooming into a 1-second window of a 24-hour session shows every data point +- **60fps interaction**: <1ms per viewport update during drag (in-memory sampled data only) +- **Full zoom fidelity**: Zooming into a 1-second window triggers an async DB fetch that fills in full-resolution data +- **Fast session loading**: Two-phase progressive — data visible in <1s, full overview in ~1-3s regardless of dataset size +- **Non-blocking UI**: DB fetches run on background thread with cancellation. A thin progress bar indicates when refinement is in progress - **Low complexity**: The core algorithm is ~80 lines (binary search + min/max loop) - **No external dependencies**: Pure C# implementation ### Negative -- **Memory usage**: Full dataset lives in memory. Capped at 50M points (~800MB). Sessions exceeding this are truncated with a UI warning. +- **Memory usage for overview**: Sampled data (~3000 points/channel) lives in memory. Much smaller than keeping the full dataset in memory. - **MinMax doesn't preserve exact X boundaries**: Downsampled first/last X values may differ from source data. This required explicit time axis ranging in `ResetZoom` instead of relying on OxyPlot auto-range (see `InvalidatePlot` gotchas in CLAUDE.md). - **List mutation pattern**: Reusing cached lists and calling `InvalidatePlot(true)` is non-obvious. `InvalidatePlot(false)` silently renders stale data. This is documented in CLAUDE.md but remains a footgun for future changes. +- **Thread safety**: `_allSessionPoints` is written on background threads (session loading) and read on the UI thread (viewport updates). Currently relies on non-overlapping access patterns rather than explicit synchronization. A future refactor should add proper locking. ### Follow-up Work -- If datasets exceed the 50M point memory cap, consider hybrid approach: keep a coarse in-memory overview + on-demand DB queries for zoomed-in detail -- Monitor whether the `GetRange()` copy in the "few enough points" path becomes a bottleneck — could be replaced with a `ListSegment` wrapper if needed +- Add thread synchronization for `_allSessionPoints` (lock or move all mutations to UI thread) +- Add cancellation token to the consumer thread for clean shutdown on `Dispose()` +- Existing DB migration path: the composite index `IX_Samples_SessionTime` is only created via `EnsureCreated`, not applied to existing databases. See GitHub issue #468 for the `EnsureCreated` → EF Core migrations switch. - PR #457 should be closed as superseded From 149f6a1c4017150ee4f668f292a94f21c527bb12 Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Sun, 12 Apr 2026 19:40:16 -0600 Subject: [PATCH 19/19] =?UTF-8?q?fix:=20address=20Qodo=20review=20?= =?UTF-8?q?=E2=80=94=20stale=20fetch,=20sampling=20tail,=20minimap=20axes,?= =?UTF-8?q?=20dispose?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Cancel in-flight DB fetch when viewport changes to prevent stale data from overwriting the current view (Qodo #2.6) - Ensure LoadSampledData seeks at maxTicks on the final iteration so the session tail is always included in sampled data (Qodo #2.5) - Replace MinimapPlotModel.ResetAllAxes() with explicit axis.Zoom() from source data bounds to avoid incorrect auto-range (Qodo #2.2) - Unsubscribe timeAxis.AxisChanged in Dispose() to prevent leaks and duplicate callbacks (Qodo #2.3) - Add tickStep guard (Math.Max(1, ...)) for tiny time ranges Co-Authored-By: Claude Opus 4.6 --- Daqifi.Desktop/Loggers/DatabaseLogger.cs | 45 +++++++++++++++++++----- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/Daqifi.Desktop/Loggers/DatabaseLogger.cs b/Daqifi.Desktop/Loggers/DatabaseLogger.cs index ca5f730f..a83c73ac 100644 --- a/Daqifi.Desktop/Loggers/DatabaseLogger.cs +++ b/Daqifi.Desktop/Loggers/DatabaseLogger.cs @@ -683,7 +683,7 @@ FROM Samples } _firstTime = new DateTime(minTicks); - var tickStep = (maxTicks - minTicks) / SAMPLED_POINTS_PER_CHANNEL; + var tickStep = Math.Max(1, (maxTicks - minTicks) / SAMPLED_POINTS_PER_CHANNEL); // Read at least channelCount rows per seek to get one sample per channel var batchSize = Math.Max(channelCount * 2, 100); @@ -717,9 +717,13 @@ ORDER BY TimestampTicks // from overlapping batches var lastAddedTimestamp = new Dictionary<(string, string), long>(); - for (var i = 0; i < SAMPLED_POINTS_PER_CHANNEL; i++) + // Use <= so the final iteration (i == SAMPLED_POINTS_PER_CHANNEL) + // seeks at maxTicks, ensuring the session tail is always included + for (var i = 0; i <= SAMPLED_POINTS_PER_CHANNEL; i++) { - var seekTimestamp = minTicks + i * tickStep; + var seekTimestamp = i < SAMPLED_POINTS_PER_CHANNEL + ? minTicks + i * tickStep + : maxTicks; seekTParam.Value = seekTimestamp; using var reader = seekCmd.ExecuteReader(); @@ -822,12 +826,20 @@ private void SetupMinimapSeries( _minimapSeries[(deviceSerial, channelName)] = minimapLine; } - MinimapPlotModel.ResetAllAxes(); - - if (minimapData.Count > 0) + // Set minimap axes from source data bounds (not auto-range, which + // reads downsampled ItemsSource and may have shifted boundaries) + var nonEmpty = minimapData.Where(d => d.downsampled.Count > 0).ToList(); + if (nonEmpty.Count > 0) { - var dataMinX = minimapData.Where(d => d.downsampled.Count > 0).Min(d => d.downsampled[0].X); - var dataMaxX = minimapData.Where(d => d.downsampled.Count > 0).Max(d => d.downsampled[^1].X); + var dataMinX = nonEmpty.Min(d => d.downsampled[0].X); + var dataMaxX = nonEmpty.Max(d => d.downsampled[^1].X); + + var minimapTimeAxis = MinimapPlotModel.Axes.FirstOrDefault(a => a.Key == "MinimapTime"); + minimapTimeAxis?.Zoom(dataMinX, dataMaxX); + + var minimapYAxis = MinimapPlotModel.Axes.FirstOrDefault(a => a.Key == "MinimapY"); + minimapYAxis?.Reset(); + _minimapSelectionRect.MinimumX = dataMinX; _minimapSelectionRect.MaximumX = dataMaxX; _minimapDimLeft.MaximumX = dataMinX; @@ -1028,6 +1040,16 @@ private void UpdateMainPlotViewport(bool highFidelity = true) _lastViewportMin = visibleMin; _lastViewportMax = visibleMax; + // Cancel any in-flight DB fetch — the viewport has changed, so its + // results would be stale and could overwrite the current view + if (_fetchCts != null) + { + _fetchCts.Cancel(); + _fetchCts.Dispose(); + _fetchCts = null; + IsRefiningData = false; + } + // During drag (highFidelity=false), always use fast in-memory data to // maintain smooth 60fps. DB fetches only happen on settle (mouse up, // zoom buttons) when highFidelity=true. @@ -1451,6 +1473,13 @@ public void Dispose() _settleTimer.Tick -= OnSettleTick; _fetchCts?.Cancel(); _fetchCts?.Dispose(); + + var timeAxis = PlotModel.Axes.FirstOrDefault(a => a.Key == "Time"); + if (timeAxis != null) + { + timeAxis.AxisChanged -= OnMainTimeAxisChanged; + } + _minimapInteraction?.Dispose(); _buffer.Dispose(); _consumerGate.Dispose();