diff --git a/CLAUDE.md b/CLAUDE.md index 3ee79e72..75765a15 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -242,6 +242,7 @@ When working on: - **Firewall/Network**: Ensure admin privileges handled properly, verify ports match - **WiFi Discovery Issues**: Check network interface detection and port configuration - **Manual IP Connections**: Ensure TCP port matches what device discovery reports +- **Plot/Minimap changes**: Read the "Plot Rendering (OxyPlot)" section below — there are non-obvious gotchas with `InvalidatePlot`, auto-range, and feedback loops. Key files: `DatabaseLogger.cs`, `MinimapInteractionController.cs`, `MinMaxDownsampler.cs` - **New features**: Add unit tests with 80% coverage minimum ## Performance Considerations @@ -252,6 +253,18 @@ When working on: - Use dependency injection for testability - Cache expensive operations +### Plot Rendering (OxyPlot) + +The logged data viewer uses viewport-aware downsampling with progressive loading for 60fps interaction with large datasets. See [ADR 001](docs/adr/001-viewport-aware-downsampling.md) for full context. Key gotchas when modifying plot code: + +- **`InvalidatePlot(true)` vs `(false)`**: Use `true` whenever `ItemsSource` or its underlying list has changed — `false` renders stale cached data +- **Don't use `ResetAllAxes()` with downsampled data**: Auto-range reads from `ItemsSource`, which may have shifted X boundaries. Use explicit `axis.Zoom(min, max)` from source data +- **Guard flag for minimap sync**: Always set `IsSyncingFromMinimap` before programmatic axis changes to prevent feedback loops +- **Reuse cached lists**: Don't allocate new `List` per frame — use `_downsampledCache` +- **High-fidelity DB fetch is async**: `FetchViewportDataFromDb` runs on a background thread. Cancel in-flight fetches via `_fetchCts` before starting new ones. Results marshal back to UI via `Dispatcher.Invoke` +- **Drag vs settle pattern**: During continuous interaction (minimap drag, main plot pan), use only in-memory sampled data (`highFidelity: false`). DB fetches happen on mouse-up or after 200ms idle (`highFidelity: true`) +- **Session switching must reset axes**: `ClearPlot()` calls `axis.Reset()` on all axes. Without this, the new session inherits the previous session's zoom range + ## Error Handling - Use try-catch at appropriate levels diff --git a/Daqifi.Desktop.Test/Helpers/MinMaxDownsamplerTests.cs b/Daqifi.Desktop.Test/Helpers/MinMaxDownsamplerTests.cs new file mode 100644 index 00000000..332f203c --- /dev/null +++ b/Daqifi.Desktop.Test/Helpers/MinMaxDownsamplerTests.cs @@ -0,0 +1,208 @@ +using Daqifi.Desktop.Helpers; +using OxyPlot; + +namespace Daqifi.Desktop.Test.Helpers; + +[TestClass] +public class MinMaxDownsamplerTests +{ + #region Downsample Tests + + [TestMethod] + public void Downsample_EmptyList_ReturnsEmpty() + { + var result = MinMaxDownsampler.Downsample(new List(), 10); + Assert.AreEqual(0, result.Count); + } + + [TestMethod] + public void Downsample_FewPointsBelowThreshold_ReturnsAll() + { + var points = new List + { + new(0, 1), new(1, 2), new(2, 3) + }; + var result = MinMaxDownsampler.Downsample(points, 10); + Assert.AreEqual(3, result.Count); + } + + [TestMethod] + public void Downsample_LargeDataset_ProducesCorrectSize() + { + var points = new List(); + for (var i = 0; i < 10000; i++) + { + points.Add(new DataPoint(i, Math.Sin(i * 0.01))); + } + + var result = MinMaxDownsampler.Downsample(points, 100); + Assert.IsTrue(result.Count <= 200); + Assert.IsTrue(result.Count > 0); + } + + [TestMethod] + public void Downsample_PreservesMinMax() + { + var points = new List(); + for (var i = 0; i < 10000; i++) + { + points.Add(new DataPoint(i, Math.Sin(i * 0.01))); + } + + var result = MinMaxDownsampler.Downsample(points, 100); + var resultMax = result.Max(p => p.Y); + var resultMin = result.Min(p => p.Y); + var sourceMax = points.Max(p => p.Y); + var sourceMin = points.Min(p => p.Y); + + Assert.IsTrue(Math.Abs(resultMax - sourceMax) < 0.05, "Should preserve approximate max"); + Assert.IsTrue(Math.Abs(resultMin - sourceMin) < 0.05, "Should preserve approximate min"); + } + + #endregion + + #region Sub-range Downsample Tests + + [TestMethod] + public void Downsample_SubRange_OperatesOnCorrectRange() + { + var points = new List(); + for (var i = 0; i < 1000; i++) + { + points.Add(new DataPoint(i, i < 500 ? 0 : 100)); + } + + // Downsample only the second half (where values are 100) + var result = MinMaxDownsampler.Downsample(points, 500, 1000, 50); + Assert.IsTrue(result.All(p => p.Y == 100), "All downsampled points from second half should be 100"); + } + + [TestMethod] + public void Downsample_SubRange_EmptyRange_ReturnsEmpty() + { + var points = new List { new(0, 1), new(1, 2) }; + var result = MinMaxDownsampler.Downsample(points, 0, 0, 10); + Assert.AreEqual(0, result.Count); + } + + [TestMethod] + public void Downsample_SubRange_SmallRange_ReturnsAll() + { + var points = new List(); + for (var i = 0; i < 100; i++) + { + points.Add(new DataPoint(i, i)); + } + + // Sub-range of 5 points with bucket count of 10 — should return all 5 + var result = MinMaxDownsampler.Downsample(points, 10, 15, 10); + Assert.AreEqual(5, result.Count); + } + + #endregion + + #region FindVisibleRange Tests + + [TestMethod] + public void FindVisibleRange_EmptyList_ReturnsZeroRange() + { + var (start, end) = MinMaxDownsampler.FindVisibleRange(new List(), 0, 10); + Assert.AreEqual(0, start); + Assert.AreEqual(0, end); + } + + [TestMethod] + public void FindVisibleRange_AllVisible_ReturnsFullRange() + { + var points = new List + { + new(0, 0), new(1, 1), new(2, 2), new(3, 3), new(4, 4) + }; + + var (start, end) = MinMaxDownsampler.FindVisibleRange(points, -1, 5); + Assert.AreEqual(0, start); + Assert.AreEqual(5, end); + } + + [TestMethod] + public void FindVisibleRange_MiddleSection_ReturnsPaddedRange() + { + var points = new List(); + for (var i = 0; i < 100; i++) + { + points.Add(new DataPoint(i, i)); + } + + var (start, end) = MinMaxDownsampler.FindVisibleRange(points, 30, 60); + + // Should include padding: one point before 30 and one point after 60 + Assert.IsTrue(start <= 29, $"Start ({start}) should be at or before index 29"); + Assert.IsTrue(end >= 62, $"End ({end}) should be at or after index 62"); + } + + [TestMethod] + public void FindVisibleRange_NoPointsInRange_IncludesAdjacentPoints() + { + var points = new List + { + new(0, 0), new(1, 1), new(10, 10), new(11, 11) + }; + + // Range 5-8 has no points; binary search lands between index 1 (X=1) and 2 (X=10) + // With padding: start backs up 1 from index 2 → 1, end advances 1 from index 2 → 3 + var (start, end) = MinMaxDownsampler.FindVisibleRange(points, 5, 8); + Assert.AreEqual(1, start, "Should include point at X=1 (one before gap)"); + Assert.AreEqual(3, end, "Should include point at X=10 (one after gap)"); + } + + [TestMethod] + public void FindVisibleRange_AtBoundaries_ClampsCorrectly() + { + var points = new List + { + new(0, 0), new(1, 1), new(2, 2) + }; + + // Range starts before data + var (start, _) = MinMaxDownsampler.FindVisibleRange(points, -10, 1); + Assert.AreEqual(0, start, "Start should be clamped to 0"); + + // Range ends after data + var (_, end) = MinMaxDownsampler.FindVisibleRange(points, 1, 100); + Assert.AreEqual(3, end, "End should be clamped to list length"); + } + + [TestMethod] + public void FindVisibleRange_SinglePoint_ReturnsIt() + { + var points = new List { new(5, 10) }; + + var (start, end) = MinMaxDownsampler.FindVisibleRange(points, 0, 10); + Assert.AreEqual(0, start); + Assert.AreEqual(1, end); + } + + [TestMethod] + public void FindVisibleRange_LargeDataset_Performance() + { + var points = new List(); + for (var i = 0; i < 1_000_000; i++) + { + points.Add(new DataPoint(i, Math.Sin(i * 0.001))); + } + + var sw = System.Diagnostics.Stopwatch.StartNew(); + for (var i = 0; i < 1000; i++) + { + MinMaxDownsampler.FindVisibleRange(points, 400000, 600000); + } + sw.Stop(); + + // 1000 binary searches on 1M points should be well under 1 second, + // even on slow CI runners. Typical desktop: < 10ms. + Assert.IsTrue(sw.ElapsedMilliseconds < 1000, + $"1000 binary searches took {sw.ElapsedMilliseconds}ms, expected < 1000ms"); + } + + #endregion +} diff --git a/Daqifi.Desktop/Helpers/MinMaxDownsampler.cs b/Daqifi.Desktop/Helpers/MinMaxDownsampler.cs index c50733dc..e6456efa 100644 --- a/Daqifi.Desktop/Helpers/MinMaxDownsampler.cs +++ b/Daqifi.Desktop/Helpers/MinMaxDownsampler.cs @@ -23,24 +23,54 @@ public static List Downsample(IReadOnlyList points, int bu return []; } - if (points.Count <= bucketCount * 2) + return Downsample(points, 0, points.Count, bucketCount); + } + + /// + /// Downsamples a sub-range of a sorted list using min/max aggregation per bucket. + /// Operates on indices [, ) + /// without copying the source list. + /// + /// Time-sorted data points. + /// Inclusive start index of the sub-range. + /// Exclusive end index of the sub-range. + /// Number of buckets to divide the time range into. + /// A downsampled list of data points preserving the visual envelope. + public static List Downsample(IReadOnlyList points, int startIndex, int endIndex, int bucketCount) + { + ArgumentNullException.ThrowIfNull(points); + ArgumentOutOfRangeException.ThrowIfNegative(startIndex); + ArgumentOutOfRangeException.ThrowIfGreaterThan(endIndex, points.Count); + + var count = endIndex - startIndex; + if (count <= 0 || bucketCount <= 0) { - return new List(points); + return []; } - var result = new List(bucketCount * 2); + if (count <= bucketCount * 2) + { + var result = new List(count); + for (var i = startIndex; i < endIndex; i++) + { + result.Add(points[i]); + } + return result; + } - var xMin = points[0].X; - var xMax = points[points.Count - 1].X; + var output = new List(bucketCount * 2); + + var xMin = points[startIndex].X; + var xMax = points[endIndex - 1].X; var xRange = xMax - xMin; if (xRange <= 0) { - return [points[0]]; + return [points[startIndex]]; } var bucketWidth = xRange / bucketCount; - var pointIndex = 0; + var pointIndex = startIndex; for (var bucket = 0; bucket < bucketCount; bucket++) { @@ -54,7 +84,7 @@ public static List Downsample(IReadOnlyList points, int bu var hasPoints = false; var isLastBucket = bucket == bucketCount - 1; - while (pointIndex < points.Count && (isLastBucket || points[pointIndex].X < bucketEnd)) + while (pointIndex < endIndex && (isLastBucket || points[pointIndex].X < bucketEnd)) { var p = points[pointIndex]; hasPoints = true; @@ -82,22 +112,100 @@ public static List Downsample(IReadOnlyList points, int bu // Emit min and max in X-order to preserve visual continuity if (minYX <= maxYX) { - result.Add(new DataPoint(minYX, minY)); + output.Add(new DataPoint(minYX, minY)); if (Math.Abs(minY - maxY) > double.Epsilon) { - result.Add(new DataPoint(maxYX, maxY)); + output.Add(new DataPoint(maxYX, maxY)); } } else { - result.Add(new DataPoint(maxYX, maxY)); + output.Add(new DataPoint(maxYX, maxY)); if (Math.Abs(minY - maxY) > double.Epsilon) { - result.Add(new DataPoint(minYX, minY)); + output.Add(new DataPoint(minYX, minY)); } } } - return result; + return output; + } + + /// + /// Finds the index range [startIndex, endIndex) of points whose X values + /// fall within [xMin, xMax], with one-point padding on each side for visual continuity. + /// Uses binary search for O(log n) performance on sorted data. + /// + /// Time-sorted data points. + /// Minimum visible X value. + /// Maximum visible X value. + /// Tuple of (inclusive start index, exclusive end index). + public static (int startIndex, int endIndex) FindVisibleRange( + IReadOnlyList sortedPoints, double xMin, double xMax) + { + if (sortedPoints.Count == 0) + { + return (0, 0); + } + + // Binary search for first index where X >= xMin, then back up 1 for continuity + var start = BinarySearchLower(sortedPoints, xMin); + if (start > 0) + { + start--; + } + + // Binary search for first index where X > xMax, then add 1 for continuity + var end = BinarySearchUpper(sortedPoints, xMax); + if (end < sortedPoints.Count) + { + end++; + } + + return (start, end); + } + + /// + /// Returns the index of the first element whose X is >= value. + /// + private static int BinarySearchLower(IReadOnlyList points, double value) + { + var lo = 0; + var hi = points.Count; + while (lo < hi) + { + var mid = lo + (hi - lo) / 2; + if (points[mid].X < value) + { + lo = mid + 1; + } + else + { + hi = mid; + } + } + return lo; + } + + /// + /// Returns the index of the first element whose X is > value. + /// + private static int BinarySearchUpper(IReadOnlyList points, double value) + { + var lo = 0; + var hi = points.Count; + while (lo < hi) + { + var mid = lo + (hi - lo) / 2; + if (points[mid].X <= value) + { + lo = mid + 1; + } + else + { + hi = mid; + } + } + return lo; } } diff --git a/Daqifi.Desktop/Loggers/DatabaseLogger.cs b/Daqifi.Desktop/Loggers/DatabaseLogger.cs index f960687a..a83c73ac 100644 --- a/Daqifi.Desktop/Loggers/DatabaseLogger.cs +++ b/Daqifi.Desktop/Loggers/DatabaseLogger.cs @@ -18,6 +18,7 @@ using EFCore.BulkExtensions; using CommunityToolkit.Mvvm.ComponentModel; using CommunityToolkit.Mvvm.Input; +using System.Windows.Threading; using Application = System.Windows.Application; using FontWeights = OxyPlot.FontWeights; @@ -99,18 +100,21 @@ public LoggedSeriesLegendItem( } } -public partial class DatabaseLogger : ObservableObject, ILogger +public partial class DatabaseLogger : ObservableObject, ILogger, IDisposable { #region Constants private const int MINIMAP_BUCKET_COUNT = 800; + private const int MAIN_PLOT_BUCKET_COUNT = 2000; + private const int INITIAL_LOAD_POINTS = 100_000; + private const int SAMPLED_POINTS_PER_CHANNEL = 3000; #endregion #region Private Data public ObservableCollection LegendItems { get; } = new(); public ObservableCollection DeviceLegendGroups { get; } = new(); private readonly Dictionary<(string deviceSerial, string channelName), List> _allSessionPoints = new(); + private readonly Dictionary<(string deviceSerial, string channelName), List> _downsampledCache = new(); private readonly BlockingCollection _buffer = new(); - private readonly Dictionary<(string deviceSerial, string channelName), List> _sessionPoints = new(); private readonly Dictionary<(string deviceSerial, string channelName), LineSeries> _minimapSeries = new(); private DateTime? _firstTime; @@ -121,6 +125,14 @@ public partial class DatabaseLogger : ObservableObject, ILogger private RectangleAnnotation _minimapDimLeft; private RectangleAnnotation _minimapDimRight; private MinimapInteractionController _minimapInteraction; + internal bool IsSyncingFromMinimap; + private double _lastViewportMin = double.NaN; + private double _lastViewportMax = double.NaN; + private bool _viewportDirty; + private DispatcherTimer _viewportThrottleTimer; + private DispatcherTimer _settleTimer; + private int? _currentSessionId; + private CancellationTokenSource _fetchCts; [ObservableProperty] private PlotModel _plotModel; @@ -143,6 +155,13 @@ public partial class DatabaseLogger : ObservableObject, ILogger /// [ObservableProperty] private bool _hasSessionData; + + /// + /// True while fetching high-fidelity data from the database in the background. + /// Bound to a subtle progress indicator in the UI. + /// + [ObservableProperty] + private bool _isRefiningData; #endregion #region Legend @@ -225,6 +244,22 @@ public DatabaseLogger(IDbContextFactory loggingContext) // Subscribe to main time axis changes for minimap sync timeAxis.AxisChanged += OnMainTimeAxisChanged; + // Throttle viewport updates from main plot interaction to 60fps + _viewportThrottleTimer = new DispatcherTimer(DispatcherPriority.Render) + { + Interval = TimeSpan.FromMilliseconds(16) + }; + _viewportThrottleTimer.Tick += OnViewportThrottleTick; + _viewportThrottleTimer.Start(); + + // Settle timer: triggers high-fidelity DB fetch 200ms after the last + // viewport change (covers both main plot pan/zoom and minimap drag) + _settleTimer = new DispatcherTimer(DispatcherPriority.Background) + { + Interval = TimeSpan.FromMilliseconds(200) + }; + _settleTimer.Tick += OnSettleTick; + // Initialize minimap PlotModel InitializeMinimapPlotModel(); @@ -325,7 +360,8 @@ private void InitializeMinimapPlotModel() MinimapPlotModel, _minimapSelectionRect, _minimapDimLeft, - _minimapDimRight); + _minimapDimRight, + this); } #endregion @@ -397,14 +433,29 @@ public void ClearPlot() Application.Current.Dispatcher.Invoke(() => { _firstTime = null; - _sessionPoints.Clear(); + _currentSessionId = null; + _lastViewportMin = double.NaN; + _lastViewportMax = double.NaN; + _fetchCts?.Cancel(); + _fetchCts?.Dispose(); + _fetchCts = null; + IsRefiningData = false; + _settleTimer.Stop(); _allSessionPoints.Clear(); + _downsampledCache.Clear(); _minimapSeries.Clear(); PlotModel.Series.Clear(); LegendItems.Clear(); DeviceLegendGroups.Clear(); PlotModel.Title = string.Empty; PlotModel.Subtitle = string.Empty; + + // Reset all axes so the new session starts at full extent + foreach (var axis in PlotModel.Axes) + { + axis.Reset(); + } + PlotModel.InvalidatePlot(true); MinimapPlotModel.Series.Clear(); @@ -420,35 +471,47 @@ public void DisplayLoggingSession(LoggingSession session) { // ClearPlot is already dispatcher-wrapped ClearPlot(); + _currentSessionId = session.ID; - // Data fetching and processing (can be on background thread) var sessionName = session.Name; var subtitle = string.Empty; - var tempSeriesList = new List(); var tempLegendItemsList = new List(); + int totalSamplesCount; + // ── Phase 1: Fast initial load (<1s) ────────────────────────── + // Get channel metadata from first timestamp (6ms via index) + // and load a small initial batch for immediate display using (var context = _loggingContext.CreateDbContext()) { context.ChangeTracker.AutoDetectChangesEnabled = false; - var dbSamples = context.Samples.AsNoTracking() - .Where(s => s.LoggingSessionID == session.ID) - .OrderBy(s => s.TimestampTicks) - .Select(s => new { s.ChannelName, s.DeviceSerialNo, s.Type, s.Color, s.TimestampTicks, s.Value }) - .ToList(); // Bring data into memory + var baseQuery = context.Samples.AsNoTracking() + .Where(s => s.LoggingSessionID == session.ID); - var samplesCount = dbSamples.Count; - const int dataPointsToShow = 1000000; + // Get the first timestamp to extract channel info (instant via composite index) + var firstSample = baseQuery + .OrderBy(s => s.TimestampTicks) + .Select(s => new { s.TimestampTicks }) + .FirstOrDefault(); - if (samplesCount > dataPointsToShow) + if (firstSample == null) { - subtitle = $"\nOnly showing {dataPointsToShow:n0} out of {samplesCount:n0} data points"; + // Empty session + Application.Current.Dispatcher.Invoke(() => + { + PlotModel.Title = sessionName; + HasSessionData = false; + PlotModel.InvalidatePlot(true); + }); + return; } - var channelInfoList = dbSamples + // Get all channels from the first timestamp (32 rows, instant) + var channelInfoList = baseQuery + .Where(s => s.TimestampTicks == firstSample.TimestampTicks) .Select(s => new { s.ChannelName, s.DeviceSerialNo, s.Type, s.Color }) - .Distinct() + .ToList() .NaturalOrderBy(s => s.ChannelName) .ToList(); @@ -459,10 +522,12 @@ public void DisplayLoggingSession(LoggingSession session) tempLegendItemsList.Add(legendItem); } - // This part still needs to be careful about _allSessionPoints access if it's used by UI directly - // For now, _allSessionPoints is used to populate series ItemsSource later on UI thread - var dataSampleCount = 0; - foreach (var sample in dbSamples) + // Load initial batch for fast display (100K rows, ~16ms via index) + foreach (var sample in baseQuery + .OrderBy(s => s.TimestampTicks) + .Select(s => new { s.ChannelName, s.DeviceSerialNo, s.TimestampTicks, s.Value }) + .Take(INITIAL_LOAD_POINTS) + .AsEnumerable()) { var key = (sample.DeviceSerialNo, sample.ChannelName); if (_firstTime == null) { _firstTime = new DateTime(sample.TimestampTicks); } @@ -472,111 +537,316 @@ public void DisplayLoggingSession(LoggingSession session) { points.Add(new DataPoint(deltaTime, sample.Value)); } - - dataSampleCount++; - if (dataSampleCount >= dataPointsToShow) - { - break; - } } - } - // Prepare downsampled minimap data on the background thread - var minimapSeriesData = new List<(string channelName, string deviceSerial, OxyColor color, List downsampled)>(); - foreach (var kvp in _allSessionPoints) - { - if (kvp.Value.Count > 0) - { - var downsampled = MinMaxDownsampler.Downsample(kvp.Value, MINIMAP_BUCKET_COUNT); - var matchingSeries = tempSeriesList.FirstOrDefault(s => - { - var parts = s.Title.Split([" : ("], StringSplitOptions.None); - return parts.Length == 2 && parts[0] == kvp.Key.channelName && parts[1].TrimEnd(')') == kvp.Key.deviceSerial; - }); - minimapSeriesData.Add((kvp.Key.channelName, kvp.Key.deviceSerial, matchingSeries?.Color ?? OxyColors.Gray, downsampled)); - } + totalSamplesCount = baseQuery.Count(); } - // Update UI-bound collections and properties on the UI thread + // Show the initial data immediately + var initialMinimapData = PrepareMinimapData(tempSeriesList); Application.Current.Dispatcher.Invoke(() => { PlotModel.Title = sessionName; - PlotModel.Subtitle = subtitle; + PlotModel.Subtitle = totalSamplesCount > INITIAL_LOAD_POINTS + ? "\nLoading full dataset..." + : string.Empty; + + SetupUiCollections(tempSeriesList, tempLegendItemsList); + SetupMinimapSeries(initialMinimapData); + HasSessionData = tempSeriesList.Count > 0; + PlotModel.InvalidatePlot(true); + }); - foreach (var legendItem in tempLegendItemsList) + // ── Phase 2: Load sampled data covering full time range (~1-3s) ── + // Instead of streaming all 10M+ rows (30s), do N targeted index + // seeks spread across the time range. Each seek reads one batch + // of interleaved channel data at that timestamp position. + // Result: ~96K rows covering the full range in ~1-3 seconds. + if (totalSamplesCount > INITIAL_LOAD_POINTS) + { + // Clear phase 1 data and reload with sampled data + foreach (var kvp in _allSessionPoints) { - LegendItems.Add(legendItem); + kvp.Value.Clear(); } + _firstTime = null; + + LoadSampledData(session.ID, tempSeriesList.Count); - // Build grouped legend by device - DeviceLegendGroups.Clear(); - var groupDict = new Dictionary(); - foreach (var legendItem in tempLegendItemsList) + // Refresh UI with sampled full-range data + var fullMinimapData = PrepareMinimapData(tempSeriesList); + Application.Current.Dispatcher.Invoke(() => { - if (!groupDict.TryGetValue(legendItem.DeviceSerialNo, out var group)) + PlotModel.Subtitle = string.Empty; + + // Update main plot series with full downsampled data + foreach (var series in PlotModel.Series.OfType()) { - group = new DeviceLegendGroup(legendItem.DeviceSerialNo); - groupDict[legendItem.DeviceSerialNo] = group; - DeviceLegendGroups.Add(group); + if (series.Tag is (string deviceSerial, string channelName) + && _allSessionPoints.TryGetValue((deviceSerial, channelName), out var points)) + { + var key = (deviceSerial, channelName); + if (!_downsampledCache.TryGetValue(key, out var cached)) + { + cached = new List(MAIN_PLOT_BUCKET_COUNT * 2); + _downsampledCache[key] = cached; + } + + cached.Clear(); + if (points.Count > MAIN_PLOT_BUCKET_COUNT * 2) + { + cached.AddRange(MinMaxDownsampler.Downsample(points, MAIN_PLOT_BUCKET_COUNT)); + } + else + { + cached.AddRange(points); + } + + series.ItemsSource = cached; + } } - group.Channels.Add(legendItem); - } - foreach (var series in tempSeriesList) + // Refresh minimap with full-range data + SetupMinimapSeries(fullMinimapData); + _lastViewportMin = double.NaN; + _lastViewportMax = double.NaN; + PlotModel.InvalidatePlot(true); + }); + } + } + catch (Exception ex) + { + _appLogger.Error(ex, "Failed in DisplayLoggingSession"); + } + } + + /// + /// Prepares downsampled minimap series data from _allSessionPoints on the background thread. + /// + private List<(string channelName, string deviceSerial, OxyColor color, List downsampled)> + PrepareMinimapData(List seriesList) + { + var result = new List<(string channelName, string deviceSerial, OxyColor color, List downsampled)>(); + foreach (var kvp in _allSessionPoints) + { + if (kvp.Value.Count > 0) + { + var downsampled = MinMaxDownsampler.Downsample(kvp.Value, MINIMAP_BUCKET_COUNT); + var matchingSeries = seriesList.FirstOrDefault(s => + s.Tag is (string ds, string cn) && ds == kvp.Key.deviceSerial && cn == kvp.Key.channelName); + result.Add((kvp.Key.channelName, kvp.Key.deviceSerial, matchingSeries?.Color ?? OxyColors.Gray, downsampled)); + } + } + return result; + } + + /// + /// Loads a uniformly sampled subset of data covering the full time range + /// using targeted index seeks. Instead of reading all N million rows, + /// divides the time range into SAMPLED_POINTS_PER_CHANNEL segments and + /// seeks to each segment boundary via the composite index. Each seek + /// reads one batch of interleaved channel data (~channelCount rows). + /// Result: ~3000 points per channel in ~1-3 seconds regardless of total dataset size. + /// + private void LoadSampledData(int sessionId, int channelCount) + { + using var context = _loggingContext.CreateDbContext(); + var connection = context.Database.GetDbConnection(); + connection.Open(); + + // Get time bounds via index (instant) + long minTicks, maxTicks; + using (var boundsCmd = connection.CreateCommand()) + { + boundsCmd.CommandText = @" + SELECT MIN(TimestampTicks), MAX(TimestampTicks) + FROM Samples + WHERE LoggingSessionID = @id"; + var idParam = boundsCmd.CreateParameter(); + idParam.ParameterName = "@id"; + idParam.Value = sessionId; + boundsCmd.Parameters.Add(idParam); + + using var reader = boundsCmd.ExecuteReader(); + if (!reader.Read() || reader.IsDBNull(0)) + { + return; + } + + minTicks = reader.GetInt64(0); + maxTicks = reader.GetInt64(1); + } + + if (minTicks >= maxTicks) + { + return; + } + + _firstTime = new DateTime(minTicks); + var tickStep = Math.Max(1, (maxTicks - minTicks) / SAMPLED_POINTS_PER_CHANNEL); + // Read at least channelCount rows per seek to get one sample per channel + var batchSize = Math.Max(channelCount * 2, 100); + + // Prepared statement for repeated seeks + using var seekCmd = connection.CreateCommand(); + seekCmd.CommandText = @" + SELECT ChannelName, DeviceSerialNo, TimestampTicks, Value + FROM Samples + WHERE LoggingSessionID = @id AND TimestampTicks >= @t + ORDER BY TimestampTicks + LIMIT @limit"; + + var seekIdParam = seekCmd.CreateParameter(); + seekIdParam.ParameterName = "@id"; + seekIdParam.Value = sessionId; + seekCmd.Parameters.Add(seekIdParam); + + var seekTParam = seekCmd.CreateParameter(); + seekTParam.ParameterName = "@t"; + seekTParam.Value = minTicks; + seekCmd.Parameters.Add(seekTParam); + + var seekLimitParam = seekCmd.CreateParameter(); + seekLimitParam.ParameterName = "@limit"; + seekLimitParam.Value = batchSize; + seekCmd.Parameters.Add(seekLimitParam); + + seekCmd.Prepare(); + + // Track which timestamps we've already added to avoid duplicates + // from overlapping batches + var lastAddedTimestamp = new Dictionary<(string, string), long>(); + + // Use <= so the final iteration (i == SAMPLED_POINTS_PER_CHANNEL) + // seeks at maxTicks, ensuring the session tail is always included + for (var i = 0; i <= SAMPLED_POINTS_PER_CHANNEL; i++) + { + var seekTimestamp = i < SAMPLED_POINTS_PER_CHANNEL + ? minTicks + i * tickStep + : maxTicks; + seekTParam.Value = seekTimestamp; + + using var reader = seekCmd.ExecuteReader(); + while (reader.Read()) + { + var channelName = reader.GetString(0); + var deviceSerialNo = reader.GetString(1); + var timestampTicks = reader.GetInt64(2); + var value = reader.GetDouble(3); + + var key = (deviceSerialNo, channelName); + + // Skip duplicate timestamps from overlapping batches + if (lastAddedTimestamp.TryGetValue(key, out var lastT) && timestampTicks <= lastT) { - PlotModel.Series.Add(series); - // Assign data to series (ItemsSource) - // The key for _allSessionPoints must match how it was populated - var key = (series.Title.Split([" : ("], StringSplitOptions.None)[1].TrimEnd(')'), series.Title.Split([" : ("], StringSplitOptions.None)[0]); - if (_allSessionPoints.TryGetValue(key, out var points)) - { - series.ItemsSource = points; - } + continue; } - // Populate minimap with downsampled series - MinimapPlotModel.Series.Clear(); - _minimapSeries.Clear(); - foreach (var (channelName, deviceSerial, color, downsampled) in minimapSeriesData) + lastAddedTimestamp[key] = timestampTicks; + + var deltaTime = (timestampTicks - _firstTime.Value.Ticks) / 10000.0; + if (_allSessionPoints.TryGetValue(key, out var points)) { - var minimapLine = new LineSeries - { - Color = color, - StrokeThickness = 1, - ItemsSource = downsampled, - XAxisKey = "MinimapTime", - YAxisKey = "MinimapY" - }; - MinimapPlotModel.Series.Add(minimapLine); - _minimapSeries[(deviceSerial, channelName)] = minimapLine; + points.Add(new DataPoint(deltaTime, value)); } + } + } + } + + /// + /// Sets up UI collections (legend items, device groups, series) on the UI thread. + /// + private void SetupUiCollections(List seriesList, List legendItems) + { + foreach (var legendItem in legendItems) + { + LegendItems.Add(legendItem); + } - MinimapPlotModel.ResetAllAxes(); + DeviceLegendGroups.Clear(); + var groupDict = new Dictionary(); + foreach (var legendItem in legendItems) + { + if (!groupDict.TryGetValue(legendItem.DeviceSerialNo, out var group)) + { + group = new DeviceLegendGroup(legendItem.DeviceSerialNo); + groupDict[legendItem.DeviceSerialNo] = group; + DeviceLegendGroups.Add(group); + } + group.Channels.Add(legendItem); + } - // Initialize selection rectangle to full data range - // Use data bounds directly since ActualMinimum/Maximum aren't set until render - if (minimapSeriesData.Count > 0) + foreach (var series in seriesList) + { + PlotModel.Series.Add(series); + if (series.Tag is (string deviceSerial, string channelName) + && _allSessionPoints.TryGetValue((deviceSerial, channelName), out var points)) + { + var key = (deviceSerial, channelName); + if (!_downsampledCache.TryGetValue(key, out var cached)) { - var dataMinX = minimapSeriesData.Where(d => d.downsampled.Count > 0).Min(d => d.downsampled[0].X); - var dataMaxX = minimapSeriesData.Where(d => d.downsampled.Count > 0).Max(d => d.downsampled[^1].X); - _minimapSelectionRect.MinimumX = dataMinX; - _minimapSelectionRect.MaximumX = dataMaxX; - _minimapDimLeft.MaximumX = dataMinX; - _minimapDimRight.MinimumX = dataMaxX; + cached = new List(MAIN_PLOT_BUCKET_COUNT * 2); + _downsampledCache[key] = cached; } - MinimapPlotModel.InvalidatePlot(true); + cached.Clear(); + if (points.Count > MAIN_PLOT_BUCKET_COUNT * 2) + { + cached.AddRange(MinMaxDownsampler.Downsample(points, MAIN_PLOT_BUCKET_COUNT)); + } + else + { + cached.AddRange(points); + } - HasSessionData = tempSeriesList.Count > 0; + series.ItemsSource = cached; + } + } + } - OnPropertyChanged("SessionPoints"); // If SessionPoints is still relevant - PlotModel.InvalidatePlot(true); - }); + /// + /// Populates minimap series and sets up the selection rectangle on the UI thread. + /// + private void SetupMinimapSeries( + List<(string channelName, string deviceSerial, OxyColor color, List downsampled)> minimapData) + { + MinimapPlotModel.Series.Clear(); + _minimapSeries.Clear(); + foreach (var (channelName, deviceSerial, color, downsampled) in minimapData) + { + var minimapLine = new LineSeries + { + Color = color, + StrokeThickness = 1, + ItemsSource = downsampled, + XAxisKey = "MinimapTime", + YAxisKey = "MinimapY" + }; + MinimapPlotModel.Series.Add(minimapLine); + _minimapSeries[(deviceSerial, channelName)] = minimapLine; } - catch (Exception ex) + + // Set minimap axes from source data bounds (not auto-range, which + // reads downsampled ItemsSource and may have shifted boundaries) + var nonEmpty = minimapData.Where(d => d.downsampled.Count > 0).ToList(); + if (nonEmpty.Count > 0) { - _appLogger.Error(ex, "Failed in DisplayLoggingSession"); + var dataMinX = nonEmpty.Min(d => d.downsampled[0].X); + var dataMaxX = nonEmpty.Max(d => d.downsampled[^1].X); + + var minimapTimeAxis = MinimapPlotModel.Axes.FirstOrDefault(a => a.Key == "MinimapTime"); + minimapTimeAxis?.Zoom(dataMinX, dataMaxX); + + var minimapYAxis = MinimapPlotModel.Axes.FirstOrDefault(a => a.Key == "MinimapY"); + minimapYAxis?.Reset(); + + _minimapSelectionRect.MinimumX = dataMinX; + _minimapSelectionRect.MaximumX = dataMaxX; + _minimapDimLeft.MaximumX = dataMinX; + _minimapDimRight.MinimumX = dataMaxX; } + + MinimapPlotModel.InvalidatePlot(true); } public void DeleteLoggingSession(LoggingSession session) @@ -657,15 +927,14 @@ public void ResumeConsumer() private (LineSeries series, LoggedSeriesLegendItem legendItem) AddChannelSeries(string channelName, string deviceSerialNo, ChannelType type, string color) { var key = (DeviceSerialNo: deviceSerialNo, channelName); - _sessionPoints.Add(key, []); _allSessionPoints.Add(key, []); var newLineSeries = new LineSeries { Title = $"{channelName} : ({deviceSerialNo})", - ItemsSource = _sessionPoints.Last().Value, // This will be empty initially, data is added later + Tag = (deviceSerialNo, channelName), Color = OxyColor.Parse(color), - IsVisible = true // Default to visible + IsVisible = true }; var legendItem = new LoggedSeriesLegendItem( @@ -677,7 +946,6 @@ public void ResumeConsumer() newLineSeries, PlotModel, this); - // LegendItems.Add(legendItem); // Removed: To be added in DisplayLoggingSession on UI thread newLineSeries.YAxisKey = type switch { @@ -686,14 +954,20 @@ public void ResumeConsumer() _ => newLineSeries.YAxisKey }; - // PlotModel.Series.Add(newLineSeries); // Removed: To be added in DisplayLoggingSession on UI thread - // OnPropertyChanged("PlotModel"); // Removed: To be called in DisplayLoggingSession on UI thread return (newLineSeries, legendItem); } #region Minimap Synchronization private void OnMainTimeAxisChanged(object? sender, AxisChangedEventArgs e) { + if (IsSyncingFromMinimap) + { + return; + } + + // Mark viewport dirty — the throttle timer will handle the actual update at 60fps + _viewportDirty = true; + var timeAxis = PlotModel.Axes.FirstOrDefault(a => a.Key == "Time"); if (timeAxis == null) { @@ -707,6 +981,380 @@ private void OnMainTimeAxisChanged(object? sender, AxisChangedEventArgs e) MinimapPlotModel.InvalidatePlot(false); } + /// + /// Throttled viewport update tick — processes dirty flag at 60fps to avoid + /// re-downsampling on every mouse move during main plot pan/zoom. + /// + private void OnViewportThrottleTick(object? sender, EventArgs e) + { + if (!_viewportDirty) + { + return; + } + + _viewportDirty = false; + UpdateMainPlotViewport(highFidelity: false); + PlotModel.InvalidatePlot(true); + + // Restart the settle timer — it will fire 200ms after the last change + _settleTimer.Stop(); + _settleTimer.Start(); + } + + /// + /// Fires 200ms after the last viewport change. Triggers a high-fidelity + /// DB fetch so zoomed-in views show full-resolution data once interaction settles. + /// + private void OnSettleTick(object? sender, EventArgs e) + { + _settleTimer.Stop(); + _lastViewportMin = double.NaN; + _lastViewportMax = double.NaN; + UpdateMainPlotViewport(highFidelity: true); + PlotModel.InvalidatePlot(true); + } + + /// + /// Re-downsamples each main plot series for the currently visible time range. + /// When zoomed in far enough that the sampled in-memory data is too sparse, + /// fetches full-resolution data from the database for the visible window. + /// Skips the update if the viewport hasn't changed since the last call. + /// + private void UpdateMainPlotViewport(bool highFidelity = true) + { + var timeAxis = PlotModel.Axes.FirstOrDefault(a => a.Key == "Time"); + if (timeAxis == null) + { + return; + } + + var visibleMin = timeAxis.ActualMinimum; + var visibleMax = timeAxis.ActualMaximum; + + // Skip if viewport hasn't changed + if (visibleMin == _lastViewportMin && visibleMax == _lastViewportMax) + { + return; + } + + _lastViewportMin = visibleMin; + _lastViewportMax = visibleMax; + + // Cancel any in-flight DB fetch — the viewport has changed, so its + // results would be stale and could overwrite the current view + if (_fetchCts != null) + { + _fetchCts.Cancel(); + _fetchCts.Dispose(); + _fetchCts = null; + IsRefiningData = false; + } + + // During drag (highFidelity=false), always use fast in-memory data to + // maintain smooth 60fps. DB fetches only happen on settle (mouse up, + // zoom buttons) when highFidelity=true. + if (!highFidelity) + { + UpdateSeriesFromMemory(visibleMin, visibleMax); + return; + } + + // Check if ANY channel's sampled in-memory data is too sparse for + // this zoom level. If so, fetch full-resolution data from the DB. + var needsDbFetch = false; + if (_currentSessionId.HasValue && _firstTime.HasValue) + { + foreach (var kvp in _allSessionPoints) + { + if (kvp.Value.Count == 0) + { + continue; + } + + // Only check channels that are actually sampled (not full datasets) + if (kvp.Value.Count < SAMPLED_POINTS_PER_CHANNEL / 2) + { + continue; + } + + var (si, ei) = MinMaxDownsampler.FindVisibleRange(kvp.Value, visibleMin, visibleMax); + var sampledVisible = ei - si; + if (sampledVisible < MAIN_PLOT_BUCKET_COUNT) + { + needsDbFetch = true; + break; + } + } + } + + if (needsDbFetch) + { + FetchViewportDataFromDb(visibleMin, visibleMax); + } + else + { + UpdateSeriesFromMemory(visibleMin, visibleMax); + } + } + + /// + /// Updates series ItemsSource from the in-memory sampled data. + /// Used when the sampled data has sufficient density for the current viewport. + /// + private void UpdateSeriesFromMemory(double visibleMin, double visibleMax) + { + foreach (var series in PlotModel.Series.OfType()) + { + if (series.Tag is not (string deviceSerial, string channelName)) + { + continue; + } + + var key = (deviceSerial, channelName); + if (!_allSessionPoints.TryGetValue(key, out var allPoints) || allPoints.Count == 0) + { + continue; + } + + var (startIdx, endIdx) = MinMaxDownsampler.FindVisibleRange(allPoints, visibleMin, visibleMax); + var visibleCount = endIdx - startIdx; + + if (!_downsampledCache.TryGetValue(key, out var cached)) + { + cached = new List(MAIN_PLOT_BUCKET_COUNT * 2); + _downsampledCache[key] = cached; + } + + if (visibleCount <= MAIN_PLOT_BUCKET_COUNT * 2) + { + cached.Clear(); + for (var i = startIdx; i < endIdx; i++) + { + cached.Add(allPoints[i]); + } + } + else + { + var downsampled = MinMaxDownsampler.Downsample(allPoints, startIdx, endIdx, MAIN_PLOT_BUCKET_COUNT); + cached.Clear(); + cached.AddRange(downsampled); + } + + if (series.ItemsSource != cached) + { + series.ItemsSource = cached; + } + } + } + + /// + /// Fetches high-resolution data from the database for the visible time window + /// using sampled index seeks (same technique as LoadSampledData). Runs on a + /// background thread to keep the UI responsive. Cancels any in-flight fetch + /// when a new one starts. Results are marshaled back to the UI thread. + /// + private void FetchViewportDataFromDb(double visibleMin, double visibleMax) + { + if (!_currentSessionId.HasValue || !_firstTime.HasValue) + { + return; + } + + // Cancel any in-flight fetch + _fetchCts?.Cancel(); + _fetchCts?.Dispose(); + _fetchCts = new CancellationTokenSource(); + var ct = _fetchCts.Token; + + var sessionId = _currentSessionId.Value; + var firstTimeTicks = _firstTime.Value.Ticks; + var channelKeys = _allSessionPoints.Keys.ToList(); + + // Convert plot X (ms) back to DB ticks with padding + var minTicks = firstTimeTicks + (long)(visibleMin * 10000.0); + var maxTicks = firstTimeTicks + (long)(visibleMax * 10000.0); + var tickRange = maxTicks - minTicks; + var padding = Math.Max(tickRange / 100, 10000); + minTicks -= padding; + maxTicks += padding; + + IsRefiningData = true; + + Task.Run(() => + { + var dbPoints = new Dictionary<(string, string), List>(); + foreach (var key in channelKeys) + { + dbPoints[key] = new List(); + } + + try + { + ct.ThrowIfCancellationRequested(); + + using var context = _loggingContext.CreateDbContext(); + var connection = context.Database.GetDbConnection(); + connection.Open(); + + // Use sampled seeks: divide the visible window into + // MAIN_PLOT_BUCKET_COUNT segments and read a small batch + // at each position. This reads ~4000 * channelCount rows + // instead of potentially millions. + var seekCount = MAIN_PLOT_BUCKET_COUNT; + var seekTickStep = (maxTicks - minTicks) / seekCount; + var batchSize = Math.Max(channelKeys.Count * 2, 100); + + using var seekCmd = connection.CreateCommand(); + seekCmd.CommandText = @" + SELECT ChannelName, DeviceSerialNo, TimestampTicks, Value + FROM Samples + WHERE LoggingSessionID = @id + AND TimestampTicks >= @t + AND TimestampTicks <= @maxT + ORDER BY TimestampTicks + LIMIT @limit"; + + var idParam = seekCmd.CreateParameter(); + idParam.ParameterName = "@id"; + idParam.Value = sessionId; + seekCmd.Parameters.Add(idParam); + + var tParam = seekCmd.CreateParameter(); + tParam.ParameterName = "@t"; + tParam.Value = minTicks; + seekCmd.Parameters.Add(tParam); + + var maxTParam = seekCmd.CreateParameter(); + maxTParam.ParameterName = "@maxT"; + maxTParam.Value = maxTicks; + seekCmd.Parameters.Add(maxTParam); + + var limitParam = seekCmd.CreateParameter(); + limitParam.ParameterName = "@limit"; + limitParam.Value = batchSize; + seekCmd.Parameters.Add(limitParam); + + seekCmd.Prepare(); + + var lastAddedTimestamp = new Dictionary<(string, string), long>(); + + for (var i = 0; i < seekCount; i++) + { + ct.ThrowIfCancellationRequested(); + + tParam.Value = minTicks + i * seekTickStep; + + using var reader = seekCmd.ExecuteReader(); + while (reader.Read()) + { + var channelName = reader.GetString(0); + var deviceSerialNo = reader.GetString(1); + var timestampTicks = reader.GetInt64(2); + var value = reader.GetDouble(3); + + var key = (deviceSerialNo, channelName); + + if (lastAddedTimestamp.TryGetValue(key, out var lastT) && timestampTicks <= lastT) + { + continue; + } + + lastAddedTimestamp[key] = timestampTicks; + var deltaTime = (timestampTicks - firstTimeTicks) / 10000.0; + + if (dbPoints.TryGetValue(key, out var points)) + { + points.Add(new DataPoint(deltaTime, value)); + } + } + } + } + catch (OperationCanceledException) + { + return; + } + catch (Exception ex) + { + _appLogger.Error(ex, "Failed to fetch viewport data from DB"); + Application.Current?.Dispatcher.Invoke(() => + { + IsRefiningData = false; + UpdateSeriesFromMemory(visibleMin, visibleMax); + PlotModel.InvalidatePlot(true); + }); + return; + } + + // Marshal results back to UI thread + Application.Current?.Dispatcher.Invoke(() => + { + if (ct.IsCancellationRequested) + { + return; + } + + foreach (var series in PlotModel.Series.OfType()) + { + if (series.Tag is not (string deviceSerial, string channelName)) + { + continue; + } + + var key = (deviceSerial, channelName); + if (!dbPoints.TryGetValue(key, out var fetchedPoints) || fetchedPoints.Count == 0) + { + continue; + } + + if (!_downsampledCache.TryGetValue(key, out var cached)) + { + cached = new List(MAIN_PLOT_BUCKET_COUNT * 2); + _downsampledCache[key] = cached; + } + + cached.Clear(); + if (fetchedPoints.Count <= MAIN_PLOT_BUCKET_COUNT * 2) + { + cached.AddRange(fetchedPoints); + } + else + { + cached.AddRange(MinMaxDownsampler.Downsample(fetchedPoints, MAIN_PLOT_BUCKET_COUNT)); + } + + if (series.ItemsSource != cached) + { + series.ItemsSource = cached; + } + } + + IsRefiningData = false; + PlotModel.InvalidatePlot(true); + }); + }, ct); + } + + /// + /// Called by the minimap interaction controller during drag to update the + /// main plot. Uses in-memory sampled data only (no DB queries) for smooth 60fps. + /// + public void OnMinimapViewportChanged() + { + UpdateMainPlotViewport(highFidelity: false); + } + + /// + /// Called when minimap interaction ends (mouse up). Fetches full-resolution + /// data from the database if the zoom level warrants it. + /// + public void OnMinimapInteractionEnded() + { + _settleTimer.Stop(); + _lastViewportMin = double.NaN; + _lastViewportMax = double.NaN; + UpdateMainPlotViewport(highFidelity: true); + } + /// /// Updates the visibility of a minimap series to match its main plot counterpart. /// @@ -742,36 +1390,99 @@ private void SaveGraph() [RelayCommand] private void ResetZoom() { - PlotModel.ResetAllAxes(); + // Reset Y axes to auto-range for amplitude + foreach (var axis in PlotModel.Axes) + { + if (axis.Key != "Time") + { + axis.Reset(); + } + } + + // Compute the full data range from source data (not downsampled) and + // explicitly set the time axis rather than relying on auto-range, which + // would use the current ItemsSource extent (potentially narrowed by + // viewport downsampling). + var fullMin = double.MaxValue; + var fullMax = double.MinValue; + foreach (var kvp in _allSessionPoints) + { + if (kvp.Value.Count > 0) + { + fullMin = Math.Min(fullMin, kvp.Value[0].X); + fullMax = Math.Max(fullMax, kvp.Value[^1].X); + } + } + + var timeAxis = PlotModel.Axes.FirstOrDefault(a => a.Key == "Time"); + if (timeAxis != null && fullMin < fullMax) + { + timeAxis.Zoom(fullMin, fullMax); + } + + _lastViewportMin = double.NaN; + _lastViewportMax = double.NaN; + UpdateMainPlotViewport(); PlotModel.InvalidatePlot(true); } [RelayCommand] private void ZoomOutX() { - PlotModel.Axes[2].ZoomAtCenter(0.8); + var timeAxis = PlotModel.Axes.FirstOrDefault(a => a.Key == "Time"); + timeAxis?.ZoomAtCenter(0.8); + UpdateMainPlotViewport(); PlotModel.InvalidatePlot(true); } [RelayCommand] private void ZoomInX() { - PlotModel.Axes[2].ZoomAtCenter(1.25); + var timeAxis = PlotModel.Axes.FirstOrDefault(a => a.Key == "Time"); + timeAxis?.ZoomAtCenter(1.25); + UpdateMainPlotViewport(); PlotModel.InvalidatePlot(true); } [RelayCommand] private void ZoomOutY() { - PlotModel.Axes[0].ZoomAtCenter(0.8); + var analogAxis = PlotModel.Axes.FirstOrDefault(a => a.Key == "Analog"); + analogAxis?.ZoomAtCenter(0.8); PlotModel.InvalidatePlot(true); } [RelayCommand] private void ZoomInY() { - PlotModel.Axes[0].ZoomAtCenter(1.25); + var analogAxis = PlotModel.Axes.FirstOrDefault(a => a.Key == "Analog"); + analogAxis?.ZoomAtCenter(1.25); PlotModel.InvalidatePlot(true); } #endregion + + #region IDisposable + /// + /// Stops timers and unsubscribes event handlers to prevent leaks. + /// + public void Dispose() + { + _viewportThrottleTimer.Stop(); + _viewportThrottleTimer.Tick -= OnViewportThrottleTick; + _settleTimer.Stop(); + _settleTimer.Tick -= OnSettleTick; + _fetchCts?.Cancel(); + _fetchCts?.Dispose(); + + var timeAxis = PlotModel.Axes.FirstOrDefault(a => a.Key == "Time"); + if (timeAxis != null) + { + timeAxis.AxisChanged -= OnMainTimeAxisChanged; + } + + _minimapInteraction?.Dispose(); + _buffer.Dispose(); + _consumerGate.Dispose(); + } + #endregion } \ No newline at end of file diff --git a/Daqifi.Desktop/Loggers/LoggingContext.cs b/Daqifi.Desktop/Loggers/LoggingContext.cs index 3ed7045f..981b62c8 100644 --- a/Daqifi.Desktop/Loggers/LoggingContext.cs +++ b/Daqifi.Desktop/Loggers/LoggingContext.cs @@ -1,4 +1,4 @@ -using Daqifi.Desktop.Channel; +using Daqifi.Desktop.Channel; using Microsoft.EntityFrameworkCore; namespace Daqifi.Desktop.Logger; @@ -18,7 +18,12 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) entity.Property(ls => ls.Name).IsRequired(); }); - modelBuilder.Entity().ToTable("Samples"); + modelBuilder.Entity(entity => + { + entity.ToTable("Samples"); + entity.HasIndex(s => new { s.LoggingSessionID, s.TimestampTicks }) + .HasDatabaseName("IX_Samples_SessionTime"); + }); modelBuilder.Entity(entity => { @@ -31,4 +36,4 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) public DbSet Sessions { get; set; } public DbSet Samples { get; set; } -} \ No newline at end of file +} diff --git a/Daqifi.Desktop/MainWindow.xaml b/Daqifi.Desktop/MainWindow.xaml index 04451046..cba2ab5f 100644 --- a/Daqifi.Desktop/MainWindow.xaml +++ b/Daqifi.Desktop/MainWindow.xaml @@ -470,6 +470,7 @@ + @@ -565,8 +566,28 @@ + + + + + + + - diff --git a/Daqifi.Desktop/View/MinimapInteractionController.cs b/Daqifi.Desktop/View/MinimapInteractionController.cs index 9bcae8e4..3038cedb 100644 --- a/Daqifi.Desktop/View/MinimapInteractionController.cs +++ b/Daqifi.Desktop/View/MinimapInteractionController.cs @@ -1,6 +1,8 @@ +using Daqifi.Desktop.Logger; using OxyPlot; using OxyPlot.Annotations; using OxyPlot.Axes; +using System.Windows.Threading; using Cursor = System.Windows.Input.Cursor; using Cursors = System.Windows.Input.Cursors; using Application = System.Windows.Application; @@ -12,7 +14,7 @@ namespace Daqifi.Desktop.View; /// Handles mouse interactions on the minimap PlotModel to enable drag/resize /// of the selection rectangle, synchronized with the main plot's time axis. /// Provides cursor feedback: resize arrows on edges, grab hand inside selection, -/// and pointer outside. +/// and pointer outside. Renders are throttled to 60fps via a DispatcherTimer. /// public class MinimapInteractionController : IDisposable { @@ -24,6 +26,7 @@ public class MinimapInteractionController : IDisposable private readonly RectangleAnnotation _dimRight; private readonly string _mainTimeAxisKey; private readonly string _minimapTimeAxisKey; + private readonly DatabaseLogger _databaseLogger; private enum DragMode { None, Pan, ResizeLeft, ResizeRight } private DragMode _dragMode = DragMode.None; @@ -33,6 +36,9 @@ private enum DragMode { None, Pan, ResizeLeft, ResizeRight } private Cursor _lastCursor; private const double EDGE_TOLERANCE_FRACTION = 0.02; + + private bool _isDirty; + private readonly DispatcherTimer _renderTimer; #endregion #region Constructor @@ -42,6 +48,7 @@ public MinimapInteractionController( RectangleAnnotation selectionRect, RectangleAnnotation dimLeft, RectangleAnnotation dimRight, + DatabaseLogger databaseLogger, string mainTimeAxisKey = "Time", string minimapTimeAxisKey = "MinimapTime") { @@ -50,12 +57,20 @@ public MinimapInteractionController( _selectionRect = selectionRect; _dimLeft = dimLeft; _dimRight = dimRight; + _databaseLogger = databaseLogger; _mainTimeAxisKey = mainTimeAxisKey; _minimapTimeAxisKey = minimapTimeAxisKey; _minimapPlotModel.MouseDown += OnMouseDown; _minimapPlotModel.MouseMove += OnMouseMove; _minimapPlotModel.MouseUp += OnMouseUp; + + _renderTimer = new DispatcherTimer(DispatcherPriority.Render) + { + Interval = TimeSpan.FromMilliseconds(16) + }; + _renderTimer.Tick += OnRenderTick; + _renderTimer.Start(); } #endregion @@ -261,6 +276,13 @@ private void OnMouseUp(object? sender, OxyMouseEventArgs e) { _dragMode = DragMode.None; + // Flush any pending dirty state and trigger high-fidelity DB fetch + // for the final viewport position (regardless of dirty flag state) + _isDirty = false; + _databaseLogger.OnMinimapInteractionEnded(); + _mainPlotModel.InvalidatePlot(true); + _minimapPlotModel.InvalidatePlot(false); + // Update cursor based on final position var minimapTimeAxis = GetMinimapTimeAxis(); if (minimapTimeAxis != null) @@ -274,6 +296,21 @@ private void OnMouseUp(object? sender, OxyMouseEventArgs e) } #endregion + #region Render Throttling + private void OnRenderTick(object? sender, EventArgs e) + { + if (!_isDirty) + { + return; + } + + _isDirty = false; + _databaseLogger.OnMinimapViewportChanged(); + _mainPlotModel.InvalidatePlot(true); + _minimapPlotModel.InvalidatePlot(false); + } + #endregion + #region Private Methods private void ApplyToMainPlot(double min, double max) { @@ -283,11 +320,19 @@ private void ApplyToMainPlot(double min, double max) return; } - mainTimeAxis.Zoom(min, max); + _databaseLogger.IsSyncingFromMinimap = true; + try + { + mainTimeAxis.Zoom(min, max); + } + finally + { + _databaseLogger.IsSyncingFromMinimap = false; + } + _dimLeft.MaximumX = min; _dimRight.MinimumX = max; - _mainPlotModel.InvalidatePlot(false); - _minimapPlotModel.InvalidatePlot(false); + _isDirty = true; } private LinearAxis? GetMinimapTimeAxis() @@ -316,10 +361,12 @@ private double GetMinimapDataMin() #region IDisposable /// - /// Unsubscribes all event handlers from the minimap PlotModel. + /// Unsubscribes all event handlers from the minimap PlotModel and stops the render timer. /// public void Dispose() { + _renderTimer.Stop(); + _renderTimer.Tick -= OnRenderTick; _minimapPlotModel.MouseDown -= OnMouseDown; _minimapPlotModel.MouseMove -= OnMouseMove; _minimapPlotModel.MouseUp -= OnMouseUp; diff --git a/docs/adr/001-viewport-aware-downsampling.md b/docs/adr/001-viewport-aware-downsampling.md new file mode 100644 index 00000000..170cb46b --- /dev/null +++ b/docs/adr/001-viewport-aware-downsampling.md @@ -0,0 +1,98 @@ +# ADR 001: Viewport-Aware Downsampling for Large Dataset Rendering + +**Status**: Accepted +**Date**: 2026-04 +**PR**: #467 (supersedes #457) + +## Context + +The DAQiFi Desktop application needs to display logged data sessions that can be very large — a typical worst case is 16 channels at 1000 Hz for 24 hours, producing ~1.38 billion data points. PR #458 added an overview minimap for navigating these sessions, but the main plot still rendered every data point, making the UI sluggish with large sessions. + +OxyPlot (our charting library) iterates every point in a series during render, even when zoomed into a tiny region. With 1M points per channel and 16 channels, that's 16M point iterations per frame — far too slow for interactive pan/zoom. + +We needed a downsampling strategy that: + +1. Keeps the main plot under ~4000 points per channel regardless of dataset size +2. Shows full detail when zoomed in (not a blurry approximation) +3. Works with the minimap's ability to select arbitrary time ranges +4. Maintains 60fps during drag/resize interactions + +## Decision + +Use **viewport-aware MinMax downsampling**: on every viewport change, binary search the source data for the visible time range, then downsample only that slice to ~4000 points using min/max aggregation per bucket. + +### How it works + +**Session loading (two-phase progressive):** +1. **Phase 1** (<1s): Load first 100K samples via index scan for immediate display +2. **Phase 2** (~1-3s): Load a sampled overview via ~3000 targeted index seeks spread across the full time range. Each seek reads one batch of interleaved channel data. Result: ~3000 points/channel covering the full range + +**Viewport updates (drag vs settle):** +1. **During interaction** (minimap drag, pan/zoom): `UpdateMainPlotViewport(highFidelity: false)` uses only in-memory sampled data. Binary searches for the visible range — O(log n) via `MinMaxDownsampler.FindVisibleRange()` — then downsamples to ~4000 points per channel via min/max aggregation. Written into a **reusable cached list** per series (no allocation). +2. **On settle** (mouse-up or 200ms idle): `UpdateMainPlotViewport(highFidelity: true)` checks if the in-memory data is too sparse for the current zoom level. If so, fires an async background DB fetch using sampled index seeks within the visible window, then marshals results back to the UI thread. A `CancellationToken` ensures only the latest fetch completes. +3. Viewport updates are **throttled to 60fps** via DispatcherTimer + dirty flag, both for minimap-driven changes and main plot pan/zoom + +### Key files + +- `MinMaxDownsampler.cs` — binary search + min/max downsampling algorithm +- `DatabaseLogger.cs` — two-phase loading, viewport updates, async DB fetch (`UpdateMainPlotViewport`, `FetchViewportDataFromDb`, `LoadSampledData`) +- `MinimapInteractionController.cs` — 60fps throttled minimap interaction with drag/settle distinction +- `LoggingContext.cs` — composite DB index `IX_Samples_SessionTime` on `(LoggingSessionID, TimestampTicks)` + +## Alternatives Considered + +### 1. Global LTTB Decimation (PR #457) + +PR #457 applied Largest Triangle Three Buckets (LTTB) decimation globally to ~5000 points per channel at load time. + +**Rejected because**: Global decimation fundamentally conflicts with the minimap. If you decimate 24 hours of data to 5000 points and then zoom into a 1-minute window, only ~3 points would be visible — the plot would be empty or misleadingly sparse. The minimap's entire purpose is to let users zoom into arbitrary ranges, so the downsampling must be viewport-aware. + +LTTB also has higher computational cost per point than MinMax (triangle area calculations vs simple comparisons), which matters when re-downsampling at 60fps. + +### 2. Pre-computed Multi-Resolution Pyramid + +Build multiple resolution levels at load time (e.g., 1:1, 1:10, 1:100, 1:1000) and select the appropriate level based on zoom. + +**Rejected because**: Significant memory overhead (nearly 2x the raw data), complex invalidation if data changes, and overkill for our use case. The binary search + linear scan approach is fast enough (~1ms for 16 channels) that on-the-fly downsampling at 60fps is feasible without pre-computation. + +### 3. GPU-Accelerated Rendering + +Replace OxyPlot with a GPU-backed charting library (e.g., SciChart, LiveCharts2 with SkiaSharp). + +**Rejected because**: Major dependency change with significant migration cost. OxyPlot is well-integrated with our WPF MVVM architecture. Viewport-aware downsampling reduces the point count enough (~64K total) that OxyPlot renders comfortably within a 16ms frame budget. If we outgrow this approach, GPU rendering remains a future option. + +### 4. Virtual Scrolling / On-Demand DB Queries (Partially Adopted) + +Only load the visible time range from SQLite on each viewport change, avoiding keeping all data in memory. + +**Initially rejected** for pure on-demand because SQLite query latency is too high for 60fps continuous interaction (minimap drag, pan). **However**, we adopted a hybrid approach: + +- **During drag**: use in-memory sampled data only (3000 points/channel, loaded at session start via sampled index seeks). This guarantees <1ms viewport updates for smooth 60fps. +- **On settle** (mouse-up or 200ms idle): fetch high-resolution data from SQLite for just the visible window via async sampled index seeks on a background thread. The composite index (`IX_Samples_SessionTime`) makes these seeks fast regardless of total dataset size. + +This gives the best of both worlds: instantaneous interaction with progressive refinement to full fidelity when the user stops moving. + +## Consequences + +### Positive + +- **60fps interaction**: <1ms per viewport update during drag (in-memory sampled data only) +- **Full zoom fidelity**: Zooming into a 1-second window triggers an async DB fetch that fills in full-resolution data +- **Fast session loading**: Two-phase progressive — data visible in <1s, full overview in ~1-3s regardless of dataset size +- **Non-blocking UI**: DB fetches run on background thread with cancellation. A thin progress bar indicates when refinement is in progress +- **Low complexity**: The core algorithm is ~80 lines (binary search + min/max loop) +- **No external dependencies**: Pure C# implementation + +### Negative + +- **Memory usage for overview**: Sampled data (~3000 points/channel) lives in memory. Much smaller than keeping the full dataset in memory. +- **MinMax doesn't preserve exact X boundaries**: Downsampled first/last X values may differ from source data. This required explicit time axis ranging in `ResetZoom` instead of relying on OxyPlot auto-range (see `InvalidatePlot` gotchas in CLAUDE.md). +- **List mutation pattern**: Reusing cached lists and calling `InvalidatePlot(true)` is non-obvious. `InvalidatePlot(false)` silently renders stale data. This is documented in CLAUDE.md but remains a footgun for future changes. +- **Thread safety**: `_allSessionPoints` is written on background threads (session loading) and read on the UI thread (viewport updates). Currently relies on non-overlapping access patterns rather than explicit synchronization. A future refactor should add proper locking. + +### Follow-up Work + +- Add thread synchronization for `_allSessionPoints` (lock or move all mutations to UI thread) +- Add cancellation token to the consumer thread for clean shutdown on `Dispose()` +- Existing DB migration path: the composite index `IX_Samples_SessionTime` is only created via `EnsureCreated`, not applied to existing databases. See GitHub issue #468 for the `EnsureCreated` → EF Core migrations switch. +- PR #457 should be closed as superseded diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 00000000..2859d2fe --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,21 @@ +# Architecture Decision Records (ADRs) + +This directory captures significant architectural decisions for the DAQiFi Desktop application. + +Each ADR documents the context, decision, alternatives considered, and consequences of a technical choice. They serve as a historical record so future contributors understand *why* something was built a certain way, not just *how*. + +## Format + +ADRs follow a lightweight template: + +- **Status**: Accepted, Superseded, or Deprecated +- **Context**: The problem or situation that prompted the decision +- **Decision**: What we chose to do +- **Alternatives Considered**: What else we evaluated and why we rejected it +- **Consequences**: Trade-offs, risks, and follow-up work + +## Index + +| # | Title | Status | Date | +|---|-------|--------|------| +| [001](001-viewport-aware-downsampling.md) | Viewport-aware downsampling for large dataset rendering | Accepted | 2026-04 |