From 6ec1832659fd4d2ce431c0afe779806d5dfa84b5 Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Thu, 19 Mar 2026 21:22:55 -0600 Subject: [PATCH 1/2] fix: scale raw ADC values in CSV and JSON SD card parsers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The firmware writes raw ADC integer counts (not voltages) to CSV and JSON log files. The CSV parser was passing these through as-is, causing values like 22 ADC counts to display as 22V instead of ~0.003V. Changes: - Add ScaleRawAnalogValues to both CSV and JSON parsers, matching the protobuf parser's formula: (raw/resolution * portRange * calM + calB) * scaleM - Parse CSV column headers to distinguish ain* (analog) from dio* (digital) columns — previously the dio pair was treated as analog - Add tests for scaling with/without calibration config and DIO handling Co-Authored-By: Claude Opus 4.6 Entire-Checkpoint: a0fb8798807b --- .../Device/SdCard/SdCardCsvFileParserTests.cs | 203 +++++++++++++++++- .../SdCard/SdCardJsonFileParserTests.cs | 64 ++++++ .../Device/SdCard/SdCardCsvFileParser.cs | 146 ++++++++++--- .../Device/SdCard/SdCardJsonFileParser.cs | 40 +++- 4 files changed, 425 insertions(+), 28 deletions(-) diff --git a/src/Daqifi.Core.Tests/Device/SdCard/SdCardCsvFileParserTests.cs b/src/Daqifi.Core.Tests/Device/SdCard/SdCardCsvFileParserTests.cs index 893e503..07961c8 100644 --- a/src/Daqifi.Core.Tests/Device/SdCard/SdCardCsvFileParserTests.cs +++ b/src/Daqifi.Core.Tests/Device/SdCard/SdCardCsvFileParserTests.cs @@ -53,9 +53,9 @@ public async Task ParseAsync_ValidFirmwareCsvLines_ReturnsCorrectSamples() } [Fact] - public async Task ParseAsync_DigitalDataIsAlwaysZero_BecauseFirmwareCsvHasNoDigitalColumn() + public async Task ParseAsync_NoDioColumn_DigitalDataIsZero() { - // Arrange — real firmware CSV has no digital data column + // Arrange — CSV without a dio column pair → digital data defaults to 0 await using var stream = SdCardTestCsvFileBuilder.BuildCsvFileSharedTimestamp( "Nyquist 1", "SN123", 50_000_000u, (1000u, new[] { 10.0, 20.0 }) @@ -65,7 +65,6 @@ public async Task ParseAsync_DigitalDataIsAlwaysZero_BecauseFirmwareCsvHasNoDigi var session = await parser.ParseAsync(stream, "test.csv"); var samples = await ToListAsync(session.Samples); - // DigitalData is always 0 in the firmware CSV format Assert.Single(samples); Assert.Equal(0u, samples[0].DigitalData); } @@ -483,6 +482,204 @@ await Assert.ThrowsAsync(async () => }); } + // ------------------------------------------------------------------------- + // ADC scaling + // ------------------------------------------------------------------------- + + [Fact] + public async Task ParseAsync_WithCalibrationConfig_ScalesRawAdcValues() + { + // Arrange — raw ADC values (e.g., 22 counts on a 16-bit ADC) + // with calibration config that should scale them to voltage + await using var stream = SdCardTestCsvFileBuilder.BuildCsvFileSharedTimestamp( + "Nyquist 1", "7E2815916200E898", 50_000_000u, + (1000u, new[] { 0.0, 22.0, 16.0 }) + ); + + var overrideConfig = new global::Daqifi.Core.Device.SdCard.SdCardDeviceConfiguration( + AnalogPortCount: 3, + DigitalPortCount: 0, + TimestampFrequency: 50_000_000u, + DeviceSerialNumber: "7E2815916200E898", + DevicePartNumber: "Nyquist 1", + FirmwareRevision: "3.4.4", + CalibrationValues: new[] { (1.0, 0.0), (1.0, 0.0), (1.0, 0.0) }, + Resolution: 65535, + PortRange: new[] { 10.0, 10.0, 10.0 }, + InternalScaleM: new[] { 1.0, 1.0, 1.0 }); + + var parser = new global::Daqifi.Core.Device.SdCard.SdCardCsvFileParser(); + var options = new global::Daqifi.Core.Device.SdCard.SdCardParseOptions + { + ConfigurationOverride = overrideConfig + }; + + // Act + var session = await parser.ParseAsync(stream, "test.csv", options); + var samples = await ToListAsync(session.Samples); + + // Assert — raw value 22 should be scaled: (22 / 65535) * 10.0 * 1.0 + 0.0 = ~0.00336 + Assert.Single(samples); + Assert.Equal(3, samples[0].AnalogValues.Count); + Assert.Equal(0.0, samples[0].AnalogValues[0], precision: 5); + Assert.Equal(22.0 / 65535.0 * 10.0, samples[0].AnalogValues[1], precision: 5); + Assert.Equal(16.0 / 65535.0 * 10.0, samples[0].AnalogValues[2], precision: 5); + } + + [Fact] + public async Task ParseAsync_WithoutCalibrationConfig_ReturnsRawValues() + { + // Arrange — no config override, no resolution → raw values pass through + await using var stream = SdCardTestCsvFileBuilder.BuildCsvFileSharedTimestamp( + "Nyquist 1", "SN001", 100u, + (1000u, new[] { 22.0, 16.0 }) + ); + + var parser = new global::Daqifi.Core.Device.SdCard.SdCardCsvFileParser(); + var session = await parser.ParseAsync(stream, "test.csv"); + var samples = await ToListAsync(session.Samples); + + // No scaling applied — raw values returned as-is + Assert.Single(samples); + Assert.Equal(22.0, samples[0].AnalogValues[0], precision: 5); + Assert.Equal(16.0, samples[0].AnalogValues[1], precision: 5); + } + + [Fact] + public async Task ParseAsync_WithCalibrationOffsets_AppliesFullFormula() + { + // Arrange — test the full scaling formula: (raw / resolution * portRange * calM + calB) * internalScaleM + await using var stream = SdCardTestCsvFileBuilder.BuildCsvFileSharedTimestamp( + "TestDevice", "SN001", 100u, + (1000u, new[] { 32768.0 }) // half-scale on 16-bit ADC + ); + + var overrideConfig = new global::Daqifi.Core.Device.SdCard.SdCardDeviceConfiguration( + AnalogPortCount: 1, + DigitalPortCount: 0, + TimestampFrequency: 100u, + DeviceSerialNumber: "SN001", + DevicePartNumber: "TestDevice", + FirmwareRevision: null, + CalibrationValues: new[] { (1.02, -0.05) }, // calM=1.02, calB=-0.05 + Resolution: 65535, + PortRange: new[] { 10.0 }, + InternalScaleM: new[] { 2.0 }); + + var parser = new global::Daqifi.Core.Device.SdCard.SdCardCsvFileParser(); + var options = new global::Daqifi.Core.Device.SdCard.SdCardParseOptions + { + ConfigurationOverride = overrideConfig + }; + + var session = await parser.ParseAsync(stream, "test.csv", options); + var samples = await ToListAsync(session.Samples); + + // Expected: (32768 / 65535 * 10.0 * 1.02 + (-0.05)) * 2.0 + var normalized = 32768.0 / 65535.0; + var expected = (normalized * 10.0 * 1.02 + (-0.05)) * 2.0; + Assert.Single(samples); + Assert.Equal(expected, samples[0].AnalogValues[0], precision: 5); + } + + // ------------------------------------------------------------------------- + // DIO column handling + // ------------------------------------------------------------------------- + + [Fact] + public async Task ParseAsync_WithDioColumn_ParsesDigitalDataSeparately() + { + // Arrange — firmware CSV with ain columns + dio column at the end + var content = + "# Device: Nyquist 1\n" + + "# Serial Number: 7E2815916200E898\n" + + "# Timestamp Tick Rate: 50000000 Hz\n" + + "ain0_ts,ain0_val,ain1_ts,ain1_val,dio_ts,dio_val\n" + + "1000,0,1000,22,1000,5\n" + + "2000,1,2000,23,2000,3\n"; + await using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); + + var parser = new global::Daqifi.Core.Device.SdCard.SdCardCsvFileParser(); + var session = await parser.ParseAsync(stream, "test.csv"); + var samples = await ToListAsync(session.Samples); + + // Assert — 2 analog channels (not 3), dio parsed as digital data + Assert.Equal(2, session.DeviceConfig!.AnalogPortCount); + Assert.Equal(1, session.DeviceConfig.DigitalPortCount); + Assert.Equal(2, samples.Count); + + // Analog values (unscaled since no config override) + Assert.Equal(2, samples[0].AnalogValues.Count); + Assert.Equal(0.0, samples[0].AnalogValues[0], precision: 5); + Assert.Equal(22.0, samples[0].AnalogValues[1], precision: 5); + + // Digital data + Assert.Equal(5u, samples[0].DigitalData); + Assert.Equal(3u, samples[1].DigitalData); + } + + [Fact] + public async Task ParseAsync_WithDioColumnAndScaling_ScalesOnlyAnalogValues() + { + // Arrange — ain columns should be scaled, dio column should not + var content = + "# Device: Nyquist 1\n" + + "# Serial Number: SN001\n" + + "# Timestamp Tick Rate: 100 Hz\n" + + "ain0_ts,ain0_val,dio_ts,dio_val\n" + + "1000,32768,1000,7\n"; + await using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); + + var overrideConfig = new global::Daqifi.Core.Device.SdCard.SdCardDeviceConfiguration( + AnalogPortCount: 1, + DigitalPortCount: 1, + TimestampFrequency: 100u, + DeviceSerialNumber: "SN001", + DevicePartNumber: "Nyquist 1", + FirmwareRevision: null, + CalibrationValues: new[] { (1.0, 0.0) }, + Resolution: 65535, + PortRange: new[] { 10.0 }, + InternalScaleM: new[] { 1.0 }); + + var parser = new global::Daqifi.Core.Device.SdCard.SdCardCsvFileParser(); + var options = new global::Daqifi.Core.Device.SdCard.SdCardParseOptions + { + ConfigurationOverride = overrideConfig + }; + + var session = await parser.ParseAsync(stream, "test.csv", options); + var samples = await ToListAsync(session.Samples); + + Assert.Single(samples); + // Analog should be scaled: 32768 / 65535 * 10.0 ≈ 5.0 + Assert.Equal(32768.0 / 65535.0 * 10.0, samples[0].AnalogValues[0], precision: 3); + // Digital should remain as-is + Assert.Equal(7u, samples[0].DigitalData); + } + + [Fact] + public async Task ParseAsync_AinColumnHeader_CorrectlyParsed() + { + // Arrange — real firmware uses ain0, ain1, ain2 etc. as column prefixes + var content = + "# Device: Nyquist 1\n" + + "# Serial Number: SN001\n" + + "# Timestamp Tick Rate: 50000000 Hz\n" + + "ain0_ts,ain0_val,ain1_ts,ain1_val,ain2_ts,ain2_val\n" + + "1000,10,1000,20,1000,30\n"; + await using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); + + var parser = new global::Daqifi.Core.Device.SdCard.SdCardCsvFileParser(); + var session = await parser.ParseAsync(stream, "test.csv"); + var samples = await ToListAsync(session.Samples); + + Assert.Equal(3, session.DeviceConfig!.AnalogPortCount); + Assert.Equal(0, session.DeviceConfig.DigitalPortCount); + Assert.Single(samples); + Assert.Equal(3, samples[0].AnalogValues.Count); + } + // ------------------------------------------------------------------------- // Helper // ------------------------------------------------------------------------- diff --git a/src/Daqifi.Core.Tests/Device/SdCard/SdCardJsonFileParserTests.cs b/src/Daqifi.Core.Tests/Device/SdCard/SdCardJsonFileParserTests.cs index 62eef20..3dedca8 100644 --- a/src/Daqifi.Core.Tests/Device/SdCard/SdCardJsonFileParserTests.cs +++ b/src/Daqifi.Core.Tests/Device/SdCard/SdCardJsonFileParserTests.cs @@ -340,6 +340,70 @@ public async Task ParseAsync_PerChannelTimestamps_ReturnsNull() Assert.Null(samples[0].AnalogTimestamps); // JSON/CSV formats don't support per-channel timestamps } + // ------------------------------------------------------------------------- + // ADC scaling + // ------------------------------------------------------------------------- + + [Fact] + public async Task ParseAsync_WithCalibrationConfig_ScalesRawAdcValues() + { + // Arrange — raw ADC integer values in JSON + await using var stream = SdCardTestJsonFileBuilder.BuildJsonFileWithIntegers( + (100u, new[] { 0, 22, 16 }, "00") + ); + + var overrideConfig = new global::Daqifi.Core.Device.SdCard.SdCardDeviceConfiguration( + AnalogPortCount: 3, + DigitalPortCount: 0, + TimestampFrequency: 100u, + DeviceSerialNumber: "SN001", + DevicePartNumber: "TestDevice", + FirmwareRevision: null, + CalibrationValues: new[] { (1.0, 0.0), (1.0, 0.0), (1.0, 0.0) }, + Resolution: 65535, + PortRange: new[] { 10.0, 10.0, 10.0 }, + InternalScaleM: new[] { 1.0, 1.0, 1.0 }); + + var parser = new global::Daqifi.Core.Device.SdCard.SdCardJsonFileParser(); + var options = new global::Daqifi.Core.Device.SdCard.SdCardParseOptions + { + SessionStartTime = new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc), + FallbackTimestampFrequency = 100, + ConfigurationOverride = overrideConfig + }; + + var session = await parser.ParseAsync(stream, "test.json", options); + var samples = await ToListAsync(session.Samples); + + // Assert — raw 22 → (22 / 65535) * 10.0 ≈ 0.00336 + Assert.Single(samples); + Assert.Equal(0.0, samples[0].AnalogValues[0], precision: 5); + Assert.Equal(22.0 / 65535.0 * 10.0, samples[0].AnalogValues[1], precision: 5); + Assert.Equal(16.0 / 65535.0 * 10.0, samples[0].AnalogValues[2], precision: 5); + } + + [Fact] + public async Task ParseAsync_WithoutCalibrationConfig_ReturnsRawValues() + { + // Arrange — no config override → values pass through as-is + await using var stream = SdCardTestJsonFileBuilder.BuildJsonFileWithIntegers( + (100u, new[] { 22, 16 }, "00") + ); + + var parser = new global::Daqifi.Core.Device.SdCard.SdCardJsonFileParser(); + var options = new global::Daqifi.Core.Device.SdCard.SdCardParseOptions + { + FallbackTimestampFrequency = 100 + }; + + var session = await parser.ParseAsync(stream, "test.json", options); + var samples = await ToListAsync(session.Samples); + + Assert.Single(samples); + Assert.Equal(22.0, samples[0].AnalogValues[0], precision: 5); + Assert.Equal(16.0, samples[0].AnalogValues[1], precision: 5); + } + private static async Task> ToListAsync(IAsyncEnumerable source) { var list = new List(); diff --git a/src/Daqifi.Core/Device/SdCard/SdCardCsvFileParser.cs b/src/Daqifi.Core/Device/SdCard/SdCardCsvFileParser.cs index edd4dec..575d128 100644 --- a/src/Daqifi.Core/Device/SdCard/SdCardCsvFileParser.cs +++ b/src/Daqifi.Core/Device/SdCard/SdCardCsvFileParser.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Linq; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -15,8 +16,9 @@ namespace Daqifi.Core.Device.SdCard; /// /// Up to three #-prefixed comment lines containing device metadata /// (device name, serial number, and timestamp tick rate). -/// A column header row: ch0_ts,ch0_val,ch1_ts,ch1_val,... -/// Data rows with interleaved per-channel timestamp/value pairs. +/// A column header row: ain0_ts,ain0_val,ain1_ts,ain1_val,...,dio_ts,dio_val +/// Data rows with interleaved per-channel timestamp/value pairs. +/// Analog values are raw ADC counts that require scaling. /// /// /// @@ -69,7 +71,8 @@ public async Task ParseAsync( EmptySamples()); } - var config = MergeConfiguration(ParseHeader(lines, options), options.ConfigurationOverride); + var (headerConfig, columnLayout) = ParseHeader(lines, options); + var config = MergeConfiguration(headerConfig, options.ConfigurationOverride); // Find the index of the first data row (after comments and column header) var dataStartIndex = FindDataStartIndex(lines); @@ -88,6 +91,7 @@ public async Task ParseAsync( lines, dataStartIndex, config, + columnLayout, fileCreatedDate, options); @@ -120,14 +124,25 @@ public async Task ParseFileAsync( } /// - /// Parses device metadata from the comment header lines. + /// Describes the column layout parsed from the CSV header row. /// - private static SdCardDeviceConfiguration ParseHeader(List lines, SdCardParseOptions options) + /// Number of analog channel column pairs (ts + val). + /// Whether the last column pair is a digital I/O pair (dio_ts, dio_val). + private sealed record CsvColumnLayout(int AnalogPairCount, bool HasDigitalPair); + + /// + /// Parses device metadata and column layout from the comment header lines. + /// + private static (SdCardDeviceConfiguration Config, CsvColumnLayout Layout) ParseHeader( + List lines, + SdCardParseOptions options) { string? deviceName = null; string? serialNumber = null; var timestampFreq = options.FallbackTimestampFrequency; var analogChannelCount = 0; + var digitalChannelCount = 0; + var hasDigitalPair = false; foreach (var line in lines) { @@ -162,12 +177,29 @@ private static SdCardDeviceConfiguration ParseHeader(List lines, SdCardP } } else if (line.Contains("_ts,", StringComparison.OrdinalIgnoreCase) || - line.StartsWith("ch", StringComparison.OrdinalIgnoreCase)) + line.StartsWith("ch", StringComparison.OrdinalIgnoreCase) || + line.StartsWith("ain", StringComparison.OrdinalIgnoreCase)) { - // Column header: ch0_ts,ch0_val,ch1_ts,ch1_val,... - // Count channel pairs (every 2 columns = 1 channel) + // Column header: ain0_ts,ain0_val,ain1_ts,ain1_val,...,dio_ts,dio_val + // Count channel pairs and identify digital columns var cols = line.Split(','); - analogChannelCount = cols.Length / 2; + var totalPairs = cols.Length / 2; + + // Check each pair to distinguish analog from digital + for (var p = 0; p < totalPairs; p++) + { + var nameCol = cols[p * 2]; // e.g., "ain0_ts" or "dio_ts" + if (nameCol.StartsWith("dio", StringComparison.OrdinalIgnoreCase)) + { + hasDigitalPair = true; + digitalChannelCount = 1; + } + else + { + analogChannelCount++; + } + } + break; } else @@ -177,14 +209,18 @@ private static SdCardDeviceConfiguration ParseHeader(List lines, SdCardP } } - return new SdCardDeviceConfiguration( + var config = new SdCardDeviceConfiguration( AnalogPortCount: analogChannelCount, - DigitalPortCount: 0, + DigitalPortCount: digitalChannelCount, TimestampFrequency: timestampFreq, DeviceSerialNumber: serialNumber, DevicePartNumber: deviceName, FirmwareRevision: null, CalibrationValues: null); + + var layout = new CsvColumnLayout(analogChannelCount, hasDigitalPair); + + return (config, layout); } /// @@ -200,9 +236,10 @@ private static int FindDataStartIndex(List lines) continue; // Comment line } - // Check if it's the column header (contains "_ts," or starts with "ch") + // Check if it's the column header (contains "_ts," or starts with "ch"/"ain") if (line.Contains("_ts,", StringComparison.OrdinalIgnoreCase) || - (line.StartsWith("ch", StringComparison.OrdinalIgnoreCase) && !char.IsDigit(line[2]))) + (line.StartsWith("ch", StringComparison.OrdinalIgnoreCase) && !char.IsDigit(line[2])) || + line.StartsWith("ain", StringComparison.OrdinalIgnoreCase)) { continue; // Column header line } @@ -217,6 +254,7 @@ private static async IAsyncEnumerable ParseCsvLines( List lines, int dataStartIndex, SdCardDeviceConfiguration config, + CsvColumnLayout columnLayout, DateTime? fileCreatedDate, SdCardParseOptions options) { @@ -230,7 +268,6 @@ private static async IAsyncEnumerable ParseCsvLines( var bytesRead = 0L; var progress = options.Progress; - var dataLines = lines.Count - dataStartIndex; var totalBytes = lines.Sum(l => l.Length + 1); // +1 for newline for (var i = dataStartIndex; i < lines.Count; i++) @@ -239,14 +276,17 @@ private static async IAsyncEnumerable ParseCsvLines( linesProcessed++; bytesRead += line.Length + 1; - var parsed = TryParseCsvDataRow(line); + var parsed = TryParseCsvDataRow(line, columnLayout); if (parsed == null) { // Skip malformed lines continue; } - var (rowTimestamp, analogValues, perChannelTimestamps) = parsed.Value; + var (rowTimestamp, rawAnalogValues, digitalData, perChannelTimestamps) = parsed.Value; + + // Scale raw ADC values using device calibration + var analogValues = ScaleRawAnalogValues(rawAnalogValues, config); // Reconstruct absolute timestamp using first channel timestamp var absoluteTime = baseTime; @@ -266,7 +306,7 @@ private static async IAsyncEnumerable ParseCsvLines( absoluteTime = baseTime.AddSeconds(elapsedSeconds); } - yield return new SdCardLogEntry(absoluteTime, analogValues, 0u, perChannelTimestamps); + yield return new SdCardLogEntry(absoluteTime, analogValues, digitalData, perChannelTimestamps); // Report progress every 100 lines for efficiency if (linesProcessed % 100 == 0 && progress != null) @@ -283,9 +323,11 @@ private static async IAsyncEnumerable ParseCsvLines( /// /// Parses a firmware CSV data row with interleaved per-channel timestamp/value pairs. - /// Format: ch0_ts,ch0_val,ch1_ts,ch1_val,... + /// Separates analog channel pairs from the digital I/O pair based on the column layout. + /// Format: ain0_ts,ain0_val,...,dio_ts,dio_val /// - private static (uint rowTimestamp, IReadOnlyList analogValues, IReadOnlyList perChannelTimestamps)? TryParseCsvDataRow(string line) + private static (uint rowTimestamp, IReadOnlyList analogValues, uint digitalData, IReadOnlyList perChannelTimestamps)? + TryParseCsvDataRow(string line, CsvColumnLayout layout) { try { @@ -296,11 +338,16 @@ private static (uint rowTimestamp, IReadOnlyList analogValues, IReadOnly return null; } - var channelCount = columns.Length / 2; - var analogValues = new List(channelCount); - var perChannelTimestamps = new List(channelCount); + var totalPairs = columns.Length / 2; + var analogPairCount = layout.AnalogPairCount > 0 + ? Math.Min(layout.AnalogPairCount, totalPairs) + : (layout.HasDigitalPair ? totalPairs - 1 : totalPairs); + + var analogValues = new List(analogPairCount); + var perChannelTimestamps = new List(analogPairCount); - for (var ch = 0; ch < channelCount; ch++) + // Parse analog channel pairs + for (var ch = 0; ch < analogPairCount; ch++) { var tsCol = columns[ch * 2]; var valCol = columns[ch * 2 + 1]; @@ -319,8 +366,25 @@ private static (uint rowTimestamp, IReadOnlyList analogValues, IReadOnly analogValues.Add(val); } + // Parse digital I/O pair if present (last pair) + uint digitalData = 0; + if (layout.HasDigitalPair && totalPairs > analogPairCount) + { + var dioIndex = analogPairCount; + var dioValCol = columns[dioIndex * 2 + 1]; + if (uint.TryParse(dioValCol, NumberStyles.None, CultureInfo.InvariantCulture, out var dioVal)) + { + digitalData = dioVal; + } + } + + if (perChannelTimestamps.Count == 0) + { + return null; + } + // Use first channel's timestamp as the row timestamp - return (perChannelTimestamps[0], analogValues, perChannelTimestamps); + return (perChannelTimestamps[0], analogValues, digitalData, perChannelTimestamps); } catch { @@ -328,6 +392,40 @@ private static (uint rowTimestamp, IReadOnlyList analogValues, IReadOnly } } + /// + /// Scales raw ADC values to real voltage using device calibration data. + /// Formula: (raw / resolution * portRange * calM + calB) * internalScaleM + /// + private static IReadOnlyList ScaleRawAnalogValues( + IReadOnlyList rawValues, + SdCardDeviceConfiguration? config) + { + if (config == null || config.Resolution == 0) + { + // No config or resolution available — return raw values as-is + return rawValues; + } + + var result = new double[rawValues.Count]; + var resolution = (double)config.Resolution; + var cal = config.CalibrationValues; + var portRange = config.PortRange; + var intScale = config.InternalScaleM; + + for (var ch = 0; ch < rawValues.Count; ch++) + { + var calM = cal != null && ch < cal.Count ? cal[ch].Slope : 1.0; + var calB = cal != null && ch < cal.Count ? cal[ch].Intercept : 0.0; + var range = portRange != null && ch < portRange.Count ? portRange[ch] : 1.0; + var scaleM = intScale != null && ch < intScale.Count ? intScale[ch] : 1.0; + + var normalized = rawValues[ch] / resolution; + result[ch] = (normalized * range * calM + calB) * scaleM; + } + + return result; + } + /// /// Merges an override configuration into a parsed configuration. /// File-parsed values are primary; the override fills in gaps (zero or null fields). diff --git a/src/Daqifi.Core/Device/SdCard/SdCardJsonFileParser.cs b/src/Daqifi.Core/Device/SdCard/SdCardJsonFileParser.cs index a99ceae..432fa50 100644 --- a/src/Daqifi.Core/Device/SdCard/SdCardJsonFileParser.cs +++ b/src/Daqifi.Core/Device/SdCard/SdCardJsonFileParser.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Linq; using System.Runtime.CompilerServices; using System.Text.Json; using System.Threading; @@ -128,7 +129,10 @@ private static async IAsyncEnumerable ParseJsonLines( continue; } - var (timestamp, analogValues, digitalData) = parsed.Value; + var (timestamp, rawAnalogValues, digitalData) = parsed.Value; + + // Scale raw ADC values using device calibration + var analogValues = ScaleRawAnalogValues(rawAnalogValues, config); // Reconstruct absolute timestamp var absoluteTime = baseTime; @@ -255,6 +259,40 @@ private static SdCardDeviceConfiguration InferConfiguration(string firstLine, Sd return MergeConfiguration(inferred, options.ConfigurationOverride); } + /// + /// Scales raw ADC values to real voltage using device calibration data. + /// Formula: (raw / resolution * portRange * calM + calB) * internalScaleM + /// + private static IReadOnlyList ScaleRawAnalogValues( + IReadOnlyList rawValues, + SdCardDeviceConfiguration? config) + { + if (config == null || config.Resolution == 0) + { + // No config or resolution available — return raw values as-is + return rawValues; + } + + var result = new double[rawValues.Count]; + var resolution = (double)config.Resolution; + var cal = config.CalibrationValues; + var portRange = config.PortRange; + var intScale = config.InternalScaleM; + + for (var ch = 0; ch < rawValues.Count; ch++) + { + var calM = cal != null && ch < cal.Count ? cal[ch].Slope : 1.0; + var calB = cal != null && ch < cal.Count ? cal[ch].Intercept : 0.0; + var range = portRange != null && ch < portRange.Count ? portRange[ch] : 1.0; + var scaleM = intScale != null && ch < intScale.Count ? intScale[ch] : 1.0; + + var normalized = rawValues[ch] / resolution; + result[ch] = (normalized * range * calM + calB) * scaleM; + } + + return result; + } + /// /// Merges an override configuration into an inferred configuration. /// Inferred (file-derived) values are primary; the override fills in gaps (zero or null fields). From 8ce9a75067056d82a9bd0819d2a8f38f272c20db Mon Sep 17 00:00:00 2001 From: Tyler Kron Date: Thu, 19 Mar 2026 21:35:46 -0600 Subject: [PATCH 2/2] fix: handle digital-only CSV rows in SD card parser Use the dio timestamp as the row timestamp when no analog channels are present, instead of returning null and silently dropping the row. Co-Authored-By: Claude Opus 4.6 Entire-Checkpoint: a0fb8798807b --- .../Device/SdCard/SdCardCsvFileParserTests.cs | 32 +++++++++++++++++++ .../Device/SdCard/SdCardCsvFileParser.cs | 13 ++++++-- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/src/Daqifi.Core.Tests/Device/SdCard/SdCardCsvFileParserTests.cs b/src/Daqifi.Core.Tests/Device/SdCard/SdCardCsvFileParserTests.cs index 07961c8..5bbf24a 100644 --- a/src/Daqifi.Core.Tests/Device/SdCard/SdCardCsvFileParserTests.cs +++ b/src/Daqifi.Core.Tests/Device/SdCard/SdCardCsvFileParserTests.cs @@ -680,6 +680,38 @@ public async Task ParseAsync_AinColumnHeader_CorrectlyParsed() Assert.Equal(3, samples[0].AnalogValues.Count); } + [Fact] + public async Task ParseAsync_DigitalOnlyCsv_ParsesDioRows() + { + // Arrange — CSV with only a dio column pair, no analog channels + var content = + "# Device: Nyquist 1\n" + + "# Serial Number: SN001\n" + + "# Timestamp Tick Rate: 100 Hz\n" + + "dio_ts,dio_val\n" + + "1000,5\n" + + "1100,3\n"; + await using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content)); + + var parser = new global::Daqifi.Core.Device.SdCard.SdCardCsvFileParser(); + var options = new global::Daqifi.Core.Device.SdCard.SdCardParseOptions + { + SessionStartTime = new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc) + }; + + var session = await parser.ParseAsync(stream, "test.csv", options); + var samples = await ToListAsync(session.Samples); + + Assert.Equal(0, session.DeviceConfig!.AnalogPortCount); + Assert.Equal(1, session.DeviceConfig.DigitalPortCount); + Assert.Equal(2, samples.Count); + Assert.Empty(samples[0].AnalogValues); + Assert.Equal(5u, samples[0].DigitalData); + Assert.Equal(3u, samples[1].DigitalData); + // Second sample should be 1 second later (100 ticks / 100 Hz) + Assert.Equal(new DateTime(2024, 1, 1, 0, 0, 1, DateTimeKind.Utc), samples[1].Timestamp); + } + // ------------------------------------------------------------------------- // Helper // ------------------------------------------------------------------------- diff --git a/src/Daqifi.Core/Device/SdCard/SdCardCsvFileParser.cs b/src/Daqifi.Core/Device/SdCard/SdCardCsvFileParser.cs index 575d128..a9938e3 100644 --- a/src/Daqifi.Core/Device/SdCard/SdCardCsvFileParser.cs +++ b/src/Daqifi.Core/Device/SdCard/SdCardCsvFileParser.cs @@ -368,23 +368,30 @@ private static (uint rowTimestamp, IReadOnlyList analogValues, uint digi // Parse digital I/O pair if present (last pair) uint digitalData = 0; + uint dioTimestamp = 0; if (layout.HasDigitalPair && totalPairs > analogPairCount) { var dioIndex = analogPairCount; + var dioTsCol = columns[dioIndex * 2]; var dioValCol = columns[dioIndex * 2 + 1]; + uint.TryParse(dioTsCol, NumberStyles.None, CultureInfo.InvariantCulture, out dioTimestamp); if (uint.TryParse(dioValCol, NumberStyles.None, CultureInfo.InvariantCulture, out var dioVal)) { digitalData = dioVal; } } - if (perChannelTimestamps.Count == 0) + // Use first analog channel's timestamp, or fall back to dio timestamp + var rowTimestamp = perChannelTimestamps.Count > 0 + ? perChannelTimestamps[0] + : dioTimestamp; + + if (perChannelTimestamps.Count == 0 && dioTimestamp == 0 && !layout.HasDigitalPair) { return null; } - // Use first channel's timestamp as the row timestamp - return (perChannelTimestamps[0], analogValues, digitalData, perChannelTimestamps); + return (rowTimestamp, analogValues, digitalData, perChannelTimestamps); } catch {