From 4d5186d3b0f9b91930533ef2b55d2bce853c86ad Mon Sep 17 00:00:00 2001 From: Mykhailo Matviiv Date: Sun, 21 Dec 2025 02:11:40 +0100 Subject: [PATCH 1/2] Expose JsonReaderValue field directly --- .../LowLevelCollectionConverterShould.cs | 62 ++++++++++++++++++ .../LowLevel/TestCollectionConverterEntity.cs | 19 ++++++ .../Converters/LowLevel/TestCompositeKey.cs | 65 +++++++++++++++++++ src/EfficientDynamoDb/Converters/DdbReader.cs | 3 +- 4 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 src/EfficientDynamoDb.IntegrationTests/DataPlane/Converters/LowLevel/LowLevelCollectionConverterShould.cs create mode 100644 src/EfficientDynamoDb.IntegrationTests/DataPlane/Converters/LowLevel/TestCollectionConverterEntity.cs create mode 100644 src/EfficientDynamoDb.IntegrationTests/DataPlane/Converters/LowLevel/TestCompositeKey.cs diff --git a/src/EfficientDynamoDb.IntegrationTests/DataPlane/Converters/LowLevel/LowLevelCollectionConverterShould.cs b/src/EfficientDynamoDb.IntegrationTests/DataPlane/Converters/LowLevel/LowLevelCollectionConverterShould.cs new file mode 100644 index 00000000..fbbce4a2 --- /dev/null +++ b/src/EfficientDynamoDb.IntegrationTests/DataPlane/Converters/LowLevel/LowLevelCollectionConverterShould.cs @@ -0,0 +1,62 @@ +using NUnit.Framework; +using Shouldly; + +namespace EfficientDynamoDb.IntegrationTests.DataPlane.Converters.LowLevel; + +[TestFixture] +public class LowLevelCollectionConverterShould +{ + private const string KeyPrefix = "effddb_tests-low_lvl_collection_converters"; + private DynamoDbContext _context = null!; + private string? _testPartitionKey; + private string? _testSortKey; + + [SetUp] + public void SetUp() + { + _context = TestHelper.CreateContext(); + } + + [TearDown] + public async Task TearDown() + { + if (_testPartitionKey != null && _testSortKey != null) + { + await _context.DeleteItemAsync(_testPartitionKey, _testSortKey); + } + } + + [Test] + public async Task ApplyConvertersByDefaultForAllTimeTypes() + { + _testPartitionKey = $"{KeyPrefix}-pk"; + _testSortKey = $"{KeyPrefix}-sk"; + + var item = new TestCollectionConverterEntity + { + PartitionKey = _testPartitionKey, + SortKey = _testSortKey, + CompositeKey = new() + { + Part1 = "part1_value", + Part2 = "part2_value", + Part3 = "part3_value" + }, + CompositeKey2 = new() + { + Part1 = "part1_value_2", + Part2 = "part2_value_2", + Part3 = "part3_value_2" + } + }; + + await _context.PutItemAsync(item); + + var retrieved = await _context.GetItem() + .WithPrimaryKey(_testPartitionKey, _testSortKey) + .WithConsistentRead(true) + .ToItemAsync(); + + retrieved.ShouldBeEquivalentTo(item); + } +} \ No newline at end of file diff --git a/src/EfficientDynamoDb.IntegrationTests/DataPlane/Converters/LowLevel/TestCollectionConverterEntity.cs b/src/EfficientDynamoDb.IntegrationTests/DataPlane/Converters/LowLevel/TestCollectionConverterEntity.cs new file mode 100644 index 00000000..6cb842d5 --- /dev/null +++ b/src/EfficientDynamoDb.IntegrationTests/DataPlane/Converters/LowLevel/TestCollectionConverterEntity.cs @@ -0,0 +1,19 @@ +using EfficientDynamoDb.Attributes; + +namespace EfficientDynamoDb.IntegrationTests.DataPlane.Converters.LowLevel; + +[DynamoDbTable(TestHelper.TestTableName)] +public class TestCollectionConverterEntity +{ + [DynamoDbProperty("pk", DynamoDbAttributeType.PartitionKey)] + public required string PartitionKey { get; init; } + + [DynamoDbProperty("sk", DynamoDbAttributeType.SortKey)] + public required string SortKey { get; init; } + + [DynamoDbProperty("compositeKey", typeof(TestCompositeKeyDdbConverter))] + public required TestCompositeKey CompositeKey { get; init; } + + [DynamoDbProperty("compositeKey2", typeof(TestCompositeKeyDdbConverter))] + public required TestCompositeKey CompositeKey2 { get; init; } +} \ No newline at end of file diff --git a/src/EfficientDynamoDb.IntegrationTests/DataPlane/Converters/LowLevel/TestCompositeKey.cs b/src/EfficientDynamoDb.IntegrationTests/DataPlane/Converters/LowLevel/TestCompositeKey.cs new file mode 100644 index 00000000..3f6c87fb --- /dev/null +++ b/src/EfficientDynamoDb.IntegrationTests/DataPlane/Converters/LowLevel/TestCompositeKey.cs @@ -0,0 +1,65 @@ +using System.Text; +using EfficientDynamoDb.Converters; +using EfficientDynamoDb.DocumentModel; + +namespace EfficientDynamoDb.IntegrationTests.DataPlane.Converters.LowLevel; + +public class TestCompositeKey +{ + public string Part1 { get; set; } = null!; + public string Part2 { get; set; } = null!; + public string Part3 { get; set; } = null!; +} + +public class TestCompositeKeyDdbConverter : DdbConverter +{ + public override TestCompositeKey Read(in AttributeValue attributeValue) + { + var list = attributeValue.AsListAttribute(); + return new() + { + Part1 = list.Items[0].AsString(), + Part2 = list.Items[1].AsString(), + Part3 = list.Items[2].AsString() + }; + } + + public override AttributeValue Write(ref TestCompositeKey value) + { + return new ListAttributeValue([ + new StringAttributeValue(value.Part1), new StringAttributeValue(value.Part2), new StringAttributeValue(value.Part3) + ]); + } + + public override TestCompositeKey Read(ref DdbReader reader) + { + ref var jsonReader = ref reader.JsonReaderValue; + jsonReader.Read(); + jsonReader.Read(); + jsonReader.Read(); + var part1 = Encoding.UTF8.GetString(jsonReader.ValueSpan); + jsonReader.Read(); + + jsonReader.Read(); + jsonReader.Read(); + jsonReader.Read(); + var part2 = Encoding.UTF8.GetString(jsonReader.ValueSpan); + jsonReader.Read(); + + jsonReader.Read(); + jsonReader.Read(); + jsonReader.Read(); + var part3 = Encoding.UTF8.GetString(jsonReader.ValueSpan); + jsonReader.Read(); + + // Last end array + jsonReader.Read(); + + return new() + { + Part1 = part1, + Part2 = part2, + Part3 = part3 + }; + } +} \ No newline at end of file diff --git a/src/EfficientDynamoDb/Converters/DdbReader.cs b/src/EfficientDynamoDb/Converters/DdbReader.cs index 46edf037..cad18470 100644 --- a/src/EfficientDynamoDb/Converters/DdbReader.cs +++ b/src/EfficientDynamoDb/Converters/DdbReader.cs @@ -8,8 +8,9 @@ namespace EfficientDynamoDb.Converters { public ref struct DdbReader { - internal Utf8JsonReader JsonReaderValue; + public Utf8JsonReader JsonReaderValue; + [Obsolete($"This property returns a copy of {nameof(JsonReaderValue)} that won't advance the underlying reader correctly. Use ref to {nameof(JsonReaderValue)} instead.")] public Utf8JsonReader JsonReader { [MethodImpl(MethodImplOptions.AggressiveInlining)] From 7abb93b0e8d97bae170964316b2ce539aa9bd0e5 Mon Sep 17 00:00:00 2001 From: Mykhailo Matviiv Date: Sun, 21 Dec 2025 03:23:26 +0100 Subject: [PATCH 2/2] Update converter docs --- .../docs/dev_guide/high_level/converters.md | 70 +++++++++++++++++-- 1 file changed, 65 insertions(+), 5 deletions(-) diff --git a/website/docs/dev_guide/high_level/converters.md b/website/docs/dev_guide/high_level/converters.md index 22e6ea88..12b866c5 100644 --- a/website/docs/dev_guide/high_level/converters.md +++ b/website/docs/dev_guide/high_level/converters.md @@ -112,8 +112,8 @@ public class CustomIntConverter : DdbConverter // Efficient zero-allocation JSON to int conversion public override int Read(ref DdbReader reader) { - if (!Utf8Parser.TryParse(reader.JsonReader.ValueSpan, out int value, out _)) - throw new DdbException($"Couldn't parse int ddb value from '{reader.JsonReader.GetString()}'."); + if (!Utf8Parser.TryParse(reader.JsonReaderValue.ValueSpan, out int value, out _)) + throw new DdbException($"Couldn't parse int ddb value from '{reader.JsonReaderValue.GetString()}'."); return value; } @@ -136,11 +136,71 @@ public class CustomIntConverter : DdbConverter ### JSON reading -When a low-level read is called, `DdbReader.JsonReader` is already pointed to the JSON value. Current attribute type is automatically parsed and can be accessed using `DdbReader.AttributeType` property. +When a low-level read is called, `DdbReader.JsonReaderValue` is already pointed to the JSON value. Current attribute type is automatically parsed and can be accessed using `DdbReader.AttributeType` property. -The `reader.JsonReader.HasValueSequence` is guaranteed to be false at this point, so it's safe to use `reader.JsonReader.ValueSpan` to access the JSON buffer. +The `reader.JsonReaderValue.HasValueSequence` is guaranteed to be false at this point, so it's safe to use `reader.JsonReaderValue.ValueSpan` to access the JSON buffer. -The `DdbReader.JsonReader.Read` method should not be explicitly called unless you are writing a converter for a non-scalar DynamoDB data type - i.e., a map, list or set. +The `ref reader.JsonReaderValue.Read()` method should not be explicitly called unless you are writing a converter for a non-scalar DynamoDB data type - i.e., a map, list or set. When reading non-scalar types, you must use `ref` to access `JsonReaderValue` to ensure the reader advances correctly through the JSON structure. + +#### Parsing DynamoDB lists and arrays + +By default, EfficientDynamoDb automatically parses DynamoDB collections (lists, sets and maps) into .NET collections and dictionaries. +However, if you need to parse a DynamoDB list (array) into a custom type, you can implement the `Read` method manually. + +When parsing a DynamoDB collection, you need to manually advance through the JSON tokens. Assuming the following DDB JSON for a list of strings: + +```json +[ + { "S": "value1" }, + { "S": "value2" }, + { "S": "value3" } +] +``` + +The following converter will parse this list into a separator-delimited string, e.g. `value1#value2#value3`: + +```csharp +public class StringListConverter : DdbConverter +{ + // High-level methods are skipped for simplicity in this example. + + public override string Read(ref DdbReader reader) + { + ref var jsonReader = ref reader.JsonReaderValue; + // jsonReader is pointing to the StartArray token + + var result = new List(); + while (jsonReader.TokenType != JsonTokenType.EndArray) + { + // Read StartObject token + jsonReader.Read(); + + // Read property name ("S" for string) + jsonReader.Read(); + + // Read string value + jsonReader.Read(); + result.Add(jsonReader.GetString()); + + // Read EndObject token + jsonReader.Read(); + } + + // Read EndArray token + jsonReader.Read(); + + return string.Join('#', result); + } +} +``` + +:::info +Always use `ref` when accessing `JsonReaderValue` to call `Read()` or access its properties. This ensures the reader state advances correctly. Using the obsolete `JsonReader` property (which returns a copy) will not advance the underlying reader and will cause parsing errors. +::: + +:::caution +Leaving the reader in invalid state can cause parsing errors for the whole entity. It is the responsibility of the converter to ensure the reader is in a valid state after reading. +::: ### JSON writing