From 521cbebfed9fb5c6a63e25b7fd76bf4515025b3f Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Wed, 26 Nov 2025 22:14:07 +0700 Subject: [PATCH 01/22] Add benchmark project --- Directory.Packages.props | 1 + .../BinaryPatchBenchmark.cs | 56 +++++++++++++++++++ .../BitSoft.BinaryTools.Benchmarks.csproj | 17 ++++++ src/BitSoft.BinaryTools.Benchmarks/Program.cs | 12 ++++ src/BitSoft.BinaryTools.sln | 6 ++ 5 files changed, 92 insertions(+) create mode 100644 src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs create mode 100644 src/BitSoft.BinaryTools.Benchmarks/BitSoft.BinaryTools.Benchmarks.csproj create mode 100644 src/BitSoft.BinaryTools.Benchmarks/Program.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 4c05ccc..a926e67 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -1,5 +1,6 @@  + diff --git a/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs b/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs new file mode 100644 index 0000000..ff2470d --- /dev/null +++ b/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs @@ -0,0 +1,56 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; +using BitSoft.BinaryTools.Patch; + +namespace BitSoft.BinaryTools.Benchmarks; + +[ShortRunJob] +[MemoryDiagnoser] +public class BinaryPatchBenchmark +{ + private byte[]? _source; + private byte[]? _modified; + + private Stream? _sourceStream; + private Stream? _modifiedStream; + private Stream? _patchStream; + + [GlobalSetup] + public void GlobalSetUp() + { + _source = new byte[1024]; + _modified = new byte[1024]; + + Array.Copy(sourceArray: _source, destinationArray: _modified, length: _source.Length); + + _sourceStream = new MemoryStream(_source); + _modifiedStream = new MemoryStream(_modified); + } + + [IterationSetup] + public void SetUp() + { + _patchStream = new MemoryStream(); + } + + [IterationCleanup] + public void Cleanup() + { + _patchStream?.Dispose(); + } + + [GlobalCleanup] + public void GlobalCleanUp() + { + _sourceStream?.Dispose(); + _modifiedStream?.Dispose(); + } + + [Benchmark] + public async Task CreateBinaryPatch() + { + await BinaryPatch.CreateAsync(source: _sourceStream!, modified: _modifiedStream!, output: _patchStream!); + } +} diff --git a/src/BitSoft.BinaryTools.Benchmarks/BitSoft.BinaryTools.Benchmarks.csproj b/src/BitSoft.BinaryTools.Benchmarks/BitSoft.BinaryTools.Benchmarks.csproj new file mode 100644 index 0000000..48507eb --- /dev/null +++ b/src/BitSoft.BinaryTools.Benchmarks/BitSoft.BinaryTools.Benchmarks.csproj @@ -0,0 +1,17 @@ + + + + Exe + net8.0 + false + + + + + + + + + + + diff --git a/src/BitSoft.BinaryTools.Benchmarks/Program.cs b/src/BitSoft.BinaryTools.Benchmarks/Program.cs new file mode 100644 index 0000000..ef77935 --- /dev/null +++ b/src/BitSoft.BinaryTools.Benchmarks/Program.cs @@ -0,0 +1,12 @@ +using System.Reflection; +using BenchmarkDotNet.Running; + +namespace BitSoft.BinaryTools.Benchmarks; + +class Program +{ + static void Main(string[] args) + { + new BenchmarkSwitcher(typeof(Program).GetTypeInfo().Assembly).Run(args); + } +} diff --git a/src/BitSoft.BinaryTools.sln b/src/BitSoft.BinaryTools.sln index 90fe38e..9c1ff12 100644 --- a/src/BitSoft.BinaryTools.sln +++ b/src/BitSoft.BinaryTools.sln @@ -19,6 +19,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "workflows", "workflows", "{ ..\.github\workflows\dotnet.yml = ..\.github\workflows\dotnet.yml EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BitSoft.BinaryTools.Benchmarks", "BitSoft.BinaryTools.Benchmarks\BitSoft.BinaryTools.Benchmarks.csproj", "{9C7C350F-7CD9-47C4-A3D3-64E9F75E5B18}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -33,6 +35,10 @@ Global {BE7275D4-BCC8-4E39-9DDC-6FE0426D650C}.Debug|Any CPU.Build.0 = Debug|Any CPU {BE7275D4-BCC8-4E39-9DDC-6FE0426D650C}.Release|Any CPU.ActiveCfg = Release|Any CPU {BE7275D4-BCC8-4E39-9DDC-6FE0426D650C}.Release|Any CPU.Build.0 = Release|Any CPU + {9C7C350F-7CD9-47C4-A3D3-64E9F75E5B18}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9C7C350F-7CD9-47C4-A3D3-64E9F75E5B18}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9C7C350F-7CD9-47C4-A3D3-64E9F75E5B18}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9C7C350F-7CD9-47C4-A3D3-64E9F75E5B18}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {2C6A0F9D-48B5-40FD-A0FF-45ECA8034BB9} = {1B0803C7-C282-44FB-B1C9-6199FD6E1122} From 22a24a15f02cfb2a22b44c0d1e34a519b8266125 Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Wed, 26 Nov 2025 22:18:25 +0700 Subject: [PATCH 02/22] Add length param --- src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs b/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs index ff2470d..1f4bea0 100644 --- a/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs +++ b/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs @@ -17,11 +17,14 @@ public class BinaryPatchBenchmark private Stream? _modifiedStream; private Stream? _patchStream; + [Params(1024 * 1024, 10 * 1024 * 1024)] + public int BufferLength { get; set; } + [GlobalSetup] public void GlobalSetUp() { - _source = new byte[1024]; - _modified = new byte[1024]; + _source = new byte[BufferLength]; + _modified = new byte[BufferLength]; Array.Copy(sourceArray: _source, destinationArray: _modified, length: _source.Length); From 8eff75da5832c99311cdb9c54b0793a89f8decc6 Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Wed, 26 Nov 2025 22:36:56 +0700 Subject: [PATCH 03/22] Fill buffers --- .../BinaryPatchBenchmark.cs | 18 ++++++++++++++++++ .../Utils/Create.cs | 11 +++++++++++ 2 files changed, 29 insertions(+) create mode 100644 src/BitSoft.BinaryTools.Benchmarks/Utils/Create.cs diff --git a/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs b/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs index 1f4bea0..212c6de 100644 --- a/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs +++ b/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs @@ -2,6 +2,7 @@ using System.IO; using System.Threading.Tasks; using BenchmarkDotNet.Attributes; +using BitSoft.BinaryTools.Benchmarks.Utils; using BitSoft.BinaryTools.Patch; namespace BitSoft.BinaryTools.Benchmarks; @@ -20,14 +21,31 @@ public class BinaryPatchBenchmark [Params(1024 * 1024, 10 * 1024 * 1024)] public int BufferLength { get; set; } + [Params(1, 3, 5)] public int ChangedBlocks { get; set; } + + [Params(16, 128, 512)] public int ChangeSize { get; set; } + [GlobalSetup] public void GlobalSetUp() { _source = new byte[BufferLength]; _modified = new byte[BufferLength]; + Create.RandomData(_source); + Array.Copy(sourceArray: _source, destinationArray: _modified, length: _source.Length); + var changeBlockSize = _source.Length / (ChangedBlocks + 1); + + for (var b = 1; b <= ChangedBlocks; b++) + { + var position = changeBlockSize * b; + + var span = _modified.AsSpan(start: position, length: ChangeSize); + + Create.RandomData(span); + } + _sourceStream = new MemoryStream(_source); _modifiedStream = new MemoryStream(_modified); } diff --git a/src/BitSoft.BinaryTools.Benchmarks/Utils/Create.cs b/src/BitSoft.BinaryTools.Benchmarks/Utils/Create.cs new file mode 100644 index 0000000..e5f08f0 --- /dev/null +++ b/src/BitSoft.BinaryTools.Benchmarks/Utils/Create.cs @@ -0,0 +1,11 @@ +using System; + +namespace BitSoft.BinaryTools.Benchmarks.Utils; + +public static class Create +{ + public static void RandomData(Span buffer) + { + Random.Shared.NextBytes(buffer); + } +} From 717ebc42adabdfd355ce3c8e955b3e8919920b06 Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Wed, 26 Nov 2025 22:54:32 +0700 Subject: [PATCH 04/22] Add benchmark results --- .../BinaryPatchBenchmark.cs | 15 +++-- src/BitSoft.BinaryTools.Benchmarks/Readme.md | 63 +++++++++++++++++++ 2 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 src/BitSoft.BinaryTools.Benchmarks/Readme.md diff --git a/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs b/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs index 212c6de..5a018b8 100644 --- a/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs +++ b/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs @@ -18,12 +18,14 @@ public class BinaryPatchBenchmark private Stream? _modifiedStream; private Stream? _patchStream; - [Params(1024 * 1024, 10 * 1024 * 1024)] + [Params(1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024)] public int BufferLength { get; set; } - [Params(1, 3, 5)] public int ChangedBlocks { get; set; } + [Params(3, 5)] public int ChangedBlocks { get; set; } - [Params(16, 128, 512)] public int ChangeSize { get; set; } + [Params(128, 512)] public int ChangeSize { get; set; } + + [Params(512, 1024, 4096)] public int BlockSize { get; set; } [GlobalSetup] public void GlobalSetUp() @@ -72,6 +74,11 @@ public void GlobalCleanUp() [Benchmark] public async Task CreateBinaryPatch() { - await BinaryPatch.CreateAsync(source: _sourceStream!, modified: _modifiedStream!, output: _patchStream!); + await BinaryPatch.CreateAsync( + source: _sourceStream!, + modified: _modifiedStream!, + output: _patchStream!, + blockSize: BlockSize + ); } } diff --git a/src/BitSoft.BinaryTools.Benchmarks/Readme.md b/src/BitSoft.BinaryTools.Benchmarks/Readme.md new file mode 100644 index 0000000..8ca7906 --- /dev/null +++ b/src/BitSoft.BinaryTools.Benchmarks/Readme.md @@ -0,0 +1,63 @@ +# Benchmarks + +``` +| Method | BufferLength | ChangedBlocks | ChangeSize | BlockSize | Mean | Error | StdDev | Allocated | +|------------------ |------------- |-------------- |----------- |---------- |---------:|-----------:|---------:|----------:| +| CreateBinaryPatch | 1048576 | 3 | 128 | 512 | 16.80 us | 7.952 us | 0.436 us | 448 B | +| CreateBinaryPatch | 1048576 | 3 | 128 | 1024 | 20.50 us | 55.666 us | 3.051 us | 448 B | +| CreateBinaryPatch | 1048576 | 3 | 128 | 4096 | 15.00 us | 47.853 us | 2.623 us | 448 B | +| CreateBinaryPatch | 1048576 | 3 | 512 | 512 | 16.68 us | 61.173 us | 3.353 us | 448 B | +| CreateBinaryPatch | 1048576 | 3 | 512 | 1024 | 18.53 us | 65.508 us | 3.591 us | 448 B | +| CreateBinaryPatch | 1048576 | 3 | 512 | 4096 | 20.43 us | 37.933 us | 2.079 us | 448 B | +| CreateBinaryPatch | 1048576 | 5 | 128 | 512 | 20.77 us | 59.992 us | 3.288 us | 448 B | +| CreateBinaryPatch | 1048576 | 5 | 128 | 1024 | 14.37 us | 35.673 us | 1.955 us | 448 B | +| CreateBinaryPatch | 1048576 | 5 | 128 | 4096 | 15.17 us | 38.672 us | 2.120 us | 448 B | +| CreateBinaryPatch | 1048576 | 5 | 512 | 512 | 15.95 us | 61.300 us | 3.360 us | 448 B | +| CreateBinaryPatch | 1048576 | 5 | 512 | 1024 | 16.00 us | 46.548 us | 2.551 us | 448 B | +| CreateBinaryPatch | 1048576 | 5 | 512 | 4096 | 17.50 us | 4.827 us | 0.265 us | 448 B | +| CreateBinaryPatch | 10485760 | 3 | 128 | 512 | 15.08 us | 45.621 us | 2.501 us | 448 B | +| CreateBinaryPatch | 10485760 | 3 | 128 | 1024 | 17.53 us | 47.934 us | 2.627 us | 448 B | +| CreateBinaryPatch | 10485760 | 3 | 128 | 4096 | 15.60 us | 17.968 us | 0.985 us | 448 B | +| CreateBinaryPatch | 10485760 | 3 | 512 | 512 | 17.13 us | 68.075 us | 3.731 us | 448 B | +| CreateBinaryPatch | 10485760 | 3 | 512 | 1024 | 15.67 us | 42.798 us | 2.346 us | 448 B | +| CreateBinaryPatch | 10485760 | 3 | 512 | 4096 | 18.53 us | 46.381 us | 2.542 us | 448 B | +| CreateBinaryPatch | 10485760 | 5 | 128 | 512 | 17.00 us | 26.311 us | 1.442 us | 448 B | +| CreateBinaryPatch | 10485760 | 5 | 128 | 1024 | 16.05 us | 4.827 us | 0.265 us | 448 B | +| CreateBinaryPatch | 10485760 | 5 | 128 | 4096 | 15.82 us | 53.200 us | 2.916 us | 448 B | +| CreateBinaryPatch | 10485760 | 5 | 512 | 512 | 15.73 us | 36.866 us | 2.021 us | 448 B | +| CreateBinaryPatch | 10485760 | 5 | 512 | 1024 | 17.63 us | 26.895 us | 1.474 us | 448 B | +| CreateBinaryPatch | 10485760 | 5 | 512 | 4096 | 16.13 us | 70.288 us | 3.853 us | 448 B | +| CreateBinaryPatch | 104857600 | 3 | 128 | 512 | 19.27 us | 12.147 us | 0.666 us | 448 B | +| CreateBinaryPatch | 104857600 | 3 | 128 | 1024 | 20.00 us | 94.375 us | 5.173 us | 448 B | +| CreateBinaryPatch | 104857600 | 3 | 128 | 4096 | 22.37 us | 13.693 us | 0.751 us | 448 B | +| CreateBinaryPatch | 104857600 | 3 | 512 | 512 | 20.40 us | 1.824 us | 0.100 us | 448 B | +| CreateBinaryPatch | 104857600 | 3 | 512 | 1024 | 19.00 us | 27.608 us | 1.513 us | 448 B | +| CreateBinaryPatch | 104857600 | 3 | 512 | 4096 | 21.50 us | 47.818 us | 2.621 us | 448 B | +| CreateBinaryPatch | 104857600 | 5 | 128 | 512 | 19.70 us | 41.722 us | 2.287 us | 448 B | +| CreateBinaryPatch | 104857600 | 5 | 128 | 1024 | 19.20 us | 37.699 us | 2.066 us | 448 B | +| CreateBinaryPatch | 104857600 | 5 | 128 | 4096 | 16.63 us | 36.956 us | 2.026 us | 448 B | +| CreateBinaryPatch | 104857600 | 5 | 512 | 512 | 18.03 us | 18.274 us | 1.002 us | 448 B | +| CreateBinaryPatch | 104857600 | 5 | 512 | 1024 | 21.80 us | 145.642 us | 7.983 us | 448 B | +| CreateBinaryPatch | 104857600 | 5 | 512 | 4096 | 16.13 us | 31.086 us | 1.704 us | 448 B | +``` +## Legends +``` + BufferLength : Value of the 'BufferLength' parameter + ChangedBlocks : Value of the 'ChangedBlocks' parameter + ChangeSize : Value of the 'ChangeSize' parameter + BlockSize : Value of the 'BlockSize' parameter + Mean : Arithmetic mean of all measurements + Error : Half of 99.9% confidence interval + StdDev : Standard deviation of all measurements + Allocated : Allocated memory per single operation (managed only, inclusive, 1KB = 1024B) + 1 us : 1 Microsecond (0.000001 sec) +``` + +## Additional info +``` +BenchmarkDotNet v0.15.6, Windows 11 (10.0.26200.7171) +AMD Ryzen 7 5800U with Radeon Graphics 1.90GHz, 1 CPU, 16 logical and 8 physical cores +.NET SDK 10.0.100 + [Host] : .NET 8.0.22 (8.0.22, 8.0.2225.52707), X64 RyuJIT x86-64-v3 + ShortRun : .NET 8.0.22 (8.0.22, 8.0.2225.52707), X64 RyuJIT x86-64-v3 +``` From 5d2627edea216f06d19814d07a522a7af8a91fd2 Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Thu, 27 Nov 2025 12:49:34 +0700 Subject: [PATCH 05/22] Add performance test --- .../Patch/BinaryPatchTests.cs | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs b/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs index d59e315..e3b4194 100644 --- a/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs +++ b/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs @@ -1,4 +1,6 @@ +using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Threading.Tasks; using BitSoft.BinaryTools.Patch; @@ -60,4 +62,40 @@ await BinaryPatch.CreateAsync( Assert.That(patched.Length, Is.EqualTo(modified.Length)); Assert.That(patched, Is.EqualTo(modified)); } -} \ No newline at end of file + + [Ignore("Performance test")] + [TestCase(10 * 1024 * 1024, 1024)] + [TestCase(10 * 1024 * 1024, 4 * 1024)] + public async Task Should_CreatePatch(int bufferLength, int blockSize) + { + // Arrange + var source = new byte[bufferLength]; + var modified = new byte[bufferLength]; + + Random.Shared.NextBytes(source); + + Array.Copy(sourceArray: source, destinationArray: modified, length: source.Length); + + using var sourceStream = new MemoryStream(source); + using var modifiedStream = new MemoryStream(modified); + using var patchStream = new MemoryStream(); + + // Act + var stopwatch = Stopwatch.StartNew(); + + await BinaryPatch.CreateAsync( + source: sourceStream, + modified: modifiedStream, + output: patchStream, + blockSize: blockSize + ); + + stopwatch.Stop(); + + // Assert + Console.WriteLine("Source length: {0}", sourceStream.Length); + Console.WriteLine("Block size: {0}", blockSize); + Console.WriteLine("Patch length: {0}", patchStream.Position); + Console.WriteLine("Time: {0:g}", stopwatch.Elapsed); + } +} From 939ebb0ab4d1d71eb926637f253dadb0320e9d30 Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Thu, 27 Nov 2025 13:18:30 +0700 Subject: [PATCH 06/22] Support variuos block length --- .../Patch/BinaryPatchTests.cs | 2 +- src/BitSoft.BinaryTools/Patch/BinaryPatch.cs | 82 +++++++++++++------ .../Patch/BlockInfoContainer.cs | 16 +++- .../Patch/CopyBlockWithLengthSegment.cs | 13 +++ .../Patch/CopyPatchSegment.cs | 8 -- .../Patch/PatchBlockInfo.cs | 4 +- .../Patch/PatchBlockInfoWithLength.cs | 7 ++ src/BitSoft.BinaryTools/Patch/PatchReader.cs | 12 ++- src/BitSoft.BinaryTools/Patch/PatchWriter.cs | 12 ++- .../Patch/ProtocolConst.cs | 7 +- 10 files changed, 119 insertions(+), 44 deletions(-) create mode 100644 src/BitSoft.BinaryTools/Patch/CopyBlockWithLengthSegment.cs delete mode 100644 src/BitSoft.BinaryTools/Patch/CopyPatchSegment.cs create mode 100644 src/BitSoft.BinaryTools/Patch/PatchBlockInfoWithLength.cs diff --git a/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs b/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs index e3b4194..38f44a4 100644 --- a/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs +++ b/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs @@ -63,7 +63,7 @@ await BinaryPatch.CreateAsync( Assert.That(patched, Is.EqualTo(modified)); } - [Ignore("Performance test")] + // [Ignore("Performance test")] [TestCase(10 * 1024 * 1024, 1024)] [TestCase(10 * 1024 * 1024, 4 * 1024)] public async Task Should_CreatePatch(int bufferLength, int blockSize) diff --git a/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs b/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs index 3bbd568..b81db19 100644 --- a/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs +++ b/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs @@ -121,14 +121,28 @@ public static async ValueTask CreateAsync( segmentStart = NotDefined; } - await writer.WriteCopyAsync( - blockIndex: block.BlockIndex, - blockLength: block.Length, - cancellationToken: cancellationToken - ); + var blockLength = blockSize; + + if (block is PatchBlockInfoWithLength blockInfoWithLength) + { + blockLength = blockInfoWithLength.Length; + + await writer.WriteCopyBlockWithLengthAsync( + blockIndex: block.BlockIndex, + blockLength: blockLength, + cancellationToken: cancellationToken + ); + } + else + { + await writer.WriteCopyBlockAsync( + blockIndex: block.BlockIndex, + cancellationToken: cancellationToken + ); + } buffer - .AsSpan(start: position + block.Length, length: bufferLength - position - block.Length - 1) + .AsSpan(start: position + blockLength, length: bufferLength - position - blockLength - 1) .CopyTo(buffer.AsSpan(start: 0)); resetHash = true; @@ -186,26 +200,41 @@ public static async ValueTask ApplyAsync( case DataPatchSegment dataPatchSegment: await output.WriteAsync(dataPatchSegment.Data, cancellationToken); break; - case CopyPatchSegment copyPatchSegment: - var targetPosition = blockSize * copyPatchSegment.BlockIndex; - source.Seek(targetPosition, SeekOrigin.Begin); - var buffer = Pool.Rent(copyPatchSegment.BlockLength); - try - { - var memory = buffer.AsMemory(start: 0, length: copyPatchSegment.BlockLength); - var count = await source.ReadAsync(memory, cancellationToken); - if (count != copyPatchSegment.BlockLength) throw new InvalidOperationException(); - await output.WriteAsync(memory, cancellationToken); - } - finally - { - Pool.Return(buffer); - } - + case CopyBlockSegment copyBlockSegment: + await CopyBlockSegmentAsync( + blockIndex: copyBlockSegment.BlockIndex, + blockLength: blockSize + ); + break; + case CopyBlockWithLengthSegment copyPatchSegment: + await CopyBlockSegmentAsync( + blockIndex: copyPatchSegment.BlockIndex, + blockLength: copyPatchSegment.BlockLength + ); break; default: throw new NotSupportedException(); } + + continue; + + async ValueTask CopyBlockSegmentAsync(int blockIndex, int blockLength) + { + var targetPosition = blockSize * blockIndex; + source.Seek(targetPosition, SeekOrigin.Begin); + var buffer = Pool.Rent(blockLength); + try + { + var memory = buffer.AsMemory(start: 0, length: blockLength); + var count = await source.ReadAsync(memory, cancellationToken); + if (count != blockLength) throw new InvalidOperationException(); + await output.WriteAsync(memory, cancellationToken); + } + finally + { + Pool.Return(buffer); + } + } } } @@ -234,7 +263,14 @@ private static async ValueTask CalculateHashesAsync( var hash = RollingHash.Create(buffer.AsSpan(start: 0, length: length)); - blockInfoContainer.Process(hash: hash, blockIndex: blockIndex, blockLength: length); + if (length == blockSize) + { + blockInfoContainer.Process(hash: hash, blockIndex: blockIndex); + } + else + { + blockInfoContainer.Process(hash: hash, blockIndex: blockIndex, blockLength: length); + } if (length < blockSize) break; diff --git a/src/BitSoft.BinaryTools/Patch/BlockInfoContainer.cs b/src/BitSoft.BinaryTools/Patch/BlockInfoContainer.cs index 37c4269..d173f00 100644 --- a/src/BitSoft.BinaryTools/Patch/BlockInfoContainer.cs +++ b/src/BitSoft.BinaryTools/Patch/BlockInfoContainer.cs @@ -6,10 +6,22 @@ internal sealed class BlockInfoContainer { private readonly Dictionary> _hashes = new(); + public void Process(RollingHash hash, int blockIndex) + { + var checksum = hash.GetChecksum(); + var block = new PatchBlockInfo(blockIndex: blockIndex, hash: checksum); + if (!_hashes.TryGetValue(checksum, out var blocks)) + { + _hashes[block.Hash] = blocks = []; + } + + blocks.Add(block); + } + public void Process(RollingHash hash, int blockIndex, int blockLength) { var checksum = hash.GetChecksum(); - var block = new PatchBlockInfo(blockIndex: blockIndex, hash: checksum, length: blockLength); + var block = new PatchBlockInfoWithLength(blockIndex: blockIndex, hash: checksum, length: blockLength); if (!_hashes.TryGetValue(checksum, out var blocks)) { _hashes[block.Hash] = blocks = []; @@ -31,4 +43,4 @@ public void Process(RollingHash hash, int blockIndex, int blockLength) return null; } -} \ No newline at end of file +} diff --git a/src/BitSoft.BinaryTools/Patch/CopyBlockWithLengthSegment.cs b/src/BitSoft.BinaryTools/Patch/CopyBlockWithLengthSegment.cs new file mode 100644 index 0000000..f348b25 --- /dev/null +++ b/src/BitSoft.BinaryTools/Patch/CopyBlockWithLengthSegment.cs @@ -0,0 +1,13 @@ +namespace BitSoft.BinaryTools.Patch; + +internal sealed class CopyBlockWithLengthSegment(int blockIndex, int blockLength) : IPatchSegment +{ + public int BlockIndex { get; } = blockIndex; + + public int BlockLength { get; } = blockLength; +} + +internal sealed class CopyBlockSegment(int blockIndex) : IPatchSegment +{ + public int BlockIndex { get; } = blockIndex; +} diff --git a/src/BitSoft.BinaryTools/Patch/CopyPatchSegment.cs b/src/BitSoft.BinaryTools/Patch/CopyPatchSegment.cs deleted file mode 100644 index c42a4ff..0000000 --- a/src/BitSoft.BinaryTools/Patch/CopyPatchSegment.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace BitSoft.BinaryTools.Patch; - -internal sealed class CopyPatchSegment(int blockIndex, int blockLength) : IPatchSegment -{ - public int BlockIndex { get; } = blockIndex; - - public int BlockLength { get; } = blockLength; -} diff --git a/src/BitSoft.BinaryTools/Patch/PatchBlockInfo.cs b/src/BitSoft.BinaryTools/Patch/PatchBlockInfo.cs index 92d5786..496783c 100644 --- a/src/BitSoft.BinaryTools/Patch/PatchBlockInfo.cs +++ b/src/BitSoft.BinaryTools/Patch/PatchBlockInfo.cs @@ -1,10 +1,8 @@ namespace BitSoft.BinaryTools.Patch; -public sealed class PatchBlockInfo(int blockIndex, uint hash, int length) +internal class PatchBlockInfo(int blockIndex, uint hash) { public int BlockIndex { get; } = blockIndex; public uint Hash { get; } = hash; - - public int Length { get; } = length; } diff --git a/src/BitSoft.BinaryTools/Patch/PatchBlockInfoWithLength.cs b/src/BitSoft.BinaryTools/Patch/PatchBlockInfoWithLength.cs new file mode 100644 index 0000000..c08949e --- /dev/null +++ b/src/BitSoft.BinaryTools/Patch/PatchBlockInfoWithLength.cs @@ -0,0 +1,7 @@ +namespace BitSoft.BinaryTools.Patch; + +internal sealed class PatchBlockInfoWithLength(int blockIndex, uint hash, int length) + : PatchBlockInfo(blockIndex, hash) +{ + public int Length { get; } = length; +} diff --git a/src/BitSoft.BinaryTools/Patch/PatchReader.cs b/src/BitSoft.BinaryTools/Patch/PatchReader.cs index 5e22ac9..103a1bc 100644 --- a/src/BitSoft.BinaryTools/Patch/PatchReader.cs +++ b/src/BitSoft.BinaryTools/Patch/PatchReader.cs @@ -45,11 +45,19 @@ public ValueTask ReadAsync(CancellationToken cancellationToken) case ProtocolConst.SegmentTypes.EndPatchSegment: Segment = null; return ValueTask.FromResult(false); - case ProtocolConst.SegmentTypes.CopyPatchSegment: + case ProtocolConst.SegmentTypes.CopyBlock: + { + var blockIndex = _reader.ReadInt32(); + Segment = new CopyBlockSegment(blockIndex: blockIndex); + break; + } + case ProtocolConst.SegmentTypes.CopyBlockWithLength: + { var blockIndex = _reader.ReadInt32(); var blockLength = _reader.ReadInt32(); - Segment = new CopyPatchSegment(blockIndex: blockIndex, blockLength: blockLength); + Segment = new CopyBlockWithLengthSegment(blockIndex: blockIndex, blockLength: blockLength); break; + } case ProtocolConst.SegmentTypes.DataPatchSegment: var length = _reader.ReadInt32(); var span = _buffer.AsSpan(start: 0, length: length); diff --git a/src/BitSoft.BinaryTools/Patch/PatchWriter.cs b/src/BitSoft.BinaryTools/Patch/PatchWriter.cs index 723cbdb..3cdd4b6 100644 --- a/src/BitSoft.BinaryTools/Patch/PatchWriter.cs +++ b/src/BitSoft.BinaryTools/Patch/PatchWriter.cs @@ -33,9 +33,17 @@ public ValueTask WriteDataAsync(ReadOnlyMemory memory, CancellationToken c return ValueTask.CompletedTask; } - public ValueTask WriteCopyAsync(int blockIndex, int blockLength, CancellationToken cancellationToken) + public ValueTask WriteCopyBlockAsync(int blockIndex, CancellationToken cancellationToken) { - _writer.Write(ProtocolConst.SegmentTypes.CopyPatchSegment); + _writer.Write(ProtocolConst.SegmentTypes.CopyBlock); + _writer.Write(blockIndex); + + return ValueTask.CompletedTask; + } + + public ValueTask WriteCopyBlockWithLengthAsync(int blockIndex, int blockLength, CancellationToken cancellationToken) + { + _writer.Write(ProtocolConst.SegmentTypes.CopyBlockWithLength); _writer.Write(blockIndex); _writer.Write(blockLength); diff --git a/src/BitSoft.BinaryTools/Patch/ProtocolConst.cs b/src/BitSoft.BinaryTools/Patch/ProtocolConst.cs index 02bae57..144348b 100644 --- a/src/BitSoft.BinaryTools/Patch/ProtocolConst.cs +++ b/src/BitSoft.BinaryTools/Patch/ProtocolConst.cs @@ -10,9 +10,10 @@ public static class ProtocolConst public static class SegmentTypes { - public const byte CopyPatchSegment = 0x1; - public const byte DataPatchSegment = 0x2; + public const byte CopyBlock = 0x1; + public const byte CopyBlockWithLength = 0x2; + public const byte DataPatchSegment = 0x3; public const byte EndPatchSegment = byte.MaxValue; } -} \ No newline at end of file +} From de72c02b241aff21bb6b6d9186ffbbb7fe5f6860 Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Thu, 27 Nov 2025 13:19:33 +0700 Subject: [PATCH 07/22] Return of the attribute --- src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs b/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs index 38f44a4..e3b4194 100644 --- a/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs +++ b/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs @@ -63,7 +63,7 @@ await BinaryPatch.CreateAsync( Assert.That(patched, Is.EqualTo(modified)); } - // [Ignore("Performance test")] + [Ignore("Performance test")] [TestCase(10 * 1024 * 1024, 1024)] [TestCase(10 * 1024 * 1024, 4 * 1024)] public async Task Should_CreatePatch(int bufferLength, int blockSize) From d5fc08bb16be393e1a6bf2a9f447c156438fc2a3 Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Fri, 28 Nov 2025 19:59:43 +0700 Subject: [PATCH 08/22] Add stream window reader --- .../Patch/StreamWindowReader.cs | 67 ++++++++++ src/BitSoft.BinaryTools/Patch/BinaryPatch.cs | 1 + .../Patch/StreamWindowReader.cs | 114 ++++++++++++++++++ 3 files changed, 182 insertions(+) create mode 100644 src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReader.cs create mode 100644 src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs diff --git a/src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReader.cs b/src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReader.cs new file mode 100644 index 0000000..bf101d6 --- /dev/null +++ b/src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReader.cs @@ -0,0 +1,67 @@ +using System; +using System.Buffers; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using BitSoft.BinaryTools.Patch; + +namespace BitSoft.BinaryTools.Tests.Patch; + +[TestFixture] +public class StreamWindowReaderTests +{ + [Test] + public async Task Should_ReturnInitialWindow() + { + // Arrange + var source = new byte[] { 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6 }; + using var sourceStream = new MemoryStream(source); + + // Act + using var reader = new StreamWindowReader(sourceStream, ArrayPool.Shared, windowSize: 2); + + await reader.MoveAsync(CancellationToken.None); + + // Assert + Assert.That(reader.Window.Length, Is.EqualTo(2)); + Assert.That(reader.Window.ToArray(), Is.EqualTo(source.AsMemory(0, 2).ToArray()).AsCollection); + } + + [Test] + public async Task Should_ReturnMovedWindow() + { + // Arrange + var source = new byte[] { 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6 }; + using var sourceStream = new MemoryStream(source); + + // Act + using var reader = new StreamWindowReader(sourceStream, ArrayPool.Shared, windowSize: 2); + + await reader.MoveAsync(CancellationToken.None); + await reader.MoveAsync(CancellationToken.None); + + // Assert + Assert.That(reader.Window.Length, Is.EqualTo(2)); + Assert.That(reader.Window.ToArray(), Is.EqualTo(source.AsMemory(1, 2).ToArray()).AsCollection); + } + + [Test] + public async Task Should_ReturnMovedWindow_When_Overlappeds() + { + // Arrange + var source = new byte[] { 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6 }; + using var sourceStream = new MemoryStream(source); + + // Act + using var reader = new StreamWindowReader(sourceStream, ArrayPool.Shared, windowSize: 2); + + await reader.MoveAsync(CancellationToken.None); // 0 + await reader.MoveAsync(CancellationToken.None); // 1 + await reader.MoveAsync(CancellationToken.None); // 2 + await reader.MoveAsync(CancellationToken.None); // 3 + + // Assert + Assert.That(reader.Window.Length, Is.EqualTo(2)); + Assert.That(reader.Window.ToArray(), Is.EqualTo(source.AsMemory(3, 2).ToArray()).AsCollection); + } +} diff --git a/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs b/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs index b81db19..e253554 100644 --- a/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs +++ b/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs @@ -28,6 +28,7 @@ public static async ValueTask CreateAsync( var blockInfoContainer = await CalculateHashesAsync(source, blockSize, cancellationToken); + using var reader = new StreamWindowReader(modified, Pool, windowSize: blockSize); using var writer = new PatchWriter(output); await writer.WriteHeaderAsync(blockSize: blockSize, cancellationToken); diff --git a/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs b/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs new file mode 100644 index 0000000..bed0106 --- /dev/null +++ b/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs @@ -0,0 +1,114 @@ +using System; +using System.Buffers; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace BitSoft.BinaryTools.Patch; + +public class StreamWindowReader : IDisposable +{ + private readonly Stream _stream; + private readonly ArrayPool _pool; + private readonly int _windowSize; + private readonly int _bufferSize; + private readonly byte[] _buffer; + + private const int NotDefined = -1; + + private int _position = NotDefined; + private int _pinnedPosition = NotDefined; + private int _size = NotDefined; + + public ReadOnlyMemory Window + { + get + { + return _position >= 0 + ? _buffer.AsMemory(start: _position, length: _windowSize) + : throw new InvalidOperationException("The stream does not contain the window."); + } + } + + public ReadOnlyMemory PinnedWindow + { + get + { + return _pinnedPosition == NotDefined + ? throw new InvalidOperationException("Pinned position was not set.") + : _buffer.AsMemory(start: _pinnedPosition, length: _position - _pinnedPosition); + } + } + + public bool Pinned => _pinnedPosition != NotDefined; + + public StreamWindowReader(Stream stream, ArrayPool pool, int windowSize) + { + _stream = stream ?? throw new ArgumentNullException(nameof(stream)); + _pool = pool ?? throw new ArgumentNullException(nameof(pool)); + _windowSize = windowSize; + _bufferSize = windowSize * 2; + + _buffer = _pool.Rent(minimumLength: _bufferSize); + } + + public async ValueTask MoveAsync(CancellationToken cancellationToken) + { + if (_position == NotDefined) + { + var count = await _stream.ReadAsync(_buffer.AsMemory(start: 0, length: _bufferSize), cancellationToken); + if (count == 0) + return false; + _position = 0; + _size = count; + return true; + } + + _position += 1; + + if (_position == _bufferSize - _windowSize) + { + if (_pinnedPosition == NotDefined) + { + var length = _size - _position; + + Array.Copy( + sourceArray: _buffer, + sourceIndex: _position, + destinationArray: _buffer, + destinationIndex: 0, + length: length + ); + + var count = await _stream.ReadAsync( + _buffer.AsMemory(start: length, length: length), + cancellationToken + ); + + _position = 0; + _size = length + count; + } + else + { + throw new InvalidOperationException("Pinned position was not reset."); + } + } + + return true; + } + + public void PinPosition() + { + _pinnedPosition = _position; + } + + public void ResetPinnedPosition() + { + _pinnedPosition = NotDefined; + } + + public void Dispose() + { + _pool.Return(_buffer); + } +} From dddc19f9f8d6f4048061c01be0869ae3e20b4a7e Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Fri, 28 Nov 2025 20:06:28 +0700 Subject: [PATCH 09/22] Add pinned window test --- .../Patch/StreamWindowReader.cs | 33 +++++++++++++++---- .../Patch/StreamWindowReader.cs | 8 +++-- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReader.cs b/src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReader.cs index bf101d6..85d386a 100644 --- a/src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReader.cs +++ b/src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReader.cs @@ -24,7 +24,7 @@ public async Task Should_ReturnInitialWindow() // Assert Assert.That(reader.Window.Length, Is.EqualTo(2)); - Assert.That(reader.Window.ToArray(), Is.EqualTo(source.AsMemory(0, 2).ToArray()).AsCollection); + Assert.That(reader.Window.ToArray(), Is.EqualTo(source.AsMemory(start: 0, length: 2).ToArray()).AsCollection); } [Test] @@ -37,16 +37,16 @@ public async Task Should_ReturnMovedWindow() // Act using var reader = new StreamWindowReader(sourceStream, ArrayPool.Shared, windowSize: 2); - await reader.MoveAsync(CancellationToken.None); - await reader.MoveAsync(CancellationToken.None); + await reader.MoveAsync(CancellationToken.None); // 0 + await reader.MoveAsync(CancellationToken.None); // 1 // Assert Assert.That(reader.Window.Length, Is.EqualTo(2)); - Assert.That(reader.Window.ToArray(), Is.EqualTo(source.AsMemory(1, 2).ToArray()).AsCollection); + Assert.That(reader.Window.ToArray(), Is.EqualTo(source.AsMemory(start: 1, length: 2).ToArray()).AsCollection); } [Test] - public async Task Should_ReturnMovedWindow_When_Overlappeds() + public async Task Should_ReturnMovedWindow_When_Overlapped() { // Arrange var source = new byte[] { 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6 }; @@ -62,6 +62,27 @@ public async Task Should_ReturnMovedWindow_When_Overlappeds() // Assert Assert.That(reader.Window.Length, Is.EqualTo(2)); - Assert.That(reader.Window.ToArray(), Is.EqualTo(source.AsMemory(3, 2).ToArray()).AsCollection); + Assert.That(reader.Window.ToArray(), Is.EqualTo(source.AsMemory(start: 3, length: 2).ToArray()).AsCollection); + } + + [Test] + public async Task Should_ReturnMovedWindow_When_Pinned() + { + // Arrange + var source = new byte[] { 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6 }; + using var sourceStream = new MemoryStream(source); + + // Act + using var reader = new StreamWindowReader(sourceStream, ArrayPool.Shared, windowSize: 2); + + await reader.MoveAsync(CancellationToken.None); // 0 + reader.PinPosition(); + await reader.MoveAsync(CancellationToken.None); // 1 + await reader.MoveAsync(CancellationToken.None); // 2 + + // Assert + Assert.That(reader.Window.Length, Is.EqualTo(2)); + Assert.That(reader.PinnedWindow.ToArray(), Is.EqualTo(source.AsMemory(start: 0, length: 2).ToArray()).AsCollection); + Assert.That(reader.Window.ToArray(), Is.EqualTo(source.AsMemory(start: 2, length: 2).ToArray()).AsCollection); } } diff --git a/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs b/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs index bed0106..550ae87 100644 --- a/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs +++ b/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs @@ -64,8 +64,6 @@ public async ValueTask MoveAsync(CancellationToken cancellationToken) return true; } - _position += 1; - if (_position == _bufferSize - _windowSize) { if (_pinnedPosition == NotDefined) @@ -85,7 +83,7 @@ public async ValueTask MoveAsync(CancellationToken cancellationToken) cancellationToken ); - _position = 0; + _position = 1; _size = length + count; } else @@ -93,6 +91,10 @@ public async ValueTask MoveAsync(CancellationToken cancellationToken) throw new InvalidOperationException("Pinned position was not reset."); } } + else + { + _position += 1; + } return true; } From 56d7f305e8b03e9204d1fd3694e299aae06c8c91 Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Fri, 28 Nov 2025 20:09:39 +0700 Subject: [PATCH 10/22] Support short final block size --- .../Patch/StreamWindowReader.cs | 21 +++++++++++++++++++ .../Patch/StreamWindowReader.cs | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReader.cs b/src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReader.cs index 85d386a..673f5ac 100644 --- a/src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReader.cs +++ b/src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReader.cs @@ -85,4 +85,25 @@ public async Task Should_ReturnMovedWindow_When_Pinned() Assert.That(reader.PinnedWindow.ToArray(), Is.EqualTo(source.AsMemory(start: 0, length: 2).ToArray()).AsCollection); Assert.That(reader.Window.ToArray(), Is.EqualTo(source.AsMemory(start: 2, length: 2).ToArray()).AsCollection); } + + [Test] + public async Task Should_ReturnMovedWindow_When_ReachEnd() + { + // Arrange + var source = new byte[] { 0x0, 0x1, 0x2, 0x3, 0x4 }; + using var sourceStream = new MemoryStream(source); + + // Act + using var reader = new StreamWindowReader(sourceStream, ArrayPool.Shared, windowSize: 2); + + await reader.MoveAsync(CancellationToken.None); // 0 + await reader.MoveAsync(CancellationToken.None); // 1 + await reader.MoveAsync(CancellationToken.None); // 2 + await reader.MoveAsync(CancellationToken.None); // 3 + await reader.MoveAsync(CancellationToken.None); // 4 + + // Assert + Assert.That(reader.Window.Length, Is.EqualTo(1)); + Assert.That(reader.Window.ToArray(), Is.EqualTo(source.AsMemory(start: 4, length: 1).ToArray()).AsCollection); + } } diff --git a/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs b/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs index 550ae87..d1bcf66 100644 --- a/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs +++ b/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs @@ -25,7 +25,7 @@ public ReadOnlyMemory Window get { return _position >= 0 - ? _buffer.AsMemory(start: _position, length: _windowSize) + ? _buffer.AsMemory(start: _position, length: Math.Min(_windowSize, _size - _position)) : throw new InvalidOperationException("The stream does not contain the window."); } } From efc43ebfa0752c7f6c1967f4ddc81a550547e351 Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Fri, 28 Nov 2025 20:14:49 +0700 Subject: [PATCH 11/22] Support stop on end rich --- .../Patch/StreamWindowReader.cs | 24 +++++++++++++++++-- .../Patch/StreamWindowReader.cs | 3 +++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReader.cs b/src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReader.cs index 673f5ac..f260110 100644 --- a/src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReader.cs +++ b/src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReader.cs @@ -82,12 +82,13 @@ public async Task Should_ReturnMovedWindow_When_Pinned() // Assert Assert.That(reader.Window.Length, Is.EqualTo(2)); - Assert.That(reader.PinnedWindow.ToArray(), Is.EqualTo(source.AsMemory(start: 0, length: 2).ToArray()).AsCollection); + Assert.That(reader.PinnedWindow.ToArray(), + Is.EqualTo(source.AsMemory(start: 0, length: 2).ToArray()).AsCollection); Assert.That(reader.Window.ToArray(), Is.EqualTo(source.AsMemory(start: 2, length: 2).ToArray()).AsCollection); } [Test] - public async Task Should_ReturnMovedWindow_When_ReachEnd() + public async Task Should_ReturnMovedWindow_When_FinalBlockIsShort() { // Arrange var source = new byte[] { 0x0, 0x1, 0x2, 0x3, 0x4 }; @@ -106,4 +107,23 @@ public async Task Should_ReturnMovedWindow_When_ReachEnd() Assert.That(reader.Window.Length, Is.EqualTo(1)); Assert.That(reader.Window.ToArray(), Is.EqualTo(source.AsMemory(start: 4, length: 1).ToArray()).AsCollection); } + + [Test] + public async Task Should_ReturnMovedWindow_When_ReachEnd() + { + // Arrange + var source = new byte[] { 0x0, 0x1, 0x2 }; + using var sourceStream = new MemoryStream(source); + + // Act + using var reader = new StreamWindowReader(sourceStream, ArrayPool.Shared, windowSize: 2); + + await reader.MoveAsync(CancellationToken.None); // 0 + await reader.MoveAsync(CancellationToken.None); // 1 + await reader.MoveAsync(CancellationToken.None); // 2 + var result = await reader.MoveAsync(CancellationToken.None); + + // Assert + Assert.That(result, Is.False); + } } diff --git a/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs b/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs index d1bcf66..4e72571 100644 --- a/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs +++ b/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs @@ -96,6 +96,9 @@ public async ValueTask MoveAsync(CancellationToken cancellationToken) _position += 1; } + if (_position == _size) + return false; + return true; } From c01b53002dd61e2762f2c95e62191565fc3ca86a Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Sat, 29 Nov 2025 15:33:10 +0700 Subject: [PATCH 12/22] Use windowed reader --- src/BitSoft.BinaryTools/Patch/BinaryPatch.cs | 182 ++++++------------ .../Patch/StreamWindowReader.cs | 33 +++- 2 files changed, 91 insertions(+), 124 deletions(-) diff --git a/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs b/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs index e253554..3088592 100644 --- a/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs +++ b/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs @@ -30,144 +30,80 @@ public static async ValueTask CreateAsync( using var reader = new StreamWindowReader(modified, Pool, windowSize: blockSize); using var writer = new PatchWriter(output); - await writer.WriteHeaderAsync(blockSize: blockSize, cancellationToken); - var bufferLength = blockSize * 2; - var buffer = Pool.Rent(minimumLength: bufferLength); - try + if (!await reader.MoveAsync(cancellationToken)) { - var length = await modified.ReadAsync(buffer.AsMemory(start: 0, length: bufferLength), cancellationToken); - if (length == 0) - return; + await writer.CompleteAsync(cancellationToken); + return; + } - const int NotDefined = -1; + RollingHash rollingHash = default; + var resetHash = true; - var segmentStart = NotDefined; - var position = 0; + while (true) + { + if (resetHash) + { + rollingHash = RollingHash.Create(reader.Window.Span); + resetHash = false; + } - RollingHash rollingHash = default; - var resetHash = true; + var block = blockInfoContainer.Match(rollingHash); - while (true) + if (block is null) { - while (position < length) + if (!reader.Pinned) + { + reader.PinPosition(); + } + else if (reader.Finished || reader.PinnedWindow.Length == blockSize) + { + await writer.WriteDataAsync(reader.PinnedWindow, cancellationToken); + reader.ResetPinnedPosition(); + } + + if (reader.Finished) { - if (resetHash) - { - var spanLength = Math.Min(blockSize, length); - var bufferSpan = buffer.AsSpan(start: 0, length: spanLength); - rollingHash = RollingHash.Create(bufferSpan); - resetHash = false; - } - - var block = blockInfoContainer.Match(rollingHash); - - if (block is null) - { - if (length <= blockSize) - { - var memory = buffer.AsMemory(start: position, length: length); - await writer.WriteDataAsync(memory, cancellationToken); - position = 0; - break; - } - - if (segmentStart == NotDefined) - { - segmentStart = position; - } - else if (position - segmentStart + 1 == blockSize) - { - var memory = buffer.AsMemory(start: segmentStart, length: position - segmentStart + 1); - await writer.WriteDataAsync(memory, cancellationToken); - - buffer - .AsSpan(start: position + 1, length: bufferLength - position - 2) - .CopyTo(buffer.AsSpan(start: 0)); - - segmentStart = NotDefined; - resetHash = true; - - break; - } - - position += 1; - - if (position == length) - { - var memory = buffer.AsMemory(start: segmentStart, length: position - segmentStart); - await writer.WriteDataAsync(memory, cancellationToken); - position = 0; - break; - } - - if (position + blockSize < length) - { - var removedByte = buffer[position - 1]; - var addedByte = buffer[position + blockSize - 1]; - rollingHash.Update(removed: removedByte, added: addedByte); - } - else - { - resetHash = true; - } - } - else - { - if (segmentStart != NotDefined) - { - var memory = buffer.AsMemory(start: segmentStart, length: position - segmentStart); - await writer.WriteDataAsync(memory, cancellationToken); - segmentStart = NotDefined; - } - - var blockLength = blockSize; - - if (block is PatchBlockInfoWithLength blockInfoWithLength) - { - blockLength = blockInfoWithLength.Length; - - await writer.WriteCopyBlockWithLengthAsync( - blockIndex: block.BlockIndex, - blockLength: blockLength, - cancellationToken: cancellationToken - ); - } - else - { - await writer.WriteCopyBlockAsync( - blockIndex: block.BlockIndex, - cancellationToken: cancellationToken - ); - } - - buffer - .AsSpan(start: position + blockLength, length: bufferLength - position - blockLength - 1) - .CopyTo(buffer.AsSpan(start: 0)); - - resetHash = true; - - break; - } + await writer.WriteDataAsync(reader.Window, cancellationToken); + break; } - length = await modified.ReadAsync( - buffer.AsMemory(start: position, length: bufferLength - position - 1), - cancellationToken: cancellationToken - ); + var firstByte = reader.Window.Span[0]; + if (await reader.MoveAsync(cancellationToken)) + { + var newByte = reader.Window.Span[reader.Window.Length - 1]; + rollingHash.Update(removed: firstByte, added: newByte); + } + } + else + { + if (reader.Pinned) + { + await writer.WriteDataAsync(reader.PinnedWindow, cancellationToken); + reader.ResetPinnedPosition(); + } - length += position; - position = 0; + if (block is PatchBlockInfoWithLength blockInfoWithLength) + { + await writer.WriteCopyBlockWithLengthAsync( + blockIndex: block.BlockIndex, + blockLength: blockInfoWithLength.Length, + cancellationToken: cancellationToken + ); + } + else + { + await writer.WriteCopyBlockAsync( + blockIndex: block.BlockIndex, + cancellationToken: cancellationToken + ); + } - if (length == 0) - break; + await reader.MoveAsync(count: blockSize, cancellationToken); + resetHash = true; } } - finally - { - Pool.Return(buffer); - } await writer.CompleteAsync(cancellationToken); } diff --git a/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs b/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs index 4e72571..7bd34fd 100644 --- a/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs +++ b/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs @@ -19,6 +19,7 @@ public class StreamWindowReader : IDisposable private int _position = NotDefined; private int _pinnedPosition = NotDefined; private int _size = NotDefined; + private bool _continureRead = true; public ReadOnlyMemory Window { @@ -42,6 +43,8 @@ public ReadOnlyMemory PinnedWindow public bool Pinned => _pinnedPosition != NotDefined; + public bool Finished => _position == _size - 1; + public StreamWindowReader(Stream stream, ArrayPool pool, int windowSize) { _stream = stream ?? throw new ArgumentNullException(nameof(stream)); @@ -52,8 +55,31 @@ public StreamWindowReader(Stream stream, ArrayPool pool, int windowSize) _buffer = _pool.Rent(minimumLength: _bufferSize); } + public async ValueTask MoveAsync(int count, CancellationToken cancellationToken) + { + if (count <= 0) + throw new ArgumentOutOfRangeException(nameof(count)); + if (count > _windowSize) + throw new ArgumentOutOfRangeException(nameof(count)); + + for (var i = 0; i < count; i++) + { + if (await MoveAsync(cancellationToken)) + { + continue; + } + + return i; + } + + return count; + } + public async ValueTask MoveAsync(CancellationToken cancellationToken) { + if (_position != NotDefined && _position == _size) + return false; + if (_position == NotDefined) { var count = await _stream.ReadAsync(_buffer.AsMemory(start: 0, length: _bufferSize), cancellationToken); @@ -64,7 +90,7 @@ public async ValueTask MoveAsync(CancellationToken cancellationToken) return true; } - if (_position == _bufferSize - _windowSize) + if (_continureRead && _position == _bufferSize - _windowSize) { if (_pinnedPosition == NotDefined) { @@ -85,6 +111,11 @@ public async ValueTask MoveAsync(CancellationToken cancellationToken) _position = 1; _size = length + count; + + if (_size < _bufferSize) + { + _continureRead = false; + } } else { From 6e488e965e600fd7fcc25b6a27ce4849db36cde4 Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Sat, 29 Nov 2025 16:55:57 +0700 Subject: [PATCH 13/22] Fix positioning for data blocks --- .../Patch/BinaryPatchTests.cs | 17 ++++++++-- ...owReader.cs => StreamWindowReaderTests.cs} | 4 +-- src/BitSoft.BinaryTools/Patch/BinaryPatch.cs | 30 ++++++++++++----- .../Patch/StreamWindowReader.cs | 33 ++++++++++--------- 4 files changed, 55 insertions(+), 29 deletions(-) rename src/BitSoft.BinaryTools.Tests/Patch/{StreamWindowReader.cs => StreamWindowReaderTests.cs} (94%) diff --git a/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs b/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs index e3b4194..272c651 100644 --- a/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs +++ b/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs @@ -13,8 +13,8 @@ public class BinaryPatchTests private static IEnumerable TestCases() { yield return new TestCaseData( - new byte[] { 0x0, 0x1, 0x0, 0x1, 0x0 }, - new byte[] { 0x0, 0x0, 0x1, 0x0, 0x0 }, + new byte[] { 0x1, 0x2, 0x3, 0x4, 0x5 }, + new byte[] { 0x1, 0x1, 0x2, 0x3, 0x4 }, 2 ); yield return new TestCaseData( @@ -63,7 +63,8 @@ await BinaryPatch.CreateAsync( Assert.That(patched, Is.EqualTo(modified)); } - [Ignore("Performance test")] + // [Ignore("Performance test")] + [TestCase(3 * 4, 4)] [TestCase(10 * 1024 * 1024, 1024)] [TestCase(10 * 1024 * 1024, 4 * 1024)] public async Task Should_CreatePatch(int bufferLength, int blockSize) @@ -98,4 +99,14 @@ await BinaryPatch.CreateAsync( Console.WriteLine("Patch length: {0}", patchStream.Position); Console.WriteLine("Time: {0:g}", stopwatch.Elapsed); } + + [Test] + public void ArrayTest() + { + // Arrange + var sourceArray = new[] { 1, 2, 3, 4 }; + + // Act + Array.Copy(sourceArray: sourceArray, sourceIndex: 2, destinationArray: sourceArray, destinationIndex: 0, length: 2); + } } diff --git a/src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReader.cs b/src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReaderTests.cs similarity index 94% rename from src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReader.cs rename to src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReaderTests.cs index f260110..72da6c9 100644 --- a/src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReader.cs +++ b/src/BitSoft.BinaryTools.Tests/Patch/StreamWindowReaderTests.cs @@ -78,13 +78,11 @@ public async Task Should_ReturnMovedWindow_When_Pinned() await reader.MoveAsync(CancellationToken.None); // 0 reader.PinPosition(); await reader.MoveAsync(CancellationToken.None); // 1 - await reader.MoveAsync(CancellationToken.None); // 2 // Assert Assert.That(reader.Window.Length, Is.EqualTo(2)); Assert.That(reader.PinnedWindow.ToArray(), - Is.EqualTo(source.AsMemory(start: 0, length: 2).ToArray()).AsCollection); - Assert.That(reader.Window.ToArray(), Is.EqualTo(source.AsMemory(start: 2, length: 2).ToArray()).AsCollection); + Is.EqualTo(source.AsMemory(start: 0, length: 1).ToArray()).AsCollection); } [Test] diff --git a/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs b/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs index 3088592..847739f 100644 --- a/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs +++ b/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs @@ -57,24 +57,32 @@ public static async ValueTask CreateAsync( { reader.PinPosition(); } - else if (reader.Finished || reader.PinnedWindow.Length == blockSize) - { - await writer.WriteDataAsync(reader.PinnedWindow, cancellationToken); - reader.ResetPinnedPosition(); - } if (reader.Finished) { - await writer.WriteDataAsync(reader.Window, cancellationToken); + if (reader.Pinned) + await writer.WriteDataAsync(reader.PinnedWindowWithCurrent, cancellationToken); + else + await writer.WriteDataAsync(reader.Window, cancellationToken); break; } + if (reader.PinnedWindowWithCurrent.Length == blockSize) + { + await writer.WriteDataAsync(reader.PinnedWindowWithCurrent, cancellationToken); + reader.ResetPinnedPosition(); + } + var firstByte = reader.Window.Span[0]; if (await reader.MoveAsync(cancellationToken)) { var newByte = reader.Window.Span[reader.Window.Length - 1]; rollingHash.Update(removed: firstByte, added: newByte); } + else + { + break; + } } else { @@ -100,8 +108,14 @@ await writer.WriteCopyBlockAsync( ); } - await reader.MoveAsync(count: blockSize, cancellationToken); - resetHash = true; + if (await reader.SlideWindowAsync(cancellationToken)) + { + resetHash = true; + } + else + { + break; + } } } diff --git a/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs b/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs index 7bd34fd..86af73e 100644 --- a/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs +++ b/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs @@ -41,6 +41,16 @@ public ReadOnlyMemory PinnedWindow } } + public ReadOnlyMemory PinnedWindowWithCurrent + { + get + { + return _pinnedPosition == NotDefined + ? throw new InvalidOperationException("Pinned position was not set.") + : _buffer.AsMemory(start: _pinnedPosition, length: _position - _pinnedPosition + 1); + } + } + public bool Pinned => _pinnedPosition != NotDefined; public bool Finished => _position == _size - 1; @@ -55,24 +65,19 @@ public StreamWindowReader(Stream stream, ArrayPool pool, int windowSize) _buffer = _pool.Rent(minimumLength: _bufferSize); } - public async ValueTask MoveAsync(int count, CancellationToken cancellationToken) + public async ValueTask SlideWindowAsync(CancellationToken cancellationToken) { - if (count <= 0) - throw new ArgumentOutOfRangeException(nameof(count)); - if (count > _windowSize) - throw new ArgumentOutOfRangeException(nameof(count)); - - for (var i = 0; i < count; i++) + for (var i = 0; i < _windowSize; i++) { if (await MoveAsync(cancellationToken)) { continue; } - return i; + return false; } - return count; + return true; } public async ValueTask MoveAsync(CancellationToken cancellationToken) @@ -90,6 +95,8 @@ public async ValueTask MoveAsync(CancellationToken cancellationToken) return true; } + _position += 1; + if (_continureRead && _position == _bufferSize - _windowSize) { if (_pinnedPosition == NotDefined) @@ -109,10 +116,10 @@ public async ValueTask MoveAsync(CancellationToken cancellationToken) cancellationToken ); - _position = 1; + _position = 0; _size = length + count; - if (_size < _bufferSize) + if (count < _size) { _continureRead = false; } @@ -122,10 +129,6 @@ public async ValueTask MoveAsync(CancellationToken cancellationToken) throw new InvalidOperationException("Pinned position was not reset."); } } - else - { - _position += 1; - } if (_position == _size) return false; From bb3eaf47468adb5fb537d639b63be9de380e9c80 Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Sat, 29 Nov 2025 16:59:43 +0700 Subject: [PATCH 14/22] Fix next data buffer segment definition --- src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs | 2 +- src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs b/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs index 272c651..46df5a3 100644 --- a/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs +++ b/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs @@ -64,7 +64,7 @@ await BinaryPatch.CreateAsync( } // [Ignore("Performance test")] - [TestCase(3 * 4, 4)] + [TestCase(4 * 4, 4)] [TestCase(10 * 1024 * 1024, 1024)] [TestCase(10 * 1024 * 1024, 4 * 1024)] public async Task Should_CreatePatch(int bufferLength, int blockSize) diff --git a/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs b/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs index 86af73e..c975774 100644 --- a/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs +++ b/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs @@ -119,7 +119,7 @@ public async ValueTask MoveAsync(CancellationToken cancellationToken) _position = 0; _size = length + count; - if (count < _size) + if (count < _windowSize) { _continureRead = false; } From 73438b0c7e57caebc18d5f1ff46a6877caaf5144 Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Sat, 29 Nov 2025 17:00:08 +0700 Subject: [PATCH 15/22] Ignore test --- src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs b/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs index 46df5a3..23eb054 100644 --- a/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs +++ b/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs @@ -63,7 +63,7 @@ await BinaryPatch.CreateAsync( Assert.That(patched, Is.EqualTo(modified)); } - // [Ignore("Performance test")] + [Ignore("Performance test")] [TestCase(4 * 4, 4)] [TestCase(10 * 1024 * 1024, 1024)] [TestCase(10 * 1024 * 1024, 4 * 1024)] From d266770f837e97ea6fc8a6fa24896ccf468b45c5 Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Sat, 29 Nov 2025 17:30:27 +0700 Subject: [PATCH 16/22] Fix hashing fuction --- .../Patch/BinaryPatchTests.cs | 60 +++++++++++++++---- src/BitSoft.BinaryTools/Patch/BinaryPatch.cs | 4 +- src/BitSoft.BinaryTools/Patch/RollingHash.cs | 13 ++-- 3 files changed, 57 insertions(+), 20 deletions(-) diff --git a/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs b/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs index 23eb054..33ea617 100644 --- a/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs +++ b/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs @@ -17,6 +17,56 @@ private static IEnumerable TestCases() new byte[] { 0x1, 0x1, 0x2, 0x3, 0x4 }, 2 ); + yield return new TestCaseData( + new byte[] { 0x1, 0x2, 0x3, 0x4, 0x5 }, + new byte[] { 0x1, 0x1, 0x2, 0x3, 0x4 }, + 3 + ); + yield return new TestCaseData( + new byte[] { 0x1, 0x2, 0x3, 0x4, 0x5 }, + new byte[] { 0x1, 0x2, 0x3, 0x4 }, + 3 + ); + yield return new TestCaseData( + new byte[] { 0x1, 0x2, 0x3, 0x4, 0x5 }, + new byte[] { 0x1, 0x2, 0x3, 0x4, 0x5, 0x6 }, + 3 + ); + yield return new TestCaseData( + new byte[] { 0x1, 0x2, 0x3, 0x4, 0x5 }, + new byte[] { 0x1, 0x7, 0x3, 0x4, 0x5, 0x6 }, + 3 + ); + yield return new TestCaseData( + new byte[] { 0x1, 0x2, 0x3, 0x4, 0x5 }, + new byte[] { 0x1, 0x7, 0x4, 0x5, 0x6 }, + 3 + ); + yield return new TestCaseData( + new byte[] { 0x1, 0x2, 0x3, 0x4, 0x5 }, + new byte[] { 0x1, 0x7, 0x8, 0x9, 0x2 }, + 3 + ); + yield return new TestCaseData( + new byte[] { 0x1, 0x2, 0x3, 0x4, 0x5 }, + new byte[] { 0x1, 0x2, 0x3, 0x9, 0x9 }, + 3 + ); + yield return new TestCaseData( + new byte[] { 0x1, 0x2, 0x3, 0x4, 0x5 }, + new byte[] { 0x1, 0x2, 0x9, 0x9, 0x9 }, + 3 + ); + yield return new TestCaseData( + new byte[] { 0x1, 0x2, 0x3, 0x4, 0x5 }, + new byte[] { 0x9, 0x9, 0x1, 0x2, 0x3 }, + 3 + ); + yield return new TestCaseData( + new byte[] { 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8 }, + new byte[] { 0x1, 0x2, 0x9, 0x4, 0x5, 0x6, 0x7, 0x8 }, + 3 + ); yield return new TestCaseData( new byte[] { 0x0, 0x1 }, new byte[] { 0x0 }, @@ -99,14 +149,4 @@ await BinaryPatch.CreateAsync( Console.WriteLine("Patch length: {0}", patchStream.Position); Console.WriteLine("Time: {0:g}", stopwatch.Elapsed); } - - [Test] - public void ArrayTest() - { - // Arrange - var sourceArray = new[] { 1, 2, 3, 4 }; - - // Act - Array.Copy(sourceArray: sourceArray, sourceIndex: 2, destinationArray: sourceArray, destinationIndex: 0, length: 2); - } } diff --git a/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs b/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs index 847739f..267ce7f 100644 --- a/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs +++ b/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs @@ -212,7 +212,9 @@ private static async ValueTask CalculateHashesAsync( if (length == 0) break; - var hash = RollingHash.Create(buffer.AsSpan(start: 0, length: length)); + var span = buffer.AsSpan(start: 0, length: length); + + var hash = RollingHash.Create(span); if (length == blockSize) { diff --git a/src/BitSoft.BinaryTools/Patch/RollingHash.cs b/src/BitSoft.BinaryTools/Patch/RollingHash.cs index 12073a3..7f86e28 100644 --- a/src/BitSoft.BinaryTools/Patch/RollingHash.cs +++ b/src/BitSoft.BinaryTools/Patch/RollingHash.cs @@ -8,14 +8,12 @@ public struct RollingHash private uint _a; private uint _b; - private uint _sumOfWindow; private readonly uint _length; - private RollingHash(uint a, uint b, uint sumOfWindow, uint length) + private RollingHash(uint a, uint b, uint length) { _a = a; _b = b; - _sumOfWindow = sumOfWindow; _length = length; } @@ -23,26 +21,23 @@ public static RollingHash Create(ReadOnlySpan data) { uint a = 1; uint b = 0; - uint sumOfWindow = 0; for (var i = 0; i < data.Length; i++) { var value = data[i]; a = (a + value) % Base; - sumOfWindow = (sumOfWindow + value) % Base; b = (b + a) % Base; } - return new RollingHash(a: a, b: b, sumOfWindow: sumOfWindow, length: (uint)data.Length); + return new RollingHash(a: a, b: b, length: (uint)data.Length); } public void Update(byte removed, byte added) { _a = (_a - removed + added) % Base; - _sumOfWindow = (_sumOfWindow - removed + added) % Base; - _b = (_b - _length * removed + _sumOfWindow) % Base; + _b = (_b - _length * removed + _a) % Base; } public uint GetChecksum() => (_b << 16) | _a; -} \ No newline at end of file +} From 8b534b55ae38fa6805f8e8f48c75d451d6c5c349 Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Sat, 29 Nov 2025 18:04:14 +0700 Subject: [PATCH 17/22] Simplify benchmark scenarios --- .../BinaryPatchBenchmark.cs | 28 ++++------- src/BitSoft.BinaryTools.Benchmarks/Readme.md | 50 +++---------------- 2 files changed, 17 insertions(+), 61 deletions(-) diff --git a/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs b/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs index 5a018b8..ca4819d 100644 --- a/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs +++ b/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs @@ -2,7 +2,6 @@ using System.IO; using System.Threading.Tasks; using BenchmarkDotNet.Attributes; -using BitSoft.BinaryTools.Benchmarks.Utils; using BitSoft.BinaryTools.Patch; namespace BitSoft.BinaryTools.Benchmarks; @@ -18,22 +17,23 @@ public class BinaryPatchBenchmark private Stream? _modifiedStream; private Stream? _patchStream; - [Params(1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024)] + [Params(1024 * 1024)] public int BufferLength { get; set; } - [Params(3, 5)] public int ChangedBlocks { get; set; } + [Params(5)] + public int ChangedBlocks { get; set; } - [Params(128, 512)] public int ChangeSize { get; set; } + [Params(512)] public int ChangeSize { get; set; } - [Params(512, 1024, 4096)] public int BlockSize { get; set; } + [Params(1024, 4096)] public int BlockSize { get; set; } - [GlobalSetup] + [IterationSetup] public void GlobalSetUp() { _source = new byte[BufferLength]; _modified = new byte[BufferLength]; - Create.RandomData(_source); + Random.Shared.NextBytes(_source); Array.Copy(sourceArray: _source, destinationArray: _modified, length: _source.Length); @@ -45,30 +45,20 @@ public void GlobalSetUp() var span = _modified.AsSpan(start: position, length: ChangeSize); - Create.RandomData(span); + Random.Shared.NextBytes(span); } _sourceStream = new MemoryStream(_source); _modifiedStream = new MemoryStream(_modified); - } - - [IterationSetup] - public void SetUp() - { _patchStream = new MemoryStream(); } [IterationCleanup] public void Cleanup() - { - _patchStream?.Dispose(); - } - - [GlobalCleanup] - public void GlobalCleanUp() { _sourceStream?.Dispose(); _modifiedStream?.Dispose(); + _patchStream?.Dispose(); } [Benchmark] diff --git a/src/BitSoft.BinaryTools.Benchmarks/Readme.md b/src/BitSoft.BinaryTools.Benchmarks/Readme.md index 8ca7906..5676366 100644 --- a/src/BitSoft.BinaryTools.Benchmarks/Readme.md +++ b/src/BitSoft.BinaryTools.Benchmarks/Readme.md @@ -1,44 +1,10 @@ # Benchmarks ``` -| Method | BufferLength | ChangedBlocks | ChangeSize | BlockSize | Mean | Error | StdDev | Allocated | -|------------------ |------------- |-------------- |----------- |---------- |---------:|-----------:|---------:|----------:| -| CreateBinaryPatch | 1048576 | 3 | 128 | 512 | 16.80 us | 7.952 us | 0.436 us | 448 B | -| CreateBinaryPatch | 1048576 | 3 | 128 | 1024 | 20.50 us | 55.666 us | 3.051 us | 448 B | -| CreateBinaryPatch | 1048576 | 3 | 128 | 4096 | 15.00 us | 47.853 us | 2.623 us | 448 B | -| CreateBinaryPatch | 1048576 | 3 | 512 | 512 | 16.68 us | 61.173 us | 3.353 us | 448 B | -| CreateBinaryPatch | 1048576 | 3 | 512 | 1024 | 18.53 us | 65.508 us | 3.591 us | 448 B | -| CreateBinaryPatch | 1048576 | 3 | 512 | 4096 | 20.43 us | 37.933 us | 2.079 us | 448 B | -| CreateBinaryPatch | 1048576 | 5 | 128 | 512 | 20.77 us | 59.992 us | 3.288 us | 448 B | -| CreateBinaryPatch | 1048576 | 5 | 128 | 1024 | 14.37 us | 35.673 us | 1.955 us | 448 B | -| CreateBinaryPatch | 1048576 | 5 | 128 | 4096 | 15.17 us | 38.672 us | 2.120 us | 448 B | -| CreateBinaryPatch | 1048576 | 5 | 512 | 512 | 15.95 us | 61.300 us | 3.360 us | 448 B | -| CreateBinaryPatch | 1048576 | 5 | 512 | 1024 | 16.00 us | 46.548 us | 2.551 us | 448 B | -| CreateBinaryPatch | 1048576 | 5 | 512 | 4096 | 17.50 us | 4.827 us | 0.265 us | 448 B | -| CreateBinaryPatch | 10485760 | 3 | 128 | 512 | 15.08 us | 45.621 us | 2.501 us | 448 B | -| CreateBinaryPatch | 10485760 | 3 | 128 | 1024 | 17.53 us | 47.934 us | 2.627 us | 448 B | -| CreateBinaryPatch | 10485760 | 3 | 128 | 4096 | 15.60 us | 17.968 us | 0.985 us | 448 B | -| CreateBinaryPatch | 10485760 | 3 | 512 | 512 | 17.13 us | 68.075 us | 3.731 us | 448 B | -| CreateBinaryPatch | 10485760 | 3 | 512 | 1024 | 15.67 us | 42.798 us | 2.346 us | 448 B | -| CreateBinaryPatch | 10485760 | 3 | 512 | 4096 | 18.53 us | 46.381 us | 2.542 us | 448 B | -| CreateBinaryPatch | 10485760 | 5 | 128 | 512 | 17.00 us | 26.311 us | 1.442 us | 448 B | -| CreateBinaryPatch | 10485760 | 5 | 128 | 1024 | 16.05 us | 4.827 us | 0.265 us | 448 B | -| CreateBinaryPatch | 10485760 | 5 | 128 | 4096 | 15.82 us | 53.200 us | 2.916 us | 448 B | -| CreateBinaryPatch | 10485760 | 5 | 512 | 512 | 15.73 us | 36.866 us | 2.021 us | 448 B | -| CreateBinaryPatch | 10485760 | 5 | 512 | 1024 | 17.63 us | 26.895 us | 1.474 us | 448 B | -| CreateBinaryPatch | 10485760 | 5 | 512 | 4096 | 16.13 us | 70.288 us | 3.853 us | 448 B | -| CreateBinaryPatch | 104857600 | 3 | 128 | 512 | 19.27 us | 12.147 us | 0.666 us | 448 B | -| CreateBinaryPatch | 104857600 | 3 | 128 | 1024 | 20.00 us | 94.375 us | 5.173 us | 448 B | -| CreateBinaryPatch | 104857600 | 3 | 128 | 4096 | 22.37 us | 13.693 us | 0.751 us | 448 B | -| CreateBinaryPatch | 104857600 | 3 | 512 | 512 | 20.40 us | 1.824 us | 0.100 us | 448 B | -| CreateBinaryPatch | 104857600 | 3 | 512 | 1024 | 19.00 us | 27.608 us | 1.513 us | 448 B | -| CreateBinaryPatch | 104857600 | 3 | 512 | 4096 | 21.50 us | 47.818 us | 2.621 us | 448 B | -| CreateBinaryPatch | 104857600 | 5 | 128 | 512 | 19.70 us | 41.722 us | 2.287 us | 448 B | -| CreateBinaryPatch | 104857600 | 5 | 128 | 1024 | 19.20 us | 37.699 us | 2.066 us | 448 B | -| CreateBinaryPatch | 104857600 | 5 | 128 | 4096 | 16.63 us | 36.956 us | 2.026 us | 448 B | -| CreateBinaryPatch | 104857600 | 5 | 512 | 512 | 18.03 us | 18.274 us | 1.002 us | 448 B | -| CreateBinaryPatch | 104857600 | 5 | 512 | 1024 | 21.80 us | 145.642 us | 7.983 us | 448 B | -| CreateBinaryPatch | 104857600 | 5 | 512 | 4096 | 16.13 us | 31.086 us | 1.704 us | 448 B | +| Method | BufferLength | ChangedBlocks | ChangeSize | BlockSize | Mean | Error | StdDev | Allocated | +|------------------ |------------- |-------------- |----------- |---------- |---------:|---------:|---------:|----------:| +| CreateBinaryPatch | 1048576 | 5 | 512 | 1024 | 31.39 ms | 13.92 ms | 0.763 ms | 2.21 MB | +| CreateBinaryPatch | 1048576 | 5 | 512 | 4096 | 34.66 ms | 13.49 ms | 0.739 ms | 2.15 MB | ``` ## Legends ``` @@ -55,9 +21,9 @@ ## Additional info ``` -BenchmarkDotNet v0.15.6, Windows 11 (10.0.26200.7171) -AMD Ryzen 7 5800U with Radeon Graphics 1.90GHz, 1 CPU, 16 logical and 8 physical cores +BenchmarkDotNet v0.15.6, macOS 26.1 (25B78) [Darwin 25.1.0] +Apple M1 Pro, 1 CPU, 10 logical and 10 physical cores .NET SDK 10.0.100 - [Host] : .NET 8.0.22 (8.0.22, 8.0.2225.52707), X64 RyuJIT x86-64-v3 - ShortRun : .NET 8.0.22 (8.0.22, 8.0.2225.52707), X64 RyuJIT x86-64-v3 + [Host] : .NET 8.0.22 (8.0.22, 8.0.2225.52707), Arm64 RyuJIT armv8.0-a + ShortRun : .NET 8.0.22 (8.0.22, 8.0.2225.52707), Arm64 RyuJIT armv8.0-a ``` From c867ec77f7cb67fabcacfd37f6c7e3629af5e2c0 Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Sat, 29 Nov 2025 18:15:02 +0700 Subject: [PATCH 18/22] Add exception details --- src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs b/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs index c975774..0af674a 100644 --- a/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs +++ b/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs @@ -126,7 +126,7 @@ public async ValueTask MoveAsync(CancellationToken cancellationToken) } else { - throw new InvalidOperationException("Pinned position was not reset."); + throw new InvalidOperationException($"Pinned position '{_pinnedPosition}' for buffer '{_bufferSize}' with window '{_windowSize}' was not reset."); } } From 91e47d878a90e658762076826a3c5e45272f1e8e Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Sat, 29 Nov 2025 18:27:37 +0700 Subject: [PATCH 19/22] Rename IsPinned property --- src/BitSoft.BinaryTools/Patch/BinaryPatch.cs | 6 +++--- src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs b/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs index 267ce7f..0eeb876 100644 --- a/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs +++ b/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs @@ -53,14 +53,14 @@ public static async ValueTask CreateAsync( if (block is null) { - if (!reader.Pinned) + if (!reader.IsPinned) { reader.PinPosition(); } if (reader.Finished) { - if (reader.Pinned) + if (reader.IsPinned) await writer.WriteDataAsync(reader.PinnedWindowWithCurrent, cancellationToken); else await writer.WriteDataAsync(reader.Window, cancellationToken); @@ -86,7 +86,7 @@ public static async ValueTask CreateAsync( } else { - if (reader.Pinned) + if (reader.IsPinned) { await writer.WriteDataAsync(reader.PinnedWindow, cancellationToken); reader.ResetPinnedPosition(); diff --git a/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs b/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs index 0af674a..2105b77 100644 --- a/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs +++ b/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs @@ -51,7 +51,7 @@ public ReadOnlyMemory PinnedWindowWithCurrent } } - public bool Pinned => _pinnedPosition != NotDefined; + public bool IsPinned => _pinnedPosition != NotDefined; public bool Finished => _position == _size - 1; @@ -126,7 +126,7 @@ public async ValueTask MoveAsync(CancellationToken cancellationToken) } else { - throw new InvalidOperationException($"Pinned position '{_pinnedPosition}' for buffer '{_bufferSize}' with window '{_windowSize}' was not reset."); + throw new InvalidOperationException($"Pinned position '{_pinnedPosition}' for buffer '{_size}/{_bufferSize}' with window '{_windowSize}' was not reset."); } } From 07ba9b61da79d0ecdc8f4d969f4ef30844b44857 Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Sat, 29 Nov 2025 19:10:21 +0700 Subject: [PATCH 20/22] Add strong hash calculator --- .../Patch/BinaryPatchTests.cs | 43 ++++++++++++++++--- src/BitSoft.BinaryTools/Patch/BinaryPatch.cs | 13 ++++-- .../Patch/BlockInfoContainer.cs | 19 +++++--- .../Patch/CopyBlockSegment.cs | 6 +++ .../Patch/CopyBlockWithLengthSegment.cs | 5 --- .../Patch/HashCalculator.cs | 35 +++++++++++++++ .../Patch/PatchBlockInfo.cs | 4 +- .../Patch/PatchBlockInfoWithLength.cs | 4 +- src/BitSoft.BinaryTools/Patch/RollingHash.cs | 2 +- .../Patch/StreamWindowReader.cs | 3 +- 10 files changed, 110 insertions(+), 24 deletions(-) create mode 100644 src/BitSoft.BinaryTools/Patch/CopyBlockSegment.cs create mode 100644 src/BitSoft.BinaryTools/Patch/HashCalculator.cs diff --git a/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs b/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs index 33ea617..26e44d8 100644 --- a/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs +++ b/src/BitSoft.BinaryTools.Tests/Patch/BinaryPatchTests.cs @@ -67,6 +67,16 @@ private static IEnumerable TestCases() new byte[] { 0x1, 0x2, 0x9, 0x4, 0x5, 0x6, 0x7, 0x8 }, 3 ); + yield return new TestCaseData( + new byte[] { 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xA, 0xB, 0xC }, + new byte[] { 0x1, 0x2, 0x3, 0x4, 0x5, 0xA, 0xA, 0xA, 0xA, 0xB, 0xA, 0xC }, + 3 + ); + yield return new TestCaseData( + new byte[] { 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9 }, + new byte[] { 0x1, 0x2, 0xA, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9 }, + 4 + ); yield return new TestCaseData( new byte[] { 0x0, 0x1 }, new byte[] { 0x0 }, @@ -114,10 +124,9 @@ await BinaryPatch.CreateAsync( } [Ignore("Performance test")] - [TestCase(4 * 4, 4)] - [TestCase(10 * 1024 * 1024, 1024)] - [TestCase(10 * 1024 * 1024, 4 * 1024)] - public async Task Should_CreatePatch(int bufferLength, int blockSize) + [TestCase(3 * 4, 4, 2, 2)] + [TestCase(100 * 4 * 4, 4, 5, 6)] + public async Task Should_CreatePatch(int bufferLength, int blockSize, int changedBlocks, int changeSize) { // Arrange var source = new byte[bufferLength]; @@ -127,6 +136,17 @@ public async Task Should_CreatePatch(int bufferLength, int blockSize) Array.Copy(sourceArray: source, destinationArray: modified, length: source.Length); + var changeBlockSize = source.Length / (changedBlocks + 1); + + for (var b = 1; b <= changedBlocks; b++) + { + var position = changeBlockSize * b; + + var span = modified.AsSpan(start: position, length: changeSize); + + Random.Shared.NextBytes(span); + } + using var sourceStream = new MemoryStream(source); using var modifiedStream = new MemoryStream(modified); using var patchStream = new MemoryStream(); @@ -147,6 +167,19 @@ await BinaryPatch.CreateAsync( Console.WriteLine("Source length: {0}", sourceStream.Length); Console.WriteLine("Block size: {0}", blockSize); Console.WriteLine("Patch length: {0}", patchStream.Position); - Console.WriteLine("Time: {0:g}", stopwatch.Elapsed); + Console.WriteLine("Create time: {0:g}", stopwatch.Elapsed); + + sourceStream.Position = 0; + patchStream.Position = 0; + + using var patchedStream = new MemoryStream(); + + stopwatch.Restart(); + await BinaryPatch.ApplyAsync(source: sourceStream, patch: patchStream, output: patchedStream); + stopwatch.Stop(); + + Console.WriteLine("Apply time: {0:g}", stopwatch.Elapsed); + + Assert.That(patchedStream.ToArray(), Is.EqualTo(modified)); } } diff --git a/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs b/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs index 0eeb876..1683d54 100644 --- a/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs +++ b/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs @@ -41,6 +41,8 @@ public static async ValueTask CreateAsync( RollingHash rollingHash = default; var resetHash = true; + using var strongHashCalculator = new HashCalculator(); + while (true) { if (resetHash) @@ -49,7 +51,9 @@ public static async ValueTask CreateAsync( resetHash = false; } - var block = blockInfoContainer.Match(rollingHash); + var strongHash = strongHashCalculator.CalculatedHash(reader.Window.Span); + + var block = blockInfoContainer.Match(rollingHash, strongHash); if (block is null) { @@ -203,6 +207,8 @@ private static async ValueTask CalculateHashesAsync( var blockIndex = 0; + using var hashCalculator = new HashCalculator(); + var buffer = Pool.Rent(blockSize); try { @@ -215,14 +221,15 @@ private static async ValueTask CalculateHashesAsync( var span = buffer.AsSpan(start: 0, length: length); var hash = RollingHash.Create(span); + var strongHash = hashCalculator.CalculatedHash(buffer, offset: 0, count: length); if (length == blockSize) { - blockInfoContainer.Process(hash: hash, blockIndex: blockIndex); + blockInfoContainer.Process(blockIndex: blockIndex, hash: hash, strongHash: strongHash); } else { - blockInfoContainer.Process(hash: hash, blockIndex: blockIndex, blockLength: length); + blockInfoContainer.Process(blockIndex: blockIndex, blockLength: length, hash: hash, strongHash); } if (length < blockSize) diff --git a/src/BitSoft.BinaryTools/Patch/BlockInfoContainer.cs b/src/BitSoft.BinaryTools/Patch/BlockInfoContainer.cs index d173f00..f149816 100644 --- a/src/BitSoft.BinaryTools/Patch/BlockInfoContainer.cs +++ b/src/BitSoft.BinaryTools/Patch/BlockInfoContainer.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; namespace BitSoft.BinaryTools.Patch; @@ -6,10 +7,10 @@ internal sealed class BlockInfoContainer { private readonly Dictionary> _hashes = new(); - public void Process(RollingHash hash, int blockIndex) + public void Process(int blockIndex, RollingHash hash, byte[] strongHash) { var checksum = hash.GetChecksum(); - var block = new PatchBlockInfo(blockIndex: blockIndex, hash: checksum); + var block = new PatchBlockInfo(blockIndex: blockIndex, hash: checksum, strongHash); if (!_hashes.TryGetValue(checksum, out var blocks)) { _hashes[block.Hash] = blocks = []; @@ -18,10 +19,15 @@ public void Process(RollingHash hash, int blockIndex) blocks.Add(block); } - public void Process(RollingHash hash, int blockIndex, int blockLength) + public void Process(int blockIndex, int blockLength, RollingHash hash, byte[] strongHash) { var checksum = hash.GetChecksum(); - var block = new PatchBlockInfoWithLength(blockIndex: blockIndex, hash: checksum, length: blockLength); + var block = new PatchBlockInfoWithLength( + blockIndex: blockIndex, + length: blockLength, + hash: checksum, + strongHash: strongHash + ); if (!_hashes.TryGetValue(checksum, out var blocks)) { _hashes[block.Hash] = blocks = []; @@ -30,14 +36,15 @@ public void Process(RollingHash hash, int blockIndex, int blockLength) blocks.Add(block); } - public PatchBlockInfo? Match(RollingHash hash) + public PatchBlockInfo? Match(RollingHash hash, ReadOnlySpan strongHash) { var checksum = hash.GetChecksum(); if (_hashes.TryGetValue(checksum, out var blocks)) { foreach (var block in blocks) { - return block; + if (block.StrongHash.SequenceEqual(strongHash)) + return block; } } diff --git a/src/BitSoft.BinaryTools/Patch/CopyBlockSegment.cs b/src/BitSoft.BinaryTools/Patch/CopyBlockSegment.cs new file mode 100644 index 0000000..b1f306a --- /dev/null +++ b/src/BitSoft.BinaryTools/Patch/CopyBlockSegment.cs @@ -0,0 +1,6 @@ +namespace BitSoft.BinaryTools.Patch; + +internal sealed class CopyBlockSegment(int blockIndex) : IPatchSegment +{ + public int BlockIndex { get; } = blockIndex; +} diff --git a/src/BitSoft.BinaryTools/Patch/CopyBlockWithLengthSegment.cs b/src/BitSoft.BinaryTools/Patch/CopyBlockWithLengthSegment.cs index f348b25..b9fa375 100644 --- a/src/BitSoft.BinaryTools/Patch/CopyBlockWithLengthSegment.cs +++ b/src/BitSoft.BinaryTools/Patch/CopyBlockWithLengthSegment.cs @@ -6,8 +6,3 @@ internal sealed class CopyBlockWithLengthSegment(int blockIndex, int blockLength public int BlockLength { get; } = blockLength; } - -internal sealed class CopyBlockSegment(int blockIndex) : IPatchSegment -{ - public int BlockIndex { get; } = blockIndex; -} diff --git a/src/BitSoft.BinaryTools/Patch/HashCalculator.cs b/src/BitSoft.BinaryTools/Patch/HashCalculator.cs new file mode 100644 index 0000000..c563216 --- /dev/null +++ b/src/BitSoft.BinaryTools/Patch/HashCalculator.cs @@ -0,0 +1,35 @@ +using System; +using System.Buffers; +using System.Security.Cryptography; + +namespace BitSoft.BinaryTools.Patch; + +internal sealed class HashCalculator : IDisposable +{ + private static ArrayPool Pool { get; } = ArrayPool.Shared; + + private readonly MD5 _md5 = MD5.Create(); + private readonly byte[] _buffer = Pool.Rent(minimumLength: 32); + + public byte[] CalculatedHash(byte[] source, int offset, int count) + { + ArgumentNullException.ThrowIfNull(source); + + return _md5.ComputeHash(source, offset: offset, count: count); + } + + public ReadOnlySpan CalculatedHash(ReadOnlySpan source) + { + if (_md5.TryComputeHash(source: source, destination: _buffer, out var bytesWritten)) + { + return _buffer.AsSpan(start: 0, length: bytesWritten); + } + + throw new InvalidOperationException("Hash calculation failed."); + } + + public void Dispose() + { + _md5.Dispose(); + } +} diff --git a/src/BitSoft.BinaryTools/Patch/PatchBlockInfo.cs b/src/BitSoft.BinaryTools/Patch/PatchBlockInfo.cs index 496783c..8db174e 100644 --- a/src/BitSoft.BinaryTools/Patch/PatchBlockInfo.cs +++ b/src/BitSoft.BinaryTools/Patch/PatchBlockInfo.cs @@ -1,8 +1,10 @@ namespace BitSoft.BinaryTools.Patch; -internal class PatchBlockInfo(int blockIndex, uint hash) +internal class PatchBlockInfo(int blockIndex, uint hash, byte[] strongHash) { public int BlockIndex { get; } = blockIndex; public uint Hash { get; } = hash; + + public byte[] StrongHash { get; } = strongHash; } diff --git a/src/BitSoft.BinaryTools/Patch/PatchBlockInfoWithLength.cs b/src/BitSoft.BinaryTools/Patch/PatchBlockInfoWithLength.cs index c08949e..46f8980 100644 --- a/src/BitSoft.BinaryTools/Patch/PatchBlockInfoWithLength.cs +++ b/src/BitSoft.BinaryTools/Patch/PatchBlockInfoWithLength.cs @@ -1,7 +1,7 @@ namespace BitSoft.BinaryTools.Patch; -internal sealed class PatchBlockInfoWithLength(int blockIndex, uint hash, int length) - : PatchBlockInfo(blockIndex, hash) +internal sealed class PatchBlockInfoWithLength(int blockIndex, uint hash, byte[] strongHash, int length) + : PatchBlockInfo(blockIndex, hash, strongHash) { public int Length { get; } = length; } diff --git a/src/BitSoft.BinaryTools/Patch/RollingHash.cs b/src/BitSoft.BinaryTools/Patch/RollingHash.cs index 7f86e28..4363f11 100644 --- a/src/BitSoft.BinaryTools/Patch/RollingHash.cs +++ b/src/BitSoft.BinaryTools/Patch/RollingHash.cs @@ -36,7 +36,7 @@ public static RollingHash Create(ReadOnlySpan data) public void Update(byte removed, byte added) { _a = (_a - removed + added) % Base; - _b = (_b - _length * removed + _a) % Base; + _b = (_b - _length * removed + _a - 1) % Base; } public uint GetChecksum() => (_b << 16) | _a; diff --git a/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs b/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs index 2105b77..789e0a4 100644 --- a/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs +++ b/src/BitSoft.BinaryTools/Patch/StreamWindowReader.cs @@ -126,7 +126,8 @@ public async ValueTask MoveAsync(CancellationToken cancellationToken) } else { - throw new InvalidOperationException($"Pinned position '{_pinnedPosition}' for buffer '{_size}/{_bufferSize}' with window '{_windowSize}' was not reset."); + throw new InvalidOperationException( + $"Pinned position '{_pinnedPosition}' for buffer '{_size}/{_bufferSize}' with window '{_windowSize}' was not reset."); } } From 321a40541a2b1186750ae0dd08a2fa4f496c5b65 Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Sat, 29 Nov 2025 19:26:30 +0700 Subject: [PATCH 21/22] Update results --- src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs | 2 +- src/BitSoft.BinaryTools.Benchmarks/Readme.md | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs b/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs index ca4819d..f77abc7 100644 --- a/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs +++ b/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs @@ -17,7 +17,7 @@ public class BinaryPatchBenchmark private Stream? _modifiedStream; private Stream? _patchStream; - [Params(1024 * 1024)] + [Params(1024 * 1024, 10 * 1024 * 1024)] public int BufferLength { get; set; } [Params(5)] diff --git a/src/BitSoft.BinaryTools.Benchmarks/Readme.md b/src/BitSoft.BinaryTools.Benchmarks/Readme.md index 5676366..77eb882 100644 --- a/src/BitSoft.BinaryTools.Benchmarks/Readme.md +++ b/src/BitSoft.BinaryTools.Benchmarks/Readme.md @@ -3,8 +3,10 @@ ``` | Method | BufferLength | ChangedBlocks | ChangeSize | BlockSize | Mean | Error | StdDev | Allocated | |------------------ |------------- |-------------- |----------- |---------- |---------:|---------:|---------:|----------:| -| CreateBinaryPatch | 1048576 | 5 | 512 | 1024 | 31.39 ms | 13.92 ms | 0.763 ms | 2.21 MB | -| CreateBinaryPatch | 1048576 | 5 | 512 | 4096 | 34.66 ms | 13.49 ms | 0.739 ms | 2.15 MB | +| CreateBinaryPatch | 1048576 | 5 | 512 | 1024 | 1.692 s | 0.0159 s | 0.0009 s | 2.29 MB | +| CreateBinaryPatch | 1048576 | 5 | 512 | 4096 | 5.764 s | 0.1578 s | 0.0087 s | 2.18 MB | +| CreateBinaryPatch | 10485760 | 5 | 512 | 1024 | 16.832 s | 0.6871 s | 0.0377 s | 34.85 MB | +| CreateBinaryPatch | 10485760 | 5 | 512 | 4096 | 57.220 s | 0.9950 s | 0.0545 s | 32.69 MB | ``` ## Legends ``` From e1edac6d3f4b3cc6d95fab4b78bc35a8f2bbd0a4 Mon Sep 17 00:00:00 2001 From: bitc0der <59016822+bitc0der@users.noreply.github.com> Date: Sat, 29 Nov 2025 19:42:42 +0700 Subject: [PATCH 22/22] Fix hash calculation --- .../BinaryPatchBenchmark.cs | 1 - src/BitSoft.BinaryTools.Benchmarks/Readme.md | 12 ++++++------ src/BitSoft.BinaryTools/Patch/BinaryPatch.cs | 16 +++++++--------- .../Patch/BlockInfoContainer.cs | 10 +++++++++- src/BitSoft.BinaryTools/Patch/HashCalculator.cs | 2 +- 5 files changed, 23 insertions(+), 18 deletions(-) diff --git a/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs b/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs index f77abc7..6c4729a 100644 --- a/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs +++ b/src/BitSoft.BinaryTools.Benchmarks/BinaryPatchBenchmark.cs @@ -38,7 +38,6 @@ public void GlobalSetUp() Array.Copy(sourceArray: _source, destinationArray: _modified, length: _source.Length); var changeBlockSize = _source.Length / (ChangedBlocks + 1); - for (var b = 1; b <= ChangedBlocks; b++) { var position = changeBlockSize * b; diff --git a/src/BitSoft.BinaryTools.Benchmarks/Readme.md b/src/BitSoft.BinaryTools.Benchmarks/Readme.md index 77eb882..ec982a0 100644 --- a/src/BitSoft.BinaryTools.Benchmarks/Readme.md +++ b/src/BitSoft.BinaryTools.Benchmarks/Readme.md @@ -1,12 +1,12 @@ # Benchmarks ``` -| Method | BufferLength | ChangedBlocks | ChangeSize | BlockSize | Mean | Error | StdDev | Allocated | -|------------------ |------------- |-------------- |----------- |---------- |---------:|---------:|---------:|----------:| -| CreateBinaryPatch | 1048576 | 5 | 512 | 1024 | 1.692 s | 0.0159 s | 0.0009 s | 2.29 MB | -| CreateBinaryPatch | 1048576 | 5 | 512 | 4096 | 5.764 s | 0.1578 s | 0.0087 s | 2.18 MB | -| CreateBinaryPatch | 10485760 | 5 | 512 | 1024 | 16.832 s | 0.6871 s | 0.0377 s | 34.85 MB | -| CreateBinaryPatch | 10485760 | 5 | 512 | 4096 | 57.220 s | 0.9950 s | 0.0545 s | 32.69 MB | +| Method | BufferLength | ChangedBlocks | ChangeSize | BlockSize | Mean | Error | StdDev | Allocated | +|------------------ |------------- |-------------- |----------- |---------- |----------:|---------:|---------:|----------:| +| CreateBinaryPatch | 1048576 | 5 | 512 | 1024 | 36.85 ms | 10.66 ms | 0.584 ms | 2.3 MB | +| CreateBinaryPatch | 1048576 | 5 | 512 | 4096 | 37.47 ms | 11.53 ms | 0.632 ms | 2.19 MB | +| CreateBinaryPatch | 10485760 | 5 | 512 | 1024 | 397.60 ms | 28.31 ms | 1.552 ms | 34.86 MB | +| CreateBinaryPatch | 10485760 | 5 | 512 | 4096 | 376.37 ms | 58.44 ms | 3.203 ms | 32.71 MB | ``` ## Legends ``` diff --git a/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs b/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs index 1683d54..f4f45d7 100644 --- a/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs +++ b/src/BitSoft.BinaryTools/Patch/BinaryPatch.cs @@ -26,7 +26,9 @@ public static async ValueTask CreateAsync( if (!output.CanWrite) throw new ArgumentException($"{nameof(output)} does not support writing.", nameof(output)); - var blockInfoContainer = await CalculateHashesAsync(source, blockSize, cancellationToken); + using var hashCalculator = new HashCalculator(); + + var blockInfoContainer = await CalculateHashesAsync(source, hashCalculator, blockSize, cancellationToken); using var reader = new StreamWindowReader(modified, Pool, windowSize: blockSize); using var writer = new PatchWriter(output); @@ -41,8 +43,6 @@ public static async ValueTask CreateAsync( RollingHash rollingHash = default; var resetHash = true; - using var strongHashCalculator = new HashCalculator(); - while (true) { if (resetHash) @@ -51,9 +51,7 @@ public static async ValueTask CreateAsync( resetHash = false; } - var strongHash = strongHashCalculator.CalculatedHash(reader.Window.Span); - - var block = blockInfoContainer.Match(rollingHash, strongHash); + var block = blockInfoContainer.Match(rollingHash, reader.Window.Span); if (block is null) { @@ -195,20 +193,20 @@ async ValueTask CopyBlockSegmentAsync(int blockIndex, int blockLength) private static async ValueTask CalculateHashesAsync( Stream source, + HashCalculator hashCalculator, int blockSize, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(source); + ArgumentNullException.ThrowIfNull(hashCalculator); if (!source.CanRead) throw new ArgumentException("source stream must be readable.", nameof(source)); - var blockInfoContainer = new BlockInfoContainer(); + var blockInfoContainer = new BlockInfoContainer(hashCalculator); var blockIndex = 0; - using var hashCalculator = new HashCalculator(); - var buffer = Pool.Rent(blockSize); try { diff --git a/src/BitSoft.BinaryTools/Patch/BlockInfoContainer.cs b/src/BitSoft.BinaryTools/Patch/BlockInfoContainer.cs index f149816..f264b8f 100644 --- a/src/BitSoft.BinaryTools/Patch/BlockInfoContainer.cs +++ b/src/BitSoft.BinaryTools/Patch/BlockInfoContainer.cs @@ -5,8 +5,14 @@ namespace BitSoft.BinaryTools.Patch; internal sealed class BlockInfoContainer { + private readonly HashCalculator _hashCalculator; private readonly Dictionary> _hashes = new(); + public BlockInfoContainer(HashCalculator hashCalculator) + { + _hashCalculator = hashCalculator ?? throw new ArgumentNullException(nameof(hashCalculator)); + } + public void Process(int blockIndex, RollingHash hash, byte[] strongHash) { var checksum = hash.GetChecksum(); @@ -36,11 +42,13 @@ public void Process(int blockIndex, int blockLength, RollingHash hash, byte[] st blocks.Add(block); } - public PatchBlockInfo? Match(RollingHash hash, ReadOnlySpan strongHash) + public PatchBlockInfo? Match(RollingHash hash, ReadOnlySpan span) { var checksum = hash.GetChecksum(); if (_hashes.TryGetValue(checksum, out var blocks)) { + var strongHash = _hashCalculator.CalculatedHash(span); + foreach (var block in blocks) { if (block.StrongHash.SequenceEqual(strongHash)) diff --git a/src/BitSoft.BinaryTools/Patch/HashCalculator.cs b/src/BitSoft.BinaryTools/Patch/HashCalculator.cs index c563216..3f3575b 100644 --- a/src/BitSoft.BinaryTools/Patch/HashCalculator.cs +++ b/src/BitSoft.BinaryTools/Patch/HashCalculator.cs @@ -9,7 +9,7 @@ internal sealed class HashCalculator : IDisposable private static ArrayPool Pool { get; } = ArrayPool.Shared; private readonly MD5 _md5 = MD5.Create(); - private readonly byte[] _buffer = Pool.Rent(minimumLength: 32); + private readonly byte[] _buffer = Pool.Rent(minimumLength: 16); public byte[] CalculatedHash(byte[] source, int offset, int count) {