From 1f0453d1caa53ea87f0b03db00392d8ed2a063a1 Mon Sep 17 00:00:00 2001 From: Slava Date: Tue, 7 Apr 2026 16:35:34 +0300 Subject: [PATCH] Support non-standard RaptorQ symbol size (> 65536) --- src/adnl/raptorq/src/base.rs | 22 +- src/adnl/raptorq/src/decoder.rs | 16 +- src/adnl/raptorq/src/encoder.rs | 6 +- src/adnl/raptorq/src/octets.rs | 5 +- src/adnl/src/overlay/broadcast.rs | 7 +- src/adnl/src/rldp/recv.rs | 17 +- src/adnl/src/rldp/send.rs | 6 +- src/adnl/tests/test_overlay.rs | 93 ++- src/node/tests/compat_test/Cargo.toml | 4 + src/node/tests/compat_test/Makefile | 7 + src/node/tests/compat_test/README.md | 31 +- .../tests/compat_test/cpp_src/CMakeLists.txt | 1 + .../compat_test/cpp_src/compat_test_node.cpp | 160 ++++- .../compat_test/cpp_src/compat_test_node.hpp | 3 + src/node/tests/compat_test/src/lib.rs | 113 +++- .../tests/compat_test/tests/test_raptorq.rs | 628 ++++++++++++++++++ 16 files changed, 1071 insertions(+), 48 deletions(-) create mode 100644 src/node/tests/compat_test/tests/test_raptorq.rs diff --git a/src/adnl/raptorq/src/base.rs b/src/adnl/raptorq/src/base.rs index 192533b..1d3e24e 100644 --- a/src/adnl/raptorq/src/base.rs +++ b/src/adnl/raptorq/src/base.rs @@ -104,7 +104,7 @@ impl EncodingPacket { #[cfg_attr(feature = "serde_support", derive(Serialize, Deserialize))] pub struct ObjectTransmissionInformation { transfer_length: u64, // Limited to u40 - symbol_size: u16, + symbol_size: u32, num_source_blocks: u8, num_sub_blocks: u16, symbol_alignment: u8, @@ -113,14 +113,14 @@ pub struct ObjectTransmissionInformation { impl ObjectTransmissionInformation { pub fn new( transfer_length: u64, - symbol_size: u16, + symbol_size: u32, source_blocks: u8, sub_blocks: u16, alignment: u8, ) -> ObjectTransmissionInformation { // See errata (https://www.rfc-editor.org/errata/eid5548) assert!(transfer_length <= 942574504275); - assert_eq!(symbol_size % alignment as u16, 0); + assert_eq!(symbol_size % alignment as u32, 0); // See section 4.4.1.2. "These parameters MUST be set so that ceil(ceil(F/T)/Z) <= K'_max." if (symbol_size != 0) && (source_blocks != 0) { @@ -140,6 +140,10 @@ impl ObjectTransmissionInformation { } } + #[deprecated(note = "\ + RFC 5052 wire format, only encodes symbol_size as u16. \ + Not used in TON — TL serialization is used instead\ + ")] pub fn deserialize(data: &[u8; 12]) -> ObjectTransmissionInformation { ObjectTransmissionInformation { transfer_length: ((data[0] as u64) << 32) @@ -147,13 +151,17 @@ impl ObjectTransmissionInformation { + ((data[2] as u64) << 16) + ((data[3] as u64) << 8) + (data[4] as u64), - symbol_size: ((data[6] as u16) << 8) + data[7] as u16, + symbol_size: ((data[6] as u32) << 8) + data[7] as u32, num_source_blocks: data[8], num_sub_blocks: ((data[9] as u16) << 8) + data[10] as u16, symbol_alignment: data[11], } } + #[deprecated(note = "\ + RFC 5052 wire format, only encodes symbol_size as u16. \ + Not used in TON — TL serialization is used instead\ + ")] pub fn serialize(&self) -> [u8; 12] { [ ((self.transfer_length >> 32) & 0xFF) as u8, @@ -175,7 +183,7 @@ impl ObjectTransmissionInformation { self.transfer_length } - pub fn symbol_size(&self) -> u16 { + pub fn symbol_size(&self) -> u32 { self.symbol_size } @@ -196,9 +204,9 @@ impl ObjectTransmissionInformation { max_packet_size: u16, decoder_memory_requirement: u64, ) -> ObjectTransmissionInformation { - let alignment = 8; + let alignment: u16 = 8; assert!(max_packet_size >= alignment); - let symbol_size = max_packet_size - (max_packet_size % alignment); + let symbol_size = (max_packet_size - (max_packet_size % alignment)) as u32; let sub_symbol_size = 8; let kt = int_div_ceil(transfer_length, symbol_size as u64); diff --git a/src/adnl/raptorq/src/decoder.rs b/src/adnl/raptorq/src/decoder.rs index 6261fa7..74b5338 100644 --- a/src/adnl/raptorq/src/decoder.rs +++ b/src/adnl/raptorq/src/decoder.rs @@ -119,7 +119,7 @@ impl Decoder { #[cfg_attr(feature = "serde_support", derive(Serialize, Deserialize))] pub struct SourceBlockDecoder { source_block_id: u8, - symbol_size: u16, + symbol_size: u32, num_sub_blocks: u16, symbol_alignment: u8, source_block_symbols: u32, @@ -137,7 +137,7 @@ impl SourceBlockDecoder { note = "Use the new2() function instead. In version 2.0, that function will replace this one" )] #[cfg(feature = "std")] - pub fn new(source_block_id: u8, symbol_size: u16, block_length: u64) -> SourceBlockDecoder { + pub fn new(source_block_id: u8, symbol_size: u32, block_length: u64) -> SourceBlockDecoder { let config = ObjectTransmissionInformation::new(0, symbol_size, 0, 1, 1); SourceBlockDecoder::new2(source_block_id, &config, block_length) } @@ -175,10 +175,8 @@ impl SourceBlockDecoder { } fn unpack_sub_blocks(&self, result: &mut [u8], symbol: &Symbol, symbol_index: usize) { - let (tl, ts, nl, ns) = partition( - (self.symbol_size / self.symbol_alignment as u16) as u32, - self.num_sub_blocks, - ); + let (tl, ts, nl, ns) = + partition(self.symbol_size / self.symbol_alignment as u32, self.num_sub_blocks); let mut symbol_offset = 0; let mut sub_block_offset = 0; @@ -283,7 +281,7 @@ impl SourceBlockDecoder { let mut encoded_indices = vec![]; // See section 5.3.3.4.2. There are S + H zero symbols to start the D vector - let mut d = vec![Symbol::zero(self.symbol_size); s + h]; + let mut d = vec![Symbol::zero(self.symbol_size as usize); s + h]; for (i, source) in self.source_symbols.iter().enumerate() { if let Some(symbol) = source { encoded_indices.push(i as u32); @@ -294,7 +292,7 @@ impl SourceBlockDecoder { // Append the extended padding symbols for i in self.source_block_symbols..num_extended_symbols { encoded_indices.push(i); - d.push(Symbol::zero(self.symbol_size)); + d.push(Symbol::zero(self.symbol_size as usize)); } for repair_packet in self.repair_packets.iter() { @@ -328,7 +326,7 @@ impl SourceBlockDecoder { sys_index: u32, p1: u32, ) -> Symbol { - let mut rebuilt = Symbol::zero(self.symbol_size); + let mut rebuilt = Symbol::zero(self.symbol_size as usize); let tuple = intermediate_tuple(source_symbol_id, lt_symbols, sys_index, p1); for i in enc_indices(tuple, lt_symbols, pi_symbols, p1) { diff --git a/src/adnl/raptorq/src/encoder.rs b/src/adnl/raptorq/src/encoder.rs index c490074..68ee6e0 100644 --- a/src/adnl/raptorq/src/encoder.rs +++ b/src/adnl/raptorq/src/encoder.rs @@ -186,7 +186,7 @@ impl SourceBlockEncoder { since = "1.3.0", note = "Use the new2() function instead. In version 2.0, that function will replace this one" )] - pub fn new(source_block_id: u8, symbol_size: u16, data: &[u8]) -> SourceBlockEncoder { + pub fn new(source_block_id: u8, symbol_size: u32, data: &[u8]) -> SourceBlockEncoder { let config = ObjectTransmissionInformation::new(0, symbol_size, 0, 1, 1); SourceBlockEncoder::new2(source_block_id, &config, data) } @@ -196,7 +196,7 @@ impl SourceBlockEncoder { if config.sub_blocks() > 1 { let mut symbols = vec![vec![]; data.len() / config.symbol_size() as usize]; let (tl, ts, nl, ns) = partition( - (config.symbol_size() / config.symbol_alignment() as u16) as u32, + config.symbol_size() / config.symbol_alignment() as u32, config.sub_blocks(), ); // Divide the block into sub-blocks and then concatenate the sub-symbols into symbols @@ -247,7 +247,7 @@ impl SourceBlockEncoder { )] pub fn with_encoding_plan( source_block_id: u8, - symbol_size: u16, + symbol_size: u32, data: &[u8], plan: &SourceBlockEncodingPlan, ) -> SourceBlockEncoder { diff --git a/src/adnl/raptorq/src/octets.rs b/src/adnl/raptorq/src/octets.rs index 2a1d4fb..7b25f9c 100644 --- a/src/adnl/raptorq/src/octets.rs +++ b/src/adnl/raptorq/src/octets.rs @@ -633,10 +633,7 @@ unsafe fn store_neon(ptr: *mut uint8x16_t, value: uint8x16_t) { #[cfg(target_arch = "arm")] use std::arch::arm::*; - // TODO: replace with vst1q_u8 when it's supported - let reinterp = vreinterpretq_u64_u8(value); - *(ptr as *mut u64) = vgetq_lane_u64(reinterp, 0); - *(ptr as *mut u64).add(1) = vgetq_lane_u64(reinterp, 1); + vst1q_u8(ptr as *mut u8, value); } #[cfg(all(target_arch = "aarch64", feature = "std"))] diff --git a/src/adnl/src/overlay/broadcast.rs b/src/adnl/src/overlay/broadcast.rs index d754556..1114119 100644 --- a/src/adnl/src/overlay/broadcast.rs +++ b/src/adnl/src/overlay/broadcast.rs @@ -1675,8 +1675,6 @@ pub(crate) struct BroadcastTwostepFecProtocol { } impl BroadcastTwostepFecProtocol { - const MAX_PART_SIZE: usize = 65536; - pub(crate) fn for_recv() -> Self { Self { extra: None, send_ctx: None } } @@ -1690,9 +1688,6 @@ impl BroadcastTwostepFecProtocol { } let k = ((neighbours as usize) * 2 - 2) / 3; let part_size = (data.len() + k - 1) / k; - if part_size >= Self::MAX_PART_SIZE { - fail!("Too big part size {part_size} in {} broadcast", Self::broadcast_type()); - } let ctx = BroadcastTwostepSendContext { neighbours, part_size }; Ok(Self { extra: Some(extra), send_ctx: Some(ctx) }) } @@ -1819,7 +1814,7 @@ impl BroadcastProtocol for BroadcastTwostepFecProtocol { bcast_id: *bcast_id, data_hash: sha256_digest(ctx.data.object), date, - encoder: RaptorqEncoder::with_data(ctx.data.object, Some(send_ctx.part_size as u16)), + encoder: RaptorqEncoder::with_data(ctx.data.object, Some(send_ctx.part_size as u32)), extra: self.extra.take().unwrap_or_default(), flags: ctx.flags, seqno: 0, diff --git a/src/adnl/src/rldp/recv.rs b/src/adnl/src/rldp/recv.rs index 89e61e5..a7cd10d 100644 --- a/src/adnl/src/rldp/recv.rs +++ b/src/adnl/src/rldp/recv.rs @@ -59,12 +59,8 @@ impl RaptorqDecoder { /// Construct with parameters pub fn with_params(params: FecTypeRaptorQ) -> Result { const MAX_SOURCE_SYMBOLS: i32 = 56_403; // K'_max per RFC 6330 §5.1.2 - if (params.symbol_size <= 0) || (params.symbol_size > u16::MAX as i32) { - fail!( - "Invalid FEC params: symbol_size must be in 1..={}, got {}", - u16::MAX, - params.symbol_size - ); + if params.symbol_size <= 0 { + fail!("Invalid FEC params: symbol_size must be > 0, got {}", params.symbol_size); } if params.data_size <= 0 { fail!("Invalid FEC params: data_size must be > 0, got {}", params.data_size); @@ -77,12 +73,15 @@ impl RaptorqDecoder { } else { // Two-step broadcast case: symbol_size was set by the sender without alignment // rounding (alignment=1). Use the same config as the encoder. + // symbol_size can exceed u16::MAX for large blocks with few validators + // (matches C++ behaviour which uses size_t for symbol_size). // // With source_blocks=1, raptorq asserts ceil(data_size/symbol_size) <= // MAX_SOURCE_SYMBOLS_PER_BLOCK (K'_max = 56403, RFC 6330 §5.1.2). // Validate before calling to prevent a panic on malformed network messages. - let source_symbols = (params.data_size + params.symbol_size - 1) / params.symbol_size; - if source_symbols > MAX_SOURCE_SYMBOLS { + let source_symbols = (params.data_size as i64 + params.symbol_size as i64 - 1) + / params.symbol_size as i64; + if source_symbols > MAX_SOURCE_SYMBOLS as i64 { fail!( "Invalid FEC params: source symbol count {source_symbols} \ exceeds raptorq limit {MAX_SOURCE_SYMBOLS} (data_size={}, symbol_size={})", @@ -92,7 +91,7 @@ impl RaptorqDecoder { } raptorq::ObjectTransmissionInformation::new( params.data_size as u64, - params.symbol_size as u16, + params.symbol_size as u32, 1, 1, 1, diff --git a/src/adnl/src/rldp/send.rs b/src/adnl/src/rldp/send.rs index 08033de..555411d 100644 --- a/src/adnl/src/rldp/send.rs +++ b/src/adnl/src/rldp/send.rs @@ -44,7 +44,7 @@ pub struct RaptorqEncoder { impl RaptorqEncoder { /// Construct over data - pub fn with_data(data: &[u8], symbol: Option) -> Self { + pub fn with_data(data: &[u8], symbol: Option) -> Self { let engine = if let Some(symbol_size) = symbol.filter(|&s| s as usize != Constraints::SYMBOL) { // symbol_size is set for the two-step FEC broadcast case where data is @@ -59,8 +59,8 @@ impl RaptorqEncoder { // and k is bounded by the neighbour count (typically a few dozen, at most // ~130), which is far below K'_max = 56403 (RFC 6330 max symbols per block). // - // sub_blocks=1: the data fits comfortably in memory since part_size < 65536 - // and kt <= k << K'_max, so no sub-block splitting is needed. + // sub_blocks=1: the data fits comfortably in memory since + // kt <= k << K'_max, so no sub-block splitting is needed. let config = raptorq::ObjectTransmissionInformation::new( data.len() as u64, symbol_size, diff --git a/src/adnl/tests/test_overlay.rs b/src/adnl/tests/test_overlay.rs index 95117e7..3deacdd 100644 --- a/src/adnl/tests/test_overlay.rs +++ b/src/adnl/tests/test_overlay.rs @@ -1627,7 +1627,7 @@ async fn test_overlay_semiprivate() -> Result<()> { fn test_overlay_raptorq() { use rand::{seq::SliceRandom, SeedableRng}; - fn run(symbol: Option) { + fn run(symbol: Option) { let seed: u64 = rand::random(); println!("test_overlay_raptorq seed: {seed}"); let mut rng = rand::rngs::StdRng::seed_from_u64(seed); @@ -1778,3 +1778,94 @@ fn test_overlay_raptorq() { println!("--- symbol=Some(771) (alignment=1) ---"); run(Some(771)); } + +/// Test that RaptorQ encode/decode works with symbol_size > 65535 (u16 limit). +/// This matches the C++ behaviour where symbol_size is size_t. +/// Simulates TwostepFec for 800KB data with 10 parties: +/// k = (10*2-2)/3 = 6, part_size = ceil(819200/6) = 136534 +#[test] +fn test_raptorq_large_symbol_size() { + use adnl::{RaptorqDecoder, RaptorqEncoder}; + use rand::Rng; + + const DATA_SIZE: usize = 800 * 1024; + + for num_parties in [5u32, 10, 20] { + let k = ((num_parties as usize) * 2 - 2) / 3; + let part_size = (DATA_SIZE + k - 1) / k; + + println!( + "--- parties={num_parties}, k={k}, part_size={part_size} (>65535: {}) ---", + part_size > 65535 + ); + + // Generate random data + let mut rng = rand::thread_rng(); + let data: Vec = (0..DATA_SIZE).map(|_| rng.gen()).collect(); + + // Encode with large symbol_size + let mut encoder = RaptorqEncoder::with_data(&data, Some(part_size as u32)); + let params = encoder.params().clone(); + println!( + " params: data_size={}, symbol_size={}, symbols_count={}", + params.data_size, params.symbol_size, params.symbols_count + ); + assert_eq!(params.data_size, DATA_SIZE as i32); + assert_eq!(params.symbol_size, part_size as i32); + + // Collect source + repair symbols + let source_count = params.symbols_count as usize; + let mut packets: Vec<(u32, Vec)> = Vec::new(); + let mut seqno = 0u32; + for _ in 0..(source_count * 3 / 2) { + let chunk = encoder.encode(&mut seqno).unwrap(); + packets.push((seqno, chunk)); + seqno += 1; + } + assert!( + packets.len() >= source_count, + "Not enough packets generated: {} < {source_count}", + packets.len() + ); + + // Decode using exactly source_count symbols (minimum required) + let mut decoder = RaptorqDecoder::with_params(params.clone()) + .expect("decoder creation must succeed for large symbol_size"); + + let mut decoded = None; + for (seq, chunk) in packets.iter().take(source_count + 2) { + if let Some(result) = decoder.decode(*seq, chunk) { + decoded = Some(result); + break; + } + } + + let result = decoded.expect("decode must succeed"); + assert_eq!(result.len(), DATA_SIZE, "decoded size mismatch"); + assert_eq!(result, data, "decoded data mismatch"); + println!(" OK: encode/decode verified for symbol_size={part_size}"); + } +} + +/// Verify that the RaptorQ encoder produces valid FEC symbols for large +/// part_size values (>65535) that match the C++ TwostepFec behaviour. +#[test] +fn test_twostep_fec_encoder_large_symbols() { + let data = vec![0x42u8; 800 * 1024]; + for neighbours in [5u32, 10, 20] { + let k = ((neighbours as usize) * 2 - 2) / 3; + let part_size = (data.len() + k - 1) / k; + + let mut encoder = RaptorqEncoder::with_data(&data, Some(part_size as u32)); + let params = encoder.params().clone(); + assert_eq!(params.data_size, data.len() as i32); + assert_eq!(params.symbol_size, part_size as i32); + + // Generate one symbol per neighbour (like broadcast-twostep.cpp) + let mut seqno = 0u32; + for _ in 0..neighbours { + let chunk = encoder.encode(&mut seqno).unwrap(); + assert_eq!(chunk.len(), part_size, "symbol size mismatch"); + } + } +} diff --git a/src/node/tests/compat_test/Cargo.toml b/src/node/tests/compat_test/Cargo.toml index 26adde0..51e4e72 100644 --- a/src/node/tests/compat_test/Cargo.toml +++ b/src/node/tests/compat_test/Cargo.toml @@ -77,3 +77,7 @@ path = "tests/test_quic_transport.rs" [[test]] name = "test_quic_overlay" path = "tests/test_quic_overlay.rs" + +[[test]] +name = "test_raptorq" +path = "tests/test_raptorq.rs" diff --git a/src/node/tests/compat_test/Makefile b/src/node/tests/compat_test/Makefile index f0f87a8..1bfa849 100644 --- a/src/node/tests/compat_test/Makefile +++ b/src/node/tests/compat_test/Makefile @@ -12,6 +12,13 @@ BUILD_DIR := $(ROOT_DIR)/build CPP_BUILD_DIR ?= $(BUILD_DIR)/cpp RUST_TARGET_DIR := $(BUILD_DIR)/rust +# Resolve CPP_SRC_PATH to absolute to avoid breakage when CMake runs from build dir +CPP_SRC_PATH_ORIG := $(CPP_SRC_PATH) +CPP_SRC_PATH := $(shell cd "$(CPP_SRC_PATH)" 2>/dev/null && pwd) +ifeq ($(CPP_SRC_PATH),) +$(error CPP_SRC_PATH "$(CPP_SRC_PATH_ORIG)" is not a valid directory) +endif + # C++ source files location CPP_TEST_SRC := $(ROOT_DIR)/cpp_src diff --git a/src/node/tests/compat_test/README.md b/src/node/tests/compat_test/README.md index 375ae2b..4182986 100644 --- a/src/node/tests/compat_test/README.md +++ b/src/node/tests/compat_test/README.md @@ -38,7 +38,8 @@ compat_test/ │ ├── test_twostep_fec_relay.rs # TwostepFec broadcast relay (6-node topology) │ ├── test_quic_transport.rs # QUIC transport: raw queries, large messages, TLS │ ├── test_quic_overlay.rs # QUIC overlay: messages and queries via QUIC -│ └── test_quic_private_overlay.rs # QUIC private overlay: ADNL vs QUIC transport +│ ├── test_quic_private_overlay.rs # QUIC private overlay: ADNL vs QUIC transport +│ └── test_raptorq.rs # RaptorQ FEC codec cross-implementation tests └── build/ # Build artifacts (gitignored) └── cpp/ # C++ binary output ``` @@ -97,7 +98,8 @@ make clean | `test_quic_transport` | 3 | 3 | 0 | Compatible | | `test_quic_overlay` | 4 | 4 | 0 | Compatible | | `test_quic_private_overlay` | 5 | 5 | 0 | Compatible | -| **Total** | **53** | **52** | **1** | | +| `test_raptorq` | 14 | 14 | 0 | Compatible | +| **Total** | **67** | **66** | **1** | | ## Test Suites @@ -205,6 +207,26 @@ Both RLDP v1 and v2, both directions, three payload sizes: - QUIC overlay query (Rust → C++) - Private overlay message (C++ → Rust, with receipt verification) +### 14. RaptorQ FEC Codec (`test_raptorq`) +Cross-implementation RaptorQ encode/decode — symbols produced by one side are fed to the other's decoder. No networking involved; the C++ test node exposes `raptorq_encode`/`raptorq_decode` commands that operate on raw data: + +| Test | Direction | Payload | Scenario | Result | +|------|-----------|---------|----------|--------| +| `test_rust_encode_cpp_decode_small` | Rust → C++ | 500 B | Source-only | PASS | +| `test_rust_encode_cpp_decode_medium` | Rust → C++ | 10 KB | With repair symbols | PASS | +| `test_rust_encode_cpp_decode_large` | Rust → C++ | 100 KB | Large payload | PASS | +| `test_cpp_encode_rust_decode_small` | C++ → Rust | 500 B | Source-only | PASS | +| `test_cpp_encode_rust_decode_medium` | C++ → Rust | 10 KB | With repair symbols | PASS | +| `test_cpp_encode_rust_decode_large` | C++ → Rust | 100 KB | Large payload | PASS | +| `test_rust_encode_cpp_decode_with_loss` | Rust → C++ | 10 KB | 2 source symbols dropped, repaired | PASS | +| `test_cpp_encode_rust_decode_with_loss` | C++ → Rust | 10 KB | 2 source symbols dropped, repaired | PASS | +| `test_params_match` | Both | 9 sizes | Parameters identical (100B–100KB) | PASS | +| `test_source_symbols_identical` | Both | 3 sizes | Source symbols byte-identical | PASS | +| `test_4mb_rust_encode_cpp_decode` | Rust → C++ | 4 MB | 4/8/16/32/64 symbols (>=64KB each), SHA-256 verified | PASS | +| `test_4mb_cpp_encode_rust_decode` | C++ → Rust | 4 MB | 4/8/16/32/64 symbols (>=64KB each), SHA-256 verified | PASS | +| `test_4mb_large_symbol_params_match` | Both | 4 MB | Parameters identical for all 5 symbol counts | PASS | +| `test_4mb_large_symbol_source_identical` | Both | 4 MB | Source symbols byte-identical for all 5 symbol counts | PASS | + ## Environment Variables | Variable | Description | Required | @@ -236,6 +258,8 @@ The C++ test node (`compat_test_node`) communicates via JSON over stdin/stdout: {"cmd": "enable_quic"} {"cmd": "send_quic_message", "peer_adnl_id": "HEX", "data": "BASE64"} {"cmd": "send_quic_query", "peer_adnl_id": "HEX", "data": "BASE64", "timeout_ms": 5000} +{"cmd": "raptorq_encode", "data": "BASE64", "symbol_size": 768, "repair_count": 2} +{"cmd": "raptorq_decode", "data_size": 10000, "symbol_size": 768, "symbols_count": 14, "symbols": [{"id": 0, "data": "BASE64"}, ...]} {"cmd": "shutdown"} // Responses: @@ -259,6 +283,7 @@ Tests use different port ranges to avoid conflicts: - `test_quic_transport`: 18000-18099 - `test_quic_overlay`: 18100-18199 - `test_quic_private_overlay`: 18200-18299 +- `test_raptorq`: 19000-19099 ## Troubleshooting @@ -268,7 +293,7 @@ Tests use different port ranges to avoid conflicts: - Verify C++20 compiler support ### Tests Timeout -- Ensure no firewall blocks UDP ports 14000-19000 on localhost +- Ensure no firewall blocks UDP ports 14000-20000 on localhost - Check that no other processes use the same ports - Try increasing sleep durations in tests if running on slow hardware diff --git a/src/node/tests/compat_test/cpp_src/CMakeLists.txt b/src/node/tests/compat_test/cpp_src/CMakeLists.txt index f0f2749..a9c16b8 100644 --- a/src/node/tests/compat_test/cpp_src/CMakeLists.txt +++ b/src/node/tests/compat_test/cpp_src/CMakeLists.txt @@ -56,6 +56,7 @@ target_include_directories(compat_test_node PRIVATE ${TON_SRC_PATH}/rldp ${TON_SRC_PATH}/rldp2 ${TON_SRC_PATH}/fec + ${TON_SRC_PATH}/tdfec ${TON_SRC_PATH}/quic ${TON_SRC_PATH}/metrics ${TON_SRC_PATH}/tddb diff --git a/src/node/tests/compat_test/cpp_src/compat_test_node.cpp b/src/node/tests/compat_test/cpp_src/compat_test_node.cpp index 896ac78..bea1dc7 100644 --- a/src/node/tests/compat_test/cpp_src/compat_test_node.cpp +++ b/src/node/tests/compat_test/cpp_src/compat_test_node.cpp @@ -220,7 +220,11 @@ void CompatTestNode::process_stdin() { void CompatTestNode::handle_command(std::string cmd_line) { if (cmd_line.empty()) return; - LOG(INFO) << "CMD: " << cmd_line; + if (cmd_line.size() <= 250) { + LOG(INFO) << "CMD: " << cmd_line; + } else { + LOG(INFO) << "CMD: " << cmd_line.substr(0, 250) << "... (" << cmd_line.size() << " bytes)"; + } auto json_res = td::json_decode(cmd_line); if (json_res.is_error()) { @@ -288,6 +292,10 @@ void CompatTestNode::handle_command(std::string cmd_line) { cmd_send_quic_message(obj); } else if (cmd == "send_quic_query") { cmd_send_quic_query(obj); + } else if (cmd == "raptorq_encode") { + cmd_raptorq_encode(obj); + } else if (cmd == "raptorq_decode") { + cmd_raptorq_decode(obj); } else if (cmd == "shutdown") { respond("{\"result\": \"shutting_down\"}"); std::_Exit(0); // Force immediate exit @@ -1261,6 +1269,156 @@ void CompatTestNode::cmd_send_quic_query(td::JsonObject &obj) { timeout, std::move(data)); } +void CompatTestNode::cmd_raptorq_encode(td::JsonObject &obj) { + auto data_b64 = get_string(obj, "data"); + auto symbol_size_raw = get_int(obj, "symbol_size", 768); + if (symbol_size_raw <= 0) { + respond_error("symbol_size must be positive"); + return; + } + auto symbol_size = static_cast(symbol_size_raw); + + if (data_b64.empty()) { + respond_error("Missing 'data' (base64)"); + return; + } + + auto data_res = td::base64_decode(data_b64); + if (data_res.is_error()) { + respond_error("Invalid base64 data: " + data_res.error().message().str()); + return; + } + auto data = td::BufferSlice(data_res.move_as_ok()); + + auto encoder = td::fec::RaptorQEncoder::create(std::move(data), symbol_size); + if (!encoder) { + respond_error("Failed to create RaptorQ encoder"); + return; + } + + auto params = encoder->get_parameters(); + + // How many symbols to generate: all source + some repair + auto num_repair_raw = get_int(obj, "repair_count", 0); + auto num_repair = static_cast(num_repair_raw < 0 ? 0 : num_repair_raw); + auto total = params.symbols_count + num_repair; + + // Need precalc to generate repair symbols + if (num_repair > 0) { + encoder->prepare_more_symbols(); + } + + std::ostringstream oss; + oss << "{\"result\": {" + << "\"data_size\": " << params.data_size + << ", \"symbol_size\": " << params.symbol_size + << ", \"symbols_count\": " << params.symbols_count + << ", \"symbols\": ["; + + for (size_t i = 0; i < total; i++) { + auto sym = encoder->gen_symbol(static_cast(i)); + if (i > 0) oss << ", "; + oss << "{\"id\": " << sym.id + << ", \"data\": \"" << td::base64_encode(sym.data.as_slice()) << "\"}"; + } + + oss << "]}}"; + respond(oss.str()); +} + +void CompatTestNode::cmd_raptorq_decode(td::JsonObject &obj) { + auto data_size_raw = get_int(obj, "data_size", 0); + auto symbol_size_raw = get_int(obj, "symbol_size", 768); + auto symbols_count_raw = get_int(obj, "symbols_count", 0); + + if (data_size_raw <= 0 || symbol_size_raw <= 0 || symbols_count_raw <= 0) { + respond_error("Missing required params: data_size, symbol_size, symbols_count"); + return; + } + auto data_size = static_cast(data_size_raw); + auto symbol_size = static_cast(symbol_size_raw); + auto symbols_count = static_cast(symbols_count_raw); + + // Parse symbols array from JSON + // We need to manually walk the JSON array + td::JsonValue *symbols_val = nullptr; + for (auto &kv : obj.field_values_) { + if (kv.first == "symbols") { + symbols_val = &kv.second; + break; + } + } + + if (!symbols_val || symbols_val->type() != td::JsonValue::Type::Array) { + respond_error("Missing or invalid 'symbols' array"); + return; + } + + td::fec::RaptorQEncoder::Parameters params; + params.data_size = data_size; + params.symbol_size = symbol_size; + params.symbols_count = symbols_count; + + auto decoder_res = td::fec::RaptorQDecoder::create(params); + if (decoder_res.is_error()) { + respond_error("Failed to create decoder: " + decoder_res.error().message().str()); + return; + } + auto decoder = decoder_res.move_as_ok(); + + auto &arr = symbols_val->get_array(); + for (auto &elem : arr) { + if (elem.type() != td::JsonValue::Type::Object) { + respond_error("Symbol must be a JSON object"); + return; + } + auto &sym_obj = elem.get_object(); + + td::uint32 sym_id = 0; + std::string sym_data_b64; + for (auto &skv : sym_obj.field_values_) { + if (skv.first == "id") { + if (skv.second.type() == td::JsonValue::Type::Number) { + sym_id = static_cast(td::to_integer(skv.second.get_number())); + } + } else if (skv.first == "data") { + if (skv.second.type() == td::JsonValue::Type::String) { + sym_data_b64 = skv.second.get_string().str(); + } + } + } + + auto sym_data_res = td::base64_decode(sym_data_b64); + if (sym_data_res.is_error()) { + respond_error("Invalid base64 in symbol data"); + return; + } + + td::fec::Symbol sym{sym_id, td::BufferSlice(sym_data_res.move_as_ok())}; + auto status = decoder->add_symbol(std::move(sym)); + if (status.is_error()) { + respond_error("add_symbol failed: " + status.message().str()); + return; + } + } + + if (!decoder->may_try_decode()) { + respond_error("Not enough symbols to decode"); + return; + } + + auto decode_res = decoder->try_decode(false); + if (decode_res.is_error()) { + respond_error("Decode failed: " + decode_res.error().message().str()); + return; + } + + auto decoded = decode_res.move_as_ok(); + auto decoded_b64 = td::base64_encode(decoded.data.as_slice()); + + respond_ok("\"data\": \"" + decoded_b64 + "\""); +} + } // namespace compat_test // ---------- Main ---------- diff --git a/src/node/tests/compat_test/cpp_src/compat_test_node.hpp b/src/node/tests/compat_test/cpp_src/compat_test_node.hpp index 111d5fa..969673f 100644 --- a/src/node/tests/compat_test/cpp_src/compat_test_node.hpp +++ b/src/node/tests/compat_test/cpp_src/compat_test_node.hpp @@ -8,6 +8,7 @@ #include "rldp/rldp.h" #include "rldp2/rldp.h" #include "quic-sender.h" +#include "td/fec/fec.h" #include "keys/keys.hpp" #include "keyring/keyring.h" #include "td/actor/actor.h" @@ -194,6 +195,8 @@ class CompatTestNode : public td::actor::Actor { void cmd_enable_quic(td::JsonObject &obj); void cmd_send_quic_message(td::JsonObject &obj); void cmd_send_quic_query(td::JsonObject &obj); + void cmd_raptorq_encode(td::JsonObject &obj); + void cmd_raptorq_decode(td::JsonObject &obj); // Helpers std::string get_string(td::JsonObject &obj, const std::string &key); diff --git a/src/node/tests/compat_test/src/lib.rs b/src/node/tests/compat_test/src/lib.rs index 30828ef..ef2c6f0 100644 --- a/src/node/tests/compat_test/src/lib.rs +++ b/src/node/tests/compat_test/src/lib.rs @@ -270,10 +270,33 @@ pub enum CppCommand { timeout_ms: i64, }, + #[serde(rename = "raptorq_encode")] + RaptorqEncode { + /// base64-encoded data to encode + data: String, + symbol_size: u32, + repair_count: u32, + }, + + #[serde(rename = "raptorq_decode")] + RaptorqDecode { + data_size: u32, + symbol_size: u32, + symbols_count: u32, + symbols: Vec, + }, + #[serde(rename = "shutdown")] Shutdown, } +/// A single RaptorQ encoded symbol (id + base64 data) +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct EncodedSymbol { + pub id: u32, + pub data: String, // base64 +} + /// Ready response from C++ node #[derive(Debug, Clone, serde::Deserialize)] pub struct ReadyResponse { @@ -311,6 +334,15 @@ pub struct ReceivedMessage { pub timestamp: i32, } +/// Result from RaptorQ encode command +#[derive(Debug, Clone)] +pub struct RaptorqEncodeResult { + pub data_size: u32, + pub symbol_size: u32, + pub symbols_count: u32, + pub symbols: Vec, +} + /// Info about the C++ node #[derive(Debug, Clone)] pub struct NodeInfo { @@ -433,11 +465,20 @@ impl CppTestNode { /// Send a command and get response pub fn send_command(&mut self, cmd: &CppCommand) -> Result { + self.send_command_with_timeout(cmd, DEFAULT_COMMAND_TIMEOUT) + } + + /// Send a command and get response with a custom timeout + pub fn send_command_with_timeout( + &mut self, + cmd: &CppCommand, + timeout: Duration, + ) -> Result { let json = serde_json::to_string(cmd)?; writeln!(self.stdin, "{}", json)?; self.stdin.flush()?; - let line = self.recv_line(DEFAULT_COMMAND_TIMEOUT)?; + let line = self.recv_line(timeout)?; if line.is_empty() { return Err(CompatTestError::InvalidResponse( @@ -451,7 +492,16 @@ impl CppTestNode { /// Extract result value, returning error if response is an error fn expect_result(&mut self, cmd: &CppCommand) -> Result { - let response = self.send_command(cmd)?; + self.expect_result_with_timeout(cmd, DEFAULT_COMMAND_TIMEOUT) + } + + /// Extract result value with a custom timeout + fn expect_result_with_timeout( + &mut self, + cmd: &CppCommand, + timeout: Duration, + ) -> Result { + let response = self.send_command_with_timeout(cmd, timeout)?; match response { CppResponse::Result { result } => Ok(result), CppResponse::Error { error } => Err(CompatTestError::CommandFailed(error)), @@ -856,6 +906,65 @@ impl CppTestNode { .map_err(|e| CompatTestError::InvalidResponse(format!("Invalid base64 answer: {}", e))) } + // ---- RaptorQ ---- + + /// Longer timeout for RaptorQ commands that transfer large base64 payloads. + const RAPTORQ_COMMAND_TIMEOUT: Duration = Duration::from_secs(60); + + /// Encode data using C++ RaptorQ encoder. + /// Returns (params, symbols) where params = (data_size, symbol_size, symbols_count). + pub fn raptorq_encode( + &mut self, + data: &[u8], + symbol_size: u32, + repair_count: u32, + ) -> Result { + let result = self.expect_result_with_timeout( + &CppCommand::RaptorqEncode { data: b64_encode(data), symbol_size, repair_count }, + Self::RAPTORQ_COMMAND_TIMEOUT, + )?; + let data_size = result["data_size"] + .as_u64() + .ok_or_else(|| CompatTestError::InvalidResponse("Missing data_size".into()))? + as u32; + let sym_size = result["symbol_size"] + .as_u64() + .ok_or_else(|| CompatTestError::InvalidResponse("Missing symbol_size".into()))? + as u32; + let symbols_count = result["symbols_count"] + .as_u64() + .ok_or_else(|| CompatTestError::InvalidResponse("Missing symbols_count".into()))? + as u32; + let symbols: Vec = serde_json::from_value(result["symbols"].clone()) + .map_err(|e| CompatTestError::InvalidResponse(format!("Bad symbols: {}", e)))?; + Ok(RaptorqEncodeResult { data_size, symbol_size: sym_size, symbols_count, symbols }) + } + + /// Decode symbols using C++ RaptorQ decoder. + /// Returns decoded data bytes. + pub fn raptorq_decode( + &mut self, + data_size: u32, + symbol_size: u32, + symbols_count: u32, + symbols: &[EncodedSymbol], + ) -> Result> { + let result = self.expect_result_with_timeout( + &CppCommand::RaptorqDecode { + data_size, + symbol_size, + symbols_count, + symbols: symbols.to_vec(), + }, + Self::RAPTORQ_COMMAND_TIMEOUT, + )?; + let data_b64 = result["data"] + .as_str() + .ok_or_else(|| CompatTestError::InvalidResponse("Expected 'data'".to_string()))?; + b64_decode(data_b64) + .map_err(|e| CompatTestError::InvalidResponse(format!("Invalid base64: {}", e))) + } + // ---- Lifecycle ---- /// Shutdown the node diff --git a/src/node/tests/compat_test/tests/test_raptorq.rs b/src/node/tests/compat_test/tests/test_raptorq.rs new file mode 100644 index 0000000..fbecc63 --- /dev/null +++ b/src/node/tests/compat_test/tests/test_raptorq.rs @@ -0,0 +1,628 @@ +/* + * Copyright (C) 2025-2026 RSquad Blockchain Lab. + * + * Licensed under the GNU General Public License v3.0. + * See the LICENSE file in the root of this repository. + * + * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. + */ +//! RaptorQ FEC cross-implementation compatibility tests. +//! +//! Tests that symbols encoded by one implementation (Rust or C++) can be +//! decoded by the other, ensuring the RaptorQ codec is wire-compatible. + +use adnl::{RaptorqDecoder, RaptorqEncoder}; +use compat_test::{skip_if_no_cpp, CppTestNode, EncodedSymbol, TestTimeout}; +use ton_api::ton::fec::type_::RaptorQ as FecTypeRaptorQ; +use ton_block::sha256_digest; + +/// Port range 19000-19099 for raptorq tests +const BASE_PORT: u16 = 19000; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +fn rust_encode( + data: &[u8], + symbol_size: Option, + repair_count: u32, +) -> (FecTypeRaptorQ, Vec) { + let mut encoder = RaptorqEncoder::with_data(data, symbol_size); + let params = encoder.params().clone(); + let source_count = params.symbols_count as u32; + let total = source_count + repair_count; + + let mut symbols = Vec::new(); + let mut seqno = 0u32; + for _ in 0..total { + let prev = seqno; + let chunk = encoder.encode(&mut seqno).expect("encode failed"); + symbols.push(EncodedSymbol { + id: seqno, + data: base64::Engine::encode(&base64::engine::general_purpose::STANDARD, &chunk), + }); + if prev == seqno { + seqno += 1; + } + } + + (params, symbols) +} + +fn rust_decode(params: &FecTypeRaptorQ, symbols: &[EncodedSymbol]) -> Vec { + let mut decoder = RaptorqDecoder::with_params(params.clone()).expect("decoder creation failed"); + for sym in symbols { + let data = base64::Engine::decode(&base64::engine::general_purpose::STANDARD, &sym.data) + .expect("bad base64"); + if let Some(result) = decoder.decode(sym.id, &data) { + return result; + } + } + panic!( + "Rust decoder failed to reconstruct (fed {} symbols, needed {})", + symbols.len(), + params.symbols_count + ); +} + +fn make_test_data(size: usize) -> Vec { + (0..size).map(|i| (i % 251) as u8).collect() +} + +// --------------------------------------------------------------------------- +// Tests: Rust encode -> C++ decode +// --------------------------------------------------------------------------- + +#[test] +fn test_rust_encode_cpp_decode_small() { + skip_if_no_cpp!(); + let _timeout = TestTimeout::new(30); + let mut cpp = CppTestNode::spawn(BASE_PORT).expect("spawn C++ node"); + + let data = make_test_data(500); + let (params, symbols) = rust_encode(&data, None, 0); + + println!( + "Rust encoded: data_size={}, symbol_size={}, symbols_count={}, generated={}", + params.data_size, + params.symbol_size, + params.symbols_count, + symbols.len() + ); + + let decoded = cpp + .raptorq_decode( + params.data_size as u32, + params.symbol_size as u32, + params.symbols_count as u32, + &symbols, + ) + .expect("C++ decode failed"); + + assert_eq!(decoded.len(), data.len(), "decoded size mismatch"); + assert_eq!(decoded, data, "decoded data mismatch"); + println!("OK: Rust encode -> C++ decode ({}B, source-only)", data.len()); +} + +#[test] +fn test_rust_encode_cpp_decode_medium() { + skip_if_no_cpp!(); + let _timeout = TestTimeout::new(30); + let mut cpp = CppTestNode::spawn(BASE_PORT + 1).expect("spawn C++ node"); + + // ~10 KB: will produce multiple source symbols with 768-byte symbol_size + let data = make_test_data(10_000); + let (params, symbols) = rust_encode(&data, None, 2); + + println!( + "Rust encoded: data_size={}, symbol_size={}, symbols_count={}, generated={}", + params.data_size, + params.symbol_size, + params.symbols_count, + symbols.len() + ); + + let decoded = cpp + .raptorq_decode( + params.data_size as u32, + params.symbol_size as u32, + params.symbols_count as u32, + &symbols, + ) + .expect("C++ decode failed"); + + assert_eq!(decoded.len(), data.len(), "decoded size mismatch"); + assert_eq!(decoded, data, "decoded data mismatch"); + println!("OK: Rust encode -> C++ decode ({}B, with repair)", data.len()); +} + +#[test] +fn test_rust_encode_cpp_decode_large() { + skip_if_no_cpp!(); + let _timeout = TestTimeout::new(60); + let mut cpp = CppTestNode::spawn(BASE_PORT + 2).expect("spawn C++ node"); + + // ~100 KB + let data = make_test_data(100_000); + let (params, symbols) = rust_encode(&data, None, 4); + + println!( + "Rust encoded: data_size={}, symbol_size={}, symbols_count={}, generated={}", + params.data_size, + params.symbol_size, + params.symbols_count, + symbols.len() + ); + + let decoded = cpp + .raptorq_decode( + params.data_size as u32, + params.symbol_size as u32, + params.symbols_count as u32, + &symbols, + ) + .expect("C++ decode failed"); + + assert_eq!(decoded.len(), data.len(), "decoded size mismatch"); + assert_eq!(decoded, data, "decoded data mismatch"); + println!("OK: Rust encode -> C++ decode ({}B, large)", data.len()); +} + +// --------------------------------------------------------------------------- +// Tests: C++ encode -> Rust decode +// --------------------------------------------------------------------------- + +#[test] +fn test_cpp_encode_rust_decode_small() { + skip_if_no_cpp!(); + let _timeout = TestTimeout::new(30); + let mut cpp = CppTestNode::spawn(BASE_PORT + 10).expect("spawn C++ node"); + + let data = make_test_data(500); + let result = cpp.raptorq_encode(&data, 768, 0).expect("C++ encode failed"); + + println!( + "C++ encoded: data_size={}, symbol_size={}, symbols_count={}, generated={}", + result.data_size, + result.symbol_size, + result.symbols_count, + result.symbols.len() + ); + + let params = FecTypeRaptorQ { + data_size: result.data_size as i32, + symbol_size: result.symbol_size as i32, + symbols_count: result.symbols_count as i32, + }; + + let decoded = rust_decode(¶ms, &result.symbols); + assert_eq!(decoded.len(), data.len(), "decoded size mismatch"); + assert_eq!(decoded, data, "decoded data mismatch"); + println!("OK: C++ encode -> Rust decode ({}B, source-only)", data.len()); +} + +#[test] +fn test_cpp_encode_rust_decode_medium() { + skip_if_no_cpp!(); + let _timeout = TestTimeout::new(30); + let mut cpp = CppTestNode::spawn(BASE_PORT + 11).expect("spawn C++ node"); + + let data = make_test_data(10_000); + let result = cpp.raptorq_encode(&data, 768, 2).expect("C++ encode failed"); + + println!( + "C++ encoded: data_size={}, symbol_size={}, symbols_count={}, generated={}", + result.data_size, + result.symbol_size, + result.symbols_count, + result.symbols.len() + ); + + let params = FecTypeRaptorQ { + data_size: result.data_size as i32, + symbol_size: result.symbol_size as i32, + symbols_count: result.symbols_count as i32, + }; + + let decoded = rust_decode(¶ms, &result.symbols); + assert_eq!(decoded.len(), data.len(), "decoded size mismatch"); + assert_eq!(decoded, data, "decoded data mismatch"); + println!("OK: C++ encode -> Rust decode ({}B, with repair)", data.len()); +} + +#[test] +fn test_cpp_encode_rust_decode_large() { + skip_if_no_cpp!(); + let _timeout = TestTimeout::new(60); + let mut cpp = CppTestNode::spawn(BASE_PORT + 12).expect("spawn C++ node"); + + let data = make_test_data(100_000); + let result = cpp.raptorq_encode(&data, 768, 4).expect("C++ encode failed"); + + println!( + "C++ encoded: data_size={}, symbol_size={}, symbols_count={}, generated={}", + result.data_size, + result.symbol_size, + result.symbols_count, + result.symbols.len() + ); + + let params = FecTypeRaptorQ { + data_size: result.data_size as i32, + symbol_size: result.symbol_size as i32, + symbols_count: result.symbols_count as i32, + }; + + let decoded = rust_decode(¶ms, &result.symbols); + assert_eq!(decoded.len(), data.len(), "decoded size mismatch"); + assert_eq!(decoded, data, "decoded data mismatch"); + println!("OK: C++ encode -> Rust decode ({}B, large)", data.len()); +} + +// --------------------------------------------------------------------------- +// Tests: repair-only decode (drop some source, keep repair) +// --------------------------------------------------------------------------- + +#[test] +fn test_rust_encode_cpp_decode_with_loss() { + skip_if_no_cpp!(); + let _timeout = TestTimeout::new(30); + let mut cpp = CppTestNode::spawn(BASE_PORT + 20).expect("spawn C++ node"); + + let data = make_test_data(10_000); + let source_count_extra = 4u32; // extra repair symbols + let (params, all_symbols) = rust_encode(&data, None, source_count_extra); + let source_count = params.symbols_count as usize; + + // Drop first 2 source symbols, keep the rest + all repair + let symbols: Vec<_> = all_symbols[2..].to_vec(); + println!( + "Rust encoded {} symbols ({}+{} repair), feeding {} (dropped 2 source) to C++", + all_symbols.len(), + source_count, + source_count_extra, + symbols.len() + ); + + let decoded = cpp + .raptorq_decode( + params.data_size as u32, + params.symbol_size as u32, + params.symbols_count as u32, + &symbols, + ) + .expect("C++ decode failed with loss"); + + assert_eq!(decoded, data, "decoded data mismatch after loss"); + println!("OK: Rust encode -> C++ decode with simulated loss ({}B)", data.len()); +} + +#[test] +fn test_cpp_encode_rust_decode_with_loss() { + skip_if_no_cpp!(); + let _timeout = TestTimeout::new(30); + let mut cpp = CppTestNode::spawn(BASE_PORT + 21).expect("spawn C++ node"); + + let data = make_test_data(10_000); + let result = cpp.raptorq_encode(&data, 768, 4).expect("C++ encode failed"); + let source_count = result.symbols_count as usize; + + // Drop first 2 source symbols, keep the rest + repair + let symbols: Vec<_> = result.symbols[2..].to_vec(); + println!( + "C++ encoded {} symbols ({}+4 repair), feeding {} (dropped 2 source) to Rust", + result.symbols.len(), + source_count, + symbols.len() + ); + + let params = FecTypeRaptorQ { + data_size: result.data_size as i32, + symbol_size: result.symbol_size as i32, + symbols_count: result.symbols_count as i32, + }; + + let decoded = rust_decode(¶ms, &symbols); + assert_eq!(decoded, data, "decoded data mismatch after loss"); + println!("OK: C++ encode -> Rust decode with simulated loss ({}B)", data.len()); +} + +// --------------------------------------------------------------------------- +// Tests: parameter agreement +// --------------------------------------------------------------------------- + +#[test] +fn test_params_match() { + skip_if_no_cpp!(); + let _timeout = TestTimeout::new(30); + let mut cpp = CppTestNode::spawn(BASE_PORT + 30).expect("spawn C++ node"); + + // Verify both implementations agree on encoding parameters for various data sizes + for &data_size in &[100, 500, 768, 769, 1536, 5000, 10000, 50000, 100000] { + let data = make_test_data(data_size); + + let rust_encoder = RaptorqEncoder::with_data(&data, None); + let rust_params = rust_encoder.params(); + + let cpp_result = cpp.raptorq_encode(&data, 768, 0).expect("C++ encode failed"); + + assert_eq!( + rust_params.data_size as u32, cpp_result.data_size, + "data_size mismatch for input size {}", + data_size + ); + assert_eq!( + rust_params.symbol_size as u32, cpp_result.symbol_size, + "symbol_size mismatch for input size {}", + data_size + ); + assert_eq!( + rust_params.symbols_count as u32, cpp_result.symbols_count, + "symbols_count mismatch for input size {}", + data_size + ); + + println!( + "params match for data_size={}: symbols_count={}, symbol_size={}", + data_size, cpp_result.symbols_count, cpp_result.symbol_size + ); + } + println!("OK: all parameter sets match between Rust and C++"); +} + +// --------------------------------------------------------------------------- +// Tests: source symbols are byte-identical +// --------------------------------------------------------------------------- + +#[test] +fn test_source_symbols_identical() { + skip_if_no_cpp!(); + let _timeout = TestTimeout::new(30); + let mut cpp = CppTestNode::spawn(BASE_PORT + 31).expect("spawn C++ node"); + + for &data_size in &[500, 5000, 50000] { + let data = make_test_data(data_size); + + let (rust_params, rust_symbols) = rust_encode(&data, None, 0); + let cpp_result = cpp.raptorq_encode(&data, 768, 0).expect("C++ encode failed"); + + let source_count = rust_params.symbols_count as usize; + assert_eq!(source_count, cpp_result.symbols_count as usize); + + for i in 0..source_count { + let rust_sym_data = base64::Engine::decode( + &base64::engine::general_purpose::STANDARD, + &rust_symbols[i].data, + ) + .expect("bad base64"); + let cpp_sym_data = base64::Engine::decode( + &base64::engine::general_purpose::STANDARD, + &cpp_result.symbols[i].data, + ) + .expect("bad base64"); + + assert_eq!( + rust_sym_data, cpp_sym_data, + "source symbol {} differs for data_size={}", + i, data_size + ); + } + println!("OK: {} source symbols identical for data_size={}", source_count, data_size); + } +} + +// --------------------------------------------------------------------------- +// Tests: 4MB large-symbol bidirectional FEC (symbol sizes >= 64 KB) +// --------------------------------------------------------------------------- + +/// 4 MB — maximum block size used in TON. +const DATA_4MB: usize = 4 * 1024 * 1024; + +/// Symbol counts to test. All yield symbol_size >= 64 KB for 4 MB data: +/// 4 → 1 MB, 8 → 512 KB, 16 → 256 KB, 32 → 128 KB, 64 → 64 KB. +const SYMBOL_COUNTS: &[u32] = &[4, 8, 16, 32, 64]; + +fn make_random_data(size: usize) -> Vec { + // LCG to generate deterministic pseudo-random data (fast, no extra deps) + let mut v = vec![0u8; size]; + let mut state: u64 = 0xDEAD_BEEF_CAFE_BABE; + for chunk in v.chunks_mut(8) { + state = state.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + let bytes = state.to_le_bytes(); + let len = chunk.len(); + chunk.copy_from_slice(&bytes[..len]); + } + v +} + +fn hash_hex(data: &[u8]) -> String { + hex::encode(sha256_digest(data)) +} + +/// symbol_size = ceil(data_len / symbol_count) +fn symbol_size_for(data_len: usize, symbol_count: u32) -> u32 { + ((data_len + symbol_count as usize - 1) / symbol_count as usize) as u32 +} + +#[test] +fn test_4mb_rust_encode_cpp_decode() { + skip_if_no_cpp!(); + let _timeout = TestTimeout::new(300); + let mut cpp = CppTestNode::spawn(BASE_PORT + 40).expect("spawn C++ node"); + + let data = make_random_data(DATA_4MB); + let original_hash = hash_hex(&data); + println!("original SHA-256: {}", original_hash); + + for &sym_count in SYMBOL_COUNTS { + let sym_size = symbol_size_for(data.len(), sym_count); + println!( + "\n--- Rust encode -> C++ decode: {} symbols, symbol_size={} ({} KB) ---", + sym_count, + sym_size, + sym_size / 1024, + ); + + let (params, symbols) = rust_encode(&data, Some(sym_size), 0); + assert_eq!( + params.symbols_count, sym_count as i32, + "unexpected symbols_count for sym_size={}", + sym_size, + ); + println!( + "Rust encoded: data_size={}, symbol_size={}, symbols_count={}", + params.data_size, params.symbol_size, params.symbols_count, + ); + + let decoded = cpp + .raptorq_decode( + params.data_size as u32, + params.symbol_size as u32, + params.symbols_count as u32, + &symbols, + ) + .expect("C++ decode failed"); + + let decoded_hash = hash_hex(&decoded); + println!("decoded SHA-256: {}", decoded_hash); + assert_eq!( + original_hash, decoded_hash, + "SHA-256 mismatch for {} symbols (sym_size={}): Rust encode -> C++ decode", + sym_count, sym_size, + ); + println!("OK: hash match ({} symbols)", sym_count); + } +} + +#[test] +fn test_4mb_cpp_encode_rust_decode() { + skip_if_no_cpp!(); + let _timeout = TestTimeout::new(300); + let mut cpp = CppTestNode::spawn(BASE_PORT + 41).expect("spawn C++ node"); + + let data = make_random_data(DATA_4MB); + let original_hash = hash_hex(&data); + println!("original SHA-256: {}", original_hash); + + for &sym_count in SYMBOL_COUNTS { + let sym_size = symbol_size_for(data.len(), sym_count); + println!( + "\n--- C++ encode -> Rust decode: {} symbols, symbol_size={} ({} KB) ---", + sym_count, + sym_size, + sym_size / 1024, + ); + + let result = cpp.raptorq_encode(&data, sym_size, 0).expect("C++ encode failed"); + assert_eq!( + result.symbols_count, sym_count, + "unexpected symbols_count for sym_size={}", + sym_size, + ); + println!( + "C++ encoded: data_size={}, symbol_size={}, symbols_count={}", + result.data_size, result.symbol_size, result.symbols_count, + ); + + let params = FecTypeRaptorQ { + data_size: result.data_size as i32, + symbol_size: result.symbol_size as i32, + symbols_count: result.symbols_count as i32, + }; + + let decoded = rust_decode(¶ms, &result.symbols); + let decoded_hash = hash_hex(&decoded); + println!("decoded SHA-256: {}", decoded_hash); + assert_eq!( + original_hash, decoded_hash, + "SHA-256 mismatch for {} symbols (sym_size={}): C++ encode -> Rust decode", + sym_count, sym_size, + ); + println!("OK: hash match ({} symbols)", sym_count); + } +} + +#[test] +fn test_4mb_large_symbol_params_match() { + skip_if_no_cpp!(); + let _timeout = TestTimeout::new(300); + let mut cpp = CppTestNode::spawn(BASE_PORT + 42).expect("spawn C++ node"); + + let data = make_random_data(DATA_4MB); + + for &sym_count in SYMBOL_COUNTS { + let sym_size = symbol_size_for(data.len(), sym_count); + + let rust_encoder = RaptorqEncoder::with_data(&data, Some(sym_size)); + let rp = rust_encoder.params(); + + let cpp_result = cpp.raptorq_encode(&data, sym_size, 0).expect("C++ encode failed"); + + assert_eq!( + rp.data_size as u32, cpp_result.data_size, + "data_size mismatch for {} symbols", + sym_count, + ); + assert_eq!( + rp.symbol_size as u32, cpp_result.symbol_size, + "symbol_size mismatch for {} symbols", + sym_count, + ); + assert_eq!( + rp.symbols_count as u32, cpp_result.symbols_count, + "symbols_count mismatch for {} symbols", + sym_count, + ); + println!( + "OK: params match for {} symbols: symbol_size={}, symbols_count={}", + sym_count, cpp_result.symbol_size, cpp_result.symbols_count, + ); + } +} + +#[test] +fn test_4mb_large_symbol_source_identical() { + skip_if_no_cpp!(); + let _timeout = TestTimeout::new(300); + let mut cpp = CppTestNode::spawn(BASE_PORT + 43).expect("spawn C++ node"); + + let data = make_random_data(DATA_4MB); + + for &sym_count in SYMBOL_COUNTS { + let sym_size = symbol_size_for(data.len(), sym_count); + let (rust_params, rust_symbols) = rust_encode(&data, Some(sym_size), 0); + let cpp_result = cpp.raptorq_encode(&data, sym_size, 0).expect("C++ encode failed"); + + let n = rust_params.symbols_count as usize; + assert_eq!(n, cpp_result.symbols_count as usize); + + for i in 0..n { + let rs = base64::Engine::decode( + &base64::engine::general_purpose::STANDARD, + &rust_symbols[i].data, + ) + .expect("bad base64"); + let cs = base64::Engine::decode( + &base64::engine::general_purpose::STANDARD, + &cpp_result.symbols[i].data, + ) + .expect("bad base64"); + + let rs_hash = hash_hex(&rs); + let cs_hash = hash_hex(&cs); + assert_eq!( + rs_hash, cs_hash, + "source symbol {} differs for {} symbols (sym_size={}): rust={} cpp={}", + i, sym_count, sym_size, rs_hash, cs_hash, + ); + } + println!( + "OK: {} source symbols identical for {} symbols (sym_size={} = {} KB)", + n, + sym_count, + sym_size, + sym_size / 1024, + ); + } +}