Skip to content

Commit 2f9529d

Browse files
authored
Merge pull request #68 from RSquad/raptorq_symbol
Support non-standard RaptorQ symbol size (> 65536)
2 parents 260a764 + 1f0453d commit 2f9529d

File tree

16 files changed

+1071
-48
lines changed

16 files changed

+1071
-48
lines changed

src/adnl/raptorq/src/base.rs

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ impl EncodingPacket {
104104
#[cfg_attr(feature = "serde_support", derive(Serialize, Deserialize))]
105105
pub struct ObjectTransmissionInformation {
106106
transfer_length: u64, // Limited to u40
107-
symbol_size: u16,
107+
symbol_size: u32,
108108
num_source_blocks: u8,
109109
num_sub_blocks: u16,
110110
symbol_alignment: u8,
@@ -113,14 +113,14 @@ pub struct ObjectTransmissionInformation {
113113
impl ObjectTransmissionInformation {
114114
pub fn new(
115115
transfer_length: u64,
116-
symbol_size: u16,
116+
symbol_size: u32,
117117
source_blocks: u8,
118118
sub_blocks: u16,
119119
alignment: u8,
120120
) -> ObjectTransmissionInformation {
121121
// See errata (https://www.rfc-editor.org/errata/eid5548)
122122
assert!(transfer_length <= 942574504275);
123-
assert_eq!(symbol_size % alignment as u16, 0);
123+
assert_eq!(symbol_size % alignment as u32, 0);
124124
// See section 4.4.1.2. "These parameters MUST be set so that ceil(ceil(F/T)/Z) <= K'_max."
125125

126126
if (symbol_size != 0) && (source_blocks != 0) {
@@ -140,20 +140,28 @@ impl ObjectTransmissionInformation {
140140
}
141141
}
142142

143+
#[deprecated(note = "\
144+
RFC 5052 wire format, only encodes symbol_size as u16. \
145+
Not used in TON — TL serialization is used instead\
146+
")]
143147
pub fn deserialize(data: &[u8; 12]) -> ObjectTransmissionInformation {
144148
ObjectTransmissionInformation {
145149
transfer_length: ((data[0] as u64) << 32)
146150
+ ((data[1] as u64) << 24)
147151
+ ((data[2] as u64) << 16)
148152
+ ((data[3] as u64) << 8)
149153
+ (data[4] as u64),
150-
symbol_size: ((data[6] as u16) << 8) + data[7] as u16,
154+
symbol_size: ((data[6] as u32) << 8) + data[7] as u32,
151155
num_source_blocks: data[8],
152156
num_sub_blocks: ((data[9] as u16) << 8) + data[10] as u16,
153157
symbol_alignment: data[11],
154158
}
155159
}
156160

161+
#[deprecated(note = "\
162+
RFC 5052 wire format, only encodes symbol_size as u16. \
163+
Not used in TON — TL serialization is used instead\
164+
")]
157165
pub fn serialize(&self) -> [u8; 12] {
158166
[
159167
((self.transfer_length >> 32) & 0xFF) as u8,
@@ -175,7 +183,7 @@ impl ObjectTransmissionInformation {
175183
self.transfer_length
176184
}
177185

178-
pub fn symbol_size(&self) -> u16 {
186+
pub fn symbol_size(&self) -> u32 {
179187
self.symbol_size
180188
}
181189

@@ -196,9 +204,9 @@ impl ObjectTransmissionInformation {
196204
max_packet_size: u16,
197205
decoder_memory_requirement: u64,
198206
) -> ObjectTransmissionInformation {
199-
let alignment = 8;
207+
let alignment: u16 = 8;
200208
assert!(max_packet_size >= alignment);
201-
let symbol_size = max_packet_size - (max_packet_size % alignment);
209+
let symbol_size = (max_packet_size - (max_packet_size % alignment)) as u32;
202210
let sub_symbol_size = 8;
203211

204212
let kt = int_div_ceil(transfer_length, symbol_size as u64);

src/adnl/raptorq/src/decoder.rs

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ impl Decoder {
119119
#[cfg_attr(feature = "serde_support", derive(Serialize, Deserialize))]
120120
pub struct SourceBlockDecoder {
121121
source_block_id: u8,
122-
symbol_size: u16,
122+
symbol_size: u32,
123123
num_sub_blocks: u16,
124124
symbol_alignment: u8,
125125
source_block_symbols: u32,
@@ -137,7 +137,7 @@ impl SourceBlockDecoder {
137137
note = "Use the new2() function instead. In version 2.0, that function will replace this one"
138138
)]
139139
#[cfg(feature = "std")]
140-
pub fn new(source_block_id: u8, symbol_size: u16, block_length: u64) -> SourceBlockDecoder {
140+
pub fn new(source_block_id: u8, symbol_size: u32, block_length: u64) -> SourceBlockDecoder {
141141
let config = ObjectTransmissionInformation::new(0, symbol_size, 0, 1, 1);
142142
SourceBlockDecoder::new2(source_block_id, &config, block_length)
143143
}
@@ -175,10 +175,8 @@ impl SourceBlockDecoder {
175175
}
176176

177177
fn unpack_sub_blocks(&self, result: &mut [u8], symbol: &Symbol, symbol_index: usize) {
178-
let (tl, ts, nl, ns) = partition(
179-
(self.symbol_size / self.symbol_alignment as u16) as u32,
180-
self.num_sub_blocks,
181-
);
178+
let (tl, ts, nl, ns) =
179+
partition(self.symbol_size / self.symbol_alignment as u32, self.num_sub_blocks);
182180

183181
let mut symbol_offset = 0;
184182
let mut sub_block_offset = 0;
@@ -283,7 +281,7 @@ impl SourceBlockDecoder {
283281

284282
let mut encoded_indices = vec![];
285283
// See section 5.3.3.4.2. There are S + H zero symbols to start the D vector
286-
let mut d = vec![Symbol::zero(self.symbol_size); s + h];
284+
let mut d = vec![Symbol::zero(self.symbol_size as usize); s + h];
287285
for (i, source) in self.source_symbols.iter().enumerate() {
288286
if let Some(symbol) = source {
289287
encoded_indices.push(i as u32);
@@ -294,7 +292,7 @@ impl SourceBlockDecoder {
294292
// Append the extended padding symbols
295293
for i in self.source_block_symbols..num_extended_symbols {
296294
encoded_indices.push(i);
297-
d.push(Symbol::zero(self.symbol_size));
295+
d.push(Symbol::zero(self.symbol_size as usize));
298296
}
299297

300298
for repair_packet in self.repair_packets.iter() {
@@ -328,7 +326,7 @@ impl SourceBlockDecoder {
328326
sys_index: u32,
329327
p1: u32,
330328
) -> Symbol {
331-
let mut rebuilt = Symbol::zero(self.symbol_size);
329+
let mut rebuilt = Symbol::zero(self.symbol_size as usize);
332330
let tuple = intermediate_tuple(source_symbol_id, lt_symbols, sys_index, p1);
333331

334332
for i in enc_indices(tuple, lt_symbols, pi_symbols, p1) {

src/adnl/raptorq/src/encoder.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ impl SourceBlockEncoder {
186186
since = "1.3.0",
187187
note = "Use the new2() function instead. In version 2.0, that function will replace this one"
188188
)]
189-
pub fn new(source_block_id: u8, symbol_size: u16, data: &[u8]) -> SourceBlockEncoder {
189+
pub fn new(source_block_id: u8, symbol_size: u32, data: &[u8]) -> SourceBlockEncoder {
190190
let config = ObjectTransmissionInformation::new(0, symbol_size, 0, 1, 1);
191191
SourceBlockEncoder::new2(source_block_id, &config, data)
192192
}
@@ -196,7 +196,7 @@ impl SourceBlockEncoder {
196196
if config.sub_blocks() > 1 {
197197
let mut symbols = vec![vec![]; data.len() / config.symbol_size() as usize];
198198
let (tl, ts, nl, ns) = partition(
199-
(config.symbol_size() / config.symbol_alignment() as u16) as u32,
199+
config.symbol_size() / config.symbol_alignment() as u32,
200200
config.sub_blocks(),
201201
);
202202
// Divide the block into sub-blocks and then concatenate the sub-symbols into symbols
@@ -247,7 +247,7 @@ impl SourceBlockEncoder {
247247
)]
248248
pub fn with_encoding_plan(
249249
source_block_id: u8,
250-
symbol_size: u16,
250+
symbol_size: u32,
251251
data: &[u8],
252252
plan: &SourceBlockEncodingPlan,
253253
) -> SourceBlockEncoder {

src/adnl/raptorq/src/octets.rs

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -633,10 +633,7 @@ unsafe fn store_neon(ptr: *mut uint8x16_t, value: uint8x16_t) {
633633
#[cfg(target_arch = "arm")]
634634
use std::arch::arm::*;
635635

636-
// TODO: replace with vst1q_u8 when it's supported
637-
let reinterp = vreinterpretq_u64_u8(value);
638-
*(ptr as *mut u64) = vgetq_lane_u64(reinterp, 0);
639-
*(ptr as *mut u64).add(1) = vgetq_lane_u64(reinterp, 1);
636+
vst1q_u8(ptr as *mut u8, value);
640637
}
641638

642639
#[cfg(all(target_arch = "aarch64", feature = "std"))]

src/adnl/src/overlay/broadcast.rs

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1675,8 +1675,6 @@ pub(crate) struct BroadcastTwostepFecProtocol {
16751675
}
16761676

16771677
impl BroadcastTwostepFecProtocol {
1678-
const MAX_PART_SIZE: usize = 65536;
1679-
16801678
pub(crate) fn for_recv() -> Self {
16811679
Self { extra: None, send_ctx: None }
16821680
}
@@ -1690,9 +1688,6 @@ impl BroadcastTwostepFecProtocol {
16901688
}
16911689
let k = ((neighbours as usize) * 2 - 2) / 3;
16921690
let part_size = (data.len() + k - 1) / k;
1693-
if part_size >= Self::MAX_PART_SIZE {
1694-
fail!("Too big part size {part_size} in {} broadcast", Self::broadcast_type());
1695-
}
16961691
let ctx = BroadcastTwostepSendContext { neighbours, part_size };
16971692
Ok(Self { extra: Some(extra), send_ctx: Some(ctx) })
16981693
}
@@ -1819,7 +1814,7 @@ impl BroadcastProtocol<BroadcastTwostepFec> for BroadcastTwostepFecProtocol {
18191814
bcast_id: *bcast_id,
18201815
data_hash: sha256_digest(ctx.data.object),
18211816
date,
1822-
encoder: RaptorqEncoder::with_data(ctx.data.object, Some(send_ctx.part_size as u16)),
1817+
encoder: RaptorqEncoder::with_data(ctx.data.object, Some(send_ctx.part_size as u32)),
18231818
extra: self.extra.take().unwrap_or_default(),
18241819
flags: ctx.flags,
18251820
seqno: 0,

src/adnl/src/rldp/recv.rs

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -59,12 +59,8 @@ impl RaptorqDecoder {
5959
/// Construct with parameters
6060
pub fn with_params(params: FecTypeRaptorQ) -> Result<Self> {
6161
const MAX_SOURCE_SYMBOLS: i32 = 56_403; // K'_max per RFC 6330 §5.1.2
62-
if (params.symbol_size <= 0) || (params.symbol_size > u16::MAX as i32) {
63-
fail!(
64-
"Invalid FEC params: symbol_size must be in 1..={}, got {}",
65-
u16::MAX,
66-
params.symbol_size
67-
);
62+
if params.symbol_size <= 0 {
63+
fail!("Invalid FEC params: symbol_size must be > 0, got {}", params.symbol_size);
6864
}
6965
if params.data_size <= 0 {
7066
fail!("Invalid FEC params: data_size must be > 0, got {}", params.data_size);
@@ -77,12 +73,15 @@ impl RaptorqDecoder {
7773
} else {
7874
// Two-step broadcast case: symbol_size was set by the sender without alignment
7975
// rounding (alignment=1). Use the same config as the encoder.
76+
// symbol_size can exceed u16::MAX for large blocks with few validators
77+
// (matches C++ behaviour which uses size_t for symbol_size).
8078
//
8179
// With source_blocks=1, raptorq asserts ceil(data_size/symbol_size) <=
8280
// MAX_SOURCE_SYMBOLS_PER_BLOCK (K'_max = 56403, RFC 6330 §5.1.2).
8381
// Validate before calling to prevent a panic on malformed network messages.
84-
let source_symbols = (params.data_size + params.symbol_size - 1) / params.symbol_size;
85-
if source_symbols > MAX_SOURCE_SYMBOLS {
82+
let source_symbols = (params.data_size as i64 + params.symbol_size as i64 - 1)
83+
/ params.symbol_size as i64;
84+
if source_symbols > MAX_SOURCE_SYMBOLS as i64 {
8685
fail!(
8786
"Invalid FEC params: source symbol count {source_symbols} \
8887
exceeds raptorq limit {MAX_SOURCE_SYMBOLS} (data_size={}, symbol_size={})",
@@ -92,7 +91,7 @@ impl RaptorqDecoder {
9291
}
9392
raptorq::ObjectTransmissionInformation::new(
9493
params.data_size as u64,
95-
params.symbol_size as u16,
94+
params.symbol_size as u32,
9695
1,
9796
1,
9897
1,

src/adnl/src/rldp/send.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ pub struct RaptorqEncoder {
4444

4545
impl RaptorqEncoder {
4646
/// Construct over data
47-
pub fn with_data(data: &[u8], symbol: Option<u16>) -> Self {
47+
pub fn with_data(data: &[u8], symbol: Option<u32>) -> Self {
4848
let engine =
4949
if let Some(symbol_size) = symbol.filter(|&s| s as usize != Constraints::SYMBOL) {
5050
// symbol_size is set for the two-step FEC broadcast case where data is
@@ -59,8 +59,8 @@ impl RaptorqEncoder {
5959
// and k is bounded by the neighbour count (typically a few dozen, at most
6060
// ~130), which is far below K'_max = 56403 (RFC 6330 max symbols per block).
6161
//
62-
// sub_blocks=1: the data fits comfortably in memory since part_size < 65536
63-
// and kt <= k << K'_max, so no sub-block splitting is needed.
62+
// sub_blocks=1: the data fits comfortably in memory since
63+
// kt <= k << K'_max, so no sub-block splitting is needed.
6464
let config = raptorq::ObjectTransmissionInformation::new(
6565
data.len() as u64,
6666
symbol_size,

src/adnl/tests/test_overlay.rs

Lines changed: 92 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1627,7 +1627,7 @@ async fn test_overlay_semiprivate() -> Result<()> {
16271627
fn test_overlay_raptorq() {
16281628
use rand::{seq::SliceRandom, SeedableRng};
16291629

1630-
fn run(symbol: Option<u16>) {
1630+
fn run(symbol: Option<u32>) {
16311631
let seed: u64 = rand::random();
16321632
println!("test_overlay_raptorq seed: {seed}");
16331633
let mut rng = rand::rngs::StdRng::seed_from_u64(seed);
@@ -1778,3 +1778,94 @@ fn test_overlay_raptorq() {
17781778
println!("--- symbol=Some(771) (alignment=1) ---");
17791779
run(Some(771));
17801780
}
1781+
1782+
/// Test that RaptorQ encode/decode works with symbol_size > 65535 (u16 limit).
1783+
/// This matches the C++ behaviour where symbol_size is size_t.
1784+
/// Simulates TwostepFec for 800KB data with 10 parties:
1785+
/// k = (10*2-2)/3 = 6, part_size = ceil(819200/6) = 136534
1786+
#[test]
1787+
fn test_raptorq_large_symbol_size() {
1788+
use adnl::{RaptorqDecoder, RaptorqEncoder};
1789+
use rand::Rng;
1790+
1791+
const DATA_SIZE: usize = 800 * 1024;
1792+
1793+
for num_parties in [5u32, 10, 20] {
1794+
let k = ((num_parties as usize) * 2 - 2) / 3;
1795+
let part_size = (DATA_SIZE + k - 1) / k;
1796+
1797+
println!(
1798+
"--- parties={num_parties}, k={k}, part_size={part_size} (>65535: {}) ---",
1799+
part_size > 65535
1800+
);
1801+
1802+
// Generate random data
1803+
let mut rng = rand::thread_rng();
1804+
let data: Vec<u8> = (0..DATA_SIZE).map(|_| rng.gen()).collect();
1805+
1806+
// Encode with large symbol_size
1807+
let mut encoder = RaptorqEncoder::with_data(&data, Some(part_size as u32));
1808+
let params = encoder.params().clone();
1809+
println!(
1810+
" params: data_size={}, symbol_size={}, symbols_count={}",
1811+
params.data_size, params.symbol_size, params.symbols_count
1812+
);
1813+
assert_eq!(params.data_size, DATA_SIZE as i32);
1814+
assert_eq!(params.symbol_size, part_size as i32);
1815+
1816+
// Collect source + repair symbols
1817+
let source_count = params.symbols_count as usize;
1818+
let mut packets: Vec<(u32, Vec<u8>)> = Vec::new();
1819+
let mut seqno = 0u32;
1820+
for _ in 0..(source_count * 3 / 2) {
1821+
let chunk = encoder.encode(&mut seqno).unwrap();
1822+
packets.push((seqno, chunk));
1823+
seqno += 1;
1824+
}
1825+
assert!(
1826+
packets.len() >= source_count,
1827+
"Not enough packets generated: {} < {source_count}",
1828+
packets.len()
1829+
);
1830+
1831+
// Decode using exactly source_count symbols (minimum required)
1832+
let mut decoder = RaptorqDecoder::with_params(params.clone())
1833+
.expect("decoder creation must succeed for large symbol_size");
1834+
1835+
let mut decoded = None;
1836+
for (seq, chunk) in packets.iter().take(source_count + 2) {
1837+
if let Some(result) = decoder.decode(*seq, chunk) {
1838+
decoded = Some(result);
1839+
break;
1840+
}
1841+
}
1842+
1843+
let result = decoded.expect("decode must succeed");
1844+
assert_eq!(result.len(), DATA_SIZE, "decoded size mismatch");
1845+
assert_eq!(result, data, "decoded data mismatch");
1846+
println!(" OK: encode/decode verified for symbol_size={part_size}");
1847+
}
1848+
}
1849+
1850+
/// Verify that the RaptorQ encoder produces valid FEC symbols for large
1851+
/// part_size values (>65535) that match the C++ TwostepFec behaviour.
1852+
#[test]
1853+
fn test_twostep_fec_encoder_large_symbols() {
1854+
let data = vec![0x42u8; 800 * 1024];
1855+
for neighbours in [5u32, 10, 20] {
1856+
let k = ((neighbours as usize) * 2 - 2) / 3;
1857+
let part_size = (data.len() + k - 1) / k;
1858+
1859+
let mut encoder = RaptorqEncoder::with_data(&data, Some(part_size as u32));
1860+
let params = encoder.params().clone();
1861+
assert_eq!(params.data_size, data.len() as i32);
1862+
assert_eq!(params.symbol_size, part_size as i32);
1863+
1864+
// Generate one symbol per neighbour (like broadcast-twostep.cpp)
1865+
let mut seqno = 0u32;
1866+
for _ in 0..neighbours {
1867+
let chunk = encoder.encode(&mut seqno).unwrap();
1868+
assert_eq!(chunk.len(), part_size, "symbol size mismatch");
1869+
}
1870+
}
1871+
}

src/node/tests/compat_test/Cargo.toml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,3 +77,7 @@ path = "tests/test_quic_transport.rs"
7777
[[test]]
7878
name = "test_quic_overlay"
7979
path = "tests/test_quic_overlay.rs"
80+
81+
[[test]]
82+
name = "test_raptorq"
83+
path = "tests/test_raptorq.rs"

src/node/tests/compat_test/Makefile

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,13 @@ BUILD_DIR := $(ROOT_DIR)/build
1212
CPP_BUILD_DIR ?= $(BUILD_DIR)/cpp
1313
RUST_TARGET_DIR := $(BUILD_DIR)/rust
1414

15+
# Resolve CPP_SRC_PATH to absolute to avoid breakage when CMake runs from build dir
16+
CPP_SRC_PATH_ORIG := $(CPP_SRC_PATH)
17+
CPP_SRC_PATH := $(shell cd "$(CPP_SRC_PATH)" 2>/dev/null && pwd)
18+
ifeq ($(CPP_SRC_PATH),)
19+
$(error CPP_SRC_PATH "$(CPP_SRC_PATH_ORIG)" is not a valid directory)
20+
endif
21+
1522
# C++ source files location
1623
CPP_TEST_SRC := $(ROOT_DIR)/cpp_src
1724

0 commit comments

Comments
 (0)