From 04cd7a00e26f2c186a1b2b0c4b844e71c9980e80 Mon Sep 17 00:00:00 2001 From: Slava Date: Wed, 8 Apr 2026 08:29:31 +0300 Subject: [PATCH] Updates for Simplex implementation --- src/adnl/src/overlay/broadcast.rs | 12 +- src/adnl/src/overlay/mod.rs | 170 +- src/adnl/src/quic/mod.rs | 936 +++- src/adnl/tests/test_overlay.rs | 5 +- src/node/consensus-common/src/adnl_overlay.rs | 112 +- src/node/consensus-common/src/lib.rs | 51 +- .../src/tests/test_adnl_overlay.rs | 51 + src/node/simplex/CHANGELOG.md | 7 + src/node/simplex/README.md | 76 +- src/node/simplex/src/lib.rs | 109 +- src/node/simplex/src/receiver.rs | 1166 +++-- src/node/simplex/src/session.rs | 101 +- src/node/simplex/src/session_processor.rs | 3912 +++++++---------- src/node/simplex/src/simplex_state.rs | 628 ++- src/node/simplex/src/startup_recovery.rs | 315 +- .../src/tests/test_candidate_resolver.rs | 67 + src/node/simplex/src/tests/test_database.rs | 43 + src/node/simplex/src/tests/test_receiver.rs | 383 +- src/node/simplex/src/tests/test_restart.rs | 497 +-- .../src/tests/test_session_processor.rs | 2741 +++++++++--- .../simplex/src/tests/test_simplex_state.rs | 1252 ++++-- .../simplex/src/tests/test_slot_bounds.rs | 114 +- src/node/simplex/tests/test_collation.rs | 99 +- src/node/simplex/tests/test_consensus.rs | 739 ++-- src/node/simplex/tests/test_restart.rs | 214 +- src/node/simplex/tests/test_validation.rs | 67 +- src/node/src/config.rs | 44 +- src/node/src/engine_operations.rs | 40 +- src/node/src/engine_traits.rs | 23 +- src/node/src/network/custom_overlay_client.rs | 2 +- src/node/src/network/full_node_overlays.rs | 4 +- src/node/src/network/node_network.rs | 242 +- .../tests/test_node_network_validator_list.rs | 20 + src/node/src/validator/consensus.rs | 33 +- src/node/src/validator/fabric.rs | 1 - .../src/validator/tests/test_session_id.rs | 229 +- .../tests/test_validator_session_listener.rs | 78 + src/node/src/validator/validator_group.rs | 336 +- src/node/src/validator/validator_manager.rs | 434 +- .../validator/validator_session_listener.rs | 106 +- .../tests/test_run_net/test_run_net_ci.sh | 7 + .../test_accelerated_consensus_session.rs | 18 - .../tests/test_fast_session.rs | 11 +- 43 files changed, 9234 insertions(+), 6261 deletions(-) create mode 100644 src/node/consensus-common/src/tests/test_adnl_overlay.rs create mode 100644 src/node/src/validator/tests/test_validator_session_listener.rs diff --git a/src/adnl/src/overlay/broadcast.rs b/src/adnl/src/overlay/broadcast.rs index 1114119..2d5c752 100644 --- a/src/adnl/src/overlay/broadcast.rs +++ b/src/adnl/src/overlay/broadcast.rs @@ -1946,16 +1946,16 @@ impl BroadcastParsed for BroadcastTwostepSimple { } pub(crate) struct BroadcastTwostepSimpleProtocol { - big_data: bool, extra: Option>, + reliable: bool, } impl BroadcastTwostepSimpleProtocol { - pub(crate) fn for_recv(big_data: bool) -> Self { - Self { big_data, extra: None } + pub(crate) fn for_recv(reliable: bool) -> Self { + Self { reliable, extra: None } } - pub(crate) fn for_send(big_data: bool, extra: Vec) -> Self { - Self { big_data, extra: Some(extra) } + pub(crate) fn for_send(reliable: bool, extra: Vec) -> Self { + Self { reliable, extra: Some(extra) } } fn calc_to_sign(bcast_id: BroadcastId, data: &[u8]) -> Result> { let to_sign = @@ -1992,7 +1992,7 @@ impl BroadcastProtocol for BroadcastTwostepSimpleProtoco } fn send_method(&self) -> BroadcastSendMethod { - if self.big_data { + if self.reliable { BroadcastSendMethod::QuicOrRldp } else { BroadcastSendMethod::Fast diff --git a/src/adnl/src/overlay/mod.rs b/src/adnl/src/overlay/mod.rs index 2399694..0a9fabd 100644 --- a/src/adnl/src/overlay/mod.rs +++ b/src/adnl/src/overlay/mod.rs @@ -377,6 +377,10 @@ impl OverlayType { } } + fn quic_requested(&self) -> bool { + matches!(self, OverlayType::Private { use_quic: true, .. }) + } + fn calc_message_prefix(&self, overlay_id: &OverlayShortId) -> Result> { match self { Self::CertifiedMembers { certificate, .. } => serialize_boxed( @@ -485,6 +489,8 @@ declare_counted!( nodes: lockfree::map::Map, NodeObject>, overlay_id: Arc, owned_broadcasts: lockfree::map::Map, + // Peers waiting for ADNL address resolution before being added to known_peers + pending_peers: lockfree::queue::Queue>, purge_broadcasts: lockfree::queue::Queue, purge_broadcasts_count: AtomicU32, queue_one_time_broadcasts: tokio::sync::mpsc::UnboundedSender<(BroadcastId, Instant)>, @@ -998,6 +1004,15 @@ impl Overlay { .map_err(|e| error!("Error putting one time broadcast into monitoring queue: {e}")) } + fn try_add_peer(&self, our_key: &Arc, peer: &Arc) -> Result { + if self.adnl.have_peer(our_key, peer)? { + self.known_peers.add(peer)?; + Ok(true) + } else { + Ok(false) + } + } + fn update_neighbours(&self, n: u32) -> Result<()> { if self.overlay_type.is_private() { let n = min(self.known_peers.all().count(), n); @@ -1428,31 +1443,8 @@ impl OverlayNode { self.add_typed_private_overlay(overlay_type, params, peers) } - /// Add semiprivate overlay - pub fn add_semiprivate_overlay( - &self, - params: OverlayParams, - overlay_key: Option<&Arc>, - root_members: &[Arc], - certificate: Option, - max_slaves: usize, - ) -> Result { - let mut root_members_full = HashMap::with_capacity(root_members.len()); - for member in root_members { - root_members_full.insert(member.clone(), lockfree::map::Map::new()); - } - let overlay_type = OverlayType::CertifiedMembers { - root_members: root_members_full, - max_slaves, - bcast_prefix: OverlayUtils::calc_message_prefix(params.overlay_id)?, - certificate, - key: overlay_key.cloned(), - }; - self.add_typed_private_overlay(overlay_type, params, root_members) - } - - /// Add private overlay peers - pub fn add_private_peers( + /// Add private peers to ADNL layer + pub fn add_private_peers_to_adnl( &self, local_adnl_key: &Arc, peers: Vec<(IpAddress, Option, Arc)>, @@ -1466,6 +1458,15 @@ impl OverlayNode { Ok(ret) } + /// Add private peers to the overlay + pub fn add_private_peers_to_overlay( + &self, + overlay_id: &Arc, + peers: &[Arc], + ) -> Result { + self.add_peers_to_overlay(overlay_id, peers, "Cannot get overlay to add peers") + } + /// Add public overlay peer pub fn add_public_peer, N2: Borrow>( &self, @@ -1551,18 +1552,27 @@ impl OverlayNode { Ok(Some(ret)) } - fn calc_src_key_for_broadcast<'a>( - &'a self, - overlay: &'a Overlay, - src_key: Option<&'a Arc>, - ) -> &'a Arc { - if let Some(source) = src_key { - source - } else if let Some(key) = overlay.overlay_key() { - key - } else { - &self.node_key + /// Add semiprivate overlay + pub fn add_semiprivate_overlay( + &self, + params: OverlayParams, + overlay_key: Option<&Arc>, + root_members: &[Arc], + certificate: Option, + max_slaves: usize, + ) -> Result { + let mut root_members_full = HashMap::with_capacity(root_members.len()); + for member in root_members { + root_members_full.insert(member.clone(), lockfree::map::Map::new()); } + let overlay_type = OverlayType::CertifiedMembers { + root_members: root_members_full, + max_slaves, + bcast_prefix: OverlayUtils::calc_message_prefix(params.overlay_id)?, + certificate, + key: overlay_key.cloned(), + }; + self.add_typed_private_overlay(overlay_type, params, root_members) } /// Broadcast message @@ -1617,10 +1627,11 @@ impl OverlayNode { }; let neighbours = overlay.calc_broadcast_twostep_neighbours(); let big_data = data.object.len() >= Self::MIN_BYTES_FEC_TWO_STEPS_BROADCAST; + let reliable = big_data || overlay.overlay_type.quic_requested(); if big_data && (neighbours >= Self::MIN_NODES_FEC_TWO_STEPS_BROADCAST) { BroadcastTwostepFecProtocol::for_send(data.object, neighbours, extra)?.send(ctx).await } else { - BroadcastTwostepSimpleProtocol::for_send(big_data, extra).send(ctx).await + BroadcastTwostepSimpleProtocol::for_send(reliable, extra).send(ctx).await } } @@ -1986,6 +1997,7 @@ impl OverlayNode { overlay_id: params.overlay_id.clone(), overlay_type, owned_broadcasts: lockfree::map::Map::new(), + pending_peers: lockfree::queue::Queue::new(), purge_broadcasts: lockfree::queue::Queue::new(), purge_broadcasts_count: AtomicU32::new(0), queue_one_time_broadcasts: sender_one_time, @@ -2028,6 +2040,11 @@ impl OverlayNode { let overlay = self.get_overlay(params.overlay_id, "Cannot add overlay")?; let handle = params.runtime.unwrap_or_else(tokio::runtime::Handle::current); handle.spawn(async move { + let local_adnl_key = if overlay.overlay_type.is_private() { + Some(overlay.overlay_key().unwrap_or(&default_key).id()) + } else { + None + }; let mut timeout_peers = 0; let mut last_one_time_broadcast = None; let mut next_ping = None; @@ -2055,6 +2072,31 @@ impl OverlayNode { if let Err(e) = overlay.update_neighbours(1) { log::error!(target: TARGET, "Error: {}", e) } + if let Some(key) = &local_adnl_key { + let mut pending = Vec::new(); + while let Some(peer) = overlay.pending_peers.pop() { + pending.push(peer); + } + for peer in pending { + match overlay.try_add_peer(key, &peer) { + Ok(true) => { + log::info!( + target: TARGET, + "Resolved pending peer {peer} in overlay {}", + overlay.overlay_id + ); + continue; + } + Err(e) => log::warn!( + target: TARGET, + "Error resolving pending peer {peer} in overlay {}: {e}", + overlay.overlay_id + ), + _ => (), + } + overlay.pending_peers.push(peer); + } + } timeout_peers = 0; } let peer = if let Some(iter) = next_ping.as_mut() { @@ -2101,6 +2143,34 @@ impl OverlayNode { Ok(added) } + fn add_peers_to_overlay( + &self, + overlay_id: &Arc, + peers: &[Arc], + msg: &str, + ) -> Result { + let overlay = self.get_overlay(overlay_id, msg)?; + let our_key = overlay.overlay_key().unwrap_or(&self.node_key).id(); + let mut ret = 0; + for peer in peers { + if peer == our_key { + continue; + } + if overlay.try_add_peer(our_key, peer)? { + ret += 1; + } else { + log::info!( + target: TARGET, + "Peer {peer} has no ADNL address yet in overlay {}, queued for later", + overlay.overlay_id + ); + overlay.pending_peers.push(peer.clone()); + } + } + overlay.update_neighbours(Self::MAX_OVERLAY_NEIGHBOURS)?; + Ok(ret) + } + fn add_typed_private_overlay( &self, overlay_type: OverlayType, @@ -2109,20 +2179,27 @@ impl OverlayNode { ) -> Result { let overlay_id = params.overlay_id; if self.add_overlay(overlay_type, params)? { - let overlay = self.get_overlay(&overlay_id, "Cannot add the private overlay")?; - let our_key = overlay.overlay_key().unwrap_or(&self.node_key).id(); - for peer in peers { - if peer != our_key { - overlay.known_peers.add(peer)?; - } - } - overlay.update_neighbours(Self::MAX_OVERLAY_NEIGHBOURS)?; + self.add_peers_to_overlay(overlay_id, peers, "Cannot add the private overlay")?; Ok(true) } else { Ok(false) } } + fn calc_src_key_for_broadcast<'a>( + &'a self, + overlay: &'a Overlay, + src_key: Option<&'a Arc>, + ) -> &'a Arc { + if let Some(source) = src_key { + source + } else if let Some(key) = overlay.overlay_key() { + key + } else { + &self.node_key + } + } + fn check_overlay_adnl_address(&self, overlay: &Arc, adnl: &Arc) -> bool { let local_adnl = overlay.overlay_key().unwrap_or(&self.node_key).id(); if local_adnl != adnl { @@ -2399,7 +2476,8 @@ impl Subscriber for OverlayNode { } Ok(Broadcast::Overlay_BroadcastTwostepSimple(bcast)) => { let big_data = bcast.data.len() >= Self::MIN_BYTES_FEC_TWO_STEPS_BROADCAST; - BroadcastTwostepSimpleProtocol::for_recv(big_data).recv(bcast, ctx).await?; + let reliable = big_data || ctx.overlay.overlay_type.quic_requested(); + BroadcastTwostepSimpleProtocol::for_recv(reliable).recv(bcast, ctx).await?; return Ok(true); } Ok(bcast) => fail!("Unsupported overlay broadcast message {:?}", bcast), diff --git a/src/adnl/src/quic/mod.rs b/src/adnl/src/quic/mod.rs index 28f3a09..110d064 100644 --- a/src/adnl/src/quic/mod.rs +++ b/src/adnl/src/quic/mod.rs @@ -16,13 +16,13 @@ use crate::{ }; use std::{ collections::{HashMap, HashSet}, - fmt, - net::SocketAddr, + fmt::{Debug, Formatter, Write}, + net::{IpAddr, SocketAddr, UdpSocket}, sync::{ atomic::{AtomicBool, AtomicU64, Ordering}, Arc, Mutex, Once, Weak, }, - time::Duration, + time::{Duration, Instant}, }; use ton_api::{ deserialize_boxed, deserialize_boxed_with_suffix, serialize_boxed, @@ -39,6 +39,17 @@ use ton_block::{ const TARGET: &str = "quic"; +/// Distinguishes connection-level failures from per-message send errors. +enum SendError { + /// Could not establish a connection (peer unreachable, handshake timeout). + /// The sender task should flush the queue — retrying individual messages + /// will hit the same handshake timeout each time. + Fatal(anyhow::Error), + /// Connection existed but the send failed (stream reset, dead connection). + /// The sender task should continue to the next message. + Temporary(anyhow::Error), +} + /// Key for the QUIC inbound connection map: (local_key_id, peer_key_id). /// Matches the C++ `AdnlPath{local_id, peer_id}` semantics so that two /// connections from the same peer address but different key pairs (e.g. @@ -72,14 +83,56 @@ struct QuicOutboundConnection { } /// Per-peer sender lifecycle guard. Uses an atomic flag to ensure exactly -/// one sender task runs per outbound peer. +/// one sender task runs per outbound peer. Also tracks connect attempts +/// and timestamps for diagnostics. struct SenderState { active: AtomicBool, + /// Number of consecutive connect attempts (reset on success). + connect_attempts: AtomicU64, + /// Timestamp of the last successful connect (`None` if never connected). + last_connect: Mutex>, + /// Timestamp of the last time the connection was seen alive by the + /// periodic checker (~every 5s). Updated by `spawn_connection_checker`. + last_alive: Mutex>, } impl SenderState { fn new() -> Arc { - Arc::new(Self { active: AtomicBool::new(false) }) + Arc::new(Self { + active: AtomicBool::new(false), + connect_attempts: AtomicU64::new(0), + last_connect: Mutex::new(None), + last_alive: Mutex::new(None), + }) + } + + fn record_connect_success(&self) { + self.connect_attempts.store(0, Ordering::Relaxed); + let now = Instant::now(); + if let Ok(mut ts) = self.last_connect.lock() { + *ts = Some(now); + } + if let Ok(mut ts) = self.last_alive.lock() { + *ts = Some(now); + } + } + + /// Called by the connection checker when the connection is alive. + fn touch_alive(&self) { + if let Ok(mut ts) = self.last_alive.lock() { + *ts = Some(Instant::now()); + } + } + + fn next_attempt(&self) -> u64 { + self.connect_attempts.fetch_add(1, Ordering::Relaxed) + 1 + } + + fn last_alive_ago(&self) -> String { + match self.last_alive.lock().ok().and_then(|ts| *ts) { + Some(ts) => format!("{:.1}s ago", ts.elapsed().as_secs_f64()), + None => "never".to_string(), + } } } @@ -106,6 +159,28 @@ impl rustls::client::ResolvesClientCert for QuicCertResolver { } } +/// Fixed single-key server cert resolver for the per-key fallback configs. +struct QuicSingleKeyServerResolver(Arc); + +impl Debug for QuicSingleKeyServerResolver { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("QuicSingleKeyServerResolver").finish() + } +} + +impl rustls::server::ResolvesServerCert for QuicSingleKeyServerResolver { + fn resolve( + &self, + _client_hello: rustls::server::ClientHello<'_>, + ) -> Option> { + Some(self.0.clone()) + } + + fn only_raw_public_keys(&self) -> bool { + true + } +} + /// Per-identity outbound state: a dedicated client config (with this identity's cert) and /// its outbound connection pool keyed by remote SocketAddr. struct LocalKeyState { @@ -192,8 +267,8 @@ impl QuicServerCertResolver { } } -impl fmt::Debug for QuicServerCertResolver { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +impl Debug for QuicServerCertResolver { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("QuicServerCertResolver").finish() } } @@ -244,8 +319,8 @@ impl QuicClientCertVerifier { } } -impl fmt::Debug for QuicClientCertVerifier { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +impl Debug for QuicClientCertVerifier { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("QuicClientCertVerifier").finish() } } @@ -333,6 +408,38 @@ fn peer_key_id_from_connection(conn: &quinn::Connection) -> Option> { key_id_from_spki(first.as_ref()).ok() } +/// Tracks rapid reconnection attempts from a remote address. +/// Used to detect C++ clients stuck in a connect-and-abandon loop due to +/// key mismatch after validator key rotation. +struct ReconnectTracker { + /// Number of connections that closed without opening any streams. + count: u32, + /// When the first failed attempt in this window was recorded. + window_start: Instant, +} + +impl ReconnectTracker { + fn record(entry: &mut ReconnectTracker, window: Duration) { + let now = Instant::now(); + if now.duration_since(entry.window_start) > window { + entry.count = 1; + entry.window_start = now; + } else { + entry.count += 1; + } + } + + fn should_fallback(entry: &ReconnectTracker, window: Duration, threshold: u32) -> bool { + let now = Instant::now(); + if now.duration_since(entry.window_start) > window || entry.count < threshold { + return false; + } + // After threshold: alternate OLD/NEW on each attempt. + // offset 0 → OLD, offset 1 → NEW, offset 2 → OLD, ... + (entry.count - threshold) % 2 == 0 + } +} + /// Per-port endpoint state: the quinn endpoint, its accept loop handle, /// and the TLS cert/key maps for identities registered on this port. struct EndpointState { @@ -341,6 +448,14 @@ struct EndpointState { local_key_names: Arc>>, /// Tracks the most recently added identity name for SNI fallback. last_added_name: Arc>>, + /// Per-key ServerConfig for fallback: when a peer keeps failing with the + /// default (newest) key, we cycle through older keys. + per_key_configs: Arc>>>, + /// Tracks rapid reconnection failures per remote IP (port-independent, + /// since QUIC clients use a new source port for each connection). + /// Shared with the accept loop; also stored here for potential future stats access. + #[allow(dead_code)] + reconnect_tracker: Arc>>, } /// Command sent to the background Tokio task that manages QUIC key operations. @@ -353,6 +468,11 @@ enum KeyCommand { bind_addr: SocketAddr, reply: tokio::sync::oneshot::Sender>, }, + RemoveKey { + key_id: Arc, + bind_addr: SocketAddr, + reply: tokio::sync::oneshot::Sender>, + }, } pub struct QuicNode { @@ -372,6 +492,8 @@ pub struct QuicNode { msg_stats: Arc, /// Channel for dispatching key operations to a Tokio-hosted background task. key_cmd_tx: tokio::sync::mpsc::UnboundedSender, + /// Aggregate error counters (reset each stats dump interval). + transport_errors: Arc, } impl QuicNode { @@ -385,6 +507,40 @@ impl QuicNode { const DEFAULT_QUERY_TIMEOUT_MS: u64 = 5000; /// Maximum number of messages buffered per outbound peer const SEND_QUEUE_CAPACITY: usize = 1024; + /// Timeout for QUIC handshake when connecting to a peer. + /// C++ ngtcp2 abandons after ~3-5s, so 5s is a reasonable upper bound. + + // --- Key fallback on rapid reconnection --- + // When a C++ client keeps connecting and immediately disconnecting (key + // mismatch after validator key rotation), the server detects the pattern + // and presents an older key as a fallback. + // + /// Enable the key fallback mechanism. When `false`, the server always + /// presents the most recently registered key (last_added_name). + const KEY_FALLBACK_ENABLED: bool = false; + /// Time window for counting rapid reconnection attempts from the same IP. + const KEY_FALLBACK_WINDOW: Duration = Duration::from_secs(60); + /// Number of connection attempts within the window before triggering fallback. + const KEY_FALLBACK_THRESHOLD: u32 = 3; + const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); + + /// Backoff schedule: first 2 attempts (cycle 1) have no delay, then groups + /// of 10 attempts (5 cycles) increase by 5s each, capped at 30s. + /// attempts 1-2 → 0s + /// attempts 3-12 → 5s + /// attempts 13-22 → 10s + /// attempts 23-32 → 15s + /// attempts 33-42 → 20s + /// attempts 43-52 → 25s + /// attempts 53+ → 30s (capped) + fn connect_backoff(prev_attempts: u64) -> Duration { + if prev_attempts < 2 { + return Duration::ZERO; + } + let group = (prev_attempts - 2) / 10; + let secs = ((group + 1) * 5).min(30); + Duration::from_secs(secs) + } /// Create a new QuicNode. No endpoints are bound — they are created lazily /// by `add_key()` when the first identity for a given port is registered. @@ -416,6 +572,7 @@ impl QuicNode { inbound_pools: Mutex::new(Vec::new()), msg_stats: MsgStats::new(), key_cmd_tx, + transport_errors: TransportErrors::new(), }); // Spawn background task that processes key commands inside the Tokio runtime. let weak = Arc::downgrade(&transport); @@ -434,6 +591,14 @@ impl QuicNode { }; let _ = reply.send(result); } + KeyCommand::RemoveKey { key_id, bind_addr, reply } => { + let result = if let Some(this) = weak.upgrade() { + this.remove_key_inner(&key_id, bind_addr) + } else { + Err(error!("QuicNode dropped")) + }; + let _ = reply.send(result); + } } } _ = token.cancelled() => break, @@ -547,6 +712,22 @@ impl QuicNode { Ok(key_id.clone()) })?; + // Build a dedicated ServerConfig for this key (used by the fallback mechanism + // when a peer keeps reconnecting and failing with the default/newest key). + if let Some(cert_entry) = endpoint_state.server_cert_keys.get(&name) { + match Self::build_single_key_server_config(cert_entry.val().clone()) { + Ok(cfg) => { + if let Ok(mut configs) = endpoint_state.per_key_configs.lock() { + configs.insert(name.clone(), Arc::new(cfg)); + } + } + Err(e) => log::warn!( + target: TARGET, + "Cannot build per-key ServerConfig for {key_id}: {e}" + ), + } + } + // Update last-added name for SNI fallback (C++ ngtcp2 doesn't send SNI) if let Ok(mut last) = endpoint_state.last_added_name.lock() { *last = Some(name); @@ -561,6 +742,68 @@ impl QuicNode { Ok(()) } + /// Unregister a local identity from a specific bind address. + /// Removes the key from the server cert resolver, local key names, and local + /// key state. After this call the QUIC server will no longer present this + /// key's RPK certificate to connecting peers. + pub fn remove_key(&self, key_id: &Arc, bind_addr: SocketAddr) -> Result<()> { + let (reply_tx, reply_rx) = tokio::sync::oneshot::channel(); + self.key_cmd_tx + .send(KeyCommand::RemoveKey { key_id: key_id.clone(), bind_addr, reply: reply_tx }) + .map_err(|_| error!("QuicNode key command channel closed"))?; + match tokio::runtime::Handle::try_current() { + Ok(_) => tokio::task::block_in_place(|| { + reply_rx + .blocking_recv() + .map_err(|_| error!("QuicNode key command reply channel dropped"))? + }), + Err(_) => reply_rx + .blocking_recv() + .map_err(|_| error!("QuicNode key command reply channel dropped"))?, + } + } + + /// Internal implementation of remove_key — always runs inside the Tokio runtime. + fn remove_key_inner(&self, key_id: &Arc, bind_addr: SocketAddr) -> Result<()> { + let port = bind_addr.port(); + let endpoints = self.endpoints.lock().map_err(|e| error!("Endpoints lock: {e}"))?; + let Some(endpoint_state) = endpoints.get(&port) else { + fail!("No QUIC endpoint on port {port} for key {key_id}"); + }; + + let name = Self::key_id_to_server_name(key_id); + + // Remove from server cert resolver map + endpoint_state.server_cert_keys.remove(&name); + + // Remove from local key name → key id mapping + endpoint_state.local_key_names.remove(&name); + + // Update last_added_name: if it was pointing to the removed key, + // switch to another remaining key (or None). + if let Ok(mut last) = endpoint_state.last_added_name.lock() { + if last.as_deref() == Some(&name) { + *last = endpoint_state.local_key_names.iter().next().map(|e| e.key().clone()); + } + } + + // Remove per-key ServerConfig for fallback + if let Ok(mut configs) = endpoint_state.per_key_configs.lock() { + configs.remove(&name); + } + + // Remove from local key state (outbound connections for this identity) + self.local_keys.remove(key_id); + + log::info!( + target: TARGET, + "Unregistered QUIC identity {} from port {}", + key_id, bind_addr.port() + ); + + Ok(()) + } + pub fn add_peer_key(&self, key_id: Arc, addr: SocketAddr) -> Result<()> { add_unbound_object_to_map_with_update(&self.peer_keys, key_id, |_| Ok(Some(addr)))?; Ok(()) @@ -576,6 +819,7 @@ impl QuicNode { adnl: Option<&AdnlNode>, peers: &AdnlPeers, ) -> Result> { + let t0 = Instant::now(); self.ensure_peer_registered(adnl, peers)?; let tag = extract_inner_tag(&data); let size = data.len(); @@ -583,15 +827,37 @@ impl QuicNode { let addr = self.addr_by_key(peers.other())?; let state = self.local_key_state(peers.local())?; let outbound = Self::get_or_create_outbound_connection(&state.outbound, addr)?; + let t_prep = t0.elapsed(); // Fast path: if connection is alive, send directly without queue overhead if let Some(ref conn) = outbound.conn { - match Self::send_via_stream(conn, &data).await { - Ok(_) => { - self.msg_stats.record(tag, size, addr, true, false); + match Self::send_via_stream_nowait(conn, &data).await { + Ok(()) => { + self.msg_stats.record(tag, size, addr, true, MsgKind::Message); + let t_total = t0.elapsed(); + if t_total > Duration::from_millis(10) { + log::warn!( + target: TARGET, + "QUIC message() SLOW to {addr}: \ + prep={:.1}ms send={:.1}ms total={:.1}ms tag={tag:08x} size={size}", + t_prep.as_secs_f64() * 1000.0, + (t_total - t_prep).as_secs_f64() * 1000.0, + t_total.as_secs_f64() * 1000.0, + ); + } else { + log::trace!( + target: TARGET, + "QUIC message() to {addr}: \ + prep={:.1}ms send={:.1}ms total={:.1}ms tag={tag:08x} size={size}", + t_prep.as_secs_f64() * 1000.0, + (t_total - t_prep).as_secs_f64() * 1000.0, + t_total.as_secs_f64() * 1000.0, + ); + } return Ok(Some(data.len())); } Err(e) => { + self.transport_errors.send_failed.fetch_add(1, Ordering::Relaxed); log::warn!( target: TARGET, "QUIC direct send to {} failed: {e}, removing dead connection, \ @@ -599,6 +865,7 @@ impl QuicNode { peers.other() ); Self::remove_dead_connection(&state.outbound, addr, conn); + self.transport_errors.dead_conn_removed.fetch_add(1, Ordering::Relaxed); } } } @@ -606,9 +873,10 @@ impl QuicNode { // Slow path: no connection (or it just died) — enqueue for the sender task // which will establish the connection and deliver if !outbound.send_queue.try_push(data) { + self.transport_errors.queue_full.fetch_add(1, Ordering::Relaxed); fail!("QUIC send queue full for peer {}", peers.other()); } - self.msg_stats.record(tag, size, addr, true, false); + self.msg_stats.record(tag, size, addr, true, MsgKind::Message); // Spawn sender task if not already running (CAS guarantees at most one per peer) if outbound @@ -653,15 +921,25 @@ impl QuicNode { let size = data.len(); let timeout_ms = timeout_ms.unwrap_or(Self::DEFAULT_QUERY_TIMEOUT_MS); let wire = serialize_boxed(&QuicQuery { data: data.into() }.into_boxed())?; - let response = self.send_query_raw(wire, peers, timeout_ms).await?; - self.msg_stats.record(tag, size, addr, true, true); + self.msg_stats.record(tag, size, addr, true, MsgKind::Query); + let response = match self.send_query_raw(wire, peers, timeout_ms).await { + Ok(r) => r, + Err(e) => { + self.msg_stats.record(tag, 0, addr, false, MsgKind::NoAnswer); + return Err(e); + } + }; if response.is_empty() { + self.msg_stats.record(tag, 0, addr, false, MsgKind::NoAnswer); return Ok(None); } let obj = deserialize_boxed(&response) .map_err(|e| error!("Cannot deserialise QUIC answer: {e}"))?; match obj.downcast::() { - Ok(QuicResponse::Quic_Answer(answer)) => Ok(Some(answer.data.to_vec())), + Ok(QuicResponse::Quic_Answer(answer)) => { + self.msg_stats.record(tag, answer.data.len(), addr, false, MsgKind::Answer); + Ok(Some(answer.data.to_vec())) + } Err(x) => fail!("Unexpected QUIC response type {x:?}"), } } @@ -700,6 +978,21 @@ impl QuicNode { async fn connect(&self, peers: &AdnlPeers, addr: SocketAddr, server_name: &str) -> Result<()> { let dst = peers.other(); let state = self.local_key_state(peers.local())?; + + // Check if a live connection already exists — avoid creating a duplicate + // that would be immediately closed and disrupt the peer's accept loop. + if let Some(entry) = state.outbound.map().get(&addr) { + if let Some(ref conn) = entry.val().conn { + if conn.close_reason().is_none() { + log::trace!( + target: TARGET, + "QUIC connect to {addr}: reusing existing live connection" + ); + return Ok(()); + } + } + } + let endpoint = { let endpoints = self.endpoints.lock().map_err(|e| error!("Endpoints lock: {e}"))?; endpoints @@ -722,7 +1015,7 @@ impl QuicNode { fail!("QUIC RPK mismatch connecting to {addr}: expected {dst}, got {peer_id}"); } - if !state.outbound.set_connection_state(addr, |found| { + let stored = state.outbound.set_connection_state(addr, |found| { if found.conn.is_none() { Ok(Some(QuicOutboundConnection { conn: Some(conn.clone()), @@ -732,12 +1025,35 @@ impl QuicNode { } else { Ok(None) } - })? { - conn.close(0u32.into(), b"Duplicate QUIC connection"); + })?; + if !stored { + // Another thread won the race. Don't close our connection — the peer + // may already be using it for inbound stream processing. Park it so + // quinn's idle timeout cleans it up gracefully. + log::debug!( + target: TARGET, + "QUIC connect to {addr}: another connection stored first, \ + parking ours for idle-timeout cleanup" + ); + self.park_superseded_connection(conn); } Ok(()) } + /// Keep a superseded connection alive until quinn's idle timeout expires, + /// so the peer's accept loop isn't disrupted by an abrupt close. + fn park_superseded_connection(&self, conn: quinn::Connection) { + let token = self.cancellation_token.clone(); + tokio::spawn(async move { + tokio::select! { + _ = conn.closed() => {} + _ = token.cancelled() => { + conn.close(0u32.into(), b"shutdown"); + } + } + }); + } + /// Obtain (or create) an outbound connection and connect in the foreground. /// Used by the query path where a live connection is required synchronously. async fn ensure_outbound_connection( @@ -753,7 +1069,10 @@ impl QuicNode { break Ok(conn); } log::info!(target: TARGET, "Try new QUIC connection to {addr} in foreground"); - self.connect(peers, addr, &server_name).await?; + if let Err(e) = self.connect(peers, addr, &server_name).await { + self.transport_errors.connect_failed.fetch_add(1, Ordering::Relaxed); + return Err(e); + } log::info!(target: TARGET, "QUIC connected to {addr} in foreground"); } } @@ -839,7 +1158,7 @@ impl QuicNode { sock.bind(&bind_addr.into()) .map_err(|e| error!("Cannot bind UDP socket to {bind_addr}: {e}"))?; sock.set_nonblocking(true).map_err(|e| error!("Cannot set non-blocking: {e}"))?; - std::net::UdpSocket::from(sock) + UdpSocket::from(sock) }; let runtime: Arc = Arc::new(quinn::TokioRuntime); let endpoint = quinn::Endpoint::new( @@ -862,6 +1181,9 @@ impl QuicNode { ), } + let per_key_configs = Arc::new(Mutex::new(HashMap::new())); + let reconnect_tracker = Arc::new(Mutex::new(HashMap::new())); + Self::spawn_accept_loop( endpoint.clone(), local_key_names.clone(), @@ -872,6 +1194,8 @@ impl QuicNode { self.cancellation_token.clone(), inbound, self.msg_stats.clone(), + per_key_configs.clone(), + reconnect_tracker.clone(), ); let state = Arc::new(EndpointState { @@ -879,6 +1203,8 @@ impl QuicNode { server_cert_keys, local_key_names, last_added_name, + per_key_configs, + reconnect_tracker, }); endpoints.insert(port, state.clone()); @@ -936,13 +1262,33 @@ impl QuicNode { bind_addr: SocketAddr, max_streams_per_connection: usize, msg_stats: Arc, + fallback_config: Option>, + reconnect_tracker: Arc>>, ) { let addr = incoming.remote_address(); + // Use fallback ServerConfig if provided (older key for rapid-reconnect peers). + let connecting = if let Some(config) = fallback_config { + match incoming.accept_with(config) { + Ok(c) => c, + Err(e) => { + log::warn!(target: TARGET, "QUIC accept_with (fallback) from {addr}: {e}"); + return; + } + } + } else { + match incoming.accept() { + Ok(c) => c, + Err(e) => { + log::warn!(target: TARGET, "QUIC accept from {addr}: {e}"); + return; + } + } + }; // Bound handshake time: C++ ngtcp2 clients abandon after ~3-5s and retry, // so a handshake still in progress after 5s is almost certainly stale. // Without this, stale Connecting futures accumulate inside quinn's endpoint, // slowing its internal event loop and delaying endpoint.accept() for new peers. - let conn = match tokio::time::timeout(Duration::from_secs(5), incoming).await { + let conn = match tokio::time::timeout(Duration::from_secs(5), connecting).await { Ok(Ok(conn)) => conn, Ok(Err(e)) => { log::warn!(target: TARGET, "QUIC handshake from {addr} on {bind_addr} failed: {e}"); @@ -954,8 +1300,6 @@ impl QuicNode { } }; - log::info!(target: TARGET, "Accepted QUIC connection from {addr} on {bind_addr}"); - let peer_key_id = match peer_key_id_from_connection(&conn) { Some(key_id) => key_id, None => { @@ -991,6 +1335,12 @@ impl QuicNode { } }; + log::info!( + target: TARGET, + "Accepted QUIC connection from {addr} on {bind_addr} \ + local {local_key_id} peer {peer_key_id}" + ); + let inbound_key = QuicInboundKey(local_key_id.clone(), peer_key_id.clone()); let had_existing = { let mut found_existing = false; @@ -1027,6 +1377,7 @@ impl QuicNode { // Accept both bi-directional streams (queries + legacy messages) and // uni-directional streams (fire-and-forget messages from the new sender). + let streams_accepted = Arc::new(AtomicU64::new(0)); let conn_bi = conn.clone(); let conn_uni = conn.clone(); let sem_bi = stream_semaphore.clone(); @@ -1037,6 +1388,8 @@ impl QuicNode { let peers_uni = peers; let stats_bi = msg_stats.clone(); let stats_uni = msg_stats; + let streams_bi = streams_accepted.clone(); + let streams_uni = streams_accepted.clone(); let bi_loop = async { loop { @@ -1047,6 +1400,7 @@ impl QuicNode { break; } }; + streams_bi.fetch_add(1, Ordering::Relaxed); let permit = match sem_bi.clone().acquire_owned().await { Ok(p) => p, Err(_) => break, @@ -1081,6 +1435,7 @@ impl QuicNode { break; } }; + streams_uni.fetch_add(1, Ordering::Relaxed); let permit = match sem_uni.clone().acquire_owned().await { Ok(p) => p, Err(_) => break, @@ -1101,18 +1456,39 @@ impl QuicNode { }; // Run both accept loops; when either exits (connection closed), both stop. + // Also monitor conn.closed() directly — if the remote peer disconnects + // without ever opening streams (e.g., C++ key-mismatch abandon), we detect + // it immediately instead of waiting for the 15s idle timeout. tokio::select! { () = bi_loop => {} () = uni_loop => {} + reason = conn.closed() => { + log::debug!( + target: TARGET, + "QUIC connection from {addr} closed early: {reason}" + ); + } } + let total_streams = streams_accepted.load(Ordering::Relaxed); let is_current = inbound.get(&inbound_key).map(|e| e.val().stable_id() == conn_id).unwrap_or(false); if is_current { inbound.remove(&inbound_key); } + + // If the connection was productive (streams were opened), clear the + // reconnect tracker for this IP. This resets the fallback state so + // future connections from this peer use the default (newest) key again. + if total_streams > 0 { + if let Ok(mut tracker) = reconnect_tracker.lock() { + tracker.remove(&addr.ip()); + } + } + log::info!( target: TARGET, - "Exit QUIC inbound receiver for {addr} (conn_id={conn_id}, removed={is_current})" + "Exit QUIC inbound receiver for {addr} \ + (conn_id={conn_id}, streams={total_streams}, removed={is_current})" ); } @@ -1123,6 +1499,33 @@ impl QuicNode { format!("{}.{}", &hex[..32], &hex[32..]) } + /// Build a ServerConfig that always presents a single fixed RPK certificate. + /// Used by the fallback mechanism to present an older key to peers that + /// keep failing with the default (newest) key. + fn build_single_key_server_config( + certified_key: Arc, + ) -> Result { + let resolver = Arc::new(QuicSingleKeyServerResolver(certified_key)); + let verifier = QuicClientCertVerifier::new(); + let mut tls_config = rustls::ServerConfig::builder() + .with_client_cert_verifier(verifier) + .with_cert_resolver(resolver); + tls_config.alpn_protocols = vec![b"ton".to_vec()]; + + let mut server_config = quinn::ServerConfig::with_crypto(Arc::new( + quinn::crypto::rustls::QuicServerConfig::try_from(tls_config) + .map_err(|e| error!("Cannot create per-key QUIC server config: {e}"))?, + )); + let mut transport_config = quinn::TransportConfig::default(); + transport_config.max_concurrent_bidi_streams(1_000u32.into()); + transport_config.max_idle_timeout(Some( + quinn::IdleTimeout::try_from(Duration::from_secs(15)).expect("15s fits in IdleTimeout"), + )); + transport_config.keep_alive_interval(Some(Duration::from_secs(5))); + server_config.transport_config(Arc::new(transport_config)); + Ok(server_config) + } + fn local_key_state(&self, src: &Arc) -> Result> { self.local_keys .get(src) @@ -1171,7 +1574,16 @@ impl QuicNode { ); match obj.downcast::() { Ok(Request::Quic_Message(msg)) => { - msg_stats.record(extract_inner_tag(&msg.data), msg.data.len(), addr, false, false); + msg_stats.record( + extract_inner_tag(&msg.data), + msg.data.len(), + addr, + false, + MsgKind::Message, + ); + // Ack immediately before processing — don't block the sender + // while we dispatch to subscribers + let _ = send.finish(); log::debug!( target: TARGET, "process_incoming_stream from {addr}: QUIC MESSAGE, \ @@ -1187,21 +1599,12 @@ impl QuicNode { break; } } - let _ = send.finish(); - log::debug!( - target: TARGET, - "process_incoming_stream from {addr}: finished send side" - ); } Ok(Request::Quic_Query(query)) => { - msg_stats.record( - extract_inner_tag(&query.data), - query.data.len(), - addr, - false, - true, - ); + let query_tag = extract_inner_tag(&query.data); + msg_stats.record(query_tag, query.data.len(), addr, false, MsgKind::Query); log::debug!(target: TARGET, "process_incoming_stream from {addr}: QUIC QUERY"); + let mut answered = false; let answer = Query::process(subscribers, &query.data, &peers).await?; if let Some(answer) = answer { let answer = match answer { @@ -1213,12 +1616,17 @@ impl QuicNode { Answer::Object(tagged) => serialize_boxed(&tagged.object)?, Answer::Raw(tagged) => tagged.object, }; + msg_stats.record(query_tag, data.len(), addr, true, MsgKind::Answer); + answered = true; let response = QuicAnswer { data: data.into() }.into_boxed(); send.write_all(&serialize_boxed(&response)?) .await .map_err(|e| error!("QUIC write answer to {addr}: {e}"))?; } } + if !answered { + msg_stats.record(query_tag, 0, addr, true, MsgKind::NoAnswer); + } let _ = send.finish(); } Err(_obj) => { @@ -1261,7 +1669,13 @@ impl QuicNode { .map_err(|e| error!("Cannot deserialize QUIC uni-stream from {addr}: {e}"))?; match obj.downcast::() { Ok(Request::Quic_Message(msg)) => { - msg_stats.record(extract_inner_tag(&msg.data), msg.data.len(), addr, false, false); + msg_stats.record( + extract_inner_tag(&msg.data), + msg.data.len(), + addr, + false, + MsgKind::Message, + ); for subscriber in subscribers { if subscriber.try_consume_custom(&msg.data, peers).await? { break; @@ -1338,32 +1752,36 @@ impl QuicNode { let delay_ms = rand::thread_rng().gen_range(500..=2500); tokio::time::sleep(Duration::from_millis(delay_ms)).await; + let peer_key = &key.1; let old_alive = inbound.get(&key).map(|e| e.val().close_reason().is_none()).unwrap_or(false); let new_alive = new_conn.close_reason().is_none(); if old_alive && new_alive { - if let Some(old) = inbound.remove(&key) { + // Don't close the old connection — it may be the remote peer's active + // outbound. Just replace it in the tracking map; the old connection's + // accept loops will continue until idle timeout. + if let Some(_old) = inbound.remove(&key) { log::info!( target: TARGET, - "Closing old duplicate inbound from {addr} (both alive after {delay_ms}ms)" + "Replacing old duplicate inbound from {addr} key {peer_key} \ + (both alive after {delay_ms}ms)" ); - old.val().close(0u32.into(), b"Replaced by new inbound"); } let nc = new_conn.clone(); let _ = add_unbound_object_to_map_with_update(&inbound, key, |_| Ok(Some(nc.clone()))); } else if new_alive { inbound.remove(&key); - let nc = new_conn.clone(); - let _ = add_unbound_object_to_map_with_update(&inbound, key, |_| Ok(Some(nc.clone()))); log::debug!( target: TARGET, - "Old inbound from {addr} already closed, keeping new" + "Old inbound from {addr} key {peer_key} already closed, keeping new" ); + let nc = new_conn.clone(); + let _ = add_unbound_object_to_map_with_update(&inbound, key, |_| Ok(Some(nc.clone()))); } else { log::debug!( target: TARGET, - "New inbound from {addr} already closed, keeping old" + "New inbound from {addr} key {peer_key} already closed, keeping old" ); } } @@ -1371,6 +1789,16 @@ impl QuicNode { /// Drain the send queue and exit. Spawned when `message()` has no live /// connection and must enqueue data for later delivery. The task establishes /// the connection, sends all queued messages, and terminates. + /// + /// On connect failure the task waits 1s and retries once. If the retry also + /// fails, all remaining queued messages are flushed (the peer is unreachable + /// and retrying each message individually would stall for the full handshake + /// timeout every time). + /// + /// When previous connect attempts have failed (counter persists in + /// `SenderState`), the task applies a stepped backoff before the first + /// attempt. Messages keep queuing during the backoff and are either + /// delivered on success or flushed on failure. async fn run_sender_task( quic: Arc, peers: AdnlPeers, @@ -1382,13 +1810,68 @@ impl QuicNode { ) { log::trace!(target: TARGET, "QUIC sender task started for {addr}"); - loop { + // Stepped backoff based on previous failed attempts (2 attempts per cycle). + let prev_attempts = sender_state.connect_attempts.load(Ordering::Relaxed); + let backoff = Self::connect_backoff(prev_attempts); + if !backoff.is_zero() { + log::info!( + target: TARGET, + "QUIC sender to {addr}: backoff {backoff:?} before connect \ + (previous attempts: {prev_attempts})" + ); + tokio::time::sleep(backoff).await; + } + + 'outer: loop { // Drain the queue while let Some(data) = send_queue.pop() { - if let Err(e) = - quic.send_message(&peers, addr, &server_name, &outbound, &data).await + match quic + .send_message(&peers, addr, &server_name, &outbound, &sender_state, &data) + .await { - log::warn!(target: TARGET, "QUIC sender to {addr} error: {e}"); + Ok(()) => {} + Err(SendError::Temporary(e)) => { + log::warn!(target: TARGET, "QUIC sender to {addr} send error: {e}"); + } + Err(SendError::Fatal(e)) => { + log::warn!(target: TARGET, "QUIC sender to {addr} connect error: {e}"); + if send_queue.is_empty() { + break 'outer; + } + // Retry once after 1s + tokio::time::sleep(Duration::from_secs(1)).await; + match quic + .send_message( + &peers, + addr, + &server_name, + &outbound, + &sender_state, + &data, + ) + .await + { + Ok(()) => {} + Err(SendError::Temporary(e)) => { + log::warn!( + target: TARGET, + "QUIC sender to {addr} send error on retry: {e}" + ); + } + Err(SendError::Fatal(e)) => { + let mut flushed = 0usize; + while send_queue.pop().is_some() { + flushed += 1; + } + log::warn!( + target: TARGET, + "QUIC sender to {addr} connect retry failed: {e}, \ + flushed {flushed} queued messages" + ); + break 'outer; + } + } + } } } @@ -1412,33 +1895,79 @@ impl QuicNode { } /// Send a single message to the peer, establishing the connection first if needed. + /// Returns `SendError::Fatal` when the connection cannot be established + /// (peer unreachable) and `SendError::Temporary` when the send itself fails + /// on an existing connection. async fn send_message( &self, peers: &AdnlPeers, addr: SocketAddr, server_name: &str, outbound: &Connections, + sender_state: &SenderState, data: &[u8], - ) -> Result<()> { - let entry = Self::get_or_create_outbound_connection(outbound, addr)?; + ) -> std::result::Result<(), SendError> { + let entry = Self::get_or_create_outbound_connection(outbound, addr) + .map_err(SendError::Temporary)?; match entry.conn { Some(ref conn) => { - if let Err(e) = Self::send_via_stream(conn, data).await { + if let Err(e) = Self::send_via_stream_nowait(conn, data).await { + self.transport_errors.send_failed.fetch_add(1, Ordering::Relaxed); log::warn!( target: TARGET, "QUIC send to {addr} failed: {e}, removing dead connection" ); Self::remove_dead_connection(outbound, addr, conn); - return Err(e); + self.transport_errors.dead_conn_removed.fetch_add(1, Ordering::Relaxed); + return Err(SendError::Temporary(e)); } } None => { - log::info!(target: TARGET, "QUIC sender: connecting to {addr}"); - self.connect(peers, addr, server_name).await?; + let attempt = sender_state.next_attempt(); + log::info!( + target: TARGET, + "QUIC sender: connecting to {addr} (attempt {attempt})" + ); + match tokio::time::timeout( + Self::CONNECT_TIMEOUT, + self.connect(peers, addr, server_name), + ) + .await + { + Ok(Ok(())) => { + sender_state.record_connect_success(); + } + Ok(Err(e)) => { + self.transport_errors.connect_failed.fetch_add(1, Ordering::Relaxed); + log::warn!( + target: TARGET, + "QUIC connect to {addr} failed: {e} \ + (attempt {attempt}, last alive: {})", + sender_state.last_alive_ago() + ); + return Err(SendError::Fatal(e)); + } + Err(_) => { + self.transport_errors.connect_failed.fetch_add(1, Ordering::Relaxed); + let msg = format!( + "QUIC connect to {addr} timed out ({}s, \ + attempt {attempt}, last alive: {})", + Self::CONNECT_TIMEOUT.as_secs(), + sender_state.last_alive_ago() + ); + log::warn!(target: TARGET, "{msg}"); + return Err(SendError::Fatal(error!("{msg}").into())); + } + } log::info!(target: TARGET, "QUIC sender: connected to {addr}"); - let entry = Self::get_or_create_outbound_connection(outbound, addr)?; + let entry = Self::get_or_create_outbound_connection(outbound, addr) + .map_err(SendError::Temporary)?; if let Some(ref conn) = entry.conn { - Self::send_via_stream(conn, data).await?; + Self::send_via_stream_nowait(conn, data).await.map_err(SendError::Temporary)?; + } else { + return Err(SendError::Temporary( + error!("QUIC connection to {addr} lost after connect").into(), + )); } } } @@ -1463,14 +1992,17 @@ impl QuicNode { match result { Ok(Ok(response)) => return Ok(response), Ok(Err(e)) => { + self.transport_errors.send_failed.fetch_add(1, Ordering::Relaxed); log::warn!( target: TARGET, "QUIC query to {} failed: {e}, removing dead connection and retrying", peers.other() ); Self::remove_dead_connection(&state.outbound, addr, conn); + self.transport_errors.dead_conn_removed.fetch_add(1, Ordering::Relaxed); } Err(_) => { + self.transport_errors.query_timeout.fetch_add(1, Ordering::Relaxed); log::warn!( target: TARGET, "QUIC query to {} timed out ({timeout_ms}ms), \ @@ -1478,6 +2010,7 @@ impl QuicNode { peers.other() ); Self::remove_dead_connection(&state.outbound, addr, conn); + self.transport_errors.dead_conn_removed.fetch_add(1, Ordering::Relaxed); } } } @@ -1493,6 +2026,50 @@ impl QuicNode { } } + /// Fire-and-forget message send: opens a bidirectional stream, writes data, + /// and returns immediately without waiting for a response. + /// Uses bidi (not uni) streams because C++ ngtcp2 servers set + /// `initial_max_streams_uni = 0` by default, rejecting uni-streams. + /// Used by `message()` and `send_message()` where the response is not needed. + async fn send_via_stream_nowait(conn: &quinn::Connection, data: &[u8]) -> Result<()> { + let t0 = Instant::now(); + let addr = conn.remote_address(); + let (mut send, _recv) = + conn.open_bi().await.map_err(|e| error!("Cannot open QUIC bi-stream: {e}"))?; + let t_open = t0.elapsed(); + send.write_all(data).await.map_err(|e| error!("QUIC stream write: {e}"))?; + let t_write = t0.elapsed(); + send.finish().map_err(|e| error!("QUIC stream finish: {e}"))?; + let t_finish = t0.elapsed(); + // Drop _recv without reading — fire-and-forget, matching C++ behavior + if t_finish > Duration::from_millis(10) { + log::warn!( + target: TARGET, + "send_via_stream_nowait SLOW to {addr}: \ + open={:.1}ms write={:.1}ms finish={:.1}ms total={:.1}ms data_len={}", + t_open.as_secs_f64() * 1000.0, + (t_write - t_open).as_secs_f64() * 1000.0, + (t_finish - t_write).as_secs_f64() * 1000.0, + t_finish.as_secs_f64() * 1000.0, + data.len() + ); + } else { + log::trace!( + target: TARGET, + "send_via_stream_nowait to {addr}: \ + open={:.1}ms write={:.1}ms finish={:.1}ms total={:.1}ms data_len={}", + t_open.as_secs_f64() * 1000.0, + (t_write - t_open).as_secs_f64() * 1000.0, + (t_finish - t_write).as_secs_f64() * 1000.0, + t_finish.as_secs_f64() * 1000.0, + data.len() + ); + } + Ok(()) + } + + /// Request-response send: opens a bidirectional stream, writes data, + /// and waits for the peer's response. Used by `send_query_raw()`. async fn send_via_stream(conn: &quinn::Connection, data: &[u8]) -> Result> { log::debug!( target: TARGET, @@ -1537,6 +2114,8 @@ impl QuicNode { cancellation_token: tokio_util::sync::CancellationToken, inbound: Arc, msg_stats: Arc, + per_key_configs: Arc>>>, + reconnect_tracker: Arc>>, ) { tokio::spawn(async move { loop { @@ -1554,12 +2133,35 @@ impl QuicNode { let addr = incoming.remote_address(); log::debug!(target: TARGET, "Accept in QUIC server on {bind_addr} from {addr}"); + // Check if this peer has been rapidly reconnecting BEFORE + // recording this attempt, so the fallback triggers on the + // connection AFTER the threshold, not the one that reaches it. + let fallback_config = Self::pick_fallback_config( + addr, + &reconnect_tracker, + &per_key_configs, + &server_cert_resolver, + ); + + // Record this attempt AFTER the fallback check, so the + // threshold triggers on the next connection, not this one. + if Self::KEY_FALLBACK_ENABLED { + if let Ok(mut tracker) = reconnect_tracker.lock() { + let entry = tracker.entry(addr.ip()).or_insert_with(|| ReconnectTracker { + count: 0, + window_start: Instant::now(), + }); + ReconnectTracker::record(entry, Self::KEY_FALLBACK_WINDOW); + } + } + let token = cancellation_token.clone(); let lkn = local_key_names.clone(); let scr = server_cert_resolver.clone(); let ib = inbound.clone(); let subs = subscribers.clone(); let stats = msg_stats.clone(); + let tracker = reconnect_tracker.clone(); tokio::spawn(async move { tokio::select! { _ = token.cancelled() => { @@ -1568,6 +2170,7 @@ impl QuicNode { _ = Self::handle_connection( incoming, lkn, scr, ib, subs, bind_addr, max_streams_per_connection, stats, + fallback_config, tracker, ) => {} } }); @@ -1577,6 +2180,52 @@ impl QuicNode { }); } + /// Pick an alternative ServerConfig if a peer has been rapidly reconnecting. + /// Returns None (use default) or Some(config with an older key). + fn pick_fallback_config( + addr: SocketAddr, + reconnect_tracker: &Mutex>, + per_key_configs: &Mutex>>, + server_cert_resolver: &QuicServerCertResolver, + ) -> Option> { + if !Self::KEY_FALLBACK_ENABLED { + return None; + } + + let should_fallback = reconnect_tracker + .lock() + .ok() + .and_then(|tracker| { + tracker.get(&addr.ip()).map(|entry| { + ReconnectTracker::should_fallback( + entry, + Self::KEY_FALLBACK_WINDOW, + Self::KEY_FALLBACK_THRESHOLD, + ) + }) + }) + .unwrap_or(false); + + if !should_fallback { + return None; + } + + // Find a key that is NOT the current default (last_added_name). + let last_name = server_cert_resolver.last_added_name.lock().ok().and_then(|g| g.clone()); + + let configs = per_key_configs.lock().ok()?; + for (name, config) in configs.iter() { + if Some(name) != last_name.as_ref() { + log::info!( + target: TARGET, + "Fallback: presenting older key to {addr} (rapid reconnection detected)" + ); + return Some(config.clone()); + } + } + None + } + /// Background task that periodically scans all outbound connection pools and /// removes dead connections. This detects peer crashes, network changes, and /// idle timeouts before the next send attempt, avoiding the 10-15s hang on @@ -1609,7 +2258,13 @@ impl QuicNode { conn.close_reason() ); Self::remove_dead_connection(outbound, addr, conn); + transport + .transport_errors + .dead_conn_removed + .fetch_add(1, Ordering::Relaxed); removed += 1; + } else { + state.sender_state.touch_alive(); } } // Fully remove entry only when connection is cleared, no sender @@ -1672,15 +2327,22 @@ impl QuicNode { let id = (conn.stable_id(), true); seen.insert(id); total += 1; - let snap = ConnSnapshot::from_stats(&s); + let since = prev + .get(&id) + .map(|p| p.connected_since) + .unwrap_or_else(Instant::now); + let snap = ConnSnapshot::new(&s, since); let delta = prev.get(&id).map(|p| snap.delta(p)).unwrap_or(snap); prev.insert(id, snap); - fmt::Write::write_fmt( + Write::write_fmt( &mut dump, format_args!( " outbound peer={addr} \ + up={} \ dtx={} bytes/{} dgrams drx={} bytes/{} dgrams \ - dlost={} pkts rtt={:?} cwnd={} mtu={} key={key_id:.8}\n", + dlost={} pkts rtt={:?} cwnd={} mtu={} \ + local={} remote={}\n", + snap.uptime_str(), delta.tx_bytes, delta.tx_dgrams, delta.rx_bytes, @@ -1689,6 +2351,10 @@ impl QuicNode { s.path.rtt, s.path.cwnd, s.path.current_mtu, + key_id, + peer_key_id_from_connection(&conn) + .map(|k| k.to_string()) + .unwrap_or_else(|| "?".to_string()), ), ) .ok(); @@ -1716,15 +2382,20 @@ impl QuicNode { let id = (conn.stable_id(), false); seen.insert(id); total += 1; - let snap = ConnSnapshot::from_stats(&s); + let since = + prev.get(&id).map(|p| p.connected_since).unwrap_or_else(Instant::now); + let snap = ConnSnapshot::new(&s, since); let delta = prev.get(&id).map(|p| snap.delta(p)).unwrap_or(snap); prev.insert(id, snap); - fmt::Write::write_fmt( + Write::write_fmt( &mut dump, format_args!( - " inbound peer={addr} local={local_id} remote={peer_id} \ + " inbound peer={addr} \ + up={} \ dtx={} bytes/{} dgrams drx={} bytes/{} dgrams \ - dlost={} pkts rtt={:?} cwnd={} mtu={}\n", + dlost={} pkts rtt={:?} cwnd={} mtu={} \ + local={} remote={}\n", + snap.uptime_str(), delta.tx_bytes, delta.tx_dgrams, delta.rx_bytes, @@ -1733,6 +2404,8 @@ impl QuicNode { s.path.rtt, s.path.cwnd, s.path.current_mtu, + local_id, + peer_id, ), ) .ok(); @@ -1748,12 +2421,11 @@ impl QuicNode { for (key, count, bytes) in &msg_entries { if current_peer != Some(key.addr) { current_peer = Some(key.addr); - fmt::Write::write_fmt(&mut dump, format_args!(" peer {}:\n", key.addr,)) - .ok(); + Write::write_fmt(&mut dump, format_args!(" peer {}:\n", key.addr,)).ok(); } let dir = if key.is_outbound { "out" } else { " in" }; - let kind = if key.is_query { "query" } else { "msg " }; - fmt::Write::write_fmt( + let kind = key.kind.label(); + Write::write_fmt( &mut dump, format_args!( " {dir}/{kind} {:#010x}({}) count={count} bytes={bytes}\n", @@ -1764,7 +2436,16 @@ impl QuicNode { .ok(); } - fmt::Write::write_fmt(&mut dump, format_args!( + let (sf, qt, cf, qf, dr) = transport.transport_errors.take(); + Write::write_fmt( + &mut dump, + format_args!( + " errors: send_failed={sf} query_timeout={qt} \ + connect_failed={cf} queue_full={qf} dead_conn_removed={dr}\n", + ), + ) + .ok(); + Write::write_fmt(&mut dump, format_args!( " total: {total} connections, {} msg entries", msg_entries.len(), )).ok(); @@ -1839,10 +2520,49 @@ fn tl_tag_name(tag: u32) -> &'static str { 0x9283ce37 => "validatorSession.blockUpdate", 0xbe7b573a => "consensus.simplex.certificate", 0xc37ef4f3 => "consensus.simplex.vote", + 0x543fba6c => "consensus.simplex.requestCandidate", _ => "unknown", } } +/// Aggregate error counters for the QUIC transport layer (lock-free, reset on each stats dump). +struct TransportErrors { + /// send_via_stream / send_via_stream_nowait failures (stream open, write, finish errors) + send_failed: AtomicU64, + /// query timeouts (deadline exceeded waiting for response) + query_timeout: AtomicU64, + /// connection establishment failures (connect / handshake errors) + connect_failed: AtomicU64, + /// messages dropped because the per-peer send queue was full + queue_full: AtomicU64, + /// dead connections removed (by checker or on send failure) + dead_conn_removed: AtomicU64, +} + +impl TransportErrors { + fn new() -> Arc { + Arc::new(Self { + send_failed: AtomicU64::new(0), + query_timeout: AtomicU64::new(0), + connect_failed: AtomicU64::new(0), + queue_full: AtomicU64::new(0), + dead_conn_removed: AtomicU64::new(0), + }) + } + + /// Take current values and reset to zero; returns (send_failed, query_timeout, + /// connect_failed, queue_full, dead_conn_removed). + fn take(&self) -> (u64, u64, u64, u64, u64) { + ( + self.send_failed.swap(0, Ordering::Relaxed), + self.query_timeout.swap(0, Ordering::Relaxed), + self.connect_failed.swap(0, Ordering::Relaxed), + self.queue_full.swap(0, Ordering::Relaxed), + self.dead_conn_removed.swap(0, Ordering::Relaxed), + ) + } +} + /// Snapshot of cumulative counters from a single connection, used to compute deltas. #[derive(Clone, Copy)] struct ConnSnapshot { @@ -1851,16 +2571,19 @@ struct ConnSnapshot { rx_bytes: u64, rx_dgrams: u64, lost_pkts: u64, + /// When this connection was first observed by the stats dumper. + connected_since: Instant, } impl ConnSnapshot { - fn from_stats(s: &quinn::ConnectionStats) -> Self { + fn new(s: &quinn::ConnectionStats, connected_since: Instant) -> Self { Self { tx_bytes: s.udp_tx.bytes, tx_dgrams: s.udp_tx.datagrams, rx_bytes: s.udp_rx.bytes, rx_dgrams: s.udp_rx.datagrams, lost_pkts: s.path.lost_packets, + connected_since, } } @@ -1871,6 +2594,23 @@ impl ConnSnapshot { rx_bytes: self.rx_bytes.saturating_sub(prev.rx_bytes), rx_dgrams: self.rx_dgrams.saturating_sub(prev.rx_dgrams), lost_pkts: self.lost_pkts.saturating_sub(prev.lost_pkts), + connected_since: self.connected_since, + } + } + + fn uptime_str(&self) -> String { + let secs = self.connected_since.elapsed().as_secs(); + if secs < 60 { + format!("{secs}s") + } else if secs < 3600 { + format!("{}m{}s", secs / 60, secs % 60) + } else if secs < 36 * 3600 { + format!("{}h{}m", secs / 3600, (secs % 3600) / 60) + } else { + let days = secs / 86400; + let hours = (secs % 86400) / 3600; + let mins = (secs % 3600) / 60; + format!("{days}d{hours}h{mins}m") } } } @@ -1897,13 +2637,33 @@ impl MsgTagCounters { } } +/// Classification of a QUIC message for telemetry. +#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +enum MsgKind { + Message, + Query, + Answer, + NoAnswer, +} + +impl MsgKind { + fn label(&self) -> &'static str { + match self { + MsgKind::Message => "msg ", + MsgKind::Query => "query ", + MsgKind::Answer => "ans ", + MsgKind::NoAnswer => "no_ans", + } + } +} + /// Per-peer, per-TL-tag message statistics key. #[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] struct MsgStatsKey { addr: SocketAddr, tag: u32, is_outbound: bool, - is_query: bool, + kind: MsgKind, } /// Tracks per-peer, per-message-kind statistics for QUIC traffic. @@ -1916,8 +2676,8 @@ impl MsgStats { Arc::new(Self { counters: lockfree::map::Map::new() }) } - fn record(&self, tag: u32, size: usize, addr: SocketAddr, is_outbound: bool, is_query: bool) { - let key = MsgStatsKey { addr, tag, is_outbound, is_query }; + fn record(&self, tag: u32, size: usize, addr: SocketAddr, is_outbound: bool, kind: MsgKind) { + let key = MsgStatsKey { addr, tag, is_outbound, kind }; if let Some(entry) = self.counters.get(&key) { entry.val().record(size); return; @@ -2017,9 +2777,9 @@ mod tests { let stats = MsgStats::new(); let addr = test_addr(1000); - stats.record(0xAA, 100, addr, true, false); - stats.record(0xAA, 200, addr, true, false); - stats.record(0xBB, 50, addr, true, true); + stats.record(0xAA, 100, addr, true, MsgKind::Message); + stats.record(0xAA, 200, addr, true, MsgKind::Message); + stats.record(0xBB, 50, addr, true, MsgKind::Query); let entries = stats.drain(); assert_eq!(entries.len(), 2); @@ -2040,9 +2800,9 @@ mod tests { let addr_a = test_addr(1000); let addr_b = test_addr(2000); - stats.record(0xAA, 10, addr_b, true, false); - stats.record(0xBB, 500, addr_a, true, false); - stats.record(0xCC, 100, addr_a, true, false); + stats.record(0xAA, 10, addr_b, true, MsgKind::Message); + stats.record(0xBB, 500, addr_a, true, MsgKind::Message); + stats.record(0xCC, 100, addr_a, true, MsgKind::Message); let entries = stats.drain(); assert_eq!(entries.len(), 3); @@ -2063,7 +2823,7 @@ mod tests { let stats = MsgStats::new(); let addr = test_addr(1000); - stats.record(0xAA, 100, addr, true, false); + stats.record(0xAA, 100, addr, true, MsgKind::Message); let entries = stats.drain(); assert_eq!(entries.len(), 1); @@ -2077,20 +2837,20 @@ mod tests { let stats = MsgStats::new(); let addr = test_addr(1000); - stats.record(0xAA, 100, addr, true, false); - stats.record(0xBB, 50, addr, false, false); + stats.record(0xAA, 100, addr, true, MsgKind::Message); + stats.record(0xBB, 50, addr, false, MsgKind::Message); // First drain: both active, counters reset let _ = stats.drain(); // Only record on 0xAA - stats.record(0xAA, 200, addr, true, false); + stats.record(0xAA, 200, addr, true, MsgKind::Message); // Second drain: 0xBB was idle → evicted let _ = stats.drain(); // Record on 0xBB again — must re-insert (was evicted) - stats.record(0xBB, 30, addr, false, false); + stats.record(0xBB, 30, addr, false, MsgKind::Message); let entries = stats.drain(); // 0xAA was idle since last drain (evicted), only 0xBB with activity is returned @@ -2104,9 +2864,9 @@ mod tests { let stats = MsgStats::new(); let addr = test_addr(1000); - stats.record(0xAA, 100, addr, true, false); // outbound msg - stats.record(0xAA, 200, addr, false, false); // inbound msg - stats.record(0xAA, 300, addr, true, true); // outbound query + stats.record(0xAA, 100, addr, true, MsgKind::Message); // outbound msg + stats.record(0xAA, 200, addr, false, MsgKind::Message); // inbound msg + stats.record(0xAA, 300, addr, true, MsgKind::Query); // outbound query let entries = stats.drain(); assert_eq!(entries.len(), 3); diff --git a/src/adnl/tests/test_overlay.rs b/src/adnl/tests/test_overlay.rs index 3deacdd..06185b4 100644 --- a/src/adnl/tests/test_overlay.rs +++ b/src/adnl/tests/test_overlay.rs @@ -56,7 +56,8 @@ use ton_block::{ #[path = "./test_utils.rs"] mod test_utils; use test_utils::{ - find_overlay_peer, get_adnl_config, init_compatibility_test, init_test, TestContext, + find_overlay_peer, get_adnl_config, init_compatibility_test, init_test, init_test_log, + TestContext, }; const KEY_TAG_DHT: usize = 1; @@ -1459,7 +1460,7 @@ async fn test_overlay_semiprivate() -> Result<()> { const SLAVE3: &str = "127.0.0.1:4206"; const SLAVE4: &str = "127.0.0.1:4207"; - crate::test_utils::init_test_log(); + init_test_log(); let mut peers = Vec::new(); diff --git a/src/node/consensus-common/src/adnl_overlay.rs b/src/node/consensus-common/src/adnl_overlay.rs index aad8d94..cd5598b 100644 --- a/src/node/consensus-common/src/adnl_overlay.rs +++ b/src/node/consensus-common/src/adnl_overlay.rs @@ -7,7 +7,7 @@ * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. */ use crate::{ - utils::MetricsHandle, BlockPayloadPtr, ConsensusNode, ConsensusOverlay, + utils::MetricsHandle, BlockPayloadPtr, ConsensusCommonFactory, ConsensusNode, ConsensusOverlay, ConsensusOverlayListenerPtr, ConsensusOverlayLogReplayListenerPtr, ConsensusOverlayManager, ConsensusOverlayManagerPtr, ConsensusOverlayPtr, OverlayTransportType, PrivateKey, PublicKeyHash, QueryResponseCallback, Result, @@ -24,6 +24,7 @@ use std::{ any::Any, collections::HashMap, future::Future, + net::{IpAddr, Ipv4Addr, SocketAddr}, pin::Pin, sync::{ atomic::{AtomicBool, Ordering}, @@ -36,7 +37,10 @@ use ton_api::{ deserialize_boxed, serialize_bare, serialize_boxed, serialize_boxed_append, ton::{ catchain::BroadcastWrapper, - consensus::simplex::{Certificate as SimplexCertificate, Vote as SimplexVote}, + consensus::{ + simplex::{Certificate as SimplexCertificate, Vote as SimplexVote}, + RequestError as ConsensusRequestError, + }, overlay::{ broadcast::BroadcastTwostepSimple, broadcast_twostep::id::Id as BroadcastTwostepId, broadcast_twostep_simple::tosign::ToSign as BroadcastTwostepSimpleToSign, @@ -49,6 +53,26 @@ use ton_block::{error, fail, sha256_digest, KeyId, KeyOption, UInt256}; const LOG_TARGET: &str = "consensus_adnl_overlay"; +fn describe_query_response_error(error: &ConsensusRequestError) -> &'static str { + match error { + ConsensusRequestError::Consensus_RequestError => "consensus.requestError", + } +} + +fn extract_query_response_error(data: &[u8]) -> Option { + let message = deserialize_boxed(data).ok()?; + let error = message.downcast::().ok()?; + Some(describe_query_response_error(&error).to_string()) +} + +fn normalize_query_response_payload(data: Vec) -> Result { + if let Some(error_name) = extract_query_response_error(data.as_slice()) { + return Err(error!("Peer returned {}", error_name)); + } + + Ok(ConsensusCommonFactory::create_block_payload(data)) +} + /// Stream tags for task processor routing in ADNL overlay #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum AdnlOverlayStreamTag { @@ -591,7 +615,7 @@ impl Peer { } // Add new address - let add_result = self.overlay_node.add_private_peers( + let add_result = self.overlay_node.add_private_peers_to_adnl( &self.src_adnl_addr, vec![(adnl_addr, quic_addr, key)], ); @@ -899,11 +923,9 @@ impl AdnlOverlay { log::debug!( target: LOG_TARGET, - "Creating new AdnlOverlay: overlay_id={}, local_id={}, nodes_count={}, transport={:?}", - overlay_id, - local_id, - nodes.len(), - transport_type, + "Creating new AdnlOverlay: overlay_id={overlay_id}, local_id={local_id}, \ + nodes_count={}, transport={transport_type:?}", + nodes.len() ); // Find local ADNL key from nodes by matching local_id @@ -916,8 +938,7 @@ impl AdnlOverlay { local_adnl_key = Some(stack.adnl.key_by_id(&node.adnl_id)?); log::trace!( target: LOG_TARGET, - "Found local ADNL key: local_id={}, adnl_id={}", - local_id, + "Found local ADNL key: local_id={local_id}, adnl_id={}", node.adnl_id ); continue; // Skip adding local node to peers @@ -949,10 +970,7 @@ impl AdnlOverlay { adnl::QuicNode::OFFSET_PORT ) })?; - let bind_addr = std::net::SocketAddr::new( - std::net::IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED), - quic_port, - ); + let bind_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), quic_port); quic.add_key(&key_bytes, local_adnl_key.id(), bind_addr)?; log::info!( target: LOG_TARGET, @@ -1577,9 +1595,7 @@ impl ConsensusOverlay for AdnlOverlay { let result = result.ok_or_else(|| error!("answer is None!"))?; let data = serialize_boxed(&result)?; - let data = crate::ConsensusCommonFactory::create_block_payload(data); - - Ok(data) + normalize_query_response_payload(data) } .await; @@ -1595,7 +1611,10 @@ impl ConsensusOverlay for AdnlOverlay { }); } - /// Send query via RLDP for large messages + /// Send query via RLDP for large messages. + /// On QUIC-enabled overlays (simplex+quic) the query is routed through QUIC + /// bi-directional streams instead, since QUIC handles flow control natively + /// and doesn't need `max_answer_size`. fn send_query_via_rldp( &self, dst_adnl_id: PublicKeyHash, @@ -1610,6 +1629,7 @@ impl ConsensusOverlay for AdnlOverlay { let overlay_node = self.stack.overlay.clone(); let stop_requested = self.stop_requested.clone(); let runtime_handle = self.runtime_handle.clone(); + let is_quic = self.is_quic_available; if stop_requested.load(Ordering::Relaxed) { log::warn!(target: LOG_TARGET, "AdnlOverlay: Overlay {} was stopped!", &overlay_id); @@ -1622,28 +1642,40 @@ impl ConsensusOverlay for AdnlOverlay { return; } - // Execute RLDP query let result = async { let query_body = deserialize_boxed(query.data())?; - let mut query_data = overlay_node.get_query_prefix(&overlay_id)?; - serialize_boxed_append(&mut query_data, &query_body)?; - - let (data, _) = overlay_node - .query_via_rldp( - &dst_adnl_id, - &TaggedByteSlice { - object: &query_data[..], - #[cfg(feature = "telemetry")] - tag: query_body.bare_object().constructor(), - }, - &overlay_id, - Some(max_answer_size), - v2, - None, - ) - .await?; - let data = data.ok_or_else(|| error!("answer is None!"))?; - Ok(crate::ConsensusCommonFactory::create_block_payload(data)) + + if is_quic { + // QUIC path: use bi-directional stream query (no max_answer_size needed) + let tagged: TaggedTlObject = query_body.into(); + let result = overlay_node + .query_via_quic(&dst_adnl_id, &tagged, &overlay_id, None) + .await?; + let result = result.ok_or_else(|| error!("QUIC query answer is None!"))?; + let data = serialize_boxed(&result)?; + normalize_query_response_payload(data) + } else { + // RLDP path: traditional large-message query + let mut query_data = overlay_node.get_query_prefix(&overlay_id)?; + serialize_boxed_append(&mut query_data, &query_body)?; + + let (data, _) = overlay_node + .query_via_rldp( + &dst_adnl_id, + &TaggedByteSlice { + object: &query_data[..], + #[cfg(feature = "telemetry")] + tag: query_body.bare_object().constructor(), + }, + &overlay_id, + Some(max_answer_size), + v2, + None, + ) + .await?; + let data = data.ok_or_else(|| error!("answer is None!"))?; + normalize_query_response_payload(data) + } } .await; @@ -1957,3 +1989,7 @@ impl ConsensusOverlayManager for AdnlOverlayManager { } } } + +#[cfg(test)] +#[path = "tests/test_adnl_overlay.rs"] +mod tests; diff --git a/src/node/consensus-common/src/lib.rs b/src/node/consensus-common/src/lib.rs index 769c176..cb0f1ee 100644 --- a/src/node/consensus-common/src/lib.rs +++ b/src/node/consensus-common/src/lib.rs @@ -116,18 +116,6 @@ pub type ValidatorBlockCandidateDecisionCallback = pub type ValidatorBlockCandidateCallback = Box) + Send>; -/// Result of a committed candidate proof download. -/// Carries the block signatures from the on-chain proof, sufficient -/// to reconstruct the FinalCert for SimplexState injection. -#[derive(Clone, Debug)] -pub struct CommittedBlockProof { - pub block_id: BlockIdExt, - pub signatures: BlockSignaturesVariant, -} - -/// Callback for SessionListener::get_committed_candidate -pub type CommittedBlockProofCallback = Box) + Send>; - /// Pointer to async request pub type AsyncRequestPtr = Arc; @@ -510,9 +498,9 @@ pub trait AsyncKeyValueStorage: Send + Sync { // Session Statistics // ============================================================================ -/// Session statistics for a committed block +/// Session statistics reported alongside validator-session block-acceptance callbacks. /// -/// Passed to on_block_committed callback to report session health metrics. +/// For Simplex these stats are currently also reused for finalized delivery. #[derive(Debug, Clone, Default)] pub struct SessionStats { /// Total number of errors during this session @@ -985,7 +973,9 @@ pub trait SessionListener: Send + Sync { /// New block is committed /// - /// Called when a block has been finalized by the consensus. + /// Called for sequential block acceptance callbacks. + /// Catchain uses this directly; Simplex now delivers finalized blocks via + /// `on_block_finalized` and must not rely on this callback. /// The `signatures` parameter contains the block signatures in variant format /// (either Ordinary for catchain-based consensus or Simplex for simplex consensus). #[allow(clippy::too_many_arguments)] @@ -1013,15 +1003,32 @@ pub trait SessionListener: Send + Sync { callback: ValidatorBlockCandidateCallback, ); - /// Download committed block proof from full-node storage/network. + /// A block has been finalized (FinalCert observed) and is ready for + /// validator-side acceptance. /// - /// Called by SessionProcessor when WaitingForFinalCert for an MC block - /// whose seqno == expected_seqno. The implementor (ValidatorGroup) fetches - /// the block proof via EngineOperations and returns BlockSignaturesVariant. + /// Called immediately when a finalization certificate is collected for a + /// slot, regardless of whether predecessors have been committed yet. + /// This is the Simplex finalized-delivery counterpart to `on_block_committed`. /// - /// Replaces the Rust-only requestCandidate2 path with a mechanism - /// compatible with both Rust and C++ validators. - fn get_committed_candidate(&self, block_id: BlockIdExt, callback: CommittedBlockProofCallback); + /// `block_id` carries the full `BlockIdExt` (shard, seqno, root_hash, + /// file_hash) so `ValidatorGroup` can derive the block identity without + /// relying on sequential `prev_block_ids` tracking. + /// + /// Has a default no-op implementation for backward compatibility with + /// legacy listeners that do not participate in finalized delivery. + #[allow(unused_variables)] + fn on_block_finalized( + &self, + block_id: BlockIdExt, + source_info: BlockSourceInfo, + root_hash: BlockHash, + file_hash: BlockHash, + data: BlockPayloadPtr, + signatures: BlockSignaturesVariant, + approve_signatures: Vec<(PublicKeyHash, BlockPayloadPtr)>, + ) { + // Default no-op for backward compatibility with legacy listeners. + } } // ============================================================================ diff --git a/src/node/consensus-common/src/tests/test_adnl_overlay.rs b/src/node/consensus-common/src/tests/test_adnl_overlay.rs new file mode 100644 index 0000000..fdeae16 --- /dev/null +++ b/src/node/consensus-common/src/tests/test_adnl_overlay.rs @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2025-2026 RSquad Blockchain Lab. + * + * Licensed under the GNU General Public License v3.0. + * See the LICENSE file in the root of this repository. + * + * This software is provided "AS IS", WITHOUT WARRANTY OF ANY KIND. + */ +//! Unit tests for `adnl_overlay.rs` helpers. + +use super::*; +use ton_api::{ + serialize_boxed, + ton::consensus::{ + simplex::candidateandcert::CandidateAndCert, RequestError as ConsensusRequestError, + }, + IntoBoxed, +}; + +#[test] +fn test_extract_query_response_error_recognizes_request_error() { + let bytes = serialize_boxed(&ConsensusRequestError::Consensus_RequestError) + .expect("serialize requestError"); + + let error_name = + extract_query_response_error(bytes.as_slice()).expect("requestError must be extracted"); + assert_eq!(error_name, "consensus.requestError"); +} + +#[test] +fn test_normalize_query_response_payload_rejects_request_error() { + let bytes = serialize_boxed(&ConsensusRequestError::Consensus_RequestError) + .expect("serialize requestError"); + + let err = + normalize_query_response_payload(bytes).expect_err("requestError must become query error"); + assert!(err.to_string().contains("consensus.requestError"), "unexpected error: {err}"); +} + +#[test] +fn test_normalize_query_response_payload_accepts_candidate_and_cert() { + let bytes = serialize_boxed( + &CandidateAndCert { candidate: Vec::::new().into(), notar: Vec::::new().into() } + .into_boxed(), + ) + .expect("serialize CandidateAndCert"); + + let payload = + normalize_query_response_payload(bytes.clone()).expect("candidateAndCert must pass"); + assert_eq!(payload.data(), &bytes); +} diff --git a/src/node/simplex/CHANGELOG.md b/src/node/simplex/CHANGELOG.md index 59b917d..d84df76 100644 --- a/src/node/simplex/CHANGELOG.md +++ b/src/node/simplex/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to the Simplex Consensus Protocol implementation will be documented in this file. +## [Unreleased] + +### Added +- Observability: peer-delivered candidate ingress counters + (`simplex_candidate_received_broadcast`, `simplex_candidate_received_query`) and a unified + `simplex_collation_starts` counter, with derivative `/s` dump support for operator debugging. + ## [0.5.0] - 2026-03-20 ### Added diff --git a/src/node/simplex/README.md b/src/node/simplex/README.md index 8fced04..e6c129f 100644 --- a/src/node/simplex/README.md +++ b/src/node/simplex/README.md @@ -7,6 +7,12 @@ Rust implementation of the Simplex consensus protocol for TON blockchain. > **C++ Reference**: Primary tracking is `ton-blockchain/ton/tree/simplex` (main repo). > Secondary: `DanShaders/ton/tree/alpenglow` (superseded). +> **Current semantics (Apr 2026):** +> - Simplex is finalized-driven. +> - Finalized blocks are delivered through `on_block_finalized()` and may arrive out of order. +> - `on_block_committed()` remains part of the shared listener interface for legacy sequential acceptance, but Simplex must not use it. +> - Missing-body handling uses `finalized_pending_body`: a finalized block can be known before its body arrives locally. + ## Overview Simplex is a consensus protocol based on the Solana Alpenglow White Paper (May 2025 v1) with modifications for TON: @@ -67,7 +73,8 @@ This crate targets wire-compatibility with the upstream **C++ Simplex** implemen - Overlay ID computation (node ordering, short ID) - `candidateAndCert.notar` encoding (voteSignatureSet) - Handle incoming `consensus.simplex.certificate` on vote channel -- `requestCandidate2` removed — replaced by `get_committed_candidate` +- `requestCandidate2` removed +- `get_committed_candidate` / `CommittedBlockProof*` removed; current Simplex relies on finalized delivery plus deferred body materialization - Shard `before_split` empty block rule - Restart support (DB persistence + startup recovery) - Certificate rebroadcast on restart @@ -88,7 +95,7 @@ This crate targets wire-compatibility with the upstream **C++ Simplex** implemen │ │ │ SessionProcessor │ │ │ - pull callback queue │ │ │ │ │ - consensus FSM │ │ │ - invoke SessionListener: │ │ │ │ │ - slot state management │ │ │ - on_candidate │ │ -│ │ │ - vote tracking │ │ │ - on_block_committed │ │ +│ │ │ - vote tracking │ │ │ - on_block_finalized │ │ │ │ └──────────────────────────────┘ │ │ - on_generate_slot │ │ │ │ │ └──────────────────────────────────┘ │ │ │ - pull main task queue │ │ @@ -146,7 +153,7 @@ This crate targets wire-compatibility with the upstream **C++ Simplex** implemen │ SessionListener (implemented by caller) │ │ - on_candidate: validate block │ │ - on_generate_slot: create new block │ -│ - on_block_committed: finalization notification │ +│ - on_block_finalized: finalized-block delivery │ └─────────────────────────────────────────────────────────────────────────────┘ ``` @@ -190,7 +197,7 @@ The original Alpenglow White Paper specifies 5 vote types for consensus. However When 2/3 stake weight is reached for a vote type, a certificate is formed: - **NotarizationCert**: Block is notarized -- **FinalizationCert**: Block is finalized (committed) +- **FinalizationCert**: Block is finalized - **SkipCert**: Slot is skipped Certificates are implicit (derived from vote counts), not explicit on-wire objects. @@ -228,7 +235,7 @@ let validators re-vote on the previous block to attempt getting a FinalizeCertif Each slot follows this flow: ``` -Collate → Broadcast → Validate → Notarize → Vote → Collect → Finalize → Commit → next slot +Collate → Broadcast → Validate → Notarize → Vote → Collect → Finalize → Deliver → next slot ``` | Phase | SessionProcessor | SimplexState | Output | @@ -240,7 +247,7 @@ Collate → Broadcast → Validate → Notarize → Vote → Collect → Finaliz | **Vote** | `broadcast_vote()` | - | Vote to network | | **Collect** | `on_vote()` | `on_vote()` → thresholds | Threshold events | | **Finalize** | - | `try_final()` | `BroadcastVote(Final)` | -| **Commit** | `handle_block_finalized()` | `BlockFinalized` event | `on_block_committed()` | +| **Deliver** | `handle_block_finalized()` | `BlockFinalized` event | `on_block_finalized()` | ## Package Structure @@ -253,7 +260,7 @@ node/simplex/ │ ├── lib.rs # Public API: Session, SimplexSession, SessionOptions, SessionFactory │ ├── block.rs # Block candidate types: RawCandidateId, Candidate, etc. │ ├── certificate.rs # Certificate types: VoteSignature, Certificate (crate-private) -│ ├── database.rs # DB persistence for restart/recommit (crate-private) +│ ├── database.rs # DB persistence for restart recovery (crate-private) │ ├── simplex_state.rs # Core consensus FSM with event-based output │ ├── session.rs # Session actor (multi-threaded wrapper, task queues) │ ├── session_processor.rs # Integrates SimplexState with network (crate-private) @@ -296,7 +303,7 @@ Entry point for integration. See `lib.rs` documentation for detailed API referen | `ConsensusSession` | Base session interface (trait, from consensus-common) | | `SimplexSession` | Simplex-specific session operations (extends `Session`) | | `SessionListener` | Callback trait (from consensus-common) | -| `SessionStats` | Session health metrics passed to `on_block_committed` | +| `SessionStats` | Session health metrics passed alongside validator callbacks | | `Receiver` | Network sender interface (trait) | | `ReceiverListener` | Network receiver callbacks (trait) | @@ -346,11 +353,12 @@ Single-threaded consensus algorithm (crate-private): - ✅ MC finalization callback - `SimplexSession::notify_mc_finalized()` posts to `set_mc_finalized_seqno()` - ✅ Missing block requests - `schedule_request_candidate()` → delayed action → `receiver.request_candidate()` - ✅ Recursive parent resolution - `PendingParentResolution`, `update_resolution_cache_chain()`, `find_first_missing_parent()` -- ✅ Round tracking - `current_round` field tracks sequential commit counter (independent of slots) +- ✅ Finalized-driven delivery - `handle_block_finalized()`, `maybe_emit_out_of_order_finalized()`, `maybe_apply_finalized_state()` +- ✅ Roundless listener model - round is not used for Simplex sequencing logic - ✅ Standstill coordination - calls `receiver.reschedule_standstill()` on finalization, `set_standstill_slots()` on finalization/skip - ✅ DB persistence - finalized blocks, candidate infos, notar certs, votes, pool state persisted to RocksDB -- ✅ Startup recovery - bootstrap load, vote replay, receiver cache restore, recommit to ValidatorGroup -- ✅ Download committed block via full-node proof for MC gap recovery (replaces requestCandidate2) +- ✅ Startup recovery - bootstrap load, vote replay, receiver cache restore, finalized-boundary restoration +- ✅ Late-join handling - finalized blocks can be known before body arrival via `finalized_pending_body` - ⚠️ Precollation parent tracking - needs fix for cross-window scenarios **Key methods:** @@ -375,15 +383,18 @@ Single-threaded consensus algorithm (crate-private): - `simplex_time:collation_latency` - Histogram for collation time - `simplex_active_weight` - Gauge for current network active weight - `simplex_validates.*` - ResultStatusCounter for validation requests -- `simplex_collates.*` - ResultStatusCounter for collation requests -- `simplex_commits.*` - ResultStatusCounter for commit requests +- `simplex_collates.*` - ResultStatusCounter for collation completion results +- `simplex_collation_starts` - Counter for all collation entry attempts +- `simplex_commits.*` - legacy-named ResultStatusCounter for finalized-delivery/apply outcomes - `simplex_precollation_requests` - Counter for precollation requests - `simplex_precollation_results` - Counter for precollation completions - `simplex_collates_precollated.*` - ResultStatusCounter for precollated block hits +- `simplex_candidate_received_broadcast` - Counter for peer-delivered broadcast candidate bodies +- `simplex_candidate_received_query` - Counter for peer-delivered query-response candidate bodies - `simplex_skipped_slots` - Counter for skipped slots -- `simplex_batch_commits` - Counter for batch commit operations -- `simplex_batch_commit_size` - Histogram for batch commit sizes -- `simplex_current_round` - Gauge for current round (sequential commit counter) +- `simplex_batch_commits` - legacy-named batch finalized-apply metric +- `simplex_batch_commit_size` - legacy-named histogram for finalized batch size +- `simplex_finalized_pending_body_count` - Gauge for finalized blocks waiting for body arrival ### SimplexState (`simplex_state.rs`) @@ -396,7 +407,7 @@ Core consensus state machine (crate-private): **Event model**: Instead of callbacks, SimplexState produces events: - `BroadcastVote(vote)` - Vote to broadcast to all validators -- `BlockFinalized(slot, block)` - Block finalized (triggers `on_block_committed`) +- `BlockFinalized(slot, block)` - Block finalized (triggers `on_block_finalized`) - `SlotSkipped(slot)` - Slot skipped (handled internally, no callback) **API**: @@ -598,8 +609,12 @@ impl SessionListener for MyListener { // Call callback with block candidate } + fn on_block_finalized(&self, block_id, round, source, root_hash, file_hash, data, signatures, approve_signatures) { + // Finalized block delivered by Simplex (may be out of order) + } + fn on_block_committed(&self, source_info, root_hash, file_hash, data, signatures, approve_signatures, stats) { - // Block was finalized in consensus + unreachable!("Simplex must not call on_block_committed()"); } fn on_block_skipped(&self, round: u32) { @@ -655,7 +670,7 @@ Multi-instance consensus tests with in-process overlay. |------|-------------|--------| | `test_simplex_consensus_basic` | Basic consensus with 7 nodes, 100 rounds | ✅ | | `test_simplex_consensus_with_failures` | Consensus with simulated failures | ✅ | -| `test_simplex_consensus_finalcert_recovery` | FinalCert recovery via `get_committed_candidate` | ✅ | +| `test_simplex_consensus_finalcert_recovery` | FinalCert recovery and finalized delivery without the old proof callback path | ✅ | | `test_simplex_consensus_shard_with_mc_notifications` | MC finalization forwarding to shards | ✅ | | `test_simplex_consensus_adnl_overlay` | ADNL overlay-based consensus | ✅ | | `test_simplex_consensus_adnl_net_gremlin` | ADNL net gremlin (packet loss/delay simulation) | ✅ | @@ -665,7 +680,7 @@ Multi-instance consensus tests with in-process overlay. **Test Configuration:** - `total_slots: u32` - Number of slots to complete (default: 100) -- `min_commit_percent: f64` - Minimum required commit rate (default: 0.5 = 50%) +- `min_finalized_percent: f64` - Minimum required finalized-delivery rate (default varies by test) - `test_timeout: Duration` - Maximum time to wait - `expect_timeout: bool` - If true, test passes on timeout @@ -692,8 +707,7 @@ Restart integration tests (public API only) validating DB-backed stop/restart re | Test | Description | |------|-------------| -| `test_single_session_restart_round_monotonicity_full_replay` | Restart with full replay; round/slot stream remains monotonic | -| `test_single_session_restart_round_monotonicity_first_commit_after_finalized` | Restart after finalized boundary; first post-restart commit keeps monotonicity | +| `test_single_session_restart_round_monotonicity_first_commit_after_finalized` | Restart after finalized boundary; resumed session keeps finalized state consistent without historical recommit | **Running:** ```bash @@ -805,7 +819,7 @@ All metrics use the `simplex_` prefix. Latency histograms use `time:` prefix (va | `simplex_process_events_calls` | FSM event processing calls | `process_simplex_events()` | | `simplex_errors` | Protocol-breaking errors | `increment_error()` | | `simplex_misbehavior` | Detected misbehavior events | `on_vote()` conflict detection | -| `simplex_batch_commits` | Batch commit operations | `try_commit_finalized_chains()` | +| `simplex_batch_commits` | Legacy batch finalized-apply metric | historical naming; sequential commit scheduler removed | | `simplex_skip_total` | Total slot skip events | `handle_slot_skipped()` | | `simplex_votes_in_notarize` | Inbound notarize votes | `on_vote()` | | `simplex_votes_in_finalize` | Inbound finalize votes | `on_vote()` | @@ -820,6 +834,9 @@ All metrics use the `simplex_` prefix. Latency histograms use `time:` prefix (va | `simplex_validation_reject` | Validation rejections | `candidate_decision_fail()` | | `simplex_validation_late_callback` | Late validation callbacks | `candidate_decision_ok/fail()` | | `simplex_health_warnings` | Health anomaly warnings (not errors) | `run_health_checks()` | +| `simplex_candidate_received_broadcast` | Peer-delivered broadcast candidate bodies (excludes local self-loop) | `on_candidate_received()` | +| `simplex_candidate_received_query` | Peer-delivered requestCandidate/query-response candidate bodies (excludes local self-loop) | `on_candidate_received()` | +| `simplex_collation_starts` | Unified collation entry attempts across async, retry, precollated, and empty-block paths | `check_collation()`, `invoke_collation()`, `invoke_collation_retry()` | | `simplex_precollation_requests` | Precollation requests sent | `invoke_precollation()` | | `simplex_precollation_results` | Precollation results received | `precollation_result()` | @@ -828,8 +845,8 @@ All metrics use the `simplex_` prefix. Latency histograms use `time:` prefix (va | Metric | Description | |--------|-------------| | `simplex_validates` | Block validation results | -| `simplex_collates` | Block collation results | -| `simplex_commits` | Block commit results | +| `simplex_collates` | Block collation completion results (`.total` only covers async listener requests) | +| `simplex_commits` | Finalized-delivery/apply results (legacy metric family name) | | `simplex_collates_precollated` | Precollated block hits | | `simplex_collates_expire` | Expired collation time slots | @@ -840,7 +857,8 @@ All metrics use the `simplex_` prefix. Latency histograms use `time:` prefix (va | `simplex_active_weight` | Active validator weight | `check_all()` | | `simplex_total_weight` | Total validator weight | `init_metrics()` | | `simplex_threshold_66` | 2/3 weight threshold | `init_metrics()` | -| `simplex_last_finalized_slot` | Last finalized slot index | `commit_single_block()` | +| `simplex_last_finalized_slot` | Last finalized slot index | `maybe_apply_finalized_state()` | +| `simplex_finalized_pending_body_count` | Finalized blocks waiting for body arrival | `handle_block_finalized()`, cleanup, materialization | | `simplex_first_non_finalized_slot` | First non-finalized slot (FSM) | `check_all()` | | `simplex_first_non_progressed_slot` | First non-progressed slot (FSM) | `check_all()` | @@ -855,7 +873,7 @@ All metrics use the `simplex_` prefix. Latency histograms use `time:` prefix (va | `time:slot_stage1_received_latency` | ms | Slot start to first candidate received | | `time:slot_stage2_notarized_latency` | ms | Slot start to first notarize vote | | `time:slot_stage3_finalized_latency` | ms | Slot start to first finalize vote | -| `simplex_batch_commit_size` | count | Blocks committed per batch | +| `simplex_batch_commit_size` | count | Blocks applied per finalized batch (legacy metric name) | #### Receiver Counters @@ -888,8 +906,10 @@ All counters and progress gauges are registered as derivative metrics via `Metri - `simplex_last_finalized_slot` -- finalized slots per second - `simplex_first_non_finalized_slot` -- FSM advancement rate -- `simplex_commits.total` -- commit throughput +- `simplex_commits.total` -- finalized-delivery throughput (legacy metric name) - `simplex_validates.total` -- validation throughput +- `simplex_collation_starts` -- collation entry attempts per second +- `simplex_candidate_received_broadcast` + `simplex_candidate_received_query` -- peer-delivered candidate-body ingress rate (sum them for total ingress) ### Health Checks diff --git a/src/node/simplex/src/lib.rs b/src/node/simplex/src/lib.rs index 6bb2aae..b8077a7 100644 --- a/src/node/simplex/src/lib.rs +++ b/src/node/simplex/src/lib.rs @@ -92,7 +92,8 @@ //! Implements callbacks via [`SessionListener`] trait (from validator-session): //! - `on_candidate` - Validate incoming block candidate //! - `on_generate_slot` - Generate new block when leader -//! - `on_block_committed` - Block finalized in consensus +//! - `on_block_finalized` - Finalized block delivered to validator side +//! - `on_block_committed` - legacy sequential callback; not used by Simplex //! - `on_block_skipped` - Slot was skipped //! //! ## Type Relationships @@ -111,7 +112,7 @@ //! SessionListener (trait, implemented by caller) //! ├── on_candidate() //! ├── on_generate_slot() -//! ├── on_block_committed() +//! ├── on_block_finalized() //! └── on_block_skipped() //! //! Receiver (trait) ──sends──► Votes, BlockBroadcasts @@ -220,7 +221,7 @@ use std::{ sync::{Arc, Weak}, time::Duration, }; -use ton_block::{fail, Result, ShardIdent}; +use ton_block::{fail, BlockIdExt, Result, ShardIdent}; /* Shared Raw Vote Data (memory-efficient storage) @@ -314,7 +315,7 @@ pub mod ton { /// Sentinel value indicating Simplex roundless mode. /// /// When Simplex uses this value for the `round` field in callbacks (`on_candidate`, -/// `on_generate_slot`, `on_block_committed`), it signals to `ValidatorGroup` that +/// `on_generate_slot`, `on_block_finalized`), it signals to `ValidatorGroup` that /// round-based invariants should be bypassed. /// /// # Rationale @@ -350,40 +351,6 @@ pub type SessionListenerPtr = Weak; /// Log replay listener pointer pub type SessionReplayListenerPtr = consensus_common::ConsensusReplayListenerPtr; -/* - RestartRecommitStrategy for session restart behavior -*/ - -/// Strategy for replaying finalized blocks to ValidatorGroup on restart. -/// -/// After session restart, the consensus state is restored from the database. -/// This strategy controls how persisted finalized history is replayed in -/// roundless mode. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum RestartRecommitStrategy { - /// Replay the full persisted finalized chain in deterministic order. - /// - /// Replay uses commit-path semantics only: - /// - non-empty finalized records emit `on_block_committed` - /// - empty finalized records are replayed internally without - /// `on_block_skipped` callbacks - /// - /// The replay source must be consistent (parent-chain continuity and seqno - /// invariants); inconsistencies fail startup recovery. - #[default] - FullReplay, - - /// Do not replay historical finalized records (C++-like behavior). - /// - /// Only restore receiver caches and resume from first new block. - /// The first `on_block_committed` after restart will be for a newly - /// produced block, not a historical one. - /// - /// **Caution**: This assumes engine state is already consistent with - /// consensus progress. Use only when that invariant is guaranteed. - FirstCommitAfterFinalized, -} - /* SessionOptions for Simplex consensus */ @@ -460,16 +427,6 @@ pub struct SessionOptions { /// Default: false (non-blocking) pub wait_for_db_init: bool, - /// Strategy for replaying finalized blocks to ValidatorGroup on restart. - /// - /// Controls how the session handles the gap between restored consensus state - /// (from DB) and ValidatorGroup's `expected_current_round` (starts at 0). - /// - /// See [`RestartRecommitStrategy`] for available options. - /// - /// Default: `FullReplay` - pub restart_recommit_strategy: RestartRecommitStrategy, - /// Use QUIC overlay transport instead of ADNL UDP for this session. /// When true, overlay messages/queries are sent via QUIC streams. /// Default: false @@ -507,12 +464,12 @@ pub struct SessionOptions { pub candidate_resolve_timeout_cap: Duration, pub candidate_resolve_cooldown: Duration, - // TODO: wire into standstill recovery egress shaping. C++ pool.cpp uses this to - // rate-limit bytes/s during standstill_resolution_task. + // Wired into Receiver standstill replay token-bucket shaping. + // C++ parity: pool.cpp standstill_resolution_task byte budget. pub standstill_max_egress_bytes_per_s: u32, - // TODO: wire into slot/vote acceptance bounds. C++ consensus.cpp and pool.cpp use this - // to reject candidates/votes from too-far-future windows. + // Wired into slot/vote acceptance bounds in SimplexState + Receiver. + // C++ parity: consensus.cpp/pool.cpp future-window rejection. pub max_leader_window_desync: u32, // TODO: wire into peer ban logic. C++ pool.cpp bans peers with bad vote/cert signatures @@ -522,6 +479,17 @@ pub struct SessionOptions { // TODO: wire into candidate resolver rate limiting. C++ candidate-resolver.cpp uses a // 1-second sliding window with this limit per peer for requestCandidate. pub candidate_resolve_rate_limit: u32, + + /// When true, collation for non-first slots in a leader window waits until + /// the parent slot is notarized (or finalized) before producing the next + /// candidate. This avoids broadcasting blocks that validators cannot yet + /// accept because C++ `WaitForParent` defers until the parent is notarized. + /// + /// When false, in-window candidate chaining uses the locally generated + /// parent immediately (optimistic pipelining). + /// + /// Default: true + pub require_notarized_parent_for_collation: bool, } impl Default for SessionOptions { @@ -543,7 +511,6 @@ impl Default for SessionOptions { standstill_timeout: Duration::from_secs(10), empty_block_mc_lag_threshold: None, wait_for_db_init: false, - restart_recommit_strategy: RestartRecommitStrategy::default(), use_quic: false, health_alert_cooldown: Duration::from_secs(30), health_stall_warning_secs: 15, @@ -560,6 +527,7 @@ impl Default for SessionOptions { max_leader_window_desync: 250, bad_signature_ban_duration: Duration::from_secs(5), candidate_resolve_rate_limit: 10, + require_notarized_parent_for_collation: true, } } } @@ -691,43 +659,38 @@ impl SessionOptions { /// /// # MC Finalization Notification /// -/// For **shard chains**, empty block generation depends on masterchain -/// finalization status. When `last_mc_finalized_seqno + 8 < new_seqno`, -/// the session generates empty blocks instead of normal blocks. +/// For **shard chains**, empty block generation depends on the masterchain-applied +/// top for that shard. When `last_mc_finalized_seqno + 8 < new_seqno`, the session +/// generates empty blocks instead of normal blocks. /// /// The higher layer (ValidatorManager) should call `notify_mc_finalized()` -/// when masterchain blocks are finalized to enable this functionality. +/// when masterchain state is updated, passing the current applied top for +/// each simplex session shard. /// /// # Example /// /// ```ignore -/// // When MC block is finalized, notify shard sessions -/// if !shard.is_masterchain() { -/// simplex_session.notify_mc_finalized(mc_block_seqno); -/// } +/// // When MC state is updated, notify simplex sessions with their applied top. +/// simplex_session.notify_mc_finalized(applied_top_for_session_shard); /// ``` pub trait SimplexSession: ConsensusSession { - /// Notify session about masterchain finalization + /// Notify session about the current applied top for its shard /// /// # Purpose /// - /// For shard sessions, this updates `last_mc_finalized_seqno` which is used by - /// `should_generate_empty_block()` to decide if an empty block should be generated. + /// This updates session-local applied-top tracking: + /// - shard sessions use it for empty-block recovery against MC-registered tops + /// - masterchain sessions use it to mirror the applied MC head for validation/empty logic /// /// # When to Call /// - /// When a masterchain block is finalized, ValidatorManager should call this for all - /// shard validator sessions with the MC block's seqno. - /// - /// # For Masterchain Sessions - /// - /// This method is a no-op for masterchain sessions (they track their own finalization - /// internally via `last_committed_seqno`). + /// When masterchain state is updated, ValidatorManager should call this for all + /// simplex validator sessions with the current applied top for that session shard. /// /// # Arguments /// - /// * `mc_block_seqno` - The seqno of the finalized masterchain block - fn notify_mc_finalized(&self, mc_block_seqno: u32); + /// * `applied_top` - Current applied top for this session shard + fn notify_mc_finalized(&self, applied_top: BlockIdExt); /// Check if the session has fully stopped (all threads have terminated). /// diff --git a/src/node/simplex/src/receiver.rs b/src/node/simplex/src/receiver.rs index 8844318..a2d857f 100644 --- a/src/node/simplex/src/receiver.rs +++ b/src/node/simplex/src/receiver.rs @@ -56,10 +56,9 @@ use crate::{ block::{SlotIndex, ValidatorIndex}, - simplex_state::MAX_FUTURE_SLOTS, ActivityNodePtr, BlockPayloadPtr, ConsensusOverlayListener, ConsensusOverlayLogReplayListener, ConsensusOverlayManagerPtr, MetricsHandle, PrivateKey, PublicKey, PublicKeyHash, RawVoteData, - SessionId, SessionNode, ValidatorWeight, + SessionId, SessionNode, SessionOptions, ValidatorWeight, }; use consensus_common::{ check_execution_time, instrument, @@ -69,7 +68,7 @@ use consensus_common::{ use crossbeam::channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender}; use rand::{seq::SliceRandom, Rng}; use std::{ - collections::HashMap, + collections::{HashMap, HashSet, VecDeque}, mem::discriminant, panic::{catch_unwind, AssertUnwindSafe}, sync::{ @@ -91,7 +90,7 @@ use ton_api::{ CandidateAndCert as CandidateAndCertBoxed, Certificate, UnsignedVote, Vote as TlVoteBoxed, }, - CandidateData, CandidateParent, + CandidateData, CandidateParent, RequestError as ConsensusRequestError, }, pub_::publickey::Overlay, rpc::consensus::simplex::RequestCandidate, @@ -122,14 +121,9 @@ const SHUFFLE_SEND_ORDER_PERIOD: Duration = Duration::from_secs(10); // Period t const ACTIVE_WEIGHT_RECOMPUTE_PERIOD: Duration = Duration::from_secs(1); // Period to recompute active weight // Candidate request constants (block repair / candidate resolver) -// Per-request network query timeout (overlay send_query deadline) -const CANDIDATE_REQUEST_TIMEOUT: Duration = Duration::from_secs(3); -// C++ parity: candidate-resolver.cpp uses indefinite retry with exponential backoff. -// bus.h defaults: initial=0.5s, multiplier=1.5, max=30.0s -const CANDIDATE_REQUEST_INITIAL_TIMEOUT: Duration = Duration::from_millis(500); -const CANDIDATE_REQUEST_TIMEOUT_MULTIPLIER: f64 = 1.5; -const CANDIDATE_REQUEST_MAX_TIMEOUT: Duration = Duration::from_secs(30); -const CANDIDATE_REQUEST_MAX_RETRIES: u32 = 50; +const CANDIDATE_QUERY_RATE_LIMIT_WINDOW: Duration = Duration::from_secs(1); +const CANDIDATE_RETRY_WARN_INTERVAL: u32 = 50; +const CANDIDATE_SOFT_GIVEUP_REPORT_INTERVAL: Duration = Duration::from_secs(30); // Standstill initial range - used before first finalization calls set_standstill_slots() // After first finalization, SessionProcessor sets the actual range via set_standstill_slots() @@ -139,6 +133,61 @@ const STANDSTILL_INITIAL_SLOT_END: u32 = 1_000_000; // Import ACTIVITY_THRESHOLD from utils.rs for consistency with SimplexState use crate::utils::ACTIVITY_THRESHOLD; +/// Runtime candidate resolver knobs sourced from SessionOptions noncritical params. +#[derive(Clone, Copy, Debug)] +pub(crate) struct CandidateResolveConfig { + pub timeout: Duration, + pub timeout_multiplier: f64, + pub timeout_cap: Duration, + pub cooldown: Duration, + pub rate_limit: u32, +} + +impl CandidateResolveConfig { + pub(crate) fn from_session_options(options: &SessionOptions) -> Self { + Self { + timeout: options.candidate_resolve_timeout, + timeout_multiplier: options.candidate_resolve_timeout_multiplier, + timeout_cap: options.candidate_resolve_timeout_cap, + cooldown: options.candidate_resolve_cooldown, + rate_limit: options.candidate_resolve_rate_limit, + } + } +} + +impl Default for CandidateResolveConfig { + fn default() -> Self { + Self::from_session_options(&SessionOptions::default()) + } +} + +#[derive(Default)] +struct SlidingWindowRateLimiter { + timestamps: VecDeque, +} + +impl SlidingWindowRateLimiter { + fn allow(&mut self, now: SystemTime, window: Duration, limit: u32) -> bool { + if limit == 0 { + return false; + } + + while let Some(front) = self.timestamps.front() { + let expired = now.duration_since(*front).map_or(false, |elapsed| elapsed >= window); + if !expired { + break; + } + self.timestamps.pop_front(); + } + + if self.timestamps.len() as u32 >= limit { + return false; + } + self.timestamps.push_back(now); + true + } +} + /* Standstill Certificate Types */ @@ -157,6 +206,19 @@ pub(crate) enum StandstillCertificateType { Final, } +/// Standstill trigger notification sent to the session layer. +/// +/// The receiver owns replay queue construction and pacing, while the +/// `SessionProcessor` owns the C++-style pool-state diagnostic dump sourced +/// from `SimplexState`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct StandstillTriggerNotification { + pub begin: u32, + pub end: u32, + pub cert_count: u32, + pub vote_count: u32, +} + /* Receiver trait and type aliases @@ -252,6 +314,30 @@ pub(crate) trait Receiver: Send + Sync { /// * `up_to_slot` - Clean up all data for slots < up_to_slot fn cleanup(&self, up_to_slot: u32); + /// Update ingress slot lower bound for vote/certificate filtering. + /// + /// This tracks the consensus `first_non_finalized_slot` frontier and is + /// intentionally independent from history retention cleanup. + /// + /// # Arguments + /// * `slot` - First acceptable slot (inclusive) for ingress filtering + fn set_ingress_slot_begin(&self, slot: u32); + + /// Update ingress progress cursor used for the future-slot horizon. + /// + /// This tracks the consensus `first_non_progressed_slot` / C++ `now_` + /// cursor and controls how far ahead votes/certificates may be accepted. + /// + /// # Arguments + /// * `slot` - First non-progressed slot for ingress horizon calculations + fn set_ingress_progress_slot(&self, slot: u32); + + /// Cancel pending candidate-repair requests for a conclusively skipped slot. + /// + /// This mirrors the C++ model where skip/final/notar state advancement + /// resolves obsolete pending work instead of continuing to retry it. + fn cancel_candidate_requests_for_slot(&self, slot: u32); + /// Request a missing candidate from peers (block repair) /// /// Called by SessionProcessor when a finalization event requires a candidate @@ -265,6 +351,15 @@ pub(crate) trait Receiver: Send + Sync { /// * `block_hash` - Block hash of the missing candidate fn request_candidate(&self, slot: u32, block_hash: UInt256); + /// Arm standstill detection. + /// + /// Called when the session is promoted to current and `Session::start(seqno)` + /// has been invoked. Before this call the standstill alarm is disarmed + /// (overlay warms up silently). Matches C++ behavior where the bridge + /// publishes the Start event only after both `create_session` and `start` + /// have completed. + fn start(&self); + /// Reschedule standstill alarm /// /// Resets the standstill timer to fire after `standstill_timeout`. @@ -381,12 +476,17 @@ pub(crate) trait ReceiverListener: Send + Sync { /// - last_activity: last receive time per validator (None if never received) fn on_activity(&self, active_weight: ValidatorWeight, last_activity: Vec>); - /// Fallback for RequestCandidate queries when resolver_cache misses. + /// Standstill alarm fired and a fresh replay snapshot was built. /// - /// Called by `handle_query()` when `want_candidate=true` but the resolver_cache - /// does not have the candidate data. Delegates to SessionProcessor which can - /// reconstruct the response from its in-memory `candidate_data_cache`, rebuild - /// an empty candidate from `CandidateInfo`, or load persisted payloads from SimplexDB. + /// The session layer uses this to emit the C++-style standstill diagnostic + /// dump from `SimplexState` at the same logical trigger point as the + /// receiver's standstill replay handling. + fn on_standstill_trigger(&self, notification: StandstillTriggerNotification); + + /// Fallback for RequestCandidate queries when resolver_cache misses requested parts. + /// + /// Delegates to SessionProcessor which can reconstruct candidate body and/or notar + /// from in-memory cache, metadata, and SimplexDB payload/certificate storage. /// /// This achieves parity with C++ `CandidateResolver::try_load_candidate_data_from_db()`. /// @@ -395,6 +495,7 @@ pub(crate) trait ReceiverListener: Send + Sync { &self, slot: SlotIndex, block_hash: UInt256, + want_candidate: bool, want_notar: bool, response_callback: QueryResponseCallback, ); @@ -439,6 +540,10 @@ struct CandidateRequestState { retry_count: u32, /// Current timeout for this request (grows with exponential backoff) current_timeout: Duration, + /// Monotonic attempt id for stale timeout/response filtering. + attempt_id: u64, + /// True while exactly one outbound query is in-flight for this request. + in_flight: bool, /// Validator index of the peer being queried source_idx: ValidatorIndex, /// Accumulated notar bytes from partial responses (C++ CandidateAndCert::merge parity). @@ -449,6 +554,8 @@ struct CandidateRequestState { /// Peers may return candidate-only while notar is still missing; cache the body so /// a later notar-only response can complete the merged result. cached_candidate: Option>, + /// Number of soft-giveup reports emitted for this request. + giveup_reports: u32, } /* @@ -766,6 +873,18 @@ struct DeduplicationKey { vote_hash: UInt256, } +#[derive(Hash, Eq, PartialEq, Clone)] +struct StandstillVoteKey { + slot: u32, + kind: u8, + candidate_hash: Option, +} + +enum StandstillReplayItem { + Vote(TlVote), + Certificate(Vec), +} + /* ReceiverImpl - internal state, single-threaded operations */ @@ -810,6 +929,8 @@ pub(crate) struct ReceiverImpl { max_candidate_query_answer_size: u64, /// Protocol version from consensus config (determines BOC serialization flags) proto_version: u32, + /// Candidate resolver runtime config sourced from SessionOptions. + candidate_resolve_config: CandidateResolveConfig, /// Metrics in_messages_bytes: metrics::Counter, out_messages_bytes: metrics::Counter, @@ -824,6 +945,8 @@ pub(crate) struct ReceiverImpl { _activity_node: ActivityNodePtr, /// Standstill timeout duration standstill_timeout: Duration, + /// Standstill replay egress budget in bytes/sec. + standstill_max_egress_bytes_per_s: u32, /// Next standstill alarm timestamp (reset on finalization and after re-broadcast) standstill_alarm: Option, /// Standstill slot range [begin, end) for vote re-broadcast @@ -839,6 +962,14 @@ pub(crate) struct ReceiverImpl { /// Stored when send_vote_impl() is called /// Format: (slot, signed_vote) our_votes: Vec<(u32, TlVote)>, + /// Dedup set for `our_votes` replay cache. + our_vote_keys: HashSet, + /// Pending standstill replay items to be sent under egress shaping. + standstill_replay_queue: VecDeque, + /// Current token bucket quota for standstill replay (bytes). + standstill_egress_quota_bytes: f64, + /// Last token bucket update time. + standstill_egress_quota_time: SystemTime, /// Candidate resolver cache (local to this thread) resolver_cache: CandidateResolverCache, /// Delayed actions to execute at scheduled times @@ -847,6 +978,8 @@ pub(crate) struct ReceiverImpl { /// Pending candidate requests (outbound): (slot, block_hash) → request state /// Used to track ongoing block repair requests to other validators pending_requests: HashMap<(SlotIndex, UInt256), CandidateRequestState>, + /// Per-peer inbound requestCandidate rate limiters. + candidate_query_rate_limiters: HashMap, /// Task queues for posting callbacks from overlay responses task_queues: Arc, /// Standstill certificate cache: slot → certificate bundle bytes @@ -858,11 +991,19 @@ pub(crate) struct ReceiverImpl { /// Format: (slot, serialized_cert_bytes) /// Reference: C++ pool.cpp last_final_cert_ last_final_cert: Option<(u32, Vec)>, - /// Finalization cursor for ingress DoS protection. - /// Updated by `cleanup()` when SessionProcessor advances finalization. + /// Ingress slot lower bound for DoS protection. + /// Updated from SessionProcessor's finalized frontier. /// Used to reject far-future votes/certificates before expensive operations /// (signature verification, dedup HashMap insertion). first_active_slot: u32, + /// Ingress progress cursor for far-future DoS protection. + /// Updated from SessionProcessor's progress frontier (`first_non_progressed_slot` / C++ `now_`). + /// Used as the base for the acceptable future horizon. + ingress_progress_slot: u32, + /// Session slots-per-leader-window config (for C++-parity slot bounds). + slots_per_leader_window: u32, + /// Session max-leader-window-desync config (for C++-parity slot bounds). + max_leader_window_desync: u32, candidate_requests_counter: metrics::Counter, candidate_request_retries_counter: metrics::Counter, candidate_request_timeouts_counter: metrics::Counter, @@ -904,17 +1045,17 @@ impl ReceiverImpl { stats.last_recv_time = Some(SystemTime::now()); } - // DoS protection: reject far-future/negative slots BEFORE expensive - // signature verification and dedup HashMap insertion. + // Keep the receiver prefilter cheap; SessionProcessor mirrors the C++ + // warning behavior while the receiver only traces Rust-side drops. let slot = Self::get_vote_slot(&vote); - if self.is_slot_out_of_bounds(slot) { - log::warn!( - "SimplexReceiver {}: REJECTED vote from source {} - slot {} out of bounds [{}, {}]", + if self.is_vote_slot_out_of_bounds(slot) { + log::trace!( + "SimplexReceiver {}: dropped vote from source {} - slot {} outside [{}, {})", self.session_id.to_hex_string(), source_idx, slot, self.first_active_slot, - self.max_acceptable_slot() + self.first_too_new_vote_slot() ); return; } @@ -1006,15 +1147,15 @@ impl ReceiverImpl { sigs ); - // DoS protection: reject far-future/negative slots before forwarding. - if self.is_slot_out_of_bounds(slot) { - log::warn!( - "SimplexReceiver {}: REJECTED certificate from source {} - slot {} out of bounds [{}, {}] kind={}", + // Mirror C++: drop finalized/old certificates cheaply, but do not reject + // a certificate only because its slot is ahead of the current vote horizon. + if self.is_certificate_slot_too_old(slot) { + log::trace!( + "SimplexReceiver {}: dropped old certificate from source {} - slot {} < {} kind={}", self.session_id.to_hex_string(), source_idx, slot, self.first_active_slot, - self.max_acceptable_slot(), kind ); return; @@ -1357,7 +1498,7 @@ impl ReceiverImpl { /// which can reconstruct the response from in-memory or DB-backed storage. fn handle_query( &mut self, - _adnl_id: PublicKeyHash, + adnl_id: PublicKeyHash, data: BlockPayloadPtr, response_callback: QueryResponseCallback, ) { @@ -1384,6 +1525,19 @@ impl ReceiverImpl { let want_candidate: bool = req.want_candidate.into(); let want_notar: bool = req.want_notar.into(); + if !self.check_candidate_query_rate_limit(&adnl_id) { + log::warn!( + "SimplexReceiver {}: requestCandidate rate-limited \ + slot={} hash={} from {}", + self.session_id.to_hex_string(), + slot, + &block_hash.to_hex_string()[..8], + key_to_base64(&adnl_id), + ); + response_callback(Err(error!("too many requests"))); + return; + } + log::trace!( "SimplexReceiver {}: requestCandidate slot={} hash={} want_candidate={} want_notar={}", self.session_id.to_hex_string(), @@ -1396,22 +1550,31 @@ impl ReceiverImpl { let candidate_bytes = if want_candidate { self.resolver_cache.get_candidate(slot, &block_hash).cloned() } else { - None + Some(Vec::new()) }; + let notar_bytes = if want_notar { + self.resolver_cache.get_notar_cert(slot, &block_hash).cloned() + } else { + Some(Vec::new()) + }; + let candidate_miss = want_candidate && candidate_bytes.is_none(); + let notar_miss = want_notar && notar_bytes.is_none(); - let cache_miss = want_candidate && candidate_bytes.is_none(); - - if cache_miss { + if candidate_miss || notar_miss { if let Some(listener) = self.listener.upgrade() { log::debug!( "SimplexReceiver {}: requestCandidate cache MISS \ - for slot={slot} hash={}, delegating to SessionProcessor", + for slot={slot} hash={} (candidate_miss={}, notar_miss={}), \ + delegating to SessionProcessor", self.session_id.to_hex_string(), &block_hash.to_hex_string()[..8], + candidate_miss, + notar_miss, ); listener.on_candidate_query_fallback( slot, block_hash, + want_candidate, want_notar, response_callback, ); @@ -1426,14 +1589,7 @@ impl ReceiverImpl { } let candidate_bytes = candidate_bytes.unwrap_or_default(); - let notar_bytes = if want_notar { - self.resolver_cache - .get_notar_cert(slot, &block_hash) - .cloned() - .unwrap_or_default() - } else { - Vec::new() - }; + let notar_bytes = notar_bytes.unwrap_or_default(); let response = CandidateAndCert { candidate: candidate_bytes.into(), @@ -1473,6 +1629,15 @@ impl ReceiverImpl { response_callback(Err(error!("Unknown query type"))); } + fn check_candidate_query_rate_limit(&mut self, adnl_id: &PublicKeyHash) -> bool { + let limiter = self.candidate_query_rate_limiters.entry(adnl_id.clone()).or_default(); + limiter.allow( + SystemTime::now(), + CANDIDATE_QUERY_RATE_LIMIT_WINDOW, + self.candidate_resolve_config.rate_limit, + ) + } + /// Cache candidate data for resolver queries fn cache_candidate(&mut self, slot: SlotIndex, block_hash: UInt256, data: Vec) { log::trace!( @@ -1575,16 +1740,14 @@ impl ReceiverImpl { Reference: C++ CandidateResolver in validator/consensus/candidate-resolver.cpp */ - /// Request a missing candidate from peers + /// Request a missing candidate from peers. /// - /// Sends a requestCandidate query to a peer and schedules retry on timeout. - /// On successful response, calls on_candidate_received on the listener. + /// C++ parity: one in-flight request per id; retries happen only after + /// the prior attempt resolves (response/timeout), with backoff + cooldown. fn request_candidate_impl(&mut self, slot: SlotIndex, block_hash: UInt256) { check_execution_time!(50_000); let key = (slot, block_hash.clone()); - - // Check if already pending if self.pending_requests.contains_key(&key) { log::trace!( "SimplexReceiver {}: request_candidate slot={} hash={} - already pending", @@ -1595,7 +1758,6 @@ impl ReceiverImpl { return; } - // Select a random peer to query (skip self) let source_idx = match self.select_peer_for_candidate_request(None) { Some(idx) => idx, None => { @@ -1610,30 +1772,22 @@ impl ReceiverImpl { }; self.candidate_requests_counter.increment(1); - - // Create request state - let request_state = CandidateRequestState { - start_time: SystemTime::now(), - retry_count: 0, - current_timeout: CANDIDATE_REQUEST_INITIAL_TIMEOUT, - source_idx, - cached_notar: None, - cached_candidate: None, - }; - self.pending_requests.insert(key.clone(), request_state); - - // Send the query - self.send_candidate_request(slot, block_hash.clone(), source_idx); - - // Schedule timeout handler - let slot_clone = slot; - let hash_clone = block_hash.clone(); - self.post_delayed_action( - SystemTime::now() + CANDIDATE_REQUEST_INITIAL_TIMEOUT, - move |receiver: &mut ReceiverImpl| { - receiver.handle_candidate_request_timeout(slot_clone, hash_clone); + self.pending_requests.insert( + key, + CandidateRequestState { + start_time: SystemTime::now(), + retry_count: 0, + current_timeout: self.candidate_resolve_config.timeout, + attempt_id: 0, + in_flight: false, + source_idx, + cached_notar: None, + cached_candidate: None, + giveup_reports: 0, }, ); + + self.send_candidate_request(slot, block_hash); } /// Select a peer for candidate request, skipping self and (optionally) a specific peer. @@ -1676,13 +1830,52 @@ impl ReceiverImpl { None } - /// Send the actual requestCandidate query to a peer - fn send_candidate_request( - &mut self, - slot: SlotIndex, - block_hash: UInt256, - source_idx: ValidatorIndex, - ) { + fn next_candidate_timeout(&self, current_timeout: Duration) -> Duration { + let next_timeout_ms = (current_timeout.as_millis() as f64 + * self.candidate_resolve_config.timeout_multiplier) + as u128; + Duration::from_millis( + next_timeout_ms.min(self.candidate_resolve_config.timeout_cap.as_millis()) as u64, + ) + } + + /// Send one requestCandidate attempt for a pending request. + fn send_candidate_request(&mut self, slot: SlotIndex, block_hash: UInt256) { + let key = (slot, block_hash.clone()); + let (source_idx, request_timeout, want_candidate, want_notar) = { + let Some(state_ro) = self.pending_requests.get(&key) else { + return; + }; + let have_candidate = state_ro.cached_candidate.is_some() + || self.resolver_cache.get_candidate(slot, &block_hash).is_some(); + let have_notar = state_ro.cached_notar.is_some() + || self.resolver_cache.get_notar_cert(slot, &block_hash).is_some(); + let want_candidate = !have_candidate; + let want_notar = !have_notar; + (state_ro.source_idx, state_ro.current_timeout, want_candidate, want_notar) + }; + + if !want_candidate && !want_notar { + log::trace!( + "SimplexReceiver {}: requestCandidate slot={slot} hash={} \ + already complete in cache, cancelling pending request", + self.session_id.to_hex_string(), + &block_hash.to_hex_string()[..8], + ); + self.pending_requests.remove(&key); + return; + } + + let attempt_id = { + let Some(state) = self.pending_requests.get_mut(&key) else { + return; + }; + // Use monotonic attempt ids for stale timeout/response filtering. + state.attempt_id = state.attempt_id.saturating_add(1); + state.in_flight = true; + state.attempt_id + }; + let peer_adnl_id = match self.sources.get(source_idx.value() as usize) { Some(stats) => stats.adnl_id.clone(), None => { @@ -1698,8 +1891,8 @@ impl ReceiverImpl { let candidate_id = CandidateId { slot: slot.value() as i32, hash: block_hash.clone() }; let request = RequestCandidate { id: candidate_id.into_boxed(), - want_candidate: true.into(), - want_notar: true.into(), + want_candidate: want_candidate.into(), + want_notar: want_notar.into(), }; let (serialized, query_name) = (serialize_boxed(&request), "requestCandidate"); @@ -1718,22 +1911,20 @@ impl ReceiverImpl { let payload = ConsensusCommonFactory::create_block_payload(serialized); log::trace!( - "SimplexReceiver {}: sending {} slot={} hash={} to validator {}", + "SimplexReceiver {}: sending {query_name} slot={slot} hash={} \ + to validator {source_idx} (attempt={attempt_id} timeout={request_timeout:?} \ + want_candidate={want_candidate} want_notar={want_notar})", self.session_id.to_hex_string(), - query_name, - slot, &block_hash.to_hex_string()[..8], - source_idx, ); // Capture data for callback (we need to move these into the closure) let slot_for_cb = slot; let hash_for_cb = block_hash.clone(); - let session_id = self.session_id.clone(); let task_queues = self.get_task_queues(); // Send query via RLDP overlay with explicit response size budget (C++ PR #2195 parity) - let timeout_deadline = SystemTime::now() + CANDIDATE_REQUEST_TIMEOUT; + let timeout_deadline = SystemTime::now() + request_timeout; self.overlay.send_query_via_rldp( peer_adnl_id, query_name.to_string(), @@ -1743,7 +1934,7 @@ impl ReceiverImpl { slot_for_cb, hash_for_cb, result, - session_id, + attempt_id, ); })); }), @@ -1752,6 +1943,13 @@ impl ReceiverImpl { self.max_candidate_query_answer_size, true, // RLDPv2 ); + + // Timeout callback for this exact attempt. + let slot_clone = slot; + let hash_clone = block_hash; + self.post_delayed_action(SystemTime::now() + request_timeout, move |receiver| { + receiver.handle_candidate_request_timeout(slot_clone, hash_clone, attempt_id); + }); } /// Get task queues for callback posting @@ -1819,50 +2017,58 @@ impl ReceiverImpl { slot: SlotIndex, block_hash: UInt256, result: Result, - _session_id: SessionId, + attempt_id: u64, ) { check_execution_time!(50_000); let key = (slot, block_hash.clone()); - // Check if request is still pending (might have been fulfilled by broadcast) - if !self.pending_requests.contains_key(&key) { - log::trace!( - "SimplexReceiver {}: candidate response for slot={} hash={} - no longer pending", - self.session_id.to_hex_string(), - slot, - &block_hash.to_hex_string()[..8] - ); - return; - } + let source_idx = match self.pending_requests.get_mut(&key) { + Some(state) if state.attempt_id == attempt_id && state.in_flight => { + state.in_flight = false; + state.source_idx.value() + } + Some(_) => { + log::trace!( + "SimplexReceiver {}: stale candidate response slot={} hash={} attempt={} ignored", + self.session_id.to_hex_string(), + slot, + &block_hash.to_hex_string()[..8], + attempt_id + ); + return; + } + None => { + log::trace!( + "SimplexReceiver {}: candidate response for slot={} hash={} - no longer pending", + self.session_id.to_hex_string(), + slot, + &block_hash.to_hex_string()[..8] + ); + return; + } + }; match result { Ok(response_payload) => { // Deserialize response let response_data = response_payload.data(); match deserialize_boxed(response_data) { - Ok(message) => { - if let Ok(response) = message.downcast::() { - // Get source_idx from pending request before removing - let source_idx = self - .pending_requests - .get(&key) - .map(|state| state.source_idx.value()) - .unwrap_or(0); - - // Successfully received response + Ok(message) => match message.downcast::() { + Ok(response) => { let candidate_bytes = response.candidate(); let notar_bytes = response.notar(); log::trace!( "SimplexReceiver {}: received candidate response slot={} hash={} \ - candidate_len={} notar_len={} from validator {}", + candidate_len={} notar_len={} from validator {} (attempt={})", self.session_id.to_hex_string(), slot, &block_hash.to_hex_string()[..8], candidate_bytes.len(), notar_bytes.len(), - source_idx + source_idx, + attempt_id ); // C++ CandidateAndCert::merge parity: cache both partial fields and @@ -1877,27 +2083,38 @@ impl ReceiverImpl { notar_bytes, ); - // If body is still missing after merge, keep pending for retry. if merged_candidate_bytes.is_empty() { log::debug!( "SimplexReceiver {}: body-empty response for slot={} hash={} \ - (notar_len={}), will retry on timeout", + (notar_len={}), scheduling retry", self.session_id.to_hex_string(), slot, &block_hash.to_hex_string()[..8], notar_bytes.len(), ); + self.retry_candidate_request( + slot, + block_hash, + false, + "missing_body_after_merge", + ); return; } if merged_notar.is_empty() { log::debug!( "SimplexReceiver {}: candidate-only partial response for \ - slot={} hash={}, keep pending until notar arrives", + slot={} hash={}, scheduling retry", self.session_id.to_hex_string(), slot, &block_hash.to_hex_string()[..8], ); + self.retry_candidate_request( + slot, + block_hash, + false, + "missing_notar_after_merge", + ); return; } @@ -1907,8 +2124,6 @@ impl ReceiverImpl { Ok(msg) => match msg.downcast::() { Ok(c) => c, Err(_) => { - // Drop cached candidate so retry can fetch a fresh body; - // also purge resolver_cache to avoid serving bad data to peers. self.resolver_cache.remove_candidate(slot, &block_hash); if let Some(state) = self.pending_requests.get_mut(&key) { state.cached_candidate = None; @@ -1917,12 +2132,16 @@ impl ReceiverImpl { "SimplexReceiver {}: unexpected candidate type in response", self.session_id.to_hex_string() ); + self.retry_candidate_request( + slot, + block_hash, + false, + "bad_candidate_type", + ); return; } }, Err(e) => { - // Drop cached candidate so retry can fetch a fresh body; - // also purge resolver_cache to avoid serving bad data to peers. self.resolver_cache.remove_candidate(slot, &block_hash); if let Some(state) = self.pending_requests.get_mut(&key) { state.cached_candidate = None; @@ -1932,6 +2151,12 @@ impl ReceiverImpl { self.session_id.to_hex_string(), e ); + self.retry_candidate_request( + slot, + block_hash, + false, + "candidate_deserialize_error", + ); return; } }; @@ -1939,7 +2164,6 @@ impl ReceiverImpl { // Remove from pending only when merged candidate+notar is complete. self.pending_requests.remove(&key); - // Call listener with source_idx, using merged notar. if let Some(listener) = self.listener.upgrade() { listener.on_candidate_received( source_idx, @@ -1947,131 +2171,428 @@ impl ReceiverImpl { Some(merged_notar), ); } - } else { - log::warn!( - "SimplexReceiver {}: unexpected response type for requestCandidate", - self.session_id.to_hex_string() - ); } - } + Err(message) => { + if message.downcast::().is_ok() { + log::debug!( + "SimplexReceiver {}: peer returned requestError for slot={} hash={}", + self.session_id.to_hex_string(), + slot, + &block_hash.to_hex_string()[..8] + ); + self.retry_candidate_request( + slot, + block_hash, + true, + "request_error", + ); + } else { + log::warn!( + "SimplexReceiver {}: \ + unexpected response type for requestCandidate", + self.session_id.to_hex_string() + ); + self.retry_candidate_request( + slot, + block_hash, + false, + "unexpected_response_type", + ); + } + } + }, Err(e) => { log::warn!( "SimplexReceiver {}: failed to deserialize candidate response: {}", self.session_id.to_hex_string(), e ); + self.retry_candidate_request( + slot, + block_hash, + false, + "response_deserialize_error", + ); } } } Err(e) => { log::trace!( - "SimplexReceiver {}: candidate request failed slot={} hash={}: {}", + "SimplexReceiver {}: candidate request failed slot={} hash={} \ + attempt={}: {}", self.session_id.to_hex_string(), slot, &block_hash.to_hex_string()[..8], + attempt_id, e ); - // Error will be handled by timeout - don't retry here to avoid duplicates + self.retry_candidate_request(slot, block_hash, true, "request_timeout_or_error"); } } } - /// Handle request timeout - retry with next peer using exponential backoff. - /// C++ parity: candidate-resolver.cpp retries indefinitely until resolved. - fn handle_candidate_request_timeout(&mut self, slot: SlotIndex, block_hash: UInt256) { + fn retry_candidate_request( + &mut self, + slot: SlotIndex, + block_hash: UInt256, + count_timeout: bool, + reason: &'static str, + ) { let key = (slot, block_hash.clone()); - - // Check if request is still pending and get current state - let (retry_count, prev_source_idx, current_timeout) = match self.pending_requests.get(&key) - { - Some(state) => (state.retry_count, state.source_idx.value(), state.current_timeout), - None => { - log::trace!( - "SimplexReceiver {}: handle_candidate_request_timeout slot={} hash={} - request already fulfilled or cancelled", - self.session_id.to_hex_string(), - slot, - &block_hash.to_hex_string()[..8] - ); + let Some(state_ro) = self.pending_requests.get(&key) else { + return; + }; + let prev_source_idx = state_ro.source_idx.value(); + let current_timeout = state_ro.current_timeout; + let start_time = state_ro.start_time; + let prev_giveup_reports = state_ro.giveup_reports; + + let next_source_idx = self + .select_peer_for_candidate_request(Some(prev_source_idx)) + .unwrap_or_else(|| ValidatorIndex::new(prev_source_idx)); + let next_timeout = self.next_candidate_timeout(current_timeout); + + let (retry_count, emit_soft_giveup) = { + let Some(state) = self.pending_requests.get_mut(&key) else { return; + }; + state.retry_count = state.retry_count.saturating_add(1); + state.source_idx = next_source_idx; + state.current_timeout = next_timeout; + state.in_flight = false; + + let elapsed = SystemTime::now().duration_since(start_time).unwrap_or_default(); + let threshold_secs = CANDIDATE_SOFT_GIVEUP_REPORT_INTERVAL + .as_secs() + .saturating_mul((prev_giveup_reports as u64).saturating_add(1)) + .max(1); + let emit_soft_giveup = elapsed >= Duration::from_secs(threshold_secs); + if emit_soft_giveup { + state.giveup_reports = state.giveup_reports.saturating_add(1); } + (state.retry_count, emit_soft_giveup) }; - self.candidate_request_timeouts_counter.increment(1); - let new_retry_count = retry_count + 1; - if new_retry_count % CANDIDATE_REQUEST_MAX_RETRIES == 0 { + if count_timeout { + self.candidate_request_timeouts_counter.increment(1); + } + self.candidate_request_retries_counter.increment(1); + + if retry_count % CANDIDATE_RETRY_WARN_INTERVAL == 0 { log::warn!( "SimplexReceiver {}: candidate request slot={slot} hash={} \ - still pending after {new_retry_count} retries, continuing", + still pending after {} retries (reason={})", self.session_id.to_hex_string(), - &block_hash.to_hex_string()[..8] + &block_hash.to_hex_string()[..8], + retry_count, + reason ); } - // Exponential backoff: timeout * multiplier, capped at max - let next_timeout_ms = - (current_timeout.as_millis() as f64 * CANDIDATE_REQUEST_TIMEOUT_MULTIPLIER) as u128; - let next_timeout = Duration::from_millis( - next_timeout_ms.min(CANDIDATE_REQUEST_MAX_TIMEOUT.as_millis()) as u64, - ); - - // Select next peer (random, excluding previous) - let next_source_idx = match self.select_peer_for_candidate_request(Some(prev_source_idx)) { - Some(idx) => idx, - None => { - // No peers available right now -- schedule a retry after backoff anyway, - // peers may come back online. - self.candidate_request_retries_counter.increment(1); - log::warn!( - "SimplexReceiver {}: no peers for candidate request slot={slot} hash={}, \ - will retry in {next_timeout:?}", - self.session_id.to_hex_string(), - &block_hash.to_hex_string()[..8] - ); - if let Some(state) = self.pending_requests.get_mut(&key) { - state.retry_count = new_retry_count; - state.current_timeout = next_timeout; - } - let slot_clone = slot; - let hash_clone = block_hash; - self.post_delayed_action( - SystemTime::now() + next_timeout, - move |receiver: &mut ReceiverImpl| { - receiver.handle_candidate_request_timeout(slot_clone, hash_clone); - }, - ); - return; - } - }; - - // Update request state - self.candidate_request_retries_counter.increment(1); - if let Some(state) = self.pending_requests.get_mut(&key) { - state.retry_count = new_retry_count; - state.source_idx = next_source_idx; - state.current_timeout = next_timeout; + if emit_soft_giveup { + self.candidate_request_giveups_counter.increment(1); + self.health_counters.candidate_giveups.fetch_add(1, Ordering::Relaxed); + log::warn!( + "SimplexReceiver {}: candidate request slot={slot} hash={} soft_giveup_report \ + retry_count={} reason={}", + self.session_id.to_hex_string(), + &block_hash.to_hex_string()[..8], + retry_count, + reason + ); } log::trace!( - "SimplexReceiver {}: retrying candidate request slot={slot} hash={} \ - to validator {next_source_idx} (retry {new_retry_count}, timeout {next_timeout:?})", + "SimplexReceiver {}: scheduling candidate retry slot={} hash={} \ + to validator {} in {:?} (retry {}, reason={}, next_query_timeout={:?})", self.session_id.to_hex_string(), - &block_hash.to_hex_string()[..8] + slot, + &block_hash.to_hex_string()[..8], + next_source_idx, + self.candidate_resolve_config.cooldown, + retry_count, + reason, + next_timeout ); - // Send to next peer - self.send_candidate_request(slot, block_hash.clone(), next_source_idx); - - // Schedule next timeout with backoff let slot_clone = slot; let hash_clone = block_hash; self.post_delayed_action( - SystemTime::now() + next_timeout, + SystemTime::now() + self.candidate_resolve_config.cooldown, move |receiver: &mut ReceiverImpl| { - receiver.handle_candidate_request_timeout(slot_clone, hash_clone); + receiver.send_candidate_request(slot_clone, hash_clone); }, ); } + /// Handle timeout for one specific attempt id. + fn handle_candidate_request_timeout( + &mut self, + slot: SlotIndex, + block_hash: UInt256, + attempt_id: u64, + ) { + let key = (slot, block_hash.clone()); + let timed_out = match self.pending_requests.get_mut(&key) { + Some(state) if state.attempt_id == attempt_id && state.in_flight => { + state.in_flight = false; + true + } + _ => false, + }; + if !timed_out { + log::trace!( + "SimplexReceiver {}: candidate timeout slot={slot} hash={} attempt={attempt_id} \ + ignored (stale or completed)", + self.session_id.to_hex_string(), + &block_hash.to_hex_string()[..8] + ); + return; + } + + self.retry_candidate_request(slot, block_hash, true, "timeout"); + } + + fn standstill_vote_key(vote: &TlVote) -> StandstillVoteKey { + match &vote.vote { + UnsignedVote::Consensus_Simplex_NotarizeVote(v) => StandstillVoteKey { + slot: *v.id.slot() as u32, + kind: 0, + candidate_hash: Some(v.id.hash().clone()), + }, + UnsignedVote::Consensus_Simplex_SkipVote(v) => { + StandstillVoteKey { slot: v.slot as u32, kind: 1, candidate_hash: None } + } + UnsignedVote::Consensus_Simplex_FinalizeVote(v) => StandstillVoteKey { + slot: *v.id.slot() as u32, + kind: 2, + candidate_hash: Some(v.id.hash().clone()), + }, + } + } + + fn cache_vote_for_standstill(&mut self, vote: TlVote) { + let key = Self::standstill_vote_key(&vote); + if !self.our_vote_keys.insert(key) { + return; + } + let slot = Self::get_vote_slot_from_inner(&vote); + self.our_votes.push((slot, vote)); + self.standstill_slot_end = self.standstill_slot_end.max(slot.saturating_add(1)); + } + + fn rebuild_standstill_vote_keys(&mut self) { + self.our_vote_keys.clear(); + for (_, vote) in &self.our_votes { + self.our_vote_keys.insert(Self::standstill_vote_key(vote)); + } + } + + fn standstill_send_recipient_count(&self) -> u64 { + self.send_order.iter().filter(|&&idx| idx != self.local_idx).count() as u64 + } + + fn standstill_cost_recipient_count(&self) -> u64 { + self.send_order.len() as u64 + } + + fn estimate_standstill_replay_item_cost(&self, item: &StandstillReplayItem) -> u64 { + let recipient_count = self.standstill_cost_recipient_count(); + if recipient_count == 0 { + return 0; + } + match item { + StandstillReplayItem::Vote(vote) => { + let serialized = + consensus_common::serialize_tl_boxed_object!(&vote.clone().into_boxed()); + serialized.len() as u64 * recipient_count + } + StandstillReplayItem::Certificate(bytes) => bytes.len() as u64 * recipient_count, + } + } + + fn send_serialized_certificate_to_all(&mut self, bytes: Vec) { + let payload = ConsensusCommonFactory::create_block_payload(bytes.into()); + let msg_size = payload.data().len() as u64; + let recipient_count = self.standstill_send_recipient_count(); + + self.out_messages_bytes.increment(msg_size * recipient_count); + self.out_bytes.increment(msg_size * recipient_count); + self.out_messages_count.increment(recipient_count); + + for &target_idx in &self.send_order { + if target_idx == self.local_idx { + continue; + } + + if let Some(stats) = self.sources.get_mut(target_idx as usize) { + stats.out_messages += 1; + stats.last_send_time = Some(SystemTime::now()); + + self.overlay.send_message( + &stats.adnl_id, + &self.local_adnl_id, + &payload, + false, // is_retransmission=false for simplex + ); + } + } + } + + fn send_standstill_replay_item(&mut self, item: StandstillReplayItem) { + match item { + StandstillReplayItem::Vote(vote) => self.send_vote_impl(vote, true), + StandstillReplayItem::Certificate(bytes) => { + self.send_serialized_certificate_to_all(bytes) + } + } + } + + fn refill_standstill_egress_quota(&mut self) { + if self.standstill_max_egress_bytes_per_s == 0 { + self.standstill_egress_quota_bytes = f64::INFINITY; + self.standstill_egress_quota_time = SystemTime::now(); + return; + } + + let now = SystemTime::now(); + if let Ok(elapsed) = now.duration_since(self.standstill_egress_quota_time) { + let max_bytes = self.standstill_max_egress_bytes_per_s as f64; + // Allow accumulation across multiple seconds; otherwise a replay item bigger + // than one-second budget could starve forever. + self.standstill_egress_quota_bytes += elapsed.as_secs_f64() * max_bytes; + } + self.standstill_egress_quota_time = now; + } + + fn reset_standstill_egress_budget_for_replay(&mut self) { + if self.standstill_max_egress_bytes_per_s == 0 { + self.standstill_egress_quota_bytes = f64::INFINITY; + self.standstill_egress_quota_time = SystemTime::now(); + return; + } + + let now = SystemTime::now(); + self.standstill_egress_quota_bytes = 0.0; + self.standstill_egress_quota_time = + now.checked_sub(Duration::from_millis(10)).unwrap_or(now); + } + + fn drain_standstill_replay_queue(&mut self) { + if self.standstill_replay_queue.is_empty() { + return; + } + + if self.standstill_max_egress_bytes_per_s == 0 { + while let Some(item) = self.standstill_replay_queue.pop_front() { + self.send_standstill_replay_item(item); + } + return; + } + + self.refill_standstill_egress_quota(); + + while let Some(front) = self.standstill_replay_queue.front() { + let cost = self.estimate_standstill_replay_item_cost(front) as f64; + if cost > 0.0 && self.standstill_egress_quota_bytes < cost { + break; + } + if cost > 0.0 { + self.standstill_egress_quota_bytes -= cost; + } + let item = self.standstill_replay_queue.pop_front().expect("queue front exists"); + self.send_standstill_replay_item(item); + } + } + + fn collect_standstill_certificate_payloads(&self, begin: u32, end: u32) -> Vec> { + let mut cert_bytes_list: Vec> = Vec::new(); + + if let Some((_slot, bytes)) = &self.last_final_cert { + cert_bytes_list.push(bytes.clone()); + } + + let mut slots: Vec = self + .standstill_certs + .keys() + .copied() + .filter(|slot| *slot >= begin && *slot < end) + .collect(); + slots.sort_unstable(); + for slot in slots { + if let Some(bundle) = self.standstill_certs.get(&slot) { + if let Some(bytes) = &bundle.notar { + cert_bytes_list.push(bytes.clone()); + } + if let Some(bytes) = &bundle.skip { + cert_bytes_list.push(bytes.clone()); + } + if let Some(bytes) = &bundle.final_ { + cert_bytes_list.push(bytes.clone()); + } + } + } + + cert_bytes_list + } + + fn collect_standstill_votes_for_replay(&self, begin: u32, end: u32) -> Vec { + let mut votes: Vec<_> = self + .our_votes + .iter() + .filter(|(slot, vote)| { + if *slot < begin || *slot >= end { + return false; + } + + let bundle = self.standstill_certs.get(slot); + match &vote.vote { + UnsignedVote::Consensus_Simplex_NotarizeVote(_) => { + bundle.map_or(true, |b| b.notar.is_none()) + } + UnsignedVote::Consensus_Simplex_SkipVote(_) => { + bundle.map_or(true, |b| b.skip.is_none()) + } + UnsignedVote::Consensus_Simplex_FinalizeVote(_) => { + bundle.map_or(true, |b| b.final_.is_none()) + } + } + }) + .map(|(_, vote)| vote.clone()) + .collect(); + + votes.sort_by(|left, right| { + let left_key = Self::standstill_vote_key(left); + let right_key = Self::standstill_vote_key(right); + left_key.slot.cmp(&right_key.slot).then(left_key.kind.cmp(&right_key.kind)) + }); + votes + } + + fn rebuild_standstill_replay_queue(&mut self, begin: u32, end: u32) -> (u32, u32, usize) { + let cert_payloads = self.collect_standstill_certificate_payloads(begin, end); + let cert_count = cert_payloads.len() as u32; + let votes_to_rebroadcast = self.collect_standstill_votes_for_replay(begin, end); + let vote_count = votes_to_rebroadcast.len() as u32; + let replaced_items = self.standstill_replay_queue.len(); + + self.standstill_replay_queue.clear(); + for bytes in cert_payloads { + self.standstill_replay_queue.push_back(StandstillReplayItem::Certificate(bytes)); + } + for vote in votes_to_rebroadcast { + self.standstill_replay_queue.push_back(StandstillReplayItem::Vote(vote)); + } + if replaced_items == 0 { + self.reset_standstill_egress_budget_for_replay(); + } + + self.standstill_certs_rebroadcast_counter.increment(cert_count as u64); + self.standstill_votes_rebroadcast_counter.increment(vote_count as u64); + + (cert_count, vote_count, replaced_items) + } + /// Send a signed vote to all validators /// /// # Arguments @@ -2083,11 +2604,7 @@ impl ReceiverImpl { // Store vote for potential standstill re-broadcast (only on first send) if !is_rebroadcast { - let slot = Self::get_vote_slot_from_inner(&vote); - self.our_votes.push((slot, vote.clone())); - // Keep standstill end large enough to include newly sent votes. - // This avoids relying on external range updates for window growth (C++ parity: alarm() uses current state). - self.standstill_slot_end = self.standstill_slot_end.max(slot.saturating_add(1)); + self.cache_vote_for_standstill(vote.clone()); } // Serialize vote for network transmission @@ -2161,8 +2678,7 @@ impl ReceiverImpl { slot, discriminant(&vote.vote) ); - self.our_votes.push((slot, vote)); - self.standstill_slot_end = self.standstill_slot_end.max(slot.saturating_add(1)); + self.cache_vote_for_standstill(vote); } /// Send block candidate to all validators @@ -2416,18 +2932,77 @@ impl ReceiverImpl { } } - /// Maximum slot the receiver will accept (inclusive). - /// Mirrors `SimplexState::max_acceptable_slot()` using the receiver's own - /// finalization cursor, which is updated by `cleanup()`. - fn max_acceptable_slot(&self) -> u32 { - self.first_active_slot.saturating_add(MAX_FUTURE_SLOTS) + /// Returns the first slot that is considered "too new" for votes. + /// + /// Mirrors C++ `pool.cpp`: + /// `(now_ / slots_per_leader_window + max_desync + 1) * slots_per_leader_window` + fn first_too_new_vote_slot(&self) -> u32 { + let current_window = self.ingress_progress_slot / self.slots_per_leader_window; + current_window + .saturating_add(self.max_leader_window_desync) + .saturating_add(1) + .saturating_mul(self.slots_per_leader_window) + } + + /// Returns `true` if a vote slot is outside the C++ vote ingress range + /// `[first_active_slot, first_too_new_vote_slot())`. + fn is_vote_slot_out_of_bounds(&self, slot: u32) -> bool { + slot < self.first_active_slot || slot >= self.first_too_new_vote_slot() + } + + /// Returns `true` if a certificate references an already finalized slot. + fn is_certificate_slot_too_old(&self, slot: u32) -> bool { + slot < self.first_active_slot } - /// Returns `true` if `slot` is outside the acceptable range - /// `[first_active_slot, first_active_slot + MAX_FUTURE_SLOTS]`. - /// Rejects both already-finalized slots and far-future slots. - fn is_slot_out_of_bounds(&self, slot: u32) -> bool { - slot < self.first_active_slot || slot > self.max_acceptable_slot() + /// Advance ingress slot lower bound used by pre-filter bounds checks. + fn set_ingress_slot_begin_impl(&mut self, slot: SlotIndex) { + let slot_value = slot.value(); + if slot_value <= self.first_active_slot { + return; + } + + log::trace!( + "SimplexReceiver {}: set_ingress_slot_begin {} -> {}", + self.session_id.to_hex_string(), + self.first_active_slot, + slot_value + ); + self.first_active_slot = slot_value; + if self.ingress_progress_slot < self.first_active_slot { + self.ingress_progress_slot = self.first_active_slot; + } + } + + /// Advance ingress progress cursor used for the future-slot upper bound. + fn set_ingress_progress_slot_impl(&mut self, slot: SlotIndex) { + let slot_value = slot.value().max(self.first_active_slot); + if slot_value <= self.ingress_progress_slot { + return; + } + + log::trace!( + "SimplexReceiver {}: set_ingress_progress_slot {} -> {}", + self.session_id.to_hex_string(), + self.ingress_progress_slot, + slot_value + ); + self.ingress_progress_slot = slot_value; + } + + fn cancel_candidate_requests_for_slot_impl(&mut self, slot: SlotIndex) { + let before = self.pending_requests.len(); + self.pending_requests.retain(|(pending_slot, _), _| *pending_slot != slot); + let removed = before.saturating_sub(self.pending_requests.len()); + + if removed > 0 { + log::trace!( + "SimplexReceiver {}: cancelled {} pending candidate requests for slot {}", + self.session_id.to_hex_string(), + removed, + slot + ); + } } /// Cleanup old slots data @@ -2448,11 +3023,14 @@ impl ReceiverImpl { up_to_slot ); - self.first_active_slot = up_to_slot.value(); + self.standstill_replay_queue.clear(); // Clean up old votes (keep votes for slot >= up_to_slot) let old_count = self.our_votes.len(); self.our_votes.retain(|(s, _)| *s >= up_to_slot.value()); + if self.our_votes.len() != old_count { + self.rebuild_standstill_vote_keys(); + } if self.our_votes.len() < old_count { log::trace!( "SimplexReceiver {}: cleaned up {} old votes (up_to_slot={})", @@ -2468,6 +3046,9 @@ impl ReceiverImpl { // Clean up resolver cache for old slots self.cleanup_resolver_cache(up_to_slot); + // Clean up in-flight candidate requests for old slots + self.pending_requests.retain(|(slot, _), _| *slot >= up_to_slot); + // Clean up standstill certificate cache let old_cert_count = self.standstill_certs.len(); self.standstill_certs.retain(|&slot, _| slot >= up_to_slot.value()); @@ -2485,6 +3066,15 @@ impl ReceiverImpl { // Reference: C++ pool.cpp only calls reschedule_standstill_resolution() in on_finalization() } + /// Arm standstill detection after session promotion to current. + fn start(&mut self) { + log::info!( + "SimplexReceiver {}: started, standstill detection armed", + self.session_id.to_hex_string(), + ); + self.reschedule_standstill(); + } + /// Reschedule standstill alarm /// /// Reference: C++ pool.cpp reschedule_standstill_resolution() @@ -2505,6 +3095,7 @@ impl ReceiverImpl { /// Reference: C++ pool.cpp alarm() fn check_standstill(&mut self) { check_execution_time!(10_000); + self.drain_standstill_replay_queue(); let now = SystemTime::now(); @@ -2523,61 +3114,38 @@ impl ReceiverImpl { let begin = self.standstill_slot_begin; let end = self.standstill_slot_end; - // 1. Re-broadcast cached certificates - let cert_count = self.rebroadcast_standstill_certificates(begin, end); - - // 2. Re-broadcast our votes in tracked range, but ONLY if matching cert doesn't exist - // Reference: C++ Tsentrizbirkom::serialize_to(messages, bundle): - // if (notarize_.has_value() && !bundle.notarize_.has_value()) { ... } - // if (skip_.has_value() && !bundle.skip_.has_value()) { ... } - // if (finalize_.has_value() && !bundle.finalize_.has_value()) { ... } - let votes_to_rebroadcast: Vec<_> = self - .our_votes - .iter() - .filter(|(slot, vote)| { - if *slot < begin || *slot >= end { - return false; - } - // Check if we have the matching cert cached - let bundle = self.standstill_certs.get(slot); - match &vote.vote { - UnsignedVote::Consensus_Simplex_NotarizeVote(_) => { - // Only send notar vote if no notar cert cached - bundle.map_or(true, |b| b.notar.is_none()) - } - UnsignedVote::Consensus_Simplex_SkipVote(_) => { - // Only send skip vote if no skip cert cached - bundle.map_or(true, |b| b.skip.is_none()) - } - UnsignedVote::Consensus_Simplex_FinalizeVote(_) => { - // Only send finalize vote if no final cert cached - bundle.map_or(true, |b| b.final_.is_none()) - } - } - }) - .map(|(_, v)| v.clone()) - .collect(); - - // Standstill detected - log summary self.standstill_triggers_counter.increment(1); self.health_counters.standstill_triggers.fetch_add(1, Ordering::Relaxed); - self.standstill_certs_rebroadcast_counter.increment(cert_count as u64); - self.standstill_votes_rebroadcast_counter.increment(votes_to_rebroadcast.len() as u64); - - log::warn!( - "SimplexReceiver {}: Standstill detected, re-broadcasting {} certs + {} votes \ - (range [{}, {}))", - self.session_id.to_hex_string(), - cert_count, - votes_to_rebroadcast.len(), - begin, - end - ); + let (cert_count, vote_count, replaced_items) = + self.rebuild_standstill_replay_queue(begin, end); + if replaced_items > 0 { + log::trace!( + "SimplexReceiver {}: replaced {} pending standstill replay items with a fresh \ + snapshot", + self.session_id.to_hex_string(), + replaced_items + ); + } - // Re-broadcast each vote in range (already signed, no loopback) - for vote in votes_to_rebroadcast { - self.send_vote_impl(vote, true /* is_rebroadcast */); + if let Some(listener) = self.listener.upgrade() { + listener.on_standstill_trigger(StandstillTriggerNotification { + begin, + end, + cert_count, + vote_count, + }); + } else { + log::warn!( + "SimplexReceiver {}: Standstill detected, re-broadcasting {} certs + {} votes \ + (range [{}, {}))", + self.session_id.to_hex_string(), + cert_count, + vote_count, + begin, + end + ); } + self.drain_standstill_replay_queue(); // Reschedule standstill timer (reschedule after re-broadcast) self.reschedule_standstill(); @@ -2588,6 +3156,10 @@ impl ReceiverImpl { /// Sets `[begin, end)` range and removes votes outside this range. /// Reference: C++ pool.cpp tracked_slots_interval() = [first_non_finalized, current_window_end) fn set_standstill_slots_impl(&mut self, begin: u32, end: u32) { + if self.standstill_slot_begin == begin && self.standstill_slot_end == end { + return; + } + log::trace!( "SimplexReceiver {}: set_standstill_slots [{}, {})", self.session_id.to_hex_string(), @@ -2597,10 +3169,14 @@ impl ReceiverImpl { self.standstill_slot_begin = begin; self.standstill_slot_end = end; + self.standstill_replay_queue.clear(); // Remove votes outside the range let old_count = self.our_votes.len(); self.our_votes.retain(|(slot, _)| *slot >= begin && *slot < end); + if self.our_votes.len() != old_count { + self.rebuild_standstill_vote_keys(); + } if self.our_votes.len() < old_count { log::trace!( "SimplexReceiver {}: removed {} votes outside standstill range", @@ -3059,6 +3635,27 @@ impl Receiver for ReceiverWrapper { })); } + fn set_ingress_slot_begin(&self, slot: u32) { + let slot = SlotIndex::new(slot); + self.task_queues.post_closure(Box::new(move |receiver: &mut ReceiverImpl| { + receiver.set_ingress_slot_begin_impl(slot); + })); + } + + fn set_ingress_progress_slot(&self, slot: u32) { + let slot = SlotIndex::new(slot); + self.task_queues.post_closure(Box::new(move |receiver: &mut ReceiverImpl| { + receiver.set_ingress_progress_slot_impl(slot); + })); + } + + fn cancel_candidate_requests_for_slot(&self, slot: u32) { + let slot = SlotIndex::new(slot); + self.task_queues.post_closure(Box::new(move |receiver: &mut ReceiverImpl| { + receiver.cancel_candidate_requests_for_slot_impl(slot); + })); + } + fn cache_notarization_cert(&self, slot: u32, block_hash: UInt256, notar_cert_data: Vec) { let slot_idx = SlotIndex::new(slot); self.task_queues.post_closure(Box::new(move |receiver: &mut ReceiverImpl| { @@ -3073,6 +3670,12 @@ impl Receiver for ReceiverWrapper { })); } + fn start(&self) { + self.task_queues.post_closure(Box::new(move |receiver: &mut ReceiverImpl| { + receiver.start(); + })); + } + fn reschedule_standstill(&self) { self.task_queues.post_closure(Box::new(move |receiver: &mut ReceiverImpl| { receiver.reschedule_standstill(); @@ -3153,9 +3756,13 @@ impl ReceiverWrapper { overlay_manager: ConsensusOverlayManagerPtr, listener: ReceiverListenerPtr, standstill_timeout: Duration, + standstill_max_egress_bytes_per_s: u32, + slots_per_leader_window: u32, + max_leader_window_desync: u32, panicked_flag: Arc, use_quic: bool, health_counters: Arc, + candidate_resolve_config: CandidateResolveConfig, ) -> Result { log::info!( "Creating SimplexReceiver for session {} (shard={}) with {} nodes", @@ -3332,6 +3939,7 @@ impl ReceiverWrapper { max_candidate_size, max_candidate_query_answer_size, proto_version, + candidate_resolve_config, in_messages_bytes: in_messages_bytes_clone, out_messages_bytes: out_messages_bytes_clone, in_broadcasts_bytes: in_broadcasts_bytes_clone, @@ -3342,17 +3950,26 @@ impl ReceiverWrapper { out_broadcasts_count: out_broadcasts_count_clone, _activity_node: activity_node.clone(), standstill_timeout, - standstill_alarm: Some(SystemTime::now() + standstill_timeout), // Initial scheduling + standstill_max_egress_bytes_per_s, + standstill_alarm: None, // Armed by Receiver::start() when session becomes current standstill_slot_begin: STANDSTILL_INITIAL_SLOT_BEGIN, standstill_slot_end: STANDSTILL_INITIAL_SLOT_END, our_votes: Vec::new(), + our_vote_keys: HashSet::new(), + standstill_replay_queue: VecDeque::new(), + standstill_egress_quota_bytes: 0.0, + standstill_egress_quota_time: SystemTime::now(), resolver_cache, delayed_actions: Vec::new(), pending_requests: HashMap::new(), + candidate_query_rate_limiters: HashMap::new(), task_queues: task_queues_clone.clone(), standstill_certs: HashMap::new(), last_final_cert: None, first_active_slot: 0, + ingress_progress_slot: 0, + slots_per_leader_window, + max_leader_window_desync, candidate_requests_counter: metrics_receiver_clone .sink() .register_counter(&"simplex_candidate_requests".into()), @@ -3509,13 +4126,16 @@ impl ReceiverWrapper { check_execution_time!(10_000); metrics_dumper.update(&metrics_receiver_clone); - if log::log_enabled!(log::Level::Debug) { + if log::log_enabled!(log::Level::Info) { let session_id_str = session_id_clone.to_hex_string(); - log::debug!("SimplexReceiver {} metrics:", &session_id_str); + log::info!("SimplexReceiver {} metrics:", &session_id_str); - metrics_dumper.dump(|string| { - log::debug!("{}{}", session_id_str, string); - }); + { + check_execution_time!(5_000); + metrics_dumper.dump(|string| { + log::info!("{}{}", session_id_str, string); + }); + } } receiver_impl.debug_dump(); diff --git a/src/node/simplex/src/session.rs b/src/node/simplex/src/session.rs index 004789e..5b4d0c9 100644 --- a/src/node/simplex/src/session.rs +++ b/src/node/simplex/src/session.rs @@ -92,7 +92,7 @@ use ton_api::ton::consensus::{ simplex::{Certificate, Vote}, CandidateData, }; -use ton_block::{error, Error, Result, ShardIdent, UInt256}; +use ton_block::{error, BlockIdExt, Error, Result, ShardIdent, UInt256}; /* Constants @@ -105,6 +105,8 @@ const TASK_QUEUE_LATENCY_WARN_DUMP_PERIOD: Duration = Duration::from_millis(2000 const SESSION_METRICS_DUMP_PERIOD_MS: u64 = 15000; // period of metrics dump const SESSION_PROFILING_DUMP_PERIOD_MS: u64 = 30000; // period of profiling dump const SESSION_HEALTH_CHECK_PERIOD_MS: u64 = 20000; +//LK: for debugging only; need to be removed in future +const SESSION_MAX_LEADER_WINDOW_DESYNC_MARGIN: u32 = 0; const LOG_TARGET_PROFILING: &str = "simplex_profiling"; // log target for profiling /* @@ -146,6 +148,12 @@ impl ReceiverListener for ReceiverListenerImpl { })); } + fn on_standstill_trigger(&self, notification: crate::receiver::StandstillTriggerNotification) { + self.task_queue.post_closure(Box::new(move |processor: &mut SessionProcessor| { + processor.on_standstill_trigger(notification); + })); + } + /// Handle incoming certificate from network fn on_certificate(&self, source_idx: u32, certificate: Certificate) { self.task_queue.post_closure(Box::new(move |processor: &mut SessionProcessor| { @@ -158,6 +166,7 @@ impl ReceiverListener for ReceiverListenerImpl { &self, slot: crate::block::SlotIndex, block_hash: UInt256, + want_candidate: bool, want_notar: bool, response_callback: consensus_common::QueryResponseCallback, ) { @@ -165,6 +174,7 @@ impl ReceiverListener for ReceiverListenerImpl { processor.handle_candidate_query_fallback( slot, block_hash, + want_candidate, want_notar, response_callback, ); @@ -407,11 +417,11 @@ impl ConsensusSession for SessionImpl { } impl SimplexSession for SessionImpl { - fn notify_mc_finalized(&self, mc_block_seqno: u32) { - // Post closure to main queue for thread-safe update of last_mc_finalized_seqno - // in SessionProcessor. This is used for shard empty block decisions. + fn notify_mc_finalized(&self, applied_top: BlockIdExt) { + // Post closure to the main queue for thread-safe applied-top tracking updates + // in SessionProcessor. This drives empty-block policy and MC validation gating. self.main_task_queue.post_closure(Box::new(move |processor: &mut SessionProcessor| { - processor.set_mc_finalized_seqno(mc_block_seqno); + processor.set_mc_finalized_block(applied_top); })); } @@ -488,7 +498,7 @@ impl SessionImpl { panicked_flag: Arc, task_queue: TaskQueuePtr, callbacks_task_queue: CallbackTaskQueuePtr, - options: SessionOptions, + mut options: SessionOptions, session_id: SessionId, shard: ShardIdent, ids: Vec, @@ -511,6 +521,19 @@ impl SessionImpl { get_elapsed_time(&session_creation_time).as_secs_f64() * 1000.0, ); + // Inflate the future-window bound at session bootstrap so receiver/state ingress + // checks tolerate a much larger slot skew during debugging. + let original_max_leader_window_desync = options.max_leader_window_desync; + options.max_leader_window_desync = options + .max_leader_window_desync + .saturating_add(SESSION_MAX_LEADER_WINDOW_DESYNC_MARGIN); + log::info!( + "Session {} bootstrap desync margin: max_leader_window_desync {} -> {}", + session_id.to_hex_string(), + original_max_leader_window_desync, + options.max_leader_window_desync, + ); + // Signal thread start based on wait_for_db_init option: // - If false: send Ok(()) now (non-blocking for caller) // - If true: wait until full initialization completes @@ -587,9 +610,13 @@ impl SessionImpl { overlay_manager.clone(), receiver_listener, options.standstill_timeout, + options.standstill_max_egress_bytes_per_s, + options.slots_per_leader_window, + options.max_leader_window_desync, panicked_flag.clone(), options.use_quic, health_counters.clone(), + crate::receiver::CandidateResolveConfig::from_session_options(&options), ) { Ok(r) => r, Err(err) => { @@ -673,6 +700,8 @@ impl SessionImpl { initial_block_seqno ); + receiver.start(); + // Phase 4a: Create session description (immutable session configuration) let description = match SessionDescription::new( &options, @@ -717,10 +746,7 @@ impl SessionImpl { // This replays votes, sets finalized boundary, applies local flags, // generates skip votes, and restores receiver cache. if !is_fresh_start { - let recovery_options = SessionStartupRecoveryOptions { - restart_recommit_strategy: options.restart_recommit_strategy, - initial_block_seqno, - }; + let recovery_options = SessionStartupRecoveryOptions { initial_block_seqno }; let recovery_processor = SessionStartupRecoveryProcessor::new( session_id.clone(), @@ -816,13 +842,16 @@ impl SessionImpl { metrics_dumper.update(processor.get_metrics_receiver()); - if log::log_enabled!(log::Level::Debug) { + if log::log_enabled!(log::Level::Info) { let session_id_str = session_id.to_hex_string(); - log::debug!("SimplexSession {} metrics:", &session_id_str); + log::info!("SimplexSession {} metrics:", &session_id_str); - metrics_dumper.dump(|string| { - log::debug!("{}{}", session_id_str, string); - }); + { + check_execution_time!(10_000); + metrics_dumper.dump(|string| { + log::info!("{}{}", session_id_str, string); + }); + } } next_metrics_dump_time = processor.get_description().get_time() @@ -989,6 +1018,9 @@ impl SessionImpl { metrics_dumper.add_derivative_metric("simplex_collates.total"); metrics_dumper.add_derivative_metric("simplex_collates.success"); metrics_dumper.add_derivative_metric("simplex_collates.failure"); + metrics_dumper.add_derivative_metric("simplex_collation_starts"); + metrics_dumper.add_derivative_metric("simplex_candidate_received_broadcast"); + metrics_dumper.add_derivative_metric("simplex_candidate_received_query"); metrics_dumper.add_derivative_metric("simplex_commits.total"); metrics_dumper.add_derivative_metric("simplex_commits.success"); metrics_dumper.add_derivative_metric("simplex_commits.failure"); @@ -1045,9 +1077,11 @@ impl SessionImpl { metrics_dumper.add_derivative_metric("simplex_first_non_progressed_slot"); metrics_dumper.add_derivative_metric("simplex_skip_total"); + metrics_dumper.add_derivative_metric("simplex_votes_in_total"); metrics_dumper.add_derivative_metric("simplex_votes_in_notarize"); metrics_dumper.add_derivative_metric("simplex_votes_in_finalize"); metrics_dumper.add_derivative_metric("simplex_votes_in_skip"); + metrics_dumper.add_derivative_metric("simplex_votes_out_total"); metrics_dumper.add_derivative_metric("simplex_votes_out_notarize"); metrics_dumper.add_derivative_metric("simplex_votes_out_finalize"); metrics_dumper.add_derivative_metric("simplex_votes_out_skip"); @@ -1059,6 +1093,43 @@ impl SessionImpl { metrics_dumper.add_derivative_metric("simplex_validation_late_callback"); metrics_dumper.add_derivative_metric("simplex_health_warnings"); + // Vote-mix ratios for skip/notar/final observability. + add_compute_percentage_metric( + &mut metrics_dumper, + "simplex_votes_in_skip_share", + "simplex_votes_in_skip", + "simplex_votes_in_total", + 0.0, + ); + add_compute_percentage_metric( + &mut metrics_dumper, + "simplex_votes_in_notarize_share", + "simplex_votes_in_notarize", + "simplex_votes_in_total", + 0.0, + ); + add_compute_percentage_metric( + &mut metrics_dumper, + "simplex_votes_in_finalize_share", + "simplex_votes_in_finalize", + "simplex_votes_in_total", + 0.0, + ); + add_compute_relative_metric( + &mut metrics_dumper, + "simplex_votes_in_skip_to_notar_ratio", + "simplex_votes_in_skip", + "simplex_votes_in_notarize", + 0.0, + ); + add_compute_relative_metric( + &mut metrics_dumper, + "simplex_votes_in_skip_to_finalize_ratio", + "simplex_votes_in_skip", + "simplex_votes_in_finalize", + 0.0, + ); + metrics_dumper } diff --git a/src/node/simplex/src/session_processor.rs b/src/node/simplex/src/session_processor.rs index 45d7df7..3889b3d 100644 --- a/src/node/simplex/src/session_processor.rs +++ b/src/node/simplex/src/session_processor.rs @@ -35,7 +35,7 @@ //! │ ┌─────────────────────────────┐ ┌─────────────────────────────────────┐ │ //! │ │ check_all() loop: │ │ Event dispatch: │ │ //! │ │ 1. check_collation() │ │ BroadcastVote → sign & send │ │ -//! │ │ 2. check_validation() │ │ BlockFinalized → notify_commit │ │ +//! │ │ 2. check_validation() │ │ BlockFinalized → apply_finalized │ │ //! │ │ 3. simplex_state.check_all()│ │ SlotSkipped → cleanup │ │ //! │ │ 4. process_simplex_events() │ └─────────────────────────────────────┘ │ //! │ │ 5. update next awake time │ │ @@ -53,7 +53,7 @@ //! //! # Consensus Loop //! -//! Each slot: `Collate → Broadcast → Validate → Notarize → Vote → Collect → Finalize → Commit` +//! Each slot: `Collate → Broadcast → Validate → Notarize → Vote → Collect → Finalize → Deliver` //! //! See `README.md` "Consensus Loop" section for the phase-to-method mapping table. //! @@ -74,15 +74,13 @@ use crate::{ CandidateInfoRecord, FinalizedBlockRecord, PoolStateRecord, SimplexDbPtr, VoteRecord, }, misbehavior::{MisbehaviorReport, VoteResult}, - receiver::{ReceiverPtr, StandstillCertificateType}, + receiver::{ReceiverPtr, StandstillCertificateType, StandstillTriggerNotification}, session_description::SessionDescription, simplex_state::{ BlockFinalizedEvent, FinalizationReachedEvent, NotarizationReachedEvent, SimplexEvent, SimplexState, SimplexStateOptions, SkipCertificateReachedEvent, SlotSkippedEvent, Vote, }, - startup_recovery::{ - CandidateHash, RestartRoundAction, SessionStartupRecoveryListener, SignatureBytes, - }, + startup_recovery::{CandidateHash, SessionStartupRecoveryListener, SignatureBytes}, task_queue::{post_callback_closure, CallbackTaskQueuePtr, TaskPtr, TaskQueuePtr}, utils::{ extract_vote_and_signature, sign_vote, threshold_33, threshold_66, verify_vote_signature, @@ -110,10 +108,8 @@ use ton_api::{ candidateid::CandidateId, candidateparent::CandidateParent, simplex::{ - candidateandcert::CandidateAndCert, vote::Vote as TlVote, - votesignature::VoteSignature as TlVoteSignature, - votesignatureset::VoteSignatureSet, Certificate, UnsignedVote, Vote as TlVoteBoxed, - VoteSignatureSet as VoteSignatureSetBoxed, + candidateandcert::CandidateAndCert, vote::Vote as TlVote, Certificate, + UnsignedVote, Vote as TlVoteBoxed, VoteSignatureSet as VoteSignatureSetBoxed, }, CandidateData, CandidateHashData, CandidateParent as CandidateParentBoxed, }, @@ -123,8 +119,8 @@ use ton_api::{ }; use ton_block::{ error, fail, sha256_digest, BlockIdExt, BlockSignaturesPure, BlockSignaturesSimplex, - BlockSignaturesVariant, BocFlags, CryptoSignature, CryptoSignaturePair, Deserializable, Error, - HashmapType, KeyId, Result, UInt256, ValidatorBaseInfo, + BlockSignaturesVariant, BocFlags, CryptoSignature, CryptoSignaturePair, Error, Result, UInt256, + ValidatorBaseInfo, }; /* @@ -138,7 +134,7 @@ const MAX_AWAKE_TIMEOUT: Duration = Duration::from_secs(86400); /// Maximum generation time for collation - warn if exceeded const MAX_GENERATION_TIME: Duration = Duration::from_millis(1000); -/// Period without commits before triggering debug dump (stalled consensus detection) +/// Period without finalizations before triggering debug dump (stalled consensus detection) /// Matches validator-session ROUND_DEBUG_PERIOD const ROUND_DEBUG_PERIOD: Duration = Duration::from_secs(15); @@ -155,14 +151,6 @@ const CANDIDATE_REQUEST_DELAY: Duration = Duration::from_secs(1); /// Under network partitions, a single request may time out; we must retry, but not spam. const CANDIDATE_REQUEST_RETRY_INTERVAL: Duration = Duration::from_secs(2); -/// Interval for re-requesting committed block proofs in WaitingForFinalCert. -const COMMITTED_PROOF_RETRY_INTERVAL: Duration = Duration::from_secs(1); - -/// Maximum parent-chain walk depth when deriving a persisted DB parent. -/// -/// This is a safety guard against corrupted parent pointers creating long/looping chains. -const MAX_DB_PARENT_WALK_HOPS: usize = 1024; - /// Maximum parent chain depth for resolution tracking /// Protects against excessive recursion in update_resolution_cache_chain const MAX_CHAIN_DEPTH: u32 = 10000; @@ -175,16 +163,44 @@ const DEEP_RECURSION_WARNING_THRESHOLD: u32 = 100; /// Candidates waiting longer than this are considered failed const MAX_PARENT_WAIT_TIME: Duration = Duration::from_secs(600); // 10 minutes -/// Integration knob: avoid generating NON-EMPTY blocks on non-committed parents. +/// Integration knob: avoid generating NON-EMPTY blocks on non-finalized parents. /// /// When `true`, shardchain sessions use the masterchain-style empty-block rule -/// (`last_committed_seqno + 1 < new_seqno`) instead of the C++ shardchain rule +/// (`finalized_head_seqno + 1 < new_seqno`) instead of the C++ shardchain rule /// (MC lag threshold). This was needed before optimistic validation was implemented. /// /// Now that ValidatorGroup uses candidate-native validation (run_validate_query_any_candidate) /// and check_validation() accepts notarized parents, this flag is set to `false` for C++ parity. const DISABLE_NON_FINALIZED_PARENTS_FOR_COLLATION: bool = false; +/// Controls whether SessionProcessor blocks validation submission on C++-style WaitForParent +/// readiness (`parent finalized/notarized + full skip-gap coverage`). +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[allow(dead_code)] +enum ParentReadinessMode { + /// Keep C++ parity behavior in SessionProcessor. + StrictWaitForParent, + /// Allow sending candidates to validator-side validation before parent readiness converges. + RelaxedAllowEarlyValidation, +} + +/// Controls whether SessionProcessor enforces MC accepted-head ordering before forwarding. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[allow(dead_code)] +enum McAcceptedHeadMode { + /// Keep MC accepted-head gate in SessionProcessor (`check_mc_validation_ready`). + StrictSessionProcessorGate, + /// Delegate MC stale protection to validator-side checks. + ValidatorSideOnly, +} + +/// Default mode for current pass: allow validator-side attempts even while parent readiness is +/// still converging in SessionProcessor. +const PARENT_READINESS_MODE: ParentReadinessMode = ParentReadinessMode::RelaxedAllowEarlyValidation; + +/// Default mode for current pass: keep MC stale protection on validator side. +const MC_ACCEPTED_HEAD_MODE: McAcceptedHeadMode = McAcceptedHeadMode::ValidatorSideOnly; + /// Tracks per-anomaly cooldowns and delta baselines for health alert deduplication. /// All timestamps use `SystemTime` (via `self.now()`) for deterministic testing. pub(crate) struct HealthAlertState { @@ -195,11 +211,15 @@ pub(crate) struct HealthAlertState { last_finalization_nonzero_at: SystemTime, last_parent_aging_warn: SystemTime, last_progress_warn: SystemTime, + last_skip_ratio_warn: SystemTime, last_standstill_warn: SystemTime, last_isolation_warn: SystemTime, prev_candidate_giveups: u64, prev_cert_verify_fails: u64, prev_last_finalized_slot: f64, + prev_votes_in_notarize: u64, + prev_votes_in_finalize: u64, + prev_votes_in_skip: u64, prev_standstill_triggers: u64, cooldown: Duration, } @@ -216,11 +236,15 @@ impl HealthAlertState { last_finalization_nonzero_at: now, last_parent_aging_warn: warn_base, last_progress_warn: warn_base, + last_skip_ratio_warn: warn_base, last_standstill_warn: warn_base, last_isolation_warn: warn_base, prev_candidate_giveups: 0, prev_cert_verify_fails: 0, prev_last_finalized_slot: 0.0, + prev_votes_in_notarize: 0, + prev_votes_in_finalize: 0, + prev_votes_in_skip: 0, prev_standstill_triggers: 0, cooldown, } @@ -403,8 +427,7 @@ struct DelayedAction { /// Validated block candidate for finalization /// -/// Contains the data needed to call on_block_committed when a block finalizes. -/// Validated candidate stored after successful validation +/// Contains validated candidate data stored after successful validation. /// Note: Currently stored but not used - we use received_candidates for finalization #[derive(Debug)] #[allow(dead_code)] @@ -437,7 +460,7 @@ struct ReceivedCandidate { #[allow(dead_code)] // May be used for debugging/diagnostics candidate_id_hash: UInt256, /// Serialized CandidateHashData TL bytes - /// Used for building BlockSignaturesSimplex during commit + /// Used for building BlockSignaturesSimplex during finalization delivery /// SHA256(candidate_hash_data_bytes) == candidate_id_hash candidate_hash_data_bytes: Vec, /// Full block ID (workchain, shard, seqno, root_hash, file_hash) @@ -502,29 +525,33 @@ struct PendingValidation { source_idx: ValidatorIndex, } -/// Block to be committed as part of batch finalization -/// -/// Collects all data needed to commit a block: slot, hash, and whether it's -/// the triggered block. Used by `collect_gapless_commit_chain()` to build the commit queue. -/// -/// Reference: C++ finalize_blocks() walks parent chain and commits each block -struct BlockToCommit { - /// Candidate identity (slot, hash) - candidate_id: RawCandidateId, - /// Is this the triggered (first) block in the finalization batch? - is_triggered_block: bool, +/// Tracks a locally generated candidate until it validates successfully. +#[derive(Debug)] +struct GeneratedCandidateValidationWatch { + /// When the local candidate was generated. + generated_at: SystemTime, + /// Whether it has already entered higher-layer validation. + validation_started: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum McValidationReadiness { + Ready, + WaitingForAcceptedHead, } /* Finalization Journal - Tracks finalized blocks that have not yet been committed (awaiting bodies / gapless chain). + Tracks finalized blocks that cannot be fully materialized yet because the + candidate body has not arrived. Once the body is known, we can deliver the + finalized callback, persist restart state, and clear local slot runtime. */ /// Finalization journal entry /// /// Records that a FinalCert was observed for (slot, hash), but we haven't -/// committed it yet (awaiting missing bodies or gapless chain from committed head). +/// materialized local finalized state yet because the candidate body is still missing. #[derive(Clone)] struct FinalizedEntry { /// The finalization event from SimplexState @@ -534,42 +561,6 @@ struct FinalizedEntry { finalized_at: SystemTime, } -/// Result of collecting a gapless commit chain -enum ChainCollectionResult { - /// Chain is ready: all bodies present, NotarCerts available, connects to committed head - Ready { - /// The parent chain to commit (oldest first, fully-bodied, gapless) - chain: Vec, - }, - - /// The finalized block is already committed (its block_id == last_committed_block_id) - AlreadyCommitted, - - /// Masterchain-only: commit is blocked because we are missing a FinalCert for the next expected seqno. - /// - /// This typically happens when we observe finalization for a later masterchain block (seqno = K), - /// but have not yet observed FinalCerts (and thus cannot commit) for intermediate seqnos. - /// - /// IMPORTANT: Unlike MissingCandidate, this is NOT resolved by requestCandidate(want_notar=true), - /// because requestCandidate responses carry only (candidate bytes + notar cert), not FinalCert. - WaitingForFinalCert { - /// Next seqno we must commit (last_committed_seqno + 1, or initial seqno) - expected_seqno: u32, - /// Triggered finalized candidate id we are trying to commit - finalized_id: RawCandidateId, - /// Seqno of that triggered finalized candidate (from received candidate body) - finalized_seqno: u32, - }, - - /// Missing candidate body or NotarCert for a block in the chain - /// Caller should request this candidate from peers (want_notar=true gets NotarCert) - MissingCandidate { - /// Exact (slot, hash) of the missing candidate - /// (could be triggered block or any ancestor in the parent chain) - missing_id: RawCandidateId, - }, -} - /* Slot runtime + outcome gating (future wiring) @@ -653,7 +644,7 @@ pub(crate) struct SessionProcessor { receiver: ReceiverPtr, /// Stop flag shared with Session main loop. /// - /// Used to suppress late callbacks/commits during session shutdown (validator-group compatibility). + /// Used to suppress late callbacks/finalizations during session shutdown (validator-group compatibility). stop_flag: Arc, /// Simplex database for persistent storage db: SimplexDbPtr, @@ -707,6 +698,13 @@ pub(crate) struct SessionProcessor { /// `on_candidate_received` self-loop, so `resolve_parent_block_id()` can /// find the parent immediately for chained precollation. generated_parent_cache: HashMap, + /// Locally generated candidates that have not yet validated successfully. + /// + /// Used to surface warnings and metrics when the self-loop or validation + /// pipeline drops our own candidate before it reaches a successful + /// `candidate_decision_ok_internal()` outcome. + generated_candidates_waiting_validation: + HashMap, /* Validation state @@ -738,6 +736,11 @@ pub(crate) struct SessionProcessor { /// Used by `handle_candidate_query_fallback()` when the receiver's `resolver_cache` misses. /// This provides C++ parity with `CandidateResolver::try_load_candidate_data_from_db()`. candidate_data_cache: HashMap>, + /// Broadcast ingress dedup: slot -> first candidate seen via broadcast path. + /// + /// Matches the spirit of C++ `PrecheckCandidateBroadcast` slot-level conflict + /// guard by rejecting a second conflicting candidate id for the same slot. + seen_broadcast_candidates: HashMap, /* Metrics @@ -764,8 +767,6 @@ pub(crate) struct SessionProcessor { validates_counter: ResultStatusCounter, /// Result status counter for collation requests collates_counter: ResultStatusCounter, - /// Result status counter for commit requests - commits_counter: ResultStatusCounter, /// Counter for precollation requests precollation_requests_counter: metrics::Counter, /// Counter for precollation results @@ -774,21 +775,20 @@ pub(crate) struct SessionProcessor { collates_precollated_counter: ResultStatusCounter, /// Result status counter for expired collation time slots collates_expire_counter: ResultStatusCounter, + /// Counter for all collation entry attempts, including async, retry, precollated, + /// and empty-block fast paths. + collation_starts_counter: metrics::Counter, /// Histogram for broadcast-to-validation complete latency broadcast_validation_latency_histogram: metrics::Histogram, /// Counter for errors during session (for SessionStats) errors_counter: metrics::Counter, - /// Counter for batch commit operations - batch_commit_counter: metrics::Counter, - /// Histogram for batch commit sizes (number of blocks committed at once) - batch_commit_size_histogram: metrics::Histogram, - /// Gauge for finalized-but-uncommitted journal size (commit lag indicator) - finalized_uncommitted_gauge: metrics::Gauge, + /// Gauge for finalized blocks still waiting for candidate body arrival + finalized_pending_body_gauge: metrics::Gauge, /* - Error tracking for SessionStats + Error tracking */ - /// Total errors count during this session (incremented on errors, passed to on_block_committed) + /// Total errors count during this session (used for session statistics) /// Atomic to allow increment_error(&self) without requiring &mut self session_errors_count: AtomicU32, @@ -810,15 +810,15 @@ pub(crate) struct SessionProcessor { Debug Reference: validator-session/src/session_processor.rs round_debug_at */ - /// Next time for stalled round debug dump (reset on each commit) - /// If current time >= round_debug_at, no commits occurred for ROUND_DEBUG_PERIOD + /// Next time for stalled round debug dump (reset on each finalization) + /// If current time >= round_debug_at, no finalizations occurred for ROUND_DEBUG_PERIOD round_debug_at: SystemTime, - /// Time of last commit (for accurate stall duration reporting) - last_commit_time: SystemTime, + /// Time of last finalization (for accurate stall duration reporting) + last_finalization_time: SystemTime, /* Slot Sequence Invariants - Ensures correct ordering of slots for commits, skips, and generation + Ensures correct ordering of slots for finalizations, skips, and generation */ /// Last slot for which generation was requested /// Must be monotonically increasing (gaps allowed) @@ -828,68 +828,62 @@ pub(crate) struct SessionProcessor { Block SeqNo Tracking Tracks expected blockchain sequence number for next block */ - /// Last committed block seqno - updated in commit_single_block(). - /// Used for strict commit sequencing and validation checks. - last_committed_seqno: Option, + /// Highest finalized non-empty block seqno materialized locally in this session. + /// Used for validation shortcuts and local progress tracking. + finalized_head_seqno: Option, - /// Last committed block slot - updated in commit_single_block() - /// Used to retrieve parent BlockIdExt for empty block generation - last_committed_slot: Option, - /// Last committed non-empty block id (parent for empty blocks) + /// Slot of the latest locally materialized finalized head. + /// Used to retrieve parent `BlockIdExt` for empty block generation. + finalized_head_slot: Option, + /// Last finalized non-empty block id (parent for empty blocks) /// /// Empty blocks inherit parent's BlockIdExt (C++ behavior), so we must keep the - /// last non-empty committed block id available for empty block generation. - last_committed_block_id: Option, + /// last non-empty finalized block id available for empty block generation. + finalized_head_block_id: Option, - /// Last committed block's before_split flag (for split/merge handling) + /// Last finalized head block's before_split flag (for split/merge handling) /// /// C++ parity: C++ always generates empty blocks when previous block has `before_split=true`. /// We track this flag to implement the same behavior in `should_generate_empty_block()`. /// /// Reference: C++ block-producer.cpp `is_before_split()` + `should_generate_empty_block()` - last_committed_before_split: bool, + finalized_head_before_split: bool, - /// Last consensus-finalized seqno - tracks the highest seqno of a block committed - /// with FinalCert (is_final=true) in this session. + /// Last consensus-finalized seqno tracked from finalized delivery in this session. /// /// C++ parity: mirrors `last_consensus_finalized_seqno_` in block-producer.cpp, which /// advances on FinalizeBlock(is_final=true) and on BlockFinalizedInMasterchain events. /// Used for `should_generate_empty_block()` on masterchain. /// - /// Updated in `commit_single_block()` when use_final_cert is true, and in - /// `set_mc_finalized_seqno()` (coupled max with last_mc_finalized_seqno). + /// Updated in `maybe_apply_finalized_state()` and in `set_mc_finalized_block()` + /// (coupled max with last_mc_finalized_seqno). last_consensus_finalized_seqno: Option, - /// Blocks that have been committed (finalized): RawCandidateId(slot, hash) - /// - /// Used during batch finalization to track which blocks in a parent chain - /// have already been committed, avoiding double-commit. + /// Blocks whose finalized state has already been materialized locally. /// - /// When a BlockFinalized event triggers batch finalization, we walk the - /// parent chain and commit each block. This set tracks which blocks - /// have already been committed so we don't commit them again. + /// Used to deduplicate repeated finalization events and late body arrivals, + /// avoiding duplicate DB writes and repeated finalized bookkeeping. /// /// Cleaned up in cleanup_old_slots() for slots older than MAX_HISTORY_SLOTS. finalized_blocks: HashSet, /* - Finalization Journal + Finalized Pending Body - Tracks finalized blocks (FinalCert observed) that have not yet been committed - to ValidatorGroup. Commitment is deferred until: - - All bodies in the uncommitted ancestor chain are present, AND - - The chain is gapless by seqno (strict invariant) + Tracks finalized blocks (FinalCert observed) whose candidate body has not + yet been received locally. Materialization is deferred until the + corresponding candidate body arrives. - Two commit triggers: - - BlockFinalizedEvent: records in journal + tries commit - - on_candidate_received: body arrival + tries commit + Two materialization triggers: + - BlockFinalizedEvent: records entry + applies immediately when body is present + - on_candidate_received: late body arrival triggers deferred finalization */ - /// Journal of finalized-but-not-yet-committed blocks + /// Finalized blocks still waiting for candidate body arrival /// /// Keyed by RawCandidateId = { slot, hash } /// Inserted when BlockFinalizedEvent arrives (even if body missing) - /// Removed when committed or cleaned up (old slots) - finalized_journal_pending_commit: HashMap, + /// Removed when body arrives and finalization is applied, or cleaned up (old slots) + finalized_pending_body: HashMap, /* Slot outcome emission gating (future wiring) @@ -909,12 +903,24 @@ pub(crate) struct SessionProcessor { Reference: C++ block-producer.cpp should_generate_empty_block() ======================================================================== */ - /// Last masterchain finalized seqno (for shardchain empty block decisions) + /// Last MC-registered top seqno for this shard session (for empty block decisions) /// - /// Updated via `set_mc_finalized_seqno()` when MC finalization events arrive. + /// Updated via `set_mc_finalized_block()` when manager notifications arrive. + /// The value is the current applied-top seqno for this session shard. /// Used by `should_generate_empty_block()` for shardchain sessions. - /// For masterchain sessions, `last_committed_seqno` is used instead. + /// Masterchain sessions also receive manager-applied top updates here so startup and + /// recovery share the same monotonic applied-top cursor. last_mc_finalized_seqno: Option, + /// Seqno fallback for the accepted normal head used by MC validation ordering. + /// + /// Seeded from `initial_block_seqno - 1`, then advanced by applied-top notifications, + /// restart recovery, and finalized non-empty blocks. + accepted_normal_head_seqno: u32, + /// Exact accepted normal head when known. + /// + /// This mirrors the C++ `last_accepted_block_` semantics closely enough to reject + /// stale same-seqno forks once an exact BlockIdExt has been observed. + accepted_normal_head_block_id: Option, /* ======================================================================== @@ -933,10 +939,6 @@ pub(crate) struct SessionProcessor { /// Candidate request throttling: (slot, hash) → next allowed request time. requested_candidates: HashMap, - /// Pending committed block proof requests via get_committed_candidate. - /// Throttling map: block_id → next allowed request time. - pending_committed_proof_requests: HashMap, - /* ======================================================================== Pending Parent Resolution (Recursive Candidate Resolution) @@ -964,6 +966,13 @@ pub(crate) struct SessionProcessor { Reference: C++ bus.h MisbehaviorReport ======================================================================== */ + /// Dedup set for finalized delivery. + /// + /// We emit at most once per finalized candidate id, even if: + /// - finalization is observed before candidate body (delayed emit path), or + /// - repeated finalization/candidate events arrive from network. + finalized_delivery_sent: HashSet, + /// Collected misbehavior reports from this session /// /// When a vote is detected as misbehavior (e.g., conflicting votes for same slot), @@ -973,7 +982,7 @@ pub(crate) struct SessionProcessor { /// Counter for detected misbehavior events misbehavior_counter: metrics::Counter, - /// Gauge: last finalized slot index (set on each commit) + /// Gauge: last finalized slot index (set on each finalization) last_finalized_slot_gauge: metrics::Gauge, /// Gauge: first non-finalized slot from FSM (set in check_all) first_non_finalized_slot_gauge: metrics::Gauge, @@ -982,13 +991,19 @@ pub(crate) struct SessionProcessor { /// Counter: total skip events skip_total_counter: metrics::Counter, /// Vote pipeline counters (in) + votes_in_total_counter: metrics::Counter, votes_in_notarize_counter: metrics::Counter, votes_in_finalize_counter: metrics::Counter, votes_in_skip_counter: metrics::Counter, /// Vote pipeline counters (out) + votes_out_total_counter: metrics::Counter, votes_out_notarize_counter: metrics::Counter, votes_out_finalize_counter: metrics::Counter, votes_out_skip_counter: metrics::Counter, + /// Local vote totals for health anomaly delta checks (inbound stream). + votes_in_notarize_total: u64, + votes_in_finalize_total: u64, + votes_in_skip_total: u64, /// Certificate counters certs_in_counter: metrics::Counter, certs_relayed_counter: metrics::Counter, @@ -999,6 +1014,22 @@ pub(crate) struct SessionProcessor { validation_late_callback_counter: metrics::Counter, /// Health warnings counter (separate from session_errors_count) health_warnings_counter: metrics::Counter, + /// Broadcast precheck drops: old slot (< first_non_finalized) + candidate_precheck_old_slot_drop_counter: metrics::Counter, + /// Broadcast precheck drops: too-far-future slot + candidate_precheck_future_slot_drop_counter: metrics::Counter, + /// Broadcast precheck drops: sender is not expected slot leader + candidate_precheck_unexpected_sender_drop_counter: metrics::Counter, + /// Broadcast precheck drops: conflicting second candidate for same slot + candidate_precheck_conflicting_slot_drop_counter: metrics::Counter, + /// Peer-delivered candidate bodies received via broadcast. + /// Excludes the local generated-block self-loop. + candidate_received_broadcast_counter: metrics::Counter, + /// Peer-delivered candidate bodies received via requestCandidate/query responses. + /// Excludes the local generated-block self-loop. + candidate_received_query_counter: metrics::Counter, + /// Locally generated candidates that failed to validate successfully. + generated_candidate_validation_missed_counter: metrics::Counter, /// Health alert state for cooldown-based anomaly detection pub(crate) health_alert_state: HealthAlertState, /// Shared health counters from receiver (standstill triggers, candidate giveups) @@ -1030,6 +1061,26 @@ impl SessionProcessor { self.description.set_time(self.now() + delta); } + #[inline] + fn record_candidate_ingress(&self, sender_idx: ValidatorIndex, is_broadcast_candidate: bool) { + // Keep ingress counters focused on peer-delivered traffic: locally generated + // blocks loop back through on_candidate_received() but are not network ingress. + if sender_idx == self.description.get_self_idx() { + return; + } + + if is_broadcast_candidate { + self.candidate_received_broadcast_counter.increment(1); + } else { + self.candidate_received_query_counter.increment(1); + } + } + + #[inline] + fn record_collation_start(&self) { + self.collation_starts_counter.increment(1); + } + /// Clear manual time override (return to real-time mode). #[allow(dead_code)] pub(crate) fn clear_time(&self) { @@ -1237,7 +1288,7 @@ impl SessionProcessor { // INVARIANT: initial_block_seqno must be > 0. // Block seqno 0 is reserved for the zerostate (genesis), so the first real block is seqno 1. - // This invariant ensures last_committed_seqno initialization (initial_block_seqno - 1) is valid. + // This invariant ensures finalized_head_seqno initialization (initial_block_seqno - 1) is valid. assert!( initial_block_seqno > 0, "INVARIANT VIOLATION: initial_block_seqno must be > 0, got {}", @@ -1255,18 +1306,24 @@ impl SessionProcessor { let simplex_state = SimplexState::new(&description, simplex_state_options)?; let initial_standstill_slots = simplex_state.get_tracked_slots_interval(); + let initial_progress_slot = simplex_state.get_first_non_progressed_slot().value(); // Initialize receiver standstill tracked range to the FSM-tracked interval (C++ parity). // Receiver defaults to a broad range, but we can set the precise initial interval immediately // because `SimplexState::new()` creates window 0 (so end = slots_per_leader_window). + receiver.set_ingress_slot_begin(initial_standstill_slots.0); + receiver.set_ingress_progress_slot(initial_progress_slot); receiver.set_standstill_slots(initial_standstill_slots.0, initial_standstill_slots.1); log::info!( "Session {} SIMPLEX MODE: require_finalized_parent=false (C++ parenting enabled). \ Optimistic validation: candidate-native path (notarized parents accepted). \ - DISABLE_NON_FINALIZED_PARENTS_FOR_COLLATION={}.", + DISABLE_NON_FINALIZED_PARENTS_FOR_COLLATION={}. \ + parent_readiness_mode={:?}, mc_accepted_head_mode={:?}.", session_id.to_hex_string(), - DISABLE_NON_FINALIZED_PARENTS_FOR_COLLATION + DISABLE_NON_FINALIZED_PARENTS_FOR_COLLATION, + PARENT_READINESS_MODE, + MC_ACCEPTED_HEAD_MODE, ); log::info!( @@ -1287,26 +1344,26 @@ impl SessionProcessor { active_weight_gauge, validates_counter, collates_counter, - commits_counter, precollation_requests_counter, precollation_results_counter, collates_precollated_counter, collates_expire_counter, + collation_starts_counter, broadcast_validation_latency_histogram, first_candidate_received_latency_histogram, first_candidate_notarized_latency_histogram, first_candidate_finalized_latency_histogram, errors_counter, - batch_commit_counter, - batch_commit_size_histogram, misbehavior_counter, last_finalized_slot_gauge, first_non_finalized_slot_gauge, first_non_progressed_slot_gauge, skip_total_counter, + votes_in_total_counter, votes_in_notarize_counter, votes_in_finalize_counter, votes_in_skip_counter, + votes_out_total_counter, votes_out_notarize_counter, votes_out_finalize_counter, votes_out_skip_counter, @@ -1317,10 +1374,17 @@ impl SessionProcessor { validation_reject_counter, validation_late_callback_counter, health_warnings_counter, + candidate_precheck_old_slot_drop_counter, + candidate_precheck_future_slot_drop_counter, + candidate_precheck_unexpected_sender_drop_counter, + candidate_precheck_conflicting_slot_drop_counter, + candidate_received_broadcast_counter, + candidate_received_query_counter, + generated_candidate_validation_missed_counter, ) = Self::init_metrics(&metrics_receiver, &description); - let finalized_uncommitted_gauge = - metrics_receiver.sink().register_gauge(&"simplex_finalized_uncommitted_count".into()); + let finalized_pending_body_gauge = + metrics_receiver.sink().register_gauge(&"simplex_finalized_pending_body_count".into()); let now = description.get_time(); let num_validators = description.get_total_nodes() as usize; @@ -1356,6 +1420,7 @@ impl SessionProcessor { earliest_collation_time: None, local_chain_head: None, generated_parent_cache: HashMap::new(), + generated_candidates_waiting_validation: HashMap::new(), // Validation state pending_validations: HashMap::new(), pending_approve: HashSet::new(), @@ -1366,6 +1431,7 @@ impl SessionProcessor { validated_candidates: VecDeque::new(), received_candidates: HashMap::new(), candidate_data_cache: HashMap::new(), + seen_broadcast_candidates: HashMap::new(), // Metrics metrics_receiver, check_all_counter, @@ -1376,16 +1442,14 @@ impl SessionProcessor { active_weight_gauge, validates_counter, collates_counter, - commits_counter, precollation_requests_counter, precollation_results_counter, collates_precollated_counter, collates_expire_counter, + collation_starts_counter, broadcast_validation_latency_histogram, errors_counter, - batch_commit_counter, - batch_commit_size_histogram, - finalized_uncommitted_gauge, + finalized_pending_body_gauge, // Error tracking (includes startup errors from before processor was created) session_errors_count: AtomicU32::new(initial_errors), // Slot stage tracking @@ -1394,31 +1458,33 @@ impl SessionProcessor { first_candidate_finalized_latency_histogram, // Debug round_debug_at: now + ROUND_DEBUG_PERIOD, - last_commit_time: now, + last_finalization_time: now, // Slot/round tracking last_generated_slot: None, - // Treat the block *before* `initial_block_seqno` as the committed head at session start. + // Treat the block *before* `initial_block_seqno` as the finalized head at session start. // // This is required for: // - empty-block generation gating (non-finalized parent / ValidatorGroup limitation), - // - validation gating (expected_seqno = last_committed_seqno + 1), + // - validation gating (expected_seqno = finalized_head_seqno + 1), // and matches C++ where the block producer tracks the parent seqno from `Start` / `base`. - last_committed_seqno: initial_block_seqno.checked_sub(1), - last_committed_slot: None, - last_committed_block_id: None, - last_committed_before_split: false, + finalized_head_seqno: initial_block_seqno.checked_sub(1), + finalized_head_slot: None, + finalized_head_block_id: None, + finalized_head_before_split: false, last_consensus_finalized_seqno: initial_block_seqno.checked_sub(1), // Batch finalization tracking finalized_blocks: HashSet::new(), - finalized_journal_pending_commit: HashMap::new(), + finalized_pending_body: HashMap::new(), slots: BTreeMap::new(), // Empty block support - last_mc_finalized_seqno: None, + last_mc_finalized_seqno: initial_block_seqno.checked_sub(1), + accepted_normal_head_seqno: initial_block_seqno.saturating_sub(1), + accepted_normal_head_block_id: None, // Candidate request tracking requested_candidates: HashMap::new(), - pending_committed_proof_requests: HashMap::new(), // Pending parent resolution pending_parent_resolutions: HashMap::new(), + finalized_delivery_sent: HashSet::new(), // Misbehavior tracking misbehavior_reports: Vec::new(), misbehavior_counter, @@ -1426,12 +1492,17 @@ impl SessionProcessor { first_non_finalized_slot_gauge, first_non_progressed_slot_gauge, skip_total_counter, + votes_in_total_counter, votes_in_notarize_counter, votes_in_finalize_counter, votes_in_skip_counter, + votes_out_total_counter, votes_out_notarize_counter, votes_out_finalize_counter, votes_out_skip_counter, + votes_in_notarize_total: 0, + votes_in_finalize_total: 0, + votes_in_skip_total: 0, certs_in_counter, certs_relayed_counter, cert_conflict_counter, @@ -1439,6 +1510,13 @@ impl SessionProcessor { validation_reject_counter, validation_late_callback_counter, health_warnings_counter, + candidate_precheck_old_slot_drop_counter, + candidate_precheck_future_slot_drop_counter, + candidate_precheck_unexpected_sender_drop_counter, + candidate_precheck_conflicting_slot_drop_counter, + candidate_received_broadcast_counter, + candidate_received_query_counter, + generated_candidate_validation_missed_counter, health_alert_state: HealthAlertState::new(now, health_alert_cooldown), receiver_health_counters, cert_verify_fails_total: 0, @@ -1505,26 +1583,26 @@ impl SessionProcessor { metrics::Gauge, // active_weight_gauge ResultStatusCounter, // validates_counter ResultStatusCounter, // collates_counter - ResultStatusCounter, // commits_counter metrics::Counter, // precollation_requests_counter metrics::Counter, // precollation_results_counter ResultStatusCounter, // collates_precollated_counter ResultStatusCounter, // collates_expire_counter + metrics::Counter, // collation_starts_counter metrics::Histogram, // broadcast_validation_latency_histogram metrics::Histogram, // first_candidate_received_latency_histogram metrics::Histogram, // first_candidate_notarized_latency_histogram metrics::Histogram, // first_candidate_finalized_latency_histogram metrics::Counter, // errors_counter - metrics::Counter, // batch_commit_counter - metrics::Histogram, // batch_commit_size_histogram metrics::Counter, // misbehavior_counter metrics::Gauge, // last_finalized_slot_gauge metrics::Gauge, // first_non_finalized_slot_gauge metrics::Gauge, // first_non_progressed_slot_gauge metrics::Counter, // skip_total_counter + metrics::Counter, // votes_in_total_counter metrics::Counter, // votes_in_notarize_counter metrics::Counter, // votes_in_finalize_counter metrics::Counter, // votes_in_skip_counter + metrics::Counter, // votes_out_total_counter metrics::Counter, // votes_out_notarize_counter metrics::Counter, // votes_out_finalize_counter metrics::Counter, // votes_out_skip_counter @@ -1535,6 +1613,13 @@ impl SessionProcessor { metrics::Counter, // validation_reject_counter metrics::Counter, // validation_late_callback_counter metrics::Counter, // health_warnings_counter + metrics::Counter, // candidate_precheck_old_slot_drop_counter + metrics::Counter, // candidate_precheck_future_slot_drop_counter + metrics::Counter, // candidate_precheck_unexpected_sender_drop_counter + metrics::Counter, // candidate_precheck_conflicting_slot_drop_counter + metrics::Counter, // candidate_received_broadcast_counter + metrics::Counter, // candidate_received_query_counter + metrics::Counter, // generated_candidate_validation_missed_counter ) { let sink = metrics_receiver.sink(); @@ -1570,7 +1655,6 @@ impl SessionProcessor { // Result status counters let validates_counter = ResultStatusCounter::new(metrics_receiver, "simplex_validates"); let collates_counter = ResultStatusCounter::new(metrics_receiver, "simplex_collates"); - let commits_counter = ResultStatusCounter::new(metrics_receiver, "simplex_commits"); // Precollation metrics let precollation_requests_counter = @@ -1581,13 +1665,11 @@ impl SessionProcessor { ResultStatusCounter::new(metrics_receiver, "simplex_collates_precollated"); let collates_expire_counter = ResultStatusCounter::new(metrics_receiver, "simplex_collates_expire"); + let collation_starts_counter = sink.register_counter(&"simplex_collation_starts".into()); // Error tracking for ValidatorSessionStats let errors_counter = sink.register_counter(&"simplex_errors".into()); - let batch_commit_counter = sink.register_counter(&"simplex_batch_commits".into()); - let batch_commit_size_histogram = - sink.register_histogram(&"simplex_batch_commit_size".into()); let misbehavior_counter = sink.register_counter(&"simplex_misbehavior".into()); let last_finalized_slot_gauge = sink.register_gauge(&"simplex_last_finalized_slot".into()); @@ -1597,9 +1679,11 @@ impl SessionProcessor { sink.register_gauge(&"simplex_first_non_progressed_slot".into()); let skip_total_counter = sink.register_counter(&"simplex_skip_total".into()); + let votes_in_total_counter = sink.register_counter(&"simplex_votes_in_total".into()); let votes_in_notarize_counter = sink.register_counter(&"simplex_votes_in_notarize".into()); let votes_in_finalize_counter = sink.register_counter(&"simplex_votes_in_finalize".into()); let votes_in_skip_counter = sink.register_counter(&"simplex_votes_in_skip".into()); + let votes_out_total_counter = sink.register_counter(&"simplex_votes_out_total".into()); let votes_out_notarize_counter = sink.register_counter(&"simplex_votes_out_notarize".into()); let votes_out_finalize_counter = @@ -1616,6 +1700,20 @@ impl SessionProcessor { sink.register_counter(&"simplex_validation_late_callback".into()); let health_warnings_counter = sink.register_counter(&"simplex_health_warnings".into()); + let candidate_precheck_old_slot_drop_counter = + sink.register_counter(&"simplex_candidate_precheck_drop_old_slot".into()); + let candidate_precheck_future_slot_drop_counter = + sink.register_counter(&"simplex_candidate_precheck_drop_future_slot".into()); + let candidate_precheck_unexpected_sender_drop_counter = + sink.register_counter(&"simplex_candidate_precheck_drop_unexpected_sender".into()); + let candidate_precheck_conflicting_slot_drop_counter = + sink.register_counter(&"simplex_candidate_precheck_drop_conflicting_slot".into()); + let candidate_received_broadcast_counter = + sink.register_counter(&"simplex_candidate_received_broadcast".into()); + let candidate_received_query_counter = + sink.register_counter(&"simplex_candidate_received_query".into()); + let generated_candidate_validation_missed_counter = + sink.register_counter(&"simplex_generated_candidate_validation_missed".into()); ( check_all_counter, @@ -1626,26 +1724,26 @@ impl SessionProcessor { active_weight_gauge, validates_counter, collates_counter, - commits_counter, precollation_requests_counter, precollation_results_counter, collates_precollated_counter, collates_expire_counter, + collation_starts_counter, broadcast_validation_latency_histogram, first_candidate_received_latency_histogram, first_candidate_notarized_latency_histogram, first_candidate_finalized_latency_histogram, errors_counter, - batch_commit_counter, - batch_commit_size_histogram, misbehavior_counter, last_finalized_slot_gauge, first_non_finalized_slot_gauge, first_non_progressed_slot_gauge, skip_total_counter, + votes_in_total_counter, votes_in_notarize_counter, votes_in_finalize_counter, votes_in_skip_counter, + votes_out_total_counter, votes_out_notarize_counter, votes_out_finalize_counter, votes_out_skip_counter, @@ -1656,6 +1754,13 @@ impl SessionProcessor { validation_reject_counter, validation_late_callback_counter, health_warnings_counter, + candidate_precheck_old_slot_drop_counter, + candidate_precheck_future_slot_drop_counter, + candidate_precheck_unexpected_sender_drop_counter, + candidate_precheck_conflicting_slot_drop_counter, + candidate_received_broadcast_counter, + candidate_received_query_counter, + generated_candidate_validation_missed_counter, ) } @@ -1676,6 +1781,98 @@ impl SessionProcessor { &self.metrics_receiver } + fn track_generated_candidate_for_validation(&mut self, candidate_id: RawCandidateId) { + self.generated_candidates_waiting_validation.insert( + candidate_id, + GeneratedCandidateValidationWatch { + generated_at: self.now(), + validation_started: false, + }, + ); + } + + fn mark_generated_candidate_validation_started(&mut self, candidate_id: &RawCandidateId) { + if let Some(watch) = self.generated_candidates_waiting_validation.get_mut(candidate_id) { + watch.validation_started = true; + } + } + + fn mark_generated_candidate_validation_succeeded(&mut self, candidate_id: &RawCandidateId) { + self.generated_candidates_waiting_validation.remove(candidate_id); + } + + fn take_generated_candidate_watch_by_slot( + &mut self, + slot: SlotIndex, + ) -> Option<(RawCandidateId, GeneratedCandidateValidationWatch)> { + let candidate_id = self + .generated_candidates_waiting_validation + .keys() + .find(|candidate_id| candidate_id.slot == slot) + .cloned()?; + let watch = self.generated_candidates_waiting_validation.remove(&candidate_id)?; + Some((candidate_id, watch)) + } + + fn note_generated_candidate_validation_missed( + &mut self, + candidate_id: &RawCandidateId, + reason: impl Into, + ) { + let reason = reason.into(); + let Some(watch) = self.generated_candidates_waiting_validation.remove(candidate_id) else { + return; + }; + + self.generated_candidate_validation_missed_counter.increment(1); + + let waited_ms = + self.now().duration_since(watch.generated_at).unwrap_or_default().as_millis(); + log::warn!( + "Session {} local_generated_candidate_missed_validation: slot={} hash={} \ + validation_started={} waited={}ms reason={}", + &self.session_id().to_hex_string()[..8], + candidate_id.slot, + &candidate_id.hash.to_hex_string()[..8], + watch.validation_started, + waited_ms, + reason, + ); + } + + fn note_generated_candidate_validation_missed_for_slot( + &mut self, + slot: SlotIndex, + reason: impl Into, + ) { + let reason = reason.into(); + if let Some((candidate_id, watch)) = self.take_generated_candidate_watch_by_slot(slot) { + self.generated_candidate_validation_missed_counter.increment(1); + + let waited_ms = + self.now().duration_since(watch.generated_at).unwrap_or_default().as_millis(); + log::warn!( + "Session {} local_generated_candidate_missed_validation: slot={} hash={} \ + validation_started={} waited={}ms reason={}", + &self.session_id().to_hex_string()[..8], + candidate_id.slot, + &candidate_id.hash.to_hex_string()[..8], + watch.validation_started, + waited_ms, + reason, + ); + return; + } + + self.generated_candidate_validation_missed_counter.increment(1); + log::warn!( + "Session {} local_generated_candidate_missed_validation: slot={} reason={}", + &self.session_id().to_hex_string()[..8], + slot, + reason, + ); + } + /* Validator index validation */ @@ -1687,13 +1884,12 @@ impl SessionProcessor { } /* - Error tracking for SessionStats + Error tracking */ /// Increment the session error counter /// /// Called when an error occurs during session processing. - /// The error count is included in SessionStats passed to on_block_committed. /// Uses atomic increment to allow calling with &self (no &mut self required). fn increment_error(&self) { self.session_errors_count.fetch_add(1, Ordering::Relaxed); @@ -1959,16 +2155,6 @@ impl SessionProcessor { deserialize_typed(bytes) } - /// Build session statistics for on_block_committed callback - /// - /// Creates a SessionStats struct with current session metrics. - /// Called before notify_block_committed to capture session health data. - fn build_session_stats(&self) -> consensus_common::SessionStats { - consensus_common::SessionStats { - errors_count: self.session_errors_count.load(Ordering::Relaxed), - } - } - /* ======================================================================== Empty Block Support (TON-specific extension for finalization recovery) @@ -1981,16 +2167,37 @@ impl SessionProcessor { ======================================================================== */ - /// Update the last masterchain finalized seqno (for shardchain decisions) + fn advance_accepted_normal_head_seqno(&mut self, seqno: u32) { + if seqno > self.accepted_normal_head_seqno { + self.accepted_normal_head_seqno = seqno; + if self + .accepted_normal_head_block_id + .as_ref() + .is_some_and(|block_id| block_id.seq_no < seqno) + { + self.accepted_normal_head_block_id = None; + } + } + } + + fn advance_accepted_normal_head_block(&mut self, block_id: BlockIdExt) { + self.advance_accepted_normal_head_seqno(block_id.seq_no); + match self.accepted_normal_head_block_id.as_ref() { + Some(current) if current >= &block_id => {} + _ => self.accepted_normal_head_block_id = Some(block_id), + } + } + + /// Update applied-top tracking from manager notification. /// - /// This should be called when a masterchain block finalization event is received - /// (similar to C++ `ConsensusBus::BlockFinalizedInMasterchain` event). - /// The seqno is used in `should_generate_empty_block()` to determine if a - /// shardchain should generate an empty block. + /// This should be called when manager forwards the current applied top for this + /// session shard (C++ `top_block_id()` / `BlockFinalizedInMasterchain` semantics). + /// The seqno is used in `should_generate_empty_block()` and the exact block id is + /// also used to seed the MC validation accepted-head cursor when known. /// /// # Arguments /// - /// * `seqno` - The masterchain block seqno that was finalized + /// * `applied_top` - Current applied top for this session shard /// /// # Reference /// @@ -2001,9 +2208,21 @@ impl SessionProcessor { /// last_consensus_finalized_seqno_ = std::max(last_mc_finalized_seqno_, last_consensus_finalized_seqno_); /// } /// ``` - pub fn set_mc_finalized_seqno(&mut self, seqno: u32) { + pub fn set_mc_finalized_block(&mut self, applied_top: BlockIdExt) { + let session_shard = self.description.get_shard(); + if applied_top.shard() != session_shard { + log::trace!( + "Session {}: ignoring MC finalization update for mismatched shard {} \ + (session shard {})", + &self.session_id().to_hex_string()[..8], + applied_top.shard(), + session_shard + ); + return; + } + let seqno = applied_top.seq_no; log::trace!( - "Session {}: set_mc_finalized_seqno={} (was {:?})", + "Session {}: set_applied_top_seqno={} (was {:?})", &self.session_id().to_hex_string()[..8], seqno, self.last_mc_finalized_seqno @@ -2012,6 +2231,7 @@ impl SessionProcessor { // last_mc_finalized_seqno_ = std::max(event->block.seqno(), last_mc_finalized_seqno_); let prev_mc = self.last_mc_finalized_seqno.unwrap_or(0); self.last_mc_finalized_seqno = Some(seqno.max(prev_mc)); + self.advance_accepted_normal_head_block(applied_top); // C++ parity: BlockFinalizedInMasterchain also couples to last_consensus_finalized_seqno_ let consensus = self.last_consensus_finalized_seqno.unwrap_or(0); let mc = self.last_mc_finalized_seqno.unwrap_or(0); @@ -2023,7 +2243,7 @@ impl SessionProcessor { /// Get the last masterchain finalized seqno /// - /// Returns `None` if no MC finalization has been reported. + /// Returns the last known applied-top seqno for this session shard. #[allow(dead_code)] pub fn last_mc_finalized_seqno(&self) -> Option { self.last_mc_finalized_seqno @@ -2072,7 +2292,7 @@ impl SessionProcessor { // C++ parity: ALWAYS generate empty if previous block has before_split flag // This is required for shard split/merge operations. // Reference: C++ block-producer.cpp is_before_split() check - if self.last_committed_before_split { + if self.finalized_head_before_split { log::debug!( "Session {} should_generate_empty_block: slot={}, seqno={} - generating EMPTY \ (prev block has before_split=true, required for split/merge)", @@ -2237,6 +2457,9 @@ impl SessionProcessor { let session_id = self.session_id().to_hex_string(); let session_prefix = &session_id[..8.min(session_id.len())]; let cooldown = self.health_alert_state.cooldown; + let skip_ratio_min_delta_votes = (self.description.get_total_nodes() as u64).max(2) / 2; + const SKIP_RATIO_WARN_THRESHOLD: f64 = 3.0; + const SKIP_RATIO_ERROR_THRESHOLD: f64 = 8.0; // 1. Progress gap: first_non_progressed - first_non_finalized > window size let first_non_finalized = self.simplex_state.get_first_non_finalized_slot().0; @@ -2268,10 +2491,10 @@ impl SessionProcessor { } } - // 2. Zero finalization speed: committed slot unchanged for too long + // 2. Zero finalization speed: finalized slot unchanged for too long let stall_warn_secs = self.description.opts().health_stall_warning_secs; let stall_err_secs = self.description.opts().health_stall_error_secs; - let current_finalized = self.last_committed_slot.map(|s| s.0 as f64).unwrap_or(0.0); + let current_finalized = self.finalized_head_slot.map(|s| s.0 as f64).unwrap_or(0.0); if current_finalized != self.health_alert_state.prev_last_finalized_slot { self.health_alert_state.last_finalization_nonzero_at = now; self.health_alert_state.prev_last_finalized_slot = current_finalized; @@ -2432,7 +2655,71 @@ impl SessionProcessor { ); } - // 8. Validator isolation: only self is active for extended period + // 8. Skip/notar/final ratio anomaly (delta-based, inbound vote stream). + let current_notar = self.votes_in_notarize_total; + let current_final = self.votes_in_finalize_total; + let current_skip = self.votes_in_skip_total; + let delta_notar = + current_notar.saturating_sub(self.health_alert_state.prev_votes_in_notarize); + let delta_final = + current_final.saturating_sub(self.health_alert_state.prev_votes_in_finalize); + let delta_skip = current_skip.saturating_sub(self.health_alert_state.prev_votes_in_skip); + let delta_total = delta_notar + delta_final + delta_skip; + + self.health_alert_state.prev_votes_in_notarize = current_notar; + self.health_alert_state.prev_votes_in_finalize = current_final; + self.health_alert_state.prev_votes_in_skip = current_skip; + + if delta_total >= skip_ratio_min_delta_votes + && now.duration_since(self.health_alert_state.last_skip_ratio_warn).unwrap_or_default() + >= cooldown + { + let progress_votes = delta_notar + delta_final; + // Compare skip traffic against the full progress-vote stream to avoid + // false positives when only one denominator is sparse in this window. + let skip_to_progress = delta_skip as f64 / (progress_votes.max(1) as f64); + let skip_to_notar = if delta_notar > 0 { + delta_skip as f64 / (delta_notar as f64) + } else { + f64::INFINITY + }; + let skip_to_final = if delta_final > 0 { + delta_skip as f64 / (delta_final as f64) + } else { + f64::INFINITY + }; + let skip_share = if delta_total > 0 { + 100.0 * (delta_skip as f64) / (delta_total as f64) + } else { + 0.0 + }; + + if skip_to_progress >= SKIP_RATIO_WARN_THRESHOLD { + self.health_alert_state.last_skip_ratio_warn = now; + self.health_warnings_counter.increment(1); + + if skip_to_progress >= SKIP_RATIO_ERROR_THRESHOLD && progress_votes == 0 { + log::error!( + "SIMPLEX_HEALTH anomaly=skip_vote_dominance session={session_prefix} \ + delta_skip={delta_skip} delta_notar={delta_notar} delta_final={delta_final} \ + skip_share={skip_share:.0}% skip_to_progress={skip_to_progress:.2} \ + skip_to_notar={skip_to_notar:.2} \ + skip_to_final={skip_to_final:.2}" + ); + self.increment_error(); + } else { + log::warn!( + "SIMPLEX_HEALTH anomaly=skip_vote_dominance session={session_prefix} \ + delta_skip={delta_skip} delta_notar={delta_notar} delta_final={delta_final} \ + skip_share={skip_share:.0}% skip_to_progress={skip_to_progress:.2} \ + skip_to_notar={skip_to_notar:.2} \ + skip_to_final={skip_to_final:.2}" + ); + } + } + } + + // 9. Validator isolation: only self is active for extended period let isolation_threshold = Duration::from_secs(60); let session_age = now.duration_since(self.session_creation_time()).unwrap_or_default(); if session_age > isolation_threshold @@ -2468,7 +2755,7 @@ impl SessionProcessor { /// - SimplexState FSM dump (via SimplexState::debug_dump) /// /// # Arguments - /// * `is_stalled` - If true, consensus is stalled (no commits for ROUND_DEBUG_PERIOD). + /// * `is_stalled` - If true, consensus is stalled (no finalizations for ROUND_DEBUG_PERIOD). /// In stall mode, full details are logged to INFO level for immediate visibility. /// In normal mode (health check), brief status goes to INFO, full details to DEBUG. /// @@ -2490,13 +2777,15 @@ impl SessionProcessor { // Stalled consensus: log error and increment error counter if is_stalled { - let time_since_commit = - now.duration_since(self.last_commit_time).map(|d| d.as_secs_f64()).unwrap_or(0.0); + let time_since_finalization = now + .duration_since(self.last_finalization_time) + .map(|d| d.as_secs_f64()) + .unwrap_or(0.0); log::error!( - "Session {} stalled (no commits for {:.1}s, slot_dur={:.1}s, threshold {:.0}s), \ + "Session {} stalled (no finalizations for {:.1}s, slot_dur={:.1}s, threshold {:.0}s), \ slot_nf={}, slot_np={}", &self.session_id().to_hex_string()[..8], - time_since_commit, + time_since_finalization, slot_dur_secs, ROUND_DEBUG_PERIOD.as_secs_f64(), fsm_first_non_finalized_slot, @@ -2619,6 +2908,22 @@ impl SessionProcessor { self.validated_candidates.len() )); + let metrics_snapshot = self.metrics_receiver.snapshot(); + let metric_counter = |name: &str| metrics_snapshot.counters.get(name).copied().unwrap_or(0); + result.push_str(&format!( + " - metrics: recv_broadcast={}, recv_query={}, drop_old={}, drop_future={}, \ + drop_unexpected_sender={}, drop_conflicting_slot={}, generated_missed_validation={}, \ + generated_waiting_validation={}\n", + metric_counter("simplex_candidate_received_broadcast"), + metric_counter("simplex_candidate_received_query"), + metric_counter("simplex_candidate_precheck_drop_old_slot"), + metric_counter("simplex_candidate_precheck_drop_future_slot"), + metric_counter("simplex_candidate_precheck_drop_unexpected_sender"), + metric_counter("simplex_candidate_precheck_drop_conflicting_slot"), + metric_counter("simplex_generated_candidate_validation_missed"), + self.generated_candidates_waiting_validation.len(), + )); + // Nodes list (with activity info) result.push_str(" - nodes:\n"); for i in 0..self.description.get_total_nodes() { @@ -2683,13 +2988,12 @@ impl SessionProcessor { result.push_str(&format!(" {}\n", line)); } - // C++ parity: standstill slot-grid dump (only on stall) - // Reference: C++ pool.cpp alarm() sb << slot << ": " << per-validator markers + // C++ parity: standstill diagnostic dump (only on stall) if is_stalled { - let grid = self.simplex_state.standstill_slot_grid_dump(&self.description); - if !grid.is_empty() { - result.push_str(" - standstill_slot_grid:\n"); - for line in grid.lines() { + let diagnostic = self.simplex_state.standstill_diagnostic_dump(&self.description); + if !diagnostic.is_empty() { + result.push_str(" - standstill_diagnostic:\n"); + for line in diagnostic.lines() { result.push_str(&format!(" {}\n", line)); } } @@ -2714,10 +3018,12 @@ impl SessionProcessor { /// with unarmed timeouts (`skip_timestamp = None`) so that no skip /// cascade fires during the startup delay. /// - /// Reference: C++ `Start` event -> `advance_present()` -> - /// `LeaderWindowObserved` -> `alarm_timestamp()`. + /// C++ reference: `start_up()` initialises state and processes + /// bootstrap votes; timeouts are armed through the event flow + /// (`LeaderWindowObserved` → `alarm_timestamp()`). In Rust the + /// equivalent arming point is this explicit `start()` call. pub(crate) fn start(&mut self) { - self.simplex_state.set_timeouts(&self.description); + self.simplex_state.reset_timeouts_on_start(&self.description); log::info!( "Session {} started: skip timeouts armed", @@ -2742,7 +3048,7 @@ impl SessionProcessor { // Stalled consensus detection let now = self.now(); - // Debug dump if no commits for ROUND_DEBUG_PERIOD (stalled consensus) + // Debug dump if no finalizations for ROUND_DEBUG_PERIOD (stalled consensus) if now >= self.round_debug_at { self.debug_dump(true); // is_stalled=true: full dump to INFO level self.round_debug_at = now + ROUND_DEBUG_PERIOD; @@ -2763,8 +3069,9 @@ impl SessionProcessor { // Process all events produced by FSM self.process_simplex_events(); - // Retry finalized chain commits/proof requests even without new inbound events. - self.try_commit_finalized_chains(); + // Keep receiver standstill slots aligned even when the FSM tracked + // interval changes outside the finalization / skip hooks. + self.sync_standstill_slots_from_state(); // Update awake time from FSM timeout if let Some(fsm_timeout) = self.simplex_state.get_next_timeout() { @@ -2922,8 +3229,22 @@ impl SessionProcessor { } } - // Don't generate if already generated or pending for this slot + // Don't generate if already generated or pending for this slot. + // However, if we have a local chain head with a deferred precollation for the + // next slot in this window, retry it — the parent may have been notarized since + // the initial deferral, but the retry in handle_notarization_reached could have + // missed it (e.g., local_chain_head was updated between defer and event). if self.slot_is_generated(current_slot) || self.slot_is_pending_generate(current_slot) { + if let Some(ref head) = self.local_chain_head { + let next_slot = SlotIndex(head.slot.0 + 1); + if head.window == current_window + && !self.slot_is_generated(next_slot) + && !self.slot_is_pending_generate(next_slot) + && !self.precollated_blocks.contains_key(&next_slot) + { + self.precollate_block(next_slot); + } + } return; } @@ -2994,6 +3315,7 @@ impl SessionProcessor { ); self.collates_precollated_counter.success(); + self.record_collation_start(); // Use precollated candidate (precollated blocks are never empty) self.generated_block(current_slot, CollationResult::Block(candidate)); @@ -3103,11 +3425,11 @@ impl SessionProcessor { if let Some(parent_block_id) = resolved_parent_block_id.clone() { log::debug!( "Session {} invoke_collation: generating EMPTY block for slot {}! \ - new_seqno={}, last_committed_seqno={:?}, last_mc_finalized_seqno={:?}", + new_seqno={}, finalized_head_seqno={:?}, last_mc_finalized_seqno={:?}", self.session_id().to_hex_string(), slot, new_seqno, - self.last_committed_seqno, + self.finalized_head_seqno, self.last_mc_finalized_seqno ); @@ -3129,6 +3451,7 @@ impl SessionProcessor { self.precollated_blocks.insert(slot, precollated_block); // Call collation complete with empty block result + self.record_collation_start(); self.on_collation_complete( slot, request_id, @@ -3143,10 +3466,10 @@ impl SessionProcessor { panic!( "Session {} INVARIANT VIOLATION: should_generate_empty_block({}) returned true \ but no parent available. First block in epoch cannot be empty. \ - last_committed_seqno={:?}, last_mc_finalized_seqno={:?}", + finalized_head_seqno={:?}, last_mc_finalized_seqno={:?}", self.session_id().to_hex_string(), new_seqno, - self.last_committed_seqno, + self.finalized_head_seqno, self.last_mc_finalized_seqno ); } @@ -3278,6 +3601,7 @@ impl SessionProcessor { }); // Notify listener + self.record_collation_start(); self.collates_counter.total_increment(); self.notify_generate_slot(slot, source_info, request, parent, callback); } @@ -3324,6 +3648,7 @@ impl SessionProcessor { CollationResult::Empty { .. } => unreachable!(), }; + let mut publish_now = false; if let Some(precollated_block) = self.precollated_blocks.get_mut(&slot) { if precollated_block.candidate.is_some() { log::error!( @@ -3348,7 +3673,24 @@ impl SessionProcessor { ); // Precollate next block - self.precollate_block(slot + 1); + // C++ parity: if this slot is still in the current leader window, + // publish the candidate immediately instead of waiting until it + // becomes `first_non_progressed_slot`. + let current_window = self.simplex_state.get_current_leader_window_idx(); + publish_now = self.description.get_window_idx(slot) == current_window; + + if publish_now { + log::trace!( + "Session {} on_collation_complete: publishing in-window future candidate \ + for slot {} immediately (C++ parity)", + self.session_id().to_hex_string(), + slot + ); + } + + if !publish_now { + self.precollate_block(slot + 1); + } } else { log::warn!( "Session {} on_collation_complete: no precollated entry for slot {} \ @@ -3358,6 +3700,15 @@ impl SessionProcessor { request_id ); } + + if publish_now { + if let Some(precollated_candidate) = + self.precollated_blocks.get(&slot).and_then(|pb| pb.candidate.clone()) + { + self.generated_block(slot, CollationResult::Block(precollated_candidate)); + self.precollate_block(slot + 1); + } + } } else { // Slot already passed - collation result came too late (expired) log::warn!( @@ -3653,6 +4004,7 @@ impl SessionProcessor { }); // Notify listener + self.record_collation_start(); self.collates_counter.total_increment(); self.notify_generate_slot(slot, source_info, request, parent, callback); } @@ -3747,6 +4099,10 @@ impl SessionProcessor { slot_window, current_window ); + self.note_generated_candidate_validation_missed_for_slot( + slot, + format!("generated_block_stale_window window={slot_window} current_window={current_window}"), + ); self.invalidate_local_chain_head(); return; } @@ -3754,15 +4110,30 @@ impl SessionProcessor { // Use FSM's progress cursor to validate this is for the current slot. // Collation follows notarized/skipped progress, not finalization. let fsm_first_non_progressed_slot = self.simplex_state.get_first_non_progressed_slot(); - if slot != fsm_first_non_progressed_slot { + if slot < fsm_first_non_progressed_slot { log::warn!( - "Session {} generated_block: slot {} != fsm first_non_progressed_slot {}", + "Session {} generated_block: slot {} < fsm first_non_progressed_slot {}", self.session_id().to_hex_string(), slot, fsm_first_non_progressed_slot ); + self.note_generated_candidate_validation_missed_for_slot( + slot, + format!( + "generated_block_old_progress_slot first_non_progressed_slot={fsm_first_non_progressed_slot}" + ), + ); return; } + if slot > fsm_first_non_progressed_slot { + log::trace!( + "Session {} generated_block: publishing future in-window slot {} \ + (first_non_progressed_slot={})", + self.session_id().to_hex_string(), + slot, + fsm_first_non_progressed_slot + ); + } log::trace!( "Session {} generated_block: using locked parent for slot {}: {:?}", @@ -3819,7 +4190,8 @@ impl SessionProcessor { let candidate_parent_info = crate::block::CandidateParentInfo { slot, hash: prepared.candidate_hash.clone() }; let raw_id = RawCandidateId { slot, hash: prepared.candidate_hash.clone() }; - self.generated_parent_cache.insert(raw_id, prepared.block_id_ext.clone()); + self.generated_parent_cache.insert(raw_id.clone(), prepared.block_id_ext.clone()); + self.track_generated_candidate_for_validation(raw_id.clone()); let slot_window = self.description.get_window_idx(slot); self.local_chain_head = Some(LocalChainHead { @@ -4231,17 +4603,38 @@ impl SessionProcessor { // the same window was just generated locally (block-producer.cpp `parent = id`). // Fall back to FSM available_base for the first slot in a window or if the // local chain head is stale. + // + // When require_notarized_parent_for_collation is enabled, the local chain head + // is only used if the parent slot is already notarized (or finalized). This + // matches C++ WaitForParent semantics where validators defer notarization until + // the parent is notarized, so broadcasting a candidate with a non-notarized + // parent is wasteful. Instead of returning early, fall through to the FSM + // available_base path — this ensures collation proceeds if the FSM already has + // a notarized base for this slot (e.g., notarization arrived but the retry in + // handle_notarization_reached didn't match the local_chain_head). let parent = if let Some(ref head) = self.local_chain_head { if head.window == target_window && head.slot + 1 == target_slot { - log::trace!( - "Session {} precollate_block: using local_chain_head for slot {} \ - (parent=s{}:{})", - &self.session_id().to_hex_string()[..8], - target_slot, - head.parent_info.slot, - &head.parent_info.hash.to_hex_string()[..8], - ); - Some(head.parent_info.clone()) + if self.description.opts().require_notarized_parent_for_collation + && !self.simplex_state.has_notarized_block(head.slot) + { + log::debug!( + "Session {} precollate_block: local_chain_head parent s{} \ + not yet notarized, falling through to FSM base", + &self.session_id().to_hex_string()[..8], + head.slot, + ); + None + } else { + log::trace!( + "Session {} precollate_block: using local_chain_head for slot {} \ + (parent=s{}:{})", + &self.session_id().to_hex_string()[..8], + target_slot, + head.parent_info.slot, + &head.parent_info.hash.to_hex_string()[..8], + ); + Some(head.parent_info.clone()) + } } else { None } @@ -4417,13 +4810,13 @@ impl SessionProcessor { return; } - // Reject far-future slots before signature verification (DoS protection) - if self.simplex_state.is_slot_too_far_ahead(tl_slot) { + // Mirror C++ vote ingress: reject slots at or beyond `first_too_new_slot`. + if self.simplex_state.is_vote_slot_too_far_ahead(tl_slot) { log::warn!( - "Session {} on_vote: REJECTED - slot {tl_slot} too far ahead (max={}) \ + "Session {} on_vote: REJECTED - slot {tl_slot} too far ahead (first_too_new={}) \ kind={tl_kind} from source_idx={source_idx}", &self.session_id().to_hex_string()[..8], - self.simplex_state.max_acceptable_slot(), + self.simplex_state.first_too_new_vote_slot(), ); return; } @@ -4487,9 +4880,21 @@ impl SessionProcessor { ); match tl_kind { - "notarize" => self.votes_in_notarize_counter.increment(1), - "finalize" => self.votes_in_finalize_counter.increment(1), - "skip" => self.votes_in_skip_counter.increment(1), + "notarize" => { + self.votes_in_total_counter.increment(1); + self.votes_in_notarize_counter.increment(1); + self.votes_in_notarize_total = self.votes_in_notarize_total.saturating_add(1); + } + "finalize" => { + self.votes_in_total_counter.increment(1); + self.votes_in_finalize_counter.increment(1); + self.votes_in_finalize_total = self.votes_in_finalize_total.saturating_add(1); + } + "skip" => { + self.votes_in_total_counter.increment(1); + self.votes_in_skip_counter.increment(1); + self.votes_in_skip_total = self.votes_in_skip_total.saturating_add(1); + } _ => {} } @@ -4708,17 +5113,6 @@ impl SessionProcessor { return; } - // Reject far-future slots before signature verification (DoS protection) - if self.simplex_state.is_slot_too_far_ahead(tl_slot) { - log::warn!( - "Session {} on_certificate: REJECTED - slot {tl_slot} too far ahead \ - (max={}) kind={tl_kind} from source_idx={source_idx}", - &self.session_id().to_hex_string()[..8], - self.simplex_state.max_acceptable_slot(), - ); - return; - } - // Parse and verify the certificate (C++ strict policy) // Certificate::from_tl performs comprehensive validation: // - Rejects invalid validator indices @@ -4843,7 +5237,7 @@ impl SessionProcessor { cert.signatures.len(), ); // NOTE: SimplexState emits: - // - BlockFinalized (commit trigger) and + // - BlockFinalized (finalization trigger) and // - FinalizationReached (standstill caching) // when the cert is stored. SessionProcessor handles those in event handlers. } @@ -5021,14 +5415,32 @@ impl SessionProcessor { let sender_idx = ValidatorIndex::new(source_idx); let slot = SlotIndex::new(slot); + let is_broadcast_candidate = notar_cert.is_none(); + let is_local_self_candidate = + is_broadcast_candidate && sender_idx == self.description.get_self_idx(); + self.record_candidate_ingress(sender_idx, is_broadcast_candidate); // Reject far-future slots (DoS protection) — before any signature verification if self.simplex_state.is_slot_too_far_ahead(slot) { + if is_broadcast_candidate { + self.candidate_precheck_future_slot_drop_counter.increment(1); + } + if is_local_self_candidate { + self.note_generated_candidate_validation_missed_for_slot( + slot, + format!( + "candidate_precheck_too_far_ahead max_acceptable_slot={}", + self.simplex_state.max_acceptable_slot() + ), + ); + } log::warn!( - "Session {} on_candidate_received: REJECTED - slot {} too far ahead (max={})", + "Session {} on_candidate_received: REJECTED precheck_drop_reason=too_far_ahead \ + slot={} max={} origin={}", &self.session_id().to_hex_string()[..8], slot, self.simplex_state.max_acceptable_slot(), + if is_broadcast_candidate { "broadcast" } else { "query" }, ); return; } @@ -5057,6 +5469,51 @@ impl SessionProcessor { return; } + if is_broadcast_candidate && sender_idx != leader_idx { + self.candidate_precheck_unexpected_sender_drop_counter.increment(1); + log::warn!( + "Session {} on_candidate_received: REJECTED \ + precheck_drop_reason=unexpected_sender \ + slot={} leader={} sender={} origin=broadcast", + &self.session_id().to_hex_string()[..8], + slot, + leader_idx, + sender_idx + ); + return; + } + + // Broadcast path must reject stale slots eagerly to avoid stale-body/db churn. + let fsm_first_non_finalized_slot = self.simplex_state.get_first_non_finalized_slot(); + if slot < fsm_first_non_finalized_slot { + if is_broadcast_candidate { + self.candidate_precheck_old_slot_drop_counter.increment(1); + if is_local_self_candidate { + self.note_generated_candidate_validation_missed_for_slot( + slot, + format!( + "candidate_precheck_old_slot first_non_finalized_slot={fsm_first_non_finalized_slot}" + ), + ); + } + log::warn!( + "Session {} on_candidate_received: REJECTED precheck_drop_reason=old_slot \ + slot={} first_non_finalized={} origin=broadcast", + &self.session_id().to_hex_string()[..8], + slot, + fsm_first_non_finalized_slot + ); + return; + } + + log::trace!( + "Session {} on_candidate_received: old slot received {} (current={}) origin=query", + self.session_id().to_hex_string(), + slot, + fsm_first_non_finalized_slot, + ); + } + // Get leader public key for signature verification let leader_key = self.description.get_source_public_key(leader_idx).clone(); @@ -5076,6 +5533,12 @@ impl SessionProcessor { ) { Ok(c) => c, Err(e) => { + if is_local_self_candidate { + self.note_generated_candidate_validation_missed_for_slot( + slot, + format!("candidate_deserialization_failed error={e}"), + ); + } log::warn!( "Session {} on_candidate_received: failed to deserialize candidate from \ sender={}, leader={}, slot={}: {}", @@ -5103,22 +5566,10 @@ impl SessionProcessor { leader_idx ); - // 4. Validate slot is reasonable - // Use FSM's finalization cursor to reject candidates from old slots. - let fsm_first_non_finalized_slot = self.simplex_state.get_first_non_finalized_slot(); - if slot < fsm_first_non_finalized_slot { - log::trace!( - "Session {} on_candidate_received: old slot received {} (current={})", - self.session_id().to_hex_string(), - slot, - fsm_first_non_finalized_slot, - ); - } - // 5. Candidates can be received via relay or requestCandidate (query response), // so the sender can differ from the slot leader. The signature is verified against // the leader's key above, so a mismatch here is not an error. - if sender_idx != leader_idx { + if sender_idx != leader_idx && !is_broadcast_candidate { log::trace!( "Session {} on_candidate_received: received leader candidate via relay/query: \ slot={slot} leader={leader_idx} sender={sender_idx}", @@ -5136,6 +5587,38 @@ impl SessionProcessor { candidate_id.slot ); + if is_broadcast_candidate { + match self.seen_broadcast_candidates.get(&slot).cloned() { + Some(existing) if existing != candidate_id => { + self.candidate_precheck_conflicting_slot_drop_counter.increment(1); + if is_local_self_candidate { + self.note_generated_candidate_validation_missed( + &candidate_id, + format!( + "candidate_precheck_conflicting_slot first_seen_slot_hash={}:{}", + existing.slot, + &existing.hash.to_hex_string()[..8] + ), + ); + } + log::warn!( + "Session {} on_candidate_received: REJECTED \ + precheck_drop_reason=conflicting_slot_candidate \ + slot={} first_seen={:?} new_candidate={:?} origin=broadcast", + &self.session_id().to_hex_string()[..8], + slot, + existing, + candidate_id + ); + return; + } + Some(_) => {} + None => { + self.seen_broadcast_candidates.insert(slot, candidate_id.clone()); + } + } + } + // Check if candidate already known. // A finalized-boundary stub (seeded by handle_block_finalized with empty data) is NOT // "already known" for this purpose -- we want the real body to overwrite it. @@ -5161,19 +5644,17 @@ impl SessionProcessor { // already have the candidate body (e.g., we missed the certificate broadcast). // Do NOT drop notar_cert in this case, otherwise the node can get permanently stuck // waiting for NotarCert while repeatedly receiving bodies. - let had_any_cert = notar_cert.is_some(); if let Some(ref cert_bytes) = notar_cert { self.process_received_notar_cert(slot, &id_hash, cert_bytes); } - if had_any_cert { - self.try_commit_finalized_chains(); + if notar_cert.is_some() { self.check_all(); } return; } // 7. Store candidate in received_candidates for finalization (even if not validated) - // This allows us to commit blocks that are finalized before validation completes + // This allows us to accept blocks that are finalized before validation completes // Reference: validator-session/src/session_processor.rs set_block_candidate let receive_time = self.now(); let block_id = raw_candidate.block.block_id(); @@ -5223,27 +5704,33 @@ impl SessionProcessor { // while the parent slot is from the Simplex FSM. These can legitimately diverge when: // 1. The FSM parent is an older notarized block // 2. The collator's chain has more finalized blocks - // Seqno validation is still performed at commit time in commit_single_block. + // Seqno validation is deferred until finalized state is materialized. log::debug!( "Session {} on_candidate_received: seqno differs from parent-based \ expectation for slot={slot}, received seqno={received_seqno}, \ expected={expected_seqno} (parent_seqno={parent_seqno}, \ - is_empty={is_empty}). Allowing through - will validate at commit.", + is_empty={is_empty}). Allowing through - finalized path will resolve it.", &self.session_id().to_hex_string()[..8], ); } } // If parent not yet received, we can't validate seqno - allow it through - // Validation will happen during commit + // Validation will happen when finalized state is applied. } else { // No parent (first block in epoch) - seqno is based on the session's initial_block_seqno // which may be > 1 if this is not the first session (e.g., after zerostate, seqno=1, but // subsequent sessions continue from their start seqno). - // We don't validate first block seqno at receive time - defer to commit time. + // We don't validate first block seqno at receive time - defer to finalized application. // INVARIANT: First block (no parent) cannot be empty // Empty blocks inherit parent's BlockIdExt, so they require a parent if is_empty { + if is_local_self_candidate { + self.note_generated_candidate_validation_missed( + &candidate_id, + "first_block_cannot_be_empty", + ); + } log::warn!( "Session {} on_candidate_received: INVARIANT VIOLATION - first block (slot={}) \ cannot be empty (empty blocks require parent). Rejecting.", @@ -5253,17 +5740,15 @@ impl SessionProcessor { return; } - // INVARIANT: First block should be slot 0 (or first slot in epoch) - // If we receive a block without parent at slot > 0, it's suspicious + // Genesis-parent candidates at slot > 0 are normal in Simplex: when early + // slots are skipped, subsequent leaders produce blocks with parent_id=None. if slot.value() != 0 { - log::warn!( - "Session {} on_candidate_received: unexpected no-parent block at slot={} \ - (expected slot 0 for first block). Allowing but logging.", + log::trace!( + "Session {} on_candidate_received: genesis-parent block at slot={} \ + (early slots were skipped)", &self.session_id().to_hex_string()[..8], slot ); - // Note: We allow this through because in some edge cases (session restart, - // fork recovery) the first block might not be at slot 0 } log::debug!( @@ -5275,7 +5760,7 @@ impl SessionProcessor { } // Extract actual block data from RawCandidate (not the TL wrapper) - // This is what on_block_committed callback expects + // This is what validation/finalization callbacks consume. let (block_data, collated_data) = match raw_candidate.block.as_block() { Some(block) => ( consensus_common::ConsensusCommonFactory::create_block_payload(block.data.clone()), @@ -5299,6 +5784,12 @@ impl SessionProcessor { let candidate_hash_data_bytes = if is_empty { // Empty blocks use candidateHashDataEmpty with CandidateId parent let Some(parent) = parent_id.as_ref() else { + if is_local_self_candidate { + self.note_generated_candidate_validation_missed( + &candidate_id, + "empty_candidate_missing_parent", + ); + } log::error!( "Session {} on_candidate_received: empty block must have parent", &self.session_id().to_hex_string()[..8] @@ -5366,6 +5857,15 @@ impl SessionProcessor { // Remove from requested_candidates if we were waiting for this self.requested_candidates.remove(&candidate_id); + // Delayed finalized delivery: + // if FinalCert arrived earlier and body has just appeared, emit now. + let pending_finalized_event = + self.finalized_pending_body.get(&candidate_id).map(|entry| entry.event.clone()); + if let Some(event) = pending_finalized_event { + self.maybe_emit_out_of_order_finalized(&candidate_id, &event); + self.maybe_apply_finalized_state(&candidate_id); + } + // DEBUG: Short pattern for quick grep (RECV = candidate received) log::debug!( "Session {} RECV candidate: slot={slot}, hash={}, seqno={received_seqno}, \ @@ -5407,22 +5907,20 @@ impl SessionProcessor { id_hash, ); - // Optimistic validation: candidates with non-committed (notarized-only) parents + // Optimistic validation: candidates with non-finalized (notarized-only) parents // are accepted and forwarded to check_validation(), which validates them as soon - // as the parent slot is notarized in the FSM. No committed-head gating. + // as the parent slot is notarized in the FSM. No finalized-head gating. if let Some(ref p) = parent_id { - let parent_is_committed = self - .last_committed_block_id + let parent_is_finalized_head = self + .finalized_head_block_id .as_ref() - .and_then(|committed| { - self.received_candidates.get(p).map(|r| &r.block_id == committed) - }) + .and_then(|head| self.received_candidates.get(p).map(|r| &r.block_id == head)) .unwrap_or(false); - if !parent_is_committed { + if !parent_is_finalized_head { log::debug!( "Session {} on_candidate_received: candidate slot={} hash={} has \ - non-committed parent (slot={}), will validate optimistically.", + non-finalized parent (slot={}), will validate optimistically.", &self.session_id().to_hex_string()[..8], slot, &id_hash.to_hex_string()[..8], @@ -5442,10 +5940,6 @@ impl SessionProcessor { self.queue_for_parent_resolution(raw_candidate, slot, leader_idx, receive_time); } - // Try to commit any finalized chains that may have become ready - // (body arrival can make finalized blocks commit-ready) - self.try_commit_finalized_chains(); - // Immediately process the new candidate (don't wait for next awake) self.check_all(); } @@ -5523,7 +6017,7 @@ impl SessionProcessor { // and SessionProcessor handles DB persistence + receiver cache updates there. // // For old slots, SimplexState intentionally avoids emitting events, - // but we still persist the cert for restart/recommit support. + // but we still persist the cert for restart recovery. if slot < first_non_finalized_slot { let candidate_id = RawCandidateId { slot, hash: block_hash.clone() }; if !self.notar_cert_store_results.contains_key(&candidate_id) { @@ -5570,111 +6064,6 @@ impl SessionProcessor { } } - /// Process finalization certificate signature-set received from query response - /// - /// Deserializes, verifies, and stores the FinalCert in SimplexState. - /// - /// Expects serialized boxed `voteSignatureSet` (same wire format as `candidateAndCert.notar`). - fn process_received_final_cert( - &mut self, - slot: SlotIndex, - block_hash: &UInt256, - final_cert_bytes: &[u8], - ) { - log::trace!( - "Session {} process_received_final_cert: slot={} hash={} bytes={}", - &self.session_id().to_hex_string()[..8], - slot, - &block_hash.to_hex_string()[..8], - final_cert_bytes.len() - ); - - // Deserialize VoteSignatureSet - let tl_sigs = match deserialize_boxed(final_cert_bytes) { - Ok(msg) => match msg.downcast::() { - Ok(sigs) => sigs, - Err(_) => { - log::warn!( - "Session {} process_received_final_cert: unexpected type, expected \ - VoteSignatureSet for slot={slot} hash={}", - &self.session_id().to_hex_string()[..8], - &block_hash.to_hex_string()[..8], - ); - return; - } - }, - Err(e) => { - log::warn!( - "Session {} process_received_final_cert: failed to deserialize \ - VoteSignatureSet for slot={slot} hash={}: {e}", - &self.session_id().to_hex_string()[..8], - &block_hash.to_hex_string()[..8], - ); - return; - } - }; - - // Verify and build certificate (matches C++ FinalCert::from_tl signature path) - match self.verify_final_cert_from_vote_signature_set(slot, block_hash, &tl_sigs) { - Ok(final_cert_ptr) => { - log::trace!( - "Session {} process_received_final_cert: verified final cert for slot={slot} \ - hash={} with {} sigs", - &self.session_id().to_hex_string()[..8], - &block_hash.to_hex_string()[..8], - final_cert_ptr.signatures.len(), - ); - - let store_result = self.simplex_state.set_finalize_certificate( - &self.description, - slot, - block_hash, - final_cert_ptr.clone(), - ); - - if let Err(e) = store_result { - log::warn!( - "Session {} process_received_final_cert: final cert conflict slot={slot} \ - hash={}: {e}", - &self.session_id().to_hex_string()[..8], - &block_hash.to_hex_string()[..8], - ); - return; - } - } - Err(e) => { - log::warn!( - "Session {} process_received_final_cert: invalid final cert for slot={slot} \ - hash={}: {e}", - &self.session_id().to_hex_string()[..8], - &block_hash.to_hex_string()[..8], - ); - } - } - } - - /// Verify finalization certificate from a `VoteSignatureSet` received via the - /// committed-proof recovery flow (`get_committed_candidate`). - /// - /// Reference: C++ FinalCert::from_tl(voteSignatureSet&&, vote, bus) - fn verify_final_cert_from_vote_signature_set( - &self, - slot: SlotIndex, - block_hash: &UInt256, - tl_sigs: &VoteSignatureSetBoxed, - ) -> Result { - let vote = crate::simplex_state::FinalizeVote { slot, block_hash: block_hash.clone() }; - - let cert = crate::certificate::FinalCert::from_tl_signatures( - tl_sigs, - vote, - &self.description, - &self.session_id(), - )?; - - Ok(Arc::new(cert)) - } - /// Verify notarization certificate from VoteSignatureSet (C++ wire format) /// /// Parse VoteSignatureSet and verify signatures. @@ -5722,6 +6111,32 @@ impl SessionProcessor { self.last_activity = last_activity; } + pub fn on_standstill_trigger(&mut self, notification: StandstillTriggerNotification) { + log::warn!("{}", self.build_standstill_trigger_log(¬ification)); + } + + fn build_standstill_trigger_log(&self, notification: &StandstillTriggerNotification) -> String { + let mut result = format!( + "Session {}: Standstill detected, re-broadcasting \ + {} certs + {} votes (range [{}, {})). Current pool state:\n", + &self.session_id().to_hex_string()[..8], + notification.cert_count, + notification.vote_count, + notification.begin, + notification.end, + ); + result.push_str(&self.simplex_state.standstill_diagnostic_dump(&self.description)); + result + } + + fn sync_standstill_slots_from_state(&self) { + let (begin, end) = self.simplex_state.get_tracked_slots_interval(); + let progress = self.simplex_state.get_first_non_progressed_slot().value(); + self.receiver.set_ingress_slot_begin(begin); + self.receiver.set_ingress_progress_slot(progress); + self.receiver.set_standstill_slots(begin, end); + } + /* ======================================================================== Recursive Parent Resolution @@ -6043,18 +6458,6 @@ impl SessionProcessor { return; } - // Check if slot has already progressed (skip old candidates) - // Use FSM's progress cursor - anything less is already done - let fsm_first_non_progressed_slot = self.simplex_state.get_first_non_progressed_slot(); - if slot < fsm_first_non_progressed_slot { - log::trace!( - "Session {} register_resolved_candidate: skipping old slot {slot} (fsm \ - first_non_progressed_slot is {fsm_first_non_progressed_slot})", - self.session_id().to_hex_string(), - ); - return; - } - log::trace!( "Session {} register_resolved_candidate: registering candidate slot={} hash={}", self.session_id().to_hex_string(), @@ -6125,14 +6528,138 @@ impl SessionProcessor { Reference: validator-session/src/session_processor.rs try_approve_block, candidate_decision_* */ + /// C++ `WaitForParent`-equivalent readiness check. + /// + /// Validates parent and skip-gap preconditions before the candidate can be sent + /// to higher-layer validation. + fn is_wait_for_parent_ready(&self, pending: &PendingValidation) -> bool { + let slot = pending.slot; + let first_non_finalized = self.simplex_state.get_first_non_finalized_slot(); + let parent_id = pending.raw_candidate.parent_id.as_ref(); + + // C++ parity (pool.cpp maybe_resolve_request): + // next_slot_after_parent = parent.has_value() ? parent->slot + 1 : 0 + let next_slot_after_parent = match parent_id { + Some(pid) => { + if pid.slot >= slot { + return false; + } + pid.slot + 1 + } + None => SlotIndex::new(0), + }; + + if slot < first_non_finalized { + return false; + } + if next_slot_after_parent < first_non_finalized { + return false; + } + + // C++ parity (pool.cpp maybe_resolve_request): + // - if parent is at finalized boundary, it must match last finalized block; + // - otherwise parent slot must be notarized with the same candidate hash. + if next_slot_after_parent == first_non_finalized { + match parent_id { + None => { + // Genesis parent at genesis boundary: both last_finalized and parent + // are None. C++ invariant: first_nonfinalized==0 <=> !last_finalized.has_value(), + // so when both are None they match and the check passes. + if first_non_finalized.value() != 0 { + return false; + } + } + Some(pid) => { + let Some((last_finalized_slot, final_cert)) = + self.simplex_state.get_last_finalize_certificate() + else { + return false; + }; + if last_finalized_slot != pid.slot || final_cert.vote.block_hash != pid.hash { + return false; + } + } + } + } else { + // next_slot_after_parent > first_non_finalized, so parent must exist and be notarized. + // Genesis parent has next_slot_after_parent=0 which can't exceed first_non_finalized. + let Some(pid) = parent_id else { + return false; + }; + let Some(notarized_hash) = + self.simplex_state.get_notarized_block_hash(&self.description, pid.slot) + else { + return false; + }; + if notarized_hash != pid.hash { + return false; + } + } + + if next_slot_after_parent == slot { + return true; + } + + // All intermediate slots must have Skip certificates. + let mut gap_slot = next_slot_after_parent; + while gap_slot < slot { + if !self.simplex_state.has_skip_certificate_for_slot(&self.description, gap_slot) { + return false; + } + gap_slot += 1; + } + + true + } + + fn check_mc_validation_ready( + &self, + pending: &PendingValidation, + ) -> Result { + let expected_tip = self.resolve_parent_normal_tip(&pending.raw_candidate); + let expected_seqno = match expected_tip.as_ref() { + Some(block_id) => block_id.seq_no, + None if pending.raw_candidate.parent_id.is_none() => { + self.description.get_initial_block_seqno().saturating_sub(1) + } + None => { + fail!("Cannot resolve parent normal tip for MC candidate"); + } + }; + + if self.accepted_normal_head_seqno < expected_seqno { + return Ok(McValidationReadiness::WaitingForAcceptedHead); + } + if self.accepted_normal_head_seqno > expected_seqno { + fail!( + "Stale MC candidate parent: accepted head seqno {} already passed expected {}", + self.accepted_normal_head_seqno, + expected_seqno + ); + } + + if let (Some(accepted_tip), Some(expected_tip)) = + (self.accepted_normal_head_block_id.as_ref(), expected_tip.as_ref()) + { + if accepted_tip != expected_tip { + fail!( + "Stale MC candidate parent: accepted head {} does not match expected {}", + accepted_tip, + expected_tip + ); + } + } + + Ok(McValidationReadiness::Ready) + } + /// Check pending validations and send to higher layer for validation /// /// Called from check_all(). Iterates all pending_validations and forwards /// each eligible candidate to the SessionListener via on_candidate(). /// - /// Validates pending candidates whose parent slot has been notarized (or finalized) - /// in the FSM. Genesis candidates (no parent) are always eligible. This enables - /// optimistic validation on notarized-only parents (C++ parity). + /// Validates pending candidates whose parent chain is `WaitForParent`-ready in the FSM: + /// parent readiness + gap skip coverage (C++ parity). fn check_validation(&mut self) { check_execution_time!(10_000); instrument!(); @@ -6140,10 +6667,11 @@ impl SessionProcessor { // Collect candidates to validate. // A candidate is eligible when: // 1. It is fully resolved (parent chain data available — enforced by register_resolved_candidate). - // 2. Its parent slot is notarized (or finalized) in the FSM, OR it is a genesis candidate. + // 2. Parent chain is C++ WaitForParent-ready (notar/final parent + gap skip coverage). // 3. It is not already being validated, approved, or rejected. let mut to_validate: Vec<(RawCandidateId, SlotIndex, ValidatorIndex, SystemTime)> = Vec::new(); + let mut to_reject: Vec<(RawCandidateId, SlotIndex, Error)> = Vec::new(); let candidate_ids: Vec = self.pending_validations.keys().cloned().collect(); for candidate_id in candidate_ids { @@ -6175,6 +6703,27 @@ impl SessionProcessor { } } + let wait_for_parent_ready = self.is_wait_for_parent_ready(pending); + if !wait_for_parent_ready + && matches!(PARENT_READINESS_MODE, ParentReadinessMode::StrictWaitForParent) + { + continue; + } + + if self.description.get_shard().is_masterchain() + && !pending.raw_candidate.block.is_empty() + && matches!(MC_ACCEPTED_HEAD_MODE, McAcceptedHeadMode::StrictSessionProcessorGate) + { + match self.check_mc_validation_ready(pending) { + Ok(McValidationReadiness::Ready) => {} + Ok(McValidationReadiness::WaitingForAcceptedHead) => continue, + Err(e) => { + to_reject.push((candidate_id.clone(), pending.slot, e)); + continue; + } + } + } + // Empty blocks skip ValidatorGroup validation but still need FSM-tip reference // check (performed in try_approve_block). C++ block-validator.cpp rejects unless // block == event->state->as_normal(). @@ -6188,20 +6737,6 @@ impl SessionProcessor { continue; } - // Non-empty block: parent slot must be notarized (or finalized) in the FSM. - // `is_fully_resolved` (checked before insertion into pending_validations) guarantees - // that parent chain data is available; this check confirms the parent reached consensus. - match pending.raw_candidate.parent_id.as_ref() { - None => { - // Genesis/first-in-epoch: always eligible - } - Some(parent_id) => { - if !self.simplex_state.has_notarized_block(parent_id.slot) { - continue; - } - } - } - to_validate.push(( candidate_id.clone(), pending.slot, @@ -6210,22 +6745,29 @@ impl SessionProcessor { )); } + for (candidate_id, slot, err) in to_reject { + self.candidate_decision_fail(slot, candidate_id, err); + } + // Process each candidate for (candidate_id, slot, source_idx, receive_time) in to_validate { self.try_approve_block(&candidate_id, slot, source_idx, receive_time); } } - /// Resolve the expected referenced BlockIdExt for an empty candidate. + /// Resolve the parent-chain normal tip (`event->state->as_normal()` in C++). /// /// Walks the parent chain through `received_candidates` until a non-empty - /// ancestor is found. Returns its `block_id`, which is the C++ equivalent - /// of `event->state->as_normal()` in `block-validator.cpp`. + /// ancestor is found. For candidates without an explicit parent, falls back to the + /// exact accepted head when it is already known. /// /// Returns `None` if the parent chain is broken, missing, or contains - /// only empty ancestors (no normal tip exists). - fn resolve_expected_empty_block(&self, raw_candidate: &RawCandidate) -> Option { - let parent_id = raw_candidate.parent_id.as_ref()?; + /// only empty ancestors and no exact accepted head is available. + fn resolve_parent_normal_tip(&self, raw_candidate: &RawCandidate) -> Option { + let parent_id = match raw_candidate.parent_id.as_ref() { + Some(parent_id) => parent_id, + None => return self.accepted_normal_head_block_id.clone(), + }; let parent = self.received_candidates.get(parent_id)?; if !parent.is_empty { return Some(parent.block_id.clone()); @@ -6270,6 +6812,7 @@ impl SessionProcessor { .entry(candidate_id.clone()) .and_modify(|c| *c += 1) .or_insert(0); + self.mark_generated_candidate_validation_started(candidate_id); // Get pending validation (now safe to borrow after mutable operations) let Some(pending) = self.pending_validations.get(candidate_id) else { @@ -6286,7 +6829,7 @@ impl SessionProcessor { // the parent chain and compare before approving. if pending.raw_candidate.block.is_empty() { let referenced_block = pending.raw_candidate.block.block_id().clone(); - let expected = self.resolve_expected_empty_block(&pending.raw_candidate); + let expected = self.resolve_parent_normal_tip(&pending.raw_candidate); let cid = candidate_id.clone(); match expected { @@ -6461,24 +7004,33 @@ impl SessionProcessor { // has round gating; in roundless Simplex we gate by "still pending"). if !self.pending_validations.contains_key(&candidate_id) { self.validation_late_callback_counter.increment(1); + self.note_generated_candidate_validation_missed( + &candidate_id, + "validation_late_callback_without_pending_entry", + ); self.pending_approve.remove(&candidate_id); self.validation_attempt_map.remove(&candidate_id); return; } - // If the block is already committed by the time validation completes, drop the result. - // (We might have advanced quickly while validation was queued in the higher layer.) - if let (Some(committed_seqno), Some(cand_seqno)) = ( - self.last_committed_seqno, + // If the block is already finalized by the time validation completes, drop the result. + if let (Some(finalized_seqno), Some(cand_seqno)) = ( + self.finalized_head_seqno, self.pending_validations .get(&candidate_id) .and_then(|p| p.raw_candidate.block.as_block().map(|b| b.id.seq_no)), ) { - if cand_seqno <= committed_seqno { + if cand_seqno <= finalized_seqno { + self.note_generated_candidate_validation_missed( + &candidate_id, + format!( + "validation_succeeded_after_finalization finalized_seqno={finalized_seqno} cand_seqno={cand_seqno}" + ), + ); log::warn!( "Session {} candidate_decision_ok: slot={slot}, hash={:?}, \ - committed_seqno={committed_seqno}, cand_seqno={cand_seqno} (drop because \ - new block is already committed)", + finalized_seqno={finalized_seqno}, cand_seqno={cand_seqno} (drop because \ + block is already finalized)", self.session_id().to_hex_string(), candidate_id, ); @@ -6509,6 +7061,10 @@ impl SessionProcessor { let pending = match self.pending_validations.remove(&candidate_id) { Some(p) => p, None => { + self.note_generated_candidate_validation_missed( + &candidate_id, + "validation_success_missing_pending_entry", + ); log::warn!( "Session {} candidate_decision_ok_internal: no pending validation for {:?}", self.session_id().to_hex_string(), @@ -6544,6 +7100,10 @@ impl SessionProcessor { let candidate = match pending.raw_candidate.resolve(None) { Ok(c) => c, Err(e) => { + self.note_generated_candidate_validation_missed( + &candidate_id, + format!("validation_success_resolve_failed error={e}"), + ); log::warn!( "Session {} candidate_decision_ok: failed to resolve candidate: {}", self.session_id().to_hex_string(), @@ -6553,6 +7113,8 @@ impl SessionProcessor { } }; + self.mark_generated_candidate_validation_succeeded(&candidate_id); + // Mark as approved self.approved.insert( candidate_id, @@ -6584,25 +7146,35 @@ impl SessionProcessor { // has round gating; in roundless Simplex we gate by "still pending"). if !self.pending_validations.contains_key(&candidate_id) { self.validation_late_callback_counter.increment(1); + self.note_generated_candidate_validation_missed( + &candidate_id, + "validation_fail_late_callback_without_pending_entry", + ); self.pending_approve.remove(&candidate_id); self.validation_attempt_map.remove(&candidate_id); return; } - // If the block is already committed by the time validation fails, drop it without retries. - if let (Some(committed_seqno), Some(cand_seqno)) = ( - self.last_committed_seqno, + // If the block is already finalized by the time validation fails, drop it without retries. + if let (Some(finalized_seqno), Some(cand_seqno)) = ( + self.finalized_head_seqno, self.pending_validations .get(&candidate_id) .and_then(|p| p.raw_candidate.block.as_block().map(|b| b.id.seq_no)), ) { log::warn!( "Session {} candidate_decision_fail: slot={slot}, hash={:?}, \ - committed_seqno={committed_seqno}, cand_seqno={cand_seqno} (drop)", + finalized_seqno={finalized_seqno}, cand_seqno={cand_seqno} (drop)", self.session_id().to_hex_string(), candidate_id, ); - if cand_seqno <= committed_seqno { + if cand_seqno <= finalized_seqno { + self.note_generated_candidate_validation_missed( + &candidate_id, + format!( + "validation_failed_after_finalization finalized_seqno={finalized_seqno} cand_seqno={cand_seqno}" + ), + ); self.pending_approve.remove(&candidate_id); self.pending_validations.remove(&candidate_id); self.validation_attempt_map.remove(&candidate_id); @@ -6653,6 +7225,10 @@ impl SessionProcessor { candidate_id, reason, ); + self.note_generated_candidate_validation_missed( + &candidate_id, + format!("validation_failed_final reason={reason}"), + ); // Remove from pending self.pending_approve.remove(&candidate_id); @@ -6782,9 +7358,18 @@ impl SessionProcessor { log::trace!("Session {} broadcast_vote: {:?}", self.session_id().to_hex_string(), vote); match &vote { - Vote::Notarize(_) => self.votes_out_notarize_counter.increment(1), - Vote::Finalize(_) => self.votes_out_finalize_counter.increment(1), - Vote::Skip(_) => self.votes_out_skip_counter.increment(1), + Vote::Notarize(_) => { + self.votes_out_total_counter.increment(1); + self.votes_out_notarize_counter.increment(1); + } + Vote::Finalize(_) => { + self.votes_out_total_counter.increment(1); + self.votes_out_finalize_counter.increment(1); + } + Vote::Skip(_) => { + self.votes_out_total_counter.increment(1); + self.votes_out_skip_counter.increment(1); + } _ => {} } @@ -6903,7 +7488,7 @@ impl SessionProcessor { /* Finalization Flow - Reference: validator-session/src/session_processor.rs notify_block_committed + Reference: Simplex finalized-driven delivery path ┌─────────────────────────────────────────────────────────────────────────────────┐ │ Finalization Flow │ @@ -6912,10 +7497,10 @@ impl SessionProcessor { │ │ │ │ ▼ │ │ handle_block_finalized(): │ - │ ├── Collect finalization signatures from SimplexState vote accounting │ - │ ├── Create signature vectors for on_block_committed │ - │ ├── notify_block_committed(source_info, root_hash, file_hash, ...) │ - │ └── Reset round state via reset_slot_state() │ + │ ├── Record pending finalization if body is still missing │ + │ ├── maybe_emit_out_of_order_finalized(...) │ + │ ├── maybe_apply_finalized_state(...) │ + │ └── Reset round state once finalized state is materialized │ │ │ │ SimplexEvent::SlotSkipped(slot) │ │ │ │ @@ -6939,7 +7524,8 @@ impl SessionProcessor { /// Schedule a candidate request with delay if not already requested /// - /// Called by `try_commit_finalized_chains()` when a candidate body or NotarCert is missing. + /// Called when we need to repair missing candidate data after learning about a + /// finalized or otherwise required block before all body/notar data is present. /// Adds the (slot, hash) to `requested_candidates` and schedules a delayed action. /// After the delay, if the candidate is still not in `received_candidates`, requests /// it from peers (with want_notar=true to get NotarCert). @@ -6952,7 +7538,7 @@ impl SessionProcessor { /// # Parameters /// - `initial_delay`: Optional delay before sending the request. /// - `None`: Use default `CANDIDATE_REQUEST_DELAY` (allows broadcast to arrive first) - /// - `Some(Duration::ZERO)`: Request immediately (for commit-critical recovery paths) + /// - `Some(Duration::ZERO)`: Request immediately (for repair-critical paths) /// - `Some(dur)`: Custom delay fn request_candidate( &mut self, @@ -6964,6 +7550,17 @@ impl SessionProcessor { let key = RawCandidateId { slot, hash: block_hash.clone() }; + if self.simplex_state.has_skip_certificate_for_slot(&self.description, slot) { + log::trace!( + "Session {} request_candidate: slot={} hash={} - skipped already, not requesting", + &self.session_id().to_hex_string()[..8], + slot, + &block_hash.to_hex_string()[..8], + ); + self.requested_candidates.remove(&key); + return; + } + // Throttle repeated requests for the same (slot,hash) to survive transient partitions. let now = self.now(); if let Some(next_allowed_at) = self.requested_candidates.get(&key) { @@ -7017,6 +7614,28 @@ impl SessionProcessor { self.post_delayed_action(expiration_time, move |processor: &mut SessionProcessor| { let candidate_id = RawCandidateId { slot, hash: block_hash.clone() }; + if !processor.requested_candidates.contains_key(&candidate_id) { + log::trace!( + "Session {} delayed_request_candidate: slot={slot} hash={} \ + - cancelled before send", + &session_id.to_hex_string()[..8], + &block_hash.to_hex_string()[..8], + ); + return; + } + if processor + .simplex_state + .has_skip_certificate_for_slot(&processor.description, slot) + { + log::trace!( + "Session {} delayed_request_candidate: slot={slot} hash={} \ + - skipped before send", + &session_id.to_hex_string()[..8], + &block_hash.to_hex_string()[..8], + ); + processor.requested_candidates.remove(&candidate_id); + return; + } let have_body = processor.has_real_candidate_body(&candidate_id); let have_notar = processor.simplex_state.get_notarize_certificate(slot, &block_hash).is_some(); @@ -7048,1465 +7667,40 @@ impl SessionProcessor { } } - /* - ======================================================================== - Batch Finalization Support (C++ finalize_blocks pattern) - ======================================================================== + fn cancel_candidate_repairs_for_slot(&mut self, slot: SlotIndex) { + let before = self.requested_candidates.len(); + self.requested_candidates.retain(|candidate_id, _| candidate_id.slot != slot); + let removed_requests = before.saturating_sub(self.requested_candidates.len()); + let removed_missing_body = self.missing_body_logged.remove(&slot.value()); - When a block finalizes, we need to commit its entire parent chain. - C++ pattern: finalize_blocks() walks parent → grandparent → ... until - reaching an already-finalized block, then commits in reverse (oldest first). + self.receiver.cancel_candidate_requests_for_slot(slot.value()); - - First (triggered) block uses FinalCert signatures - - Parent blocks use NotarCert signatures - - MC optimization: skip parent walk for masterchain - */ + if removed_requests > 0 || removed_missing_body { + log::trace!( + "Session {} cancel_candidate_repairs_for_slot: slot={slot} \ + removed_requests={removed_requests} removed_missing_body={removed_missing_body}", + &self.session_id().to_hex_string()[..8] + ); + } + } - /// Collect a gapless commit chain from finalized block to committed head - /// - /// Walks from finalized block following parent_id pointers until reaching - /// the block that matches `last_committed_block_id`. - /// - /// # Algorithm - /// 1. For each block in the chain, verify body exists in `received_candidates` - /// 2. Stop when `received.block_id == last_committed_block_id` - /// 3. For non-triggered, non-empty blocks: verify NotarCert exists - /// 4. Return chain in commit order (oldest first) + /// Handle block finalized event /// - /// # Returns - /// - `Ready { chain }`: Chain is committable (all bodies + NotarCerts present) - /// - `AlreadyCommitted`: The finalized block is already the committed head - /// - `MissingCandidate { missing_id }`: Body or NotarCert missing, request from peers + /// Called when FSM determines a block has finalization certificate. + /// Records finalization and applies finalized-driven delivery/state updates. /// - /// # Seqno gap handling - /// - **Non-masterchain**: if we successfully walk from finalized block to committed head via parent pointers, - /// the chain is gapless by construction (each block's seqno = parent.seqno + 1 for non-empty). - /// - **Masterchain**: we do NOT allow committing non-empty parent blocks with NotarCert-only ("approve") - /// signatures. If the finalized masterchain block's seqno is ahead of expected, we return - /// `WaitingForFinalCert` instead of trying to fill gaps via approve-commits. + /// This function ALWAYS processes the finalization (never blocks FSM event processing). + /// If bodies are missing, the finalization is recorded in the journal and local application + /// is deferred until bodies arrive (triggered by on_candidate_received). /// - /// # Reference - /// C++ `finalize_blocks_inner()` in consensus.cpp: - /// - Walks parent chain collecting candidates - /// - Uses NotarCert for non-triggered blocks - /// - Uses FinalCert for triggered block - fn collect_gapless_commit_chain(&self, finalized_id: &RawCandidateId) -> ChainCollectionResult { - let mut chain = Vec::new(); - let mut current_id = finalized_id.clone(); - let mut is_first = true; - let mut triggered_is_empty: Option = None; - - // Track previous (child) block's seqno and empty status for invariant check - // We walk child -> parent, so we check: child.seqno = parent.seqno + 1 (non-empty) or = (empty) - let mut prev_child_seqno: Option = None; - let mut prev_child_is_empty: Option = None; - - loop { - // 1. Check if body exists - let received = match self.received_candidates.get(¤t_id) { - Some(r) => r, - None => { - log::trace!( - "Session {} collect_gapless_commit_chain: missing body for slot={} hash={}", - &self.session_id().to_hex_string()[..8], - current_id.slot, - ¤t_id.hash.to_hex_string()[..8] - ); - return ChainCollectionResult::MissingCandidate { missing_id: current_id }; - } - }; - - // 1b. Finalized-boundary stub detection. - // - // Stubs are inserted by handle_block_finalized() for parent-resolution boundaries. - // They are not committable bodies. - // - // - Triggered block is a stub: treat as missing body and request it. - // - Non-triggered ancestor is a stub: stop walking (boundary reached). - if received.candidate_hash_data_bytes.is_empty() { - if is_first { - log::debug!( - "Session {} collect_gapless_commit_chain: triggered finalized block is \ - still a boundary stub, waiting for body: slot={} hash={}", - &self.session_id().to_hex_string()[..8], - current_id.slot, - ¤t_id.hash.to_hex_string()[..8], - ); - return ChainCollectionResult::MissingCandidate { - missing_id: current_id.clone(), - }; - } - - log::trace!( - "Session {} collect_gapless_commit_chain: reached finalized boundary stub \ - at slot={}, stopping walk", - &self.session_id().to_hex_string()[..8], - current_id.slot, - ); - break; - } - - let current_seqno = received.block_id.seq_no; - - if is_first && triggered_is_empty.is_none() { - triggered_is_empty = Some(received.is_empty); - } - - // Late-finalization fast path (gremlin / out-of-order FinalCert delivery): - // - // It is possible to receive `BlockFinalizedEvent` for a block that is already behind - // the current committed head by seqno. This happens when: - // - we committed this block earlier as part of committing some finalized descendant, and - // - the FinalCert for this ancestor arrives later (or is observed later). - // - // In this case, walking parent_id pointers will never reach the committed head - // (because the committed head is a DESCENDANT), and we'll hit the session boundary - // (`parent_id == None`). C++ handles this via `finalized_blocks_[id].done` and returns; - // Rust should treat it as "already committed" and do nothing. - // - // NOTE: We only apply this check to the triggered finalized candidate (is_first), - // not to intermediate parents in the walk. - if is_first { - if let Some(committed_seqno) = self.last_committed_seqno { - if current_seqno < committed_seqno { - log::debug!( - "Session {} collect_gapless_commit_chain: finalized candidate is \ - behind committed head, treating as already committed. \ - triggered_slot={} triggered_seqno={current_seqno} \ - committed_seqno={committed_seqno}", - &self.session_id().to_hex_string()[..8], - current_id.slot, - ); + /// See Finalization Flow diagram above for full flow. + fn handle_block_finalized(&mut self, event: BlockFinalizedEvent) { + check_execution_time!(50_000); + instrument!(); - #[cfg(debug_assertions)] - { - // Debug-only safety: ensure this "old finalized" candidate is an ancestor of the - // committed head in the candidate-parent chain. If not, it's a fork / inconsistency. - if let Some(committed_slot) = self.last_committed_slot { - let head_candidate_id = self - .finalized_blocks - .iter() - .find(|id| id.slot == committed_slot) - .cloned(); - - if let Some(mut cursor) = head_candidate_id { - let mut depth: u32 = 0; - let mut found = false; - - while depth < MAX_CHAIN_DEPTH { - if cursor == *finalized_id { - found = true; - break; - } - let Some(rcv) = self.received_candidates.get(&cursor) - else { - break; - }; - let Some(parent) = &rcv.parent_id else { - break; - }; - cursor = parent.clone(); - depth += 1; - } - - assert!( - found, - "Session {} CHAIN INVARIANT VIOLATION: finalized candidate is behind committed head \ - (triggered_seqno={} < committed_seqno={}) but is NOT an ancestor of the committed head \ - in candidate-parent chain. This indicates a fork or state inconsistency.", - &self.session_id().to_hex_string()[..8], - current_seqno, - committed_seqno - ); - } else { - log::debug!( - "Session {} collect_gapless_commit_chain: debug ancestry \ - check skipped (cannot locate committed head candidate for \ - last_committed_slot={committed_slot})", - &self.session_id().to_hex_string()[..8], - ); - } - } - } - - return ChainCollectionResult::AlreadyCommitted; - } - } - } - - // 2. Check if we reached the committed head - // KEY: Compare block_id, not membership in finalized_blocks set - if let Some(ref committed_block_id) = self.last_committed_block_id { - if &received.block_id == committed_block_id { - log::trace!( - "Session {} collect_gapless_commit_chain: reached committed head at \ - slot={} seqno={current_seqno}", - &self.session_id().to_hex_string()[..8], - current_id.slot, - ); - - // INVARIANT CHECK: verify child->parent seqno relationship with committed head - if let (Some(child_seqno), Some(child_is_empty)) = - (prev_child_seqno, prev_child_is_empty) - { - let expected_child_seqno = if child_is_empty { - current_seqno // Empty: child.seqno = parent.seqno - } else { - current_seqno + 1 // Non-empty: child.seqno = parent.seqno + 1 - }; - - assert!( - child_seqno == expected_child_seqno, - "Session {} SEQNO INVARIANT VIOLATION at committed head! \ - Child seqno={}, parent (committed head) seqno={}, child_is_empty={}, expected_child_seqno={}. \ - This indicates corrupted parent chain data - refusing to commit.", - &self.session_id().to_hex_string()[..8], - child_seqno, - current_seqno, - child_is_empty, - expected_child_seqno - ); - } - - break; // Don't include committed head in chain - } - } - - // Masterchain parity (C++ consensus.cpp::finalize_blocks_inner): - // - // C++ has an early-return on MC when `maybe_final_cert` is null, which prevents - // committing notarized parents (create_simplex_approve) on masterchain. Only the - // finalized (FinalCert) target is committed on MC; parents are resolved only to - // obtain their ids. - // - // Rust equivalent: if the triggered finalized candidate is NON-empty on MC, commit - // only this single block (do not walk/commit parents). - // - // CRITICAL: We must verify seqno continuity BEFORE using this fast-path! - // If triggered_seqno > last_committed_seqno + 1, there are missing intermediate - // masterchain blocks. We must NOT commit any NotarCert-only ("approve") blocks on MC, - // so we wait for the missing FinalCert(s) instead. - if is_first && self.description.get_shard().is_masterchain() && !received.is_empty { - // Check seqno continuity from committed head - let expected_seqno = match self.last_committed_seqno { - Some(prev) => prev + 1, - None => self.description.get_initial_block_seqno(), - }; - - if current_seqno == expected_seqno { - // Gapless - safe to use MC fast-path - log::trace!( - "Session {} collect_gapless_commit_chain: MC mode - non-empty triggered, \ - single commit for slot={} seqno={current_seqno}", - &self.session_id().to_hex_string()[..8], - current_id.slot, - ); - return ChainCollectionResult::Ready { - chain: vec![BlockToCommit { - candidate_id: current_id, - is_triggered_block: true, - }], - }; - } else if current_seqno > expected_seqno { - log::debug!( - "Session {} collect_gapless_commit_chain: MC FINAL-ONLY invariant: \ - finalized block is ahead of committed head. Waiting for FinalCert of \ - expected seqno. triggered=s{}:{} triggered_seqno={current_seqno} \ - expected_seqno={expected_seqno} last_committed_seqno={:?}", - &self.session_id().to_hex_string()[..8], - current_id.slot, - ¤t_id.hash.to_hex_string()[..8], - self.last_committed_seqno, - ); - return ChainCollectionResult::WaitingForFinalCert { - expected_seqno, - finalized_id: finalized_id.clone(), - finalized_seqno: current_seqno, - }; - } else { - // current_seqno < expected_seqno: This should be caught by late-finalization - // fast path above. If we reach here, something is very wrong. - log::warn!( - "Session {} collect_gapless_commit_chain: MC unexpected seqno - \ - triggered_slot={} has seqno={current_seqno}, expected={expected_seqno}. \ - Should have been caught by late-finalization check.", - &self.session_id().to_hex_string()[..8], - current_id.slot, - ); - // Fall through to normal flow - } - } - - // 3. INVARIANT CHECK: verify child->parent seqno relationship - // prev_child (if any) should have seqno = current_seqno + 1 (non-empty) or = current_seqno (empty) - if let (Some(child_seqno), Some(child_is_empty)) = - (prev_child_seqno, prev_child_is_empty) - { - let expected_child_seqno = if child_is_empty { - current_seqno // Empty child: child.seqno = parent.seqno - } else { - current_seqno + 1 // Non-empty child: child.seqno = parent.seqno + 1 - }; - - assert!( - child_seqno == expected_child_seqno, - "Session {} SEQNO INVARIANT VIOLATION! \ - Child seqno={}, parent slot={} seqno={}, child_is_empty={}, expected_child_seqno={}. \ - This indicates corrupted parent chain data - refusing to commit.", - &self.session_id().to_hex_string()[..8], - child_seqno, - current_id.slot, - current_seqno, - child_is_empty, - expected_child_seqno - ); - } - - // 4. For non-triggered, non-empty blocks: verify NotarCert exists - // (Triggered block uses FinalCert; empty blocks don't need signatures) - // - // Even on masterchain, if we decide to commit parent non-empty blocks (catch-up path), - // we must have a NotarCert to build a valid signature set for accept_block. - if !is_first && !received.is_empty { - if self - .simplex_state - .get_notarize_certificate(current_id.slot, ¤t_id.hash) - .is_none() - { - log::debug!( - "Session {} collect_gapless_commit_chain: missing NotarCert for slot={} \ - hash={}", - &self.session_id().to_hex_string()[..8], - current_id.slot, - ¤t_id.hash.to_hex_string()[..8], - ); - // Request candidate (want_notar=true) to get NotarCert - return ChainCollectionResult::MissingCandidate { missing_id: current_id }; - } - } - - // 5. Add to chain - log::trace!( - "Session {} collect_gapless_commit_chain: adding slot={}, hash={}, \ - seqno={current_seqno}, is_empty={}, is_triggered={is_first}", - &self.session_id().to_hex_string()[..8], - current_id.slot, - ¤t_id.hash.to_hex_string()[..8], - received.is_empty, - ); - chain.push(BlockToCommit { - candidate_id: current_id.clone(), - is_triggered_block: is_first, - }); - is_first = false; - - // Remember this block's info for next iteration's invariant check - prev_child_seqno = Some(current_seqno); - prev_child_is_empty = Some(received.is_empty); - - // Masterchain parity: if the triggered finalized candidate was empty, FinalCert is - // propagated through empties to the nearest non-empty ancestor. On MC we should not - // notar-commit further parents, so we stop after adding the first non-empty ancestor. - // - // CRITICAL: Before stopping, verify seqno continuity from the committed head! - // If this block's seqno doesn't directly follow last_committed_seqno, there are - // missing intermediate masterchain blocks. We must NOT commit NotarCert-only blocks on MC, - // so we wait for missing FinalCert(s) instead. - // This MC early-stop is ONLY for the "empty-triggered → nearest non-empty ancestor" case. - // For a non-empty triggered finalized block, we must be able to catch up under partitions - // by walking parents when there is a seqno gap. - if self.description.get_shard().is_masterchain() - && !received.is_empty - && triggered_is_empty == Some(true) - { - let expected_seqno = match self.last_committed_seqno { - Some(prev) => prev + 1, - None => self.description.get_initial_block_seqno(), - }; - - if current_seqno == expected_seqno { - // Gapless - safe to stop parent walk - log::trace!( - "Session {} collect_gapless_commit_chain: MC mode - reached nearest \ - non-empty ancestor (gapless), stopping at slot={} seqno={current_seqno}", - &self.session_id().to_hex_string()[..8], - current_id.slot, - ); - break; - } else if current_seqno > expected_seqno { - log::debug!( - "Session {} collect_gapless_commit_chain: MC FINAL-ONLY invariant: \ - empty-triggered FinalCert resolves to non-empty ancestor with seqno gap. \ - Waiting for FinalCert of expected seqno. triggered=s{}:{} ancestor=s{}:{} \ - ancestor_seqno={current_seqno} expected_seqno={expected_seqno} \ - last_committed_seqno={:?}", - &self.session_id().to_hex_string()[..8], - finalized_id.slot, - &finalized_id.hash.to_hex_string()[..8], - current_id.slot, - ¤t_id.hash.to_hex_string()[..8], - self.last_committed_seqno, - ); - return ChainCollectionResult::WaitingForFinalCert { - expected_seqno, - finalized_id: finalized_id.clone(), - finalized_seqno: current_seqno, - }; - } else { - // current_seqno < expected_seqno: This block is older than committed head. - // This should have been caught by the committed head check at the start. - log::warn!( - "Session {} collect_gapless_commit_chain: MC ancestor has seqno \ - {current_seqno} < expected {expected_seqno}. This should not happen.", - &self.session_id().to_hex_string()[..8], - ); - break; - } - } - - // 6. Move to parent - match &received.parent_id { - Some(parent) => { - log::trace!( - "Session {} collect_gapless_commit_chain: moving to parent slot={}, \ - hash={}", - &self.session_id().to_hex_string()[..8], - parent.slot, - &parent.hash.to_hex_string()[..8], - ); - current_id = parent.clone(); - } - None => { - // Genesis/epoch start - verify seqno is initial - let initial_seqno = self.description.get_initial_block_seqno(); - assert!( - received.is_empty || current_seqno == initial_seqno, - "Session {} SEQNO INVARIANT VIOLATION: genesis block has seqno={}, expected initial={}. \ - This indicates corrupted parent chain data - refusing to commit.", - &self.session_id().to_hex_string()[..8], - current_seqno, - initial_seqno - ); - - assert!( - self.last_committed_block_id.is_none(), - "Session {} CHAIN INVARIANT VIOLATION: hit genesis but last_committed exists. \ - Expected to reach committed head via parent chain but reached genesis instead. \ - This indicates broken parent chain or state inconsistency - refusing to commit.", - &self.session_id().to_hex_string()[..8] - ); - - // First block in session - OK to break - log::trace!( - "Session {} collect_gapless_commit_chain: slot={} has no parent \ - (genesis/epoch start)", - &self.session_id().to_hex_string()[..8], - current_id.slot, - ); - break; - } - } - } - - // Reverse to get commit order (oldest first) - chain.reverse(); - - if chain.is_empty() { - log::trace!( - "Session {} collect_gapless_commit_chain: finalized block is already committed", - &self.session_id().to_hex_string()[..8] - ); - ChainCollectionResult::AlreadyCommitted - } else { - log::trace!( - "Session {} collect_gapless_commit_chain: collected {} blocks for commit", - &self.session_id().to_hex_string()[..8], - chain.len() - ); - ChainCollectionResult::Ready { chain } - } - } - - /// Commit a single block with seqno validation and proper signatures - /// - /// This function: - /// 1. Validates seqno == last_committed_seqno + 1 (panics on mismatch) - /// 2. Prepares signatures: - /// - FinalCert for the committed block selected by `final_sig_target` - /// (nearest non-empty ancestor when the finalized candidate is empty) - /// - NotarCert for other non-empty blocks (create_simplex_approve) - /// 3. Marks slot outcome (Commit or Skip) for emission - /// 4. Round is derived from slot at emit time - /// - /// # Arguments - /// * `block_info` - Block to commit - /// * `triggered_event` - The original BlockFinalizedEvent (for triggered block's FinalCert) - /// - /// # Reference - /// C++ finalize_blocks(): - /// - `is_first_block`: FinalCert → create_simplex - /// - else: NotarCert → create_simplex_approve - fn commit_single_block( - &mut self, - block_info: &BlockToCommit, - triggered_event: &BlockFinalizedEvent, - final_sig_target: Option<&RawCandidateId>, - final_sig_context: &(SlotIndex, Vec), - ) { - let candidate_id = block_info.candidate_id.clone(); - let slot = candidate_id.slot; - let block_hash = &candidate_id.hash; - - // Get received candidate data - let received = match self.received_candidates.get(&candidate_id) { - Some(r) => { - r.clone() // Clone to avoid borrow issues - } - None => { - // This should not happen if collect_parent_chain worked correctly - log::error!( - "Session {} commit_single_block: CRITICAL - no received candidate for \ - slot={slot} hash={}", - &self.session_id().to_hex_string()[..8], - &block_hash.to_hex_string()[..8], - ); - self.increment_error(); - return; - } - }; - - let is_empty_block = received.is_empty; - let seqno = received.block_id.seq_no; - - // Seqno validation: commits must be sequential by seqno on top of the committed head. - // This is a fundamental invariant (not a temporary limitation). - // - // NOTE: We intentionally do NOT derive expected seqno from `received.parent_id` here, - // because the parent candidate body may be missing even when the block is finalized - // by votes (network loss / out-of-order). Committed-chain tracking is the source of truth. - let expected_seqno_from_committed = match (is_empty_block, self.last_committed_seqno) { - (false, Some(prev)) => prev + 1, - (false, None) => self.description.get_initial_block_seqno(), - (true, Some(prev)) => prev, // empty block re-signs the committed head - (true, None) => { - // INVARIANT: first block cannot be empty - panic!( - "Session {} INVARIANT VIOLATION: empty committed block has no parent at slot {}", - self.session_id().to_hex_string(), - slot - ); - } - }; - - // STRICT SEQNO INVARIANT: - // collect_gapless_commit_chain() guarantees the chain is gapless from committed head. - // Any mismatch here indicates a bug in the chain collection algorithm. - assert!( - seqno == expected_seqno_from_committed, - "Session {} SEQNO INVARIANT VIOLATION in commit_single_block at slot={}. \ - Block has seqno={}, expected={}. is_empty={}. \ - This should never happen - collect_gapless_commit_chain() guarantees gapless chains.", - &self.session_id().to_hex_string()[..8], - slot, - seqno, - expected_seqno_from_committed, - is_empty_block - ); - - // Update committed seqno tracking: - // - non-empty blocks advance seqno to the actual seqno - // - empty blocks keep seqno unchanged - if !is_empty_block { - self.last_committed_seqno = Some(seqno); - } - - // Track last committed slot for diagnostics/recovery. - self.last_committed_slot = Some(slot); - - // Track the block id for the last finalized seqno. - // For empty blocks this is the re-signed parent block id (same as previous non-empty id). - self.last_committed_block_id = Some(received.block_id.clone()); - - // Extract and track before_split flag for split/merge handling - // C++ parity: C++ checks `is_before_split(prev_block_data)` in should_generate_empty_block() - // We extract it here during commit and cache it for the next collation decision. - if !is_empty_block { - // Only update for non-empty blocks (empty blocks re-use parent's BlockIdExt) - if let Ok(before_split) = crate::utils::extract_before_split_flag(received.data.data()) - { - self.last_committed_before_split = before_split; - if before_split { - log::info!( - "Session {} commit_single_block: block at slot={slot} seqno={seqno} has \ - before_split=true (next block MUST be empty for split/merge)", - &self.session_id().to_hex_string()[..8], - ); - } - } else { - // Failed to extract - log at trace level (expected in tests with dummy block data) - log::trace!( - "Session {} commit_single_block: failed to extract before_split flag for \ - slot={slot}, assuming false", - &self.session_id().to_hex_string()[..8], - ); - self.last_committed_before_split = false; - } - } - - // ===== Common state updates (for both empty and non-empty) ===== - - // Track as finalized - self.finalized_blocks.insert(candidate_id.clone()); - - log::trace!( - "Session {} commit_single_block: slot={}, seqno={}, is_triggered={}, is_empty={}", - &self.session_id().to_hex_string()[..8], - slot, - seqno, - block_info.is_triggered_block, - is_empty_block - ); - - // ===== Branch: empty vs non-empty block handling ===== - - // Persisted flag for `FinalizedBlockRecord::is_final` (C++ parity): - // - Non-empty: true iff this commit uses FinalCert signatures. - // - Empty: true iff FinalCert is active for this empty candidate (propagation case). - let record_is_final: bool; - - if is_empty_block { - // Empty blocks inherit parent's BlockIdExt and should NOT trigger on_block_committed. - // C++ only commits non-empty blocks. No ValidatorGroup callback needed. - - // C++ parity: empty finalized records store `is_final = maybe_final_cert.not_null()`. - // We approximate this using `final_sig_target`: - // - If there is a non-empty FinalCert target in this batch, FinalCert is active for - // empties at/after that target slot. - // - If this batch contains no non-empty blocks (all empties until an already-finalized - // ancestor), FinalCert is still considered active for these empties. - record_is_final = match final_sig_target { - Some(target) => slot >= target.slot, - None => true, - }; - - // DEBUG: Short pattern for quick grep (EMPTY = empty block processed) - log::debug!( - "Session {} EMPTY BLOCK: slot={}, seqno={} (no ValidatorGroup callback)", - &self.session_id().to_hex_string()[..8], - slot, - seqno, - ); - // TRACE: Method name pattern for detailed tracking - log::trace!( - "Session {} commit_single_block: empty block slot={slot}, seqno={seqno}, hash={} \ - - no on_block_committed", - self.session_id().to_hex_string(), - block_hash.to_hex_string(), - ); - } else { - // Non-empty block: prepare signatures and call notify_block_committed directly - - // Determine whether this committed non-empty block should carry FinalCert signatures. - // - // C++ parity: - // - If the finalized (triggered) candidate is non-empty, FinalCert applies to that candidate. - // - If the finalized candidate is empty, FinalCert is propagated through empties and applies - // to the nearest non-empty ancestor that is being finalized in this batch. - // - Other non-empty blocks in the chain use NotarCert (create_simplex_approve). - let use_final_cert = final_sig_target.is_some_and(|id| id == &candidate_id); - record_is_final = use_final_cert; - - // MASTERCHAIN INVARIANT (C++ parity): - // Masterchain blocks MUST be accepted only with final signatures (FinalCert). - // C++ AcceptBlockQuery rejects non-final signature sets on masterchain. - assert!( - !self.description.get_shard().is_masterchain() || use_final_cert, - "Session {} INVARIANT VIOLATION: masterchain non-empty commit without FinalCert (approve-only) is forbidden. \ - slot={} seqno={} hash={} triggered={} final_sig_target={:?}", - &self.session_id().to_hex_string()[..8], - slot, - seqno, - &block_hash.to_hex_string()[..8], - block_info.is_triggered_block, - final_sig_target.map(|id| (id.slot, id.hash.to_hex_string())) - ); - - // Prepare signature sets. - // - FinalCert: primary signatures from FinalCert, approve_signatures from NotarCert (if any) - // - NotarCert: both sets from NotarCert (same as create_simplex_approve) - let (signatures, approve_signatures) = if use_final_cert { - self.prepare_triggered_block_signatures(triggered_event, slot, block_hash) - } else { - self.prepare_parent_block_signatures(slot, block_hash) - }; - - // Signature verification context: - // - For FinalCert commits: bind to the finalized candidate's (slot, hash_data) - // - For NotarCert commits: bind to this candidate's (slot, hash_data) - let (sig_slot, sig_candidate_hash_data_bytes) = if use_final_cert { - (final_sig_context.0, final_sig_context.1.clone()) - } else { - (slot, received.candidate_hash_data_bytes.clone()) - }; - - // Create source info with SIMPLEX_ROUNDLESS - let source_public_key = - self.description.get_source_public_key(received.source_idx).clone(); - let source_info = crate::BlockSourceInfo { - source: source_public_key, - priority: BlockCandidatePriority { - round: SIMPLEX_ROUNDLESS, - first_block_round: SIMPLEX_ROUNDLESS, - priority: 0, - }, - }; - - // DEBUG: Short pattern for quick grep (COMMIT = block committed) - log::debug!( - "Session {} COMMIT: slot={}, seqno={}, hash={}, from=v{:03}, sigs={}, is_final={}", - &self.session_id().to_hex_string()[..8], - slot, - seqno, - &received.root_hash.to_hex_string()[..8], - received.source_idx, - signatures.len(), - use_final_cert, - ); - // TRACE: Method name pattern for detailed tracking - log::trace!( - "Session {} commit_single_block: COMMIT at slot={slot}, seqno={seqno}, \ - root_hash={}, file_hash={}, source={}, triggered={}, is_final={use_final_cert}", - self.session_id().to_hex_string(), - received.root_hash.to_hex_string(), - received.file_hash.to_hex_string(), - received.source_idx, - block_info.is_triggered_block, - ); - - let stats = self.build_session_stats(); - self.notify_block_committed( - source_info, - received.root_hash.clone(), - received.file_hash.clone(), - received.data.clone(), - signatures, - approve_signatures, - sig_slot, - sig_candidate_hash_data_bytes, - use_final_cert, - stats, - ); - - // Increment commits counter - self.commits_counter.success(); - - // C++ parity: block-producer.cpp advances last_consensus_finalized_seqno_ - // only on FinalizeBlock when is_final() is true. This is the Rust equivalent. - if use_final_cert { - let prev = self.last_consensus_finalized_seqno.unwrap_or(0); - if seqno > prev { - self.last_consensus_finalized_seqno = Some(seqno); - log::debug!( - "Session {} commit_single_block: advanced last_consensus_finalized_seqno \ - {} -> {} (slot={}, is_final=true)", - &self.session_id().to_hex_string()[..8], - prev, - seqno, - slot - ); - } - } - } - - // ===== Common finalization (for both empty and non-empty) ===== - self.last_finalized_slot_gauge.set(slot.0 as f64); - - // Reset stalled round debug timer on each slot processed - let now = self.now(); - self.round_debug_at = now + ROUND_DEBUG_PERIOD; - self.last_commit_time = now; - - // ===== Persist finalized block to database ===== - // Reference: C++ consensus.cpp finalize_blocks() - // - Masterchain: co_await db->set(...) — blocking write - // - Non-masterchain: db->set(...).start().detach() — fire-and-forget - // - // C++ parity: for EMPTY candidates, C++ persists finalizedBlock records ONLY on non-masterchain - // (`else if (!owning_bus()->shard.is_masterchain()) { db->set(...).start().detach(); }`). - // On masterchain, empty finalized records are not persisted. - // - // IMPORTANT: On masterchain, since empty candidates are not persisted, persisted - // `finalizedBlock` records must form a contiguous parent chain across *persisted* blocks. - // If a non-empty block's consensus parent is an empty candidate, store the nearest - // non-empty ancestor as `parent` (skipping empty slots) to keep the DB chain consistent - // across restarts (matches C++ load_from_db chain filtering intent). - let record_parent = if self.description.get_shard().is_masterchain() && !is_empty_block { - let mut parent = received.parent_id.clone(); - let mut hops = 0usize; - while let Some(pid) = parent.clone() { - // Safety: avoid pathological loops if state is corrupted. - if hops > MAX_DB_PARENT_WALK_HOPS { - log::error!( - "Session {} commit_block: exceeded parent walk limit while computing DB \ - parent for slot={}", - &self.session_id().to_hex_string()[..8], - slot.value(), - ); - self.increment_error(); - break; - } - hops += 1; - - match self.received_candidates.get(&pid) { - Some(p) if p.is_empty => { - parent = p.parent_id.clone(); - } - _ => break, - } - } - parent - } else { - received.parent_id.clone() - }; - - let record = FinalizedBlockRecord { - candidate_id, - block_id: received.block_id.clone(), - parent: record_parent, - is_final: record_is_final, - }; - if is_empty_block && self.description.get_shard().is_masterchain() { - // MC + empty: do not persist (C++ parity) - log::trace!( - "Session {} commit_block: skipping finalized block DB write for empty MC slot={} \ - (C++ parity)", - &self.session_id().to_hex_string()[..8], - slot.value(), - ); - } else if self.description.get_shard().is_masterchain() { - // Masterchain: blocking write (C++ co_await pattern) - log::trace!( - "Session {} commit_block: saving finalized block (MC, blocking) slot={}, \ - is_final={}", - &self.session_id().to_hex_string()[..8], - slot.value(), - record.is_final, - ); - match self.db.save_finalized_block_async(&record) { - Ok(result) => { - if let Err(e) = result.wait() { - log::error!( - "Session {} commit_block: failed to store finalized block for \ - slot={}: {e}", - &self.session_id().to_hex_string()[..8], - slot.value(), - ); - self.increment_error(); - } - } - Err(e) => { - log::error!( - "Session {} commit_block: failed to create finalized block save for \ - slot={}: {e}", - &self.session_id().to_hex_string()[..8], - slot.value(), - ); - self.increment_error(); - } - } - } else { - // Non-masterchain: fire-and-forget (C++ .start().detach() pattern) - log::trace!( - "Session {} commit_block: saving finalized block (non-MC, fire-and-forget) \ - slot={}, is_final={}", - &self.session_id().to_hex_string()[..8], - slot.value(), - record.is_final, - ); - if let Err(e) = self.db.save_finalized_block(&record) { - log::error!( - "Session {} commit_block: failed to store finalized block (non-MC) for \ - slot={}: {e}", - &self.session_id().to_hex_string()[..8], - slot.value(), - ); - self.increment_error(); - } - } - } - - /// Prepare signatures for triggered (first) block using FinalCert - /// - /// Reference: C++ create_simplex() for first block - fn prepare_triggered_block_signatures( - &self, - event: &BlockFinalizedEvent, - _slot: SlotIndex, - _block_hash: &UInt256, - ) -> ( - Vec<(crate::PublicKeyHash, crate::BlockPayloadPtr)>, - Vec<(crate::PublicKeyHash, crate::BlockPayloadPtr)>, - ) { - let certificate = &event.certificate; - - // Finalize signatures from FinalCert - let signatures: Vec<(crate::PublicKeyHash, crate::BlockPayloadPtr)> = certificate - .signatures - .iter() - .map(|vote_sig| { - let public_key_hash = - self.description.get_source_public_key_hash(vote_sig.validator_idx); - let signature = consensus_common::ConsensusCommonFactory::create_block_payload( - vote_sig.signature.clone().into(), - ); - (public_key_hash.clone(), signature) - }) - .collect(); - - // Approve signatures from NotarCert (if available) - let approve_signatures = self.get_notarize_signatures(event.slot, &event.block_hash); - - (signatures, approve_signatures) - } - - /// Prepare signatures for parent block using NotarCert - /// - /// Reference: C++ create_simplex_approve() for parent blocks - fn prepare_parent_block_signatures( - &self, - slot: SlotIndex, - block_hash: &UInt256, - ) -> ( - Vec<(crate::PublicKeyHash, crate::BlockPayloadPtr)>, - Vec<(crate::PublicKeyHash, crate::BlockPayloadPtr)>, - ) { - // MASTERCHAIN INVARIANT: - // Parent-block "approve" signatures (NotarCert-only) must NEVER be used for masterchain commits. - assert!( - !self.description.get_shard().is_masterchain(), - "Session {} INVARIANT VIOLATION: attempted to prepare NotarCert-only signatures \ - for masterchain slot={} hash={}. This corresponds to C++ create_simplex_approve(), which is forbidden on MC.", - &self.session_id().to_hex_string()[..8], - slot, - &block_hash.to_hex_string()[..8] - ); - // For parent blocks, we use NotarCert for both signature sets - // (no finalization certificate available, only notarization) - let approve_signatures = self.get_notarize_signatures(slot, block_hash); - - // Primary signatures are also from NotarCert for parent blocks - // Reference: C++ create_simplex_approve uses notarization signatures - (approve_signatures.clone(), approve_signatures) - } - - /// Get notarization signatures for a block - fn get_notarize_signatures( - &self, - slot: SlotIndex, - block_hash: &UInt256, - ) -> Vec<(crate::PublicKeyHash, crate::BlockPayloadPtr)> { - if let Some(notar_cert) = self.simplex_state.get_notarize_certificate(slot, block_hash) { - notar_cert - .signatures - .iter() - .map(|vote_sig| { - let public_key_hash = - self.description.get_source_public_key_hash(vote_sig.validator_idx); - let signature = consensus_common::ConsensusCommonFactory::create_block_payload( - vote_sig.signature.clone().into(), - ); - (public_key_hash.clone(), signature) - }) - .collect() - } else { - log::warn!( - "Session {} get_notarize_signatures: no NotarCert for slot={slot}, hash={} - \ - using empty signatures", - &self.session_id().to_hex_string()[..8], - &block_hash.to_hex_string()[..8], - ); - Vec::new() - } - } - - #[cfg(debug_assertions)] - /// Debug-only precheck: verify that committing this chain would satisfy the strict seqno invariant - /// - /// This produces clearer diagnostics BEFORE the invariant fires in commit_single_block(). - /// Only enabled in debug builds (release builds skip this overhead). - fn debug_precheck_gapless_chain(&self, chain: &[BlockToCommit]) { - let mut expected_seqno = match self.last_committed_seqno { - Some(prev) => prev + 1, - None => self.description.get_initial_block_seqno(), - }; - - for (idx, block_info) in chain.iter().enumerate() { - let Some(received) = self.received_candidates.get(&block_info.candidate_id) else { - log::error!( - "Session {} debug_precheck_gapless_chain: body must exist for candidate_id \ - slot={}", - &self.description.get_session_id().to_hex_string()[..8], - block_info.candidate_id.slot, - ); - return; - }; - - let actual_seqno = received.block_id.seq_no; - let is_empty = received.is_empty; - - if is_empty { - // Empty blocks keep same seqno - let expected_for_empty = expected_seqno.saturating_sub(1); - assert_eq!( - actual_seqno, expected_for_empty, - "debug_precheck: empty block at chain[{}] (slot={}) has seqno={}, expected={}", - idx, block_info.candidate_id.slot, actual_seqno, expected_for_empty - ); - } else { - // Non-empty blocks must match and advance - assert_eq!( - actual_seqno, expected_seqno, - "debug_precheck: non-empty block at chain[{}] (slot={}) has seqno={}, expected={}", - idx, block_info.candidate_id.slot, actual_seqno, expected_seqno - ); - expected_seqno = actual_seqno + 1; - } - } - - log::trace!( - "Session {} debug_precheck_gapless_chain: verified {} blocks are gapless (seqno \ - range: {} -> {})", - &self.session_id().to_hex_string()[..8], - chain.len(), - self.last_committed_seqno - .map(|s| s + 1) - .unwrap_or(self.description.get_initial_block_seqno()), - expected_seqno - 1, - ); - } - - /// Attempt to commit all finalized blocks that are now ready - /// - /// Called from two triggers: - /// - `handle_block_finalized()` after recording finalization - /// - `on_candidate_received()` after body arrival / resolution cache update - /// - /// For each finalized-but-uncommitted block, check if it's commit-ready: - /// - If ready: commit the chain and remove from journal - /// - If already committed: remove from journal - /// - If missing bodies/NotarCerts: request them and keep in journal - /// - /// This function is idempotent and safe to call multiple times. - fn try_commit_finalized_chains(&mut self) { - // Collect keys to process, sorted by (seqno, slot) for deterministic - // oldest-first commit ordering (avoid arbitrary HashMap iteration order). - let mut finalized_keys: Vec = - self.finalized_journal_pending_commit.keys().cloned().collect(); - - if finalized_keys.is_empty() { - return; - } - - finalized_keys.sort_unstable_by_key(|id| { - let seqno = - self.received_candidates.get(id).map(|r| r.block_id.seq_no).unwrap_or(u32::MAX); - (seqno, id.slot.0) - }); - - log::trace!( - "Session {} try_commit_finalized_chains: checking {} finalized blocks", - &self.session_id().to_hex_string()[..8], - finalized_keys.len() - ); - - let mut committed_keys = Vec::new(); - - for finalized_id in finalized_keys { - // Get the finalized entry (clone to avoid borrow conflicts) - let entry = match self.finalized_journal_pending_commit.get(&finalized_id) { - Some(e) => e.clone(), - None => continue, // Already removed (committed in this iteration) - }; - - // Collect gapless commit chain (new unified function) - match self.collect_gapless_commit_chain(&finalized_id) { - ChainCollectionResult::Ready { chain } => { - log::debug!( - "Session {} try_commit_finalized_chains: committing {} blocks \ - (triggered=s{}:{})", - &self.session_id().to_hex_string()[..8], - chain.len(), - finalized_id.slot, - &finalized_id.hash.to_hex_string()[..8], - ); - - // Optional: debug-only gapless precheck - #[cfg(debug_assertions)] - self.debug_precheck_gapless_chain(&chain); - - // Commit the chain - self.commit_finalized_chain(&entry.event, chain); - - // Mark for removal from journal - committed_keys.push(finalized_id); - } - - ChainCollectionResult::AlreadyCommitted => { - log::debug!( - "Session {} try_commit_finalized_chains: s{}:{} already committed", - &self.session_id().to_hex_string()[..8], - finalized_id.slot, - &finalized_id.hash.to_hex_string()[..8] - ); - // Remove from journal - committed_keys.push(finalized_id); - } - - ChainCollectionResult::MissingCandidate { missing_id } => { - if self.missing_body_logged.insert(missing_id.slot.0) { - log::debug!( - "Session {} try_commit_finalized_chains: s{}:{} waiting for s{}:{} \ - (body or NotarCert)", - &self.session_id().to_hex_string()[..8], - finalized_id.slot, - &finalized_id.hash.to_hex_string()[..8], - missing_id.slot, - &missing_id.hash.to_hex_string()[..8], - ); - } - - // Request the missing candidate (includes body + NotarCert with want_notar=true) - self.request_candidate(missing_id.slot, missing_id.hash, None); - - // Keep in journal - will retry when candidate arrives - } - - ChainCollectionResult::WaitingForFinalCert { - expected_seqno, - finalized_seqno, - .. - } => { - log::debug!( - "Session {} try_commit_finalized_chains: MC waiting for FinalCert of \ - expected seqno={expected_seqno} (triggered=s{}:{} seqno={finalized_seqno} \ - last_committed_seqno={:?})", - &self.session_id().to_hex_string()[..8], - finalized_id.slot, - &finalized_id.hash.to_hex_string()[..8], - self.last_committed_seqno, - ); - // Attempt to recover the missing FinalCert via get_committed_candidate. - // - // We must request FinalCert signatures for the *next committable* masterchain block - // (seqno == expected_seqno). We locate it by walking the finalized block's parent - // chain until we find a non-empty candidate with that seqno. - // - // NOTE: If we don't have bodies for some ancestors, we request them first (v1/v2), - // and will retry on the next on_candidate_received() / retry tick. - let mut cursor = finalized_id.clone(); - let mut depth: u32 = 0; - - loop { - if depth >= MAX_CHAIN_DEPTH { - log::warn!( - "Session {} try_commit_finalized_chains: MC WaitingForFinalCert - \ - exceeded MAX_CHAIN_DEPTH while walking parents (triggered=s{}:{})", - &self.session_id().to_hex_string()[..8], - finalized_id.slot, - &finalized_id.hash.to_hex_string()[..8], - ); - break; - } - - let Some(rcv) = self.received_candidates.get(&cursor) else { - // Need body to know seqno; request it. - log::debug!( - "Session {} try_commit_finalized_chains: MC WaitingForFinalCert - \ - missing body for ancestor s{}:{}; requesting candidate", - &self.session_id().to_hex_string()[..8], - cursor.slot, - &cursor.hash.to_hex_string()[..8], - ); - // Commit-critical recovery: request immediately (skip initial 1s delay). - self.request_candidate( - cursor.slot, - cursor.hash.clone(), - Some(Duration::ZERO), - ); - break; - }; - - if !rcv.is_empty && rcv.block_id.seq_no == expected_seqno { - let have_final = self - .simplex_state - .get_finalize_certificate(cursor.slot, &cursor.hash) - .is_some(); - - if have_final { - log::trace!( - "Session {} try_commit_finalized_chains: MC \ - WaitingForFinalCert - already have FinalCert for expected \ - seqno={expected_seqno} at s{}:{}", - &self.session_id().to_hex_string()[..8], - cursor.slot, - &cursor.hash.to_hex_string()[..8], - ); - // MC gap recovery: - // If we already obtained the missing FinalCert for the next committable - // masterchain block (seqno == expected_seqno), commit it immediately. - // - // This preserves the C++ "final-only on masterchain" invariant while - // allowing Rust to catch up under partitions by committing the missing - // FinalCert block(s) before the triggered finalized block. - if let Some(final_cert) = self - .simplex_state - .get_finalize_certificate(cursor.slot, &cursor.hash) - { - let commit_target = cursor.clone(); - match self.collect_gapless_commit_chain(&commit_target) { - ChainCollectionResult::Ready { chain } => { - log::debug!( - "Session {} try_commit_finalized_chains: MC gap \ - recovery - committing expected \ - seqno={expected_seqno} at s{}:{} (chain_len={})", - &self.session_id().to_hex_string()[..8], - commit_target.slot, - &commit_target.hash.to_hex_string()[..8], - chain.len(), - ); - let synthetic_event = BlockFinalizedEvent { - slot: commit_target.slot, - block_hash: commit_target.hash.clone(), - block_id: None, - certificate: final_cert.clone(), - }; - self.commit_finalized_chain(&synthetic_event, chain); - } - ChainCollectionResult::AlreadyCommitted => { - // Nothing to do. - } - ChainCollectionResult::MissingCandidate { missing_id } => { - // Should be rare (we just had body), but handle defensively. - self.request_candidate( - missing_id.slot, - missing_id.hash, - None, - ); - } - ChainCollectionResult::WaitingForFinalCert { - expected_seqno: inner_expected_seqno, - finalized_id: inner_finalized_id, - finalized_seqno: inner_finalized_seqno, - } => { - log::error!( - "Session {} try_commit_finalized_chains: MC gap \ - recovery invariant violated - \ - collect_gapless_commit_chain returned \ - WaitingForFinalCert for commit_target s{}:{} \ - (seqno={inner_finalized_seqno}) \ - outer_expected_seqno={expected_seqno} \ - inner_expected_seqno={inner_expected_seqno} \ - last_committed_seqno={:?}", - &self.session_id().to_hex_string()[..8], - inner_finalized_id.slot, - &inner_finalized_id.hash.to_hex_string()[..8], - self.last_committed_seqno, - ); - self.increment_error(); - debug_assert!( - false, - "MC gap recovery invariant violated: commit_target s{}:{} (seqno={}) \ - still reported as ahead of committed head (inner_expected_seqno={}, last_committed_seqno={:?})", - inner_finalized_id.slot, - &inner_finalized_id.hash.to_hex_string()[..8], - inner_finalized_seqno, - inner_expected_seqno, - self.last_committed_seqno, - ); - } - } - } - } else { - // Request committed block proof from full node - // (C++-compatible mechanism via full node block proof) - let now = self.now(); - let block_id = rcv.block_id.clone(); - - if let Some(&next_allowed) = - self.pending_committed_proof_requests.get(&block_id) - { - if now < next_allowed { - break; - } - } - - log::debug!( - "Session {} WaitingForFinalCert: requesting committed \ - proof for seqno={} block_id={} at s{}:{}", - &self.session_id().to_hex_string()[..8], - expected_seqno, - block_id, - cursor.slot, - &cursor.hash.to_hex_string()[..8], - ); - - self.pending_committed_proof_requests - .insert(block_id.clone(), now + COMMITTED_PROOF_RETRY_INTERVAL); - - self.notify_get_committed_candidate(block_id); - } - break; - } - - let Some(parent) = &rcv.parent_id else { - // Can't walk further (session boundary). Keep waiting. - break; - }; - - cursor = parent.clone(); - depth += 1; - } - } - } - } - - // Remove committed entries from journal - let did_commit = !committed_keys.is_empty(); - for key in committed_keys { - self.finalized_journal_pending_commit.remove(&key); - } - - // If something was committed, newly-unblocked chains may now be ready. - // Reschedule check_all so the session loop re-enters this function. - if did_commit { - self.set_next_awake_time(self.now()); - } - - self.finalized_uncommitted_gauge.set(self.finalized_journal_pending_commit.len() as f64); - } - - /// Commit a finalized chain that has been verified as commit-ready - /// - /// This is the ONLY entry point to `commit_single_block()` for finalization-triggered commits. - /// The chain MUST have been verified by `check_commit_readiness()` to be gapless and fully-bodied. - /// - /// # Arguments - /// * `triggered_event` - The original BlockFinalizedEvent (for FinalCert signatures) - /// * `chain` - The parent chain to commit (oldest first), from `check_commit_readiness` - fn commit_finalized_chain( - &mut self, - triggered_event: &BlockFinalizedEvent, - chain: Vec, - ) { - let slot = triggered_event.slot; - let block_hash = &triggered_event.block_hash; - let batch_size = chain.len(); - - // Derive FinalCert signature context and target commit (existing logic) - let triggered_id = RawCandidateId { slot, hash: block_hash.clone() }; - let Some(triggered_received) = self.received_candidates.get(&triggered_id) else { - log::error!( - "Session {} commit_finalized_chain: triggered candidate must exist slot={}", - &self.description.get_session_id().to_hex_string()[..8], - slot - ); - return; - }; - - let final_sig_context: (SlotIndex, Vec) = - (triggered_id.slot, triggered_received.candidate_hash_data_bytes.clone()); - - // Pick the nearest non-empty candidate in this batch for FinalCert application - let final_sig_target: Option = chain.iter().rev().find_map(|b| { - self.received_candidates - .get(&b.candidate_id) - .and_then(|rcv| (!rcv.is_empty).then_some(b.candidate_id.clone())) - }); - - // Log batch commit start - let cert_weight = triggered_event.certificate.total_weight(&self.description); - let total_weight = self.description.get_total_weight(); - log::debug!( - "Session {} FINALIZED: slot={}, hash={}, batch={} blocks, weight={}/{} ({:.0}%)", - &self.session_id().to_hex_string()[..8], - slot, - &block_hash.to_hex_string()[..8], - batch_size, - cert_weight, - total_weight, - 100.0 * cert_weight as f64 / total_weight as f64 - ); - - // INVARIANT CHECK - Certificate must have sufficient weight (existing) - let threshold = threshold_66(total_weight); - debug_assert!( - cert_weight >= threshold, - "finalization certificate weight {} below threshold {}", - cert_weight, - threshold - ); - - // Record slot duration (for triggered block's slot) - if let Ok(duration) = self.now().duration_since(self.slot_started_at(slot)) { - self.slot_duration_histogram.record(duration.as_millis() as f64); - } - - // Track first finalized candidate (stage 3 latency) - if !self.slot_first_candidate_finalized(slot) { - self.slot_set_first_candidate_finalized(slot, true); - if let Ok(latency) = self.now().duration_since(self.slot_started_at(slot)) { - self.first_candidate_finalized_latency_histogram.record(latency.as_millis() as f64); - log::trace!( - "Session {}: first block finalized in {:.3}ms", - &self.session_id().to_hex_string()[..8], - latency.as_secs_f64() * 1000.0 - ); - } - } - - // Commit each block in the parent chain (oldest first) - // commit_single_block handles: signatures, mark_slot_outcome (round derived at emit time) - for block_info in &chain { - self.commit_single_block( - block_info, - triggered_event, - final_sig_target.as_ref(), - &final_sig_context, - ); - } - - // Record batch metrics - self.batch_commit_counter.increment(1); - self.batch_commit_size_histogram.record(batch_size as f64); - - // Reset per-slot state for triggered slot - // (parent slots were already finalized in previous events or are being cleaned up) - self.reset_slot_state(slot); - - log::trace!( - "Session {} commit_finalized_chain: completed batch commit of {batch_size} blocks, \ - triggered_slot={slot}", - &self.session_id().to_hex_string()[..8], - ); - } - - /// Handle block finalized event - /// - /// Called when FSM determines a block has finalization certificate. - /// Records finalization and attempts commit via unified scheduler. - /// - /// This function ALWAYS processes the finalization (never blocks FSM event processing). - /// If bodies are missing, the finalization is recorded in the journal and commitment - /// is deferred until bodies arrive (triggered by on_candidate_received). - /// - /// See Finalization Flow diagram above for full flow. - fn handle_block_finalized(&mut self, event: BlockFinalizedEvent) { - check_execution_time!(50_000); - instrument!(); - - let slot = event.slot; - let block_hash = &event.block_hash; - let finalized_id = RawCandidateId { slot, hash: block_hash.clone() }; + let slot = event.slot; + let block_hash = &event.block_hash; + let finalized_id = RawCandidateId { slot, hash: block_hash.clone() }; // INVARIANT CHECK - Certificate must have sufficient weight (>=2/3+1) // Reference: C++ pool.cpp - certificate is only created when threshold is reached @@ -8535,56 +7729,16 @@ impl SessionProcessor { self.increment_error(); } - // ALWAYS record finalization in journal (even if body missing or not yet commit-ready) - let entry = FinalizedEntry { event: event.clone(), finalized_at: self.now() }; - self.finalized_journal_pending_commit.insert(finalized_id.clone(), entry); - - // NOTE: last_consensus_finalized_seqno is NOT advanced here. - // C++ parity: block-producer.cpp advances last_consensus_finalized_seqno_ only on - // FinalizeBlock(is_final=true), which happens AFTER the state-resolver commits. - // In Rust, the equivalent is commit_single_block() with use_final_cert=true. - - // Seed a finalized-boundary entry into received_candidates for parent resolution. - // C++ parity: StateResolver::resolve_state_inner() treats finalized blocks as boundaries - // and stops recursing into their ancestors. Rust needs the same behavior for live sessions, - // not only for restart recovery. - if let Some(ref block_id) = event.block_id { - if !self.received_candidates.contains_key(&finalized_id) { - self.received_candidates.insert( - finalized_id.clone(), - ReceivedCandidate { - slot, - source_idx: self.description.get_self_idx(), - candidate_id_hash: block_hash.clone(), - candidate_hash_data_bytes: Vec::new(), - block_id: block_id.clone(), - root_hash: block_id.root_hash.clone(), - file_hash: block_id.file_hash.clone(), - data: consensus_common::ConsensusCommonFactory::create_block_payload( - Vec::new(), - ), - collated_data: - consensus_common::ConsensusCommonFactory::create_block_payload( - Vec::new(), - ), - receive_time: self.now(), - is_empty: false, - parent_id: None, - is_fully_resolved: true, - }, - ); - log::debug!( - "Session {} handle_block_finalized: seeded finalized boundary for slot={} \ - seqno={} (for parent resolution)", - &self.session_id().to_hex_string()[..8], - slot, - block_id.seq_no() - ); - - // Resolve any pending parent resolutions that were waiting for this candidate - self.update_resolution_cache_chain(&finalized_id); - self.try_resolve_waiting_candidates(&finalized_id); - } + // Keep only pending finalizations that still wait for candidate body. + let should_store_for_later = self + .received_candidates + .get(&finalized_id) + .map(|r| r.candidate_hash_data_bytes.is_empty() || r.is_empty) + .unwrap_or(true); + if should_store_for_later { + let entry = FinalizedEntry { event: event.clone(), finalized_at: self.now() }; + self.finalized_pending_body.insert(finalized_id.clone(), entry); + self.finalized_pending_body_gauge.set(self.finalized_pending_body.len() as f64); } log::debug!( @@ -8600,9 +7754,9 @@ impl SessionProcessor { // Note: Certificate caching for standstill is handled in handle_finalization_reached() // which is triggered by SimplexEvent::FinalizationReached (emitted after BlockFinalized). - // Attempt commit via unified scheduler - // (may commit immediately if ready, or defer if bodies missing) - self.try_commit_finalized_chains(); + // Finalized delivery (with delayed body support + dedup). + self.maybe_emit_out_of_order_finalized(&finalized_id, &event); + self.maybe_apply_finalized_state(&finalized_id); // Continue FSM event processing (do NOT push event back to queue) } @@ -8660,6 +7814,19 @@ impl SessionProcessor { let first_non_progressed_slot = self.simplex_state.get_first_non_progressed_slot(); let first_non_finalized_slot = self.simplex_state.get_first_non_finalized_slot(); + let stale_generated_candidates: Vec = self + .generated_candidates_waiting_validation + .keys() + .filter(|candidate_id| candidate_id.slot < up_to_slot) + .cloned() + .collect(); + for candidate_id in stale_generated_candidates { + self.note_generated_candidate_validation_missed( + &candidate_id, + format!("cleanup_old_candidates up_to_slot={up_to_slot}"), + ); + } + // Clean up validation state collections (session-level, keyed by RawCandidateId) self.pending_validations.retain(|id, _| id.slot >= up_to_slot); self.pending_approve.retain(|id| id.slot >= up_to_slot); @@ -8676,42 +7843,17 @@ impl SessionProcessor { // Clean up candidate_data_cache in sync with received_candidates self.candidate_data_cache.retain(|id, _| id.slot >= up_to_slot); + self.seen_broadcast_candidates.retain(|slot, _| *slot >= up_to_slot); // Remove stale finalized-journal entries for old slots. - { - let now = self.now(); - let session_id_hex = self.session_id().to_hex_string(); - let mut stale_count = 0u32; - self.finalized_journal_pending_commit.retain(|id, entry| { - if id.slot < up_to_slot { - let age_secs = now - .duration_since(entry.finalized_at) - .map(|d| d.as_secs_f64()) - .unwrap_or(0.0); - log::warn!( - "Session {} cleanup: removing stale finalized-journal entry slot={} \ - (finalized {:.1}s ago, never committed)", - &session_id_hex[..8], - id.slot, - age_secs, - ); - stale_count += 1; - false - } else { - true - } - }); - if stale_count > 0 { - self.session_errors_count - .fetch_add(stale_count, std::sync::atomic::Ordering::Relaxed); - self.errors_counter.increment(stale_count as u64); - self.finalized_uncommitted_gauge - .set(self.finalized_journal_pending_commit.len() as f64); - } - } + // Prune entries for slots older than the FSM cursor. + // This map is a transient "finalized but body not yet received" buffer. + self.finalized_pending_body.retain(|id, _| id.slot >= up_to_slot); + self.finalized_pending_body_gauge.set(self.finalized_pending_body.len() as f64); // Prune log-throttle set to prevent unbounded growth over long sessions self.missing_body_logged.retain(|&slot| slot >= up_to_slot.value()); + self.finalized_delivery_sent.retain(|id| id.slot >= up_to_slot); // Remove pending candidate requests for slots < up_to_slot self.requested_candidates.retain(|id, _| id.slot >= up_to_slot); @@ -8918,6 +8060,25 @@ impl SessionProcessor { self.increment_error(); } } + + // When require_notarized_parent_for_collation is active, precollation for the + // next slot may have been deferred because this parent wasn't yet notarized. + // Now that the notarization arrived, retry precollation. + if self.description.opts().require_notarized_parent_for_collation { + if let Some(ref head) = self.local_chain_head { + if head.slot == event.slot { + let next_slot = SlotIndex(head.slot.0 + 1); + log::debug!( + "Session {} handle_notarization_reached: parent s{} now notarized, \ + retrying precollate_block for slot {}", + &self.session_id().to_hex_string()[..8], + event.slot, + next_slot, + ); + self.precollate_block(next_slot); + } + } + } } /// Handle skip certificate reached event @@ -9065,7 +8226,7 @@ impl SessionProcessor { // Update standstill tracked slots range let (begin, end) = self.simplex_state.get_tracked_slots_interval(); - self.receiver.set_standstill_slots(begin, end); + self.sync_standstill_slots_from_state(); log::trace!( "Session {} update_standstill_after_final_cert: slot={} tracked_slots=[{}, {})", @@ -9096,19 +8257,22 @@ impl SessionProcessor { // FSM already updated first_non_finalized_slot and cleaned up internally // Reset per-slot state for this slot self.reset_slot_state(slot); + self.cancel_candidate_repairs_for_slot(slot); // Update standstill tracked slots range (but DO NOT reschedule standstill on skip) // Reference: C++ pool.cpp on_skip() does NOT call reschedule_standstill_resolution() let (begin, end) = self.simplex_state.get_tracked_slots_interval(); - self.receiver.set_standstill_slots(begin, end); + self.sync_standstill_slots_from_state(); // Cancel any precollations for the skipped slot self.remove_precollated_block(slot); log::trace!( - "Session {} handle_slot_skipped: completed slot={}", + "Session {} handle_slot_skipped: completed slot={} tracked_slots=[{}, {})", &self.session_id().to_hex_string()[..8], - slot + slot, + begin, + end ); } @@ -9248,14 +8412,220 @@ impl SessionProcessor { log::trace!("SessionProcessor::notify_generate_slot: on_generate_slot finish"); } - }); + }); + } + + /// Emit finalized callback if all required data is available. + /// + /// This helper is called from: + /// - `handle_block_finalized` (immediate path), and + /// - `on_candidate_received` (delayed path when body arrives after FinalCert). + fn maybe_emit_out_of_order_finalized( + &mut self, + finalized_id: &RawCandidateId, + event: &BlockFinalizedEvent, + ) { + if self.finalized_delivery_sent.contains(finalized_id) { + return; + } + + let Some(received) = self.received_candidates.get(finalized_id) else { + // Candidate body not known yet. + return; + }; + if received.candidate_hash_data_bytes.is_empty() || received.is_empty { + // Boundary stubs and empty blocks are not delivered via on_block_finalized. + return; + } + + let source_idx = received.source_idx; + let source_public_key = self.description.get_source_public_key(source_idx).clone(); + let source_info = crate::BlockSourceInfo { + source: source_public_key, + priority: crate::BlockCandidatePriority { + round: SIMPLEX_ROUNDLESS, + priority: 0, + first_block_round: SIMPLEX_ROUNDLESS, + }, + }; + + let sigs: Vec<(crate::PublicKeyHash, crate::BlockPayloadPtr)> = event + .certificate + .signatures + .iter() + .map(|s| { + ( + self.description.get_source_public_key_hash(s.validator_idx).clone(), + consensus_common::ConsensusCommonFactory::create_block_payload( + s.signature.clone(), + ), + ) + }) + .collect(); + + let delivered = self.notify_block_finalized( + received.block_id.clone(), + source_info, + received.root_hash.clone(), + received.file_hash.clone(), + received.data.clone(), + sigs, + Vec::new(), + event.slot, + received.candidate_hash_data_bytes.clone(), + ); + if delivered { + self.finalized_delivery_sent.insert(finalized_id.clone()); + self.finalized_pending_body.remove(finalized_id); + self.finalized_pending_body_gauge.set(self.finalized_pending_body.len() as f64); + } + } + + /// Apply finalized-driven local state once the candidate body is available. + /// + /// This replaces the old sequential commit path for Simplex sessions: + /// - updates local finalized/head cursors + /// - persists finalized records for restart recovery + /// - clears per-slot runtime once finalization is materially applied + fn maybe_apply_finalized_state(&mut self, finalized_id: &RawCandidateId) { + if self.finalized_blocks.contains(finalized_id) { + return; + } + + let Some(received) = self.received_candidates.get(finalized_id).cloned() else { + return; + }; + + if received.candidate_hash_data_bytes.is_empty() { + return; + } + + let slot = received.slot; + let seqno = received.block_id.seq_no(); + + let slot_started_at = self.slot_started_at(slot); + if let Ok(duration) = self.now().duration_since(slot_started_at) { + self.slot_duration_histogram.record(duration.as_millis() as f64); + } + + if !self.slot_first_candidate_finalized(slot) { + self.slot_set_first_candidate_finalized(slot, true); + if let Ok(latency) = self.now().duration_since(slot_started_at) { + self.first_candidate_finalized_latency_histogram.record(latency.as_millis() as f64); + } + } + + let previous_head_seqno = self.finalized_head_seqno.unwrap_or(0); + let should_replace_head = match self.finalized_head_slot { + Some(current_slot) => { + seqno > previous_head_seqno + || (seqno == previous_head_seqno && slot >= current_slot) + } + None => true, + }; + + if !received.is_empty { + if seqno > previous_head_seqno { + self.finalized_head_seqno = Some(seqno); + } + + self.advance_accepted_normal_head_block(received.block_id.clone()); + + if should_replace_head { + if let Ok(before_split) = + crate::utils::extract_before_split_flag(received.data.data()) + { + self.finalized_head_before_split = before_split; + if before_split { + log::info!( + "Session {} maybe_apply_finalized_state: block at slot={slot} seqno={seqno} has \ + before_split=true (next block MUST be empty for split/merge)", + &self.session_id().to_hex_string()[..8], + ); + } + } else { + log::trace!( + "Session {} maybe_apply_finalized_state: failed to extract before_split flag for \ + slot={slot}, assuming false", + &self.session_id().to_hex_string()[..8], + ); + self.finalized_head_before_split = false; + } + } + } + + if should_replace_head { + self.finalized_head_slot = Some(slot); + self.finalized_head_block_id = Some(received.block_id.clone()); + } + + if seqno > self.last_consensus_finalized_seqno.unwrap_or(0) { + self.last_consensus_finalized_seqno = Some(seqno); + } + + self.finalized_blocks.insert(finalized_id.clone()); + + self.last_finalized_slot_gauge.set(slot.0 as f64); + let now = self.now(); + self.round_debug_at = now + ROUND_DEBUG_PERIOD; + self.last_finalization_time = now; + + let record = FinalizedBlockRecord { + candidate_id: finalized_id.clone(), + block_id: received.block_id.clone(), + parent: received.parent_id.clone(), + is_final: true, + }; + + if self.description.get_shard().is_masterchain() { + match self.db.save_finalized_block_async(&record) { + Ok(result) => { + if let Err(e) = result.wait() { + log::error!( + "Session {} maybe_apply_finalized_state: failed to store finalized block for \ + slot={}: {e}", + &self.session_id().to_hex_string()[..8], + slot.value(), + ); + self.increment_error(); + } + } + Err(e) => { + log::error!( + "Session {} maybe_apply_finalized_state: failed to create finalized block save for \ + slot={}: {e}", + &self.session_id().to_hex_string()[..8], + slot.value(), + ); + self.increment_error(); + } + } + } else if let Err(e) = self.db.save_finalized_block(&record) { + log::error!( + "Session {} maybe_apply_finalized_state: failed to store finalized block for slot={}: {e}", + &self.session_id().to_hex_string()[..8], + slot.value(), + ); + self.increment_error(); + } + + self.finalized_pending_body.remove(finalized_id); + self.finalized_pending_body_gauge.set(self.finalized_pending_body.len() as f64); + if self.simplex_state.is_slot_progressed(&self.description, slot) { + self.reset_slot_state(slot); + } else { + log::trace!( + "Session {} maybe_apply_finalized_state: skipping reset_slot_state for \ + non-progressed slot={slot}", + &self.session_id().to_hex_string()[..8], + ); + } } - /// Notify listener about a committed block - /// - /// Called when a block has been committed with sufficient signatures. - fn notify_block_committed( + /// Emit `on_block_finalized` to the listener for finalized-driven acceptance. + fn notify_block_finalized( &self, + block_id: BlockIdExt, source_info: crate::BlockSourceInfo, root_hash: crate::BlockHash, file_hash: crate::BlockHash, @@ -9264,36 +8634,33 @@ impl SessionProcessor { approve_signatures: Vec<(crate::PublicKeyHash, crate::BlockPayloadPtr)>, slot: SlotIndex, candidate_hash_data_bytes: Vec, - is_final: bool, - stats: consensus_common::SessionStats, - ) { + ) -> bool { check_execution_time!(20_000); log::trace!( - "Session {} notify_block_committed: posting on_block_committed event for \ - root_hash={:x}", + "Session {} notify_block_finalized: posting for block_id={} slot={}", self.session_id().to_hex_string(), - root_hash, + block_id, + slot, ); let listener = self.listener.clone(); - // Build BlockSignaturesVariant::Simplex with proper context for signature verification let signatures_variant = match self.build_simplex_signatures_variant( &signatures, slot, candidate_hash_data_bytes, - is_final, + true, // always final ) { Ok(v) => v, Err(e) => { log::error!( - "Session {} notify_block_committed: failed to build signatures variant: {}", + "Session {} notify_block_finalized: failed to build signatures variant: {}", self.session_id().to_hex_string(), e ); self.increment_error(); - return; + return false; } }; @@ -9301,21 +8668,18 @@ impl SessionProcessor { check_execution_time!(20_000); if let Some(listener) = listener.upgrade() { - log::trace!("SessionProcessor::notify_block_committed: on_block_committed start"); - - listener.on_block_committed( + listener.on_block_finalized( + block_id, source_info, root_hash, file_hash, data, signatures_variant, approve_signatures, - stats, ); - - log::trace!("SessionProcessor::notify_block_committed: on_block_committed finish"); } }); + true } /// Build BlockSignaturesVariant::Simplex from raw signature pairs with context @@ -9437,185 +8801,11 @@ impl SessionProcessor { Ok(BlockSignaturesVariant::Simplex(simplex_signatures)) } - // ======================================================================== - // Download committed block proof for MC gap recovery - // ======================================================================== - - /// Convert BlockSignaturesSimplex from a block proof into VoteSignatureSet - /// TL bytes that process_received_final_cert expects. - /// - /// Maps pure_signatures (node_id_short → CryptoSignature) back to - /// VoteSignature (validator_idx → signature bytes) using SessionDescription. - fn convert_proof_to_final_cert_bytes( - &self, - sigs: &BlockSignaturesSimplex, - ) -> Result<(SlotIndex, UInt256, Vec)> { - let candidate_data_bytes = sigs.candidate_data_bytes()?; - let candidate_hash = sha256_digest(&candidate_data_bytes); - let block_hash = UInt256::from_slice(&candidate_hash); - - let mut votes = Vec::new(); - sigs.pure_signatures.signatures().iterate_slices(|_key, ref mut slice| { - let pair = CryptoSignaturePair::construct_from(slice)?; - let key_id = KeyId::from_data(*pair.node_id_short.as_slice()); - let node_id: crate::PublicKeyHash = key_id; - - match self.description.get_source_index(&node_id) { - Ok(val_idx) => { - votes.push( - TlVoteSignature { - who: val_idx.value() as i32, - signature: pair.sign.as_bytes().to_vec().into(), - } - .into_boxed(), - ); - } - Err(_) => { - log::trace!( - "Session {} convert_proof_to_final_cert_bytes: \ - unknown signer {} (skipping)", - &self.session_id().to_hex_string()[..8], - node_id, - ); - } - } - Ok(true) - })?; - - if votes.is_empty() { - fail!("No known signers in proof for slot={}", sigs.slot); - } - - let tl_set = VoteSignatureSet { votes: votes.into() }.into_boxed(); - let bytes = serialize_boxed(&tl_set)?; - - let slot = SlotIndex::new(sigs.slot); - Ok((slot, block_hash, bytes)) - } - - /// Handle committed block proof received from ValidatorGroup. - /// - /// Converts proof signatures to VoteSignatureSet bytes and feeds them - /// through the existing process_received_final_cert → set_finalize_certificate - /// → try_commit_finalized_chains flow. - fn process_committed_proof_result( - &mut self, - block_id: BlockIdExt, - result: Result, - ) { - self.pending_committed_proof_requests.remove(&block_id); - let proof = match result { - Ok(p) => p, - Err(e) => { - log::warn!( - "Session {} process_committed_proof_result: failed for {}: {}", - &self.session_id().to_hex_string()[..8], - block_id, - e - ); - return; - } - }; - - if proof.block_id != block_id { - log::warn!( - "Session {} process_committed_proof_result: \ - proof identity mismatch: requested {} but got {}", - &self.session_id().to_hex_string()[..8], - block_id, - proof.block_id, - ); - return; - } - - let simplex_sigs = match &proof.signatures { - BlockSignaturesVariant::Simplex(s) if s.is_final => s, - _ => { - log::warn!( - "Session {} process_committed_proof_result: \ - expected Simplex(is_final=true) for {}", - &self.session_id().to_hex_string()[..8], - block_id, - ); - return; - } - }; - - match self.convert_proof_to_final_cert_bytes(simplex_sigs) { - Ok((slot, block_hash, final_cert_bytes)) => { - log::debug!( - "Session {} process_committed_proof_result: \ - converted proof for {} → slot={} hash={} ({} bytes)", - &self.session_id().to_hex_string()[..8], - block_id, - slot, - &block_hash.to_hex_string()[..8], - final_cert_bytes.len(), - ); - self.process_received_final_cert(slot, &block_hash, &final_cert_bytes); - self.try_commit_finalized_chains(); - } - Err(e) => { - log::warn!( - "Session {} process_committed_proof_result: \ - conversion failed for {}: {}", - &self.session_id().to_hex_string()[..8], - block_id, - e - ); - } - } - } - - /// Request committed block proof from full-node via SessionListener. - /// - /// Posts callback through invoke_session_callback (SXCB thread), which calls - /// listener.get_committed_candidate(). The result is posted back to SXMAIN - /// via task_queue.post_closure → process_committed_proof_result. - fn notify_get_committed_candidate(&self, block_id: BlockIdExt) { - check_execution_time!(20_000); - - log::trace!( - "Session {} notify_get_committed_candidate: requesting proof for {}", - &self.session_id().to_hex_string()[..8], - block_id, - ); - - let listener = self.listener.clone(); - let task_queue = self.task_queue.clone(); - let session_id = self.session_id().clone(); - - self.invoke_session_callback(move || { - if let Some(listener) = listener.upgrade() { - let task_queue_inner = task_queue; - let block_id_for_log = block_id.clone(); - - listener.get_committed_candidate( - block_id.clone(), - Box::new(move |result| { - crate::task_queue::post_closure( - &task_queue_inner, - move |processor: &mut SessionProcessor| { - processor.process_committed_proof_result(block_id, result); - }, - ); - }), - ); - - log::trace!( - "notify_get_committed_candidate: posted for {} (session {})", - block_id_for_log, - session_id.to_hex_string(), - ); - } - }); - } - /// Handle RequestCandidate query fallback when receiver's resolver_cache misses. /// /// Called from SXRCV thread via ReceiverListener when a peer's RequestCandidate query - /// cannot be answered from the in-memory resolver_cache. Attempts to reconstruct the - /// response from: + /// cannot be fully answered from the in-memory resolver_cache. Attempts to reconstruct + /// requested candidate body and/or notar parts from: /// 1. `candidate_data_cache` (in-memory, fast path) /// 2. SimplexDB `CandidateInfoRecord` (empty blocks only -- reconstructed from metadata) /// @@ -9625,10 +8815,12 @@ impl SessionProcessor { /// the validator manager. /// /// Reference: C++ `CandidateResolver::try_load_candidate_data_from_db()` + /// TODO: LK: move DB operations to background thread pub fn handle_candidate_query_fallback( &mut self, slot: SlotIndex, block_hash: UInt256, + want_candidate: bool, want_notar: bool, response_callback: crate::QueryResponseCallback, ) { @@ -9637,117 +8829,102 @@ impl SessionProcessor { let candidate_id = RawCandidateId { slot, hash: block_hash.clone() }; let session_hex = &self.session_id().to_hex_string()[..8]; - // 1. Fast path: check in-memory candidate_data_cache - if let Some(candidate_bytes) = self.candidate_data_cache.get(&candidate_id) { - log::debug!( - "Session {} candidate_query_fallback: cache HIT for slot={} hash={} ({}B)", - session_hex, - slot, - &block_hash.to_hex_string()[..8], - candidate_bytes.len() - ); - let notar_bytes = if want_notar { - self.load_notar_cert_bytes_from_db(&candidate_id) - } else { - Vec::new() - }; - Self::send_candidate_and_cert_response( - candidate_bytes.clone(), - notar_bytes, - response_callback, - ); - return; - } + // Candidate and notar can be requested independently. Build each part + // from the best available source and return partials when only one part exists. + let mut candidate_bytes = Vec::new(); - // 2. DB path: load CandidateInfoRecord for metadata - let candidate_info = match self.load_candidate_info_from_db(&candidate_id) { - Some(info) => info, - None => { + if want_candidate { + // 1. Fast path: in-memory candidate_data_cache + if let Some(bytes) = self.candidate_data_cache.get(&candidate_id) { log::debug!( - "Session {} candidate_query_fallback: NOT FOUND for slot={} hash={}", - session_hex, - slot, - &block_hash.to_hex_string()[..8] + "Session {session_hex} candidate_query_fallback: \ + candidate cache HIT for slot={slot} hash={} ({}B)", + &block_hash.to_hex_string()[..8], + bytes.len() ); - Self::send_empty_candidate_response(response_callback); - return; - } - }; - - let notar_bytes = - if want_notar { self.load_notar_cert_bytes_from_db(&candidate_id) } else { Vec::new() }; + candidate_bytes.clone_from(bytes); + } else { + // 2. DB path: candidate metadata + let candidate_info = self.load_candidate_info_from_db(&candidate_id); - // 3. Try persisted payload from DB first (works for both empty and non-empty blocks, - // since save_candidate_payload_async persists payloads for all candidates). - { - const DB_TIMEOUT: Duration = Duration::from_secs(2); - match self.db.load_candidate_payload_by_id(&candidate_id, DB_TIMEOUT) { - Ok(Some(payload_bytes)) => { - log::debug!( - "Session {} candidate_query_fallback: loaded payload from DB for slot={} ({}B)", - session_hex, - slot, - payload_bytes.len() - ); - Self::send_candidate_and_cert_response( - payload_bytes, - notar_bytes, - response_callback, - ); - return; + // 3. Persisted payload (works for both empty and non-empty blocks) + const DB_TIMEOUT: Duration = Duration::from_secs(2); + match self.db.load_candidate_payload_by_id(&candidate_id, DB_TIMEOUT) { + Ok(Some(payload_bytes)) => { + log::debug!( + "Session {session_hex} candidate_query_fallback: \ + loaded payload from DB for slot={slot} ({}B)", + payload_bytes.len() + ); + candidate_bytes = payload_bytes; + } + Ok(None) => {} + Err(e) => { + log::warn!( + "Session {session_hex} candidate_query_fallback: \ + DB payload load error for slot={slot}: {e}" + ); + } } - Ok(None) => {} - Err(e) => { - log::warn!( - "Session {} candidate_query_fallback: DB payload load error for slot={}: {}", - session_hex, - slot, - e - ); + + // 4. Metadata reconstruction for empty blocks when payload missing. + if candidate_bytes.is_empty() { + if let Some(info) = candidate_info.as_ref() { + let is_empty = matches!( + info.candidate_hash_data, + CandidateHashData::Consensus_CandidateHashDataEmpty(_) + ); + if is_empty { + match self + .reconstruct_empty_candidate_data_from_info(&candidate_id, info) + { + Ok(bytes) => { + log::debug!( + "Session {session_hex} candidate_query_fallback: \ + reconstructed empty block for slot={slot} ({}B)", + bytes.len() + ); + candidate_bytes = bytes; + } + Err(e) => { + log::warn!( + "Session {session_hex} candidate_query_fallback: \ + failed to reconstruct empty block for slot={slot}: {e}" + ); + } + } + } + } } } } - // 4. DB payload not available: try metadata reconstruction for empty blocks - let is_empty = matches!( - candidate_info.candidate_hash_data, - CandidateHashData::Consensus_CandidateHashDataEmpty(_) - ); + let notar_bytes = + if want_notar { self.load_notar_cert_bytes_from_db(&candidate_id) } else { Vec::new() }; - if is_empty { - match self.reconstruct_empty_candidate_data_from_info(&candidate_id, &candidate_info) { - Ok(bytes) => { - log::debug!( - "Session {} candidate_query_fallback: reconstructed empty block for slot={} ({}B)", - session_hex, - slot, - bytes.len() - ); - Self::send_candidate_and_cert_response(bytes, notar_bytes, response_callback); - return; - } - Err(e) => { - log::warn!( - "Session {} candidate_query_fallback: failed to reconstruct empty block \ - for slot={}: {}", - session_hex, - slot, - e - ); - } - } + if candidate_bytes.is_empty() && notar_bytes.is_empty() { + log::debug!( + "Session {} candidate_query_fallback: NOT FOUND for slot={} hash={} \ + (want_candidate={}, want_notar={})", + session_hex, + slot, + &block_hash.to_hex_string()[..8], + want_candidate, + want_notar, + ); + } else { + log::debug!( + "Session {} candidate_query_fallback: responding slot={} hash={} \ + candidate_bytes={} notar_bytes={}", + session_hex, + slot, + &block_hash.to_hex_string()[..8], + candidate_bytes.len(), + notar_bytes.len() + ); } - // 5. Not in memory, DB, or reconstructable: return notar-only if available (partial merge). - log::debug!( - "Session {} candidate_query_fallback: block NOT FOUND for slot={} hash={}, \ - returning notar_only={}", - session_hex, - slot, - &block_hash.to_hex_string()[..8], - !notar_bytes.is_empty() - ); - Self::send_candidate_and_cert_response(Vec::new(), notar_bytes, response_callback); + Self::send_candidate_and_cert_response(candidate_bytes, notar_bytes, response_callback); } /// Load CandidateInfoRecord from DB (blocking, used for rare query fallback). @@ -9808,11 +8985,6 @@ impl SessionProcessor { response_callback(result); } - /// Send empty CandidateAndCert response (when fallback has nothing to return). - fn send_empty_candidate_response(response_callback: crate::QueryResponseCallback) { - Self::send_candidate_and_cert_response(Vec::new(), Vec::new(), response_callback); - } - /// Reconstruct CandidateData::Consensus_Empty bytes from CandidateInfoRecord. fn reconstruct_empty_candidate_data_from_info( &self, @@ -9875,6 +9047,9 @@ impl SessionStartupRecoveryListener for SessionProcessor { slot.value() ); self.simplex_state.set_first_non_finalized_slot(slot); + self.receiver.set_ingress_slot_begin(slot.value()); + self.receiver + .set_ingress_progress_slot(self.simplex_state.get_first_non_progressed_slot().value()); } fn recovery_on_vote( @@ -10031,6 +9206,11 @@ impl SessionStartupRecoveryListener for SessionProcessor { let block_hash = block.candidate_id.hash.clone(); let block_id = block.block_id.clone(); let candidate_id = RawCandidateId { slot, hash: block_hash.clone() }; + let is_empty = block + .parent + .as_ref() + .and_then(|parent_id| self.received_candidates.get(parent_id)) + .is_some_and(|parent| parent.block_id == block_id); // Skip if already present (shouldn't happen, but be safe) if self.received_candidates.contains_key(&candidate_id) { @@ -10051,7 +9231,7 @@ impl SessionStartupRecoveryListener for SessionProcessor { data: consensus_common::ConsensusCommonFactory::create_block_payload(Vec::new()), collated_data: consensus_common::ConsensusCommonFactory::create_block_payload(Vec::new()), receive_time: self.now(), - is_empty: false, + is_empty, parent_id: block.parent.clone(), is_fully_resolved: true, }, @@ -10087,12 +9267,20 @@ impl SessionStartupRecoveryListener for SessionProcessor { .map(|p| format!("s{}:{}", p.slot.value(), &p.hash.to_hex_string()[..8])), ); - if self.received_candidates.contains_key(&candidate_id) { + let is_fully_resolved = self.compute_is_fully_resolved(&parent); + + if let Some(existing) = self.received_candidates.get_mut(&candidate_id) { + existing.source_idx = leader_idx; + existing.candidate_hash_data_bytes = candidate_hash_data_bytes; + existing.block_id.clone_from(&block_id); + existing.root_hash.clone_from(&block_id.root_hash); + existing.file_hash.clone_from(&block_id.file_hash); + existing.is_empty = is_empty; + existing.parent_id = parent; + existing.is_fully_resolved = is_fully_resolved; return; } - let is_fully_resolved = self.compute_is_fully_resolved(&parent); - self.received_candidates.insert( candidate_id.clone(), ReceivedCandidate { @@ -10154,19 +9342,21 @@ impl SessionStartupRecoveryListener for SessionProcessor { ); // Update last_committed tracking to reflect the restart state - self.last_committed_seqno = Some(seqno); - self.last_committed_slot = Some(slot); - self.last_committed_block_id = Some(block_id.clone()); + self.finalized_head_seqno = Some(seqno); + self.finalized_head_slot = Some(slot); + self.finalized_head_block_id = Some(block_id.clone()); + self.last_mc_finalized_seqno = Some(self.last_mc_finalized_seqno.unwrap_or(0).max(seqno)); self.last_consensus_finalized_seqno = Some(seqno); + self.advance_accepted_normal_head_block(block_id.clone()); // Note: We do NOT set available_base here anymore. This is now done in // recovery_finalize_parent_chain() after all kept votes are restored, // because the kept votes may finalize additional slots. - // Note: We do NOT call notify_block_committed here because: - // 1. C++ only publishes BlockFinalized event, not a full re-acceptance + // Note: We do NOT notify ValidatorGroup here because: + // 1. C++ only republishes finalized state, not a fresh accept callback // 2. The block was already accepted before restart - // 3. Recommit execution (if enabled) handles ValidatorGroup notification separately + // 3. Restart recovery now restores state only; no historical replay callbacks remain } fn recovery_finalize_parent_chain(&mut self) { @@ -10442,7 +9632,7 @@ impl SessionStartupRecoveryListener for SessionProcessor { // 4) Update receiver standstill tracked range and timer // This also prunes cached votes outside [begin, end). - self.receiver.set_standstill_slots(begin, end); + self.sync_standstill_slots_from_state(); self.receiver.reschedule_standstill(); log::info!( @@ -10453,122 +9643,6 @@ impl SessionStartupRecoveryListener for SessionProcessor { self.session_id().to_hex_string(), ); } - - fn recovery_apply_restart_recommit_actions( - &mut self, - actions: &[RestartRoundAction], - get_candidate: &mut dyn FnMut( - &RestartRoundAction, - ) - -> Result, - ) -> Result<()> { - log::info!( - target: "startup_recovery", - "Session {}: applying {} restart recommit actions", - self.session_id().to_hex_string(), - actions.len() - ); - - let mut committed = 0u32; - - for action in actions { - let RestartRoundAction::Commit { - slot, - block_id, - leader_idx, - root_hash, - file_hash, - candidate_hash, - candidate_hash_data_bytes, - is_empty, - .. - } = action; - - if *is_empty { - log::debug!( - target: "startup_recovery", - "Session {}: replayed empty finalized record slot={}, seqno={} (no callback)", - &self.session_id().to_hex_string()[..8], - slot.value(), - block_id.seq_no() - ); - self.last_committed_slot = Some(*slot); - continue; - } - - log::debug!( - target: "startup_recovery", - "Session {}: restart recommit COMMIT slot={}, round=ROUNDLESS, seqno={}", - &self.session_id().to_hex_string()[..8], - slot.value(), - block_id.seq_no() - ); - - // Fetch candidate via closure (must exist for non-empty actions). - let candidate = get_candidate(action).map_err(|e| { - error!("restart replay failed to fetch candidate for slot {}: {e}", slot.value()) - })?; - - // Get signatures from restored notar certificate - // After vote replay, simplex_state should have the certificates. - let signatures = self.get_notarize_signatures(*slot, candidate_hash); - if signatures.is_empty() { - fail!( - "restart replay missing notar cert for slot {} hash {}", - slot.value(), - candidate_hash.to_hex_string() - ); - } - - // For restart recommit, use notar signatures for both sets - // (same as prepare_parent_block_signatures) - let approve_signatures = signatures.clone(); - - // Build source info (SIMPLEX_ROUNDLESS) - let source_public_key = self.description.get_source_public_key(*leader_idx).clone(); - let source_info = crate::BlockSourceInfo { - source: source_public_key, - priority: BlockCandidatePriority { - round: SIMPLEX_ROUNDLESS, // Simplex roundless mode - first_block_round: SIMPLEX_ROUNDLESS, // Must match round for consistency - priority: 0, - }, - }; - - // Build session stats - let stats = self.build_session_stats(); - - // Notify listener about the commit (SIMPLEX_ROUNDLESS) - // is_final = true for all replayed finalized blocks - self.notify_block_committed( - source_info, - root_hash.clone(), - file_hash.clone(), - candidate.data.clone(), - signatures, - approve_signatures, - *slot, - candidate_hash_data_bytes.clone(), - true, // is_final - stats, - ); - - // Update seqno tracking - self.last_committed_seqno = Some(block_id.seq_no()); - self.last_committed_slot = Some(*slot); - - committed += 1; - } - - log::info!( - target: "startup_recovery", - "Session {}: restart recommit complete: {} committed", - self.session_id().to_hex_string(), - committed - ); - - Ok(()) - } } /* diff --git a/src/node/simplex/src/simplex_state.rs b/src/node/simplex/src/simplex_state.rs index 9cc05a3..d1d2f8d 100644 --- a/src/node/simplex/src/simplex_state.rs +++ b/src/node/simplex/src/simplex_state.rs @@ -257,16 +257,6 @@ use crate::{ session_description::SessionDescription, RawVoteData, ValidatorWeight, }; - -/// Maximum number of slots ahead of `first_non_finalized_slot` that the FSM -/// will accept. Any vote, candidate, or certificate referencing a slot beyond -/// this horizon is rejected to prevent a Byzantine validator from triggering -/// unbounded window/slot allocation (DoS). -/// -/// Note: C++ has no equivalent cap. This is a Rust-only defense-in-depth measure. -/// 10,000 is generous enough to never affect liveness under normal conditions. -pub const MAX_FUTURE_SLOTS: u32 = 10_000; - use std::{ cmp, collections::{BinaryHeap, HashMap, HashSet, VecDeque}, @@ -324,27 +314,6 @@ pub struct SimplexStateOptions { /// /// C++ uses notarized parent check (not finalized) for collation availability. pub require_finalized_parent: bool, - - /// Use notarized-parent chain semantics for parenting/progress (C++ pool `now_` model). - /// - /// When `false` (legacy ParentReady-driven window progression): - /// - Leader window advancement / timeout scheduling is driven via `on_window_base_ready()` (finalization) - /// - First-slot parent readiness is tracked per-window (`LeaderWindow.available_bases`) - /// - `first_non_progressed_slot` / `Slot.available_base` / `Slot.skipped` are still tracked for consistency, - /// but do not drive leader-window progression - /// - /// When `true` (C++ pool.cpp parity, default for `cpp_compatible()`): - /// - Progress cursor `first_non_progressed_slot` advances on **(notarized OR skipped)**, - /// like C++ `PoolImpl::advance_present()` / `maybe_publish_new_leader_windows()` - /// - Per-slot `available_base` (optional-of-optional) is the canonical parent chain: - /// - `None` = base unknown - /// - `Some(None)` = genesis base - /// - `Some(Some(id))` = concrete parent candidate id - /// - Leader window advancement / timeout scheduling follows the progress cursor - /// - /// Both modes maintain the tracking state (`available_base`, `skipped`, `first_non_progressed_slot`) - /// to keep `SimplexState` internally consistent. - pub use_notarized_parent_chain: bool, } impl Default for SimplexStateOptions { @@ -355,8 +324,6 @@ impl Default for SimplexStateOptions { allow_skip_after_notarize: true, // C++ allows notarized blocks as parents (not just finalized) require_finalized_parent: false, - // C++ pool.cpp parity: notarized-parent chain drives window progression. - use_notarized_parent_chain: true, } } } @@ -374,7 +341,6 @@ impl SimplexStateOptions { enable_fallback_protocol: true, allow_skip_after_notarize: false, require_finalized_parent: false, - use_notarized_parent_chain: false, } } @@ -388,7 +354,6 @@ impl SimplexStateOptions { enable_fallback_protocol: false, allow_skip_after_notarize: true, require_finalized_parent: true, - use_notarized_parent_chain: false, } } } @@ -685,9 +650,8 @@ struct Slot { /// - `Some(None)` = genesis base (RawParentId{}) /// - `Some(Some(id))` = concrete parent candidate id /// - /// This field is always maintained for state consistency. When - /// `SimplexStateOptions::use_notarized_parent_chain` is enabled, it is used - /// to propagate bases across (notarized OR skipped) slots like C++ pool.cpp. + /// This field tracks the canonical parent chain across (notarized OR skipped) + /// slots like C++ pool.cpp. available_base: Option, /// Pending block candidate waiting for parent/conditions @@ -1370,9 +1334,8 @@ pub(crate) struct SimplexState { /// /// Mirrors C++ `PoolImpl::now_` (pool.cpp maybe_publish_new_leader_windows()). /// - /// This field is always maintained for state consistency. When - /// `SimplexStateOptions::use_notarized_parent_chain` is enabled, it is used - /// to drive leader-window progression / timeout scheduling. + /// This field drives leader-window progression / timeout scheduling and + /// mirrors C++ `PoolImpl::now_`. first_non_progressed_slot: SlotIndex, /* @@ -1386,6 +1349,12 @@ pub(crate) struct SimplexState { /// Timestamp when current skip_slot times out skip_timestamp: Option, + /// Fixed per-window base from which all slot deadlines are derived. + /// C++ `timeout_base_` (consensus.cpp): set to `now + first_block_timeout` + /// when a new leader window starts, never modified within the window. + /// Slot at offset k has deadline `timeout_base + (k+1) * target_rate`. + timeout_base: Option, + /// First block timeout (adaptive) first_block_timeout: Duration, @@ -1399,6 +1368,10 @@ pub(crate) struct SimplexState { */ /// Slots per leader window slots_per_leader_window: u32, + /// Maximum accepted leader-window desync. + /// + /// C++ parity: `params_.max_leader_window_desync` bound used in consensus/pool ingress. + max_leader_window_desync: u32, /// SimplexState options (fallback protocol, etc.) opts: SimplexStateOptions, @@ -1476,9 +1449,11 @@ impl SimplexState { first_non_progressed_slot: SlotIndex(0), skip_slot: SlotIndex(0), skip_timestamp: None, + timeout_base: None, first_block_timeout, target_rate_timeout, slots_per_leader_window: slots_per_window, + max_leader_window_desync: desc.opts().max_leader_window_desync, opts, window_reject_count: 0, }; @@ -1552,10 +1527,10 @@ impl SimplexState { /// Ensure window exists at index. /// - /// Defense-in-depth: refuses to allocate beyond `MAX_FUTURE_SLOTS` horizon + /// Defense-in-depth: refuses to allocate beyond configured future horizon /// even if the caller forgot to pre-validate. fn ensure_window_exists(&mut self, idx: WindowIndex) { - let max_slot = self.first_non_finalized_slot.value() + MAX_FUTURE_SLOTS; + let max_slot = self.max_acceptable_slot().value(); let max_window = WindowIndex(max_slot / self.slots_per_leader_window + 1); if idx > max_window { self.window_reject_count += 1; @@ -1651,7 +1626,6 @@ impl SimplexState { if self.first_non_finalized_slot > self.first_non_progressed_slot { self.first_non_progressed_slot = self.first_non_finalized_slot; } - log::trace!( "SimplexState::set_first_non_finalized_slot: setting to {} (slots_per_window={})", self.first_non_finalized_slot.value(), @@ -1714,6 +1688,7 @@ impl SimplexState { }; let window_idx = desc.get_window_idx(slot); let offset = desc.get_slot_offset_in_window(slot) as usize; + let clear_pending_on_skip = self.opts.enable_fallback_protocol; // After restart recovery sets `first_non_finalized_slot`, we may prune old leader windows // by advancing `leader_window_offset`. Votes for slots in pruned windows are irrelevant @@ -1772,7 +1747,12 @@ impl SimplexState { window.slots[offset].is_voted = true; window.slots[offset].voted_skip = true; window.slots[offset].is_bad_window = true; - window.slots[offset].pending_block = None; + // C++ mode preserves pending candidate on skip/restart-skip: + // consensus.cpp gates candidate intake by voted_notar, not voted_skip. + // Keep Alpenglow behavior unchanged (clear pending on skip). + if clear_pending_on_skip { + window.slots[offset].pending_block = None; + } log::trace!( "SimplexState::mark_slot_voted_on_restart: slot {} marked voted_skip=true", slot.value() @@ -1810,6 +1790,7 @@ impl SimplexState { // end_slot = window * slots_per_leader_window let start_slot = (first_nonannounced_window.value() - 1) * slots_per_window; let end_slot = first_nonannounced_window.value() * slots_per_window; + let clear_pending_on_skip = self.opts.enable_fallback_protocol; let mut skip_count = 0u32; for slot_num in start_slot..end_slot { @@ -1840,7 +1821,11 @@ impl SimplexState { window.slots[offset].is_voted = true; window.slots[offset].voted_skip = true; window.slots[offset].is_bad_window = true; - window.slots[offset].pending_block = None; + // C++ mode preserves pending candidate across restart skip voting. + // Keep Alpenglow fallback behavior unchanged. + if clear_pending_on_skip { + window.slots[offset].pending_block = None; + } log::trace!( "SimplexState::generate_restart_skip_votes: queueing skip for slot {}", @@ -1918,6 +1903,16 @@ impl SimplexState { self.try_skip_window(window_idx); } + #[cfg(test)] + pub fn on_block_notarized_for_test( + &mut self, + desc: &SessionDescription, + slot: SlotIndex, + block_hash: UInt256, + ) { + self.on_block_notarized(desc, slot, block_hash); + } + #[cfg(test)] pub fn set_first_non_finalized_slot_for_test(&mut self, slot: SlotIndex) { self.first_non_finalized_slot = slot; @@ -2194,25 +2189,38 @@ impl SimplexState { /// for i ∈ windowSlots(s) do // set timeouts for all slots /// schedule event Timeout(i) at time clock()+Δtimeout+(i−s+1)·Δblock /// ``` - pub(crate) fn set_timeouts(&mut self, desc: &SessionDescription) { + fn set_timeouts(&mut self, desc: &SessionDescription) { let window_start = self.current_leader_window_idx * self.slots_per_leader_window; self.skip_slot = window_start; - //TODO: LK: in C++ first slot in a window has timeout first_block_timeout without target_rate_timeout - self.skip_timestamp = - Some(desc.get_time() + self.first_block_timeout + self.target_rate_timeout); - - log::warn!( + // C++ consensus.cpp: + // timeout_base_ = Timestamp::in(first_block_timeout_); // base = now + first_block + // alarm_timestamp() = Timestamp::in(target_rate, timeout_base_); // alarm = base + target_rate + // First alarm fires at: now + first_block_timeout + target_rate (both modes). + self.timeout_base = Some(desc.get_time() + self.first_block_timeout); + let first_timeout = self.first_block_timeout + self.target_rate_timeout; + self.skip_timestamp = Some(desc.get_time() + first_timeout); + + log::debug!( "SimplexState::set_timeouts: ({}/{}) scheduling timeout in {:.3}s \ (first_block={:.3}s, target_rate={:.3}s)", self.current_leader_window_idx, self.skip_slot, - (self.first_block_timeout + self.target_rate_timeout).as_secs_f64(), + first_timeout.as_secs_f64(), self.first_block_timeout.as_secs_f64(), self.target_rate_timeout.as_secs_f64(), ); } + /// Arm/reset startup timeouts when session processing actually starts. + /// + /// This is called by SessionProcessor on the first active tick (after startup/recovery), + /// so skip timers are anchored to "ready" time instead of FSM construction time. + pub(crate) fn reset_timeouts_on_start(&mut self, desc: &SessionDescription) { + self.restore_default_timeouts(desc); + self.set_timeouts(desc); + } + /// Restore default timeouts (reset adaptive backoff) fn restore_default_timeouts(&mut self, desc: &SessionDescription) { self.target_rate_timeout = desc.opts().target_rate; @@ -2344,14 +2352,16 @@ impl SimplexState { // Alpenglow: trySkipWindow(s) self.try_skip_window(window_idx); - // C++ compatibility: skip entire remaining window at once, then BREAK. - // Reference: C++ consensus.cpp alarm() lines 120-133: + // C++ compatibility: skip entire remaining window at once, then STOP. + // Reference: C++ consensus.cpp alarm(): // C++ fires alarm once and skips ALL remaining slots in the window, - // then sets timeout_slot_ = window_end and reschedules. - // Between alarm firings, incoming events (NotarizationObserved, - // skip certs from peers) can advance timeout_slot_ past active slots. - // We break after one window to give incoming events a chance to - // advance skip_slot before we vote skip for more slots. + // then sets timeout_slot_ = window_end. Crucially, C++ does NOT + // schedule a new alarm here — the next alarm is only armed when + // LeaderWindowObserved fires for the next window (which applies + // first_block_timeout). Without this, the skip timer races ahead + // of actual window advancement, firing for future windows with + // only target_rate delay instead of first_block_timeout + target_rate, + // causing nodes to vote skip before leaders can produce blocks. if !self.opts.enable_fallback_protocol { let window_end_slot = (window_idx + 1) * self.slots_per_leader_window; if self.skip_slot < window_end_slot { @@ -2363,9 +2373,11 @@ impl SimplexState { ); self.skip_slot = window_end_slot; } - // Schedule next timeout at target_rate from now (not accumulated) - skip_timestamp = desc.get_time() + self.target_rate_timeout; - self.skip_timestamp = Some(skip_timestamp); + // Do NOT reschedule — let advance_leader_window_on_progress_cursor() + // re-arm via set_timeouts() with proper first_block_timeout when + // the next window actually starts. + self.skip_timestamp = None; + self.timeout_base = None; break; } } @@ -2374,9 +2386,9 @@ impl SimplexState { /// Apply adaptive timeout backoff based on previous window's timeout history /// - /// This is used by both: - /// - `on_window_base_ready()` (legacy finalization-driven window progression) - /// - `advance_leader_window_on_progress_cursor()` (notarized-parent-chain mode) + /// This is used by helper paths that populate available bases for collation, + /// including `on_window_base_ready()` and + /// `advance_leader_window_on_progress_cursor()`. /// /// Reference: C++ pool.cpp (adaptive backoff logic in window progression) /// @@ -2659,18 +2671,19 @@ impl SimplexState { )); } - // Reject far-future slots (DoS protection) - if self.is_slot_too_far_ahead(slot) { + // Reject far-future vote slots using the C++ `first_too_new_slot` rule. + if self.is_vote_slot_too_far_ahead(slot) { log::warn!( - "SimplexState::on_vote: ({}/{}) REJECTED - slot too far ahead (max={})", + "SimplexState::on_vote: ({}/{}) REJECTED - slot too far ahead \ + (first_too_new={})", window_idx, slot, - self.max_acceptable_slot() + self.first_too_new_vote_slot() ); return VoteResult::Rejected(format!( - "slot {} too far ahead (max={})", + "slot {} too far ahead (first_too_new={})", slot, - self.max_acceptable_slot() + self.first_too_new_vote_slot() )); } @@ -3729,61 +3742,19 @@ impl SimplexState { } self.propagate_base_after_notarization(desc, parent_info.clone()); } + // C++ parity: finalization path must run the progress cursor walk + // (`advance_present`) before leader-window publication logic. + self.advance_progress_cursor(desc); - // Choose window advancement strategy based on mode log::trace!( - "SimplexState::check_thresholds: ({}/{}) window advancement strategy: \ - use_notarized_parent_chain={} current_window={} first_non_progressed_slot={}", + "SimplexState::check_thresholds: ({}/{}) advancing leader window on \ + progress cursor current_window={} first_non_progressed_slot={}", window_idx, slot_id, - self.opts.use_notarized_parent_chain, self.current_leader_window_idx, self.first_non_progressed_slot ); - if self.opts.use_notarized_parent_chain { - // Behavioral mode: advance leader window based on progress cursor (not finalization). - // Reference: C++ pool.cpp maybe_publish_new_leader_windows() - self.advance_leader_window_on_progress_cursor(desc); - } else { - // Trigger ParentReady for the next window - // When a block is finalized, it becomes a valid parent for the next window's first slot - // Reference: Alpenglow Algorithm 1 "upon ParentReady(window, hash(b))" - // - // Note: C++ reference has this in pool.cpp comment but NOT implemented. - // We implement it here: finalized block in window W becomes parent for window W+1. - // - // No recursion risk: on_window_base_ready -> check_pending_blocks -> try_notar - // only broadcasts votes, doesn't call check_thresholds_and_trigger. - let next_window_idx = - slot_id.window_index(self.slots_per_leader_window) + 1; - - log::trace!( - "SimplexState::check_thresholds: ({}/{}) triggering ParentReady for {} parent={}:{}", - window_idx, - slot_id, - next_window_idx, - slot_id, - &block.to_hex_string()[..8] - ); - - // Call on_window_base_ready to handle all the logic: - // - Add to available_bases - // - Check pending blocks - // - Update timeouts with adaptive backoff - // Note: This cannot fail because: - // - next_window_idx is small (no overflow) - // - parent slot < next window start slot (by construction) - if let Err(e) = - self.on_window_base_ready(desc, next_window_idx, Some(parent_info)) - { - log::error!( - "SimplexState::check_thresholds: ({}/{}) ParentReady failed: {}", - window_idx, - slot_id, - e - ); - } - } + self.advance_leader_window_on_progress_cursor(desc); break; } @@ -3863,15 +3834,11 @@ impl SimplexState { // Reference: C++ pool.cpp on_skip() → slot.skipped=true, propagate base if needed self.propagate_base_after_skip_cert(desc, slot_id); - // When notarized-parent chain mode is enabled, trigger leader window advancement - // based on progress cursor instead of waiting for finalization. - // Otherwise, use the legacy per-window propagation approach. log::trace!( "SimplexState::check_thresholds: ({}/{}) window advancement after skip: \ - use_notarized_parent_chain={} current_window={} first_non_progressed_slot={}", + current_window={} first_non_progressed_slot={}", window_idx, slot_id, - self.opts.use_notarized_parent_chain, self.current_leader_window_idx, self.first_non_progressed_slot ); @@ -3889,70 +3856,7 @@ impl SimplexState { ); } - if self.opts.use_notarized_parent_chain { - self.advance_leader_window_on_progress_cursor(desc); - } else { - // Check if this is the last slot in the window BEFORE cleanup - // If so, and if no block was finalized in this window, we need to - // propagate the available bases to the next window (including genesis/None) - // This handles the startup case where an entire window is skipped. - let current_window_idx = slot_id.window_index(self.slots_per_leader_window); - let slot_offset_in_window = slot_id.offset_in_window(self.slots_per_leader_window); - let is_last_slot_in_window = - slot_offset_in_window == self.slots_per_leader_window - 1; - - // Capture bases BEFORE cleanup (window may be removed by cleanup) - let bases_to_propagate: Option> = if is_last_slot_in_window { - let next_window_idx = current_window_idx + 1; - - // Check if next window already has available bases (from finalization) - let next_window_has_bases = self - .get_window(next_window_idx) - .map(|w| !w.available_bases.is_empty()) - .unwrap_or(false); - - if !next_window_has_bases { - // Capture current window's available bases before cleanup - self.get_window(current_window_idx) - .map(|w| w.available_bases.iter().cloned().collect()) - } else { - None - } - } else { - None - }; - - // Propagate bases to next window after cleanup - if let Some(bases) = bases_to_propagate { - if !bases.is_empty() { - let next_window_idx = current_window_idx + 1; - - log::trace!( - "SimplexState: Last slot {} of window {} skipped without finalization, \ - propagating {} available base(s) to window {}", - slot_id, - current_window_idx, - bases.len(), - next_window_idx - ); - - for parent in bases { - // Use on_window_base_ready to handle the logic consistently - // Note: No recursion risk (same as BlockFinalized case) - if let Err(e) = - self.on_window_base_ready(desc, next_window_idx, parent.clone()) - { - log::error!( - "SimplexState: SlotSkipped failed to propagate parent {:?} to window {}: {}", - parent, - next_window_idx, - e - ); - } - } - } - } - } + self.advance_leader_window_on_progress_cursor(desc); } } @@ -4012,54 +3916,54 @@ impl SimplexState { let parent_info = CandidateParentInfo { slot, hash: block_hash.clone() }; self.propagate_base_after_notarization(desc, parent_info.clone()); - // When notarized-parent chain mode is enabled, trigger leader window advancement - // based on progress cursor instead of waiting for finalization. // Reference: C++ pool.cpp maybe_publish_new_leader_windows() log::trace!( "SimplexState::on_block_notarized: ({}/{}) window advancement check: \ - use_notarized_parent_chain={} first_non_progressed_slot={}", + first_non_progressed_slot={}", window_idx, slot, - self.opts.use_notarized_parent_chain, self.first_non_progressed_slot ); - if self.opts.use_notarized_parent_chain { - self.advance_leader_window_on_progress_cursor(desc); - } + self.advance_leader_window_on_progress_cursor(desc); - // C++ compatibility: advance skip timer when NotarCert arrives - // Reference: C++ consensus.cpp lines 228-243 (NotarizationObserved handler) - // When a NotarCert is observed, C++ advances timeout_slot_ to slot+1 and - // reschedules the alarm to now + target_rate. This prevents the skip cascade - // from racing ahead of active block production. + // C++ compatibility: advance skip timer when NotarCert arrives. + // Reference: C++ consensus.cpp NotarizationObserved handler. + // + // C++ computes the deadline from a fixed per-window timeout_base_: + // alarm = timeout_base_ + (timeout_slot_ - window_start) * target_rate + // This anchors all deadlines to the window start time, not to "now". // - // Important: do NOT shrink skip_timestamp below the current scheduled value. - // During the first_block_timeout window, the skip timer is intentionally set - // far in the future to give all nodes time to join the overlay. Setting it to - // now + target_rate here would bypass that protection entirely. + // Guard: C++ checks `timeout_slot_ <= event->id.slot + 1`. + // Since C++ timeout_slot_ = Rust skip_slot + 1, this maps to: + // skip_slot + 1 <= slot + 1 → skip_slot <= slot + // This prevents stale updates and also prevents overwriting a deadline + // that was freshly set by advance_leader_window_on_progress_cursor when + // notarization of the last window slot caused a window transition. if !self.opts.enable_fallback_protocol { let next_slot = slot + 1; - if self.skip_slot <= next_slot { - let new_timestamp = desc.get_time() + self.target_rate_timeout; - // Only update skip_timestamp if it would be later than current, - // preserving the first_block_timeout window. - let effective_timestamp = match self.skip_timestamp { - Some(current) if current > new_timestamp => current, - _ => new_timestamp, - }; - log::debug!( - "SimplexState::on_block_notarized: advancing skip timer: \ - skip_slot {} -> {next_slot}, new timeout in {:?}{}", - self.skip_slot, - self.target_rate_timeout, - if effective_timestamp != new_timestamp { - " (preserved first_block_timeout)" - } else { - "" - } - ); - self.skip_slot = next_slot; - self.skip_timestamp = Some(effective_timestamp); + if self.skip_slot <= slot { + if let Some(base) = self.timeout_base { + let window_start = + self.current_leader_window_idx.window_start(self.slots_per_leader_window); + + // C++ timeout_slot_ = slot+2 normally, slot+1 at window end. + // Rust skip_slot = C++ timeout_slot_ - 1. + let is_window_end = next_slot.value() % self.slots_per_leader_window == 0; + let cpp_timeout_slot = + if is_window_end { next_slot.value() } else { next_slot.value() + 1 }; + + let offset = cpp_timeout_slot - window_start.value(); + let new_deadline = base + self.target_rate_timeout * offset; + + log::debug!( + "SimplexState::on_block_notarized: advancing skip timer: \ + skip_slot {} -> {next_slot}, deadline at base+{}*target_rate", + self.skip_slot, + offset, + ); + self.skip_slot = next_slot; + self.skip_timestamp = Some(new_deadline); + } } } @@ -4174,6 +4078,7 @@ impl SimplexState { /// # Errors /// /// Returns error if window_idx would cause overflow or parent slot is invalid. + #[cfg(test)] pub fn on_window_base_ready( &mut self, desc: &SessionDescription, @@ -4353,7 +4258,6 @@ impl SimplexState { ) -> bool { let window_idx = desc.get_window_idx(slot); let offset = desc.get_slot_offset_in_window(slot) as usize; - let is_first = desc.is_first_in_window(slot); self.ensure_window_exists(window_idx); @@ -4402,75 +4306,22 @@ impl SimplexState { } // Check can_vote_notar - let can_vote_notar = if self.opts.use_notarized_parent_chain { - // C++ pool.cpp parity: - // Parent readiness is determined by per-slot `available_base` chain (not ParentReady/available_bases). - // - // Reference: C++ pool.cpp `SlotState::available_base` and request/resolve logic that only - // allows extending the chain from the known base. - let expected_base = self.get_slot_available_base(desc, slot); - let (base_known, expected_parent): (bool, CandidateParent) = match expected_base { - Some(parent) => (true, parent), - None => (false, None), - }; - - let candidate_parent: CandidateParent = parent.cloned(); - let matches_parent = base_known && expected_parent == candidate_parent; - - log::trace!( - "SimplexState::try_notar: ({window_idx}/{slot}) notarized-parent chain: \ - base_known={base_known} expected_base={} candidate_parent={} matches={}", - Self::format_parent(expected_parent.as_ref()), - Self::format_parent(parent), - matches_parent - ); - - matches_parent - } else if is_first { - // Alpenglow: firstSlot: ParentReady(hashparent) ∈ state[s] - let parent_key: CandidateParent = parent.cloned(); - let has_parent = self - .get_window(window_idx) - .map(|w| w.available_bases.contains(&parent_key)) - .unwrap_or(false); - - log::trace!( - "SimplexState::try_notar: ({}/{}) first_in_window, parent={} in_bases={}", - window_idx, - slot, - Self::format_parent(parent), - has_parent - ); - has_parent - } else { - // Alpenglow: not firstSlot: VotedNotar(hashparent) ∈ state[s-1] - let Some(parent) = parent else { - log::trace!( - "SimplexState::try_notar: ({}/{}) non-first slot, no parent -> cannot vote", - window_idx, - slot - ); - return false; - }; - let prev_slot = slot - 1; - let prev_window_idx = desc.get_window_idx(prev_slot); - let prev_offset = desc.get_slot_offset_in_window(prev_slot) as usize; + let expected_base = self.get_slot_available_base(desc, slot); + let (base_known, expected_parent): (bool, CandidateParent) = match expected_base { + Some(parent) => (true, parent), + None => (false, None), + }; - let voted_notar = self - .get_window(prev_window_idx) - .and_then(|w| w.slots[prev_offset].voted_notar.as_ref()); - let matches_parent = voted_notar.map(|voted| voted == parent).unwrap_or(false); + let candidate_parent: CandidateParent = parent.cloned(); + let can_vote_notar = base_known && expected_parent == candidate_parent; - log::trace!( - "SimplexState::try_notar: ({}/{}) parent={} prev_voted={} matches={}", - window_idx, - slot, - Self::format_parent(Some(parent)), - Self::format_parent(voted_notar), - matches_parent - ); - matches_parent - }; + log::trace!( + "SimplexState::try_notar: ({window_idx}/{slot}) canonical parent check: \ + base_known={base_known} expected_base={} candidate_parent={} matches={}", + Self::format_parent(expected_parent.as_ref()), + Self::format_parent(parent), + can_vote_notar + ); if can_vote_notar { log::trace!( @@ -4832,16 +4683,43 @@ impl SimplexState { self.first_non_finalized_slot } - /// Returns the maximum slot the FSM will accept (inclusive). + /// Returns the configured future-slot span for present/progress horizon checks. + #[inline] + fn max_future_slot_span(&self) -> u32 { + self.max_leader_window_desync.saturating_mul(self.slots_per_leader_window) + } + + /// Returns the maximum slot candidate precheck will accept (inclusive). + /// + /// Mirrors C++ `PrecheckCandidateBroadcast`: + /// `slot > now_ + max_leader_window_desync * slots_per_leader_window`. pub fn max_acceptable_slot(&self) -> SlotIndex { - self.first_non_finalized_slot + MAX_FUTURE_SLOTS + SlotIndex::new( + self.first_non_progressed_slot.value().saturating_add(self.max_future_slot_span()), + ) } - /// Returns `true` if `slot` exceeds the acceptable future horizon. + /// Returns `true` if a candidate slot exceeds the acceptable future horizon. pub fn is_slot_too_far_ahead(&self, slot: SlotIndex) -> bool { slot > self.max_acceptable_slot() } + /// Returns the first slot that is considered "too new" for votes/certificates. + /// + /// Mirrors C++ `pool.cpp`: + /// `(now_ / slots_per_window + max_desync + 1) * slots_per_window` + pub fn first_too_new_vote_slot(&self) -> SlotIndex { + let current_window = self.first_non_progressed_slot.value() / self.slots_per_leader_window; + let first_too_new_window = + current_window.saturating_add(self.max_leader_window_desync).saturating_add(1); + SlotIndex::new(first_too_new_window.saturating_mul(self.slots_per_leader_window)) + } + + /// Returns `true` if a vote slot is beyond the C++ `first_too_new_slot` bound. + pub fn is_vote_slot_too_far_ahead(&self, slot: SlotIndex) -> bool { + slot >= self.first_too_new_vote_slot() + } + /// Get first non-progressed slot (progress cursor) /// /// This is the first slot that has NOT progressed yet, where "progressed" means @@ -4994,6 +4872,41 @@ impl SimplexState { false } + /// Returns the notarized block hash for a slot, if known. + /// + /// For finalized slots, this prefers the finalization certificate hash and falls back + /// to notarization certificate hash from persisted vote state. + pub fn get_notarized_block_hash( + &self, + desc: &SessionDescription, + slot: SlotIndex, + ) -> Option { + if slot < self.first_non_finalized_slot { + let sv = self.slot_votes.get(&slot)?; + if let Some(cert) = &sv.finalize_certificate { + return Some(cert.vote.block_hash.clone()); + } + if let Some(cert) = &sv.notarize_certificate { + return Some(cert.vote.block_hash.clone()); + } + return None; + } + + self.get_slot_ref(desc, slot) + .and_then(|s| s.observed_notar_certificate.as_ref().map(|c| c.hash.clone())) + } + + /// Check if a slot has reached Skip certificate state. + /// + /// This mirrors C++ `slot->state->is_skipped()` checks used by `WaitForParent`. + pub fn has_skip_certificate_for_slot( + &self, + desc: &SessionDescription, + slot: SlotIndex, + ) -> bool { + self.is_slot_skipped_cert(desc, slot) + } + /// Check if a slot is finalized (ItsOver flag) /// /// Used for debug logging to show consensus progress. @@ -5514,22 +5427,11 @@ impl SimplexState { if self.first_non_finalized_slot > self.first_non_progressed_slot { self.first_non_progressed_slot = self.first_non_finalized_slot; } + // C++ parity: after finalization, `now_` is advanced with + // `advance_present()` before leader-window publication. + self.advance_progress_cursor(desc); - // Advance leader windows using the active mode's strategy. - if self.opts.use_notarized_parent_chain { - self.advance_leader_window_on_progress_cursor(desc); - } else { - let next_window_idx = slot.window_index(self.slots_per_leader_window) + 1; - if let Err(e) = self.on_window_base_ready(desc, next_window_idx, Some(parent_info)) { - log::error!( - "SimplexState::set_finalize_certificate: ParentReady failed for w{} parent={}:{}: {}", - next_window_idx, - slot, - &block_hash.to_hex_string()[..8], - e - ); - } - } + self.advance_leader_window_on_progress_cursor(desc); Ok(true) } @@ -5647,14 +5549,11 @@ impl SimplexState { // Only finalization advances it (C++ state.h notify_finalized()). // The progress cursor (first_non_progressed_slot) DOES advance on skip. - // Advance progress cursor - if self.opts.use_notarized_parent_chain { - // Advance first_non_progressed_slot if this slot was blocking progress - if slot == self.first_non_progressed_slot { - self.advance_progress_cursor(desc); - } - self.advance_leader_window_on_progress_cursor(desc); + // Advance progress cursor / leader window if this slot was blocking progress. + if slot == self.first_non_progressed_slot { + self.advance_progress_cursor(desc); } + self.advance_leader_window_on_progress_cursor(desc); // Emit SlotSkipped event so SessionProcessor can progress/cleanup state. // This mirrors the threshold-driven path which emits SlotSkipped when the @@ -5704,6 +5603,7 @@ impl SimplexState { } /// Get finalization certificate for a specific candidate (slot, hash), if present. + #[cfg(test)] pub fn get_finalize_certificate( &self, slot: SlotIndex, @@ -5760,22 +5660,15 @@ impl SimplexState { - `Slot.skipped` (skip certificate flag, C++ `SlotState::skipped`) - `SimplexState.first_non_progressed_slot` (progress cursor, C++ `PoolImpl::now_`) - The tracking state is **always maintained** for consistency, regardless of - the `SimplexStateOptions::use_notarized_parent_chain` flag. + This tracking state drives the active C++-parity progression model. - When `use_notarized_parent_chain` is **disabled** (legacy ParentReady-driven mode): - - Tracking state is updated but does not drive leader-window progression - - Leader window advancement / timeout scheduling is driven by `on_window_base_ready()` (finalization) - - When `use_notarized_parent_chain` is **enabled** (C++ pool.cpp parity, default for `cpp_compatible()`): - - Tracking state drives leader window advancement / timeout scheduling - `first_non_progressed_slot` cursor determines when to advance `current_leader_window_idx` - Parent readiness for notarization follows `available_base` chain (not `available_bases`) This design allows: - State consistency: no mode-dependent null/partial state - Easy testing: always inspect tracking state in tests - - Clean migration: flip flag without restructuring core FSM logic + - Clear C++ parity: a single progression model without legacy branching Reference: C++ pool.cpp `PoolImpl::now_`, `SlotState::available_base`, `on_notarization()`, `on_skip()`, `maybe_publish_new_leader_windows()` @@ -5788,8 +5681,7 @@ impl SimplexState { /// Note: the Rust implementation uses max-merge (`add_available_base_max`) instead of /// unconditional assignment, to prevent regression when duplicate/late notarizations arrive. /// - /// This is always called when a block is notarized, regardless of mode. - /// The tracked state is used for progress when `use_notarized_parent_chain` is enabled. + /// This is always called when a block is notarized. fn propagate_base_after_notarization( &mut self, desc: &SessionDescription, @@ -5809,11 +5701,8 @@ impl SimplexState { // Advance progress cursor through any progressed slots self.advance_progress_cursor(desc); - // In notarized-parent chain mode, base propagation can make pending blocks voteable. - // Retry pending blocks immediately to match C++ pool behavior. - if self.opts.use_notarized_parent_chain { - self.check_pending_blocks(desc); - } + // Base propagation can make pending blocks voteable immediately. + self.check_pending_blocks(desc); } /// Set available base for the first non-finalized slot after restart recovery @@ -5861,8 +5750,7 @@ impl SimplexState { /// allows `check_pending_blocks` / `try_notar` to find the base for any pending /// block regardless of skip-cert arrival order. /// - /// This is always called when a slot is skipped, regardless of mode. - /// The tracked state is used for progress when `use_notarized_parent_chain` is enabled. + /// This is always called when a slot is skipped. fn propagate_base_after_skip_cert(&mut self, desc: &SessionDescription, slot: SlotIndex) { // Mark slot as skipped (skip certificate reached) if let Some(slot_state) = self.get_slot_mut(desc, slot) { @@ -5907,47 +5795,18 @@ impl SimplexState { } } - // C++ compatibility: advance skip timer when SkipCert arrives - // Reference: C++ consensus.cpp lines 228-248 (NotarizationObserved handler) - // C++ advances timeout_slot_ on both NotarCert and SkipCert (via LeaderWindowObserved). - // Without this, the Rust skip cascade takes ~27s for 27 slots (1s/slot) while - // C++ processes entire windows at once and advances the timer on each event. - // - // Important: do NOT shrink skip_timestamp below the current scheduled value - // to preserve the first_block_timeout window. - if !self.opts.enable_fallback_protocol { - let next_slot = slot + 1; - if self.skip_slot <= next_slot { - let new_timestamp = desc.get_time() + self.target_rate_timeout; - let effective_timestamp = match self.skip_timestamp { - Some(current) if current > new_timestamp => current, - _ => new_timestamp, - }; - log::debug!( - "SimplexState::propagate_base_after_skip_cert: advancing skip timer: \ - skip_slot {} -> {}, new timeout in {:?}{}", - self.skip_slot, - next_slot, - self.target_rate_timeout, - if effective_timestamp != new_timestamp { - " (preserved first_block_timeout)" - } else { - "" - } - ); - self.skip_slot = next_slot; - self.skip_timestamp = Some(effective_timestamp); - } - } + // C++ parity: do NOT advance the skip timer on skip certs. + // C++ consensus.cpp never touches the alarm on skip certificates — they + // flow through the pool layer only. Window-crossing skip certs trigger + // advance_progress_cursor → advance_leader_window_on_progress_cursor → + // set_timeouts(), which properly re-arms with fresh timeout_base. Within + // the same window, the fixed-base schedule handles deadlines correctly. // Advance progress cursor through any progressed slots self.advance_progress_cursor(desc); - // In notarized-parent chain mode, base propagation can make pending blocks voteable. - // Retry pending blocks immediately to match C++ pool behavior. - if self.opts.use_notarized_parent_chain { - self.check_pending_blocks(desc); - } + // Base propagation can make pending blocks voteable immediately. + self.check_pending_blocks(desc); } /// Advance progress cursor through all progressed slots @@ -6019,7 +5878,7 @@ impl SimplexState { /// Reference: C++ pool.cpp maybe_publish_new_leader_windows() /// /// This triggers timeout scheduling for the new window and applies adaptive backoff. - /// Only called when `SimplexStateOptions::use_notarized_parent_chain` is enabled. + /// Called by the active C++-parity progression path. /// /// # Ordering guarantee (C++ parity: PR #2195) /// @@ -6408,6 +6267,25 @@ impl SimplexState { } } + /// Produce the C++-style standstill diagnostic dump. + /// + /// Mirrors `pool.cpp::alarm()` by emitting the latest final certificate + /// summary first (if any), followed by the per-slot grid. + pub fn standstill_diagnostic_dump(&self, desc: &SessionDescription) -> String { + let mut sb = String::default(); + + if let Some((slot, cert)) = self.get_last_finalize_certificate() { + sb.push_str(&format!( + "Last final cert is for slot={} hash={}\n", + slot.value(), + cert.vote.block_hash.to_hex_string(), + )); + } + + sb.push_str(&self.standstill_slot_grid_dump(desc)); + sb + } + /// Produce C++-style standstill slot-grid dump. /// /// For each slot in the tracked range [begin, end), outputs one line: diff --git a/src/node/simplex/src/startup_recovery.rs b/src/node/simplex/src/startup_recovery.rs index 63b4c50..cd2e424 100644 --- a/src/node/simplex/src/startup_recovery.rs +++ b/src/node/simplex/src/startup_recovery.rs @@ -27,13 +27,11 @@ //! │ - vote replay (Phase 6.6 order) │ //! │ - set finalized boundary + apply local flags │ //! │ - restore receiver caches │ -//! │ - recommit to ValidatorGroup │ //! └─────────────────────────────────────────────────────────────────┘ //! ``` //! //! # Key Components //! -//! - [`RestartRoundAction`]: Action to take for each round during restart replay //! - [`SessionStartupRecoveryListener`]: Object-safe trait for recovery operations //! - [`SessionStartupRecoveryProcessor`]: Coordinator that loads bootstrap and drives recovery //! @@ -49,9 +47,8 @@ use crate::{ session_description::SessionDescription, simplex_state::Vote, utils::extract_vote_and_signature, - BlockHash, RawVoteData, RestartRecommitStrategy, SessionId, + RawVoteData, SessionId, }; -use consensus_common::ValidatorBlockCandidatePtr; use std::{ collections::{HashMap, HashSet}, sync::Arc, @@ -64,7 +61,7 @@ use ton_api::{ }, IntoBoxed, }; -use ton_block::{error, fail, BlockIdExt, Result, UInt256}; +use ton_block::{error, BlockIdExt, Result, UInt256}; /* Constants @@ -83,46 +80,6 @@ pub(crate) type CandidateHash = UInt256; /// Signature bytes (Ed25519 signature) pub(crate) type SignatureBytes = Vec; -/* - RestartRoundAction - action for each round during restart replay -*/ - -/// Action to take for each round during restart replay to ValidatorGroup. -/// -/// Built by `SessionStartupRecoveryProcessor::build_restart_recommit_actions()` -/// based on `RestartRecommitStrategy` and finalized block records from bootstrap. -#[derive(Debug, Clone)] -pub(crate) enum RestartRoundAction { - /// Replay this finalized record through the commit path. - /// - /// Non-empty records fetch approved candidate payload and emit - /// `on_block_committed`. Empty records are replayed as internal cursor - /// progress only (no `on_block_skipped` callbacks in roundless mode). - Commit { - /// Slot index - slot: SlotIndex, - /// Block ID with seqno - block_id: BlockIdExt, - /// Leader validator index - leader_idx: ValidatorIndex, - /// Root hash for candidate lookup - root_hash: BlockHash, - /// File hash for candidate lookup - file_hash: BlockHash, - /// Collated data hash for candidate lookup - _collated_data_hash: BlockHash, - /// Candidate hash for certificate lookup - candidate_hash: CandidateHash, - /// Pre-serialized CandidateHashData bytes (for BlockSignaturesVariant::Simplex) - candidate_hash_data_bytes: Vec, - /// Whether this finalized record is an empty block. - /// - /// Empty records are still part of deterministic replay ordering, but - /// they do not emit `on_block_committed` callbacks. - is_empty: bool, - }, -} - /* SessionStartupRecoveryListener - object-safe trait for recovery operations */ @@ -190,7 +147,7 @@ pub(crate) trait SessionStartupRecoveryListener { /// Restore kept `BroadcastVote` events to the front of the queue. /// - /// Called after recommit replay so votes are broadcast on first `check_all()`. + /// Called after startup cache restoration so votes are broadcast on first `check_all()`. fn recovery_restore_startup_votes(&mut self, votes: Vec); // ======================================================================== @@ -313,9 +270,8 @@ pub(crate) trait SessionStartupRecoveryListener { /// Seed notarization certificate into simplex_state. /// /// Used during restart to populate simplex_state.slot_votes with parsed - /// notar certs so that `get_notarize_signatures` can find them during recommit. - /// This is separate from `recovery_cache_notarization_cert` which only caches - /// raw bytes in receiver for network queries. + /// notar certs. This is separate from `recovery_cache_notarization_cert` + /// which only caches raw bytes in receiver for network queries. fn recovery_seed_notarize_certificate( &mut self, slot: SlotIndex, @@ -333,24 +289,6 @@ pub(crate) trait SessionStartupRecoveryListener { /// This is intentionally separate from `recovery_restore_startup_votes`: it restores /// historical state from DB, whereas startup votes are freshly generated on restart. fn recovery_restore_receiver_standstill_cache(&mut self, votes: &[VoteRecord]); - - // ======================================================================== - // Recommit replay (applies existing notify paths internally) - // ======================================================================== - - /// Apply restart recommit actions using existing notify paths. - /// - /// Full replay in roundless mode is commit-path only: - /// - non-empty records emit `notify_block_committed()` - /// - empty records advance internal replay cursors only - /// - /// The `get_candidate` closure is used to fetch candidate payloads for - /// non-empty records. - fn recovery_apply_restart_recommit_actions( - &mut self, - actions: &[RestartRoundAction], - get_candidate: &mut dyn FnMut(&RestartRoundAction) -> Result, - ) -> Result<()>; } /* @@ -362,20 +300,16 @@ pub(crate) trait SessionStartupRecoveryListener { /// Extracted to simplify testing and reduce coupling. #[derive(Clone, Copy, Debug)] pub(crate) struct SessionStartupRecoveryOptions { - /// Strategy for replaying finalized blocks - pub restart_recommit_strategy: RestartRecommitStrategy, - /// Initial block seqno (for seqno matching during recommit) + /// Initial block seqno passed by session start; kept for future policy hooks. + #[allow(dead_code)] // Reserved for future restart policies. pub initial_block_seqno: u32, } impl SessionStartupRecoveryOptions { /// Create from SessionOptions and initial_block_seqno #[allow(dead_code)] // Available for future use - pub fn new( - restart_recommit_strategy: RestartRecommitStrategy, - initial_block_seqno: u32, - ) -> Self { - Self { restart_recommit_strategy, initial_block_seqno } + pub fn new(initial_block_seqno: u32) -> Self { + Self { initial_block_seqno } } } @@ -388,7 +322,7 @@ impl SessionStartupRecoveryOptions { /// Coordinates the startup recovery stage: /// 1. Loads bootstrap from DB (in constructor, cancellable) /// 2. Computes recovery identity (self_idx, validator keys) -/// 3. Builds restore/recommit plans +/// 3. Builds restore plans /// 4. Drives recovery through `SessionStartupRecoveryListener` /// /// Dropped before entering the main processing loop. @@ -399,9 +333,6 @@ pub(crate) struct SessionStartupRecoveryProcessor { /// Session description (for leader key lookup during candidate reconstruction) _description: Arc, - /// Recovery options (strategy, initial seqno) - options: SessionStartupRecoveryOptions, - /// Self validator index (cached from description) self_idx: ValidatorIndex, @@ -428,7 +359,7 @@ impl SessionStartupRecoveryProcessor { pub fn new( session_id: SessionId, description: Arc, - options: SessionStartupRecoveryOptions, + _options: SessionStartupRecoveryOptions, bootstrap: Bootstrap, ) -> Self { let self_idx = description.get_self_idx(); @@ -447,7 +378,6 @@ impl SessionStartupRecoveryProcessor { Self { session_id, _description: description, - options, self_idx, bootstrap: Some(bootstrap), candidate_info_map, @@ -485,8 +415,7 @@ impl SessionStartupRecoveryProcessor { /// 2. Generates restart skip votes /// 3. Drains startup events (keeps BroadcastVote only) /// 4. Restores receiver caches (notar certs, candidate bytes) - /// 5. Builds and applies restart recommit actions - /// 6. Restores kept votes + /// 5. Restores kept votes /// /// After this method returns, the processor is consumed and can be dropped. /// @@ -582,12 +511,11 @@ impl SessionStartupRecoveryProcessor { kept_votes.len() ); - // Step 6: Seed current_round for restart replay + // Step 6: Seed current_round compatibility hook. // - // Startup recommit actions (step 11) replay historical rounds to ValidatorGroup. - // To keep round accounting consistent (and avoid double-counting), we start the - // replay from round 0 and let step 11 advance `current_round` as it applies - // skip/commit actions. + // Simplex now restores finalized state directly without historical recommit, + // so this remains a compatibility callback for the recovery pipeline. The + // slot-based SessionProcessor currently treats it as a no-op. log::debug!( target: LOG_TARGET, "Session {}: step 6/12 - seeding current_round=0 (finalized_blocks={})", @@ -662,32 +590,23 @@ impl SessionStartupRecoveryProcessor { ); listener.recovery_restore_receiver_standstill_cache(&session_boot.votes); - // Step 11: Build and apply restart recommit actions + // Step 11: Restore kept votes log::debug!( target: LOG_TARGET, - "Session {}: step 11/12 - applying restart recommit actions (strategy={:?})", - self.session_id.to_hex_string(), - self.options.restart_recommit_strategy - ); - self.apply_restart_recommit(listener, &session_boot.finalized_blocks)?; - - // Step 12: Restore kept votes - log::debug!( - target: LOG_TARGET, - "Session {}: step 12/12 - restoring {} kept votes", + "Session {}: step 11/12 - restoring {} kept votes", self.session_id.to_hex_string(), kept_votes.len() ); listener.recovery_restore_startup_votes(kept_votes); - // Step 13: Finalize parent chain setup - // IMPORTANT: This must happen AFTER step 12 (kept votes restoration) because + // Step 12: Finalize parent chain setup + // IMPORTANT: This must happen AFTER step 11 (kept votes restoration) because // the kept votes may finalize additional slots, advancing first_non_finalized_slot. // We need to set available_base for the CURRENT first_non_finalized_slot, not the // one from the DB (which was outdated). log::debug!( target: LOG_TARGET, - "Session {}: step 13/13 - finalizing parent chain setup", + "Session {}: step 12/12 - finalizing parent chain setup", self.session_id.to_hex_string() ); listener.recovery_finalize_parent_chain(); @@ -915,7 +834,7 @@ impl SessionStartupRecoveryProcessor { /// /// This does two things: /// 1. Cache raw bytes in receiver for network queries (`requestCandidate(want_notar=true)`) - /// 2. Parse and seed into simplex_state for restart recommit (`get_notarize_signatures`) + /// 2. Parse and seed into simplex_state for restored certificate state fn restore_notar_cert_cache( &self, listener: &mut dyn SessionStartupRecoveryListener, @@ -932,7 +851,7 @@ impl SessionStartupRecoveryProcessor { cert.notar_cert_bytes.to_vec(), ); - // 2. Parse and seed into simplex_state for recommit + // 2. Parse and seed into simplex_state for restored certificate state match crate::certificate::NotarCert::from_tl_bytes_for_candidate( &cert.notar_cert_bytes, cert.candidate_id.slot, @@ -1258,167 +1177,6 @@ impl SessionStartupRecoveryProcessor { } } - /// Build and apply restart recommit actions. - fn apply_restart_recommit( - &self, - listener: &mut dyn SessionStartupRecoveryListener, - finalized_blocks: &[FinalizedBlockRecord], - ) -> Result<()> { - // Build recommit actions based on strategy - let actions = self.build_restart_recommit_actions(finalized_blocks)?; - - if actions.is_empty() { - log::info!( - target: LOG_TARGET, - "Session {}: no recommit actions to apply", - self.session_id.to_hex_string() - ); - return Ok(()); - } - - log::info!( - target: LOG_TARGET, - "Session {}: applying {} recommit actions (strategy={:?})", - self.session_id.to_hex_string(), - actions.len(), - self.options.restart_recommit_strategy - ); - - // Apply actions through listener. - // Non-empty blocks cannot be fetched (no get_approved_candidate delegation in - // simplex -- C++ resolves candidates from its own DB, not validator manager). - // The get_candidate closure returns an error for non-empty blocks, causing - // the replay to skip them gracefully. - listener.recovery_apply_restart_recommit_actions(&actions, &mut |action| { - let RestartRoundAction::Commit { slot, is_empty, .. } = action; - if *is_empty { - fail!("fetch called for empty replay action at slot {}", slot.value()); - } - - Err(error!( - "non-empty block candidate fetch not supported in simplex recovery (slot {})", - slot.value() - )) - })?; - - Ok(()) - } - - /// Build restart recommit actions based on strategy. - pub fn build_restart_recommit_actions( - &self, - finalized_blocks: &[FinalizedBlockRecord], - ) -> Result> { - match self.options.restart_recommit_strategy { - RestartRecommitStrategy::FullReplay => self.build_full_replay_actions(finalized_blocks), - RestartRecommitStrategy::FirstCommitAfterFinalized => { - // C++-like mode: restore state only, no historical replay callbacks. - Ok(Vec::new()) - } - } - } - - /// Build full replay actions (all finalized blocks in order). - fn build_full_replay_actions( - &self, - finalized_blocks: &[FinalizedBlockRecord], - ) -> Result> { - let mut actions = Vec::with_capacity(finalized_blocks.len()); - let mut expected_parent: Option = None; - let mut last_slot: Option = None; - let mut last_committed_seqno = self.options.initial_block_seqno.checked_sub(1); - - for block in finalized_blocks { - let slot = block.candidate_id.slot; - let candidate_hash = block.candidate_id.hash.clone(); - let block_seqno = block.block_id.seq_no(); - - if let Some(prev_slot) = last_slot { - if slot.value() <= prev_slot.value() { - fail!( - "full replay requires strict slot monotonicity: prev_slot={}, slot={}", - prev_slot.value(), - slot.value() - ); - } - } - - if block.parent != expected_parent { - fail!( - "full replay chain discontinuity at slot {}: expected parent {:?}, actual {:?}", - slot.value(), - expected_parent, - block.parent - ); - } - - // Look up candidate info - let candidate_info = self.candidate_info_map.get(&candidate_hash).ok_or_else(|| { - error!( - "missing candidate info for replay slot {} hash {}", - slot.value(), - candidate_hash.to_hex_string() - ) - })?; - let is_empty = - Self::is_empty_block_candidate_hash_data(&candidate_info.candidate_hash_data); - - let expected_seqno = match (is_empty, last_committed_seqno) { - (false, Some(prev)) => prev.saturating_add(1), - (false, None) => self.options.initial_block_seqno, - (true, Some(prev)) => prev, - (true, None) => { - fail!("full replay cannot start from empty block at slot {}", slot.value()); - } - }; - - if block_seqno != expected_seqno { - fail!( - "full replay seqno mismatch at slot {}: expected {}, actual {}, is_empty={}", - slot.value(), - expected_seqno, - block_seqno, - is_empty - ); - } - - // Extract block_id and collated_data_hash from CandidateHashData - let (root_hash, file_hash, collated_data_hash) = - Self::extract_hashes_from_candidate_hash_data(&candidate_info.candidate_hash_data) - .ok_or_else(|| { - error!( - "failed to extract hashes from candidate hash data for replay slot {}", - slot.value() - ) - })?; - - // Serialize CandidateHashData to bytes - let candidate_hash_data_bytes = - Self::serialize_candidate_hash_data(&candidate_info.candidate_hash_data); - - // Build commit action - actions.push(RestartRoundAction::Commit { - slot, - block_id: block.block_id.clone(), - leader_idx: ValidatorIndex(candidate_info.leader_idx), - root_hash, - file_hash, - _collated_data_hash: collated_data_hash, - candidate_hash, - candidate_hash_data_bytes, - is_empty, - }); - - if !is_empty { - last_committed_seqno = Some(block_seqno); - } - last_slot = Some(slot); - expected_parent = Some(block.candidate_id.clone()); - } - - Ok(actions) - } - /// Deserialize a vote record into (Vote, SignatureBytes). fn deserialize_vote_record(vote_record: &VoteRecord) -> Option<(Vote, SignatureBytes)> { let msg = match deserialize_boxed(vote_record.data.as_slice()) { @@ -1450,31 +1208,4 @@ impl SessionStartupRecoveryProcessor { } } } - - /// Extract root_hash, file_hash, and collated_data_hash from CandidateHashData. - fn extract_hashes_from_candidate_hash_data( - data: &CandidateHashData, - ) -> Option<(BlockHash, BlockHash, BlockHash)> { - match data { - CandidateHashData::Consensus_CandidateHashDataOrdinary(ordinary) => { - let root_hash = UInt256::from_slice(ordinary.block.root_hash.as_slice()); - let file_hash = UInt256::from_slice(ordinary.block.file_hash.as_slice()); - let collated_data_hash = - UInt256::from_slice(ordinary.collated_file_hash.as_slice()); - Some((root_hash, file_hash, collated_data_hash)) - } - CandidateHashData::Consensus_CandidateHashDataEmpty(empty) => { - // Empty blocks don't have collated_data_hash - let root_hash = UInt256::from_slice(empty.block.root_hash.as_slice()); - let file_hash = UInt256::from_slice(empty.block.file_hash.as_slice()); - let collated_data_hash = UInt256::default(); - Some((root_hash, file_hash, collated_data_hash)) - } - } - } - - /// Serialize CandidateHashData to TL bytes. - fn serialize_candidate_hash_data(data: &CandidateHashData) -> Vec { - serialize_boxed(data).map(|v| v.into()).unwrap_or_default() - } } diff --git a/src/node/simplex/src/tests/test_candidate_resolver.rs b/src/node/simplex/src/tests/test_candidate_resolver.rs index 13e47b7..9e39f65 100644 --- a/src/node/simplex/src/tests/test_candidate_resolver.rs +++ b/src/node/simplex/src/tests/test_candidate_resolver.rs @@ -237,9 +237,12 @@ fn test_merge_candidate_response_parts_body_then_notar_completes_merge() { start_time: SystemTime::now(), retry_count: 0, current_timeout: Duration::from_millis(500), + attempt_id: 0, + in_flight: false, source_idx: ValidatorIndex::new(0), cached_notar: None, cached_candidate: None, + giveup_reports: 0, }; // First partial response: candidate body only -> notar remains missing. @@ -288,9 +291,12 @@ fn test_merge_candidate_response_parts_uses_locally_cached_notar() { start_time: SystemTime::now(), retry_count: 0, current_timeout: Duration::from_millis(500), + attempt_id: 0, + in_flight: false, source_idx: ValidatorIndex::new(1), cached_notar: None, cached_candidate: None, + giveup_reports: 0, }; // No notar in this response, but resolver cache already has one. @@ -308,3 +314,64 @@ fn test_merge_candidate_response_parts_uses_locally_cached_notar() { "candidate-only response should complete when notar already exists in local cache" ); } + +#[test] +fn test_merge_candidate_response_parts_notar_then_body_completes_merge() { + let slot = SlotIndex::new(99); + let block_hash = UInt256::rand(); + let candidate_bytes = vec![5, 4, 3, 2, 1]; + let notar_bytes = vec![9, 9, 9]; + + let mut cache = super::CandidateResolverCache::new(); + let mut state = super::CandidateRequestState { + start_time: SystemTime::now(), + retry_count: 0, + current_timeout: Duration::from_millis(500), + attempt_id: 0, + in_flight: false, + source_idx: ValidatorIndex::new(1), + cached_notar: None, + cached_candidate: None, + giveup_reports: 0, + }; + + // First partial response: notar only. + let (merged_candidate_1, merged_notar_1) = super::ReceiverImpl::merge_candidate_response_parts( + &mut cache, + Some(&mut state), + slot, + &block_hash, + &[], + ¬ar_bytes, + ); + assert!( + merged_candidate_1.is_empty(), + "notar-only partial response must not be considered complete" + ); + assert_eq!(merged_notar_1, notar_bytes); + + // Second partial response: candidate only, merged result should include cached notar. + let (merged_candidate_2, merged_notar_2) = super::ReceiverImpl::merge_candidate_response_parts( + &mut cache, + Some(&mut state), + slot, + &block_hash, + &candidate_bytes, + &[], + ); + assert_eq!(merged_candidate_2, candidate_bytes); + assert_eq!(merged_notar_2, notar_bytes); +} + +#[test] +fn test_sliding_window_rate_limiter_enforces_window_limit() { + let mut limiter = super::SlidingWindowRateLimiter::default(); + let window = Duration::from_secs(1); + let now = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000); + + assert!(limiter.allow(now, window, 2)); + assert!(limiter.allow(now, window, 2)); + assert!(!limiter.allow(now, window, 2)); + assert!(limiter.allow(now + Duration::from_millis(1_001), window, 2)); + assert!(!limiter.allow(now + Duration::from_millis(1_001), window, 0)); +} diff --git a/src/node/simplex/src/tests/test_database.rs b/src/node/simplex/src/tests/test_database.rs index 1187f8c..061e8a5 100644 --- a/src/node/simplex/src/tests/test_database.rs +++ b/src/node/simplex/src/tests/test_database.rs @@ -198,6 +198,49 @@ fn test_save_finalized_block_with_parent() { db.mark_for_destroy(); } +#[test] +fn test_load_finalized_blocks_keeps_empty_mc_immediate_parent_chain() { + let (_db_root, db) = create_test_db("test_load_finalized_blocks_keeps_empty_mc_chain"); + + let c1 = create_candidate_id(1, 0xA1); + let c2 = create_candidate_id(2, 0xA2); + let c3 = create_candidate_id(3, 0xA3); + + db.save_finalized_block(&FinalizedBlockRecord { + candidate_id: c1.clone(), + block_id: create_block_id(1000), + parent: None, + is_final: true, + }) + .unwrap(); + db.save_finalized_block(&FinalizedBlockRecord { + candidate_id: c2.clone(), + block_id: create_block_id(1000), + parent: Some(c1.clone()), + is_final: true, + }) + .unwrap(); + db.save_finalized_block(&FinalizedBlockRecord { + candidate_id: c3.clone(), + block_id: create_block_id(1001), + parent: Some(c2.clone()), + is_final: true, + }) + .unwrap(); + + db.sync(Some(Duration::from_secs(5))).unwrap(); + + let records = db.load_finalized_blocks().unwrap(); + assert_eq!(records.len(), 3); + assert!(records[0].parent.is_none()); + assert_eq!(records[1].parent.as_ref(), Some(&c1)); + assert_eq!(records[1].block_id.seq_no, 1000); + assert_eq!(records[2].parent.as_ref(), Some(&c2)); + assert_eq!(records[2].block_id.seq_no, 1001); + + db.mark_for_destroy(); +} + #[test] fn test_load_multiple_finalized_blocks_sorted() { let (_db_root, db) = create_test_db("test_load_multiple_finalized_blocks_sorted"); diff --git a/src/node/simplex/src/tests/test_receiver.rs b/src/node/simplex/src/tests/test_receiver.rs index 9b58db2..4e1d554 100644 --- a/src/node/simplex/src/tests/test_receiver.rs +++ b/src/node/simplex/src/tests/test_receiver.rs @@ -112,6 +112,10 @@ struct ReceiverStats { active_weight_updates: AtomicU32, /// Last active weight value last_active_weight: AtomicU64, + /// Number of standstill trigger notifications received + standstill_triggers: AtomicU32, + /// Latest standstill trigger notifications + standstill_notifications: Mutex>, /// Receiver index for logging receiver_idx: u32, } @@ -125,6 +129,8 @@ impl ReceiverStats { received_certificates: Mutex::new(Vec::new()), active_weight_updates: AtomicU32::new(0), last_active_weight: AtomicU64::new(0), + standstill_triggers: AtomicU32::new(0), + standstill_notifications: Mutex::new(Vec::new()), receiver_idx, } } @@ -181,6 +187,11 @@ impl ReceiverListener for TestReceiverListener { ); } + fn on_standstill_trigger(&self, notification: crate::receiver::StandstillTriggerNotification) { + self.stats.standstill_triggers.fetch_add(1, Ordering::Relaxed); + self.stats.standstill_notifications.lock().unwrap().push(notification); + } + fn on_certificate(&self, source_idx: u32, certificate: CertificateBoxed) { let count = self.stats.certificates_received.fetch_add(1, Ordering::Relaxed) + 1; self.stats.received_certificates.lock().unwrap().push(certificate.clone()); @@ -198,6 +209,7 @@ impl ReceiverListener for TestReceiverListener { &self, _slot: crate::block::SlotIndex, _block_hash: UInt256, + _want_candidate: bool, _want_notar: bool, response_callback: consensus_common::QueryResponseCallback, ) { @@ -259,9 +271,13 @@ impl ReceiverInstance { overlay_manager, listener_weak, Duration::from_secs(10), // standstill_timeout + 50 << 17, // standstill_max_egress_bytes_per_s + 1, // slots_per_leader_window + 250, // max_leader_window_desync panicked_flag, false, health_counters, + crate::receiver::CandidateResolveConfig::default(), )?; Ok(Self { idx, receiver, stats, _listener: listener, private_key, session_id }) @@ -479,7 +495,7 @@ fn run_receiver_test( // Generate session ID let mut rng = rand::thread_rng(); - let session_id: UInt256 = UInt256::from(rng.gen::<[u8; 32]>()); + let session_id: UInt256 = UInt256::from(rng.r#gen::<[u8; 32]>()); log::info!("Session ID: {}", session_id.to_hex_string()); @@ -693,9 +709,13 @@ fn test_receiver_candidate_resolver() { overlay_manager.clone(), Arc::downgrade(&listener0_arc), Duration::from_secs(10), + 50 << 17, + 1, + 250, panicked_flag0, false, health_counters0, + crate::receiver::CandidateResolveConfig::default(), ) .expect("Failed to create receiver 0"); @@ -783,9 +803,13 @@ fn test_receiver_candidate_resolver() { overlay_manager.clone(), Arc::downgrade(&listener1_arc), Duration::from_secs(10), + 50 << 17, + 1, + 250, panicked_flag1, false, health_counters1, + crate::receiver::CandidateResolveConfig::default(), ) .expect("Failed to create receiver 1"); @@ -804,9 +828,13 @@ fn test_receiver_candidate_resolver() { overlay_manager.clone(), Arc::downgrade(&listener2_arc), Duration::from_secs(10), + 50 << 17, + 1, + 250, panicked_flag2, false, health_counters2, + crate::receiver::CandidateResolveConfig::default(), ) .expect("Failed to create receiver 2"); @@ -856,7 +884,9 @@ fn test_receiver_candidate_resolver() { r2_broadcasts ); - println!("✓ Candidate resolver test passed: late-joining receivers successfully retrieved missed candidate"); + println!( + "✓ Candidate resolver test passed: late-joining receivers successfully retrieved missed candidate" + ); } /// Test that candidate resolver works with a large candidate payload (~1 MB) @@ -900,9 +930,13 @@ fn test_receiver_candidate_resolver_large_payload() { overlay_manager.clone(), Arc::downgrade(&listener0_arc), Duration::from_secs(10), + 50 << 17, + 1, + 250, Arc::new(AtomicBool::new(false)), false, Arc::new(crate::receiver::ReceiverHealthCounters::new()), + crate::receiver::CandidateResolveConfig::default(), ) .expect("Failed to create receiver 0"); @@ -972,9 +1006,13 @@ fn test_receiver_candidate_resolver_large_payload() { overlay_manager.clone(), Arc::downgrade(&listener1_arc), Duration::from_secs(10), + 50 << 17, + 1, + 250, Arc::new(AtomicBool::new(false)), false, Arc::new(crate::receiver::ReceiverHealthCounters::new()), + crate::receiver::CandidateResolveConfig::default(), ) .expect("Failed to create receiver 1"); @@ -1003,7 +1041,9 @@ fn test_receiver_candidate_resolver_large_payload() { r1_broadcasts ); - println!("✓ Large candidate resolver test passed: ~1 MB candidate successfully retrieved via RLDP path"); + println!( + "✓ Large candidate resolver test passed: ~1 MB candidate successfully retrieved via RLDP path" + ); } // ============================================================================ @@ -1096,9 +1136,13 @@ fn test_receiver_send_certificate_and_standstill_rebroadcasts_cached_certificate overlay_manager.clone(), Arc::downgrade(&listener0_arc), Duration::from_millis(200), + 50 << 17, + 1, + 250, Arc::new(AtomicBool::new(false)), false, Arc::new(crate::receiver::ReceiverHealthCounters::new()), + crate::receiver::CandidateResolveConfig::default(), ) .expect("Failed to create receiver 0"); @@ -1115,9 +1159,13 @@ fn test_receiver_send_certificate_and_standstill_rebroadcasts_cached_certificate overlay_manager.clone(), Arc::downgrade(&listener1_arc), Duration::from_millis(200), + 50 << 17, + 1, + 250, Arc::new(AtomicBool::new(false)), false, Arc::new(crate::receiver::ReceiverHealthCounters::new()), + crate::receiver::CandidateResolveConfig::default(), ) .expect("Failed to create receiver 1"); @@ -1203,9 +1251,13 @@ fn test_receiver_standstill_rebroadcasts_cached_local_votes() { overlay_manager.clone(), Arc::downgrade(&listener0_arc), Duration::from_millis(200), + 50 << 17, + 1, + 250, Arc::new(AtomicBool::new(false)), false, Arc::new(crate::receiver::ReceiverHealthCounters::new()), + crate::receiver::CandidateResolveConfig::default(), ) .expect("Failed to create receiver 0"); @@ -1222,9 +1274,13 @@ fn test_receiver_standstill_rebroadcasts_cached_local_votes() { overlay_manager.clone(), Arc::downgrade(&listener1_arc), Duration::from_millis(200), + 50 << 17, + 1, + 250, Arc::new(AtomicBool::new(false)), false, Arc::new(crate::receiver::ReceiverHealthCounters::new()), + crate::receiver::CandidateResolveConfig::default(), ) .expect("Failed to create receiver 1"); @@ -1256,6 +1312,319 @@ fn test_receiver_standstill_rebroadcasts_cached_local_votes() { ); } +#[test] +fn test_receiver_standstill_replay_respects_egress_budget() { + let _ = env_logger::Builder::new().filter_level(log::LevelFilter::Trace).try_init(); + + let overlay_manager = SessionFactory::create_in_process_overlay_manager(2); + let session_id = UInt256::rand(); + + let keys: Vec<_> = + (0..2).map(|_| Ed25519KeyOption::generate().expect("Failed to generate key")).collect(); + let nodes: Vec = keys + .iter() + .map(|k| SessionNode { public_key: k.clone(), adnl_id: k.id().clone(), weight: 1 }) + .collect(); + + let cert = make_skip_certificate(5); + let cert_bytes = serialize_boxed(&cert).expect("serialize cert"); + let low_egress_budget = (cert_bytes.len() as u32).max(1); // bytes/sec + + let shard = ShardIdent::masterchain(); + let max_candidate_size = 8 << 20; + let max_candidate_query_answer_size: u64 = max_candidate_size as u64 + (1 << 20); + + let (listener0, _stats0) = TestReceiverListener::create(0); + let listener0_arc: Arc = listener0.clone(); + let receiver0 = crate::receiver::ReceiverWrapper::create( + session_id.clone(), + &shard, + max_candidate_size, + max_candidate_query_answer_size, + 0, + &nodes, + &keys[0], + overlay_manager.clone(), + Arc::downgrade(&listener0_arc), + Duration::from_millis(200), + low_egress_budget, + 1, + 250, + Arc::new(AtomicBool::new(false)), + false, + Arc::new(crate::receiver::ReceiverHealthCounters::new()), + crate::receiver::CandidateResolveConfig::default(), + ) + .expect("Failed to create receiver 0"); + + let (listener1, stats1) = TestReceiverListener::create(1); + let listener1_arc: Arc = listener1.clone(); + let receiver1 = crate::receiver::ReceiverWrapper::create( + session_id.clone(), + &shard, + max_candidate_size, + max_candidate_query_answer_size, + 0, + &nodes, + &keys[1], + overlay_manager.clone(), + Arc::downgrade(&listener1_arc), + Duration::from_millis(200), + low_egress_budget, + 1, + 250, + Arc::new(AtomicBool::new(false)), + false, + Arc::new(crate::receiver::ReceiverHealthCounters::new()), + crate::receiver::CandidateResolveConfig::default(), + ) + .expect("Failed to create receiver 1"); + + thread::sleep(Duration::from_millis(500)); + + // Cache one cert for standstill replay. + receiver0.cache_standstill_certificate( + 5, + crate::receiver::StandstillCertificateType::Skip, + cert_bytes, + ); + receiver0.set_standstill_slots(0, 10); + receiver0.reschedule_standstill(); + + // Budget is intentionally low: replay should not burst immediately. + thread::sleep(Duration::from_millis(500)); + assert_eq!( + stats1.certificates_received.load(Ordering::Relaxed), + 0, + "standstill replay should be paced by egress budget, not sent in the first 500ms" + ); + + wait_until(Duration::from_secs(8), || { + stats1.certificates_received.load(Ordering::Relaxed) >= 1 + }); + + receiver0.stop(); + receiver1.stop(); + thread::sleep(Duration::from_millis(200)); +} + +#[test] +fn test_receiver_standstill_rebuilds_pending_queue_from_fresh_state() { + let _ = env_logger::Builder::new().filter_level(log::LevelFilter::Trace).try_init(); + + let overlay_manager = SessionFactory::create_in_process_overlay_manager(2); + let session_id = UInt256::rand(); + + let keys: Vec<_> = + (0..2).map(|_| Ed25519KeyOption::generate().expect("Failed to generate key")).collect(); + let nodes: Vec = keys + .iter() + .map(|k| SessionNode { public_key: k.clone(), adnl_id: k.id().clone(), weight: 1 }) + .collect(); + + let vote = crate::simplex_state::Vote::Skip(crate::simplex_state::SkipVote { + slot: crate::block::SlotIndex::new(3), + }); + let tl_vote = crate::utils::sign_vote(&vote, &session_id, &keys[0]).expect("sign_vote failed"); + let vote_bytes = serialize_boxed(&tl_vote).expect("serialize vote"); + let signed_vote = match tl_vote { + TlVoteBoxed::Consensus_Simplex_Vote(inner) => inner, + }; + + let cert = make_skip_certificate(3); + let cert_bytes = serialize_boxed(&cert).expect("serialize cert"); + let low_egress_budget = (vote_bytes.len().max(cert_bytes.len()) as u32).max(1); + + let shard = ShardIdent::masterchain(); + let max_candidate_size = 8 << 20; + let max_candidate_query_answer_size: u64 = max_candidate_size as u64 + (1 << 20); + + let (listener0, stats0) = TestReceiverListener::create(0); + let listener0_arc: Arc = listener0.clone(); + let receiver0 = crate::receiver::ReceiverWrapper::create( + session_id.clone(), + &shard, + max_candidate_size, + max_candidate_query_answer_size, + 0, + &nodes, + &keys[0], + overlay_manager.clone(), + Arc::downgrade(&listener0_arc), + Duration::from_millis(200), + low_egress_budget, + 1, + 250, + Arc::new(AtomicBool::new(false)), + false, + Arc::new(crate::receiver::ReceiverHealthCounters::new()), + crate::receiver::CandidateResolveConfig::default(), + ) + .expect("Failed to create receiver 0"); + + let (listener1, stats1) = TestReceiverListener::create(1); + let listener1_arc: Arc = listener1.clone(); + let receiver1 = crate::receiver::ReceiverWrapper::create( + session_id.clone(), + &shard, + max_candidate_size, + max_candidate_query_answer_size, + 0, + &nodes, + &keys[1], + overlay_manager.clone(), + Arc::downgrade(&listener1_arc), + Duration::from_millis(200), + low_egress_budget, + 1, + 250, + Arc::new(AtomicBool::new(false)), + false, + Arc::new(crate::receiver::ReceiverHealthCounters::new()), + crate::receiver::CandidateResolveConfig::default(), + ) + .expect("Failed to create receiver 1"); + + thread::sleep(Duration::from_millis(500)); + + receiver0.cache_our_vote_for_standstill(signed_vote); + receiver0.set_standstill_slots(0, 10); + receiver0.reschedule_standstill(); + + wait_until(Duration::from_secs(2), || stats0.standstill_triggers.load(Ordering::Relaxed) >= 1); + + assert_eq!( + stats1.votes_received.load(Ordering::Relaxed), + 0, + "low egress budget should keep the first standstill snapshot pending" + ); + assert_eq!( + stats1.certificates_received.load(Ordering::Relaxed), + 0, + "no certificate should be replayed before the snapshot is refreshed" + ); + + receiver0.cache_standstill_certificate( + 3, + crate::receiver::StandstillCertificateType::Skip, + cert_bytes, + ); + + wait_until(Duration::from_secs(3), || stats0.standstill_triggers.load(Ordering::Relaxed) >= 2); + wait_until(Duration::from_secs(6), || { + stats1.certificates_received.load(Ordering::Relaxed) >= 1 + }); + + receiver0.stop(); + receiver1.stop(); + thread::sleep(Duration::from_millis(200)); + + assert_eq!( + stats1.votes_received.load(Ordering::Relaxed), + 0, + "fresh standstill snapshot must drop the stale queued vote once a matching cert is cached" + ); + assert!( + stats1.certificates_received.load(Ordering::Relaxed) >= 1, + "expected the refreshed standstill snapshot to replay the new certificate" + ); +} + +#[test] +fn test_receiver_standstill_cache_deduplicates_equivalent_local_votes() { + let _ = env_logger::Builder::new().filter_level(log::LevelFilter::Trace).try_init(); + + let overlay_manager = SessionFactory::create_in_process_overlay_manager(2); + let session_id = UInt256::rand(); + + let keys: Vec<_> = + (0..2).map(|_| Ed25519KeyOption::generate().expect("Failed to generate key")).collect(); + let nodes: Vec = keys + .iter() + .map(|k| SessionNode { public_key: k.clone(), adnl_id: k.id().clone(), weight: 1 }) + .collect(); + + let shard = ShardIdent::masterchain(); + let max_candidate_size = 8 << 20; + let max_candidate_query_answer_size: u64 = max_candidate_size as u64 + (1 << 20); + + let (listener0, _stats0) = TestReceiverListener::create(0); + let listener0_arc: Arc = listener0.clone(); + let receiver0 = crate::receiver::ReceiverWrapper::create( + session_id.clone(), + &shard, + max_candidate_size, + max_candidate_query_answer_size, + 0, + &nodes, + &keys[0], + overlay_manager.clone(), + Arc::downgrade(&listener0_arc), + Duration::from_millis(600), + 50 << 17, + 1, + 250, + Arc::new(AtomicBool::new(false)), + false, + Arc::new(crate::receiver::ReceiverHealthCounters::new()), + crate::receiver::CandidateResolveConfig::default(), + ) + .expect("Failed to create receiver 0"); + + let (listener1, stats1) = TestReceiverListener::create(1); + let listener1_arc: Arc = listener1.clone(); + let receiver1 = crate::receiver::ReceiverWrapper::create( + session_id.clone(), + &shard, + max_candidate_size, + max_candidate_query_answer_size, + 0, + &nodes, + &keys[1], + overlay_manager.clone(), + Arc::downgrade(&listener1_arc), + Duration::from_millis(600), + 50 << 17, + 1, + 250, + Arc::new(AtomicBool::new(false)), + false, + Arc::new(crate::receiver::ReceiverHealthCounters::new()), + crate::receiver::CandidateResolveConfig::default(), + ) + .expect("Failed to create receiver 1"); + + thread::sleep(Duration::from_millis(500)); + + let vote = crate::simplex_state::Vote::Skip(crate::simplex_state::SkipVote { + slot: crate::block::SlotIndex::new(7), + }); + let tl_vote = crate::utils::sign_vote(&vote, &session_id, &keys[0]).expect("sign_vote failed"); + let signed = match tl_vote { + TlVoteBoxed::Consensus_Simplex_Vote(inner) => inner, + }; + + // Cache the same signed vote twice (restart-restored + local overlap). + receiver0.cache_our_vote_for_standstill(signed.clone()); + receiver0.cache_our_vote_for_standstill(signed); + + receiver0.set_standstill_slots(0, 10); + receiver0.reschedule_standstill(); + + wait_until(Duration::from_secs(2), || stats1.votes_received.load(Ordering::Relaxed) >= 1); + thread::sleep(Duration::from_millis(150)); + + receiver0.stop(); + receiver1.stop(); + thread::sleep(Duration::from_millis(200)); + + assert_eq!( + stats1.votes_received.load(Ordering::Relaxed), + 1, + "equivalent local votes should be replayed once during standstill" + ); +} + #[test] fn test_receiver_standstill_cache_does_not_overwrite_existing_certificate() { let _ = env_logger::Builder::new().filter_level(log::LevelFilter::Trace).try_init(); @@ -1287,9 +1656,13 @@ fn test_receiver_standstill_cache_does_not_overwrite_existing_certificate() { overlay_manager.clone(), Arc::downgrade(&listener0_arc), Duration::from_millis(200), + 50 << 17, + 1, + 250, Arc::new(AtomicBool::new(false)), false, Arc::new(crate::receiver::ReceiverHealthCounters::new()), + crate::receiver::CandidateResolveConfig::default(), ) .expect("Failed to create receiver 0"); @@ -1306,9 +1679,13 @@ fn test_receiver_standstill_cache_does_not_overwrite_existing_certificate() { overlay_manager.clone(), Arc::downgrade(&listener1_arc), Duration::from_millis(200), + 50 << 17, + 1, + 250, Arc::new(AtomicBool::new(false)), false, Arc::new(crate::receiver::ReceiverHealthCounters::new()), + crate::receiver::CandidateResolveConfig::default(), ) .expect("Failed to create receiver 1"); diff --git a/src/node/simplex/src/tests/test_restart.rs b/src/node/simplex/src/tests/test_restart.rs index e4439ba..4511342 100644 --- a/src/node/simplex/src/tests/test_restart.rs +++ b/src/node/simplex/src/tests/test_restart.rs @@ -8,11 +8,10 @@ */ //! Simplex restart and recovery unit tests //! -//! Tests for startup recovery, restart recommit, and related functionality. +//! Tests for startup recovery and related functionality. //! //! ## Test Categories //! -//! - `RestartRecommitStrategy` defaults //! - `SessionStartupRecoveryOptions` construction //! - Future: `SessionStartupRecoveryProcessor` tests with mock listener @@ -26,11 +25,11 @@ use crate::{ session_description::SessionDescription, simplex_state::{NotarizeVote, Vote}, startup_recovery::{ - RestartRoundAction, SessionStartupRecoveryListener, SessionStartupRecoveryOptions, + SessionStartupRecoveryListener, SessionStartupRecoveryOptions, SessionStartupRecoveryProcessor, }, utils::sign_vote, - RawBuffer, RestartRecommitStrategy, SessionId, SessionNode, SessionOptions, + SessionId, SessionNode, SessionOptions, }; use std::{sync::Arc, time::SystemTime}; use ton_api::{ @@ -47,30 +46,16 @@ use ton_api::{ IntoBoxed, }; use ton_block::{ - sha256_digest, BlockIdExt, BocFlags, BocWriter, BuilderData, Ed25519KeyOption, Result, - ShardIdent, UInt256, + sha256_digest, BlockIdExt, BocFlags, BocWriter, BuilderData, Ed25519KeyOption, ShardIdent, + UInt256, }; -#[test] -fn test_restart_recommit_strategy_default() { - assert_eq!(RestartRecommitStrategy::default(), RestartRecommitStrategy::FullReplay); -} - #[test] fn test_session_startup_recovery_options_new() { - let opts = - SessionStartupRecoveryOptions::new(RestartRecommitStrategy::FirstCommitAfterFinalized, 100); - assert_eq!(opts.restart_recommit_strategy, RestartRecommitStrategy::FirstCommitAfterFinalized); + let opts = SessionStartupRecoveryOptions::new(100); assert_eq!(opts.initial_block_seqno, 100); } -#[test] -fn test_session_startup_recovery_options_default_strategy() { - let opts = SessionStartupRecoveryOptions::new(RestartRecommitStrategy::default(), 1); - assert_eq!(opts.restart_recommit_strategy, RestartRecommitStrategy::FullReplay); - assert_eq!(opts.initial_block_seqno, 1); -} - // ============================================================================ // Test helpers // ============================================================================ @@ -115,7 +100,7 @@ fn create_test_desc_with_validators(count: usize, local_idx: usize) -> Arc RawCandidateId { @@ -133,18 +118,6 @@ fn make_block_id(seqno: u32) -> BlockIdExt { } } -fn make_candidate_hash_data( - root_hash: UInt256, - file_hash: UInt256, - collated_file_hash: UInt256, -) -> CandidateHashData { - CandidateHashData::Consensus_CandidateHashDataOrdinary(CandidateHashDataOrdinary { - block: BlockIdExt { shard_id: ShardIdent::masterchain(), seq_no: 0, root_hash, file_hash }, - collated_file_hash: collated_file_hash.into(), - parent: CandidateParent::Consensus_CandidateWithoutParents, - }) -} - fn make_candidate_hash_data_with_parent( block_id: BlockIdExt, collated_file_hash: UInt256, @@ -202,321 +175,6 @@ fn make_validator_session_candidate_bytes( consensus_common::serialize_tl_boxed_object!(&tl_candidate.into_boxed()) } -#[test] -fn test_restart_recommit_strategy_first_commit_after_finalized_builds_no_actions() { - let session_id = SessionId::default(); - let options = - SessionStartupRecoveryOptions::new(RestartRecommitStrategy::FirstCommitAfterFinalized, 100); - - let bootstrap = Bootstrap::default(); // contents unused for this strategy - let desc = create_test_desc(); - let proc = SessionStartupRecoveryProcessor::new(session_id, desc, options, bootstrap); - - let finalized_blocks = vec![ - FinalizedBlockRecord { - candidate_id: make_candidate_id(1, 0xA1), - block_id: make_block_id(100), - parent: None, - is_final: true, - }, - FinalizedBlockRecord { - candidate_id: make_candidate_id(2, 0xA2), - block_id: make_block_id(101), - parent: None, - is_final: true, - }, - ]; - - let actions = proc.build_restart_recommit_actions(&finalized_blocks).expect("build actions"); - assert!(actions.is_empty(), "FirstCommitAfterFinalized should not replay history"); -} - -#[test] -fn test_restart_recommit_strategy_full_replay_builds_commit_actions_for_sequential_seqno() { - let session_id = SessionId::default(); - let options = SessionStartupRecoveryOptions::new(RestartRecommitStrategy::FullReplay, 100); - - let c1 = make_candidate_id(1, 0xA1); - let c2 = make_candidate_id(2, 0xA2); - let c3 = make_candidate_id(3, 0xA3); - - let finalized_blocks = vec![ - FinalizedBlockRecord { - candidate_id: c1.clone(), - block_id: make_block_id(100), - parent: None, - is_final: true, - }, - FinalizedBlockRecord { - candidate_id: c2.clone(), - block_id: make_block_id(101), - parent: Some(c1.clone()), - is_final: true, - }, - FinalizedBlockRecord { - candidate_id: c3.clone(), - block_id: make_block_id(102), - parent: Some(c2.clone()), - is_final: true, - }, - ]; - - let candidate_infos = vec![ - CandidateInfoRecord { - candidate_id: c1.clone(), - leader_idx: 1, - candidate_hash_data: make_candidate_hash_data( - UInt256::from([1u8; 32]), - UInt256::from([2u8; 32]), - UInt256::from([3u8; 32]), - ), - signature: RawBuffer::default(), - }, - CandidateInfoRecord { - candidate_id: c2.clone(), - leader_idx: 2, - candidate_hash_data: make_candidate_hash_data( - UInt256::from([4u8; 32]), - UInt256::from([5u8; 32]), - UInt256::from([6u8; 32]), - ), - signature: RawBuffer::default(), - }, - CandidateInfoRecord { - candidate_id: c3.clone(), - leader_idx: 3, - candidate_hash_data: make_candidate_hash_data( - UInt256::from([7u8; 32]), - UInt256::from([8u8; 32]), - UInt256::from([9u8; 32]), - ), - signature: RawBuffer::default(), - }, - ]; - - let bootstrap = Bootstrap { candidate_infos, ..Bootstrap::default() }; - let desc = create_test_desc(); - let proc = SessionStartupRecoveryProcessor::new(session_id, desc, options, bootstrap); - - let actions = proc.build_restart_recommit_actions(&finalized_blocks).expect("build actions"); - assert_eq!(actions.len(), 3); - - // All actions should be Commit in slot order - for (i, action) in actions.iter().enumerate() { - let expected_slot = SlotIndex::new((i + 1) as u32); - match action { - RestartRoundAction::Commit { slot, block_id, leader_idx, is_empty, .. } => { - assert_eq!(*slot, expected_slot); - assert_eq!(block_id.seq_no(), 100 + i as u32); - assert_eq!(leader_idx.value(), (i + 1) as u32); - assert!(!is_empty); - } - } - } -} - -#[test] -fn test_restart_recommit_full_replay_replays_empty_record_without_skip_action() { - let session_id = SessionId::default(); - let options = SessionStartupRecoveryOptions::new(RestartRecommitStrategy::FullReplay, 10); - - let c1 = make_candidate_id(1, 0xA1); - let c2 = make_candidate_id(2, 0xA2); - let c3 = make_candidate_id(3, 0xA3); - let b1 = make_block_id(10); - let b2 = make_block_id(10); // empty record keeps parent seqno - let b3 = make_block_id(11); - - let finalized_blocks = vec![ - FinalizedBlockRecord { - candidate_id: c1.clone(), - block_id: b1.clone(), - parent: None, - is_final: true, - }, - FinalizedBlockRecord { - candidate_id: c2.clone(), - block_id: b2.clone(), - parent: Some(c1.clone()), - is_final: true, - }, - FinalizedBlockRecord { - candidate_id: c3.clone(), - block_id: b3.clone(), - parent: Some(c2.clone()), - is_final: true, - }, - ]; - - let bootstrap = Bootstrap { - candidate_infos: vec![ - CandidateInfoRecord { - candidate_id: c1.clone(), - leader_idx: 0, - candidate_hash_data: make_candidate_hash_data_with_parent( - b1, - UInt256::from([0x11; 32]), - None, - ), - signature: RawBuffer::default(), - }, - CandidateInfoRecord { - candidate_id: c2.clone(), - leader_idx: 0, - candidate_hash_data: make_candidate_hash_data_empty(b2, c1.clone()), - signature: RawBuffer::default(), - }, - CandidateInfoRecord { - candidate_id: c3.clone(), - leader_idx: 0, - candidate_hash_data: make_candidate_hash_data_with_parent( - b3, - UInt256::from([0x33; 32]), - Some(c2.clone()), - ), - signature: RawBuffer::default(), - }, - ], - ..Bootstrap::default() - }; - - let desc = create_test_desc(); - let proc = SessionStartupRecoveryProcessor::new(session_id, desc, options, bootstrap); - let actions = proc.build_restart_recommit_actions(&finalized_blocks).expect("build actions"); - assert_eq!(actions.len(), 3); - - match &actions[0] { - RestartRoundAction::Commit { slot, block_id, is_empty, .. } => { - assert_eq!(*slot, SlotIndex::new(1)); - assert_eq!(block_id.seq_no(), 10); - assert!(!is_empty); - } - } - match &actions[1] { - RestartRoundAction::Commit { slot, block_id, is_empty, .. } => { - assert_eq!(*slot, SlotIndex::new(2)); - assert_eq!(block_id.seq_no(), 10); - assert!(*is_empty); - } - } - match &actions[2] { - RestartRoundAction::Commit { slot, block_id, is_empty, .. } => { - assert_eq!(*slot, SlotIndex::new(3)); - assert_eq!(block_id.seq_no(), 11); - assert!(!is_empty); - } - } -} - -#[test] -fn test_restart_recommit_full_replay_fails_on_missing_candidate_info() { - let session_id = SessionId::default(); - let options = SessionStartupRecoveryOptions::new(RestartRecommitStrategy::FullReplay, 1); - - let c1 = make_candidate_id(1, 0xA1); - let finalized_blocks = vec![FinalizedBlockRecord { - candidate_id: c1, - block_id: make_block_id(1), - parent: None, - is_final: true, - }]; - - let bootstrap = Bootstrap::default(); // no candidate_infos - let desc = create_test_desc(); - let proc = SessionStartupRecoveryProcessor::new(session_id, desc, options, bootstrap); - - let err = proc.build_restart_recommit_actions(&finalized_blocks).expect_err("must fail"); - assert!(err.to_string().contains("missing candidate info"), "unexpected error: {}", err); -} - -#[test] -fn test_restart_recommit_full_replay_fails_on_seqno_mismatch_ahead() { - let session_id = SessionId::default(); - let options = SessionStartupRecoveryOptions::new(RestartRecommitStrategy::FullReplay, 10); - - let c1 = make_candidate_id(1, 0xA1); - let finalized_blocks = vec![FinalizedBlockRecord { - candidate_id: c1.clone(), - block_id: make_block_id(12), // expected 10 - parent: None, - is_final: true, - }]; - - let bootstrap = Bootstrap { - candidate_infos: vec![CandidateInfoRecord { - candidate_id: c1, - leader_idx: 0, - candidate_hash_data: make_candidate_hash_data( - UInt256::default(), - UInt256::default(), - UInt256::default(), - ), - signature: RawBuffer::default(), - }], - ..Bootstrap::default() - }; - let desc = create_test_desc(); - let proc = SessionStartupRecoveryProcessor::new(session_id, desc, options, bootstrap); - - let err = proc.build_restart_recommit_actions(&finalized_blocks).expect_err("must fail"); - assert!(err.to_string().contains("seqno mismatch"), "unexpected error: {}", err); -} - -#[test] -fn test_restart_recommit_full_replay_fails_on_parent_chain_discontinuity() { - let session_id = SessionId::default(); - let options = SessionStartupRecoveryOptions::new(RestartRecommitStrategy::FullReplay, 1); - - let c1 = make_candidate_id(1, 0xA1); - let c2 = make_candidate_id(2, 0xA2); - let wrong_parent = make_candidate_id(99, 0xFF); - let finalized_blocks = vec![ - FinalizedBlockRecord { - candidate_id: c1.clone(), - block_id: make_block_id(1), - parent: None, - is_final: true, - }, - FinalizedBlockRecord { - candidate_id: c2.clone(), - block_id: make_block_id(2), - parent: Some(wrong_parent), - is_final: true, - }, - ]; - - let bootstrap = Bootstrap { - candidate_infos: vec![ - CandidateInfoRecord { - candidate_id: c1, - leader_idx: 0, - candidate_hash_data: make_candidate_hash_data( - UInt256::default(), - UInt256::default(), - UInt256::default(), - ), - signature: RawBuffer::default(), - }, - CandidateInfoRecord { - candidate_id: c2, - leader_idx: 0, - candidate_hash_data: make_candidate_hash_data( - UInt256::default(), - UInt256::default(), - UInt256::default(), - ), - signature: RawBuffer::default(), - }, - ], - ..Bootstrap::default() - }; - let desc = create_test_desc(); - let proc = SessionStartupRecoveryProcessor::new(session_id, desc, options, bootstrap); - - let err = proc.build_restart_recommit_actions(&finalized_blocks).expect_err("must fail"); - assert!(err.to_string().contains("chain discontinuity"), "unexpected error: {}", err); -} - // ============================================================================ // Startup recovery orchestration (apply_bootstrap) with a mock listener // ============================================================================ @@ -531,12 +189,12 @@ enum RecoveryCall { DrainStartupEvents, SeedCurrentRound, SeedFinalizedBlock, + SeedReceivedCandidates, NotifyLastFinalized, CacheNotarCert, CacheCandidateBytes, RestoreStandstillCache, RestoreStartupVotes, - ApplyRestartRecommitActions, } #[derive(Default)] @@ -553,9 +211,9 @@ struct MockRecoveryListener { restored_standstill_cache_votes_len: Option, drained_votes: Vec, restored_votes: Vec, - recommit_called: bool, seeded_current_round: Option, seeded_finalized_blocks: Vec<(SlotIndex, UInt256)>, + seeded_received_candidates: Vec, last_finalized_notification: Option<(SlotIndex, UInt256, u32)>, } @@ -620,8 +278,9 @@ impl SessionStartupRecoveryListener for MockRecoveryListener { self.seeded_finalized_blocks.push((slot, block_hash)); } - fn recovery_seed_received_candidates(&mut self, _finalized_blocks: &[FinalizedBlockRecord]) { - // Mock: no-op, received_candidates not tracked in tests + fn recovery_seed_received_candidates(&mut self, finalized_blocks: &[FinalizedBlockRecord]) { + self.call_log.push(RecoveryCall::SeedReceivedCandidates); + self.seeded_received_candidates = finalized_blocks.to_vec(); } fn recovery_seed_candidate_for_parent_resolution( @@ -678,19 +337,6 @@ impl SessionStartupRecoveryListener for MockRecoveryListener { self.call_log.push(RecoveryCall::CacheCandidateBytes); self.cached_candidates.push((slot, candidate_hash, candidate_data_bytes)); } - - fn recovery_apply_restart_recommit_actions( - &mut self, - _actions: &[RestartRoundAction], - _get_candidate: &mut dyn FnMut( - &RestartRoundAction, - ) - -> Result, - ) -> Result<()> { - self.call_log.push(RecoveryCall::ApplyRestartRecommitActions); - self.recommit_called = true; - Ok(()) - } } fn make_vote_record( @@ -708,8 +354,7 @@ fn make_vote_record( #[test] fn test_apply_bootstrap_calls_expected_listener_methods_first_commit_strategy() { let session_id = SessionId::default(); - let options = - SessionStartupRecoveryOptions::new(RestartRecommitStrategy::FirstCommitAfterFinalized, 1); + let options = SessionStartupRecoveryOptions::new(1); // Create 2-validator description where self is validator 1 let self_idx = ValidatorIndex::new(1); let desc = create_test_desc_with_validators(2, 1); @@ -851,10 +496,10 @@ fn test_apply_bootstrap_calls_expected_listener_methods_first_commit_strategy() assert_eq!(listener.set_first_nonannounced_window, Some(WindowIndex::new(2))); assert_eq!(listener.generate_skip_calls, 1); - // Step 5 & 12: drain/restore votes + // Step 5 & 11: drain/restore votes assert_eq!(listener.restored_votes, drained_votes); - // Step 6: current_round seeded for restart replay (starts from 0; step 11 advances it) + // Step 6: current_round compatibility hook seeded to 0 assert_eq!(listener.seeded_current_round, Some(0)); // Step 7: finalized_blocks set seeded (2 blocks) @@ -876,16 +521,88 @@ fn test_apply_bootstrap_calls_expected_listener_methods_first_commit_strategy() // Step 10b: standstill cache rebuild invoked with full bootstrap votes slice assert_eq!(listener.restored_standstill_cache_votes_len, Some(2)); - // Strategy FirstCommitAfterFinalized => no historical recommit actions. - assert!(!listener.recommit_called); assert!(listener.cached_candidates.is_empty()); } +#[test] +fn test_apply_bootstrap_seeds_persisted_empty_mc_chain_before_last_finalized_notification() { + let session_id = SessionId::default(); + let options = SessionStartupRecoveryOptions::new(1); + + let c1 = make_candidate_id(10, 0xA1); + let c2 = make_candidate_id(11, 0xA2); + let c3 = make_candidate_id(12, 0xA3); + let finalized_blocks = vec![ + FinalizedBlockRecord { + candidate_id: c1.clone(), + block_id: make_block_id(100), + parent: None, + is_final: true, + }, + FinalizedBlockRecord { + candidate_id: c2.clone(), + block_id: make_block_id(100), + parent: Some(c1.clone()), + is_final: true, + }, + FinalizedBlockRecord { + candidate_id: c3.clone(), + block_id: make_block_id(101), + parent: Some(c2.clone()), + is_final: true, + }, + ]; + + let bootstrap = Bootstrap { + finalized_blocks: finalized_blocks.clone(), + candidate_infos: vec![], + notar_certs: vec![], + votes: vec![], + pool_state: None, + candidate_payloads: vec![], + }; + + let desc = create_test_desc(); + let proc = SessionStartupRecoveryProcessor::new(session_id, desc, options, bootstrap); + let mut listener = MockRecoveryListener::default().with_drained_votes(vec![]); + + proc.apply_bootstrap(&mut listener).expect("apply_bootstrap failed"); + + assert_eq!(listener.seeded_received_candidates.len(), finalized_blocks.len()); + for (seeded, expected) in + listener.seeded_received_candidates.iter().zip(finalized_blocks.iter()) + { + assert_eq!(seeded.candidate_id, expected.candidate_id); + assert_eq!(seeded.block_id, expected.block_id); + assert_eq!(seeded.parent, expected.parent); + assert_eq!(seeded.is_final, expected.is_final); + } + + let seed_pos = listener + .call_log + .iter() + .position(|c| *c == RecoveryCall::SeedReceivedCandidates) + .expect("expected SeedReceivedCandidates call"); + let notify_pos = listener + .call_log + .iter() + .position(|c| *c == RecoveryCall::NotifyLastFinalized) + .expect("expected NotifyLastFinalized call"); + assert!( + seed_pos < notify_pos, + "received-candidate seeding must happen before the last-finalized notification" + ); + assert_eq!( + listener.last_finalized_notification, + Some((c3.slot, c3.hash, 101)), + "the last-finalized notification must still target the newest final record" + ); +} + #[test] fn test_apply_bootstrap_does_not_generate_skip_votes_when_first_nonannounced_window_zero() { let session_id = SessionId::default(); - let options = - SessionStartupRecoveryOptions::new(RestartRecommitStrategy::FirstCommitAfterFinalized, 1); + let options = SessionStartupRecoveryOptions::new(1); let bootstrap = Bootstrap { finalized_blocks: vec![FinalizedBlockRecord { @@ -908,7 +625,7 @@ fn test_apply_bootstrap_does_not_generate_skip_votes_when_first_nonannounced_win proc.apply_bootstrap(&mut listener).expect("apply_bootstrap failed"); assert_eq!(listener.set_first_nonannounced_window, Some(WindowIndex::new(0))); assert_eq!(listener.generate_skip_calls, 0); - // Step 6: current_round seeded for restart replay (starts from 0; step 11 advances it) + // Step 6: current_round compatibility hook seeded to 0 assert_eq!(listener.seeded_current_round, Some(0)); // Step 7: finalized_blocks set seeded (1 block) assert_eq!(listener.seeded_finalized_blocks.len(), 1); @@ -1023,27 +740,12 @@ impl SessionStartupRecoveryListener for CandidateCacheListener { ) { self.cached_candidates.push((slot, candidate_hash, candidate_data_bytes)); } - - fn recovery_apply_restart_recommit_actions( - &mut self, - actions: &[RestartRoundAction], - _get_candidate: &mut dyn FnMut( - &RestartRoundAction, - ) - -> Result, - ) -> Result<()> { - // These candidate-cache tests use `FirstCommitAfterFinalized`, which must not - // re-accept historical blocks. - assert!(actions.is_empty(), "FirstCommitAfterFinalized should not replay actions"); - Ok(()) - } } #[test] fn test_restart_restore_candidate_bytes_roundtrip_empty_and_non_empty() { let session_id = SessionId::default(); - let options = - SessionStartupRecoveryOptions::new(RestartRecommitStrategy::FirstCommitAfterFinalized, 0); + let options = SessionStartupRecoveryOptions::new(0); let desc = create_test_desc(); // single validator is enough for this test let shard = ShardIdent::masterchain(); @@ -1183,8 +885,7 @@ fn test_restart_restore_candidate_bytes_roundtrip_empty_and_non_empty() { #[test] fn test_restart_restore_candidate_bytes_skips_non_empty_and_keeps_empty() { let session_id = SessionId::default(); - let options = - SessionStartupRecoveryOptions::new(RestartRecommitStrategy::FirstCommitAfterFinalized, 0); + let options = SessionStartupRecoveryOptions::new(0); let desc = create_test_desc(); let shard = ShardIdent::masterchain(); @@ -1333,16 +1034,6 @@ fn test_restart_restore_candidate_bytes_skips_non_empty_and_keeps_empty() { ) { self.cached.push((slot, candidate_hash, candidate_data_bytes)); } - fn recovery_apply_restart_recommit_actions( - &mut self, - _actions: &[RestartRoundAction], - _get_candidate: &mut dyn FnMut( - &RestartRoundAction, - ) - -> Result, - ) -> Result<()> { - Ok(()) - } } let mut listener = FailFetchListener::default(); diff --git a/src/node/simplex/src/tests/test_session_processor.rs b/src/node/simplex/src/tests/test_session_processor.rs index 5f799b5..6f5f960 100644 --- a/src/node/simplex/src/tests/test_session_processor.rs +++ b/src/node/simplex/src/tests/test_session_processor.rs @@ -92,6 +92,28 @@ fn create_test_desc( create_test_desc_with_opts(nodes, local_idx, &SessionOptions::default()) } +fn create_test_desc_for_shard_with_opts( + nodes: &[SessionNode], + local_idx: usize, + shard: ShardIdent, + opts: &SessionOptions, +) -> Arc { + let local_key = nodes[local_idx].public_key.clone(); + Arc::new( + crate::session_description::SessionDescription::new( + opts, + SessionId::default(), + 1, // initial_block_seqno + nodes, + local_key, + &shard, + SystemTime::now(), + None, // metrics + ) + .unwrap(), + ) +} + // ============================================================================ // Mock Receiver // ============================================================================ @@ -120,6 +142,14 @@ enum ReceiverAction { CacheLastFinalCertificate { slot: u32, bytes_len: usize }, /// cleanup() was called Cleanup { up_to_slot: u32 }, + /// set_ingress_slot_begin() was called + SetIngressSlotBegin { slot: u32 }, + /// set_ingress_progress_slot() was called + SetIngressProgressSlot { slot: u32 }, + /// cancel_candidate_requests_for_slot() was called + CancelCandidateRequestsForSlot { slot: u32 }, + /// set_standstill_slots() was called + SetStandstillSlots { begin: u32, end: u32 }, /// request_candidate() was called RequestCandidate { slot: u32, block_hash: UInt256 }, } @@ -128,11 +158,15 @@ enum ReceiverAction { struct MockReceiver { /// Recorded actions (sent votes, broadcasts, etc.) actions: Arc>>, + last_standstill_slots: Arc>>, } impl MockReceiver { fn new() -> Arc { - Arc::new(Self { actions: Arc::new(Mutex::new(VecDeque::new())) }) + Arc::new(Self { + actions: Arc::new(Mutex::new(VecDeque::new())), + last_standstill_slots: Arc::new(Mutex::new(None)), + }) } /// Get all recorded actions (drains the queue) @@ -178,6 +212,21 @@ impl Receiver for MockReceiver { self.actions.lock().unwrap().push_back(ReceiverAction::Cleanup { up_to_slot }); } + fn set_ingress_slot_begin(&self, slot: u32) { + self.actions.lock().unwrap().push_back(ReceiverAction::SetIngressSlotBegin { slot }); + } + + fn set_ingress_progress_slot(&self, slot: u32) { + self.actions.lock().unwrap().push_back(ReceiverAction::SetIngressProgressSlot { slot }); + } + + fn cancel_candidate_requests_for_slot(&self, slot: u32) { + self.actions + .lock() + .unwrap() + .push_back(ReceiverAction::CancelCandidateRequestsForSlot { slot }); + } + fn request_candidate(&self, slot: u32, block_hash: UInt256) { self.actions .lock() @@ -185,12 +234,19 @@ impl Receiver for MockReceiver { .push_back(ReceiverAction::RequestCandidate { slot, block_hash }); } + fn start(&self) {} + fn reschedule_standstill(&self) { // No-op for tests } - fn set_standstill_slots(&self, _begin: u32, _end: u32) { - // No-op for tests + fn set_standstill_slots(&self, begin: u32, end: u32) { + let mut last = self.last_standstill_slots.lock().unwrap(); + if *last == Some((begin, end)) { + return; + } + *last = Some((begin, end)); + self.actions.lock().unwrap().push_back(ReceiverAction::SetStandstillSlots { begin, end }); } fn send_certificate(&self, certificate: CertificateBoxed) { @@ -238,7 +294,6 @@ impl consensus_common::SessionListener for MockListener { _collated_data: BlockPayloadPtr, _callback: ValidatorBlockCandidateDecisionCallback, ) { - // No-op for simple tests } fn on_generate_slot( @@ -248,7 +303,6 @@ impl consensus_common::SessionListener for MockListener { _parent: CollationParentHint, _callback: ValidatorBlockCandidateCallback, ) { - // No-op for simple tests } fn on_block_committed( @@ -261,12 +315,12 @@ impl consensus_common::SessionListener for MockListener { _approve_signatures: Vec<(PublicKeyHash, BlockPayloadPtr)>, _stats: SessionStats, ) { - // No-op for simple tests + panic!( + "on_block_committed must not be called for Simplex sessions (finalized-driven only)" + ); } - fn on_block_skipped(&self, _round: u32) { - // No-op for simple tests - } + fn on_block_skipped(&self, _round: u32) {} fn get_approved_candidate( &self, @@ -276,16 +330,6 @@ impl consensus_common::SessionListener for MockListener { _collated_data_hash: UInt256, _callback: ValidatorBlockCandidateCallback, ) { - // No-op for simple tests - } - - fn get_committed_candidate( - &self, - block_id: BlockIdExt, - callback: consensus_common::CommittedBlockProofCallback, - ) { - log::info!("get_committed_candidate: STUB for block_id={}", block_id); - callback(Err(error!("get_committed_candidate not implemented in test"))); } } @@ -306,6 +350,8 @@ enum ListenerEvent { }, /// on_block_skipped was called (not used in SIMPLEX_ROUNDLESS mode) Skipped { round: u32 }, + /// on_block_finalized was called (out-of-order delivery) + Finalized { block_id: BlockIdExt, root_hash: UInt256 }, } // NOTE: dummy_source_info helper removed - was only used by emission model tests @@ -357,28 +403,35 @@ impl consensus_common::SessionListener for RecordingListener { fn on_block_committed( &self, _source_info: BlockSourceInfo, - root_hash: UInt256, + _root_hash: UInt256, _file_hash: UInt256, _data: BlockPayloadPtr, - signatures: BlockSignaturesVariant, + _signatures: BlockSignaturesVariant, _approve_signatures: Vec<(PublicKeyHash, BlockPayloadPtr)>, _stats: SessionStats, ) { - // Extract slot and is_final from Simplex signatures - // SIMPLEX_ROUNDLESS: Use slot instead of round (round is always u32::MAX) - let (slot, is_final) = match &signatures { - BlockSignaturesVariant::Simplex(s) => (s.slot, s.is_final), - _ => (0, false), // other variants not expected in simplex tests - }; - - self.events.lock().unwrap().push(ListenerEvent::Committed { slot, root_hash, is_final }); + panic!( + "on_block_committed must not be called for Simplex sessions (finalized-driven only)" + ); } fn on_block_skipped(&self, round: u32) { - // NOTE: Not called in SIMPLEX_ROUNDLESS mode self.events.lock().unwrap().push(ListenerEvent::Skipped { round }); } + fn on_block_finalized( + &self, + block_id: BlockIdExt, + _source_info: BlockSourceInfo, + root_hash: UInt256, + _file_hash: UInt256, + _data: BlockPayloadPtr, + _signatures: BlockSignaturesVariant, + _approve_signatures: Vec<(PublicKeyHash, BlockPayloadPtr)>, + ) { + self.events.lock().unwrap().push(ListenerEvent::Finalized { block_id, root_hash }); + } + fn get_approved_candidate( &self, _source: PublicKey, @@ -389,15 +442,6 @@ impl consensus_common::SessionListener for RecordingListener { ) { // No-op } - - fn get_committed_candidate( - &self, - block_id: BlockIdExt, - callback: consensus_common::CommittedBlockProofCallback, - ) { - log::info!("get_committed_candidate: STUB for block_id={}", block_id); - callback(Err(error!("get_committed_candidate not implemented in test"))); - } } // ============================================================================ @@ -511,21 +555,7 @@ fn create_test_desc_with_opts( local_idx: usize, opts: &SessionOptions, ) -> Arc { - let local_key = nodes[local_idx].public_key.clone(); - let shard = ShardIdent::masterchain(); - Arc::new( - crate::session_description::SessionDescription::new( - opts, - SessionId::default(), - 1, // initial_block_seqno - nodes, - local_key, - &shard, - SystemTime::now(), - None, // metrics - ) - .unwrap(), - ) + create_test_desc_for_shard_with_opts(nodes, local_idx, ShardIdent::masterchain(), opts) } impl TestFixture { @@ -534,10 +564,33 @@ impl TestFixture { Self::new_with_opts(validator_count, SessionOptions::default()) } - /// Create a test fixture with N validators and custom session options - fn new_with_opts(validator_count: u32, opts: SessionOptions) -> Self { + fn new_shard(validator_count: u32) -> Self { + Self::new_with_shard_and_local_idx( + validator_count, + 0, + ShardIdent::with_tagged_prefix(0, 0x8000_0000_0000_0000).unwrap(), + SessionOptions::default(), + ) + } + + /// Create a test fixture with N validators, custom options, and local_idx + fn new_with_local_idx(validator_count: u32, local_idx: usize, opts: SessionOptions) -> Self { + Self::new_with_shard_and_local_idx( + validator_count, + local_idx, + ShardIdent::masterchain(), + opts, + ) + } + + fn new_with_shard_and_local_idx( + validator_count: u32, + local_idx: usize, + shard: ShardIdent, + opts: SessionOptions, + ) -> Self { let nodes = create_test_validators(validator_count); - let description = create_test_desc_with_opts(&nodes, 0, &opts); + let description = create_test_desc_for_shard_with_opts(&nodes, local_idx, shard, &opts); let listener: Arc = Arc::new(MockListener); @@ -561,7 +614,7 @@ impl TestFixture { receiver.clone() as crate::receiver::ReceiverPtr, stop_flag, db, - 0, // initial_errors + 0, health_counters, ) .unwrap(); @@ -569,6 +622,11 @@ impl TestFixture { Self { nodes, description, processor, receiver, task_queue } } + /// Create a test fixture with N validators and custom session options + fn new_with_opts(validator_count: u32, opts: SessionOptions) -> Self { + Self::new_with_shard_and_local_idx(validator_count, 0, ShardIdent::masterchain(), opts) + } + /// Advance time by duration fn advance_time(&mut self, delta: Duration) { self.processor.advance_time(delta); @@ -586,6 +644,10 @@ impl TestFixture { } } +fn metrics_counter(processor: &SessionProcessor, name: &str) -> u64 { + processor.get_metrics_receiver().snapshot().counters.get(name).copied().unwrap_or(0) +} + // ============================================================================ // Certificate helpers // ============================================================================ @@ -616,6 +678,26 @@ fn build_skip_certificate_tl( Certificate { vote: unsigned_vote, signatures: sig_set }.into_boxed() } +fn make_test_final_cert(slot: SlotIndex, block_hash: UInt256) -> crate::certificate::FinalCertPtr { + Arc::new(crate::certificate::Certificate { + vote: crate::simplex_state::FinalizeVote { slot, block_hash }, + signatures: vec![ + crate::certificate::VoteSignature { + validator_idx: ValidatorIndex::new(0), + signature: vec![1u8; 64], + }, + crate::certificate::VoteSignature { + validator_idx: ValidatorIndex::new(1), + signature: vec![2u8; 64], + }, + crate::certificate::VoteSignature { + validator_idx: ValidatorIndex::new(2), + signature: vec![3u8; 64], + }, + ], + }) +} + // ============================================================================ // Basic Tests // ============================================================================ @@ -629,7 +711,7 @@ fn test_session_processor_creation() { #[test] fn test_manual_clock_control() { - let mut fixture = TestFixture::new(4); + let mut fixture = TestFixture::new_shard(4); // Seed manual time to a deterministic value first. // @@ -663,12 +745,12 @@ fn test_genesis_collation_expected_seqno_uses_initial_block_seqno() { // // Without this, collation fails with: // `seqno mismatch: candidate has seqno=1, expected=0 (derived from parent=None)` - let mut fixture = TestFixture::new(4); + let mut fixture = TestFixture::new_shard(4); // initial_block_seqno is set to 1 in create_test_desc() assert_eq!(fixture.description.get_initial_block_seqno(), 1); - // Ensure the expected seqno does NOT depend on last_committed_seqno when parent=None. - fixture.processor.last_committed_seqno = Some(123); + // Ensure the expected seqno does NOT depend on finalized_head_seqno when parent=None. + fixture.processor.finalized_head_seqno = Some(123); let slot = SlotIndex::new(132); @@ -696,9 +778,9 @@ fn test_genesis_collation_expected_seqno_uses_initial_block_seqno() { fn test_should_generate_empty_block_uses_committed_head_at_session_start() { // With DISABLE_NON_FINALIZED_PARENTS_FOR_COLLATION=false (optimistic validation), // shardchains use the MC lag threshold rule for empty-block generation, not the - // committed-head rule. Only masterchain still uses committed-head gating. + // finalized-head rule. Only masterchain still uses finalized-head gating. // - // Verify that masterchain sessions still use committed-head empty-block gating. + // Verify that masterchain sessions still use finalized-head empty-block gating. let nodes = create_test_validators(4); let local_idx = 0; let initial_block_seqno = 47; @@ -744,96 +826,197 @@ fn test_should_generate_empty_block_uses_committed_head_at_session_start() { ) .unwrap(); - assert_eq!(processor.last_committed_seqno, Some(46)); + assert_eq!(processor.finalized_head_seqno, Some(46)); // Slot 0 is the initial `first_non_progressed_slot` in fresh state. - // MC: new_seqno=48, committed=46 -> 46+1=47 < 48 -> empty + // MC: new_seqno=48, finalized_head=46 -> 46+1=47 < 48 -> empty assert!(processor.should_generate_empty_block(SlotIndex::new(0), 48)); - // MC: new_seqno=47, committed=46 -> 46+1=47 == 47 -> NOT empty + // MC: new_seqno=47, finalized_head=46 -> 46+1=47 == 47 -> NOT empty assert!(!processor.should_generate_empty_block(SlotIndex::new(0), 47)); } #[test] -fn test_masterchain_out_of_order_finalization_waits_for_missing_final_cert() { - // Deterministic corner case: - // - committed head seqno = 100 - // - we observe a finalized (FinalCert) candidate with seqno = 103 - // Masterchain invariant: we must NOT commit any intermediate non-empty blocks using NotarCert-only signatures. - // Expected behavior: keep the finalized entry in journal and report WaitingForFinalCert(expected_seqno=101). - let mut fixture = TestFixture::new(4); +fn test_out_of_order_finalized_delivery_emits_immediately_when_body_present() { + let mut opts = SessionOptions::default(); + opts.use_callback_thread = false; + let mut fixture = TestFixture::new_with_opts(4, opts); - // Seed committed head - let committed_block_id = - BlockIdExt::with_params(ShardIdent::masterchain(), 100, UInt256::rand(), UInt256::rand()); - fixture.processor.last_committed_seqno = Some(100); - fixture.processor.last_committed_block_id = Some(committed_block_id); + let recording = RecordingListener::new(); + let listener: Arc = recording.clone(); + fixture.processor.listener = Arc::downgrade(&listener); - // Create a received non-empty candidate that is "finalized" but ahead by seqno. - let slot = SlotIndex::new(200); - let block_hash = UInt256::rand(); - let candidate_id = RawCandidateId { slot, hash: block_hash.clone() }; + // Set finalized head high so this test isolates finalized-delivery callback. + fixture.processor.finalized_head_seqno = Some(100); + fixture.processor.finalized_head_block_id = Some(BlockIdExt::with_params( + ShardIdent::masterchain(), + 100, + UInt256::rand(), + UInt256::rand(), + )); - let finalized_block_id = - BlockIdExt::with_params(ShardIdent::masterchain(), 103, UInt256::rand(), UInt256::rand()); + let slot = 103u32; + let (leader_source, candidate_id, broadcast) = + make_signed_block_broadcast(&fixture, slot, vec![7u8, 8, 9, 10]); + fixture.processor.on_candidate_received(leader_source, broadcast, None); - fixture.processor.received_candidates.insert( - candidate_id.clone(), - ReceivedCandidate { - slot, - source_idx: ValidatorIndex::new(0), - candidate_id_hash: block_hash.clone(), - candidate_hash_data_bytes: vec![1, 2, 3], // non-empty (required for signature context, if ever used) - block_id: finalized_block_id.clone(), - root_hash: finalized_block_id.root_hash.clone(), - file_hash: finalized_block_id.file_hash.clone(), - data: consensus_common::ConsensusCommonFactory::create_block_payload(vec![0xAA].into()), - collated_data: consensus_common::ConsensusCommonFactory::create_block_payload( - vec![0xBB].into(), - ), - receive_time: fixture.description.get_time(), - is_empty: false, - parent_id: None, - is_fully_resolved: true, - }, + let received = fixture + .processor + .received_candidates + .get(&candidate_id) + .expect("candidate should be present") + .clone(); + + let event = BlockFinalizedEvent { + slot: candidate_id.slot, + block_hash: candidate_id.hash.clone(), + block_id: Some(received.block_id.clone()), + certificate: make_test_final_cert(candidate_id.slot, candidate_id.hash.clone()), + }; + fixture.processor.handle_block_finalized(event); + + let events = recording.drain_events(); + assert!( + events.iter().any(|e| matches!( + e, + ListenerEvent::Finalized { block_id, root_hash } + if block_id == &received.block_id && root_hash == &received.root_hash + )), + "out-of-order finalized callback must be emitted when body is already known" + ); + assert!( + !events.iter().any(|e| matches!(e, ListenerEvent::Committed { .. })), + "on_block_committed must be suppressed in out-of-order mode" + ); + assert!( + fixture.processor.finalized_pending_body.is_empty(), + "no pending-body retention expected when finalized body is already present" ); + assert!( + fixture.processor.requested_candidates.is_empty(), + "finalized-driven mode must not request missing candidates" + ); +} - // Directly verify the collector result first (no journal side effects) - match fixture.processor.collect_gapless_commit_chain(&candidate_id) { - ChainCollectionResult::WaitingForFinalCert { - expected_seqno, - finalized_id, - finalized_seqno, - } => { - assert_eq!(expected_seqno, 101); - assert_eq!(finalized_id, candidate_id); - assert_eq!(finalized_seqno, 103); - } - other => panic!("unexpected chain collection result: {:?}", mem::discriminant(&other)), - } +#[test] +fn test_out_of_order_finalized_delivery_emits_when_body_arrives_late_and_dedups() { + let mut opts = SessionOptions::default(); + opts.use_callback_thread = false; + let mut fixture = TestFixture::new_with_opts(4, opts); - // Insert a journal entry and ensure try_commit does NOT schedule requestCandidate and does NOT commit. - let dummy_final_cert: crate::certificate::FinalCertPtr = - Arc::new(crate::certificate::Certificate { - vote: crate::simplex_state::FinalizeVote { slot, block_hash: block_hash.clone() }, - signatures: Vec::new(), - }); - let dummy_event = BlockFinalizedEvent { + let recording = RecordingListener::new(); + let listener: Arc = recording.clone(); + fixture.processor.listener = Arc::downgrade(&listener); + + // Set finalized head high so this test isolates finalized-delivery callback. + fixture.processor.finalized_head_seqno = Some(70); + fixture.processor.finalized_head_block_id = Some(BlockIdExt::with_params( + ShardIdent::masterchain(), + 70, + UInt256::rand(), + UInt256::rand(), + )); + + let slot = 77u32; + let block_data = vec![0xA5, 0x5A, 0xCC, 0x33]; + let (leader_source, candidate_id, broadcast) = + make_signed_block_broadcast(&fixture, slot, block_data.clone()); + + // Finalization observed first (before full candidate body is known). + let root_hash = UInt256::from_slice(&sha256_digest(&block_data)); + let block_id = BlockIdExt { + shard_id: fixture.processor.description.get_shard().clone(), + seq_no: slot, + root_hash: root_hash.clone(), + file_hash: root_hash, + }; + let event = BlockFinalizedEvent { + slot: candidate_id.slot, + block_hash: candidate_id.hash.clone(), + block_id: Some(block_id.clone()), + certificate: make_test_final_cert(candidate_id.slot, candidate_id.hash.clone()), + }; + fixture.processor.handle_block_finalized(event.clone()); + + assert!( + recording.drain_events().is_empty(), + "no finalized callback before candidate body is available" + ); + assert!( + fixture.processor.requested_candidates.is_empty(), + "out-of-order mode must not request candidates on finalization" + ); + + // Body arrives later: delayed finalized-delivery path should emit callback now. + fixture.processor.on_candidate_received(leader_source, broadcast, None); + let events_after_body = recording.drain_events(); + let finalized_count = events_after_body + .iter() + .filter(|e| matches!(e, ListenerEvent::Finalized { block_id: id, .. } if id == &block_id)) + .count(); + assert_eq!( + finalized_count, 1, + "exactly one finalized callback expected after late body arrival" + ); + assert!( + !events_after_body.iter().any(|e| matches!(e, ListenerEvent::Committed { .. })), + "on_block_committed must stay suppressed in out-of-order mode" + ); + assert!( + fixture.processor.requested_candidates.is_empty(), + "finalized-driven mode must not request missing candidates after body arrival" + ); + + // Duplicate finalization observation should not re-emit callback. + fixture.processor.handle_block_finalized(event); + let events_after_duplicate = recording.drain_events(); + assert!( + !events_after_duplicate + .iter() + .any(|e| matches!(e, ListenerEvent::Finalized { block_id: id, .. } if id == &block_id)), + "duplicate finalized observation must be deduplicated" + ); +} + +#[test] +fn test_out_of_order_mode_does_not_run_commit_chain_recovery_for_missing_body() { + let mut opts = SessionOptions::default(); + opts.use_callback_thread = false; + let mut fixture = TestFixture::new_with_opts(4, opts); + + // No candidate body is inserted. + let slot = SlotIndex::new(55); + let block_hash = UInt256::rand(); + let block_id = + BlockIdExt::with_params(ShardIdent::masterchain(), 55, UInt256::rand(), UInt256::rand()); + let finalized_id = RawCandidateId { slot, hash: block_hash.clone() }; + + let event = BlockFinalizedEvent { slot, block_hash: block_hash.clone(), - block_id: Some(finalized_block_id), - certificate: dummy_final_cert, + block_id: Some(block_id), + certificate: make_test_final_cert(slot, block_hash), }; - fixture.processor.finalized_journal_pending_commit.insert( - candidate_id.clone(), - FinalizedEntry { event: dummy_event, finalized_at: fixture.description.get_time() }, - ); + fixture.processor.handle_block_finalized(event); - fixture.processor.try_commit_finalized_chains(); + assert!( + fixture.processor.finalized_pending_body.contains_key(&finalized_id), + "finalization should be buffered until body arrival" + ); + assert!( + !fixture.processor.received_candidates.contains_key(&finalized_id), + "finalized-driven mode must not seed stubs for missing bodies" + ); + assert!( + fixture.processor.requested_candidates.is_empty(), + "finalized-driven mode must not request missing candidates" + ); - assert_eq!(fixture.processor.last_committed_seqno, Some(100)); - assert!(fixture.processor.requested_candidates.is_empty()); - assert!(fixture.processor.finalized_journal_pending_commit.contains_key(&candidate_id)); - assert!(!fixture.processor.finalized_blocks.contains(&candidate_id)); + // Periodic scheduler path should also avoid candidate recovery. + fixture.processor.check_all(); + assert!( + fixture.processor.requested_candidates.is_empty(), + "check_all must not trigger candidate requests in finalized-driven mode" + ); } #[test] @@ -897,144 +1080,92 @@ fn test_update_resolution_cache_chain_handles_deep_descendant_chains() { } #[test] -fn test_masterchain_waiting_for_final_cert_commits_expected_seqno_when_available() { - // Masterchain catch-up invariant: - // - A finalized (FinalCert) candidate may arrive ahead of the committed head (seqno gap). - // - We must not commit intermediate MC blocks using NotarCert-only signatures. - // - Once the missing FinalCert for the *next committable* seqno is available, we should - // commit that block to advance the committed head and unblock progress. +fn test_check_all_updates_awake_time() { let mut fixture = TestFixture::new(4); - // Seed committed head at seqno 10 - let committed_block_id = - BlockIdExt::with_params(ShardIdent::masterchain(), 10, UInt256::rand(), UInt256::rand()); - fixture.processor.last_committed_seqno = Some(10); - fixture.processor.last_committed_block_id = Some(committed_block_id); - - // Next committable block (expected_seqno = 11) - let slot_11 = SlotIndex::new(21); - let hash_11 = UInt256::rand(); - let id_11 = RawCandidateId { slot: slot_11, hash: hash_11.clone() }; - let block_id_11 = - BlockIdExt::with_params(ShardIdent::masterchain(), 11, UInt256::rand(), UInt256::rand()); - fixture.processor.received_candidates.insert( - id_11.clone(), - ReceivedCandidate { - slot: slot_11, - source_idx: ValidatorIndex::new(0), - candidate_id_hash: hash_11.clone(), - candidate_hash_data_bytes: vec![1, 2, 3], - block_id: block_id_11.clone(), - root_hash: block_id_11.root_hash.clone(), - file_hash: block_id_11.file_hash.clone(), - data: consensus_common::ConsensusCommonFactory::create_block_payload(vec![0xAA].into()), - collated_data: consensus_common::ConsensusCommonFactory::create_block_payload( - vec![0xBB].into(), - ), - receive_time: fixture.description.get_time(), - is_empty: false, - parent_id: None, - is_fully_resolved: true, - }, - ); - - // Provide the missing FinalCert for seqno 11 (sufficient weight: 3/4). - let final_cert_11 = Arc::new(crate::certificate::Certificate { - vote: crate::simplex_state::FinalizeVote { slot: slot_11, block_hash: hash_11.clone() }, - signatures: vec![ - // NOTE: SessionProcessor expects 64-byte Ed25519 signatures when building BlockSignaturesSimplex. - crate::certificate::VoteSignature::new(ValidatorIndex::new(0), vec![0u8; 64]), - crate::certificate::VoteSignature::new(ValidatorIndex::new(1), vec![1u8; 64]), - crate::certificate::VoteSignature::new(ValidatorIndex::new(2), vec![2u8; 64]), - ], - }); - fixture - .processor - .simplex_state - .set_finalize_certificate(&fixture.description, slot_11, &hash_11, final_cert_11) - .expect("failed to store finalize cert for expected_seqno"); - - // Finalized candidate observed ahead by seqno (seqno 12, expected is 11). - let slot_12 = SlotIndex::new(24); - let hash_12 = UInt256::rand(); - let id_12 = RawCandidateId { slot: slot_12, hash: hash_12.clone() }; - let block_id_12 = - BlockIdExt::with_params(ShardIdent::masterchain(), 12, UInt256::rand(), UInt256::rand()); - fixture.processor.received_candidates.insert( - id_12.clone(), - ReceivedCandidate { - slot: slot_12, - source_idx: ValidatorIndex::new(0), - candidate_id_hash: hash_12.clone(), - candidate_hash_data_bytes: vec![4, 5, 6], - block_id: block_id_12.clone(), - root_hash: block_id_12.root_hash.clone(), - file_hash: block_id_12.file_hash.clone(), - data: consensus_common::ConsensusCommonFactory::create_block_payload(vec![0xCC].into()), - collated_data: consensus_common::ConsensusCommonFactory::create_block_payload( - vec![0xDD].into(), - ), - receive_time: fixture.description.get_time(), - is_empty: false, - parent_id: Some(id_11.clone()), // walkback target for expected_seqno=11 - is_fully_resolved: true, - }, - ); + let before = fixture.processor.get_next_awake_time(); - // Journal entry for the ahead-of-head finalized block (certificate contents are irrelevant - // for the gap-recovery commit because we commit using the missing block's FinalCert). - let dummy_final_cert_12: crate::certificate::FinalCertPtr = - Arc::new(crate::certificate::Certificate { - vote: crate::simplex_state::FinalizeVote { slot: slot_12, block_hash: hash_12.clone() }, - signatures: vec![ - crate::certificate::VoteSignature::new(ValidatorIndex::new(0), vec![0u8; 64]), - crate::certificate::VoteSignature::new(ValidatorIndex::new(1), vec![1u8; 64]), - crate::certificate::VoteSignature::new(ValidatorIndex::new(2), vec![2u8; 64]), - ], - }); - let event_12 = BlockFinalizedEvent { - slot: slot_12, - block_hash: hash_12.clone(), - block_id: Some(block_id_12), - certificate: dummy_final_cert_12, - }; - fixture.processor.finalized_journal_pending_commit.insert( - id_12.clone(), - FinalizedEntry { event: event_12, finalized_at: fixture.description.get_time() }, - ); + // Call check_all (should reset awake time) + fixture.processor.check_all(); - // Act: the processor should commit the missing expected_seqno block (seqno 11). - fixture.processor.try_commit_finalized_chains(); + let after = fixture.processor.get_next_awake_time(); - assert_eq!(fixture.processor.last_committed_seqno, Some(11)); - assert!(fixture.processor.finalized_blocks.contains(&id_11)); - assert!(fixture.processor.finalized_journal_pending_commit.contains_key(&id_12)); - assert!( - !fixture.processor.finalized_blocks.contains(&id_12), - "ahead-of-head finalized block should remain pending until the next commit pass" - ); + // Awake time should be updated (pushed into future) + assert!(after >= before); } #[test] -fn test_check_all_updates_awake_time() { +fn test_start_arms_timeout_from_current_time_after_cold_delay() { let mut fixture = TestFixture::new(4); + let base_time = SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000); + fixture.processor.set_time(base_time); - let before = fixture.processor.get_next_awake_time(); + // Constructor path must not arm startup skip timers. + assert!( + fixture.processor.simplex_state.get_next_timeout().is_none(), + "startup timeout must not be armed before start()" + ); - // Call check_all (should reset awake time) - fixture.processor.check_all(); + // Simulate prolonged cold startup / delayed readiness. + fixture.processor.advance_time(Duration::from_secs(120)); + let now = fixture.description.get_time(); - let after = fixture.processor.get_next_awake_time(); + fixture.processor.start(); - // Awake time should be updated (pushed into future) - assert!(after >= before); + let timeout = fixture + .processor + .simplex_state + .get_next_timeout() + .expect("startup timeout must be scheduled after start()"); + let opts = fixture.description.opts(); + let expected = now + opts.first_block_timeout + opts.target_rate; + assert_eq!( + timeout, expected, + "startup timeout must be anchored to start() time, not constructor time \ + (C++ ref: alarm = now + first_block_timeout + target_rate)" + ); } #[test] fn test_receiver_records_no_actions_initially() { let fixture = TestFixture::new(4); let actions = fixture.drain_receiver_actions(); - assert!(actions.is_empty(), "Expected no receiver actions initially"); + assert!( + actions.iter().all(|a| { + matches!(a, ReceiverAction::SetIngressSlotBegin { slot: 0 }) + || matches!(a, ReceiverAction::SetIngressProgressSlot { slot: 0 }) + || matches!(a, ReceiverAction::SetStandstillSlots { begin: 0, .. }) + }), + "Expected only initial ingress/standstill sync actions" + ); +} + +#[test] +fn test_check_all_syncs_standstill_slots_when_tracked_interval_changes() { + let mut fixture = TestFixture::new(4); + fixture.drain_receiver_actions(); + + fixture.processor.check_all(); + let initial_actions = fixture.drain_receiver_actions(); + assert!( + !initial_actions.iter().any(|a| matches!(a, ReceiverAction::SetStandstillSlots { .. })), + "unchanged tracked interval should not resync standstill slots" + ); + + fixture.processor.simplex_state.set_first_non_finalized_slot(crate::block::SlotIndex::new(4)); + let (expected_begin, expected_end) = + fixture.processor.simplex_state.get_tracked_slots_interval(); + + fixture.processor.check_all(); + let actions = fixture.drain_receiver_actions(); + assert!( + actions.iter().any(|a| matches!( + a, + ReceiverAction::SetStandstillSlots { begin, end } + if *begin == expected_begin && *end == expected_end + )), + "check_all must resync receiver standstill slots when the FSM tracked interval changes" + ); } // ============================================================================ @@ -1091,6 +1222,34 @@ fn test_on_certificate_relays_and_caches_skip_certificate_once() { ); } +#[test] +fn test_future_certificate_is_not_rejected_like_cpp() { + let mut fixture = TestFixture::new(4); + fixture.drain_receiver_actions(); + + let slot = fixture.processor.simplex_state.first_too_new_vote_slot().value(); + let tl_cert = + build_skip_certificate_tl(&SessionId::default(), &fixture.nodes, slot, &[0, 1, 2]); + + fixture.processor.on_certificate(1, tl_cert); + + assert!( + fixture.processor.simplex_state.has_skip_certificate(SlotIndex::new(slot)), + "certificate at the vote too-new boundary should still be stored" + ); + assert_eq!( + fixture.processor.simplex_state.get_first_non_progressed_slot(), + SlotIndex::new(0), + "out-of-order future certificate must not advance progress past unresolved earlier slots" + ); + + let actions = fixture.drain_receiver_actions(); + assert!( + actions.iter().any(|a| matches!(a, ReceiverAction::SendCertificate { .. })), + "accepted future certificate should still be relayed" + ); +} + #[test] fn test_handle_finalization_reached_caches_final_certificate_for_standstill() { let mut fixture = TestFixture::new(4); @@ -1135,6 +1294,116 @@ fn test_handle_finalization_reached_caches_final_certificate_for_standstill() { ); } +#[test] +fn test_update_standstill_after_final_cert_updates_ingress_when_cleanup_is_skipped() { + let mut fixture = TestFixture::new(4); + + // Regression: for early slots, history cleanup is skipped (up_to_slot == 0). + fixture.processor.cleanup_old_slots(crate::block::SlotIndex::new(8)); + let cleanup_actions = fixture.drain_receiver_actions(); + assert!( + !cleanup_actions.iter().any(|a| matches!(a, ReceiverAction::Cleanup { .. })), + "receiver.cleanup must not run before MAX_HISTORY_SLOTS" + ); + + // Simulate finalized frontier advancement and final-cert hook. + fixture.processor.simplex_state.set_first_non_finalized_slot(crate::block::SlotIndex::new(9)); + fixture.processor.update_standstill_after_final_cert(crate::block::SlotIndex::new(8)); + + let actions = fixture.drain_receiver_actions(); + assert!( + actions.iter().any(|a| matches!(a, ReceiverAction::SetIngressSlotBegin { slot: 9 })), + "final-cert path must advance receiver ingress lower bound even when cleanup is skipped" + ); + assert!( + actions.iter().any(|a| matches!(a, ReceiverAction::SetIngressProgressSlot { slot: 9 })), + "final-cert path must advance receiver ingress progress cursor" + ); +} + +#[test] +fn test_recovery_set_first_non_finalized_slot_updates_receiver_ingress() { + let mut fixture = TestFixture::new(4); + + fixture.processor.recovery_set_first_non_finalized_slot(crate::block::SlotIndex::new(9)); + + let actions = fixture.drain_receiver_actions(); + assert!( + actions.iter().any(|a| matches!(a, ReceiverAction::SetIngressSlotBegin { slot: 9 })), + "recovery must synchronize receiver ingress lower bound with restored frontier" + ); + assert!( + actions.iter().any(|a| matches!(a, ReceiverAction::SetIngressProgressSlot { slot: 9 })), + "recovery must synchronize receiver ingress progress cursor with restored frontier" + ); +} + +#[test] +fn test_skip_certificate_syncs_progress_cursor_without_advancing_finalized_frontier() { + let mut fixture = TestFixture::new(4); + fixture.drain_receiver_actions(); + + let slot = 0u32; + let tl_cert = + build_skip_certificate_tl(&SessionId::default(), &fixture.nodes, slot, &[0, 1, 2]); + + fixture.processor.on_certificate(1, tl_cert); + + let actions = fixture.drain_receiver_actions(); + assert!( + actions.iter().any(|a| matches!(a, ReceiverAction::SetIngressSlotBegin { slot: 0 })), + "skip progress must keep ingress lower bound at the finalized frontier" + ); + assert!( + actions.iter().any(|a| matches!(a, ReceiverAction::SetIngressProgressSlot { slot: 1 })), + "skip certificate must advance ingress progress cursor to the next slot" + ); +} + +#[test] +fn test_skip_certificate_cancels_stale_candidate_request_repairs() { + let mut fixture = TestFixture::new(4); + fixture.drain_receiver_actions(); + + let slot = crate::block::SlotIndex::new(0); + let block_hash = UInt256::rand(); + let candidate_id = RawCandidateId { slot, hash: block_hash.clone() }; + + fixture.processor.request_candidate(slot, block_hash.clone(), Some(Duration::from_secs(1))); + assert!( + fixture.processor.requested_candidates.contains_key(&candidate_id), + "request should be scheduled before skip" + ); + + let tl_cert = + build_skip_certificate_tl(&SessionId::default(), &fixture.nodes, slot.value(), &[0, 1, 2]); + fixture.processor.on_certificate(1, tl_cert); + + assert!( + !fixture.processor.requested_candidates.contains_key(&candidate_id), + "skip must cancel scheduled requestCandidate repairs for the skipped slot" + ); + + let actions_after_skip = fixture.drain_receiver_actions(); + assert!( + actions_after_skip + .iter() + .any(|a| matches!(a, ReceiverAction::CancelCandidateRequestsForSlot { slot: 0 })), + "skip must cancel receiver-side pending requests for the skipped slot" + ); + + fixture.advance_time(Duration::from_secs(2)); + fixture.processor.check_all(); + + let actions_after_delay = fixture.drain_receiver_actions(); + assert!( + !actions_after_delay + .iter() + .any(|a| matches!(a, ReceiverAction::RequestCandidate { slot: 0, .. })), + "cancelled delayed request must not fire after the slot is skipped" + ); +} + #[test] fn test_handle_notarization_reached_requests_missing_candidate_body() { let mut fixture = TestFixture::new(4); @@ -1221,9 +1490,14 @@ fn test_time_isolation_between_tests() { let fixture2 = TestFixture::new(4); let time2 = fixture2.description.get_time(); - // Times should be close (both created around same time) + // Times should be reasonably close (both created around same time). + // Fixture creation can occasionally be slow on contended CI hosts. let diff = time2.duration_since(time1).unwrap_or_else(|_| time1.duration_since(time2).unwrap()); - assert!(diff < Duration::from_millis(100), "Test fixtures should have similar initial times"); + assert!( + diff < Duration::from_secs(2), + "Test fixtures should have similar initial times (diff={:?})", + diff + ); } // ============================================================================ @@ -1238,13 +1512,13 @@ fn test_time_isolation_between_tests() { /// - slot 3: finalized (FinalCert) /// - parent chain: 1 → 2 → 3 (all non-empty blocks) /// -/// Expected: THREE COMMITS emitted in order (not skip/skip/commit): -/// - commit(round=1, is_final=false, sigs=NotarCert/approve) -/// - commit(round=2, is_final=false, sigs=NotarCert/approve) -/// - commit(round=3, is_final=true, sigs=FinalCert/final) +/// Expected: THREE finalization deliveries emitted in order: +/// - delivery(round=1, is_final=false, sigs=NotarCert/approve) +/// - delivery(round=2, is_final=false, sigs=NotarCert/approve) +/// - delivery(round=3, is_final=true, sigs=FinalCert/final) /// /// This verifies C++ `finalize_blocks()` parity: -/// - Parent blocks CAN be committed (even if not finalized) +/// - Parent blocks CAN be finalized (even if body not yet received) /// - Parent blocks use NotarCert/`create_simplex_approve` signatures /// - Triggered block uses FinalCert/`create_simplex` signatures /// - No panic on is_triggered_block=false @@ -1289,7 +1563,7 @@ fn test_simplex_roundless_constant_value() { /// /// We keep `require_finalized_parent=false` (C++ mode) to avoid deadlock when a slot is /// notarized but not finalized/skipped yet. ValidatorGroup limitation is handled separately -/// by forcing EMPTY collation on non-committed parents. +/// by forcing EMPTY collation on non-finalized parents. #[test] fn test_simplex_state_options_require_finalized_parent() { // Default (cpp_compatible) should have require_finalized_parent=false @@ -1388,18 +1662,18 @@ fn test_slot_started_at_tracking() { } #[test] -fn test_candidate_decision_fail_drops_late_failure_for_committed_block() { +fn test_candidate_decision_fail_drops_late_failure_for_finalized_block() { // Regression: in roundless Simplex, validation callbacks can arrive late (after the block is - // already committed). In this case we must drop the result and NOT schedule retries / WARN. + // already finalized). In this case we must drop the result and NOT schedule retries / WARN. let mut fixture = TestFixture::new(4); let slot = SlotIndex::new(5); let candidate_id = RawCandidateId { slot, hash: UInt256::rand() }; - // Pretend we've already committed past this block seqno. - fixture.processor.last_committed_seqno = Some(100); + // Pretend we've already finalized past this block seqno. + fixture.processor.finalized_head_seqno = Some(100); - // Create a non-empty RawCandidate with seqno <= last_committed_seqno. + // Create a non-empty RawCandidate with seqno <= finalized_head_seqno. let block_id = BlockIdExt::with_params(ShardIdent::masterchain(), 42, UInt256::rand(), UInt256::rand()); let creator = fixture.nodes[0].public_key.clone(); @@ -1549,9 +1823,34 @@ fn notarize_slot(fixture: &mut TestFixture, slot: SlotIndex, block_hash: &UInt25 .expect("set_notarize_certificate failed"); } +/// Helper: drive the FSM to skip-cert a slot. +fn skip_slot(fixture: &mut TestFixture, slot: SlotIndex) { + let signatures = vec![ + crate::certificate::VoteSignature::new(ValidatorIndex::new(0), vec![0]), + crate::certificate::VoteSignature::new(ValidatorIndex::new(1), vec![1]), + crate::certificate::VoteSignature::new(ValidatorIndex::new(2), vec![2]), + ]; + let vote = crate::simplex_state::SkipVote { slot }; + let cert = Arc::new(crate::certificate::Certificate { vote, signatures }); + + fixture + .processor + .simplex_state + .set_skip_certificate(&fixture.description, slot, cert) + .expect("set_skip_certificate failed"); +} + +fn strict_wait_for_parent_mode_enabled() -> bool { + matches!(PARENT_READINESS_MODE, ParentReadinessMode::StrictWaitForParent) +} + +fn strict_mc_session_gate_mode_enabled() -> bool { + matches!(MC_ACCEPTED_HEAD_MODE, McAcceptedHeadMode::StrictSessionProcessorGate) +} + #[test] fn test_check_validation_forwards_candidate_with_notarized_parent() { - let mut fixture = TestFixture::new(4); + let mut fixture = TestFixture::new_shard(4); let parent_slot = SlotIndex::new(0); let parent_hash = UInt256::rand(); @@ -1565,12 +1864,21 @@ fn test_check_validation_forwards_candidate_with_notarized_parent() { let time = fixture.description.get_time(); insert_pending_validation(&mut fixture.processor, &child_id, raw_candidate, time); - // Before notarization: check_validation should NOT forward the candidate + // Before notarization: + // - strict mode: candidate stays blocked by WaitForParent + // - relaxed mode: candidate is forwarded early to validator-side checks fixture.processor.check_validation(); - assert!( - !fixture.processor.pending_approve.contains(&child_id), - "candidate must not be forwarded when parent is not notarized" - ); + if strict_wait_for_parent_mode_enabled() { + assert!( + !fixture.processor.pending_approve.contains(&child_id), + "strict mode: candidate must not be forwarded when parent is not notarized" + ); + } else { + assert!( + fixture.processor.pending_approve.contains(&child_id), + "relaxed mode: candidate should be forwarded before parent notarization" + ); + } // Notarize the parent slot notarize_slot(&mut fixture, parent_slot, &parent_hash); @@ -1604,14 +1912,21 @@ fn test_check_validation_blocks_candidate_with_non_notarized_parent() { // Parent is NOT notarized — candidate must stay in pending_validations fixture.processor.check_validation(); - assert!( - !fixture.processor.pending_approve.contains(&child_id), - "candidate must NOT be forwarded when parent slot is not notarized" - ); - assert!( - fixture.processor.pending_validations.contains_key(&child_id), - "candidate must remain in pending_validations" - ); + if strict_wait_for_parent_mode_enabled() { + assert!( + !fixture.processor.pending_approve.contains(&child_id), + "strict mode: candidate must NOT be forwarded when parent slot is not notarized" + ); + assert!( + fixture.processor.pending_validations.contains_key(&child_id), + "strict mode: candidate must remain in pending_validations" + ); + } else { + assert!( + fixture.processor.pending_approve.contains(&child_id), + "relaxed mode: candidate should be forwarded even when parent is not notarized" + ); + } } #[test] @@ -1664,11 +1979,14 @@ fn test_check_validation_auto_approves_empty_blocks() { &child_id, parent_block_id, true, - Some(parent_id), + Some(parent_id.clone()), ); let time = fixture.description.get_time(); insert_pending_validation(&mut fixture.processor, &child_id, raw_candidate, time); + // WaitForParent parity: parent must be notarized before candidate is eligible. + notarize_slot(&mut fixture, parent_slot, &parent_id.hash); + // Empty blocks with a matching referenced block should be auto-approved. // C++ block-validator.cpp accepts when block == event->state->as_normal(). fixture.processor.check_validation(); @@ -1709,11 +2027,14 @@ fn test_empty_block_accepted_when_referenced_block_matches_parent() { &child_id, parent_block_id, true, - Some(parent_id), + Some(parent_id.clone()), ); let time = fixture.description.get_time(); insert_pending_validation(&mut fixture.processor, &child_id, raw_candidate, time); + // WaitForParent parity: parent must be notarized before candidate is eligible. + notarize_slot(&mut fixture, parent_slot, &parent_id.hash); + fixture.processor.check_validation(); assert!( !fixture.processor.pending_validations.contains_key(&child_id), @@ -1753,11 +2074,14 @@ fn test_empty_block_rejected_when_referenced_block_differs() { &child_id, wrong_block_id, true, - Some(parent_id), + Some(parent_id.clone()), ); let time = fixture.description.get_time(); insert_pending_validation(&mut fixture.processor, &child_id, raw_candidate, time); + // WaitForParent parity: parent must be notarized before candidate is eligible. + notarize_slot(&mut fixture, parent_slot, &parent_id.hash); + // C++ block-validator.cpp rejects empty candidates whose referenced block // does not match event->state->as_normal(). Rust must do the same. fixture.processor.check_validation(); @@ -1843,7 +2167,7 @@ fn test_check_validation_chains_notarized_parent_to_descendant() { // Validates that a candidate chain A -> B works: // - A has no parent (genesis), B has A as parent // - Only B's parent slot needs to be notarized for B to pass - let mut fixture = TestFixture::new(4); + let mut fixture = TestFixture::new_shard(4); let slot_a = SlotIndex::new(0); let hash_a = UInt256::rand(); @@ -1859,16 +2183,25 @@ fn test_check_validation_chains_notarized_parent_to_descendant() { insert_pending_validation(&mut fixture.processor, &id_a, raw_a, time); insert_pending_validation(&mut fixture.processor, &id_b, raw_b, time); - // First check_validation: A (genesis) should pass, B should not (parent not notarized) + // First check_validation: + // - A (genesis) should pass + // - B should wait in strict mode, but can pass in relaxed mode fixture.processor.check_validation(); assert!( fixture.processor.pending_approve.contains(&id_a), "genesis candidate A must be forwarded" ); - assert!( - !fixture.processor.pending_approve.contains(&id_b), - "candidate B must wait until parent slot is notarized" - ); + if strict_wait_for_parent_mode_enabled() { + assert!( + !fixture.processor.pending_approve.contains(&id_b), + "strict mode: candidate B must wait until parent slot is notarized" + ); + } else { + assert!( + fixture.processor.pending_approve.contains(&id_b), + "relaxed mode: candidate B may be forwarded before parent notarization" + ); + } // Notarize slot 0 (A's slot) notarize_slot(&mut fixture, slot_a, &hash_a); @@ -1881,6 +2214,102 @@ fn test_check_validation_chains_notarized_parent_to_descendant() { ); } +#[test] +fn test_check_validation_wait_for_parent_requires_gap_skip_certificates() { + let mut fixture = TestFixture::new_shard(4); + + // Candidate at slot 3 references parent at slot 0. + // Readiness requires: + // - parent slot 0 notarized + // - skip certificates for slots 1 and 2 + let parent_slot = SlotIndex::new(0); + let parent_hash = UInt256::rand(); + let parent_id = RawCandidateId { slot: parent_slot, hash: parent_hash.clone() }; + + let child_slot = SlotIndex::new(3); + let child_id = RawCandidateId { slot: child_slot, hash: UInt256::rand() }; + let raw_candidate = + make_test_non_empty_candidate(child_id.clone(), Some(parent_id.clone()), &fixture.nodes); + let time = fixture.description.get_time(); + insert_pending_validation(&mut fixture.processor, &child_id, raw_candidate, time); + + // Parent notarized, but gap skips missing: + // - strict mode: still blocked + // - relaxed mode: forwarded early + notarize_slot(&mut fixture, parent_slot, &parent_hash); + fixture.processor.check_validation(); + if strict_wait_for_parent_mode_enabled() { + assert!( + !fixture.processor.pending_approve.contains(&child_id), + "strict mode: candidate must be blocked until all intermediate slots are skip-certified" + ); + } else { + assert!( + fixture.processor.pending_approve.contains(&child_id), + "relaxed mode: candidate can be forwarded before skip-gap coverage converges" + ); + } + + // Add skip cert for slot 1 only: + // - strict mode: still blocked + // - relaxed mode: remains forwarded + skip_slot(&mut fixture, SlotIndex::new(1)); + fixture.processor.check_validation(); + if strict_wait_for_parent_mode_enabled() { + assert!( + !fixture.processor.pending_approve.contains(&child_id), + "strict mode: candidate must remain blocked when gap skip coverage is partial" + ); + } else { + assert!( + fixture.processor.pending_approve.contains(&child_id), + "relaxed mode: candidate remains eligible while skip-gap coverage is partial" + ); + } + + // Add skip cert for slot 2 -> now eligible. + skip_slot(&mut fixture, SlotIndex::new(2)); + fixture.processor.check_validation(); + assert!( + fixture.processor.pending_approve.contains(&child_id), + "candidate must be forwarded once parent and full skip-gap readiness are satisfied" + ); +} + +#[test] +fn test_check_validation_wait_for_parent_rejects_parent_hash_mismatch() { + let mut fixture = TestFixture::new(4); + + let parent_slot = SlotIndex::new(0); + let notarized_parent_hash = UInt256::from([0x21; 32]); + let mismatched_parent_hash = UInt256::from([0x99; 32]); + let parent_id = RawCandidateId { slot: parent_slot, hash: mismatched_parent_hash }; + + let child_slot = SlotIndex::new(1); + let child_id = RawCandidateId { slot: child_slot, hash: UInt256::rand() }; + let raw_candidate = + make_test_non_empty_candidate(child_id.clone(), Some(parent_id.clone()), &fixture.nodes); + let time = fixture.description.get_time(); + insert_pending_validation(&mut fixture.processor, &child_id, raw_candidate, time); + + // Notarize the slot with a different hash than candidate.parent_id. + notarize_slot(&mut fixture, parent_slot, ¬arized_parent_hash); + fixture.processor.check_validation(); + + if strict_wait_for_parent_mode_enabled() { + assert!( + !fixture.processor.pending_approve.contains(&child_id), + "strict mode: candidate must not be forwarded when parent hash mismatches notarized block" + ); + } else { + assert!( + fixture.processor.pending_approve.contains(&child_id) + || fixture.processor.rejected.contains(&child_id), + "relaxed mode: candidate should not be stalled only by WaitForParent mismatch" + ); + } +} + // ============================================================================ // Health check anomaly tests // ============================================================================ @@ -1894,10 +2323,14 @@ fn reset_health_alert_time(processor: &mut SessionProcessor, base: SystemTime) { s.last_activity_warn = base; s.last_parent_aging_warn = base; s.last_cert_fail_warn = base; + s.last_skip_ratio_warn = base; s.last_standstill_warn = base; s.last_finalization_speed_warn = base; s.last_finalization_nonzero_at = base; s.last_candidate_giveup_warn = base; + s.prev_votes_in_notarize = 0; + s.prev_votes_in_finalize = 0; + s.prev_votes_in_skip = 0; } #[test] @@ -1957,6 +2390,58 @@ fn test_health_check_candidate_giveup_anomaly() { assert_eq!(fixture.processor.health_alert_state.prev_candidate_giveups, 2); } +#[test] +fn test_health_check_skip_vote_dominance_anomaly() { + let mut fixture = TestFixture::new(4); + + let base_time = SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000); + fixture.processor.set_time(base_time); + reset_health_alert_time(&mut fixture.processor, base_time); + + fixture.processor.run_health_checks(); + + // High skip-dominant window (delta-based): should trigger skip ratio anomaly. + fixture.processor.votes_in_skip_total = 24; + fixture.processor.votes_in_notarize_total = 2; + fixture.processor.votes_in_finalize_total = 1; + + fixture.processor.set_time(base_time + Duration::from_secs(31)); + fixture.processor.run_health_checks(); + + assert_eq!(fixture.processor.health_alert_state.prev_votes_in_skip, 24); + assert_eq!(fixture.processor.health_alert_state.prev_votes_in_notarize, 2); + assert_eq!(fixture.processor.health_alert_state.prev_votes_in_finalize, 1); + assert_eq!( + fixture.processor.health_alert_state.last_skip_ratio_warn, + base_time + Duration::from_secs(31) + ); +} + +#[test] +fn test_health_check_skip_vote_dominance_ignores_sparse_zero_denominator() { + let mut fixture = TestFixture::new(4); + + let base_time = SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000); + fixture.processor.set_time(base_time); + reset_health_alert_time(&mut fixture.processor, base_time); + + fixture.processor.run_health_checks(); + + // One stream is absent in the current window, but the overall progress vote + // stream is still healthy enough that skip traffic is not dominant. + fixture.processor.votes_in_skip_total = 3; + fixture.processor.votes_in_notarize_total = 0; + fixture.processor.votes_in_finalize_total = 10; + + fixture.processor.set_time(base_time + Duration::from_secs(31)); + fixture.processor.run_health_checks(); + + assert_eq!(fixture.processor.health_alert_state.prev_votes_in_skip, 3); + assert_eq!(fixture.processor.health_alert_state.prev_votes_in_notarize, 0); + assert_eq!(fixture.processor.health_alert_state.prev_votes_in_finalize, 10); + assert_eq!(fixture.processor.health_alert_state.last_skip_ratio_warn, base_time); +} + #[test] fn test_health_check_cooldown_prevents_spam() { let mut fixture = TestFixture::new(4); @@ -2139,6 +2624,37 @@ fn test_check_collation_pacing_gate_is_idempotent() { assert_eq!(fixture.processor.get_next_awake_time(), gate_time); } +#[test] +fn test_collation_starts_metric_tracks_async_generation_requests() { + let opts = SessionOptions { slots_per_leader_window: 1, ..Default::default() }; + let mut fixture = TestFixture::new_with_opts(4, opts); + + fixture.processor.check_collation(); + + assert_eq!(metrics_counter(&fixture.processor, "simplex_collation_starts"), 1); + assert_eq!(metrics_counter(&fixture.processor, "simplex_collates.total"), 1); +} + +#[test] +fn test_collation_starts_metric_tracks_precollated_fast_path() { + let opts = SessionOptions { slots_per_leader_window: 1, ..Default::default() }; + let mut fixture = TestFixture::new_with_opts(4, opts); + let slot = SlotIndex::new(0); + let request = AsyncRequestImpl::new(11, false, fixture.description.get_time()); + let candidate = make_local_collated_candidate(&fixture, 1, 0x41); + + fixture + .processor + .precollated_blocks + .insert(slot, PrecollatedBlock { request, candidate: Some(candidate), parent: None }); + + fixture.processor.check_collation(); + + assert_eq!(metrics_counter(&fixture.processor, "simplex_collation_starts"), 1); + assert_eq!(metrics_counter(&fixture.processor, "simplex_collates.total"), 0); + assert_eq!(metrics_counter(&fixture.processor, "simplex_collates_precollated.success"), 1); +} + // ============================================================================ // Candidate Query Fallback Tests (C++ parity: CandidateResolver DB fallback) // ============================================================================ @@ -2158,7 +2674,7 @@ fn test_candidate_query_fallback_cache_hit() { tx.send(result).unwrap(); }); - fixture.processor.handle_candidate_query_fallback(slot, block_hash, false, callback); + fixture.processor.handle_candidate_query_fallback(slot, block_hash, true, false, callback); let result = rx.recv_timeout(Duration::from_secs(2)).expect("callback not called"); let payload = result.expect("response should be Ok"); @@ -2193,7 +2709,7 @@ fn test_candidate_query_fallback_miss_returns_empty() { tx.send(result).unwrap(); }); - fixture.processor.handle_candidate_query_fallback(slot, block_hash, false, callback); + fixture.processor.handle_candidate_query_fallback(slot, block_hash, true, false, callback); let result = rx.recv_timeout(Duration::from_secs(5)).expect("callback not called"); let payload = result.expect("response should be Ok even for empty"); @@ -2211,24 +2727,20 @@ fn test_candidate_query_fallback_miss_returns_empty() { assert!(inner.candidate.is_empty(), "candidate bytes should be empty when not found"); } -#[test] -fn test_candidate_data_cache_populated_on_candidate_received() { - let _ = env_logger::Builder::new().filter_level(log::LevelFilter::Debug).try_init(); - let mut fixture = TestFixture::new(4); - - // Use slot 0 so that validator 0 (local) is the slot leader - let slot = 0u32; - let block_data = vec![1u8, 2, 3, 4, 5]; +fn make_signed_block_broadcast( + fixture: &TestFixture, + slot: u32, + block_data: Vec, +) -> (u32, RawCandidateId, CandidateData) { let collated_data: Vec = vec![]; let root_hash = UInt256::from_slice(&sha256_digest(&block_data)); - let shard = ShardIdent::masterchain(); + let shard = fixture.processor.description.get_shard().clone(); - // Build uncompressed TL candidate (same approach as test_receiver_candidate_resolver) let tl_inner = TlCandidate { src: UInt256::default(), round: slot as i32, root_hash: root_hash.clone(), - data: block_data.clone().into(), + data: block_data.into(), collated_data: collated_data.clone().into(), }; let candidate_bytes = consensus_common::serialize_tl_boxed_object!(&tl_inner.into_boxed()); @@ -2237,7 +2749,7 @@ fn test_candidate_data_cache_populated_on_candidate_received() { shard_id: shard, seq_no: slot, root_hash: root_hash.clone(), - file_hash: root_hash.clone(), + file_hash: root_hash, }; let collated_file_hash = UInt256::from_slice(&sha256_digest(&collated_data)); @@ -2249,7 +2761,8 @@ fn test_candidate_data_cache_populated_on_candidate_received() { ); let session_id = fixture.processor.session_id().clone(); - let leader_key = fixture.processor.description.get_source_public_key(ValidatorIndex::new(0)); + let leader_idx = fixture.processor.description.get_leader(SlotIndex::new(slot)); + let leader_key = fixture.processor.description.get_source_public_key(leader_idx); let signature = crate::utils::sign_candidate_u32(&session_id, slot, &candidate_hash, leader_key) .expect("signing failed"); @@ -2261,14 +2774,52 @@ fn test_candidate_data_cache_populated_on_candidate_received() { signature: signature.into(), }); - let candidate_id = RawCandidateId { slot: SlotIndex::new(slot), hash: candidate_hash.clone() }; + let candidate_id = RawCandidateId { slot: SlotIndex::new(slot), hash: candidate_hash }; + (leader_idx.value(), candidate_id, broadcast) +} - assert!( - !fixture.processor.candidate_data_cache.contains_key(&candidate_id), - "cache should be empty before on_candidate_received" +fn make_local_collated_candidate( + fixture: &TestFixture, + seqno: u32, + tag: u8, +) -> Arc { + let block_boc = make_test_boc(&[tag], BocFlags::all()); + let collated_boc = make_test_boc(&[tag.wrapping_add(1)], BocFlags::Crc32); + let root_hash = UInt256::from_slice(&sha256_digest(&block_boc)); + let block_id = BlockIdExt { + shard_id: fixture.processor.description.get_shard().clone(), + seq_no: seqno, + root_hash: root_hash.clone(), + file_hash: root_hash, + }; + let collated_file_hash = UInt256::from_slice(&sha256_digest(&collated_boc)); + let self_idx = fixture.description.get_self_idx().value() as usize; + + Arc::new(crate::ValidatorBlockCandidate { + public_key: fixture.nodes[self_idx].public_key.clone(), + id: block_id, + collated_file_hash, + data: consensus_common::ConsensusCommonFactory::create_block_payload(block_boc), + collated_data: consensus_common::ConsensusCommonFactory::create_block_payload(collated_boc), + }) +} + +#[test] +fn test_candidate_data_cache_populated_on_candidate_received() { + let _ = env_logger::Builder::new().filter_level(log::LevelFilter::Debug).try_init(); + let mut fixture = TestFixture::new(4); + + // Use slot 0 so that validator 0 (local) is the slot leader + let slot = 0u32; + let (leader_source, candidate_id, broadcast) = + make_signed_block_broadcast(&fixture, slot, vec![1u8, 2, 3, 4, 5]); + + assert!( + !fixture.processor.candidate_data_cache.contains_key(&candidate_id), + "cache should be empty before on_candidate_received" ); - fixture.processor.on_candidate_received(0, broadcast, None); + fixture.processor.on_candidate_received(leader_source, broadcast, None); assert!( fixture.processor.candidate_data_cache.contains_key(&candidate_id), @@ -2281,6 +2832,194 @@ fn test_candidate_data_cache_populated_on_candidate_received() { ); } +#[test] +fn test_candidate_ingress_metrics_split_broadcast_and_query() { + let mut fixture = TestFixture::new(4); + + let (leader_source, _, broadcast) = + make_signed_block_broadcast(&fixture, 1, vec![1u8, 2, 3, 4, 5]); + fixture.processor.on_candidate_received(leader_source, broadcast, None); + + assert_eq!(metrics_counter(&fixture.processor, "simplex_candidate_received_broadcast"), 1); + assert_eq!(metrics_counter(&fixture.processor, "simplex_candidate_received_query"), 0); + + let (_, _, query_candidate) = make_signed_block_broadcast(&fixture, 2, vec![9u8, 8, 7, 6]); + fixture.processor.on_candidate_received( + ValidatorIndex::new(3).value(), + query_candidate, + Some(Vec::new()), + ); + + assert_eq!(metrics_counter(&fixture.processor, "simplex_candidate_received_broadcast"), 1); + assert_eq!(metrics_counter(&fixture.processor, "simplex_candidate_received_query"), 1); +} + +#[test] +fn test_old_slot_broadcast_is_dropped_without_persistence_side_effects() { + let mut fixture = TestFixture::new(4); + fixture.processor.simplex_state.set_first_non_finalized_slot(SlotIndex::new(1)); + + let slot = 0u32; + let (leader_source, candidate_id, broadcast) = + make_signed_block_broadcast(&fixture, slot, vec![9u8, 8, 7, 6]); + + fixture.processor.on_candidate_received(leader_source, broadcast, None /* broadcast */); + + assert!( + !fixture.processor.candidate_data_cache.contains_key(&candidate_id), + "old-slot broadcast must not populate candidate_data_cache" + ); + assert!( + !fixture.processor.received_candidates.contains_key(&candidate_id), + "old-slot broadcast must not populate received_candidates" + ); + assert!( + !fixture.processor.seen_broadcast_candidates.contains_key(&SlotIndex::new(slot)), + "old-slot broadcast should be dropped before broadcast dedup state is updated" + ); +} + +#[test] +fn test_candidate_precheck_keeps_simple_addition_rule() { + let mut fixture = TestFixture::new(4); + + let slot = fixture.processor.simplex_state.max_acceptable_slot().value().saturating_add(1); + let (leader_source, candidate_id, broadcast) = + make_signed_block_broadcast(&fixture, slot, vec![7u8, 7, 7, 7]); + + fixture.processor.on_candidate_received(leader_source, broadcast, None); + + assert!( + !fixture.processor.candidate_data_cache.contains_key(&candidate_id), + "candidate above the simple-addition bound must be dropped before caching" + ); + assert_eq!( + metrics_counter(&fixture.processor, "simplex_candidate_precheck_drop_future_slot"), + 1 + ); +} + +#[test] +fn test_candidate_precheck_progress_gap_uses_progress_cursor() { + let mut fixture = TestFixture::new(4); + fixture.drain_receiver_actions(); + + let skip_cert = build_skip_certificate_tl(&SessionId::default(), &fixture.nodes, 0, &[0, 1, 2]); + fixture.processor.on_certificate(1, skip_cert); + fixture.drain_receiver_actions(); + + assert_eq!(fixture.processor.simplex_state.get_first_non_finalized_slot(), SlotIndex::new(0)); + assert_eq!(fixture.processor.simplex_state.get_first_non_progressed_slot(), SlotIndex::new(1)); + + let slot = fixture.processor.simplex_state.max_acceptable_slot().value(); + let (leader_source, candidate_id, broadcast) = + make_signed_block_broadcast(&fixture, slot, vec![8u8, 8, 8, 8]); + + fixture.processor.on_candidate_received(leader_source, broadcast, None); + + assert!( + fixture.processor.candidate_data_cache.contains_key(&candidate_id), + "candidate at the progress-anchored boundary should survive precheck even when finalization lags" + ); +} + +#[test] +fn test_register_resolved_candidate_keeps_slot_behind_progress_cursor_until_finalized() { + let mut fixture = TestFixture::new(4); + fixture.drain_receiver_actions(); + + let skip_cert = build_skip_certificate_tl(&SessionId::default(), &fixture.nodes, 0, &[0, 1, 2]); + fixture.processor.on_certificate(1, skip_cert); + fixture.drain_receiver_actions(); + + assert_eq!(fixture.processor.simplex_state.get_first_non_finalized_slot(), SlotIndex::new(0)); + assert_eq!(fixture.processor.simplex_state.get_first_non_progressed_slot(), SlotIndex::new(1)); + + let slot = SlotIndex::new(0); + let candidate_id = RawCandidateId { slot, hash: UInt256::rand() }; + let raw_candidate = make_test_non_empty_candidate(candidate_id.clone(), None, &fixture.nodes); + let receive_time = fixture.description.get_time(); + + fixture.processor.register_resolved_candidate( + raw_candidate, + slot, + fixture.description.get_self_idx(), + receive_time, + ); + + assert!( + fixture.processor.pending_validations.contains_key(&candidate_id), + "candidate behind first_non_progressed_slot must stay eligible until the slot is finalized" + ); +} + +#[test] +fn test_conflicting_second_broadcast_same_slot_is_dropped_by_precheck() { + let mut fixture = TestFixture::new(4); + let slot = 0u32; + + let (leader_source, first_id, first_broadcast) = + make_signed_block_broadcast(&fixture, slot, vec![1u8, 1, 1, 1]); + fixture.processor.on_candidate_received( + leader_source, + first_broadcast, + None, /* broadcast */ + ); + + assert!( + fixture.processor.received_candidates.contains_key(&first_id), + "first broadcast candidate should be accepted" + ); + + let (_, second_id, second_broadcast) = + make_signed_block_broadcast(&fixture, slot, vec![2u8, 2, 2, 2]); + assert_ne!(first_id, second_id, "test setup must create conflicting candidate ids"); + + fixture.processor.on_candidate_received( + leader_source, + second_broadcast, + None, /* broadcast */ + ); + + assert!( + !fixture.processor.received_candidates.contains_key(&second_id), + "conflicting second broadcast for same slot must be dropped" + ); + assert!( + !fixture.processor.candidate_data_cache.contains_key(&second_id), + "conflicting second broadcast must not be persisted in candidate_data_cache" + ); + assert_eq!( + fixture.processor.seen_broadcast_candidates.get(&SlotIndex::new(slot)).cloned(), + Some(first_id), + "slot dedup state should keep first accepted broadcast candidate id" + ); +} + +#[test] +fn test_broadcast_from_unexpected_sender_is_dropped_by_precheck() { + let mut fixture = TestFixture::new(4); + let slot = 0u32; + let (leader_source, candidate_id, broadcast) = + make_signed_block_broadcast(&fixture, slot, vec![3u8, 4, 5, 6]); + let unexpected_sender = (leader_source + 1) % 4; + + fixture.processor.on_candidate_received( + unexpected_sender, + broadcast, + None, /* broadcast */ + ); + + assert!( + !fixture.processor.received_candidates.contains_key(&candidate_id), + "broadcast from non-leader sender must be dropped by precheck" + ); + assert!( + !fixture.processor.candidate_data_cache.contains_key(&candidate_id), + "broadcast from non-leader sender must not be persisted" + ); +} + // ============================================================================ // Protocol Parity Tests (stub body, partial merge, finalized seqno) // ============================================================================ @@ -2338,51 +3077,6 @@ fn test_has_real_candidate_body_returns_false_for_stub() { ); } -#[test] -fn test_handle_block_finalized_requests_triggered_stub_body_when_committed_head_exists() { - let mut fixture = TestFixture::new(4); - - // Simulate an already-committed head, so triggered finalized block enters - // collect_gapless_commit_chain() as a non-genesis continuation. - fixture.processor.last_committed_seqno = Some(100); - fixture.processor.last_committed_block_id = Some(BlockIdExt::with_params( - ShardIdent::masterchain(), - 100, - UInt256::rand(), - UInt256::rand(), - )); - - let slot = SlotIndex::new(555); - let block_hash = UInt256::rand(); - let candidate_id = RawCandidateId { slot, hash: block_hash.clone() }; - let finalized_block_id = - BlockIdExt::with_params(ShardIdent::masterchain(), 101, UInt256::rand(), UInt256::rand()); - - let final_cert = Arc::new(crate::certificate::Certificate { - vote: crate::simplex_state::FinalizeVote { slot, block_hash: block_hash.clone() }, - signatures: vec![ - crate::certificate::VoteSignature::new(ValidatorIndex::new(0), vec![0u8; 64]), - crate::certificate::VoteSignature::new(ValidatorIndex::new(1), vec![1u8; 64]), - crate::certificate::VoteSignature::new(ValidatorIndex::new(2), vec![2u8; 64]), - ], - }); - - fixture.processor.handle_block_finalized(BlockFinalizedEvent { - slot, - block_hash: block_hash.clone(), - block_id: Some(finalized_block_id), - certificate: final_cert, - }); - - assert!( - fixture.processor.requested_candidates.contains_key(&candidate_id), - "triggered finalized-boundary stub must be treated as missing body and requested" - ); - - // The core regression guard is scheduling requestCandidate at processor level. - // (Receiver send timing is exercised by dedicated delayed-action tests.) -} - #[test] fn test_candidate_query_fallback_returns_notar_only_when_body_missing() { let mut fixture = TestFixture::new(4); @@ -2395,50 +3089,309 @@ fn test_candidate_query_fallback_returns_notar_only_when_body_missing() { }); // No candidate in cache or DB => should return empty/empty - fixture.processor.handle_candidate_query_fallback(slot, block_hash, false, callback); + fixture.processor.handle_candidate_query_fallback(slot, block_hash, false, true, callback); let result = rx.recv().unwrap(); assert!(result.is_ok(), "should return Ok even when nothing found"); } #[test] -fn test_set_mc_finalized_seqno_couples_consensus_finalized_seqno() { +fn test_initial_mc_tracking_seeds_from_initial_block_seqno() { + let fixture = TestFixture::new(4); + + assert_eq!(fixture.processor.last_mc_finalized_seqno, Some(0)); + assert_eq!(fixture.processor.last_consensus_finalized_seqno, Some(0)); + assert_eq!(fixture.processor.accepted_normal_head_seqno, 0); + assert!( + fixture.processor.accepted_normal_head_block_id.is_none(), + "exact accepted head should be unknown until manager/recovery/finalization supplies a block id" + ); +} + +#[test] +fn test_set_mc_finalized_block_couples_consensus_finalized_seqno() { let mut fixture = TestFixture::new(4); // Initially 0 + assert_eq!(fixture.processor.last_mc_finalized_seqno, Some(0)); assert_eq!(fixture.processor.last_consensus_finalized_seqno, Some(0)); - // Set MC finalized to 42 - fixture.processor.set_mc_finalized_seqno(42); + // Set MC-registered top to seqno 42 for this session shard. + let mc_registered_top = + BlockIdExt::with_params(ShardIdent::masterchain(), 42, UInt256::rand(), UInt256::rand()); + fixture.processor.set_mc_finalized_block(mc_registered_top.clone()); // C++ parity: consensus finalized should advance to max(mc, consensus) + assert_eq!(fixture.processor.last_mc_finalized_seqno, Some(42)); assert_eq!( fixture.processor.last_consensus_finalized_seqno, Some(42), - "set_mc_finalized_seqno should couple to last_consensus_finalized_seqno via max()" + "set_mc_finalized_block should couple to last_consensus_finalized_seqno via max()" ); + assert_eq!(fixture.processor.accepted_normal_head_seqno, 42); + assert_eq!(fixture.processor.accepted_normal_head_block_id.as_ref(), Some(&mc_registered_top)); - // Set consensus finalized higher via direct field (simulating a final commit) + // Set consensus finalized higher via direct field (simulating a finalization) fixture.processor.last_consensus_finalized_seqno = Some(100); // Set MC finalized lower => should NOT decrease consensus - fixture.processor.set_mc_finalized_seqno(50); + let older_top = + BlockIdExt::with_params(ShardIdent::masterchain(), 50, UInt256::rand(), UInt256::rand()); + fixture.processor.set_mc_finalized_block(older_top); assert_eq!( fixture.processor.last_consensus_finalized_seqno, Some(100), - "set_mc_finalized_seqno must not decrease last_consensus_finalized_seqno" + "set_mc_finalized_block must not decrease last_consensus_finalized_seqno" ); // Monotonic MC seqno: out-of-order MC event with lower seqno must not regress fixture.processor.last_mc_finalized_seqno = Some(200); - fixture.processor.set_mc_finalized_seqno(150); + let out_of_order_top = + BlockIdExt::with_params(ShardIdent::masterchain(), 150, UInt256::rand(), UInt256::rand()); + fixture.processor.set_mc_finalized_block(out_of_order_top); assert_eq!( fixture.processor.last_mc_finalized_seqno, Some(200), - "set_mc_finalized_seqno must keep last_mc_finalized_seqno monotonic" + "set_mc_finalized_block must keep last_mc_finalized_seqno monotonic" ); } +#[test] +fn test_set_mc_finalized_block_ignores_mismatched_shard() { + let mut fixture = TestFixture::new(4); + assert!(fixture.description.get_shard().is_masterchain()); + + fixture.processor.last_mc_finalized_seqno = Some(123); + fixture.processor.last_consensus_finalized_seqno = Some(123); + + let shard_block = BlockIdExt::with_params( + ShardIdent::with_tagged_prefix(0, 0x8000_0000_0000_0000).unwrap(), + 777, + UInt256::rand(), + UInt256::rand(), + ); + fixture.processor.set_mc_finalized_block(shard_block); + + assert_eq!( + fixture.processor.last_mc_finalized_seqno, + Some(123), + "mismatched shard update must be ignored" + ); + assert_eq!( + fixture.processor.last_consensus_finalized_seqno, + Some(123), + "consensus finalized must not change on mismatched shard update" + ); +} + +#[test] +fn test_check_validation_waits_for_mc_applied_head_before_submitting() { + let mut fixture = TestFixture::new(4); + + let parent_slot = SlotIndex::new(0); + let child_slot = SlotIndex::new(1); + let parent_id = RawCandidateId { slot: parent_slot, hash: UInt256::rand() }; + let child_id = RawCandidateId { slot: child_slot, hash: UInt256::rand() }; + let parent_block_id = + BlockIdExt::with_params(ShardIdent::masterchain(), 1, UInt256::rand(), UInt256::rand()); + + insert_received_candidate( + &mut fixture.processor, + &parent_id, + parent_block_id.clone(), + false, + None, + ); + + let raw_candidate = + make_test_non_empty_candidate(child_id.clone(), Some(parent_id.clone()), &fixture.nodes); + let time = fixture.description.get_time(); + insert_pending_validation(&mut fixture.processor, &child_id, raw_candidate, time); + notarize_slot(&mut fixture, parent_slot, &parent_id.hash); + + fixture.processor.check_validation(); + if strict_mc_session_gate_mode_enabled() { + assert!( + fixture.processor.pending_validations.contains_key(&child_id), + "strict MC gate: candidate must stay pending until accepted head reaches parent normal tip" + ); + assert!( + !fixture.processor.pending_approve.contains(&child_id), + "strict MC gate: candidate must not be submitted while accepted head is behind" + ); + assert!( + !fixture.processor.rejected.contains(&child_id), + "strict MC gate: candidate should wait, not reject, while applied head is behind" + ); + } else { + assert!( + fixture.processor.pending_approve.contains(&child_id), + "validator-side MC mode: candidate should be submitted without SessionProcessor wait" + ); + assert!( + !fixture.processor.rejected.contains(&child_id), + "validator-side MC mode: candidate should not be rejected by SessionProcessor gate" + ); + } + + fixture.processor.set_mc_finalized_block(parent_block_id); + fixture.processor.check_validation(); + assert!( + fixture.processor.pending_approve.contains(&child_id), + "candidate should be submitted once the exact accepted head reaches the parent normal tip" + ); +} + +#[test] +fn test_check_validation_rejects_mc_candidate_with_wrong_exact_parent_head() { + let mut fixture = TestFixture::new(4); + + let parent_slot = SlotIndex::new(0); + let child_slot = SlotIndex::new(1); + let parent_id = RawCandidateId { slot: parent_slot, hash: UInt256::rand() }; + let child_id = RawCandidateId { slot: child_slot, hash: UInt256::rand() }; + let accepted_block_id = BlockIdExt::with_params( + ShardIdent::masterchain(), + 1, + UInt256::from([0x11; 32]), + UInt256::from([0x12; 32]), + ); + let different_parent_block_id = BlockIdExt::with_params( + ShardIdent::masterchain(), + 1, + UInt256::from([0x21; 32]), + UInt256::from([0x22; 32]), + ); + + fixture.processor.set_mc_finalized_block(accepted_block_id); + insert_received_candidate( + &mut fixture.processor, + &parent_id, + different_parent_block_id, + false, + None, + ); + + let raw_candidate = + make_test_non_empty_candidate(child_id.clone(), Some(parent_id.clone()), &fixture.nodes); + let time = fixture.description.get_time(); + insert_pending_validation(&mut fixture.processor, &child_id, raw_candidate, time); + notarize_slot(&mut fixture, parent_slot, &parent_id.hash); + + fixture.processor.check_validation(); + if strict_mc_session_gate_mode_enabled() { + assert!( + fixture.processor.rejected.contains(&child_id), + "strict MC gate: candidate must be rejected when exact accepted head disagrees with parent" + ); + assert!( + !fixture.processor.pending_validations.contains_key(&child_id), + "strict MC gate: stale candidate should be removed from pending after rejection" + ); + } else { + assert!( + fixture.processor.pending_approve.contains(&child_id), + "validator-side MC mode: candidate should be submitted without exact-head rejection" + ); + assert!( + !fixture.processor.rejected.contains(&child_id), + "validator-side MC mode: SessionProcessor should not reject on exact-head mismatch" + ); + } +} + +#[test] +fn test_resolve_parent_normal_tip_walks_empty_parent_chain() { + let mut fixture = TestFixture::new(4); + + let root_id = RawCandidateId { slot: SlotIndex::new(0), hash: UInt256::from([0x01; 32]) }; + let empty_a_id = RawCandidateId { slot: SlotIndex::new(1), hash: UInt256::from([0x02; 32]) }; + let empty_b_id = RawCandidateId { slot: SlotIndex::new(2), hash: UInt256::from([0x03; 32]) }; + let child_id = RawCandidateId { slot: SlotIndex::new(3), hash: UInt256::from([0x04; 32]) }; + let root_block_id = BlockIdExt::with_params( + ShardIdent::masterchain(), + 1, + UInt256::from([0x31; 32]), + UInt256::from([0x32; 32]), + ); + + insert_received_candidate(&mut fixture.processor, &root_id, root_block_id.clone(), false, None); + insert_received_candidate( + &mut fixture.processor, + &empty_a_id, + root_block_id.clone(), + true, + Some(root_id.clone()), + ); + insert_received_candidate( + &mut fixture.processor, + &empty_b_id, + root_block_id.clone(), + true, + Some(empty_a_id.clone()), + ); + + let raw_candidate = + make_test_empty_candidate_with_block(child_id, empty_b_id, root_block_id.clone()); + assert_eq!(fixture.processor.resolve_parent_normal_tip(&raw_candidate), Some(root_block_id)); +} + +#[test] +fn test_recovery_seed_received_candidates_preserves_persisted_empty_records() { + let mut fixture = TestFixture::new(4); + + let c1 = RawCandidateId { slot: SlotIndex::new(1), hash: UInt256::from([0xA1; 32]) }; + let c2 = RawCandidateId { slot: SlotIndex::new(2), hash: UInt256::from([0xA2; 32]) }; + let c3 = RawCandidateId { slot: SlotIndex::new(3), hash: UInt256::from([0xA3; 32]) }; + let b1 = BlockIdExt::with_params( + ShardIdent::masterchain(), + 10, + UInt256::from([0xB1; 32]), + UInt256::from([0xB2; 32]), + ); + let b2 = b1.clone(); + let b3 = BlockIdExt::with_params( + ShardIdent::masterchain(), + 11, + UInt256::from([0xB3; 32]), + UInt256::from([0xB4; 32]), + ); + + fixture.processor.recovery_seed_received_candidates(&[ + FinalizedBlockRecord { + candidate_id: c1.clone(), + block_id: b1.clone(), + parent: None, + is_final: true, + }, + FinalizedBlockRecord { + candidate_id: c2.clone(), + block_id: b2, + parent: Some(c1.clone()), + is_final: true, + }, + FinalizedBlockRecord { + candidate_id: c3.clone(), + block_id: b3.clone(), + parent: Some(c2.clone()), + is_final: true, + }, + ]); + + let root = fixture.processor.received_candidates.get(&c1).expect("root record"); + assert!(!root.is_empty); + + let empty = fixture.processor.received_candidates.get(&c2).expect("empty record"); + assert!(empty.is_empty, "persisted empty MC record must remain marked empty on recovery"); + assert_eq!(empty.parent_id.as_ref(), Some(&c1)); + assert_eq!(empty.block_id, b1); + + let child = fixture.processor.received_candidates.get(&c3).expect("child record"); + assert!(!child.is_empty); + assert_eq!(child.parent_id.as_ref(), Some(&c2)); + assert_eq!(child.block_id, b3); +} + // ============================================================================ // Foreign Certificate Relay Regression Tests (C++ parity) // ============================================================================ @@ -2573,11 +3526,11 @@ fn test_recovery_drain_startup_events_drops_certificate_relay_events() { } // ============================================================================ -// Gapless commit scheduler hardening tests +// Finalized journal cleanup tests // ============================================================================ -/// Verify that `cleanup_old_candidates` removes stale journal entries for old slots -/// and increments the session error counter accordingly. +/// Verify that `cleanup_old_candidates` prunes stale finalized-journal entries for old slots +/// without treating them as session errors. #[test] fn test_journal_cleanup_removes_stale_entries() { let mut fixture = TestFixture::new(4); @@ -2605,7 +3558,7 @@ fn test_journal_cleanup_removes_stale_entries() { let now = fixture.description.get_time(); - fixture.processor.finalized_journal_pending_commit.insert( + fixture.processor.finalized_pending_body.insert( old_id.clone(), FinalizedEntry { event: BlockFinalizedEvent { @@ -2618,7 +3571,7 @@ fn test_journal_cleanup_removes_stale_entries() { }, ); - fixture.processor.finalized_journal_pending_commit.insert( + fixture.processor.finalized_pending_body.insert( current_id.clone(), FinalizedEntry { event: BlockFinalizedEvent { @@ -2631,7 +3584,7 @@ fn test_journal_cleanup_removes_stale_entries() { }, ); - assert_eq!(fixture.processor.finalized_journal_pending_commit.len(), 2); + assert_eq!(fixture.processor.finalized_pending_body.len(), 2); let errors_before = fixture.processor.session_errors_count.load(std::sync::atomic::Ordering::Relaxed); @@ -2639,334 +3592,339 @@ fn test_journal_cleanup_removes_stale_entries() { // Cleanup slots < 10 — old_slot(5) should be removed, current_slot(20) kept. fixture.processor.cleanup_old_candidates(SlotIndex::new(10)); - assert_eq!(fixture.processor.finalized_journal_pending_commit.len(), 1); - assert!(!fixture.processor.finalized_journal_pending_commit.contains_key(&old_id)); - assert!(fixture.processor.finalized_journal_pending_commit.contains_key(¤t_id)); + assert_eq!(fixture.processor.finalized_pending_body.len(), 1); + assert!(!fixture.processor.finalized_pending_body.contains_key(&old_id)); + assert!(fixture.processor.finalized_pending_body.contains_key(¤t_id)); let errors_after = fixture.processor.session_errors_count.load(std::sync::atomic::Ordering::Relaxed); - assert_eq!(errors_after - errors_before, 1, "stale journal entry should increment error count"); + assert_eq!( + errors_after, errors_before, + "stale finalized-journal entries should be pruned without incrementing error count" + ); } -/// Verify that the scheduler processes entries in seqno-ascending order, -/// not arbitrary HashMap order. -/// Both entries are WaitingForFinalCert on MC (seqno ahead of committed head) -/// so neither can commit. Both stay pending in the journal. +// ============================================================================ +// on_block_finalized / maybe_apply_finalized_state tests +// ============================================================================ + +/// Finalized block with body present must advance `finalized_head_seqno` +/// and `last_consensus_finalized_seqno`. #[test] -fn test_try_commit_processes_in_seqno_order() { - let mut fixture = TestFixture::new(4); +fn test_finalized_with_body_advances_committed_seqno() { + let mut opts = SessionOptions::default(); + opts.use_callback_thread = false; + let mut fixture = TestFixture::new_with_opts(4, opts); - // TestFixture defaults to masterchain. Committed head seqno = 10. - let committed_block_id = ton_block::BlockIdExt::with_params( - ton_block::ShardIdent::masterchain(), - 10, - UInt256::rand(), - UInt256::rand(), - ); - fixture.processor.last_committed_seqno = Some(10); - fixture.processor.last_committed_block_id = Some(committed_block_id); + let recording = RecordingListener::new(); + let listener: Arc = recording.clone(); + fixture.processor.listener = Arc::downgrade(&listener); - // Both seqnos are ahead of expected (11), so MC fast-path returns - // WaitingForFinalCert and both entries remain in the journal. - let slot_a = SlotIndex::new(30); - let hash_a = UInt256::rand(); - let id_a = RawCandidateId { slot: slot_a, hash: hash_a.clone() }; - let block_id_a = ton_block::BlockIdExt::with_params( - ton_block::ShardIdent::masterchain(), - 13, // ahead: expected=11 - UInt256::rand(), - UInt256::rand(), + assert_eq!(fixture.processor.finalized_head_seqno, Some(0)); + assert_eq!(fixture.processor.last_consensus_finalized_seqno, Some(0)); + + let slot = 5u32; + let (leader_source, candidate_id, broadcast) = + make_signed_block_broadcast(&fixture, slot, vec![1, 2, 3, 4]); + fixture.processor.on_candidate_received(leader_source, broadcast, None); + + let received = fixture + .processor + .received_candidates + .get(&candidate_id) + .expect("candidate must be present") + .clone(); + + let event = BlockFinalizedEvent { + slot: candidate_id.slot, + block_hash: candidate_id.hash.clone(), + block_id: Some(received.block_id.clone()), + certificate: make_test_final_cert(candidate_id.slot, candidate_id.hash.clone()), + }; + fixture.processor.handle_block_finalized(event); + + assert_eq!( + fixture.processor.finalized_head_seqno, + Some(received.block_id.seq_no()), + "finalized_head_seqno must advance to finalized block seqno" + ); + assert_eq!( + fixture.processor.last_consensus_finalized_seqno, + Some(received.block_id.seq_no()), + "last_consensus_finalized_seqno must advance to finalized block seqno" + ); + assert!( + fixture.processor.finalized_blocks.contains(&candidate_id), + "candidate must be in finalized_blocks set" ); - let slot_b = SlotIndex::new(25); - let hash_b = UInt256::rand(); - let id_b = RawCandidateId { slot: slot_b, hash: hash_b.clone() }; - let block_id_b = ton_block::BlockIdExt::with_params( - ton_block::ShardIdent::masterchain(), - 12, // ahead: expected=11 - UInt256::rand(), - UInt256::rand(), + let events = recording.drain_events(); + assert!( + events.iter().any(|e| matches!(e, ListenerEvent::Finalized { .. })), + "on_block_finalized callback must be emitted" ); +} - for (id, slot, hash, block_id) in - [(&id_a, slot_a, &hash_a, &block_id_a), (&id_b, slot_b, &hash_b, &block_id_b)] - { - fixture.processor.received_candidates.insert( - id.clone(), - ReceivedCandidate { - slot, - source_idx: ValidatorIndex::new(0), - candidate_id_hash: hash.clone(), - candidate_hash_data_bytes: vec![1, 2, 3], - block_id: block_id.clone(), - root_hash: block_id.root_hash.clone(), - file_hash: block_id.file_hash.clone(), - data: consensus_common::ConsensusCommonFactory::create_block_payload( - vec![0xAA].into(), - ), - collated_data: consensus_common::ConsensusCommonFactory::create_block_payload( - vec![0xBB].into(), - ), - receive_time: fixture.description.get_time(), - is_empty: false, - parent_id: None, - is_fully_resolved: true, - }, - ); +/// Out-of-order finalization: higher seqno finalized first, then lower seqno. +/// Both must advance cursors monotonically (never decrease). +#[test] +fn test_finalized_out_of_order_seqno_advances_monotonically() { + let mut opts = SessionOptions::default(); + opts.use_callback_thread = false; + let mut fixture = TestFixture::new_with_opts(4, opts); - let cert: crate::certificate::FinalCertPtr = Arc::new(crate::certificate::Certificate { - vote: crate::simplex_state::FinalizeVote { slot, block_hash: hash.clone() }, - signatures: vec![ - crate::certificate::VoteSignature::new(ValidatorIndex::new(0), vec![0u8; 64]), - crate::certificate::VoteSignature::new(ValidatorIndex::new(1), vec![1u8; 64]), - crate::certificate::VoteSignature::new(ValidatorIndex::new(2), vec![2u8; 64]), - ], - }); - fixture.processor.finalized_journal_pending_commit.insert( - id.clone(), - FinalizedEntry { - event: BlockFinalizedEvent { - slot, - block_hash: hash.clone(), - block_id: Some(block_id.clone()), - certificate: cert, - }, - finalized_at: fixture.description.get_time(), - }, - ); - } + let recording = RecordingListener::new(); + let listener: Arc = recording.clone(); + fixture.processor.listener = Arc::downgrade(&listener); + + let slot_high = 10u32; + let (leader_high, id_high, broadcast_high) = + make_signed_block_broadcast(&fixture, slot_high, vec![10, 20, 30]); + fixture.processor.on_candidate_received(leader_high, broadcast_high, None); + + let received_high = fixture.processor.received_candidates.get(&id_high).unwrap().clone(); + + let event_high = BlockFinalizedEvent { + slot: id_high.slot, + block_hash: id_high.hash.clone(), + block_id: Some(received_high.block_id.clone()), + certificate: make_test_final_cert(id_high.slot, id_high.hash.clone()), + }; + fixture.processor.handle_block_finalized(event_high); - fixture.processor.try_commit_finalized_chains(); + let seqno_after_high = fixture.processor.finalized_head_seqno; + let consensus_after_high = fixture.processor.last_consensus_finalized_seqno; + + let slot_low = 3u32; + let (leader_low, id_low, broadcast_low) = + make_signed_block_broadcast(&fixture, slot_low, vec![40, 50, 60]); + fixture.processor.on_candidate_received(leader_low, broadcast_low, None); + + let received_low = fixture.processor.received_candidates.get(&id_low).unwrap().clone(); + + let event_low = BlockFinalizedEvent { + slot: id_low.slot, + block_hash: id_low.hash.clone(), + block_id: Some(received_low.block_id.clone()), + certificate: make_test_final_cert(id_low.slot, id_low.hash.clone()), + }; + fixture.processor.handle_block_finalized(event_low); + + assert!( + fixture.processor.finalized_head_seqno >= seqno_after_high, + "finalized_head_seqno must not decrease after lower-seqno finalization: \ + before={:?} after={:?}", + seqno_after_high, + fixture.processor.finalized_head_seqno, + ); + assert!( + fixture.processor.last_consensus_finalized_seqno >= consensus_after_high, + "last_consensus_finalized_seqno must not decrease after lower-seqno finalization: \ + before={:?} after={:?}", + consensus_after_high, + fixture.processor.last_consensus_finalized_seqno, + ); + assert!(fixture.processor.finalized_blocks.contains(&id_high)); + assert!(fixture.processor.finalized_blocks.contains(&id_low)); - // Both entries remain pending — MC blocks ahead of committed head wait for FinalCert. - assert!(fixture.processor.finalized_journal_pending_commit.contains_key(&id_a)); - assert!(fixture.processor.finalized_journal_pending_commit.contains_key(&id_b)); - assert_eq!(fixture.processor.last_committed_seqno, Some(10)); + let events = recording.drain_events(); + let finalized_count = + events.iter().filter(|e| matches!(e, ListenerEvent::Finalized { .. })).count(); + assert_eq!(finalized_count, 2, "both blocks must emit on_block_finalized callbacks"); } -/// Verify the finalized_uncommitted_gauge is updated correctly. +/// Duplicate finalization for the same candidate must be deduplicated: +/// second call must not re-emit callback and must not modify cursors. #[test] -fn test_finalized_uncommitted_gauge_tracks_journal_size() { - let mut fixture = TestFixture::new(4); +fn test_finalized_duplicate_is_idempotent() { + let mut opts = SessionOptions::default(); + opts.use_callback_thread = false; + let mut fixture = TestFixture::new_with_opts(4, opts); - // Empty journal — gauge should be 0 (function runs without panic). - fixture.processor.try_commit_finalized_chains(); + let recording = RecordingListener::new(); + let listener: Arc = recording.clone(); + fixture.processor.listener = Arc::downgrade(&listener); - // Add a journal entry that will become AlreadyCommitted - let slot = SlotIndex::new(5); - let hash = UInt256::rand(); - let id = RawCandidateId { slot, hash: hash.clone() }; + let slot = 7u32; + let (leader_source, candidate_id, broadcast) = + make_signed_block_broadcast(&fixture, slot, vec![0xDE, 0xAD]); + fixture.processor.on_candidate_received(leader_source, broadcast, None); - fixture.processor.last_committed_seqno = Some(100); - let committed_block_id = ton_block::BlockIdExt::with_params( - ton_block::ShardIdent::masterchain(), - 100, - UInt256::rand(), - UInt256::rand(), - ); - fixture.processor.last_committed_block_id = Some(committed_block_id.clone()); + let received = fixture.processor.received_candidates.get(&candidate_id).unwrap().clone(); - // Insert a received candidate with seqno < committed so collect_gapless returns AlreadyCommitted - fixture.processor.received_candidates.insert( - id.clone(), - ReceivedCandidate { - slot, - source_idx: ValidatorIndex::new(0), - candidate_id_hash: hash.clone(), - candidate_hash_data_bytes: vec![1, 2, 3], - block_id: committed_block_id.clone(), - root_hash: committed_block_id.root_hash.clone(), - file_hash: committed_block_id.file_hash.clone(), - data: consensus_common::ConsensusCommonFactory::create_block_payload(vec![0xAA].into()), - collated_data: consensus_common::ConsensusCommonFactory::create_block_payload( - vec![0xBB].into(), - ), - receive_time: fixture.description.get_time(), - is_empty: false, - parent_id: None, - is_fully_resolved: true, - }, - ); + let event = BlockFinalizedEvent { + slot: candidate_id.slot, + block_hash: candidate_id.hash.clone(), + block_id: Some(received.block_id.clone()), + certificate: make_test_final_cert(candidate_id.slot, candidate_id.hash.clone()), + }; + fixture.processor.handle_block_finalized(event.clone()); - let cert: crate::certificate::FinalCertPtr = Arc::new(crate::certificate::Certificate { - vote: crate::simplex_state::FinalizeVote { slot, block_hash: hash.clone() }, - signatures: Vec::new(), - }); - fixture.processor.finalized_journal_pending_commit.insert( - id.clone(), - FinalizedEntry { - event: BlockFinalizedEvent { - slot, - block_hash: hash, - block_id: Some(committed_block_id), - certificate: cert, - }, - finalized_at: fixture.description.get_time(), - }, - ); + let seqno_after_first = fixture.processor.finalized_head_seqno; + let consensus_after_first = fixture.processor.last_consensus_finalized_seqno; + let _ = recording.drain_events(); - assert_eq!(fixture.processor.finalized_journal_pending_commit.len(), 1); + fixture.processor.handle_block_finalized(event); - fixture.processor.try_commit_finalized_chains(); + assert_eq!(fixture.processor.finalized_head_seqno, seqno_after_first); + assert_eq!(fixture.processor.last_consensus_finalized_seqno, consensus_after_first); - // The AlreadyCommitted entry should be removed + let events_after_dup = recording.drain_events(); assert!( - fixture.processor.finalized_journal_pending_commit.is_empty(), - "AlreadyCommitted entry should be removed from journal" + !events_after_dup.iter().any(|e| matches!(e, ListenerEvent::Finalized { .. })), + "duplicate finalization must not re-emit on_block_finalized callback" ); } -/// Verify that seqno-sorted iteration commits sequential chains in a single pass -/// and schedules an immediate re-check via set_next_awake_time(now). +/// Empty-block finalization must NOT advance `finalized_head_seqno` (empty blocks +/// keep parent seqno), but the candidate must still be recorded in `finalized_blocks`. #[test] -fn test_sorted_pass_commits_sequential_chains_and_reschedules() { - let mut fixture = TestFixture::new(4); +fn test_finalized_empty_block_does_not_advance_seqno() { + let mut opts = SessionOptions::default(); + opts.use_callback_thread = false; + let mut fixture = TestFixture::new_with_opts(4, opts); - // Committed head at seqno 10 - let committed_block_id = ton_block::BlockIdExt::with_params( - ton_block::ShardIdent::masterchain(), - 10, + fixture.processor.finalized_head_seqno = Some(50); + fixture.processor.finalized_head_block_id = Some(BlockIdExt::with_params( + ShardIdent::masterchain(), + 50, UInt256::rand(), UInt256::rand(), - ); - fixture.processor.last_committed_seqno = Some(10); - fixture.processor.last_committed_block_id = Some(committed_block_id.clone()); + )); - // Build chain: slot_a (seqno 11, parent=boundary) → slot_b (seqno 12, parent=slot_a) - // On MC with matching expected_seqno, the MC fast-path commits the single block. - // After committing slot_a (seqno=11), re-loop should pick up slot_b (seqno=12). - let slot_a = SlotIndex::new(20); - let hash_a = UInt256::rand(); - let id_a = RawCandidateId { slot: slot_a, hash: hash_a.clone() }; - let block_id_a = ton_block::BlockIdExt::with_params( - ton_block::ShardIdent::masterchain(), - 11, - UInt256::rand(), - UInt256::rand(), - ); + let slot = SlotIndex::new(22); + let block_hash = UInt256::rand(); + let candidate_id = RawCandidateId { slot, hash: block_hash.clone() }; - let slot_b = SlotIndex::new(25); - let hash_b = UInt256::rand(); - let id_b = RawCandidateId { slot: slot_b, hash: hash_b.clone() }; - let block_id_b = ton_block::BlockIdExt::with_params( - ton_block::ShardIdent::masterchain(), - 12, + let block_id = BlockIdExt::with_params( + ShardIdent::masterchain(), + 50, // same seqno as parent — empty block UInt256::rand(), UInt256::rand(), ); - // slot_a: parent = None (session boundary → MC fast-path single-commit if seqno matches) fixture.processor.received_candidates.insert( - id_a.clone(), + candidate_id.clone(), ReceivedCandidate { - slot: slot_a, + slot, source_idx: ValidatorIndex::new(0), - candidate_id_hash: hash_a.clone(), + candidate_id_hash: block_hash.clone(), candidate_hash_data_bytes: vec![1, 2, 3], - block_id: block_id_a.clone(), - root_hash: block_id_a.root_hash.clone(), - file_hash: block_id_a.file_hash.clone(), + block_id: block_id.clone(), + root_hash: block_id.root_hash.clone(), + file_hash: block_id.file_hash.clone(), data: consensus_common::ConsensusCommonFactory::create_block_payload(vec![0xAA].into()), collated_data: consensus_common::ConsensusCommonFactory::create_block_payload( vec![0xBB].into(), ), receive_time: fixture.description.get_time(), - is_empty: false, + is_empty: true, parent_id: None, is_fully_resolved: true, }, ); - // slot_b: parent = slot_a (will be WaitingForFinalCert initially since expected=11, seqno=12) - fixture.processor.received_candidates.insert( - id_b.clone(), - ReceivedCandidate { - slot: slot_b, - source_idx: ValidatorIndex::new(0), - candidate_id_hash: hash_b.clone(), - candidate_hash_data_bytes: vec![4, 5, 6], - block_id: block_id_b.clone(), - root_hash: block_id_b.root_hash.clone(), - file_hash: block_id_b.file_hash.clone(), - data: consensus_common::ConsensusCommonFactory::create_block_payload(vec![0xCC].into()), - collated_data: consensus_common::ConsensusCommonFactory::create_block_payload( - vec![0xDD].into(), - ), - receive_time: fixture.description.get_time(), - is_empty: false, - parent_id: Some(id_a.clone()), - is_fully_resolved: true, - }, + let event = BlockFinalizedEvent { + slot, + block_hash: block_hash.clone(), + block_id: Some(block_id), + certificate: make_test_final_cert(slot, block_hash), + }; + fixture.processor.handle_block_finalized(event); + + assert_eq!( + fixture.processor.finalized_head_seqno, + Some(50), + "finalized_head_seqno must not advance for empty-block finalization" ); + assert!( + fixture.processor.finalized_blocks.contains(&candidate_id), + "empty-block candidate must be recorded in finalized_blocks" + ); +} + +/// Multiple finalized blocks with bodies arriving in reverse seqno order: +/// `finalized_head_seqno` must reflect the highest seqno seen. +#[test] +fn test_finalized_reverse_order_keeps_highest_seqno() { + let mut opts = SessionOptions::default(); + opts.use_callback_thread = false; + let mut fixture = TestFixture::new_with_opts(4, opts); + + let recording = RecordingListener::new(); + let listener: Arc = recording.clone(); + fixture.processor.listener = Arc::downgrade(&listener); + + let slots: Vec = vec![20, 15, 10, 5]; + let mut highest_seqno = 0u32; + + for &slot in &slots { + let (leader, cid, bcast) = + make_signed_block_broadcast(&fixture, slot, vec![slot as u8, 0xFF]); + fixture.processor.on_candidate_received(leader, bcast, None); + + let received = fixture.processor.received_candidates.get(&cid).unwrap().clone(); + let seqno = received.block_id.seq_no(); + if seqno > highest_seqno { + highest_seqno = seqno; + } + + let event = BlockFinalizedEvent { + slot: cid.slot, + block_hash: cid.hash.clone(), + block_id: Some(received.block_id.clone()), + certificate: make_test_final_cert(cid.slot, cid.hash.clone()), + }; + fixture.processor.handle_block_finalized(event); + } + + assert_eq!( + fixture.processor.finalized_head_seqno, + Some(highest_seqno), + "finalized_head_seqno must be the highest seqno across all out-of-order finalizations" + ); + assert_eq!( + fixture.processor.last_consensus_finalized_seqno, + Some(highest_seqno), + "last_consensus_finalized_seqno must be the highest seqno across all out-of-order finalizations" + ); + + let events = recording.drain_events(); + let finalized_count = + events.iter().filter(|e| matches!(e, ListenerEvent::Finalized { .. })).count(); + assert_eq!(finalized_count, slots.len(), "all blocks must emit on_block_finalized"); +} + +/// Verify that `finalized_pending_body` is cleaned up when +/// `maybe_apply_finalized_state` runs (body present at finalization time). +#[test] +fn test_finalized_clears_journal_entry_on_apply() { + let mut opts = SessionOptions::default(); + opts.use_callback_thread = false; + let mut fixture = TestFixture::new_with_opts(4, opts); - // Provide FinalCert for both (MC requires FinalCert for non-empty blocks) - for (slot, hash) in [(&slot_a, &hash_a), (&slot_b, &hash_b)] { - let final_cert = Arc::new(crate::certificate::Certificate { - vote: crate::simplex_state::FinalizeVote { slot: *slot, block_hash: hash.clone() }, - signatures: vec![ - crate::certificate::VoteSignature::new(ValidatorIndex::new(0), vec![0u8; 64]), - crate::certificate::VoteSignature::new(ValidatorIndex::new(1), vec![1u8; 64]), - crate::certificate::VoteSignature::new(ValidatorIndex::new(2), vec![2u8; 64]), - ], - }); - fixture - .processor - .simplex_state - .set_finalize_certificate(&fixture.description, *slot, hash, final_cert) - .expect("store final cert"); - } - - // Journal entries for both - for (id, slot, hash, block_id) in - [(&id_a, slot_a, &hash_a, &block_id_a), (&id_b, slot_b, &hash_b, &block_id_b)] - { - let cert: crate::certificate::FinalCertPtr = Arc::new(crate::certificate::Certificate { - vote: crate::simplex_state::FinalizeVote { slot, block_hash: hash.clone() }, - signatures: vec![ - crate::certificate::VoteSignature::new(ValidatorIndex::new(0), vec![0u8; 64]), - crate::certificate::VoteSignature::new(ValidatorIndex::new(1), vec![1u8; 64]), - crate::certificate::VoteSignature::new(ValidatorIndex::new(2), vec![2u8; 64]), - ], - }); - fixture.processor.finalized_journal_pending_commit.insert( - id.clone(), - FinalizedEntry { - event: BlockFinalizedEvent { - slot, - block_hash: hash.clone(), - block_id: Some(block_id.clone()), - certificate: cert, - }, - finalized_at: fixture.description.get_time(), - }, - ); - } + let slot = 12u32; + let (leader_source, candidate_id, broadcast) = + make_signed_block_broadcast(&fixture, slot, vec![0xCA, 0xFE]); + fixture.processor.on_candidate_received(leader_source, broadcast, None); - // Push next_awake_time into the future so we can verify it gets pulled back after commit. - fixture.processor.reset_next_awake_time(); - assert!( - fixture.processor.get_next_awake_time() > fixture.description.get_time(), - "next_awake_time should be in the future before commit" - ); + let received = fixture.processor.received_candidates.get(&candidate_id).unwrap().clone(); - // Because finalized_keys are sorted by seqno, slot_a (seqno=11) is processed - // first. After it commits, last_committed_seqno advances to 11, so when the - // iteration reaches slot_b (seqno=12), expected_seqno matches and it commits too. - fixture.processor.try_commit_finalized_chains(); + let event = BlockFinalizedEvent { + slot: candidate_id.slot, + block_hash: candidate_id.hash.clone(), + block_id: Some(received.block_id.clone()), + certificate: make_test_final_cert(candidate_id.slot, candidate_id.hash.clone()), + }; + fixture.processor.handle_block_finalized(event); - assert_eq!( - fixture.processor.last_committed_seqno, - Some(12), - "sorted iteration should commit both seqno 11 and 12 in one pass" - ); assert!( - fixture.processor.finalized_journal_pending_commit.is_empty(), - "all journal entries should be committed and removed" + fixture.processor.finalized_pending_body.is_empty(), + "journal must be empty after finalization with body present" ); - // Commits happened, so an immediate re-check should be scheduled. assert!( - fixture.processor.get_next_awake_time() <= fixture.description.get_time(), - "next_awake_time should be <= now after a successful commit" + fixture.processor.finalized_blocks.contains(&candidate_id), + "candidate must be in finalized_blocks set" ); } @@ -3030,12 +3988,12 @@ fn test_process_validated_candidates_before_fsm_timeout() { // expected. The key property is that the notarize vote was emitted. } -/// Verify that the `log::warn!` for "drop because new block is already -/// committed" only fires when `cand_seqno <= committed_seqno`, i.e. the -/// candidate is actually dropped. When `cand_seqno > committed_seqno` +/// Verify that the `log::warn!` for "drop because block is already +/// finalized" only fires when `cand_seqno <= finalized_seqno`, i.e. the +/// candidate is actually dropped. When `cand_seqno > finalized_seqno` /// the candidate must proceed to `validated_candidates`. #[test] -fn test_candidate_decision_ok_does_not_drop_when_cand_seqno_greater_than_committed() { +fn test_candidate_decision_ok_does_not_drop_when_cand_seqno_greater_than_finalized() { let mut fixture = TestFixture::new(4); let slot = SlotIndex::new(0); @@ -3046,11 +4004,11 @@ fn test_candidate_decision_ok_does_not_drop_when_cand_seqno_greater_than_committ let time = fixture.description.get_time(); insert_pending_validation(&mut fixture.processor, &candidate_id, raw_candidate, time); - // Set last_committed_seqno to a value BELOW the candidate's seqno. + // Set finalized_head_seqno to a value BELOW the candidate's seqno. // make_test_non_empty_candidate uses slot.value()+1 as seq_no, so for - // slot 0 the candidate seqno = 1. Setting committed to 0 means - // cand_seqno (1) > committed_seqno (0) → candidate must NOT be dropped. - fixture.processor.last_committed_seqno = Some(0); + // slot 0 the candidate seqno = 1. Setting finalized_head to 0 means + // cand_seqno (1) > finalized_seqno (0) → candidate must NOT be dropped. + fixture.processor.finalized_head_seqno = Some(0); // Call the public wrapper which contains the guard. let validity_start = time; @@ -3068,6 +4026,38 @@ fn test_candidate_decision_ok_does_not_drop_when_cand_seqno_greater_than_committ ); } +#[test] +fn test_generated_candidate_validation_missed_metric_increments_on_final_rejection() { + let mut fixture = TestFixture::new(4); + + let slot = SlotIndex::new(0); + let candidate_id = RawCandidateId { slot, hash: UInt256::rand() }; + let raw_candidate = make_test_non_empty_candidate(candidate_id.clone(), None, &fixture.nodes); + let time = fixture.description.get_time(); + insert_pending_validation(&mut fixture.processor, &candidate_id, raw_candidate, time); + + fixture.processor.track_generated_candidate_for_validation(candidate_id.clone()); + fixture.processor.mark_generated_candidate_validation_started(&candidate_id); + fixture.processor.candidate_decision_fail( + slot, + candidate_id.clone(), + error!("validator rejected"), + ); + + assert_eq!( + metrics_counter(&fixture.processor, "simplex_generated_candidate_validation_missed"), + 1 + ); + assert!( + !fixture.processor.generated_candidates_waiting_validation.contains_key(&candidate_id), + "tracking entry must be removed after the miss is recorded" + ); + assert!( + fixture.processor.rejected.contains(&candidate_id), + "final rejection should still mark the candidate as rejected" + ); +} + // ============================================================================ // Candidate Chaining Tests (C++ parity) // ============================================================================ @@ -3193,3 +4183,464 @@ fn test_multi_slot_window_session_creation() { assert_eq!(fixture.description.opts().slots_per_leader_window, 4); assert!(fixture.processor.local_chain_head.is_none()); } + +#[test] +fn test_on_collation_complete_publishes_future_slot_in_current_window() { + // C++ parity: for multi-slot windows, candidates generated for future slots in + // the same active leader window are published immediately. + let opts = SessionOptions { slots_per_leader_window: 4, ..Default::default() }; + let mut fixture = TestFixture::new_with_opts(4, opts); + + let slot = SlotIndex::new(1); + assert_eq!( + fixture.processor.simplex_state.get_first_non_progressed_slot(), + SlotIndex::new(0), + "precondition: progress cursor starts at slot 0" + ); + assert_eq!( + fixture.description.get_window_idx(slot), + fixture.processor.simplex_state.get_current_leader_window_idx(), + "precondition: slot 1 is in current leader window" + ); + + let request_id = 77; + let request = AsyncRequestImpl::new(request_id, false, fixture.description.get_time()); + fixture + .processor + .precollated_blocks + .insert(slot, PrecollatedBlock { request, candidate: None, parent: None }); + + let block_id = + BlockIdExt::with_params(ShardIdent::masterchain(), 1, UInt256::rand(), UInt256::rand()); + let block_boc = make_test_boc(&[0x31], BocFlags::all()); + let collated_boc = make_test_boc(&[0x32], BocFlags::Crc32); + let candidate = crate::ValidatorBlockCandidate { + public_key: fixture.nodes[0].public_key.clone(), + id: block_id, + collated_file_hash: UInt256::from_slice(&sha256_digest(&collated_boc)), + data: consensus_common::ConsensusCommonFactory::create_block_payload(block_boc), + collated_data: consensus_common::ConsensusCommonFactory::create_block_payload(collated_boc), + }; + + fixture.processor.on_collation_complete( + slot, + request_id, + CollationResult::Block(Arc::new(candidate)), + ); + + let actions = fixture.drain_receiver_actions(); + assert!( + actions.iter().any( + |a| matches!(a, ReceiverAction::SendBlockBroadcast { slot: s, .. } if *s == slot.value()) + ), + "future in-window candidate must be broadcast immediately (C++ parity)" + ); + assert!( + fixture.processor.slot_is_generated(slot), + "future in-window slot must be marked generated after immediate publish" + ); +} + +// ============================================================================ +// require_notarized_parent_for_collation tests +// ============================================================================ + +/// When the flag is enabled and the local chain head's parent slot is NOT notarized, +/// precollate_block must defer (return early without invoking collation). +#[test] +fn test_precollate_defers_when_parent_not_notarized_flag_enabled() { + let opts = SessionOptions { + slots_per_leader_window: 4, + require_notarized_parent_for_collation: true, + ..Default::default() + }; + let mut fixture = TestFixture::new_with_opts(4, opts); + + let parent_hash = UInt256::from([0xA1; 32]); + let block_id = BlockIdExt::with_params( + ShardIdent::masterchain(), + 1, + UInt256::from([0xA2; 32]), + UInt256::from([0xA3; 32]), + ); + + fixture.processor.local_chain_head = Some(LocalChainHead { + window: WindowIndex::new(0), + slot: SlotIndex::new(0), + parent_info: crate::block::CandidateParentInfo { + slot: SlotIndex::new(0), + hash: parent_hash.clone(), + }, + block_id: block_id.clone(), + }); + fixture + .processor + .generated_parent_cache + .insert(RawCandidateId { slot: SlotIndex::new(0), hash: parent_hash.clone() }, block_id); + + assert!( + !fixture.processor.simplex_state.has_notarized_block(SlotIndex::new(0)), + "precondition: slot 0 not notarized" + ); + + fixture.processor.precollate_block(SlotIndex::new(1)); + + assert!( + !fixture.processor.precollated_blocks.contains_key(&SlotIndex::new(1)), + "collation must NOT be invoked when parent is not notarized and flag is enabled" + ); +} + +/// When the flag is disabled, precollate_block uses the local chain head even +/// if the parent slot is not notarized (legacy optimistic pipelining). +#[test] +fn test_precollate_proceeds_when_flag_disabled() { + let opts = SessionOptions { + slots_per_leader_window: 4, + require_notarized_parent_for_collation: false, + ..Default::default() + }; + let mut fixture = TestFixture::new_with_opts(4, opts); + + let parent_hash = UInt256::from([0xB1; 32]); + let block_id = BlockIdExt::with_params( + ShardIdent::masterchain(), + 1, + UInt256::from([0xB2; 32]), + UInt256::from([0xB3; 32]), + ); + + fixture.processor.local_chain_head = Some(LocalChainHead { + window: WindowIndex::new(0), + slot: SlotIndex::new(0), + parent_info: crate::block::CandidateParentInfo { + slot: SlotIndex::new(0), + hash: parent_hash.clone(), + }, + block_id: block_id.clone(), + }); + fixture + .processor + .generated_parent_cache + .insert(RawCandidateId { slot: SlotIndex::new(0), hash: parent_hash.clone() }, block_id); + + assert!( + !fixture.processor.simplex_state.has_notarized_block(SlotIndex::new(0)), + "precondition: slot 0 not notarized" + ); + + fixture.processor.precollate_block(SlotIndex::new(1)); + + assert!( + fixture.processor.precollated_blocks.contains_key(&SlotIndex::new(1)), + "collation MUST proceed when flag is disabled (legacy behavior)" + ); +} + +/// When the flag is enabled and the parent IS notarized, precollation proceeds normally. +#[test] +fn test_precollate_proceeds_when_parent_notarized_flag_enabled() { + let opts = SessionOptions { + slots_per_leader_window: 4, + require_notarized_parent_for_collation: true, + ..Default::default() + }; + let mut fixture = TestFixture::new_with_opts(4, opts); + + let parent_hash = UInt256::from([0xC1; 32]); + let block_id = BlockIdExt::with_params( + ShardIdent::masterchain(), + 1, + UInt256::from([0xC2; 32]), + UInt256::from([0xC3; 32]), + ); + + fixture.processor.local_chain_head = Some(LocalChainHead { + window: WindowIndex::new(0), + slot: SlotIndex::new(0), + parent_info: crate::block::CandidateParentInfo { + slot: SlotIndex::new(0), + hash: parent_hash.clone(), + }, + block_id: block_id.clone(), + }); + fixture + .processor + .generated_parent_cache + .insert(RawCandidateId { slot: SlotIndex::new(0), hash: parent_hash.clone() }, block_id); + + // Manually mark slot 0 as notarized in the FSM + fixture.processor.simplex_state.on_block_notarized_for_test( + &fixture.description, + SlotIndex::new(0), + parent_hash.clone(), + ); + assert!( + fixture.processor.simplex_state.has_notarized_block(SlotIndex::new(0)), + "precondition: slot 0 must be notarized" + ); + + // Advance consensus-finalized seqno so should_generate_empty_block returns false + // (initial_block_seqno=1, parent seq_no=1, new_seqno=2, need finalized >= 1) + fixture.processor.last_consensus_finalized_seqno = Some(1); + + fixture.processor.precollate_block(SlotIndex::new(1)); + + assert!( + fixture.processor.precollated_blocks.contains_key(&SlotIndex::new(1)), + "collation must proceed when parent is notarized and flag is enabled" + ); +} + +/// When require_notarized_parent_for_collation is enabled and the local chain head +/// parent becomes notarized, check_collation retries the deferred precollation even +/// if handle_notarization_reached didn't trigger (e.g., local_chain_head was stale). +#[test] +fn test_check_collation_retries_deferred_precollation_after_notarization() { + let opts = SessionOptions { + slots_per_leader_window: 4, + require_notarized_parent_for_collation: true, + ..Default::default() + }; + let mut fixture = TestFixture::new_with_opts(4, opts); + + let parent_hash = UInt256::from([0xD1; 32]); + let block_id = BlockIdExt::with_params( + ShardIdent::masterchain(), + 1, + UInt256::from([0xD2; 32]), + UInt256::from([0xD3; 32]), + ); + + // Simulate: slot 0 was locally generated + fixture.processor.local_chain_head = Some(LocalChainHead { + window: WindowIndex::new(0), + slot: SlotIndex::new(0), + parent_info: crate::block::CandidateParentInfo { + slot: SlotIndex::new(0), + hash: parent_hash.clone(), + }, + block_id: block_id.clone(), + }); + fixture + .processor + .generated_parent_cache + .insert(RawCandidateId { slot: SlotIndex::new(0), hash: parent_hash.clone() }, block_id); + fixture.processor.slot_set_generated(SlotIndex::new(0), true); + fixture.processor.last_consensus_finalized_seqno = Some(1); + + // Slot 0 NOT notarized → precollation for slot 1 should defer (no FSM base either) + assert!(!fixture.processor.simplex_state.has_notarized_block(SlotIndex::new(0))); + + // check_collation: slot 0 already generated → tries precollate_block(1) as retry. + // But parent not notarized AND no FSM base → still deferred. + fixture.processor.check_collation(); + assert!( + !fixture.processor.precollated_blocks.contains_key(&SlotIndex::new(1)), + "slot 1 must not be collated yet (parent not notarized, no FSM base)" + ); + + // Now notarize slot 0 — this propagates FSM base to slot 1. + fixture.processor.simplex_state.on_block_notarized_for_test( + &fixture.description, + SlotIndex::new(0), + parent_hash.clone(), + ); + assert!(fixture.processor.simplex_state.has_notarized_block(SlotIndex::new(0))); + + // check_collation: slot 0 generated → retries precollate_block(1). + // Now the local chain head parent IS notarized → precollation proceeds. + fixture.processor.check_collation(); + assert!( + fixture.processor.precollated_blocks.contains_key(&SlotIndex::new(1)), + "check_collation must retry deferred precollation after parent notarization" + ); +} + +/// The default SessionOptions has require_notarized_parent_for_collation = true. +#[test] +fn test_require_notarized_parent_default_is_true() { + let opts = SessionOptions::default(); + assert!( + opts.require_notarized_parent_for_collation, + "require_notarized_parent_for_collation must default to true" + ); +} + +/// End-to-end session processor test: first leader absent, timeout fires, +/// skip certificate arrives, window advances, second leader (us) collates. +/// +/// Uses time simulation via set_time/advance_time and injects skip certificates +/// from other validators to drive the full timeout → skip → collation pipeline. +#[test] +fn test_second_leader_collates_after_timeout_skip() { + // 4 validators, 1 slot per window → window 0 = slot 0 (leader v0), window 1 = slot 1 (leader v1). + // We are v1 (second leader). + let opts = SessionOptions { + slots_per_leader_window: 1, + first_block_timeout: Duration::from_secs(3), + target_rate: Duration::from_millis(500), + ..Default::default() + }; + let mut fixture = TestFixture::new_with_local_idx(4, 1, opts.clone()); + + assert_eq!( + fixture.description.get_self_idx(), + ValidatorIndex::new(1), + "precondition: we must be validator 1" + ); + assert_eq!( + fixture.description.get_leader(SlotIndex::new(0)), + ValidatorIndex::new(0), + "precondition: v0 leads window 0" + ); + assert_eq!( + fixture.description.get_leader(SlotIndex::new(1)), + ValidatorIndex::new(1), + "precondition: v1 leads window 1" + ); + + // Set deterministic base time and start the session (arms timeouts). + let base_time = SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000); + fixture.processor.set_time(base_time); + fixture.processor.start(); + fixture.drain_receiver_actions(); // clear startup actions + + // Advance time past first_block_timeout + target_rate to trigger timeout for slot 0. + // C++ timeout fires at: base + first_block_timeout + target_rate = 3.5s + fixture.advance_time(opts.first_block_timeout + opts.target_rate + Duration::from_millis(1)); + + // check_all: processes timeout → our node broadcasts SkipVote for slot 0. + fixture.processor.check_all(); + + let actions_after_timeout = fixture.drain_receiver_actions(); + assert!( + actions_after_timeout.iter().any(|a| matches!(a, ReceiverAction::SendVote { .. })), + "we must broadcast a skip vote after timeout" + ); + + // Inject skip certificate from other validators for slot 0 (quorum = 3 of 4). + // We already voted skip, so we include ourselves (v1) plus v2, v3. + let skip_cert = build_skip_certificate_tl( + fixture.description.get_session_id(), + &fixture.nodes, + 0, + &[1, 2, 3], + ); + fixture.processor.on_certificate(2, skip_cert); + + // check_all: processes skip cert → window advances to 1 → check_collation sees + // we are leader for slot 1 → invoke_collation creates a collation request. + fixture.processor.check_all(); + + // The second leader must have initiated collation for slot 1. + let slot1 = SlotIndex::new(1); + assert!( + fixture.processor.precollated_blocks.contains_key(&slot1) + || fixture.processor.slot_is_pending_generate(slot1), + "second leader (v1) must initiate collation for slot 1 after window 0 skip. \ + precollated_blocks={:?}, pending_generate={}", + fixture.processor.precollated_blocks.keys().collect::>(), + fixture.processor.slot_is_pending_generate(slot1), + ); +} + +/// Late-join scenario: node receives only a finalization certificate for a far slot +/// (no candidate body), then receives a child candidate whose parent is that +/// bodyless-but-finalized block. The node must not stall and must start validation. +/// +/// Real-network flow: +/// 1. Node joins late, receives FinalCert for slot 50 (no body) +/// 2. FSM advances first_non_finalized_slot past slot 50 +/// 3. The finalization is recorded in finalized_pending_body (body missing) +/// 4. Skip certificates are injected for intermediate slots (51..54) +/// 5. A new candidate for slot 55 arrives with parent = slot 50 block +/// 6. is_wait_for_parent_ready passes: parent matches get_last_finalize_certificate() +/// 7. check_validation proceeds — candidate enters validation pipeline +#[test] +fn test_late_join_finalization_cert_without_body_then_child_validates() { + let mut fixture = TestFixture::new_shard(4); + + let far_slot = SlotIndex::new(50); + let far_block_hash = UInt256::rand(); + + // 1. Inject finalization certificate for slot 50 into the FSM (simulating + // a late-join node receiving a FinalCert from the network without the body). + let signatures = vec![ + crate::certificate::VoteSignature::new(ValidatorIndex::new(0), vec![10]), + crate::certificate::VoteSignature::new(ValidatorIndex::new(1), vec![11]), + crate::certificate::VoteSignature::new(ValidatorIndex::new(2), vec![12]), + ]; + let finalize_vote = + crate::simplex_state::FinalizeVote { slot: far_slot, block_hash: far_block_hash.clone() }; + let final_cert = Arc::new(crate::certificate::Certificate { vote: finalize_vote, signatures }); + + fixture + .processor + .simplex_state + .set_finalize_certificate( + &fixture.description, + far_slot, + &far_block_hash, + final_cert.clone(), + ) + .expect("set_finalize_certificate failed"); + + // Verify FSM advanced past slot 50. + let first_nf = fixture.processor.simplex_state.get_first_non_finalized_slot(); + assert!( + first_nf > far_slot, + "first_non_finalized_slot must advance past the finalized slot: got {first_nf}, expected > {far_slot}" + ); + + // Process events emitted by set_finalize_certificate (BlockFinalized + FinalizationReached). + fixture.processor.check_all(); + + // The finalization should be in finalized_pending_body since no body exists. + let far_id = RawCandidateId { slot: far_slot, hash: far_block_hash.clone() }; + assert!( + fixture.processor.finalized_pending_body.contains_key(&far_id), + "finalization for bodyless slot 50 must be recorded in finalized_pending_body" + ); + + // finalized_head_seqno must NOT advance (no body to materialize). + let head_before = fixture.processor.finalized_head_seqno; + + // 2. Inject skip certificates for the gap slots 51..55 (exclusive). + for gap in 51..55u32 { + skip_slot(&mut fixture, SlotIndex::new(gap)); + } + + // 3. Create and insert a child candidate for slot 55, parented on slot 50. + let child_slot = SlotIndex::new(55); + let child_hash = UInt256::rand(); + let child_id = RawCandidateId { slot: child_slot, hash: child_hash.clone() }; + let parent_id = RawCandidateId { slot: far_slot, hash: far_block_hash.clone() }; + + let raw_candidate = + make_test_non_empty_candidate(child_id.clone(), Some(parent_id.clone()), &fixture.nodes); + let time = fixture.description.get_time(); + insert_pending_validation(&mut fixture.processor, &child_id, raw_candidate, time); + + // 4. Drive validation: check_validation should find the child candidate eligible + // because its parent matches the last finalize certificate and gaps are skip-covered. + fixture.processor.check_validation(); + + assert!( + fixture.processor.pending_approve.contains(&child_id), + "child candidate at slot 55 must enter validation pipeline (not stall). \ + The parent at slot 50 has a finalization certificate even though the body is missing." + ); + + // finalized_head_seqno must remain unchanged (parent body still missing). + assert_eq!( + fixture.processor.finalized_head_seqno, head_before, + "finalized_head_seqno must NOT advance when the parent body is still missing" + ); + + // finalized_pending_body must still contain the slot 50 entry. + assert!( + fixture.processor.finalized_pending_body.contains_key(&far_id), + "finalized_pending_body must retain slot 50 entry until body arrives" + ); +} diff --git a/src/node/simplex/src/tests/test_simplex_state.rs b/src/node/simplex/src/tests/test_simplex_state.rs index 3ac3276..ce0203c 100644 --- a/src/node/simplex/src/tests/test_simplex_state.rs +++ b/src/node/simplex/src/tests/test_simplex_state.rs @@ -17,7 +17,11 @@ use crate::{ misbehavior::VoteResult, RawVoteData, SessionId, SessionNode, }; -use std::{iter::from_fn, sync::Arc, time::SystemTime}; +use std::{ + iter::from_fn, + sync::Arc, + time::{Duration, SystemTime}, +}; use ton_block::{BlockIdExt, Ed25519KeyOption, ShardIdent, UInt256}; /// Test helper trait to provide on_vote with default raw_vote @@ -119,20 +123,9 @@ fn opts_cpp() -> SimplexStateOptions { SimplexStateOptions::cpp_compatible() } -/// Helper to create SimplexStateOptions for legacy (finalization-driven) mode. -/// -/// This keeps the old ParentReady / `available_bases` behavior for window progression. -fn opts_cpp_legacy() -> SimplexStateOptions { - let mut opts = opts_cpp(); - opts.use_notarized_parent_chain = false; - opts -} - /// Helper to create SimplexStateOptions with notarized-parent chain enabled (pool.cpp parity mode) fn opts_notarized_parent_chain() -> SimplexStateOptions { - let mut opts = opts_cpp(); - opts.use_notarized_parent_chain = true; - opts + opts_cpp() } /// Helper to create test candidate for FSM tests @@ -194,7 +187,6 @@ fn test_new_creates_initial_state() { assert_eq!(state.first_non_finalized_slot, SlotIndex::new(0)); assert_eq!(state.current_leader_window_idx, WindowIndex::new(0)); assert!(!state.has_pending_events()); - // Timeouts start unarmed; SessionProcessor::start() calls set_timeouts(). assert!(state.get_next_timeout().is_none()); // Window 0 should have None (genesis) as available base @@ -838,235 +830,6 @@ fn test_finalize_threshold_66_triggers_block_finalized() { assert_eq!(state.first_non_finalized_slot, SlotIndex::new(1)); } -#[test] -fn test_block_finalized_triggers_parent_ready_for_next_window() { - // When a block is finalized, it should become an available parent for the next window - // This is the "ParentReady" publishing that was missing from C++ reference - let desc = create_test_desc(4, 2); // 2 slots per window - let mut state = - SimplexState::new(&desc, opts_cpp_legacy()).expect("Failed to create SimplexState"); - - // Window 0 starts with genesis (None) as available base - assert!(state.get_window(WindowIndex::new(0)).unwrap().available_bases.contains(&None)); - - // Finalize slot 0 (in window 0) - let mut block = BlockIdExt::default(); - block.root_hash = UInt256::from_slice(&[0xAA; 32]); - - let vote = Vote::Finalize(FinalizeVote { - slot: SlotIndex::new(0), - block_hash: block.root_hash.clone(), - }); - - // Need 3 out of 4 for threshold_66 - state.on_vote_test(&desc, ValidatorIndex::new(0), vote.clone(), Vec::new()).unwrap(); - state.on_vote_test(&desc, ValidatorIndex::new(1), vote.clone(), Vec::new()).unwrap(); - state.on_vote_test(&desc, ValidatorIndex::new(2), vote, Vec::new()).unwrap(); - - // Clear events - while state.pull_event().is_some() {} - - // Check that window 1 now has the finalized block as an available parent - let next_window = state.get_window(WindowIndex::new(1)); - assert!(next_window.is_some(), "Window 1 should exist after finalization"); - - let expected_parent = - Some(CandidateParentInfo { slot: SlotIndex::new(0), hash: block.root_hash.clone() }); - assert!( - next_window.unwrap().available_bases.contains(&expected_parent), - "Window 1 should have finalized block from slot 0 as available parent. Got: {:?}", - next_window.unwrap().available_bases - ); - - // Verify current_leader_window_idx was updated - assert_eq!( - state.current_leader_window_idx, - WindowIndex::new(1), - "Should advance to next window" - ); -} - -#[test] -fn test_parent_ready_enables_pending_block_notarization() { - // When ParentReady is triggered (via finalization), pending blocks should be checked - let desc = create_test_desc(4, 2); // 2 slots per window - let mut state = - SimplexState::new(&desc, opts_cpp_legacy()).expect("Failed to create SimplexState"); - - // Create a candidate for slot 2 (first slot of window 1) that references a parent - let parent_hash = UInt256::from_slice(&[0xBB; 32]); - - // This candidate needs a parent from slot 0 - let candidate = create_test_candidate( - 2, - UInt256::from_slice(&[0xCC; 32]), - BlockIdExt::default(), - Some((0, parent_hash.clone())), - 0, - ); - - // Submit candidate - it should be stored as pending (no parent available) - state.on_candidate(&desc, candidate).unwrap(); - let events: Vec<_> = from_fn(|| state.pull_event()).collect(); - assert!( - !events.iter().any(|e| matches!(e, SimplexEvent::BroadcastVote(Vote::Notarize(_)))), - "Should NOT broadcast notarize vote - parent not available yet" - ); - - // Verify candidate is pending - assert!( - state.get_window(WindowIndex::new(1)).unwrap().slots[0].pending_block.is_some(), - "Candidate should be stored as pending" - ); - - // Now finalize slot 0 with matching parent hash - let mut block = BlockIdExt::default(); - block.root_hash = parent_hash.clone(); - - let vote = Vote::Finalize(FinalizeVote { - slot: SlotIndex::new(0), - block_hash: block.root_hash.clone(), - }); - - state.on_vote_test(&desc, ValidatorIndex::new(0), vote.clone(), Vec::new()).unwrap(); - state.on_vote_test(&desc, ValidatorIndex::new(1), vote.clone(), Vec::new()).unwrap(); - state.on_vote_test(&desc, ValidatorIndex::new(2), vote, Vec::new()).unwrap(); - - // Check that BlockFinalized was emitted and ParentReady logic ran - let events: Vec<_> = from_fn(|| state.pull_event()).collect(); - assert!( - events.iter().any(|e| matches!(e, SimplexEvent::BlockFinalized(_))), - "Should emit BlockFinalized" - ); - - // The pending candidate should now have been processed via check_pending_blocks - // and a NotarVote should have been broadcast - assert!( - events.iter().any(|e| matches!(e, SimplexEvent::BroadcastVote(Vote::Notarize(_)))), - "Should broadcast NotarVote for pending candidate after parent ready" - ); - - // Pending block should be cleared - assert!( - state.get_window(WindowIndex::new(1)).unwrap().slots[0].pending_block.is_none(), - "Pending block should be cleared after notarization" - ); -} - -#[test] -fn test_genesis_propagates_to_next_window_on_full_skip() { - // When an entire window is skipped (no finalization), the genesis (None) parent - // should propagate to the next window as an available base. - // This handles the startup case where network is slow and first window times out. - let desc = create_test_desc(4, 2); // 2 slots per window - let mut state = - SimplexState::new(&desc, opts_cpp_legacy()).expect("Failed to create SimplexState"); - - // Window 0 starts with genesis (None) as available base - assert!(state.get_window(WindowIndex::new(0)).unwrap().available_bases.contains(&None)); - - // Skip slot 0 (need 3 out of 4 for threshold_66) - let skip_vote_0 = Vote::Skip(SkipVote { slot: SlotIndex::new(0) }); - state.on_vote_test(&desc, ValidatorIndex::new(0), skip_vote_0.clone(), Vec::new()).unwrap(); - state.on_vote_test(&desc, ValidatorIndex::new(1), skip_vote_0.clone(), Vec::new()).unwrap(); - state.on_vote_test(&desc, ValidatorIndex::new(2), skip_vote_0, Vec::new()).unwrap(); - - // Clear events - while state.pull_event().is_some() {} - - // Slot 0 should be skipped; C++ parity: first_non_finalized_slot stays at 0 - assert_eq!(state.first_non_finalized_slot, SlotIndex::new(0)); - // But first_non_progressed_slot advances - assert_eq!(state.first_non_progressed_slot, SlotIndex::new(1)); - - // Window 1 should NOT have genesis yet (slot 0 was not the last slot in window 0) - // Note: window 1 may or may not exist at this point - let window1_has_genesis = state - .get_window(WindowIndex::new(1)) - .map(|w| w.available_bases.contains(&None)) - .unwrap_or(false); - assert!( - !window1_has_genesis, - "Window 1 should not have genesis yet (slot 0 is not last in window)" - ); - - // Skip slot 1 (last slot in window 0) - let skip_vote_1 = Vote::Skip(SkipVote { slot: SlotIndex::new(1) }); - state.on_vote_test(&desc, ValidatorIndex::new(0), skip_vote_1.clone(), Vec::new()).unwrap(); - state.on_vote_test(&desc, ValidatorIndex::new(1), skip_vote_1.clone(), Vec::new()).unwrap(); - state.on_vote_test(&desc, ValidatorIndex::new(2), skip_vote_1, Vec::new()).unwrap(); - - // Clear events - while state.pull_event().is_some() {} - - // C++ parity: first_non_finalized_slot still at 0, progress cursor at 2 - assert_eq!(state.first_non_finalized_slot, SlotIndex::new(0)); - assert_eq!(state.first_non_progressed_slot, SlotIndex::new(2)); - - // Window 1 should now have genesis (None) as available base - // This was propagated from window 0 since no finalization occurred - let window1 = state.get_window(WindowIndex::new(1)); - assert!(window1.is_some(), "Window 1 should exist"); - assert!( - window1.unwrap().available_bases.contains(&None), - "Window 1 should have genesis (None) as available base after window 0 was fully skipped. Got: {:?}", - window1.unwrap().available_bases - ); -} - -#[test] -fn test_genesis_not_propagated_if_finalization_occurred() { - // If a block was finalized in the window, genesis should NOT be propagated - // (the finalized block becomes the parent instead) - let desc = create_test_desc(4, 2); // 2 slots per window - let mut state = - SimplexState::new(&desc, opts_cpp_legacy()).expect("Failed to create SimplexState"); - - // Finalize slot 0 - let mut block = BlockIdExt::default(); - block.root_hash = UInt256::from_slice(&[0xAA; 32]); - - let finalize_vote = Vote::Finalize(FinalizeVote { - slot: SlotIndex::new(0), - block_hash: block.root_hash.clone(), - }); - state.on_vote_test(&desc, ValidatorIndex::new(0), finalize_vote.clone(), Vec::new()).unwrap(); - state.on_vote_test(&desc, ValidatorIndex::new(1), finalize_vote.clone(), Vec::new()).unwrap(); - state.on_vote_test(&desc, ValidatorIndex::new(2), finalize_vote, Vec::new()).unwrap(); - - // Clear events - while state.pull_event().is_some() {} - - // Window 1 should have the finalized block as parent, not genesis - let window1 = state.get_window(WindowIndex::new(1)).expect("Window 1 should exist"); - let expected_parent = - Some(CandidateParentInfo { slot: SlotIndex::new(0), hash: block.root_hash.clone() }); - assert!( - window1.available_bases.contains(&expected_parent), - "Window 1 should have finalized block as parent" - ); - - // Now skip slot 1 (last slot in window 0) - let skip_vote = Vote::Skip(SkipVote { slot: SlotIndex::new(1) }); - state.on_vote_test(&desc, ValidatorIndex::new(0), skip_vote.clone(), Vec::new()).unwrap(); - state.on_vote_test(&desc, ValidatorIndex::new(1), skip_vote.clone(), Vec::new()).unwrap(); - state.on_vote_test(&desc, ValidatorIndex::new(2), skip_vote, Vec::new()).unwrap(); - - // Clear events - while state.pull_event().is_some() {} - - // Window 1 should still NOT have genesis (None) - only the finalized block - let window1 = state.get_window(WindowIndex::new(1)).expect("Window 1 should exist"); - assert!( - !window1.available_bases.contains(&None), - "Window 1 should NOT have genesis - finalization already provided a parent" - ); - assert!( - window1.available_bases.contains(&expected_parent), - "Window 1 should still have finalized block as parent" - ); -} - #[test] fn test_safe_to_skip_broadcasts_skip_fallback_but_no_slot_skipped() { // SafeToSkip triggers at 1/3 threshold and calls try_skip_window @@ -1588,47 +1351,6 @@ fn test_invalid_leader_in_candidate() { assert!(result.is_err(), "Invalid leader should return error"); } -#[test] -fn test_on_window_base_ready_makes_pending_available() { - let desc = create_test_desc(4, 2); - let mut state = - SimplexState::new(&desc, opts_cpp_legacy()).expect("Failed to create SimplexState"); - - // Test scenario: window 1 (slots 2-3), parent from slot 1 - // First, create window 1 - state.ensure_window_exists(WindowIndex::new(1)); - - // Create candidate for slot 2 (first slot in window 1) with a parent from slot 1 - let parent_hash = UInt256::from_slice(&[0xAA; 32]); - let parent_info = CandidateParentInfo { slot: SlotIndex::new(1), hash: parent_hash.clone() }; - - let candidate = create_test_candidate( - 2, // First slot in window 1 - UInt256::default(), - BlockIdExt::default(), - Some((1, parent_hash.clone())), - 0, - ); - - // Candidate should be stored as pending (parent not available in window 1) - state.on_candidate(&desc, candidate).unwrap(); - let events: Vec<_> = from_fn(|| state.pull_event()).collect(); - assert!( - !events.iter().any(|e| matches!(e, SimplexEvent::BroadcastVote(Vote::Notarize(_)))), - "Should not broadcast without parent" - ); - - // Now make the parent available for window 1 - state.on_window_base_ready(&desc, WindowIndex::new(1), Some(parent_info)).unwrap(); - - // Should now have broadcast NotarVote - let events: Vec<_> = from_fn(|| state.pull_event()).collect(); - assert!( - events.iter().any(|e| matches!(e, SimplexEvent::BroadcastVote(Vote::Notarize(_)))), - "Should broadcast after parent ready" - ); -} - /* ======================================================================== Timeout Tests @@ -1638,12 +1360,17 @@ fn test_on_window_base_ready_makes_pending_available() { #[test] fn test_timeout_management() { let desc = create_test_desc(4, 2); - let state = SimplexState::new(&desc, opts_cpp()).expect("Failed to create SimplexState"); + let mut state = SimplexState::new(&desc, opts_cpp()).expect("Failed to create SimplexState"); // FSM is created with unarmed timeouts (skip_timestamp = None). - // SessionProcessor::start() is responsible for calling set_timeouts(). + // SessionProcessor::start() is responsible for calling reset_timeouts_on_start(). assert!(state.get_next_timeout().is_none(), "FSM must start with unarmed timeouts"); assert_eq!(state.skip_slot, SlotIndex::new(0)); + + // Arm timeouts (simulating start()) + state.reset_timeouts_on_start(&desc); + assert!(state.get_next_timeout().is_some(), "reset_timeouts_on_start must set skip_timestamp"); + assert_eq!(state.skip_slot, SlotIndex::new(0)); } #[test] @@ -1653,8 +1380,8 @@ fn test_unarmed_fsm_no_skip_cascade_after_delay() { let desc = create_test_desc(4, 2); let mut state = SimplexState::new(&desc, opts_cpp()).expect("Failed to create SimplexState"); - // Simulate 60 s overlay warmup delay - let future = desc.get_time() + std::time::Duration::from_secs(60); + // Simulate 60 s overlay warmup delay without arming timeouts + let future = desc.get_time() + Duration::from_secs(60); desc.set_time(future); state.check_all(&desc); @@ -1669,16 +1396,16 @@ fn test_unarmed_fsm_no_skip_cascade_after_delay() { } #[test] -fn test_set_timeouts_enables_skip_after_expiry() { - // After set_timeouts() the skip timer fires once the deadline elapses. +fn test_armed_timeouts_enable_skip_after_expiry() { + // After reset_timeouts_on_start() the skip timer fires once the deadline elapses. let desc = create_test_desc(4, 2); let mut state = SimplexState::new(&desc, opts_cpp()).expect("Failed to create SimplexState"); let t0 = desc.get_time(); // Arm at t0 - state.set_timeouts(&desc); - assert!(state.get_next_timeout().is_some(), "set_timeouts must set skip_timestamp"); + state.reset_timeouts_on_start(&desc); + assert!(state.get_next_timeout().is_some()); // Immediately after arming, check_all should produce no skips state.check_all(&desc); @@ -1691,7 +1418,7 @@ fn test_set_timeouts_enables_skip_after_expiry() { assert_eq!(skip_count, 0, "no skip votes before timeout expires"); // Advance past first_block_timeout + target_rate (defaults: 3s + 1s = 4s) - desc.set_time(t0 + std::time::Duration::from_secs(5)); + desc.set_time(t0 + Duration::from_secs(5)); state.check_all(&desc); let mut skip_count = 0; @@ -2008,122 +1735,6 @@ fn test_get_available_parent_nonexistent_window() { Tests for scenarios where votes arrive before the block candidate. */ -#[test] -fn test_late_candidate_finalization_still_proceeds() { - // Scenario: Finalize votes arrive and reach threshold BEFORE the candidate arrives. - // The FSM should: - // 1. Emit BlockFinalized event (with block_id = None since candidate wasn't seen) - // 2. Advance to next slot - // 3. Ignore the late candidate when it arrives (slot already finalized) - // 4. Continue processing the next slot normally - - let desc = create_test_desc(4, 2); // 4 validators, 2 slots per window - let mut state = - SimplexState::new(&desc, opts_cpp_legacy()).expect("Failed to create SimplexState"); - - // Create a candidate hash (the hash that will be in the votes) - let candidate_hash = UInt256::from_slice(&[0xAA; 32]); - - // Step 1: Send finalize votes for slot 0 (without having received the candidate) - // Need 3 out of 4 for threshold_66 - let vote = Vote::Finalize(FinalizeVote { - slot: SlotIndex::new(0), - block_hash: candidate_hash.clone(), - }); - - state.on_vote_test(&desc, ValidatorIndex::new(0), vote.clone(), Vec::new()).unwrap(); - state.on_vote_test(&desc, ValidatorIndex::new(1), vote.clone(), Vec::new()).unwrap(); - state.on_vote_test(&desc, ValidatorIndex::new(2), vote, Vec::new()).unwrap(); - - // Step 2: Verify BlockFinalized was emitted - let events: Vec<_> = from_fn(|| state.pull_event()).collect(); - - let finalized_event = events.iter().find_map(|e| { - if let SimplexEvent::BlockFinalized(ev) = e { - Some(ev) - } else { - None - } - }); - - assert!(finalized_event.is_some(), "Expected BlockFinalized event"); - let finalized = finalized_event.unwrap(); - - // block_id should be None because on_candidate wasn't called - assert_eq!(finalized.slot, SlotIndex::new(0)); - assert_eq!(finalized.block_hash, candidate_hash); - assert!(finalized.block_id.is_none(), "block_id should be None for late candidate scenario"); - - // Step 3: State should have advanced - assert_eq!( - state.first_non_finalized_slot, - SlotIndex::new(1), - "Should advance to slot 1 after finalization" - ); - - // Step 4: Now the late candidate arrives for slot 0 - let block_id = BlockIdExt::with_params( - ShardIdent::masterchain(), - 0, - candidate_hash.clone(), - UInt256::default(), - ); - let late_candidate = create_test_candidate(0, candidate_hash.clone(), block_id, None, 0); - - // on_candidate should succeed but do nothing (slot already finalized) - let result = state.on_candidate(&desc, late_candidate); - assert!(result.is_ok(), "on_candidate should succeed for late candidate"); - - // No new events should be generated - let events_after: Vec<_> = from_fn(|| state.pull_event()).collect(); - assert!( - events_after.is_empty(), - "No events should be generated for late candidate in finalized slot" - ); - - // Step 5: Continue with slot 1 - the FSM should still work normally - // Window 1 should have the finalized block from slot 0 as an available parent - let window1 = state.get_window(WindowIndex::new(1)); - assert!(window1.is_some(), "Window 1 should exist"); - - let expected_parent = - Some(CandidateParentInfo { slot: SlotIndex::new(0), hash: candidate_hash.clone() }); - assert!( - window1.unwrap().available_bases.contains(&expected_parent), - "Window 1 should have finalized slot 0 as available parent" - ); - - // Create a candidate for slot 2 (first slot of window 1) with parent from slot 0 - let slot2_hash = UInt256::from_slice(&[0xBB; 32]); - let slot2_block_id = BlockIdExt::with_params( - ShardIdent::masterchain(), - 1, - slot2_hash.clone(), - UInt256::default(), - ); - let slot2_candidate = create_test_candidate( - 2, - slot2_hash.clone(), - slot2_block_id, - Some((0, candidate_hash.clone())), // Parent is the finalized slot 0 - 1, // Leader for window 1 - ); - - // This should trigger a notarize vote (ParentReady was set by finalization) - state.on_candidate(&desc, slot2_candidate).unwrap(); - - let slot2_events: Vec<_> = from_fn(|| state.pull_event()).collect(); - let has_notarize = slot2_events.iter().any(|e| { - matches!(e, SimplexEvent::BroadcastVote(Vote::Notarize(v)) if v.slot == SlotIndex::new(2)) - }); - - assert!( - has_notarize, - "FSM should broadcast notarize vote for slot 2 after late candidate scenario. Events: {:?}", - slot2_events - ); -} - #[test] fn test_late_candidate_with_notarize_votes_also_proceeds() { // More comprehensive test: Both notarize AND finalize votes arrive before candidate @@ -3259,8 +2870,7 @@ fn test_skip_events_emitted_when_threshold_reached() { // Test that SlotSkipped events are emitted immediately when // threshold is reached, regardless of slot order. let desc = create_test_desc(4, 2); - let mut state = - SimplexState::new(&desc, opts_cpp_legacy()).expect("Failed to create SimplexState"); + let mut state = SimplexState::new(&desc, opts_cpp()).expect("Failed to create SimplexState"); // Skip slot 1 first (before slot 0) let vote1 = Vote::Skip(SkipVote { slot: SlotIndex::new(1) }); @@ -4057,52 +3667,17 @@ fn test_cpp_mode_local_notarize_after_skip() { } #[test] -fn test_restart_voted_notar_parent_chain() { - // After restart with voted_notar(parent) in slot 0, try_notar for slot 1 (same window) - // must accept that parent and broadcast NotarizeVote. - // Reference: C++ consensus.cpp try_notarize() uses voted_notar from prev slot as parent. +fn test_notarized_parent_chain_state_tracked_in_default_mode_on_notarization() { + // Notarized-parent chain fields (`available_base`, `skipped`, + // `first_non_progressed_slot`) are maintained in the active C++-parity mode. let desc = create_test_desc(4, 2); - let mut state = - SimplexState::new(&desc, opts_cpp_legacy()).expect("Failed to create SimplexState"); + let mut state = SimplexState::new(&desc, opts_cpp()).expect("Failed to create state"); - let parent_hash = UInt256::from([0x44u8; 32]); - state.mark_slot_voted_on_restart( - &desc, - &Vote::Notarize(NotarizeVote { slot: SlotIndex::new(0), block_hash: parent_hash.clone() }), - ); - - // Non-first slot in same window: slot 1 uses voted_notar(slot 0) as parent proof. - let child_hash = UInt256::from([0x55u8; 32]); - let ok = state.try_notar( - &desc, - SlotIndex::new(1), - &child_hash, - Some(&crate::block::CandidateParentInfo { slot: SlotIndex::new(0), hash: parent_hash }), - ); - assert!(ok, "try_notar should succeed when parent matches voted_notar(prev_slot)"); - - let mut saw_notar_1 = false; - while let Some(ev) = state.pull_event() { - if matches!(ev, SimplexEvent::BroadcastVote(Vote::Notarize(NotarizeVote { slot, .. })) if slot == SlotIndex::new(1)) - { - saw_notar_1 = true; - } - } - assert!(saw_notar_1, "expected notarize broadcast for slot 1"); -} - -#[test] -fn test_notarized_parent_chain_state_tracked_in_default_mode_on_notarization() { - // Notarized-parent chain fields (`available_base`, `skipped`, `first_non_progressed_slot`) are maintained - // in all modes for state consistency, even when `use_notarized_parent_chain=false`. - let desc = create_test_desc(4, 2); - let mut state = SimplexState::new(&desc, opts_cpp_legacy()).expect("Failed to create state"); - - let h0 = UInt256::from([0xC0u8; 32]); - let vote0 = Vote::Notarize(NotarizeVote { slot: SlotIndex::new(0), block_hash: h0.clone() }); - state.on_vote_test(&desc, ValidatorIndex::new(0), vote0.clone(), vec![1]).unwrap(); - state.on_vote_test(&desc, ValidatorIndex::new(1), vote0.clone(), vec![2]).unwrap(); - state.on_vote_test(&desc, ValidatorIndex::new(2), vote0, vec![3]).unwrap(); + let h0 = UInt256::from([0xC0u8; 32]); + let vote0 = Vote::Notarize(NotarizeVote { slot: SlotIndex::new(0), block_hash: h0.clone() }); + state.on_vote_test(&desc, ValidatorIndex::new(0), vote0.clone(), vec![1]).unwrap(); + state.on_vote_test(&desc, ValidatorIndex::new(1), vote0.clone(), vec![2]).unwrap(); + state.on_vote_test(&desc, ValidatorIndex::new(2), vote0, vec![3]).unwrap(); // Tracking: slot0 notarized => progress cursor advances to 1 and slot1 base is set. assert_eq!(state.first_non_progressed_slot, SlotIndex::new(1)); @@ -4113,16 +3688,15 @@ fn test_notarized_parent_chain_state_tracked_in_default_mode_on_notarization() { "slot1 base must be set from notarized slot0 (tracking only)" ); - // Behavior remains legacy: window progression is still driven by ParentReady/finalization. + // Slot 0 is still inside window 0, so the leader window does not advance yet. assert_eq!(state.current_leader_window_idx, WindowIndex::new(0)); } #[test] fn test_notarized_parent_chain_state_tracked_in_default_mode_on_skip_cert() { - // Notarized-parent chain fields are maintained in all modes for state consistency, - // even when `use_notarized_parent_chain=false`. + // Skip certificates must update the active C++-parity tracking state too. let desc = create_test_desc(4, 2); - let mut state = SimplexState::new(&desc, opts_cpp_legacy()).expect("Failed to create state"); + let mut state = SimplexState::new(&desc, opts_cpp()).expect("Failed to create state"); let vote0 = Vote::Skip(SkipVote { slot: SlotIndex::new(0) }); state.on_vote_test(&desc, ValidatorIndex::new(0), vote0.clone(), vec![1]).unwrap(); @@ -4139,7 +3713,7 @@ fn test_notarized_parent_chain_state_tracked_in_default_mode_on_skip_cert() { "slot1 base must be propagated genesis from skipped slot0" ); - // Behavior remains legacy: window progression is still driven by ParentReady/finalization. + // Slot 0 is still inside window 0, so the leader window does not advance yet. assert_eq!(state.current_leader_window_idx, WindowIndex::new(0)); } @@ -4177,11 +3751,11 @@ fn test_alpenglow_mode_with_notarized_parent_chain_tracking() { "slot1 base must be set in Alpenglow mode" ); - // Behavioral mode is still legacy (finalization-driven) unless use_notarized_parent_chain=true + // Slot 0 is still inside window 0, so the leader window does not advance yet. assert_eq!( state.current_leader_window_idx, WindowIndex::new(0), - "Alpenglow mode should use legacy window advancement (finalization-driven)" + "Alpenglow mode should still remain in window 0 until progress crosses the boundary" ); } @@ -4868,6 +4442,54 @@ fn test_external_finalize_certificate_for_missed_finalization_recovery() { assert_eq!(state.first_non_finalized_slot, SlotIndex::new(1)); } +#[test] +fn test_set_finalize_certificate_advances_progress_cursor_past_pre_skipped_slots() { + // Regression: if slots after the finalized slot are already skipped, + // finalization must run progress-cursor advancement (`advance_present` parity) + // before leader-window advancement so we don't stop on a baseless skipped slot. + let desc = create_test_desc(4, 4); // 4 slots per window + let mut state = + SimplexState::new(&desc, opts_notarized_parent_chain()).expect("Failed to create state"); + let signers = vec![ValidatorIndex::new(0), ValidatorIndex::new(1), ValidatorIndex::new(2)]; + + // Pre-mark slots 1..3 as skipped (out of order, before slot 0 finalization). + for s in 1..=3u32 { + let cert = create_test_skip_cert(&desc, SlotIndex::new(s), &signers); + state.set_skip_certificate(&desc, SlotIndex::new(s), cert).unwrap(); + } + drain_events(&mut state); + + // Slot 0 is still not progressed, so cursor must stay at 0. + assert_eq!(state.get_first_non_progressed_slot(), SlotIndex::new(0)); + + // Finalize slot 0 (without prior explicit notarization event ingestion). + let slot0 = SlotIndex::new(0); + let block_hash = UInt256::from([0xA5; 32]); + let final_cert = create_test_final_cert(&desc, slot0, block_hash.clone(), &signers); + let stored = state + .set_finalize_certificate(&desc, slot0, &block_hash, final_cert) + .expect("set_finalize_certificate should succeed"); + assert!(stored, "finalize certificate should be stored"); + + // Cursor must skip over already-progressed slots 1..3 and land on slot 4. + assert_eq!( + state.get_first_non_progressed_slot(), + SlotIndex::new(4), + "progress cursor must advance past pre-skipped slots after finalization" + ); + + // Slot 4 must have finalized parent as available base. + let expected_parent = Some(CandidateParentInfo { slot: slot0, hash: block_hash }); + assert_eq!( + state.get_slot_available_base(&desc, SlotIndex::new(4)), + Some(expected_parent), + "slot 4 must inherit base from finalized slot 0" + ); + + // Window should advance from 0 to 1 (slot 4 is first slot of window 1). + assert_eq!(state.current_leader_window_idx, WindowIndex::new(1)); +} + #[test] fn test_set_finalize_certificate_emits_block_finalized_and_finalization_reached_for_tracked_slot() { // External finalize cert ingestion must emit: @@ -4948,7 +4570,7 @@ fn test_set_finalize_certificate_emits_block_finalized_and_finalization_reached_ - First increase backoff, then complete a window without timeouts and verify restore path. - test_notarized_parent_chain_advances_window_on_full_window_skip: - - With `use_notarized_parent_chain=true`, skip-cert all slots in a window and verify + - With the default C++-parity progression path, skip-cert all slots in a window and verify `first_non_progressed_slot` crosses the boundary and timeouts are scheduled for the next window. - test_notarized_parent_chain_base_propagation_with_multiple_skipped_intervals: @@ -4964,9 +4586,8 @@ fn test_set_finalize_certificate_emits_block_finalized_and_finalization_reached_ fn test_reject_far_future_vote() { let desc = create_test_desc(4, 2); let mut state = SimplexState::new(&desc, opts_cpp()).expect("create"); + let far_slot = state.first_too_new_vote_slot(); - // first_non_finalized_slot = 0, so max acceptable = MAX_FUTURE_SLOTS - let far_slot = SlotIndex::new(MAX_FUTURE_SLOTS + 1); let vote = Vote::Notarize(NotarizeVote { slot: far_slot, block_hash: UInt256::rand() }); let result = state.on_vote_test(&desc, ValidatorIndex::new(1), vote, vec![]); @@ -4982,10 +4603,10 @@ fn test_reject_far_future_vote() { fn test_accept_vote_at_boundary() { let desc = create_test_desc(4, 2); let mut state = SimplexState::new(&desc, opts_cpp()).expect("create"); + let boundary_slot = state.first_too_new_vote_slot() - 1; - // Slot exactly at the boundary should be accepted (not rejected by bounds check). + // Slot immediately before first_too_new should be accepted (not rejected by bounds check). // It may still return Rejected/Applied depending on FSM state, but NOT "too far ahead". - let boundary_slot = SlotIndex::new(MAX_FUTURE_SLOTS); let vote = Vote::Notarize(NotarizeVote { slot: boundary_slot, block_hash: UInt256::rand() }); let result = state.on_vote_test(&desc, ValidatorIndex::new(1), vote, vec![]); @@ -5005,8 +4626,10 @@ fn test_accept_vote_at_boundary() { fn test_reject_far_future_candidate() { let desc = create_test_desc(4, 2); let mut state = SimplexState::new(&desc, opts_cpp()).expect("create"); + let max_future_slots = + desc.opts().max_leader_window_desync.saturating_mul(desc.opts().slots_per_leader_window); - let far_slot = MAX_FUTURE_SLOTS + 1; + let far_slot = max_future_slots + 1; let candidate = create_test_candidate(far_slot, UInt256::rand(), BlockIdExt::default(), None, 0); @@ -5027,8 +4650,10 @@ fn test_reject_far_future_candidate() { fn test_reject_far_future_window_base_ready() { let desc = create_test_desc(4, 2); let mut state = SimplexState::new(&desc, opts_cpp()).expect("create"); + let max_future_slots = + desc.opts().max_leader_window_desync.saturating_mul(desc.opts().slots_per_leader_window); - let far_window = WindowIndex::new((MAX_FUTURE_SLOTS / 2) + 100); + let far_window = WindowIndex::new((max_future_slots / 2) + 100); let initial_len = state.leader_windows.len(); let result = state.on_window_base_ready(&desc, far_window, None); assert!(result.is_ok(), "far-future window base should be silently dropped"); @@ -5055,23 +4680,27 @@ fn test_ensure_window_exists_capped() { assert_eq!( state.leader_windows.len(), initial_len, - "ensure_window_exists should refuse to allocate beyond MAX_FUTURE_SLOTS cap" + "ensure_window_exists should refuse to allocate beyond configured desync cap" ); } #[test] -fn test_far_future_with_advanced_finalization() { +fn test_vote_bound_with_advanced_finalization() { let desc = create_test_desc(4, 2); let mut state = SimplexState::new(&desc, opts_cpp()).expect("create"); + let expected_first_too_new = SlotIndex::new( + ((5000 / desc.opts().slots_per_leader_window) + desc.opts().max_leader_window_desync + 1) + * desc.opts().slots_per_leader_window, + ); // Advance finalization cursor state.set_first_non_finalized_slot(SlotIndex::new(5000)); - // Now max acceptable = 5000 + MAX_FUTURE_SLOTS - assert_eq!(state.max_acceptable_slot(), SlotIndex::new(5000 + MAX_FUTURE_SLOTS)); + let first_too_new = state.first_too_new_vote_slot(); + assert_eq!(first_too_new, expected_first_too_new); - // Vote at max + 1 should be rejected - let vote = Vote::Skip(SkipVote { slot: SlotIndex::new(5000 + MAX_FUTURE_SLOTS + 1) }); + // Vote at first_too_new must be rejected. + let vote = Vote::Skip(SkipVote { slot: first_too_new }); let result = state.on_vote_test(&desc, ValidatorIndex::new(1), vote, vec![]); match result { VoteResult::Rejected(reason) => { @@ -5080,8 +4709,8 @@ fn test_far_future_with_advanced_finalization() { other => panic!("Expected Rejected, got {:?}", other), } - // Vote at max should be accepted (bounds-wise) - let vote = Vote::Skip(SkipVote { slot: SlotIndex::new(5000 + MAX_FUTURE_SLOTS) }); + // Vote immediately before first_too_new should still pass the bounds check. + let vote = Vote::Skip(SkipVote { slot: first_too_new - 1 }); let result = state.on_vote_test(&desc, ValidatorIndex::new(1), vote, vec![]); match result { VoteResult::Rejected(reason) => { @@ -5099,13 +4728,66 @@ fn test_far_future_with_advanced_finalization() { fn test_is_slot_too_far_ahead_helper() { let desc = create_test_desc(4, 2); let state = SimplexState::new(&desc, opts_cpp()).expect("create"); + let max_future_slots = + desc.opts().max_leader_window_desync.saturating_mul(desc.opts().slots_per_leader_window); assert!(!state.is_slot_too_far_ahead(SlotIndex::new(0))); - assert!(!state.is_slot_too_far_ahead(SlotIndex::new(MAX_FUTURE_SLOTS))); - assert!(state.is_slot_too_far_ahead(SlotIndex::new(MAX_FUTURE_SLOTS + 1))); + assert!(!state.is_slot_too_far_ahead(SlotIndex::new(max_future_slots))); + assert!(state.is_slot_too_far_ahead(SlotIndex::new(max_future_slots + 1))); assert!(state.is_slot_too_far_ahead(SlotIndex::new(u32::MAX))); } +#[test] +fn test_is_vote_slot_too_far_ahead_helper() { + let desc = create_test_desc(4, 2); + let state = SimplexState::new(&desc, opts_cpp()).expect("create"); + let first_too_new = state.first_too_new_vote_slot(); + + assert!(!state.is_vote_slot_too_far_ahead(first_too_new - 1)); + assert!(state.is_vote_slot_too_far_ahead(first_too_new)); + assert!(state.is_vote_slot_too_far_ahead(SlotIndex::new(u32::MAX))); +} + +#[test] +fn test_max_acceptable_slot_uses_progress_cursor_after_skip() { + let desc = create_test_desc(4, 2); + let mut state = + SimplexState::new(&desc, opts_notarized_parent_chain()).expect("Failed to create state"); + let max_future_slots = + desc.opts().max_leader_window_desync.saturating_mul(desc.opts().slots_per_leader_window); + + let signers = vec![ValidatorIndex::new(0), ValidatorIndex::new(1), ValidatorIndex::new(2)]; + let skip_cert = create_test_skip_cert(&desc, SlotIndex::new(0), &signers); + state.set_skip_certificate(&desc, SlotIndex::new(0), skip_cert).expect("should not error"); + + assert_eq!(state.get_first_non_finalized_slot(), SlotIndex::new(0)); + assert_eq!(state.get_first_non_progressed_slot(), SlotIndex::new(1)); + assert_eq!(state.max_acceptable_slot(), SlotIndex::new(1 + max_future_slots)); + assert!(!state.is_slot_too_far_ahead(SlotIndex::new(1 + max_future_slots))); + assert!(state.is_slot_too_far_ahead(SlotIndex::new(2 + max_future_slots))); +} + +#[test] +fn test_vote_bound_uses_progress_cursor_after_skip() { + let desc = create_test_desc(4, 2); + let mut state = + SimplexState::new(&desc, opts_notarized_parent_chain()).expect("Failed to create state"); + let expected_first_too_new = SlotIndex::new( + ((1 / desc.opts().slots_per_leader_window) + desc.opts().max_leader_window_desync + 1) + * desc.opts().slots_per_leader_window, + ); + + let signers = vec![ValidatorIndex::new(0), ValidatorIndex::new(1), ValidatorIndex::new(2)]; + let skip_cert = create_test_skip_cert(&desc, SlotIndex::new(0), &signers); + state.set_skip_certificate(&desc, SlotIndex::new(0), skip_cert).expect("should not error"); + + assert_eq!(state.get_first_non_finalized_slot(), SlotIndex::new(0)); + assert_eq!(state.get_first_non_progressed_slot(), SlotIndex::new(1)); + assert_eq!(state.first_too_new_vote_slot(), expected_first_too_new); + assert!(!state.is_vote_slot_too_far_ahead(expected_first_too_new - 1)); + assert!(state.is_vote_slot_too_far_ahead(expected_first_too_new)); +} + #[test] fn test_standstill_slot_grid_dump_empty_state() { let desc = create_test_desc(4, 2); @@ -5202,6 +4884,57 @@ fn test_standstill_slot_grid_dump_with_certs() { assert!(lines[0].starts_with("0: NNNN."), "Expected NNNN. in: {}", lines[0]); } +#[test] +fn test_standstill_diagnostic_dump_includes_last_final_cert_summary() { + let desc = create_test_desc(4, 2); + let mut state = SimplexState::new(&desc, opts_cpp()).expect("create"); + + let block_id = BlockIdExt::with_params( + ShardIdent::masterchain(), + 1, + UInt256::from([3u8; 32]), + UInt256::from([4u8; 32]), + ); + let candidate_hash = UInt256::from([0xCCu8; 32]); + let candidate = create_test_candidate(0, candidate_hash.clone(), block_id, None, 0); + state.on_candidate(&desc, candidate).unwrap(); + + for v in 0..3 { + let notar_vote = Vote::Notarize(NotarizeVote { + slot: SlotIndex::new(0), + block_hash: candidate_hash.clone(), + }); + state.on_vote_test(&desc, ValidatorIndex::new(v), notar_vote, vec![v as u8]).unwrap(); + } + for v in 0..3 { + let finalize_vote = Vote::Finalize(FinalizeVote { + slot: SlotIndex::new(0), + block_hash: candidate_hash.clone(), + }); + state + .on_vote_test(&desc, ValidatorIndex::new(v), finalize_vote, vec![10 + v as u8]) + .unwrap(); + } + + while state.has_pending_events() { + let _ = state.pull_event(); + } + + let dump = state.standstill_diagnostic_dump(&desc); + assert!( + dump.contains("Last final cert is for slot=0"), + "expected last-final summary in diagnostic dump: {dump}" + ); + assert!( + dump.contains(&candidate_hash.to_hex_string()), + "expected final-cert hash in diagnostic dump: {dump}" + ); + assert!( + dump.lines().any(|line| line.starts_with("1: ")), + "expected slot-grid line in diagnostic dump: {dump}" + ); +} + #[test] fn test_available_base_max_merge_keeps_higher_slot() { // When two propagations compete for the same target slot, max-merge must @@ -5443,6 +5176,146 @@ fn test_candidate_stored_as_pending_despite_skip_vote_cpp_mode() { ); } +#[test] +fn test_cpp_mode_try_skip_window_preserves_existing_pending_block() { + // Regression: in C++ mode, Skip must NOT drop an already buffered candidate. + let desc = create_test_desc(4, 2); + let mut state = SimplexState::new(&desc, opts_cpp()).expect("Failed to create state"); + + let parent_hash = UInt256::from([0x91; 32]); + let child_hash = UInt256::from([0x92; 32]); + let candidate = + create_test_candidate(1, child_hash, BlockIdExt::default(), Some((0, parent_hash)), 0); + state.on_candidate(&desc, candidate).unwrap(); + + assert!( + state.get_window(WindowIndex::new(0)).unwrap().slots[1].pending_block.is_some(), + "precondition: slot 1 candidate must be pending before skip" + ); + + state.try_skip_window(WindowIndex::new(0)); + + assert!( + state.get_window(WindowIndex::new(0)).unwrap().slots[1].pending_block.is_some(), + "C++ mode must preserve pending_block on skip" + ); +} + +#[test] +fn test_cpp_mode_restart_skip_paths_preserve_existing_pending_block() { + // Regression: restart skip paths in C++ mode must preserve pending candidates. + let desc = create_test_desc(4, 2); + let mut state = SimplexState::new(&desc, opts_cpp()).expect("Failed to create state"); + + let parent_hash = UInt256::from([0xA1; 32]); + let child_hash = UInt256::from([0xA2; 32]); + let candidate = + create_test_candidate(1, child_hash, BlockIdExt::default(), Some((0, parent_hash)), 0); + state.on_candidate(&desc, candidate).unwrap(); + + assert!( + state.get_window(WindowIndex::new(0)).unwrap().slots[1].pending_block.is_some(), + "precondition: slot 1 candidate must be pending before restart skips" + ); + + // 1) Direct restart-skip replay + state.mark_slot_voted_on_restart(&desc, &Vote::Skip(SkipVote { slot: SlotIndex::new(1) })); + assert!( + state.get_window(WindowIndex::new(0)).unwrap().slots[1].pending_block.is_some(), + "mark_slot_voted_on_restart(skip) must preserve pending_block in C++ mode" + ); + + // 2) Startup-generated restart skips for previous window [0,1] + let _ = state.generate_restart_skip_votes(WindowIndex::new(1), 2); + assert!( + state.get_window(WindowIndex::new(0)).unwrap().slots[1].pending_block.is_some(), + "generate_restart_skip_votes must preserve pending_block in C++ mode" + ); +} + +#[test] +fn test_cold_start_delayed_parent_recovery_notarizes_pending_cpp_mode() { + // Regression scenario: + // - cold startup delay before first active tick + // - candidate buffered while parent/state is unavailable + // - later parent availability must notarize buffered candidate (no deadlock) + let desc = create_test_desc(4, 2); + let base_time = SystemTime::UNIX_EPOCH + Duration::from_secs(1_700_000_000); + desc.set_time(base_time); + + let mut state = SimplexState::new(&desc, opts_cpp()).expect("Failed to create state"); + assert!(state.get_next_timeout().is_none(), "constructor path must not arm startup timeout"); + + // Candidate for slot 1 depends on slot 0 parent that is not available yet. + let parent_hash = UInt256::from([0xB1; 32]); + let child_hash = UInt256::from([0xB2; 32]); + let candidate = create_test_candidate( + 1, + child_hash.clone(), + BlockIdExt::default(), + Some((0, parent_hash.clone())), + 0, + ); + state.on_candidate(&desc, candidate).unwrap(); + assert!( + state.get_window(WindowIndex::new(0)).unwrap().slots[1].pending_block.is_some(), + "candidate should be buffered while parent/state is unavailable" + ); + + // Simulate long cold-start delay before the first active processing tick. + desc.set_time(base_time + Duration::from_secs(120)); + state.reset_timeouts_on_start(&desc); + state.check_all(&desc); + + // Timeout must be anchored at startup readiness, so there is no immediate skip storm. + let early_events: Vec<_> = from_fn(|| state.pull_event()).collect(); + assert!( + !early_events.iter().any(|e| matches!(e, SimplexEvent::BroadcastVote(Vote::Skip(_)))), + "must not emit immediate SkipVote right after startup timeout reset" + ); + + // Delayed parent availability: notarization for slot 0 arrives later. + let notarize_slot0 = + Vote::Notarize(NotarizeVote { slot: SlotIndex::new(0), block_hash: parent_hash }); + state.on_vote_test(&desc, ValidatorIndex::new(0), notarize_slot0.clone(), vec![1]).unwrap(); + state.on_vote_test(&desc, ValidatorIndex::new(1), notarize_slot0.clone(), vec![2]).unwrap(); + state.on_vote_test(&desc, ValidatorIndex::new(2), notarize_slot0, vec![3]).unwrap(); + + let events: Vec<_> = from_fn(|| state.pull_event()).collect(); + assert!( + events.iter().any(|e| matches!( + e, + SimplexEvent::BroadcastVote(Vote::Notarize(NotarizeVote { slot, block_hash })) + if *slot == SlotIndex::new(1) && *block_hash == child_hash + )), + "pending child candidate must be retried and notarized after parent becomes available" + ); + assert!( + state.get_window(WindowIndex::new(0)).unwrap().slots[1].pending_block.is_none(), + "pending_block must be cleared after successful notarization" + ); +} + +#[test] +fn test_alpenglow_mode_skip_clears_existing_pending_block() { + // Guardrail: fallback/Alpenglow mode keeps pendingBlocks[k] <- ⊥ on skip. + let desc = create_test_desc(4, 2); + let mut state = SimplexState::new(&desc, opts_alpenglow()).expect("Failed to create state"); + + let parent_hash = UInt256::from([0xC1; 32]); + let child_hash = UInt256::from([0xC2; 32]); + let candidate = + create_test_candidate(1, child_hash, BlockIdExt::default(), Some((0, parent_hash)), 0); + state.on_candidate(&desc, candidate).unwrap(); + assert!(state.get_window(WindowIndex::new(0)).unwrap().slots[1].pending_block.is_some()); + + state.try_skip_window(WindowIndex::new(0)); + assert!( + state.get_window(WindowIndex::new(0)).unwrap().slots[1].pending_block.is_none(), + "Alpenglow mode must clear pending_block on skip" + ); +} + #[test] fn test_pending_block_notarized_after_base_propagates_via_skip_certs() { // Full lifecycle: candidate stored as pending after skip vote, then notarized @@ -5828,8 +5701,8 @@ fn test_try_notar_not_blocked_by_its_over_after_finalize_restart_cpp_mode() { #[test] fn test_notarized_parent_chain_genesis_base_propagates_across_skipped_windows() { - // Regression test for bootstrap deadlock: when use_notarized_parent_chain=true (default - // C++ compat mode), skipping an entire window must propagate the available base to the + // Regression test for bootstrap deadlock: in the default C++-parity progression path, + // skipping an entire window must propagate the available base to the // next window via advance_leader_window_on_progress_cursor(). // // Without the fix, advance_leader_window_on_progress_cursor() only advanced the window @@ -5947,3 +5820,420 @@ fn test_notarized_parent_chain_base_propagates_across_multiple_skipped_windows() "window 3 must have available parent after windows 0+1+2 all skipped" ); } + +// ========================================================================= +// Fixed-base deadline tests (C++ timeout_base_ parity) +// +// C++ consensus.cpp stores a fixed per-window timeout_base_ and computes +// all slot deadlines as: timeout_base + (offset) * target_rate. +// These tests verify that Rust reproduces the exact same schedule. +// ========================================================================= + +#[test] +fn test_set_timeouts_arms_timeout_base() { + // set_timeouts must set timeout_base = now + first_block_timeout + // and skip_timestamp = timeout_base + target_rate. + let desc = create_test_desc(4, 4); + let mut state = + SimplexState::new(&desc, opts_notarized_parent_chain()).expect("Failed to create state"); + + let t0 = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); + desc.set_time(t0); + state.reset_timeouts_on_start(&desc); + + let first_block = desc.opts().first_block_timeout; // 3s default + let target_rate = desc.opts().target_rate; // 1s default + + assert_eq!( + state.timeout_base, + Some(t0 + first_block), + "timeout_base must be t0 + first_block_timeout" + ); + assert_eq!( + state.skip_timestamp, + Some(t0 + first_block + target_rate), + "skip_timestamp must be timeout_base + target_rate" + ); + assert_eq!(state.skip_slot, SlotIndex::new(0)); +} + +#[test] +fn test_notarization_rearm_uses_fixed_base_not_sliding() { + // Concrete scenario from C++ parity analysis: + // first_block_timeout=3s, target_rate=1s, 4 slots per window. + // + // Window starts at t0: + // timeout_base = t0 + 3s + // slot 0 deadline = t0 + 4s (base + 1*rate) + // + // Slot 0 notarizes "early" at t0 + 2s: + // C++ deadline for slot 1 = base + 2*rate = t0 + 5s (anchored to base) + // Old Rust would give: max(t0+4, t0+3) = t0 + 4s (sliding from now) + // + // After fix, Rust must produce the C++ answer: t0 + 5s. + let desc = create_test_desc(4, 4); + let mut state = + SimplexState::new(&desc, opts_notarized_parent_chain()).expect("Failed to create state"); + + let t0 = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); + desc.set_time(t0); + state.reset_timeouts_on_start(&desc); + + let first_block = desc.opts().first_block_timeout; // 3s + let target_rate = desc.opts().target_rate; // 1s + + // Advance to t0+2s and notarize slot 0 (3 out of 4 validators) + desc.set_time(t0 + Duration::from_secs(2)); + let block_hash = UInt256::from_slice(&[0xAA; 32]); + let vote = + Vote::Notarize(NotarizeVote { slot: SlotIndex::new(0), block_hash: block_hash.clone() }); + state.on_vote_test(&desc, ValidatorIndex::new(0), vote.clone(), vec![1]).unwrap(); + state.on_vote_test(&desc, ValidatorIndex::new(1), vote.clone(), vec![2]).unwrap(); + state.on_vote_test(&desc, ValidatorIndex::new(2), vote, vec![3]).unwrap(); + while state.pull_event().is_some() {} + + // skip_slot should advance to 1 (watching slot 1 now) + assert_eq!( + state.skip_slot, + SlotIndex::new(1), + "skip_slot must advance to 1 after notarization" + ); + + // C++ formula: alarm = timeout_base + (timeout_slot - window_start) * target_rate + // timeout_slot = slot+2 = 2, window_start = 0 → offset = 2 + // deadline = (t0+3) + 2*1 = t0+5 + let expected_deadline = t0 + first_block + target_rate * 2; + assert_eq!( + state.skip_timestamp, + Some(expected_deadline), + "deadline must be anchored to timeout_base, not to 'now' (expected t0+5s, NOT t0+3s)" + ); + + // timeout_base must NOT change on notarization + assert_eq!( + state.timeout_base, + Some(t0 + first_block), + "timeout_base must remain fixed within the window" + ); +} + +#[test] +fn test_notarization_rearm_successive_slots() { + // Notarize slots 0, 1, 2 in rapid succession — deadlines must follow the + // fixed schedule: base+2*rate, base+3*rate, base+4*rate. + let desc = create_test_desc(4, 4); + let mut state = + SimplexState::new(&desc, opts_notarized_parent_chain()).expect("Failed to create state"); + + let t0 = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); + desc.set_time(t0); + state.reset_timeouts_on_start(&desc); + + let first_block = desc.opts().first_block_timeout; + let target_rate = desc.opts().target_rate; + let base = t0 + first_block; + + for slot_num in 0u32..3 { + desc.set_time(t0 + Duration::from_millis(500 * (slot_num as u64 + 1))); + let hash = UInt256::from_slice(&[slot_num as u8 + 1; 32]); + + let parent = if slot_num == 0 { + None + } else { + Some((slot_num - 1, UInt256::from_slice(&[slot_num as u8; 32]))) + }; + let candidate = + create_test_candidate(slot_num, hash.clone(), BlockIdExt::default(), parent, 0); + let _ = state.on_candidate(&desc, candidate); + while state.pull_event().is_some() {} + + let vote = + Vote::Notarize(NotarizeVote { slot: SlotIndex::new(slot_num), block_hash: hash }); + state.on_vote_test(&desc, ValidatorIndex::new(0), vote.clone(), vec![1]).unwrap(); + state.on_vote_test(&desc, ValidatorIndex::new(1), vote.clone(), vec![2]).unwrap(); + state.on_vote_test(&desc, ValidatorIndex::new(2), vote, vec![3]).unwrap(); + while state.pull_event().is_some() {} + + // C++ timeout_slot_ = slot+2 (non-end-of-window) → offset = slot+2 + let expected = base + target_rate * (slot_num + 2); + assert_eq!( + state.skip_timestamp, + Some(expected), + "slot {} deadline must be base + {}*rate", + slot_num, + slot_num + 2 + ); + } +} + +#[test] +fn test_notarization_window_end_transitions_to_new_window() { + // When the last slot of a window is notarized, the progress cursor crosses + // into the next window. C++ handles this via LeaderWindowObserved which + // resets the timer. In Rust, advance_leader_window_on_progress_cursor → + // set_timeouts re-arms with fresh timeout_base for the new window. + // + // The guard `skip_slot <= slot` (C++ parity) prevents the per-notarization + // timer update from overwriting the freshly set window 1 schedule. + let desc = create_test_desc(4, 4); + let mut state = + SimplexState::new(&desc, opts_notarized_parent_chain()).expect("Failed to create state"); + + let t0 = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); + desc.set_time(t0); + state.reset_timeouts_on_start(&desc); + + let target_rate = desc.opts().target_rate; + + // Notarize all 4 slots in window 0 + for slot_num in 0u32..4 { + desc.set_time(t0 + Duration::from_millis(500 * (slot_num as u64 + 1))); + let hash = UInt256::from_slice(&[slot_num as u8 + 1; 32]); + + let parent = if slot_num == 0 { + None + } else { + Some((slot_num - 1, UInt256::from_slice(&[slot_num as u8; 32]))) + }; + let candidate = + create_test_candidate(slot_num, hash.clone(), BlockIdExt::default(), parent, 0); + let _ = state.on_candidate(&desc, candidate); + while state.pull_event().is_some() {} + + let vote = + Vote::Notarize(NotarizeVote { slot: SlotIndex::new(slot_num), block_hash: hash }); + state.on_vote_test(&desc, ValidatorIndex::new(0), vote.clone(), vec![1]).unwrap(); + state.on_vote_test(&desc, ValidatorIndex::new(1), vote.clone(), vec![2]).unwrap(); + state.on_vote_test(&desc, ValidatorIndex::new(2), vote, vec![3]).unwrap(); + while state.pull_event().is_some() {} + } + + // Window transition should have occurred + assert_eq!( + state.current_leader_window_idx, + WindowIndex::new(1), + "window must advance to 1 after all slots notarized" + ); + + // set_timeouts for window 1 was called at t0+2s (time of last notarization). + // No adaptive backoff because window 0 had no timeouts (had_timeouts=false). + let t_last = t0 + Duration::from_millis(2000); + let first_block = desc.opts().first_block_timeout; // restored to default (no backoff) + let new_base = t_last + first_block; + assert_eq!(state.timeout_base, Some(new_base), "timeout_base must be freshly set for window 1"); + assert_eq!( + state.skip_timestamp, + Some(new_base + target_rate), + "skip_timestamp must be base + target_rate for window 1" + ); + assert_eq!(state.skip_slot, SlotIndex::new(4), "skip_slot must be at window 1 start"); +} + +#[test] +fn test_skip_cert_does_not_move_timer() { + // C++ does NOT touch the consensus alarm when a skip certificate arrives. + // Skip certs flow through the pool layer only. + let desc = create_test_desc(4, 4); + let mut state = + SimplexState::new(&desc, opts_notarized_parent_chain()).expect("Failed to create state"); + + let t0 = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); + desc.set_time(t0); + state.reset_timeouts_on_start(&desc); + + let first_block = desc.opts().first_block_timeout; + let target_rate = desc.opts().target_rate; + + let skip_slot_before = state.skip_slot; + let skip_ts_before = state.skip_timestamp; + let base_before = state.timeout_base; + + assert_eq!(skip_slot_before, SlotIndex::new(0)); + assert_eq!(skip_ts_before, Some(t0 + first_block + target_rate)); + + // Advance time and submit skip cert for slot 0 (3 out of 4) + desc.set_time(t0 + Duration::from_millis(500)); + let skip = Vote::Skip(SkipVote { slot: SlotIndex::new(0) }); + state.on_vote_test(&desc, ValidatorIndex::new(0), skip.clone(), Vec::new()).unwrap(); + state.on_vote_test(&desc, ValidatorIndex::new(1), skip.clone(), Vec::new()).unwrap(); + state.on_vote_test(&desc, ValidatorIndex::new(2), skip, Vec::new()).unwrap(); + while state.pull_event().is_some() {} + + // Timer state must be UNCHANGED (skip_slot, skip_timestamp, timeout_base) + assert_eq!(state.skip_slot, skip_slot_before, "skip_slot must NOT advance on skip cert"); + assert_eq!(state.skip_timestamp, skip_ts_before, "skip_timestamp must NOT change on skip cert"); + assert_eq!(state.timeout_base, base_before, "timeout_base must NOT change on skip cert"); +} + +#[test] +fn test_window_skip_clears_timeout_base() { + // When process_timeouts fires the C++ window-skip, both skip_timestamp + // and timeout_base must be cleared (None). + let desc = create_test_desc(4, 4); + let mut state = + SimplexState::new(&desc, opts_notarized_parent_chain()).expect("Failed to create state"); + + let t0 = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); + desc.set_time(t0); + state.reset_timeouts_on_start(&desc); + + assert!(state.timeout_base.is_some(), "base must be armed after start"); + + // Advance well past the first deadline to trigger process_timeouts + desc.set_time(t0 + Duration::from_secs(10)); + state.check_all(&desc); + while state.pull_event().is_some() {} + + assert!(state.skip_timestamp.is_none(), "skip_timestamp must be None after C++ window-skip"); + assert!(state.timeout_base.is_none(), "timeout_base must be None after C++ window-skip"); + // skip_slot should be at window end (4) + assert_eq!( + state.skip_slot, + SlotIndex::new(4), + "skip_slot must be at window end after C++ window-skip" + ); +} + +#[test] +fn test_new_window_rearms_timeout_base() { + // After a window skip, when progress crosses into a new window, + // advance_leader_window_on_progress_cursor → set_timeouts must + // re-arm timeout_base with (possibly backed-off) first_block_timeout. + let desc = create_test_desc(4, 4); + let mut state = + SimplexState::new(&desc, opts_notarized_parent_chain()).expect("Failed to create state"); + + let t0 = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000_000); + desc.set_time(t0); + state.reset_timeouts_on_start(&desc); + + let target_rate = desc.opts().target_rate; + let backoff_factor = desc.opts().timeout_increase_factor; // 1.05 default + + // Trigger timeout to skip window 0 + desc.set_time(t0 + Duration::from_secs(10)); + state.check_all(&desc); + while state.pull_event().is_some() {} + assert!(state.timeout_base.is_none(), "base cleared after window skip"); + + // Now feed skip certs for all 4 slots (to let progress cursor cross window boundary) + let t1 = t0 + Duration::from_secs(11); + desc.set_time(t1); + for slot_num in 0u32..4 { + let skip = Vote::Skip(SkipVote { slot: SlotIndex::new(slot_num) }); + state.on_vote_test(&desc, ValidatorIndex::new(0), skip.clone(), Vec::new()).unwrap(); + state.on_vote_test(&desc, ValidatorIndex::new(1), skip.clone(), Vec::new()).unwrap(); + state.on_vote_test(&desc, ValidatorIndex::new(2), skip, Vec::new()).unwrap(); + } + while state.pull_event().is_some() {} + + // Window 0 had timeouts (had_timeouts=true), so adaptive backoff applies: + // first_block_timeout *= timeout_increase_factor (1.05) + let backed_off_first_block = desc.opts().first_block_timeout.mul_f64(backoff_factor); + + // Progress cursor should have advanced past window 0, triggering + // advance_leader_window_on_progress_cursor → set_timeouts for window 1 + assert_eq!( + state.current_leader_window_idx, + WindowIndex::new(1), + "window must advance to 1 after skip certs" + ); + assert_eq!( + state.timeout_base, + Some(t1 + backed_off_first_block), + "timeout_base must be re-armed for new window (with backoff)" + ); + assert_eq!( + state.skip_timestamp, + Some(t1 + backed_off_first_block + target_rate), + "skip_timestamp must be armed for new window (with backoff)" + ); + assert_eq!(state.skip_slot, SlotIndex::new(4), "skip_slot must be at start of window 1"); +} + +/// End-to-end: first leader absent -> full first-window skip -> second leader collates & notarizes. +/// +/// Scenario (4 validators, 2 slots/window): +/// Window 0 (leader=v0): skip slot 0, skip slot 1 → window 0 fully skipped. +/// Window 1 (leader=v1): candidate at slot 2 with genesis parent → notarized by quorum. +/// +/// This closes the test gap identified in the plan: existing tests verify base propagation +/// across skipped windows but do NOT assert that the second leader can successfully +/// submit a candidate and achieve notarization after the first window is entirely skipped. +#[test] +fn test_second_leader_collates_after_full_first_window_skip() { + let desc = create_test_desc(4, 2); // 4 validators, 2 slots per window + let mut state = + SimplexState::new(&desc, opts_notarized_parent_chain()).expect("Failed to create state"); + + // -- Skip entire window 0 (leader=v0 absent) -- + + // Skip slot 0 (quorum = 3 out of 4) + let skip0 = Vote::Skip(SkipVote { slot: SlotIndex::new(0) }); + state.on_vote_test(&desc, ValidatorIndex::new(0), skip0.clone(), Vec::new()).unwrap(); + state.on_vote_test(&desc, ValidatorIndex::new(1), skip0.clone(), Vec::new()).unwrap(); + state.on_vote_test(&desc, ValidatorIndex::new(2), skip0, Vec::new()).unwrap(); + while state.pull_event().is_some() {} + + // Skip slot 1 (last slot in window 0) + let skip1 = Vote::Skip(SkipVote { slot: SlotIndex::new(1) }); + state.on_vote_test(&desc, ValidatorIndex::new(0), skip1.clone(), Vec::new()).unwrap(); + state.on_vote_test(&desc, ValidatorIndex::new(1), skip1.clone(), Vec::new()).unwrap(); + state.on_vote_test(&desc, ValidatorIndex::new(2), skip1, Vec::new()).unwrap(); + while state.pull_event().is_some() {} + + // Verify window advanced and second leader has an available parent + assert_eq!( + state.current_leader_window_idx, + WindowIndex::new(1), + "window must advance to 1 after full first-window skip" + ); + assert_eq!(state.first_non_progressed_slot, SlotIndex::new(2)); + assert!( + state.has_available_parent(&desc, SlotIndex::new(2)), + "second leader must have available parent (genesis base propagated)" + ); + + // -- Second leader (v1) submits candidate for slot 2 with genesis parent -- + + let block2 = BlockIdExt::with_params( + ShardIdent::masterchain(), + 1, + UInt256::from([0x22; 32]), + UInt256::from([0x33; 32]), + ); + let candidate2 = create_test_candidate(2, UInt256::from([0x22; 32]), block2.clone(), None, 1); + state.on_candidate(&desc, candidate2).expect("second leader candidate must be accepted"); + + // Drain events — expect our own NotarVote to be broadcast + let events: Vec<_> = from_fn(|| state.pull_event()).collect(); + assert!( + events.iter().any(|e| matches!(e, SimplexEvent::BroadcastVote(Vote::Notarize(v)) + if v.slot == SlotIndex::new(2))), + "our node must broadcast a NotarVote for slot 2 after second leader's candidate" + ); + + // -- Notarize slot 2 with quorum votes -- + + let notar2 = Vote::Notarize(NotarizeVote { + slot: SlotIndex::new(2), + block_hash: block2.root_hash.clone(), + }); + state.on_vote_test(&desc, ValidatorIndex::new(1), notar2.clone(), Vec::new()).unwrap(); + state.on_vote_test(&desc, ValidatorIndex::new(2), notar2.clone(), Vec::new()).unwrap(); + state.on_vote_test(&desc, ValidatorIndex::new(3), notar2, Vec::new()).unwrap(); + while state.pull_event().is_some() {} + + // Verify slot 2 is notarized + assert!( + state.has_notarized_block(SlotIndex::new(2)), + "slot 2 must be notarized after quorum votes — second leader recovery works" + ); + + // Progress cursor must advance past slot 2 + assert!( + state.first_non_progressed_slot > SlotIndex::new(2), + "progress cursor must advance past notarized slot 2, got {}", + state.first_non_progressed_slot + ); +} diff --git a/src/node/simplex/src/tests/test_slot_bounds.rs b/src/node/simplex/src/tests/test_slot_bounds.rs index 8ce320c..7517d8c 100644 --- a/src/node/simplex/src/tests/test_slot_bounds.rs +++ b/src/node/simplex/src/tests/test_slot_bounds.rs @@ -8,8 +8,9 @@ */ //! Tests for receiver-level slot bounds checking //! -//! Verifies that the receiver rejects far-future and already-finalized slots -//! before expensive operations (signature verification, dedup HashMap insertion). +//! Verifies that the receiver: +//! - rejects vote slots using the C++ window-aligned `first_too_new_slot` rule +//! - drops already-finalized certificates, but does not cap them with the vote bound use super::*; use ton_api::{ @@ -24,59 +25,103 @@ use ton_api::{ }; use ton_block::UInt256; -/// Mirrors `ReceiverImpl::max_acceptable_slot` logic. -fn max_acceptable(first_active: u32) -> u32 { - first_active.saturating_add(MAX_FUTURE_SLOTS) +fn max_future_span() -> u32 { + let opts = crate::SessionOptions::default(); + opts.max_leader_window_desync.saturating_mul(opts.slots_per_leader_window) } -/// Mirrors `ReceiverImpl::is_slot_out_of_bounds` logic. -fn is_out_of_bounds(first_active: u32, slot: u32) -> bool { - slot < first_active || slot > max_acceptable(first_active) +fn slots_per_window() -> u32 { + crate::SessionOptions::default().slots_per_leader_window +} + +/// Mirrors candidate-style upper bound logic used outside the receiver. +fn candidate_max_acceptable(progress_slot: u32) -> u32 { + progress_slot.saturating_add(max_future_span()) +} + +/// Mirrors `ReceiverImpl::first_too_new_vote_slot` logic. +fn first_too_new_vote_slot(progress_slot: u32) -> u32 { + let spw = slots_per_window(); + (progress_slot / spw) + .saturating_add(crate::SessionOptions::default().max_leader_window_desync) + .saturating_add(1) + .saturating_mul(spw) +} + +/// Mirrors `ReceiverImpl::is_vote_slot_out_of_bounds` logic. +fn vote_is_out_of_bounds(first_active: u32, progress_slot: u32, slot: u32) -> bool { + slot < first_active || slot >= first_too_new_vote_slot(progress_slot) +} + +/// Mirrors `ReceiverImpl::is_certificate_slot_too_old` logic. +fn certificate_is_too_old(first_active: u32, slot: u32) -> bool { + slot < first_active +} + +#[test] +fn test_candidate_max_acceptable_slot_at_zero() { + assert_eq!(candidate_max_acceptable(0), max_future_span()); } #[test] -fn test_max_acceptable_slot_at_zero() { - assert_eq!(max_acceptable(0), MAX_FUTURE_SLOTS); +fn test_candidate_max_acceptable_slot_with_offset() { + assert_eq!(candidate_max_acceptable(5000), 5000 + max_future_span()); } #[test] -fn test_max_acceptable_slot_with_offset() { - assert_eq!(max_acceptable(5000), 5000 + MAX_FUTURE_SLOTS); +fn test_candidate_max_acceptable_slot_saturates() { + assert_eq!(candidate_max_acceptable(u32::MAX - 100), u32::MAX); + assert_eq!(candidate_max_acceptable(u32::MAX), u32::MAX); } #[test] -fn test_max_acceptable_slot_saturates() { - assert_eq!(max_acceptable(u32::MAX - 100), u32::MAX); - assert_eq!(max_acceptable(u32::MAX), u32::MAX); +fn test_first_too_new_vote_slot_is_window_aligned() { + let spw = slots_per_window(); + let max_desync = crate::SessionOptions::default().max_leader_window_desync; + assert_eq!(first_too_new_vote_slot(0), ((0 / spw) + max_desync + 1) * spw); + assert_eq!(first_too_new_vote_slot(spw + 1), (((spw + 1) / spw) + max_desync + 1) * spw); } #[test] -fn test_bounds_rejects_far_future_vote() { - assert!(is_out_of_bounds(0, MAX_FUTURE_SLOTS + 1)); - assert!(is_out_of_bounds(0, u32::MAX)); +fn test_vote_bounds_accept_last_slot_before_boundary() { + let boundary = first_too_new_vote_slot(0); + assert!(!vote_is_out_of_bounds(0, 0, 0)); + assert!(!vote_is_out_of_bounds(0, 0, boundary - 1)); } #[test] -fn test_bounds_accepts_boundary_slot() { - assert!(!is_out_of_bounds(0, MAX_FUTURE_SLOTS)); - assert!(!is_out_of_bounds(0, 0)); +fn test_vote_bounds_reject_boundary_slot() { + let boundary = first_too_new_vote_slot(0); + assert!(vote_is_out_of_bounds(0, 0, boundary)); + assert!(vote_is_out_of_bounds(0, 0, u32::MAX)); } #[test] -fn test_bounds_rejects_old_finalized_slot() { - assert!(is_out_of_bounds(5000, 4999)); - assert!(is_out_of_bounds(5000, 0)); +fn test_vote_bounds_reject_old_finalized_slot() { + assert!(vote_is_out_of_bounds(5000, 5000, 4999)); + assert!(vote_is_out_of_bounds(5000, 5000, 0)); } #[test] -fn test_bounds_accepts_range_with_advanced_finalization() { - assert!(!is_out_of_bounds(5000, 5000)); - assert!(!is_out_of_bounds(5000, 5000 + MAX_FUTURE_SLOTS)); +fn test_receiver_tracks_progress_not_finalization_for_vote_bound() { + let spw = slots_per_window(); + let progress_slot = spw * 2 + 1; + let slot_accepted_by_progress = first_too_new_vote_slot(progress_slot) - 1; + + assert!(!vote_is_out_of_bounds(0, progress_slot, slot_accepted_by_progress)); + assert!( + vote_is_out_of_bounds(0, 0, slot_accepted_by_progress), + "same slot would be rejected if the upper bound were still anchored on finalization" + ); } #[test] -fn test_bounds_rejects_beyond_range_with_advanced_finalization() { - assert!(is_out_of_bounds(5000, 5001 + MAX_FUTURE_SLOTS)); +fn test_certificate_bounds_only_reject_old_slots() { + let boundary = first_too_new_vote_slot(0); + assert!(certificate_is_too_old(5000, 4999)); + assert!(!certificate_is_too_old(5000, 5000)); + assert!(!certificate_is_too_old(0, boundary)); + assert!(!certificate_is_too_old(0, u32::MAX)); } #[test] @@ -102,11 +147,12 @@ fn test_get_vote_slot_extracts_skip() { } #[test] -fn test_negative_tl_slot_wraps_to_large_u32_and_is_rejected() { - // TL uses i32; negative values cast via `as u32` produce large numbers - // that must be rejected by the bounds check. +fn test_negative_tl_slot_wraps_to_large_u32_and_is_rejected_on_vote_path() { + // TL uses i32; negative values cast via `as u32` produce large numbers. + // Vote prefilter must reject them as too new; certificate negatives are + // validated downstream in SessionProcessor before certificate verification. let negative_slot: i32 = -1; let slot_as_u32 = negative_slot as u32; // wraps to u32::MAX - assert!(is_out_of_bounds(0, slot_as_u32)); - assert!(is_out_of_bounds(5000, slot_as_u32)); + assert!(vote_is_out_of_bounds(0, 0, slot_as_u32)); + assert!(vote_is_out_of_bounds(5000, 5000, slot_as_u32)); } diff --git a/src/node/simplex/tests/test_collation.rs b/src/node/simplex/tests/test_collation.rs index 0be6bdd..f7592a6 100644 --- a/src/node/simplex/tests/test_collation.rs +++ b/src/node/simplex/tests/test_collation.rs @@ -26,7 +26,7 @@ use std::{ time::{Duration, Instant, SystemTime}, }; use ton_block::{ - error, sha256_digest, BlockIdExt, BlockSignaturesVariant, BocFlags, BocWriter, BuilderData, + sha256_digest, BlockIdExt, BlockSignaturesVariant, BocFlags, BocWriter, BuilderData, Ed25519KeyOption, ShardIdent, UInt256, }; @@ -52,12 +52,16 @@ struct CollationTestListener { collation_requested: Arc, /// Collation count collation_count: Arc, + /// Set to true when the self-collated block reaches validation + candidate_validated: Arc, + /// Validation callback count + candidate_count: Arc, /// Public key for generating candidates public_key: PublicKey, - /// Next expected seqno for collation - increases after each successful collation + /// Next expected seqno for collation — updated on finalization next_expected_collation_seqno: Arc, - /// Next expected seqno for commit - initialized with initial_block_seqno, +1 for each commit - next_expected_commit_seqno: Arc, + /// Maximum finalized seqno observed (monotonically advances via fetch_max) + max_finalized_seqno: Arc, } impl SessionListener for CollationTestListener { @@ -74,6 +78,8 @@ impl SessionListener for CollationTestListener { source_info.priority.round, root_hash ); + self.candidate_validated.store(true, Ordering::Release); + self.candidate_count.fetch_add(1, Ordering::Relaxed); // Accept the candidate callback(Ok(SystemTime::now())); } @@ -98,20 +104,11 @@ impl SessionListener for CollationTestListener { self.collation_requested.store(true, Ordering::Release); self.collation_count.fetch_add(1, Ordering::Relaxed); - // Derive seqno from explicit parent hint or use stable counter value for implicit case let seqno = match &parent { consensus_common::CollationParentHint::Implicit => { - // Keep seqno stable across retries for the same slot. - // The counter is advanced on commit. - self.next_expected_collation_seqno.load(Ordering::SeqCst) - } - consensus_common::CollationParentHint::Explicit(parent_id) => { - // Explicit parent: derive seqno from parent (parent_seqno + 1) - let derived_seqno = parent_id.seq_no + 1; - // Update counter to match derived seqno for next iteration - self.next_expected_collation_seqno.store(derived_seqno + 1, Ordering::SeqCst); - derived_seqno + self.max_finalized_seqno.load(Ordering::SeqCst) } + consensus_common::CollationParentHint::Explicit(parent_id) => parent_id.seq_no + 1, }; // Generate dummy candidate with proper hashes @@ -157,27 +154,40 @@ impl SessionListener for CollationTestListener { fn on_block_committed( &self, - source_info: simplex::BlockSourceInfo, - root_hash: BlockHash, + _source_info: simplex::BlockSourceInfo, + _root_hash: BlockHash, _file_hash: BlockHash, _data: BlockPayloadPtr, _signatures: BlockSignaturesVariant, _approve_signatures: Vec<(PublicKeyHash, BlockPayloadPtr)>, _stats: consensus_common::SessionStats, + ) { + panic!( + "on_block_committed must not be called for Simplex sessions (finalized-driven only)" + ); + } + + fn on_block_finalized( + &self, + block_id: BlockIdExt, + source_info: simplex::BlockSourceInfo, + _root_hash: BlockHash, + _file_hash: BlockHash, + _data: BlockPayloadPtr, + _signatures: BlockSignaturesVariant, + _approve_signatures: Vec<(PublicKeyHash, BlockPayloadPtr)>, ) { let slot = source_info.priority.round; + let seqno = block_id.seq_no; - // Increment next_expected_commit_seqno and update next_expected_collation_seqno - let committed_seqno = self.next_expected_commit_seqno.fetch_add(1, Ordering::SeqCst); - let next_commit_seqno = committed_seqno + 1; - self.next_expected_collation_seqno.store(next_commit_seqno, Ordering::SeqCst); + self.max_finalized_seqno.fetch_max(seqno + 1, Ordering::SeqCst); + self.next_expected_collation_seqno.fetch_max(seqno + 1, Ordering::SeqCst); log::info!( - "CollationTestListener::on_block_committed: slot={}, hash={:?}, committed_seqno={}, next_expected={}", + "CollationTestListener::on_block_finalized: slot={}, seqno={}, block_id={}", slot, - root_hash, - committed_seqno, - next_commit_seqno + seqno, + block_id, ); } @@ -195,15 +205,6 @@ impl SessionListener for CollationTestListener { ) { // Not used in this test } - - fn get_committed_candidate( - &self, - block_id: BlockIdExt, - callback: consensus_common::CommittedBlockProofCallback, - ) { - log::info!("get_committed_candidate: STUB for block_id={}", block_id); - callback(Err(error!("get_committed_candidate not implemented in test"))); - } } /* @@ -314,7 +315,7 @@ fn run_collation_test() { .collect(); let db_path = format!("{}/{}_{}", DB_PATH, TEST_NAME, rand_name); let mut rng = rand::thread_rng(); - let session_id: UInt256 = UInt256::from(rng.gen::<[u8; 32]>()); + let session_id: UInt256 = UInt256::from(rng.r#gen::<[u8; 32]>()); // Session options - fast timing for quick test let session_opts = SessionOptions { @@ -328,14 +329,18 @@ fn run_collation_test() { // Create listener with tracking let collation_requested = Arc::new(AtomicBool::new(false)); let collation_count = Arc::new(AtomicU32::new(0)); + let candidate_validated = Arc::new(AtomicBool::new(false)); + let candidate_count = Arc::new(AtomicU32::new(0)); let initial_block_seqno = 1; // First block will have seqno=1 (seqno 0 is zerostate) let listener = Arc::new(CollationTestListener { collation_requested: collation_requested.clone(), collation_count: collation_count.clone(), + candidate_validated: candidate_validated.clone(), + candidate_count: candidate_count.clone(), public_key, next_expected_collation_seqno: Arc::new(AtomicU32::new(initial_block_seqno)), - next_expected_commit_seqno: Arc::new(AtomicU32::new(initial_block_seqno)), + max_finalized_seqno: Arc::new(AtomicU32::new(initial_block_seqno)), }); let session_listener: Arc = listener.clone(); @@ -361,14 +366,20 @@ fn run_collation_test() { log::info!("Session created, waiting for collation callback..."); - // Wait for collation callback + // Wait for collation and self-validation callbacks let test_start = Instant::now(); let mut collation_triggered = false; + let mut validation_triggered = false; while test_start.elapsed() < COLLATION_TIMEOUT { if collation_requested.load(Ordering::Acquire) { collation_triggered = true; - log::info!("COLLATION CALLBACK TRIGGERED after {:?}", test_start.elapsed()); + } + if candidate_validated.load(Ordering::Acquire) { + validation_triggered = true; + } + if collation_triggered && validation_triggered { + log::info!("COLLATION+VALIDATION CALLBACKS TRIGGERED after {:?}", test_start.elapsed()); break; } thread::sleep(Duration::from_millis(50)); @@ -382,10 +393,13 @@ fn run_collation_test() { // Report results let final_count = collation_count.load(Ordering::Relaxed); + let final_candidate_count = candidate_count.load(Ordering::Relaxed); log::info!( - "Test completed: collation_triggered={}, collation_count={}", + "Test completed: collation_triggered={}, validation_triggered={}, collation_count={}, candidate_count={}", collation_triggered, - final_count + validation_triggered, + final_count, + final_candidate_count ); // Assert @@ -394,6 +408,11 @@ fn run_collation_test() { "Collation callback was NOT triggered within {:?}", COLLATION_TIMEOUT ); + assert!( + validation_triggered, + "Self-collated block did NOT reach on_candidate within {:?}", + COLLATION_TIMEOUT + ); // Assert no errors were logged during the test let errors = error_count.load(Ordering::Relaxed); diff --git a/src/node/simplex/tests/test_consensus.rs b/src/node/simplex/tests/test_consensus.rs index aff2eb5..e69be0c 100644 --- a/src/node/simplex/tests/test_consensus.rs +++ b/src/node/simplex/tests/test_consensus.rs @@ -21,7 +21,7 @@ use serde::{Deserialize, Serialize}; use simplex::*; use spin::mutex::SpinMutex; use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, fs::{self, File}, io::{self, Cursor, LineWriter, Write}, path::Path, @@ -124,7 +124,7 @@ struct TestConfig { total_rounds: u32, /// Minimum percentage of commits required (0.0 - 1.0) /// Default 0.5 means at least 50% of rounds must be commits (not skips) - min_commit_percent: f64, + min_finalized_percent: f64, /// Number of validator nodes in the test node_count: usize, /// Probability of generation failure (0.0 - 1.0) @@ -219,7 +219,7 @@ impl Default for TestConfig { fn default() -> Self { Self { total_rounds: 100, - min_commit_percent: 0.5, // At least 50% commits (not skips) + min_finalized_percent: 0.5, // At least 50% commits (not skips) node_count: 11, generation_failure_probability: 0.0, candidate_rejection_probability: 0.0, @@ -346,12 +346,9 @@ fn print_latency_table_footer() { Session instance */ -/// Shared storage of committed block proofs for get_committed_candidate. -/// Populated by on_block_committed across all instances; queried by get_committed_candidate. -/// Keyed by root_hash (unique per block). All instances share one map via Arc>. -/// Race condition: multiple instances may insert the same block concurrently — this is safe -/// because the data is identical (same block, same signatures) so the last write wins. -type CommittedBlocksMap = Arc>>; +/// Shared storage of finalized block root hashes for harness-level introspection. +/// All instances share one set via Arc>. +type FinalizedBlocksMap = Arc>>; /// Session instance for a single validator node struct SessionInstance { @@ -362,22 +359,24 @@ struct SessionInstance { is_collator: Arc, collation_count: Arc, on_candidate_count: Arc, - on_block_committed_count: Arc, + on_block_finalized_count: Arc, config: TestConfig, - current_round: Arc, - /// Commit latencies in milliseconds (for statistical analysis) + /// Finalization latencies in milliseconds (for statistical analysis) commit_latencies: Arc>>, - /// Next expected seqno for commit - initialized with initial_block_seqno, +1 for each non-empty commit. - /// Shared with listener so it's updated during startup recovery before SessionInstance is wired. - next_expected_commit_seqno: Arc, - /// Session errors count - accumulated from SessionStats on each commit + /// Maximum finalized seqno observed so far. Updated atomically in + /// `on_block_finalized`; used for restart-gremlin recovery progress calculation. + max_finalized_seqno: Arc, + /// Set of seqnos for which a finalization has been delivered. + /// Invariant: every seqno appears at most once (asserted in `on_block_finalized`). + finalized_seqnos: Arc>>, + /// Session errors count session_errors_count: Arc, /// Approved candidates storage for get_approved_candidate() during restart recovery. /// Keyed by root_hash to match lookup semantics. approved_candidates: Arc>>>, - /// Shared committed block proofs for get_committed_candidate - committed_blocks: CommittedBlocksMap, + /// Shared finalized block roots for harness-level introspection. + finalized_blocks: FinalizedBlocksMap, _session: SessionPtr, _listener: Arc, } @@ -389,13 +388,10 @@ struct SessionInstanceListener { /// before session creation to support get_approved_candidate() during startup recovery. approved_candidates: Arc>>>, - /// SeqNo counter - shared with SessionInstance but available immediately. - /// Updated by on_block_committed() even before SessionInstance is wired, - /// which is critical for restart recommit to align the seqno tracking. - /// Used by on_generate_slot() to determine which seqno to use for new blocks. - next_expected_commit_seqno: Arc, - /// Shared committed block proofs for get_committed_candidate - committed_blocks: CommittedBlocksMap, + /// Maximum finalized seqno - shared with SessionInstance, available immediately. + max_finalized_seqno: Arc, + /// Finalized seqnos set - shared with SessionInstance, available immediately. + finalized_seqnos: Arc>>, } impl SessionInstance { @@ -415,8 +411,12 @@ impl SessionInstance { self.on_candidate_count.load(Ordering::Relaxed) } - fn on_block_committed_count(&self) -> u32 { - self.on_block_committed_count.load(Ordering::Relaxed) + fn on_block_finalized_count(&self) -> u32 { + self.on_block_finalized_count.load(Ordering::Relaxed) + } + + fn unique_finalized_seqno_count(&self) -> usize { + self.finalized_seqnos.lock().map(|s| s.len()).unwrap_or(0) } fn session_errors_count(&self) -> u32 { @@ -432,15 +432,12 @@ impl SessionInstance { } fn finish_slot(&self, slot: u32) { - // SIMPLEX_ROUNDLESS: Track progress by commit count - let commits = self.on_block_committed_count.load(Ordering::SeqCst); - self.current_round.store(commits, Ordering::SeqCst); - - if commits >= self.config.total_rounds { + let finalized = self.on_block_finalized_count.load(Ordering::SeqCst); + if finalized >= self.config.total_rounds { self.batch_processed.store(true, Ordering::Release); log::info!( - "Test finished after {} commits for source #{} (slot={})", - commits, + "Test finished after {} finalized blocks for source #{} (slot={})", + finalized, self.source_index, slot ); @@ -475,16 +472,6 @@ impl SessionListener for SessionInstance { // Extract slot from the embedded collated data (set by collator) let slot = collated_data.slot; - // With optimistic validation, candidates can be collated on notarized (not yet - // committed) parents, so the candidate seqno may be ahead of the committed seqno. - let committed_seqno = self.next_expected_commit_seqno.load(Ordering::SeqCst); - assert!( - collated_data.seqno >= committed_seqno, - "candidate seqno {} must be >= committed seqno {}", - collated_data.seqno, - committed_seqno, - ); - log::info!( "SessionListener::on_candidate: new candidate for \ slot {} from source {} with hash {:?} appeared with latency {} ms (self source #{})", @@ -587,22 +574,13 @@ impl SessionListener for SessionInstance { return; } - // Derive seqno from explicit parent hint or use counter for implicit case. - // - // IMPORTANT: Multiple on_generate_slot calls can happen before on_block_committed - // (collation retry / timeout), so we must use consistent seqno for all of them. - // Only ONE block per slot will actually be accepted; the others will fail - // validation with "seqno mismatch" which is correct behavior. let seqno = match &parent { consensus_common::CollationParentHint::Implicit => { - // Genesis / bootstrap case: use commit_seqno (don't increment - may retry) - self.next_expected_commit_seqno.load(Ordering::SeqCst) - } - consensus_common::CollationParentHint::Explicit(parent_id) => { - // Explicit parent: derive seqno from parent (parent_seqno + 1) - // This matches C++ behavior where block seqno = parent seqno + 1 - parent_id.seq_no + 1 + // Implicit is only expected for the very first (genesis) slot + // when no parent block exists yet. + self.max_finalized_seqno.load(Ordering::SeqCst) } + consensus_common::CollationParentHint::Explicit(parent_id) => parent_id.seq_no + 1, }; // Use seqno as the slot value for embedded data (since slot isn't exposed in API) @@ -656,37 +634,45 @@ impl SessionListener for SessionInstance { fn on_block_committed( &self, + _source_info: simplex::BlockSourceInfo, + _root_hash: BlockHash, + _file_hash: BlockHash, + _data: BlockPayloadPtr, + _signatures: BlockSignaturesVariant, + _approve_signatures: Vec<(PublicKeyHash, BlockPayloadPtr)>, + _stats: consensus_common::SessionStats, + ) { + panic!( + "on_block_committed must not be called for Simplex sessions (finalized-driven only)" + ); + } + + fn on_block_finalized( + &self, + block_id: BlockIdExt, source_info: simplex::BlockSourceInfo, root_hash: BlockHash, - file_hash: BlockHash, + _file_hash: BlockHash, data: BlockPayloadPtr, signatures: BlockSignaturesVariant, _approve_signatures: Vec<(PublicKeyHash, BlockPayloadPtr)>, - stats: consensus_common::SessionStats, ) { - // SIMPLEX_ROUNDLESS: Assert round is always u32::MAX assert_eq!( source_info.priority.round, SIMPLEX_ROUNDLESS, - "on_block_committed: round must be SIMPLEX_ROUNDLESS in roundless mode" + "on_block_finalized: round must be SIMPLEX_ROUNDLESS in roundless mode" ); - // Extract slot from signatures let slot = match &signatures { BlockSignaturesVariant::Simplex(s) => s.slot, _ => 0, }; - self.on_block_committed_count.fetch_add(1, Ordering::Relaxed); - // Track session errors from stats - self.session_errors_count.store(stats.errors_count, Ordering::Relaxed); + let seqno = block_id.seq_no(); + self.on_block_finalized_count.fetch_add(1, Ordering::Relaxed); let now = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap().as_millis() as u64; - // For non-empty blocks: this is DummyCollatedData bytes - // For empty blocks: this is empty let data_bytes = data.data(); - - // Skip latency tracking for empty blocks let latency = if !data_bytes.is_empty() { let collated_data = DummyCollatedData::from_bytes(data_bytes); now - collated_data.creation_timestamp @@ -694,32 +680,13 @@ impl SessionListener for SessionInstance { 0 }; - // Extract collated_data for source/seqno tracking (default if empty block) - let collated_data = if !data_bytes.is_empty() { - DummyCollatedData::from_bytes(data_bytes) - } else { - DummyCollatedData { creation_timestamp: now, slot, seqno: 0, source_index: 0 } - }; - - // Record latency for statistical analysis if let Ok(mut latencies) = self.commit_latencies.lock() { latencies.push(latency); } - // Detect empty block (empty data means it's an empty block that inherits parent's seqno) - let _is_empty_block = data_bytes.is_empty(); - - // Seqno tracking is updated in SessionInstanceListener::on_block_committed - // (and shared via Arc), so do NOT mutate next_expected_commit_seqno here. - let _next_commit_seqno = self.next_expected_commit_seqno.load(Ordering::SeqCst); - - // Source tracking: which validator produced the committed block (from dummy payload) - let _block_source = collated_data.source_index; - let seqno = collated_data.seqno; - log::info!( - "SessionListener::on_block_committed: new block from source {} with hash {:?} \ - committed at slot={}, seqno={}, latency={} ms (source #{})", + "SessionListener::on_block_finalized: block from source {} hash {:?} \ + finalized at slot={}, seqno={}, latency={} ms (source #{})", source_info.source.id(), root_hash, slot, @@ -728,20 +695,8 @@ impl SessionListener for SessionInstance { self.source_index ); - // Store committed block proof in shared map. - // All instances insert the same block with identical signatures, - // so concurrent inserts are a benign idempotent overwrite. - if let Ok(mut map) = self.committed_blocks.lock() { - let block_id = BlockIdExt::with_params( - self.config.shard.clone(), - collated_data.seqno, - root_hash.clone(), - file_hash.clone(), - ); - map.insert( - root_hash.clone(), - consensus_common::CommittedBlockProof { block_id, signatures: signatures.clone() }, - ); + if let Ok(mut map) = self.finalized_blocks.lock() { + map.insert(root_hash.clone()); } self.finish_slot(slot); @@ -810,34 +765,6 @@ impl SessionListener for SessionInstance { } } } - - fn get_committed_candidate( - &self, - block_id: BlockIdExt, - callback: consensus_common::CommittedBlockProofCallback, - ) { - let root_hash = block_id.root_hash.clone(); - let proof = self.committed_blocks.lock().ok().and_then(|map| map.get(&root_hash).cloned()); - - match proof { - Some(p) => { - log::info!( - "get_committed_candidate: FOUND proof for {} (source #{})", - block_id, - self.source_index - ); - callback(Ok(p)); - } - None => { - log::warn!( - "get_committed_candidate: NOT FOUND {} in shared map (source #{})", - block_id, - self.source_index - ); - callback(Err(error!("committed block {block_id} not found in shared map"))); - } - } - } } /* @@ -884,68 +811,81 @@ impl SessionListener for SessionInstanceListener { fn on_block_committed( &self, + _source_info: simplex::BlockSourceInfo, + _root_hash: BlockHash, + _file_hash: BlockHash, + _data: BlockPayloadPtr, + _signatures: BlockSignaturesVariant, + _approve_signatures: Vec<(PublicKeyHash, BlockPayloadPtr)>, + _stats: consensus_common::SessionStats, + ) { + panic!( + "on_block_committed must not be called for Simplex sessions (finalized-driven only)" + ); + } + + fn on_block_finalized( + &self, + block_id: BlockIdExt, source_info: simplex::BlockSourceInfo, root_hash: BlockHash, file_hash: BlockHash, data: BlockPayloadPtr, signatures: BlockSignaturesVariant, approve_signatures: Vec<(PublicKeyHash, BlockPayloadPtr)>, - stats: consensus_common::SessionStats, ) { - // SIMPLEX_ROUNDLESS: Assert round is always u32::MAX assert_eq!( source_info.priority.round, SIMPLEX_ROUNDLESS, - "SessionInstanceListener::on_block_committed: round must be SIMPLEX_ROUNDLESS" + "SessionInstanceListener::on_block_finalized: round must be SIMPLEX_ROUNDLESS" ); - // CRITICAL: Always update seqno counters, even if SessionInstance is not wired yet. - // This ensures restart recommit correctly aligns the seqno tracking. + let data_bytes = data.data(); let is_empty = data_bytes.is_empty(); - - // SIMPLEX_ROUNDLESS: Extract slot from signatures instead of using round let slot = match &signatures { BlockSignaturesVariant::Simplex(s) => s.slot, _ => 0, }; - // For non-empty blocks, advance the seqno counter. - // Empty blocks don't consume a seqno. if !is_empty { - let collated = DummyCollatedData::from_bytes(data_bytes); - let committed_seqno = collated.seqno; - let next_commit = committed_seqno + 1; - self.next_expected_commit_seqno.store(next_commit, Ordering::SeqCst); + let seqno = block_id.seq_no(); + // Atomically advance max_finalized_seqno (out-of-order safe). + self.max_finalized_seqno.fetch_max(seqno + 1, Ordering::SeqCst); log::trace!( - "SessionInstanceListener::on_block_committed: slot={}, seqno={}, next_commit={}", + "SessionInstanceListener::on_block_finalized: slot={}, seqno={}", slot, - committed_seqno, - next_commit + seqno ); } else { - log::trace!( - "SessionInstanceListener::on_block_committed: slot={}, empty block (no seqno change)", + log::trace!("SessionInstanceListener::on_block_finalized: slot={}, empty block", slot); + } + + // Invariant: exactly one finalization per seqno (checked even before + // SessionInstance is wired, so recovery duplicates are caught too). + if let Ok(mut seen) = self.finalized_seqnos.lock() { + let seqno = block_id.seq_no(); + assert!( + seen.insert(seqno), + "DUPLICATE finalization for seqno {} (listener, slot={})", + seqno, slot ); } - // Delegate to SessionInstance if wired if let Some(instance) = self.instance.lock().upgrade() { let instance = instance.lock(); - instance.on_block_committed( + instance.on_block_finalized( + block_id, source_info, root_hash, file_hash, data, signatures, approve_signatures, - stats, ); } } fn on_block_skipped(&self, round: u32) { - // IMPORTANT: Skipped rounds do NOT advance block seqno. - // (Empty blocks inherit parent's BlockIdExt and are reported via on_block_skipped.) if let Some(instance) = self.instance.lock().upgrade() { let instance = instance.lock(); instance.on_block_skipped(round); @@ -992,35 +932,6 @@ impl SessionListener for SessionInstanceListener { ))); } } - - fn get_committed_candidate( - &self, - block_id: BlockIdExt, - callback: consensus_common::CommittedBlockProofCallback, - ) { - // Access committed_blocks directly — works even before SessionInstance is wired - let root_hash = block_id.root_hash.clone(); - let proof = self.committed_blocks.lock().ok().and_then(|map| map.get(&root_hash).cloned()); - - match proof { - Some(p) => { - log::info!( - "SessionInstanceListener::get_committed_candidate: \ - FOUND proof for {}", - block_id - ); - callback(Ok(p)); - } - None => { - log::warn!( - "SessionInstanceListener::get_committed_candidate: \ - NOT FOUND {} in shared map", - block_id - ); - callback(Err(error!("committed block {block_id} not found in shared map"))); - } - } - } } /* @@ -1250,14 +1161,14 @@ where }, ..Default::default() }; + // Integration stress tests can temporarily run far ahead in slot space while still + // converging; keep a wide future-slot horizon to avoid synthetic test stalls. + session_opts.max_leader_window_desync = 10_000; if let Some(st) = config.standstill_timeout { session_opts.standstill_timeout = st; } - // Shared committed block proofs. - // All session instances share this map so any instance's on_block_committed - // stores the proof, and any instance's get_committed_candidate can read it. - let committed_blocks: CommittedBlocksMap = Arc::new(Mutex::new(HashMap::new())); + let finalized_blocks: FinalizedBlocksMap = Arc::new(Mutex::new(HashSet::new())); // Create session instances let mut instances = Vec::with_capacity(config.node_count); @@ -1273,25 +1184,19 @@ where Mutex>>, > = Arc::new(Mutex::new(HashMap::new())); - // SeqNo counter - created before listener so it can be updated during recovery - let next_expected_commit_seqno = Arc::new(AtomicU32::new(initial_block_seqno)); + let max_finalized_seqno = Arc::new(AtomicU32::new(initial_block_seqno)); + let finalized_seqnos: Arc>> = Arc::new(Mutex::new(HashSet::new())); let listener = Arc::new(SessionInstanceListener { instance: SpinMutex::new(Weak::new()), approved_candidates: approved_candidates.clone(), - next_expected_commit_seqno: next_expected_commit_seqno.clone(), - committed_blocks: committed_blocks.clone(), + max_finalized_seqno: max_finalized_seqno.clone(), + finalized_seqnos: finalized_seqnos.clone(), }); let session_listener: Arc = listener.clone(); - // Use shard from config let shard = config.shard.clone(); - - // Use per-node overlay manager (same for in-process, different for ADNL) let overlay_manager = overlay_managers[i].clone(); - - // Each node needs its own db_path to avoid RocksDB lock contention - // (all nodes run in the same process during tests) let db_path = format!("{}_node{}", db_path_base, i); let session = SessionFactory::create_session( @@ -1313,18 +1218,15 @@ where collation_requested: Arc::new(AtomicBool::new(false)), collation_count: Arc::new(AtomicU32::new(0)), on_candidate_count: Arc::new(AtomicU32::new(0)), - on_block_committed_count: Arc::new(AtomicU32::new(0)), + on_block_finalized_count: Arc::new(AtomicU32::new(0)), is_collator: Arc::new(AtomicBool::new(false)), config: config.clone(), - current_round: Arc::new(AtomicU32::new(0)), commit_latencies: Arc::new(Mutex::new(Vec::new())), - // SeqNo tracking: shared with listener so recovery updates reach here - next_expected_commit_seqno: next_expected_commit_seqno.clone(), - // Session errors - accumulated from SessionStats + max_finalized_seqno: max_finalized_seqno.clone(), + finalized_seqnos: finalized_seqnos.clone(), session_errors_count: Arc::new(AtomicU32::new(0)), - // Approved candidates storage - shared with listener for startup recovery approved_candidates: approved_candidates.clone(), - committed_blocks: committed_blocks.clone(), + finalized_blocks: finalized_blocks.clone(), source_index: i as u32, _session: session, _listener: listener.clone(), @@ -1351,55 +1253,66 @@ where // MC notification thread for shard sessions let mc_thread_stop_requested = Arc::new(AtomicBool::new(false)); let mc_thread_stopped = Arc::new(AtomicBool::new(false)); - let mut mc_thread_handle: Option> = - if let Some(mc_interval) = config.mc_notification_interval { - // Collect weak pointers to all sessions - let session_weak_ptrs: Vec> = instances - .iter() - .map(|inst| { - Arc::downgrade(&inst.lock()._session) as Weak - }) - .collect(); - - let stop_requested = mc_thread_stop_requested.clone(); - let stopped = mc_thread_stopped.clone(); - let test_name = config.test_name.clone(); - - Some(thread::spawn(move || { - log::info!("[MC-Thread] Started MC notification thread for test '{}'", test_name); - let mut mc_seqno: u32 = 0; - - while !stop_requested.load(Ordering::SeqCst) { - thread::sleep(mc_interval); - - if stop_requested.load(Ordering::SeqCst) { - break; - } + let mut mc_thread_handle: Option> = if let Some(mc_interval) = + config.mc_notification_interval + { + // Collect weak pointers to all sessions + let session_targets: Vec<(Weak, ShardIdent)> = instances + .iter() + .map(|inst| { + let guard = inst.lock(); + ( + Arc::downgrade(&guard._session) as Weak, + guard.config.shard.clone(), + ) + }) + .collect(); - // Notify all sessions about MC finalization - let mut notified_count = 0; - for weak_session in &session_weak_ptrs { - if let Some(session) = weak_session.upgrade() { - session.notify_mc_finalized(mc_seqno); - notified_count += 1; - } - } + let stop_requested = mc_thread_stop_requested.clone(); + let stopped = mc_thread_stopped.clone(); + let test_name = config.test_name.clone(); - log::debug!( - "[MC-Thread] Notified {} sessions about MC block seqno={}", - notified_count, - mc_seqno - ); - mc_seqno += 1; + Some(thread::spawn(move || { + log::info!("[MC-Thread] Started MC notification thread for test '{}'", test_name); + let mut mc_seqno: u32 = 0; + + while !stop_requested.load(Ordering::SeqCst) { + thread::sleep(mc_interval); + + if stop_requested.load(Ordering::SeqCst) { + break; } - stopped.store(true, Ordering::SeqCst); - log::info!("[MC-Thread] MC notification thread stopped"); - })) - } else { - mc_thread_stopped.store(true, Ordering::SeqCst); // No thread to stop - None - }; + // Notify all sessions about MC finalization + let mut notified_count = 0; + for (weak_session, shard) in &session_targets { + if let Some(session) = weak_session.upgrade() { + let mc_registered_top = BlockIdExt::with_params( + shard.clone(), + mc_seqno, + UInt256::rand(), + UInt256::rand(), + ); + session.notify_mc_finalized(mc_registered_top); + notified_count += 1; + } + } + + log::debug!( + "[MC-Thread] Notified {} sessions about MC block seqno={}", + notified_count, + mc_seqno + ); + mc_seqno += 1; + } + + stopped.store(true, Ordering::SeqCst); + log::info!("[MC-Thread] MC notification thread stopped"); + })) + } else { + mc_thread_stopped.store(true, Ordering::SeqCst); // No thread to stop + None + }; // Wait for all instances to finish or timeout let test_start = Instant::now(); @@ -1482,12 +1395,12 @@ where let inst = inst.lock(); if inst._session.is_panicked() { log::error!( - "Test '{}' detected PANIC in session {} (instance idx={}, finished={}, commits={})", + "Test '{}' detected PANIC in session {} (instance idx={}, finished={}, finalized={})", config.test_name, session_id.to_hex_string(), idx, inst.is_finished(), - inst.current_round.load(Ordering::SeqCst), + inst.on_block_finalized_count(), ); panic!( "Test '{}' failed: session panicked (instance idx={})", @@ -1597,12 +1510,18 @@ where // The new session's startup recovery will call get_approved_candidate() // to restore candidates from persistent storage. These candidates were // stored by the old session's on_generate_slot and on_candidate calls. - let (old_approved_candidates, prev_next_seqno, prev_commits) = { + let ( + old_approved_candidates, + prev_next_seqno, + prev_commits, + finalized_seqnos, + ) = { let inst = instances[node_idx].lock(); ( inst.approved_candidates.clone(), - inst.next_expected_commit_seqno.load(Ordering::SeqCst), - inst.on_block_committed_count.load(Ordering::SeqCst), + inst.max_finalized_seqno.load(Ordering::SeqCst), + inst.on_block_finalized_count.load(Ordering::SeqCst), + inst.finalized_seqnos.clone(), ) }; let candidates_count = @@ -1613,20 +1532,18 @@ where ); // Create seqno counters BEFORE listener - they will be updated by - // on_block_committed during recovery, before SessionInstance is wired. - // Preserve the previous baseline to avoid seqno regression across restart. + // on_block_finalized during recovery, before SessionInstance is wired. // Recovery callbacks may move it forward further before SessionInstance is wired. - let next_expected_commit_seqno = + // Preserve the finalized seqno set across restarts so the duplicate-finalization + // invariant remains global for the whole test, not just the latest process lifetime. + let max_finalized_seqno = Arc::new(AtomicU32::new(prev_next_seqno.max(ctx.initial_block_seqno))); - // Create a new listener that will be linked to the new session instance. - // Pass the OLD approved_candidates so get_approved_candidate() works during create_session. - // Pass the NEW seqno counters so they're updated by on_block_committed during recovery. let new_listener = Arc::new(SessionInstanceListener { instance: SpinMutex::new(Weak::new()), approved_candidates: old_approved_candidates.clone(), - next_expected_commit_seqno: next_expected_commit_seqno.clone(), - committed_blocks: committed_blocks.clone(), + max_finalized_seqno: max_finalized_seqno.clone(), + finalized_seqnos: finalized_seqnos.clone(), }); let session_listener: Arc = new_listener.clone(); @@ -1647,9 +1564,9 @@ where session.start(ctx.initial_block_seqno); // Create a completely new SessionInstance with fresh state. // The seqno trackers are shared with the listener - they were already - // updated by on_block_committed during recovery (before this point). + // updated by on_block_finalized during recovery (before this point). let recovered_next_seqno = - next_expected_commit_seqno.load(Ordering::SeqCst); + max_finalized_seqno.load(Ordering::SeqCst); let recovered_commits = recovered_next_seqno .saturating_sub(ctx.initial_block_seqno) .max(prev_commits); @@ -1665,26 +1582,21 @@ where let new_instance = Arc::new(SpinMutex::new(SessionInstance { public_key: ctx.local_key.clone(), - // Preserve progress reconstructed during startup recovery. - // Without this, restarted nodes can require a full extra - // `total_rounds` worth of commits and hit timeout. batch_processed: Arc::new(AtomicBool::new(recovered_finished)), collation_requested: Arc::new(AtomicBool::new(false)), collation_count: Arc::new(AtomicU32::new(0)), on_candidate_count: Arc::new(AtomicU32::new(0)), - on_block_committed_count: Arc::new(AtomicU32::new( + on_block_finalized_count: Arc::new(AtomicU32::new( recovered_commits, )), is_collator: Arc::new(AtomicBool::new(false)), config: config.clone(), - current_round: Arc::new(AtomicU32::new(recovered_commits)), commit_latencies: Arc::new(Mutex::new(Vec::new())), - // SeqNo tracking: shared with listener - already updated by recovery - next_expected_commit_seqno: next_expected_commit_seqno.clone(), + max_finalized_seqno: max_finalized_seqno.clone(), + finalized_seqnos: finalized_seqnos.clone(), session_errors_count: Arc::new(AtomicU32::new(0)), - // Share the preserved approved_candidates with the new instance. approved_candidates: old_approved_candidates.clone(), - committed_blocks: committed_blocks.clone(), + finalized_blocks: finalized_blocks.clone(), source_index: node_idx as u32, _session: session, _listener: new_listener.clone(), @@ -1706,18 +1618,21 @@ where old_inst.collation_count = new_inst.collation_count.clone(); old_inst.on_candidate_count = new_inst.on_candidate_count.clone(); - old_inst.on_block_committed_count = - new_inst.on_block_committed_count.clone(); + old_inst.on_block_finalized_count = + new_inst.on_block_finalized_count.clone(); old_inst.is_collator = new_inst.is_collator.clone(); - old_inst.current_round = new_inst.current_round.clone(); old_inst.commit_latencies = new_inst.commit_latencies.clone(); - old_inst.next_expected_commit_seqno = - new_inst.next_expected_commit_seqno.clone(); + old_inst + .max_finalized_seqno + .clone_from(&new_inst.max_finalized_seqno); + old_inst + .finalized_seqnos + .clone_from(&new_inst.finalized_seqnos); old_inst.session_errors_count = new_inst.session_errors_count.clone(); old_inst.approved_candidates = new_inst.approved_candidates.clone(); - old_inst.committed_blocks = new_inst.committed_blocks.clone(); + old_inst.finalized_blocks = new_inst.finalized_blocks.clone(); old_inst._session = new_inst._session.clone(); old_inst._listener = new_inst._listener.clone(); } @@ -1898,19 +1813,26 @@ where let inst = instance.lock(); let is_finished = inst.is_finished(); log::info!( - "Instance {}: finished={}, collation_requested={}, collation_count={}, candidate_count={}, commit_count={}", + "Instance {}: finished={}, collation_requested={}, collation_count={}, candidate_count={}, finalized_count={}", index, inst.is_finished(), inst.collation_requested(), inst.collation_count(), inst.on_candidate_count(), - inst.on_block_committed_count() + inst.on_block_finalized_count() ); + let finalized_count = inst.on_block_finalized_count(); + let unique_seqnos = inst.unique_finalized_seqno_count(); drop(inst); assert!(is_finished); + assert_eq!( + finalized_count as usize, unique_seqnos, + "Instance {}: finalized_count ({}) != unique seqno count ({}) — duplicate finalization detected", + index, finalized_count, unique_seqnos + ); } - // Log commit latency statistics table + // Log finalization latency statistics table log::info!(""); log::info!("=== COMMIT LATENCY STATISTICS ==="); print_latency_table_header(); @@ -1950,7 +1872,6 @@ where } // Assert no session errors occurred during the test - // Errors are tracked via SessionStats passed to on_block_committed let total_errors: u32 = instances.iter().map(|inst| inst.lock().session_errors_count()).sum(); assert!( total_errors == 0, @@ -1981,7 +1902,7 @@ fn test_simplex_consensus_basic() { run_simplex_consensus_test( TestConfig { total_rounds: 100, - min_commit_percent: 0.5, // At least 50% commits + min_finalized_percent: 0.5, // At least 50% commits node_count: 7, generation_failure_probability: 0.0, candidate_rejection_probability: 0.0, @@ -2004,24 +1925,24 @@ fn test_simplex_consensus_basic() { |instances| { // Verify commit rate meets minimum requirement let config = &instances[0].lock().config.clone(); - let min_commits = (config.total_rounds as f64 * config.min_commit_percent) as u32; + let min_finalized = (config.total_rounds as f64 * config.min_finalized_percent) as u32; for (idx, instance) in instances.iter().enumerate() { - let commits = instance.lock().on_block_committed_count(); + let commits = instance.lock().on_block_finalized_count(); log::info!( "Instance {}: {} commits out of {} total_rounds (min required: {})", idx, commits, config.total_rounds, - min_commits + min_finalized ); assert!( - commits >= min_commits, + commits >= min_finalized, "Instance {} has {} commits but requires at least {} ({}% of {} rounds)", idx, commits, - min_commits, - config.min_commit_percent * 100.0, + min_finalized, + config.min_finalized_percent * 100.0, config.total_rounds ); } @@ -2035,7 +1956,7 @@ fn test_simplex_consensus_with_failures() { run_simplex_consensus_test( TestConfig { total_rounds: 30, - min_commit_percent: 0.3, // Lower threshold due to failures + min_finalized_percent: 0.3, // Lower threshold due to failures node_count: 11, generation_failure_probability: 0.1, candidate_rejection_probability: 0.1, @@ -2060,24 +1981,24 @@ fn test_simplex_consensus_with_failures() { |instances| { // Verify commit rate meets minimum requirement let config = &instances[0].lock().config.clone(); - let min_commits = (config.total_rounds as f64 * config.min_commit_percent) as u32; + let min_finalized = (config.total_rounds as f64 * config.min_finalized_percent) as u32; for (idx, instance) in instances.iter().enumerate() { - let commits = instance.lock().on_block_committed_count(); + let commits = instance.lock().on_block_finalized_count(); log::info!( "Instance {}: {} commits out of {} total_rounds (min required: {})", idx, commits, config.total_rounds, - min_commits + min_finalized ); assert!( - commits >= min_commits, + commits >= min_finalized, "Instance {} has {} commits but requires at least {} ({}% of {} rounds)", idx, commits, - min_commits, - config.min_commit_percent * 100.0, + min_finalized, + config.min_finalized_percent * 100.0, config.total_rounds ); } @@ -2088,21 +2009,19 @@ fn test_simplex_consensus_with_failures() { /// FinalCert-recovery gremlin test /// /// Validates that a node with heavy message loss recovers missing FinalCerts -/// via `get_committed_candidate` (shared committed blocks map) rather than -/// relying on a Rust-only direct recovery path. +/// via finalized delivery rather than relying on a direct recovery path. /// /// Setup: 7 MC nodes. Node 0 gets 40% broadcast loss + 30% message loss. /// Other nodes have no loss and form a stable 6/7 majority (threshold=5). /// Node 0 will miss FinalizeVotes for some slots, hitting `WaitingForFinalCert`. -/// Recovery: `get_committed_candidate` reads proof from shared `CommittedBlocksMap` -/// (populated by other instances' `on_block_committed`), converts to VoteSignatureSet, -/// injects via `process_received_final_cert`, and resumes normal commit flow. +/// Recovery: the node eventually receives the finalization certificate from peers +/// and the finalized block is delivered via `on_block_finalized`. #[test] fn test_simplex_consensus_finalcert_recovery() { run_simplex_consensus_test( TestConfig { total_rounds: 60, - min_commit_percent: 0.3, + min_finalized_percent: 0.3, node_count: 7, generation_failure_probability: 0.0, candidate_rejection_probability: 0.0, @@ -2129,11 +2048,11 @@ fn test_simplex_consensus_finalcert_recovery() { }, |instances| { let config = &instances[0].lock().config.clone(); - let min_commits = (config.total_rounds as f64 * config.min_commit_percent) as u32; + let min_finalized = (config.total_rounds as f64 * config.min_finalized_percent) as u32; // All nodes (including the lossy node 0) must reach min commit count for (idx, instance) in instances.iter().enumerate() { - let commits = instance.lock().on_block_committed_count(); + let commits = instance.lock().on_block_finalized_count(); let errors = instance.lock().session_errors_count.load(Ordering::Relaxed); log::info!( "[finalcert-recovery] Instance {}: {} commits, {} errors \ @@ -2141,30 +2060,29 @@ fn test_simplex_consensus_finalcert_recovery() { idx, commits, errors, - min_commits, + min_finalized, idx == 0 ); assert!( - commits >= min_commits, + commits >= min_finalized, "Instance {} has {} commits but requires at least {} ({}% of {} rounds)", idx, commits, - min_commits, - config.min_commit_percent * 100.0, + min_finalized, + config.min_finalized_percent * 100.0, config.total_rounds ); } - // Verify shared committed blocks map was populated - let committed_count = - instances[0].lock().committed_blocks.lock().map(|m| m.len()).unwrap_or(0); + let finalized_count = + instances[0].lock().finalized_blocks.lock().map(|m| m.len()).unwrap_or(0); log::info!( - "[finalcert-recovery] Shared committed blocks map: {} entries", - committed_count + "[finalcert-recovery] Shared finalized blocks map: {} entries", + finalized_count ); assert!( - committed_count > 0, - "CommittedBlocksMap should have entries from on_block_committed" + finalized_count > 0, + "FinalizedBlocksMap should have entries from on_block_finalized" ); }, ); @@ -2179,7 +2097,7 @@ fn test_simplex_consensus_shard_with_mc_notifications() { run_simplex_consensus_test( TestConfig { total_rounds: 100, - min_commit_percent: 0.5, // At least 50% commits + min_finalized_percent: 0.5, // At least 50% commits node_count: 7, generation_failure_probability: 0.0, candidate_rejection_probability: 0.0, @@ -2204,24 +2122,24 @@ fn test_simplex_consensus_shard_with_mc_notifications() { |instances| { // Verify commit rate meets minimum requirement let config = &instances[0].lock().config.clone(); - let min_commits = (config.total_rounds as f64 * config.min_commit_percent) as u32; + let min_finalized = (config.total_rounds as f64 * config.min_finalized_percent) as u32; for (idx, instance) in instances.iter().enumerate() { - let commits = instance.lock().on_block_committed_count(); + let commits = instance.lock().on_block_finalized_count(); log::info!( "Instance {}: {} commits out of {} total_rounds (min required: {})", idx, commits, config.total_rounds, - min_commits + min_finalized ); assert!( - commits >= min_commits, + commits >= min_finalized, "Instance {} has {} commits but requires at least {} ({}% of {} rounds)", idx, commits, - min_commits, - config.min_commit_percent * 100.0, + min_finalized, + config.min_finalized_percent * 100.0, config.total_rounds ); } @@ -2238,7 +2156,7 @@ fn test_simplex_consensus_adnl_overlay() { run_simplex_consensus_test( TestConfig { total_rounds: 50, // Fewer rounds due to higher network latency - min_commit_percent: 0.5, + min_finalized_percent: 0.5, node_count: 5, // Smaller network for faster test generation_failure_probability: 0.0, candidate_rejection_probability: 0.0, @@ -2261,24 +2179,24 @@ fn test_simplex_consensus_adnl_overlay() { |instances| { // Verify commit rate meets minimum requirement let config = &instances[0].lock().config.clone(); - let min_commits = (config.total_rounds as f64 * config.min_commit_percent) as u32; + let min_finalized = (config.total_rounds as f64 * config.min_finalized_percent) as u32; for (idx, instance) in instances.iter().enumerate() { - let commits = instance.lock().on_block_committed_count(); + let commits = instance.lock().on_block_finalized_count(); log::info!( "Instance {}: {} commits out of {} total_rounds (min required: {})", idx, commits, config.total_rounds, - min_commits + min_finalized ); assert!( - commits >= min_commits, + commits >= min_finalized, "Instance {} has {} commits but requires at least {} ({}% of {} rounds)", idx, commits, - min_commits, - config.min_commit_percent * 100.0, + min_finalized, + config.min_finalized_percent * 100.0, config.total_rounds ); } @@ -2294,7 +2212,7 @@ fn test_simplex_consensus_adnl_net_gremlin() { run_simplex_consensus_test( TestConfig { total_rounds: 30, - min_commit_percent: 0.4, // allow some skips under partitions + min_finalized_percent: 0.4, // allow some skips under partitions node_count: 5, generation_failure_probability: 0.0, candidate_rejection_probability: 0.0, @@ -2322,24 +2240,24 @@ fn test_simplex_consensus_adnl_net_gremlin() { |instances| { // Verify commit rate meets minimum requirement (best-effort under partitions). let config = &instances[0].lock().config.clone(); - let min_commits = (config.total_rounds as f64 * config.min_commit_percent) as u32; + let min_finalized = (config.total_rounds as f64 * config.min_finalized_percent) as u32; for (idx, instance) in instances.iter().enumerate() { - let commits = instance.lock().on_block_committed_count(); + let commits = instance.lock().on_block_finalized_count(); log::info!( "Instance {}: {} commits out of {} total_rounds (min required: {})", idx, commits, config.total_rounds, - min_commits + min_finalized ); assert!( - commits >= min_commits, + commits >= min_finalized, "Instance {} has {} commits but requires at least {} ({}% of {} rounds)", idx, commits, - min_commits, - config.min_commit_percent * 100.0, + min_finalized, + config.min_finalized_percent * 100.0, config.total_rounds ); } @@ -2357,7 +2275,7 @@ fn test_simplex_consensus_restart_gremlin() { run_simplex_consensus_test( TestConfig { total_rounds: 50, - min_commit_percent: 0.3, + min_finalized_percent: 0.3, node_count: 5, generation_failure_probability: 0.0, candidate_rejection_probability: 0.0, @@ -2386,10 +2304,10 @@ fn test_simplex_consensus_restart_gremlin() { }, |instances| { let config = &instances[0].lock().config.clone(); - let min_commits = (config.total_rounds as f64 * config.min_commit_percent) as u32; + let min_finalized = (config.total_rounds as f64 * config.min_finalized_percent) as u32; for (idx, instance) in instances.iter().enumerate() { - let commits = instance.lock().on_block_committed_count(); + let commits = instance.lock().on_block_finalized_count(); log::info!( "Instance {}: {} commits out of {} total_rounds", idx, @@ -2397,14 +2315,14 @@ fn test_simplex_consensus_restart_gremlin() { config.total_rounds ); // Note: restarted nodes may have fewer commits if they were down during commit phase. - // We use a lower min_commit_percent to account for this. + // We use a lower min_finalized_percent to account for this. assert!( - commits >= min_commits, + commits >= min_finalized, "Instance {} has {} commits but requires at least {} ({}% of {} rounds)", idx, commits, - min_commits, - config.min_commit_percent * 100.0, + min_finalized, + config.min_finalized_percent * 100.0, config.total_rounds ); } @@ -2533,11 +2451,12 @@ fn test_simplex_start_gate() { first_block_timeout: Duration::from_millis(1000), slots_per_leader_window: 1, wait_for_db_init: true, + max_leader_window_desync: 10_000, ..Default::default() }; - let committed_blocks: CommittedBlocksMap = Arc::new(Mutex::new(HashMap::new())); - let commit_counters: Vec> = + let finalized_blocks: FinalizedBlocksMap = Arc::new(Mutex::new(HashSet::new())); + let finalized_counters: Vec> = (0..node_count).map(|_| Arc::new(AtomicU32::new(0))).collect(); let mut sessions: Vec = Vec::new(); // Keep instances alive so the listener Weak pointers remain valid. @@ -2545,7 +2464,7 @@ fn test_simplex_start_gate() { let config = TestConfig { total_rounds: 10, - min_commit_percent: 0.5, + min_finalized_percent: 0.5, node_count, generation_failure_probability: 0.0, candidate_rejection_probability: 0.0, @@ -2572,13 +2491,14 @@ fn test_simplex_start_gate() { let approved_candidates: Arc< Mutex>>, > = Arc::new(Mutex::new(HashMap::new())); - let next_expected_commit_seqno = Arc::new(AtomicU32::new(initial_block_seqno)); + let max_finalized_seqno = Arc::new(AtomicU32::new(initial_block_seqno)); + let finalized_seqnos: Arc>> = Arc::new(Mutex::new(HashSet::new())); let listener = Arc::new(SessionInstanceListener { instance: SpinMutex::new(Weak::new()), approved_candidates: approved_candidates.clone(), - next_expected_commit_seqno: next_expected_commit_seqno.clone(), - committed_blocks: committed_blocks.clone(), + max_finalized_seqno: max_finalized_seqno.clone(), + finalized_seqnos: finalized_seqnos.clone(), }); let session_listener: Arc = listener.clone(); @@ -2601,15 +2521,15 @@ fn test_simplex_start_gate() { collation_requested: Arc::new(AtomicBool::new(false)), collation_count: Arc::new(AtomicU32::new(0)), on_candidate_count: Arc::new(AtomicU32::new(0)), - on_block_committed_count: commit_counters[i].clone(), + on_block_finalized_count: finalized_counters[i].clone(), is_collator: Arc::new(AtomicBool::new(false)), config: config.clone(), - current_round: Arc::new(AtomicU32::new(0)), commit_latencies: Arc::new(Mutex::new(Vec::new())), - next_expected_commit_seqno, + max_finalized_seqno, + finalized_seqnos, session_errors_count: Arc::new(AtomicU32::new(0)), approved_candidates, - committed_blocks: committed_blocks.clone(), + finalized_blocks: finalized_blocks.clone(), _session: session.clone(), _listener: listener.clone(), })); @@ -2620,10 +2540,10 @@ fn test_simplex_start_gate() { instances.push(session_instance); } - // Phase 1: verify no commits while sessions are gated (overlay is warming up) - log::info!("[start_gate] Phase 1: verifying no commits for 2s without start()"); + // Phase 1a: verify no commits while sessions are gated (overlay is warming up) + log::info!("[start_gate] Phase 1a: verifying no commits for 2s without start()"); thread::sleep(Duration::from_secs(2)); - for (i, counter) in commit_counters.iter().enumerate() { + for (i, counter) in finalized_counters.iter().enumerate() { let commits = counter.load(Ordering::Relaxed); assert_eq!( commits, 0, @@ -2631,7 +2551,25 @@ fn test_simplex_start_gate() { i, commits ); } - log::info!("[start_gate] Phase 1 passed: zero commits before start()"); + log::info!("[start_gate] Phase 1a passed: zero commits before start()"); + + // Phase 1b: prolonged cold-start delay (regression guard). + // Even after a longer pre-start delay, no session may produce commits before start(). + const EXTRA_COLD_START_DELAY: Duration = Duration::from_secs(5); + log::info!( + "[start_gate] Phase 1b: verifying no commits after extra {:?} cold delay", + EXTRA_COLD_START_DELAY + ); + thread::sleep(EXTRA_COLD_START_DELAY); + for (i, counter) in finalized_counters.iter().enumerate() { + let commits = counter.load(Ordering::Relaxed); + assert_eq!( + commits, 0, + "Node {} committed {} blocks during prolonged cold-start delay before start()", + i, commits + ); + } + log::info!("[start_gate] Phase 1b passed: zero commits during prolonged cold delay"); // Phase 2: call start(seqno) on all sessions, then wait for commits log::info!( @@ -2643,29 +2581,29 @@ fn test_simplex_start_gate() { } let deadline = Instant::now() + Duration::from_secs(30); - let min_commits = 3u32; + let min_finalized = 3u32; loop { thread::sleep(Duration::from_millis(200)); let all_committed = - commit_counters.iter().all(|c| c.load(Ordering::Relaxed) >= min_commits); + finalized_counters.iter().all(|c| c.load(Ordering::Relaxed) >= min_finalized); if all_committed { break; } if Instant::now() > deadline { - for (i, counter) in commit_counters.iter().enumerate() { + for (i, counter) in finalized_counters.iter().enumerate() { log::error!("[start_gate] Node {} commits: {}", i, counter.load(Ordering::Relaxed)); } panic!( "Timed out waiting for {} commits after start() — \ sessions did not begin processing after start gate was released", - min_commits + min_finalized ); } } log::info!( "[start_gate] Phase 2 passed: all nodes committed >= {} blocks after start()", - min_commits + min_finalized ); for session in &sessions { @@ -2690,7 +2628,7 @@ fn test_simplex_consensus_candidate_chaining() { run_simplex_consensus_test( TestConfig { total_rounds: 20, - min_commit_percent: 0.3, + min_finalized_percent: 0.3, node_count: 4, generation_failure_probability: 0.0, candidate_rejection_probability: 0.0, @@ -2712,25 +2650,84 @@ fn test_simplex_consensus_candidate_chaining() { }, |instances| { let config = &instances[0].lock().config.clone(); - let min_commits = (config.total_rounds as f64 * config.min_commit_percent) as u32; + let min_finalized = (config.total_rounds as f64 * config.min_finalized_percent) as u32; for (idx, instance) in instances.iter().enumerate() { - let commits = instance.lock().on_block_committed_count(); + let commits = instance.lock().on_block_finalized_count(); log::info!( "[chaining] Instance {}: {} commits out of {} total_rounds (min required: {})", idx, commits, config.total_rounds, - min_commits + min_finalized ); assert!( - commits >= min_commits, + commits >= min_finalized, "Instance {} has {} commits but requires at least {} ({}% of {} rounds). \ Candidate chaining may not be working correctly.", idx, commits, - min_commits, - config.min_commit_percent * 100.0, + min_finalized, + config.min_finalized_percent * 100.0, + config.total_rounds + ); + } + }, + ); +} + +/// Candidate chaining in multi-slot windows should remain live under moderate +/// packet loss, validating in-window publication parity behavior. +#[test] +fn test_simplex_consensus_candidate_chaining_with_lossy_overlay() { + run_simplex_consensus_test( + TestConfig { + total_rounds: 20, + min_finalized_percent: 0.2, + node_count: 4, + generation_failure_probability: 0.0, + candidate_rejection_probability: 0.0, + max_collations: 2500, + target_rate: Duration::from_millis(300), + first_block_timeout: Duration::from_millis(3000), + test_name: "simplex_candidate_chaining_lossy".to_string(), + test_timeout: Duration::from_secs(150), + expect_timeout: false, + shard: ShardIdent::masterchain(), + mc_notification_interval: None, + overlay_type: OverlayType::InProcess, + net_gremlin: None, + restart_gremlin: None, + lossy_overlay: Some(consensus_common::LossyOverlayOpts { + lost_broadcast_probability: 0.15, + lost_message_probability: 0.1, + lost_query_probability: 0.1, + ..Default::default() + }), + lossy_overlay_node_indices: Some(vec![0]), + standstill_timeout: None, + slots_per_leader_window: Some(4), + }, + |instances| { + let config = &instances[0].lock().config.clone(); + let min_finalized = (config.total_rounds as f64 * config.min_finalized_percent) as u32; + + for (idx, instance) in instances.iter().enumerate() { + let commits = instance.lock().on_block_finalized_count(); + log::info!( + "[chaining-lossy] Instance {}: {} commits out of {} rounds (min required: {})", + idx, + commits, + config.total_rounds, + min_finalized + ); + assert!( + commits >= min_finalized, + "Instance {} has {} commits but requires at least {} ({}% of {} rounds)", + idx, + commits, + min_finalized, + config.min_finalized_percent * 100.0, config.total_rounds ); } diff --git a/src/node/simplex/tests/test_restart.rs b/src/node/simplex/tests/test_restart.rs index 1db5cc0..169fe5b 100644 --- a/src/node/simplex/tests/test_restart.rs +++ b/src/node/simplex/tests/test_restart.rs @@ -67,17 +67,22 @@ struct RestartSingleSessionListener { // Progress tracking (slot-based, not round-based for SIMPLEX_ROUNDLESS mode) last_slot_seen: AtomicU32, - last_committed_slot: AtomicU32, + last_finalized_slot: AtomicU32, + finalized_blocks_count: AtomicU32, collation_count: AtomicU32, // Restart markers restart_started: AtomicBool, first_slot_after_restart: AtomicU32, // u32::MAX means unset - seqno_updates_enabled: AtomicBool, - // Seqno tracking (simulates engine head in tests) - next_expected_collation_seqno: AtomicU32, - next_expected_commit_seqno: AtomicU32, + /// Maximum finalized seqno observed so far (out-of-order safe). + /// Used for `initial_block_seqno` on restart and as fallback in + /// `on_generate_slot` Implicit (genesis-only) case. + max_finalized_seqno: AtomicU32, + + /// Tracks finalized seqno -> BlockIdExt. + /// Invariant: each seqno is finalized exactly once with the same block identity. + finalized_seqnos: Mutex>, // Recovery verification approved_candidate_requests: AtomicU32, @@ -92,13 +97,13 @@ impl RestartSingleSessionListener { public_key, candidates: Mutex::new(HashMap::new()), last_slot_seen: AtomicU32::new(0), - last_committed_slot: AtomicU32::new(0), + last_finalized_slot: AtomicU32::new(0), + finalized_blocks_count: AtomicU32::new(0), collation_count: AtomicU32::new(0), restart_started: AtomicBool::new(false), first_slot_after_restart: AtomicU32::new(u32::MAX), - seqno_updates_enabled: AtomicBool::new(true), - next_expected_collation_seqno: AtomicU32::new(initial_block_seqno), - next_expected_commit_seqno: AtomicU32::new(initial_block_seqno), + max_finalized_seqno: AtomicU32::new(initial_block_seqno), + finalized_seqnos: Mutex::new(HashMap::new()), approved_candidate_requests: AtomicU32::new(0), max_errors_count: AtomicU32::new(0), } @@ -107,13 +112,10 @@ impl RestartSingleSessionListener { fn mark_restart_started(&self) { self.restart_started.store(true, Ordering::Release); self.first_slot_after_restart.store(u32::MAX, Ordering::Release); - // During restart recovery (FullReplay), we may get replay callbacks before - // the first new on_generate_slot. Those must NOT advance seqno trackers. - self.seqno_updates_enabled.store(false, Ordering::Release); } fn initial_block_seqno_for_restart(&self) -> u32 { - self.next_expected_commit_seqno.load(Ordering::SeqCst) + self.max_finalized_seqno.load(Ordering::SeqCst) } #[allow(dead_code)] @@ -121,8 +123,12 @@ impl RestartSingleSessionListener { self.last_slot_seen.load(Ordering::SeqCst) } - fn last_committed_slot(&self) -> u32 { - self.last_committed_slot.load(Ordering::SeqCst) + fn last_finalized_slot(&self) -> u32 { + self.last_finalized_slot.load(Ordering::SeqCst) + } + + fn finalized_blocks_count(&self) -> u32 { + self.finalized_blocks_count.load(Ordering::SeqCst) } fn first_slot_after_restart(&self) -> Option { @@ -178,36 +184,19 @@ impl SessionListener for RestartSingleSessionListener { let collation_num = self.collation_count.fetch_add(1, Ordering::SeqCst); if self.restart_started.load(Ordering::Acquire) { - // Capture first slot observed after restart (use collation count as proxy) - if self - .first_slot_after_restart + self.first_slot_after_restart .compare_exchange(u32::MAX, collation_num, Ordering::SeqCst, Ordering::SeqCst) - .is_ok() - { - // Now we are in normal operation after restart. - self.seqno_updates_enabled.store(true, Ordering::Release); - } + .ok(); } self.last_slot_seen.fetch_max(collation_num, Ordering::SeqCst); - // Derive seqno from explicit parent hint or use counter for implicit case. - // - // Important: `on_generate_slot` can be called multiple times for the same slot - // (collation retry / timeout). Until a block is actually committed, we MUST - // keep returning the same seqno. let seqno = match &parent { consensus_common::CollationParentHint::Implicit => { - // Genesis / bootstrap case: use counter (don't increment - may retry) - self.next_expected_collation_seqno.load(Ordering::SeqCst) - } - consensus_common::CollationParentHint::Explicit(parent_id) => { - // Explicit parent: derive seqno from parent (parent_seqno + 1) - let derived_seqno = parent_id.seq_no + 1; - // Update counter to match derived seqno for next iteration - self.next_expected_collation_seqno.store(derived_seqno, Ordering::SeqCst); - derived_seqno + // Genesis only — no parent block exists yet. + self.max_finalized_seqno.load(Ordering::SeqCst) } + consensus_common::CollationParentHint::Explicit(parent_id) => parent_id.seq_no + 1, }; log::info!( @@ -264,12 +253,26 @@ impl SessionListener for RestartSingleSessionListener { _source_info: BlockSourceInfo, _root_hash: BlockHash, _file_hash: BlockHash, + _data: BlockPayloadPtr, + _signatures: BlockSignaturesVariant, + _approve_signatures: Vec<(PublicKeyHash, BlockPayloadPtr)>, + _stats: consensus_common::SessionStats, + ) { + panic!( + "on_block_committed must not be called for Simplex sessions (finalized-driven only)" + ); + } + + fn on_block_finalized( + &self, + block_id: BlockIdExt, + _source_info: BlockSourceInfo, + _root_hash: BlockHash, + _file_hash: BlockHash, data: BlockPayloadPtr, signatures: BlockSignaturesVariant, _approve_signatures: Vec<(PublicKeyHash, BlockPayloadPtr)>, - stats: consensus_common::SessionStats, ) { - // Extract slot from Simplex signatures (SIMPLEX_ROUNDLESS: round is always u32::MAX) let slot = match &signatures { BlockSignaturesVariant::Simplex(s) => s.slot, _ => unreachable!( @@ -277,36 +280,31 @@ impl SessionListener for RestartSingleSessionListener { ), }; self.last_slot_seen.fetch_max(slot, Ordering::SeqCst); - self.last_committed_slot.fetch_max(slot, Ordering::SeqCst); - - // Track max errors count - self.max_errors_count.fetch_max(stats.errors_count, Ordering::SeqCst); - - // For normal operation, advance seqno trackers. - // For restart recommit callbacks (before the first on_generate_slot after restart), - // keep seqno trackers stable to match engine-head semantics. - if self.seqno_updates_enabled.load(Ordering::Acquire) { - let is_empty = data.data().is_empty(); - if is_empty { - // Empty blocks inherit parent's seqno, do not advance commit seqno. - let current = self.next_expected_commit_seqno.load(Ordering::SeqCst); - self.next_expected_collation_seqno.store(current, Ordering::SeqCst); - } else { - let committed = self.next_expected_commit_seqno.fetch_add(1, Ordering::SeqCst); - let next = committed + 1; - self.next_expected_collation_seqno.store(next, Ordering::SeqCst); + self.last_finalized_slot.fetch_max(slot, Ordering::SeqCst); + self.finalized_blocks_count.fetch_add(1, Ordering::SeqCst); + + let seqno = block_id.seq_no(); + + // Invariant: each seqno is finalized exactly once with the same block identity. + if let Ok(mut seen) = self.finalized_seqnos.lock() { + if let Some(prev) = seen.get(&seqno) { + assert_eq!( + *prev, block_id, + "DUPLICATE finalization for seqno {seqno} with different block_id: \ + prev={prev:?} new={block_id:?} (slot={slot})" + ); + panic!("DUPLICATE finalization for seqno {} (slot={})", seqno, slot); } + seen.insert(seqno, block_id.clone()); + } + + if !data.data().is_empty() { + self.max_finalized_seqno.fetch_max(seqno + 1, Ordering::SeqCst); } } fn on_block_skipped(&self, _round: u32) { - // NOTE: In SIMPLEX_ROUNDLESS mode, on_block_skipped is not called. - // Keep this for trait completeness. - if self.seqno_updates_enabled.load(Ordering::Acquire) { - // Reset collation seqno to commit seqno (flush precollations) - let current_commit_seqno = self.next_expected_commit_seqno.load(Ordering::SeqCst); - self.next_expected_collation_seqno.store(current_commit_seqno, Ordering::SeqCst); - } + unreachable!("on_block_skipped should not be called in Simplex"); } fn get_approved_candidate( @@ -336,15 +334,6 @@ impl SessionListener for RestartSingleSessionListener { ))), } } - - fn get_committed_candidate( - &self, - block_id: BlockIdExt, - callback: consensus_common::CommittedBlockProofCallback, - ) { - log::info!("get_committed_candidate: STUB for block_id={block_id}"); - callback(Err(error!("get_committed_candidate not implemented in test"))); - } } /* @@ -438,7 +427,7 @@ fn init_test_logger(test_name: &str) -> Arc { Test runner */ -fn run_single_node_restart_test(test_name: &str, strategy: RestartRecommitStrategy) { +fn run_single_node_restart_test(test_name: &str) { if !is_test_logging_enabled() { return; } @@ -467,13 +456,14 @@ fn run_single_node_restart_test(test_name: &str, strategy: RestartRecommitStrate let session_id: UInt256 = UInt256::from(rng.gen::<[u8; 32]>()); // Session options: fast timings for test speed and deterministic init + // Keep timings relaxed enough to avoid debug-mode skip storms, which can + // produce sparse finalized snapshots and make restart behavior flaky. let session_opts = SessionOptions { proto_version: 0, - target_rate: Duration::from_millis(50), - first_block_timeout: Duration::from_millis(300), + target_rate: Duration::from_millis(100), + first_block_timeout: Duration::from_millis(900), slots_per_leader_window: 1, wait_for_db_init: true, - restart_recommit_strategy: strategy, ..Default::default() }; @@ -484,7 +474,7 @@ fn run_single_node_restart_test(test_name: &str, strategy: RestartRecommitStrate let listener = Arc::new(RestartSingleSessionListener::new(public_key.clone(), initial_block_seqno)); let session_listener: Arc = listener.clone(); - log::info!("Starting session phase 1: strategy={:?}, db_path={}", strategy, db_path); + log::info!("Starting session phase 1: finalized-driven, db_path={}", db_path); let session_1 = SessionFactory::create_session( &session_opts, @@ -506,29 +496,28 @@ fn run_single_node_restart_test(test_name: &str, strategy: RestartRecommitStrate log::error!("PANIC-1: session panicked during phase 1 (restart test '{}')", test_name); panic!("session panicked during phase 1"); } - // Wait for N finalized (committed) slots before restart. - // We use `last_committed_slot` to track finalized slots. - if listener.last_committed_slot() + 1 >= rounds_before_restart { + // Wait for N finalized callbacks before restart. + if listener.finalized_blocks_count() >= rounds_before_restart { break; } thread::sleep(Duration::from_millis(50)); } - if listener.last_committed_slot() + 1 < rounds_before_restart { + if listener.finalized_blocks_count() < rounds_before_restart { log::error!( - "TIMEOUT in phase 1: did not reach {} committed slots within {:?} (last_committed_slot={})", - rounds_before_restart, - PHASE_TIMEOUT, - listener.last_committed_slot(), + "TIMEOUT in phase 1: did not reach {rounds_before_restart} finalized callbacks \ + within {PHASE_TIMEOUT:?} (finalized_count={}, last_finalized_slot={})", + listener.finalized_blocks_count(), + listener.last_finalized_slot(), ); panic!( - "phase 1 did not reach {} committed slots within {:?} (last_committed_slot={})", - rounds_before_restart, - PHASE_TIMEOUT, - listener.last_committed_slot(), + "phase 1 did not reach {rounds_before_restart} finalized callbacks \ + within {PHASE_TIMEOUT:?} (finalized_count={}, last_finalized_slot={})", + listener.finalized_blocks_count(), + listener.last_finalized_slot(), ); } - let last_committed_slot_before = listener.last_committed_slot(); + let last_finalized_slot_before = listener.last_finalized_slot(); let collation_before = listener.collation_count(); // Stop session 1 and give some time for DB handles to close @@ -540,10 +529,9 @@ fn run_single_node_restart_test(test_name: &str, strategy: RestartRecommitStrate let restart_initial_seqno = listener.initial_block_seqno_for_restart(); log::info!( - "Starting session phase 2 (restart): last_committed_slot_before={}, restart_initial_seqno={}, db_path={}", - last_committed_slot_before, - restart_initial_seqno, - db_path + "Starting session phase 2 (restart): \ + last_finalized_slot_before={last_finalized_slot_before}, \ + restart_initial_seqno={restart_initial_seqno}, db_path={db_path}" ); let session_2 = SessionFactory::create_session( @@ -582,13 +570,13 @@ fn run_single_node_restart_test(test_name: &str, strategy: RestartRecommitStrate } }; - // Note: first_after is collation count (starting from 0), last_committed_slot is from signatures + // Note: first_after is collation count (starting from 0), last_finalized_slot is from signatures // After restart, the collation count resets but we just need to verify we got a collation request // The slot monotonicity is now verified by seqno tracking, not by comparing collation counts log::info!( - "Restart slot check: first_collation_after_restart={}, last_committed_slot_before={}", + "Restart slot check: first_collation_after_restart={}, last_finalized_slot_before={}", first_after, - last_committed_slot_before + last_finalized_slot_before ); // Also require that collation actually happened after restart @@ -605,10 +593,9 @@ fn run_single_node_restart_test(test_name: &str, strategy: RestartRecommitStrate } if listener.collation_count() < collation_before + 2 { log::error!( - "TIMEOUT in phase 2: expected at least 2 new collations after restart (before={}, after={}) within {:?}", - collation_before, - listener.collation_count(), - PHASE_TIMEOUT + "TIMEOUT in phase 2: expected at least 2 new collations after restart \ + (before={collation_before}, after={}) within {PHASE_TIMEOUT:?}", + listener.collation_count() ); panic!( "expected at least 2 new collations after restart (before={}, after={}) within {:?}", @@ -631,12 +618,10 @@ fn run_single_node_restart_test(test_name: &str, strategy: RestartRecommitStrate thread::sleep(Duration::from_millis(100)); log::info!( - "Restart test '{}' complete: strategy={:?}, last_committed_slot_before={}, first_collation_after_restart={}, collations_before={}, collations_after={}, approved_candidate_requests={}", - test_name, - strategy, - last_committed_slot_before, - first_after, - collation_before, + "Restart test '{test_name}' complete: \ + last_finalized_slot_before={last_finalized_slot_before}, \ + first_collation_after_restart={first_after}, collations_before={collation_before}, \ + collations_after={}, approved_candidate_requests={}", listener.collation_count(), listener.approved_candidate_requests() ); @@ -648,16 +633,5 @@ fn run_single_node_restart_test(test_name: &str, strategy: RestartRecommitStrate #[test] fn test_single_session_restart_round_monotonicity_first_commit_after_finalized() { - run_single_node_restart_test( - "simplex_restart_single_first_commit", - RestartRecommitStrategy::FirstCommitAfterFinalized, - ); -} - -#[test] -fn test_single_session_restart_round_monotonicity_full_replay() { - run_single_node_restart_test( - "simplex_restart_single_full_replay", - RestartRecommitStrategy::FullReplay, - ); + run_single_node_restart_test("simplex_restart_single_first_commit"); } diff --git a/src/node/simplex/tests/test_validation.rs b/src/node/simplex/tests/test_validation.rs index b5985cf..15ff043 100644 --- a/src/node/simplex/tests/test_validation.rs +++ b/src/node/simplex/tests/test_validation.rs @@ -26,7 +26,7 @@ use std::{ time::{Duration, Instant, SystemTime}, }; use ton_block::{ - error, sha256_digest, BlockIdExt, BlockSignaturesVariant, BocFlags, BocWriter, BuilderData, + sha256_digest, BlockIdExt, BlockSignaturesVariant, BocFlags, BocWriter, BuilderData, Ed25519KeyOption, ShardIdent, UInt256, }; @@ -60,10 +60,10 @@ struct ValidationTestListener { collation_count: Arc, /// Public key for generating candidates public_key: PublicKey, - /// Next expected seqno for collation - increases after each successful collation + /// Next expected seqno for collation — updated on finalization next_expected_collation_seqno: Arc, - /// Next expected seqno for commit - initialized with initial_block_seqno, +1 for each commit - next_expected_commit_seqno: Arc, + /// Maximum finalized seqno observed (monotonically advances via fetch_max) + max_finalized_seqno: Arc, } impl SessionListener for ValidationTestListener { @@ -111,20 +111,11 @@ impl SessionListener for ValidationTestListener { self.collation_requested.store(true, Ordering::Release); self.collation_count.fetch_add(1, Ordering::Relaxed); - // Derive seqno from explicit parent hint or use stable counter value for implicit case let seqno = match &parent { consensus_common::CollationParentHint::Implicit => { - // Keep seqno stable across retries for the same slot. - // The counter is advanced on commit. - self.next_expected_collation_seqno.load(Ordering::SeqCst) - } - consensus_common::CollationParentHint::Explicit(parent_id) => { - // Explicit parent: derive seqno from parent (parent_seqno + 1) - let derived_seqno = parent_id.seq_no + 1; - // Update counter to match derived seqno for next iteration - self.next_expected_collation_seqno.store(derived_seqno + 1, Ordering::SeqCst); - derived_seqno + self.max_finalized_seqno.load(Ordering::SeqCst) } + consensus_common::CollationParentHint::Explicit(parent_id) => parent_id.seq_no + 1, }; // Generate dummy candidate with proper hashes @@ -175,28 +166,41 @@ impl SessionListener for ValidationTestListener { fn on_block_committed( &self, - source_info: simplex::BlockSourceInfo, - root_hash: BlockHash, + _source_info: simplex::BlockSourceInfo, + _root_hash: BlockHash, _file_hash: BlockHash, _data: BlockPayloadPtr, _signatures: BlockSignaturesVariant, _approve_signatures: Vec<(PublicKeyHash, BlockPayloadPtr)>, _stats: consensus_common::SessionStats, + ) { + panic!( + "on_block_committed must not be called for Simplex sessions (finalized-driven only)" + ); + } + + fn on_block_finalized( + &self, + block_id: BlockIdExt, + source_info: simplex::BlockSourceInfo, + _root_hash: BlockHash, + _file_hash: BlockHash, + _data: BlockPayloadPtr, + _signatures: BlockSignaturesVariant, + _approve_signatures: Vec<(PublicKeyHash, BlockPayloadPtr)>, ) { let slot = source_info.priority.round; + let seqno = block_id.seq_no; - // Increment next_expected_commit_seqno and update next_expected_collation_seqno - let committed_seqno = self.next_expected_commit_seqno.fetch_add(1, Ordering::SeqCst); - let next_commit_seqno = committed_seqno + 1; - self.next_expected_collation_seqno.store(next_commit_seqno, Ordering::SeqCst); + self.max_finalized_seqno.fetch_max(seqno + 1, Ordering::SeqCst); + self.next_expected_collation_seqno.fetch_max(seqno + 1, Ordering::SeqCst); log::info!( - "ValidationTestListener[{}]::on_block_committed: slot={}, hash={:?}, committed_seqno={}, next_expected={}", + "ValidationTestListener[{}]::on_block_finalized: slot={}, seqno={}, block_id={}", self.node_idx, slot, - root_hash, - committed_seqno, - next_commit_seqno + seqno, + block_id, ); } @@ -214,15 +218,6 @@ impl SessionListener for ValidationTestListener { ) { // Not used in this test } - - fn get_committed_candidate( - &self, - block_id: BlockIdExt, - callback: consensus_common::CommittedBlockProofCallback, - ) { - log::info!("get_committed_candidate: STUB for block_id={block_id}"); - callback(Err(error!("get_committed_candidate not implemented in test"))); - } } /* @@ -374,7 +369,7 @@ fn run_validation_test() { collation_count: collation_count_0.clone(), public_key: private_key_0.clone(), next_expected_collation_seqno: Arc::new(AtomicU32::new(initial_block_seqno)), - next_expected_commit_seqno: Arc::new(AtomicU32::new(initial_block_seqno)), + max_finalized_seqno: Arc::new(AtomicU32::new(initial_block_seqno)), }); let listener_1 = Arc::new(ValidationTestListener { @@ -385,7 +380,7 @@ fn run_validation_test() { collation_count: collation_count_1.clone(), public_key: private_key_1.clone(), next_expected_collation_seqno: Arc::new(AtomicU32::new(initial_block_seqno)), - next_expected_commit_seqno: Arc::new(AtomicU32::new(initial_block_seqno)), + max_finalized_seqno: Arc::new(AtomicU32::new(initial_block_seqno)), }); let session_listener_0: Arc = listener_0.clone(); diff --git a/src/node/src/config.rs b/src/node/src/config.rs index d48bd12..d73abd8 100644 --- a/src/node/src/config.rs +++ b/src/node/src/config.rs @@ -1177,7 +1177,7 @@ impl TonNodeConfig { pub enum ConfigEvent { AddValidatorAdnlKey(Arc, i32), - //RemoveValidatorAdnlKey(Arc, i32) + RemoveValidatorAdnlKey(Arc, i32), } #[async_trait::async_trait] @@ -1484,13 +1484,21 @@ impl NodeConfigHandler { } async fn revision_validator_keys( + self: &Arc, validator_keys: &Arc, config: &mut TonNodeConfig, + subscribers: &[Arc], ) -> Result<()> { if let Some(config_validator_keys) = &config.validator_keys { if config_validator_keys.len() > 2 { let oldest_validator_key = NodeConfigHandler::get_oldest_validator_key(config); if let Some(oldest_key) = oldest_validator_key { + log::info!( + "revision_validator_keys: removing oldest key \ + election_id={} adnl={:?}", + oldest_key.election_id, + oldest_key.validator_adnl_key_id, + ); config .remove_validator_key( oldest_key.validator_key_id.clone(), @@ -1499,8 +1507,34 @@ impl NodeConfigHandler { .await?; validator_keys.remove(&oldest_key)?; config.remove_key_from_key_ring(&oldest_key.validator_key_id.clone()).await?; - if let Some(adnl_key_id) = oldest_key.validator_adnl_key_id { - config.remove_key_from_key_ring(&adnl_key_id).await?; + if let Some(adnl_key_id) = &oldest_key.validator_adnl_key_id { + config.remove_key_from_key_ring(adnl_key_id).await?; + // Notify subscribers to clean up ADNL/QUIC/DHT state + // for the removed key. + let adnl_key_bytes = base64_decode(adnl_key_id)?; + let adnl_key_id = + KeyId::from_data(adnl_key_bytes[..].try_into().map_err(|_| { + error!("Invalid ADNL key length for {adnl_key_id}") + })?); + let election_id = oldest_key.election_id; + for subscriber in subscribers.iter() { + let subscriber = subscriber.clone(); + let adnl_key_id = adnl_key_id.clone(); + self.runtime_handle.spawn(async move { + if let Err(e) = subscriber + .event(ConfigEvent::RemoveValidatorAdnlKey( + adnl_key_id, + election_id, + )) + .await + { + log::warn!( + "revision_validator_keys: \ + RemoveValidatorAdnlKey subscriber error: {e:?}" + ); + } + }); + } } } } @@ -1537,8 +1571,8 @@ impl NodeConfigHandler { }); } - // check validator keys - Self::revision_validator_keys(validator_keys, config).await?; + // check validator keys — remove oldest if more than 2 + self.revision_validator_keys(validator_keys, config, subscribers).await?; config.save_to_file(&config.file_name).await?; Ok(()) } diff --git a/src/node/src/engine_operations.rs b/src/node/src/engine_operations.rs index 668eaed..cdaab3f 100644 --- a/src/node/src/engine_operations.rs +++ b/src/node/src/engine_operations.rs @@ -46,8 +46,9 @@ use ton_api::ton::{ ton_node::broadcast::{BlockBroadcast, ExternalMessageBroadcast, NewShardBlockBroadcast}, }; use ton_block::{ - error, fail, AccountIdPrefixFull, BlockIdExt, BlockSignaturesVariant, Cell, CellsFactory, - ConfigParams, CryptoSignaturePair, KeyId, Message, OutMsgQueue, Result, ShardIdent, UInt256, + error, fail, AccountIdPrefixFull, BlockIdExt, BlockSignaturesVariant, BocFlags, BocWriter, + Cell, CellsFactory, ConfigParams, CryptoSignaturePair, KeyId, Message, OutMsgQueue, Result, + ShardIdent, UInt256, }; use validator_session::{BlockHash, SessionId, ValidatorBlockCandidate}; @@ -130,23 +131,30 @@ impl EngineOperations for Engine { /// Register the local node's participation in a validator list and update network overlays. /// /// Delegates to [`PrivateOverlayOperations::set_validator_list`] for key matching and - /// ADNL setup, then refreshes private and custom overlays **only** when the network - /// layer is fully ready (`network_ready == true`). Overlay updates require the ADNL key - /// to be loaded into the ADNL stack first, which is why they happen here rather than - /// at the call site. + /// ADNL setup. Membership is decided by pubkey match only; overlay updates are + /// performed only when per-list network context is actually available. async fn set_validator_list( &self, validator_list_id: UInt256, validators: &[CatchainNode], ) -> Result { - let outcome = - self.validator_network().set_validator_list(validator_list_id, validators).await?; - - if matches!(&outcome, ValidatorListOutcome::Selected { network_ready: true, .. }) { - let state = self.load_last_applied_mc_state().await?; - let config = state.config_params()?; - self.overlays_router()?.update_private_overlays(config).await?; - self.overlays_router()?.update_custom_overlays(None).await?; + let network = self.validator_network(); + let outcome = network.set_validator_list(validator_list_id.clone(), validators).await?; + + if matches!(&outcome, ValidatorListOutcome::Selected { .. }) { + if network.has_validator_list_context(&validator_list_id) { + let state = self.load_last_applied_mc_state().await?; + let config = state.config_params()?; + self.overlays_router()?.update_private_overlays(config).await?; + self.overlays_router()?.update_custom_overlays(None).await?; + } else { + log::warn!( + target: "validator_manager", + "Validator list {:x} selected by pubkey, but network context is not ready yet; \ + overlay refresh is deferred", + validator_list_id + ); + } } Ok(outcome) } @@ -1067,6 +1075,10 @@ impl EngineOperations for Engine { block_root: &Cell, ) -> Result<()> { log::trace!("send_block_candidate_broadcast {}", id); + self.cache_block_candidate( + id, + BocWriter::with_flags([block_root.clone()], BocFlags::all())?.write_to_vec()?, + )?; self.overlays_router()? .send_block_candidate_broadcast(id, cc_seqno, validator_set_hash, block_root) .await?; diff --git a/src/node/src/engine_traits.rs b/src/node/src/engine_traits.rs index 6f6ca39..7608a7b 100644 --- a/src/node/src/engine_traits.rs +++ b/src/node/src/engine_traits.rs @@ -96,29 +96,20 @@ pub struct ValidatorKeyBinding { /// # C++ counterpart /// /// C++ uses `get_validator()` (`manager.cpp`) which returns a `PublicKeyHash` -/// (zero = not a validator). There is no explicit `network_ready` concept in C++ because -/// ADNL identity is always resolvable (falling back to the validator public key hash when -/// `addr` is zero, see `create_validator_group()` in `manager.cpp`). The `network_ready` flag is a -/// Rust-specific extension that decouples validator membership from ADNL/overlay readiness. -/// Rust still records validator membership immediately, while overlay activation is retried -/// until the network layer finishes loading the ADNL key. +/// (zero = not a validator). Membership is determined by pubkey-in-set only; +/// ADNL/overlay readiness is handled in transport/context paths, not in the +/// membership outcome itself. /// /// # Variants /// -/// - `Selected { key, matching_keys, network_ready }` -- local node's public key is in the +/// - `Selected { key, matching_keys }` -- local node's public key is in the /// validator set. `key` is the first selected local key used for network setup, while /// `matching_keys` preserves all local matches in C++ `temp_keys_` order so shard subsets -/// can still choose the right local validator key. `network_ready` is `true` when the -/// corresponding ADNL key and overlay infrastructure are operational; `false` when the -/// pubkey matched but ADNL setup is still pending. +/// can still choose the right local validator key. /// - `NotValidator` -- no local key matches the validator set. #[derive(Debug)] pub enum ValidatorListOutcome { - Selected { - key: Arc, - matching_keys: Vec>, - network_ready: bool, - }, + Selected { key: Arc, matching_keys: Vec> }, NotValidator, } @@ -130,6 +121,8 @@ pub trait PrivateOverlayOperations: Sync + Send { validators: &[CatchainNode], ) -> Result; + fn has_validator_list_context(&self, validator_list_id: &UInt256) -> bool; + fn activate_validator_list(&self, validator_list_id: UInt256) -> Result<()>; fn remove_validator_list(&self, validator_list_id: UInt256) -> Result; diff --git a/src/node/src/network/custom_overlay_client.rs b/src/node/src/network/custom_overlay_client.rs index 1e8023f..2d4d2ce 100644 --- a/src/node/src/network/custom_overlay_client.rs +++ b/src/node/src/network/custom_overlay_client.rs @@ -446,7 +446,7 @@ impl CustomOverlayClient { peer ); self.overlay_node - .add_private_peers(&local_key, vec![(adnl_addr, quic_addr, key)])?; + .add_private_peers_to_adnl(&local_key, vec![(adnl_addr, quic_addr, key)])?; } Ok(None) => { log::warn!("{}: find address for {} failed", self.id, &peer); diff --git a/src/node/src/network/full_node_overlays.rs b/src/node/src/network/full_node_overlays.rs index 3e9ee83..4e332ce 100644 --- a/src/node/src/network/full_node_overlays.rs +++ b/src/node/src/network/full_node_overlays.rs @@ -543,8 +543,8 @@ impl FullNodeOverlaysRouter { /// Look up the local ADNL key for the given validator set. /// /// Returns `None` both when the node is not a validator and when it is a validator - /// but the ADNL/overlay context is not yet ready (the `network_ready == false` case - /// in [`ValidatorListOutcome`]). Callers must tolerate `None` gracefully. + /// but the ADNL/overlay context is not yet ready. Callers must tolerate `None` + /// gracefully. fn try_get_our_key( self: &Arc, validators: &ValidatorSet, diff --git a/src/node/src/network/node_network.rs b/src/node/src/network/node_network.rs index 682ae38..b58d5d2 100644 --- a/src/node/src/network/node_network.rs +++ b/src/node/src/network/node_network.rs @@ -87,11 +87,11 @@ struct ValidatorContext { /// Select the local node's entry from the validator list by local-key order. /// /// Returns `(Some(node), adnl_missing)` where `adnl_missing` is true when the matched -/// validator's ADNL ID is not among the locally known ADNL keys. This mirrors the C++ +/// validator's ADNL identity cannot be resolved from locally known ADNL keys, including +/// C++-parity fallback to validator pubkey short ID. This mirrors the C++ /// `get_validator()` function (`manager.cpp`) which iterates `temp_keys_` and returns the /// first local key that belongs to the validator set. C++ does not consider ADNL readiness -/// at this layer; the `adnl_missing` flag is a Rust-specific diagnostic for the -/// network-readiness model. +/// at this layer; the `adnl_missing` flag is a Rust-side diagnostic only. fn select_local_validator_candidate<'a>( validators: &'a [CatchainNode], validator_key_ids: &[Arc], @@ -99,7 +99,8 @@ fn select_local_validator_candidate<'a>( ) -> (Option<&'a CatchainNode>, bool) { for key_id in validator_key_ids { if let Some(local_validator) = validators.iter().find(|val| val.public_key.id() == key_id) { - let adnl_missing = !validator_adnl_key_ids.contains(&local_validator.adnl_id); + let adnl_missing = !validator_adnl_key_ids.contains(&local_validator.adnl_id) + && !validator_adnl_key_ids.contains(local_validator.public_key.id()); return (Some(local_validator), adnl_missing); } } @@ -116,12 +117,20 @@ fn collect_local_validator_candidates<'a>( .collect() } +fn local_validator_adnl_candidates(local_validator: &CatchainNode) -> Vec> { + let mut ids = vec![local_validator.adnl_id.clone()]; + let fallback = local_validator.public_key.id().clone(); + if fallback != local_validator.adnl_id.clone() { + ids.push(fallback); + } + ids +} + declare_counted!( struct ValidatorSetContext { validator_peers: Vec>, validator_key: Arc, validator_adnl_key: Arc, - election_id: usize, } ); @@ -440,7 +449,7 @@ impl NodeNetwork { callback(val.adnl_id.clone()); } None => { - overlay.add_private_peers( + overlay.add_private_peers_to_adnl( &local_adnl_id, vec![(adnl_addr, quic_addr, key)], )?; @@ -553,6 +562,72 @@ impl NodeNetwork { ); Ok(true) } + + async fn resolve_validator_adnl_key( + &self, + validator_list_id: UInt256, + local_validator: &CatchainNode, + election_id: i32, + ) -> Option> { + let candidate_ids = local_validator_adnl_candidates(local_validator); + + for candidate_id in &candidate_ids { + if let Ok(adnl_key) = self.network_context.stack.adnl.key_by_id(candidate_id) { + if candidate_id.as_ref() != local_validator.adnl_id.as_ref() { + log::warn!( + target: "validator_manager", + "set_validator_list {:x}: using validator-pubkey ADNL fallback {} \ + instead of descriptor ADNL {}", + validator_list_id, + candidate_id, + local_validator.adnl_id.as_ref() + ); + } + return Some(adnl_key); + } + } + + for candidate_id in &candidate_ids { + if let Err(e) = + self.load_and_store_validator_adnl_key(candidate_id.clone(), election_id).await + { + log::warn!( + target: "validator_manager", + "set_validator_list {:x}: cannot load/store ADNL key {} (election_id={}): {}", + validator_list_id, + candidate_id, + election_id, + e + ); + continue; + } + + if let Ok(adnl_key) = self.network_context.stack.adnl.key_by_id(candidate_id) { + if candidate_id.as_ref() != local_validator.adnl_id.as_ref() { + log::warn!( + target: "validator_manager", + "set_validator_list {:x}: loaded validator-pubkey ADNL fallback {} \ + instead of descriptor ADNL {}", + validator_list_id, + candidate_id, + local_validator.adnl_id.as_ref() + ); + } + return Some(adnl_key); + } + } + + log::warn!( + target: "validator_manager", + "set_validator_list {:x}: validator pubkey {} matched but no ADNL key is available \ + yet (descriptor_adnl={}, pubkey_fallback={}); continuing with membership only", + validator_list_id, + hex::encode(local_validator.public_key.id().data()), + local_validator.adnl_id.as_ref(), + local_validator.public_key.id().as_ref(), + ); + None + } } #[async_trait::async_trait] @@ -565,8 +640,11 @@ impl PrivateOverlayOperations for NodeNetwork { /// 3. Fetch peer addresses via DHT; queue missing peers for background resolution /// 4. Create the `ValidatorSetContext` and register overlay peers /// - /// Returns `Selected { network_ready: false }` when the pubkey matches but ADNL setup - /// fails -- the caller should retry next round without treating this as non-membership. + /// Returns `Selected` whenever pubkey membership is confirmed. + /// + /// ADNL/network setup is best-effort: if ADNL key/context cannot be prepared yet, + /// the function logs warnings and returns membership-only `Selected` so validator + /// lifecycle decisions are not blocked by transport readiness. async fn set_validator_list( &self, validator_list_id: UInt256, @@ -631,76 +709,22 @@ impl PrivateOverlayOperations for NodeNetwork { log::warn!( target: "validator_manager", "set_validator_list {:x}: public key {} matches local key but ADNL id {} \ - is not in actual ADNL key set ({} keys). Possible config/key-binding issue.", + is not in actual ADNL key set ({} keys). Will also try C++-style \ + pubkey-derived ADNL fallback.", validator_list_id, hex::encode(local_validator.public_key.id().data()), hex::encode(local_validator.adnl_id.data()), validator_adnl_key_ids.len() ); } - let local_validator_adnl_key = match self - .network_context - .stack - .adnl - .key_by_id(&local_validator.adnl_id) - { - Ok(adnl_key) => adnl_key, - Err(e) => { - log::warn!("error load adnl validator key (first attempt): {e}"); - match self - .load_and_store_validator_adnl_key(local_validator.adnl_id.clone(), election_id) - .await - { - Ok(true) => {} - Ok(false) if pubkey_matched_but_adnl_missing => { - log::warn!( - target: "validator_manager", - "set_validator_list {:x}: ADNL key {} not available yet \ - (pubkey matched, network not ready; will retry next round)", - validator_list_id, local_validator.adnl_id, - ); - return Ok(ValidatorListOutcome::Selected { - key: local_validator_key.clone(), - matching_keys: matching_local_keys.clone(), - network_ready: false, - }); - } - Ok(false) => { - fail!("can't load and store adnl key (id: {})", &local_validator.adnl_id); - } - Err(e) if pubkey_matched_but_adnl_missing => { - log::warn!( - target: "validator_manager", - "set_validator_list {:x}: ADNL key {} load failed for matched \ - validator pubkey (network pending, will retry): {e}", - validator_list_id, local_validator.adnl_id, - ); - return Ok(ValidatorListOutcome::Selected { - key: local_validator_key.clone(), - matching_keys: matching_local_keys.clone(), - network_ready: false, - }); - } - Err(e) => return Err(e), - } - match self.network_context.stack.adnl.key_by_id(&local_validator.adnl_id) { - Ok(key) => key, - Err(e) if pubkey_matched_but_adnl_missing => { - log::warn!( - target: "validator_manager", - "set_validator_list {:x}: ADNL key {} still not loadable \ - after store (pubkey matched, network pending): {e}", - validator_list_id, local_validator.adnl_id, - ); - return Ok(ValidatorListOutcome::Selected { - key: local_validator_key.clone(), - matching_keys: matching_local_keys.clone(), - network_ready: false, - }); - } - Err(e) => return Err(e.into()), - } - } + let local_validator_adnl_key = self + .resolve_validator_adnl_key(validator_list_id.clone(), &local_validator, election_id) + .await; + let Some(local_validator_adnl_key) = local_validator_adnl_key else { + return Ok(ValidatorListOutcome::Selected { + key: local_validator_key.clone(), + matching_keys: matching_local_keys, + }); }; let mut peers = Vec::new(); @@ -731,7 +755,7 @@ impl PrivateOverlayOperations for NodeNetwork { self.network_context .stack .overlay - .add_private_peers(local_validator_adnl_key.id(), peers)?; + .add_private_peers_to_adnl(local_validator_adnl_key.id(), peers)?; let context = self.try_add_counted_object( &self.validator_context.sets_contexts, @@ -741,7 +765,6 @@ impl PrivateOverlayOperations for NodeNetwork { validator_peers: peers_ids.clone(), validator_key: local_validator_key.clone(), validator_adnl_key: local_validator_adnl_key.clone(), - election_id: election_id as usize, counter: self.network_context.engine_allocated.validator_sets.clone().into(), }; #[cfg(feature = "telemetry")] @@ -812,10 +835,13 @@ impl PrivateOverlayOperations for NodeNetwork { Ok(ValidatorListOutcome::Selected { key: context.validator_key.clone(), matching_keys: matching_local_keys, - network_ready: true, }) } + fn has_validator_list_context(&self, validator_list_id: &UInt256) -> bool { + self.validator_context.sets_contexts.get(validator_list_id).is_some() + } + fn activate_validator_list(&self, validator_list_id: UInt256) -> Result<()> { log::trace!("activate_validator_list {:x}", validator_list_id); self.validator_context.current_set.insert(0, validator_list_id); @@ -873,10 +899,25 @@ impl PrivateOverlayOperations for NodeNetwork { ) -> Result> { let validator_set_context = self.validator_context.sets_contexts.get(&validator_list_id).ok_or_else(|| { - error!("bad validator_list_id ({})!", validator_list_id.to_hex_string()) + error!( + "validator list context is not ready ({})", + validator_list_id.to_hex_string() + ) + })?; + let adnl_key = self + .network_context + .stack + .adnl + .key_by_id(validator_set_context.val().validator_adnl_key.id()) + .map_err(|e| { + error!( + "validator list context exists but ADNL key is not loaded \ + ({}, key={}): {}", + validator_list_id.to_hex_string(), + validator_set_context.val().validator_adnl_key.id(), + e + ) })?; - let adnl_key = - self.network_context.stack.adnl.key_by_tag(validator_set_context.val().election_id)?; if USE_CATCHAIN_ADNL_OVERLAY { let overlay = self.network_context.catchain_overlay_manager.start_overlay( @@ -957,14 +998,43 @@ impl NodeConfigSubscriber for NodeNetwork { match sender { ConfigEvent::AddValidatorAdnlKey(validator_adnl_key_id, election_id) => { self.load_and_store_validator_adnl_key(validator_adnl_key_id, election_id).await - } // Unused - // ConfigEvent::RemoveValidatorAdnlKey(validator_adnl_key_id, election_id) => { - // log::info!("config event (RemoveValidatorAdnlKey) id: {}.", &validator_adnl_key_id); - // self.network_context.adnl.delete_key(&validator_adnl_key_id, election_id as usize)?; - // let status = self.validator_context.actual_local_adnl_keys.remove(&validator_adnl_key_id).is_some(); - // log::info!("config event (RemoveValidatorAdnlKey) id: {} finished({}).", &validator_adnl_key_id, &status); - // return Ok(status); - // } + } + ConfigEvent::RemoveValidatorAdnlKey(validator_adnl_key_id, election_id) => { + log::info!( + "config event (RemoveValidatorAdnlKey) id: {} election: {}", + &validator_adnl_key_id, + election_id + ); + // Remove from ADNL peer table and key config + self.network_context + .stack + .adnl + .delete_key(&validator_adnl_key_id, election_id as usize)?; + // Remove from QUIC server cert resolver + if let Some(quic) = &self.network_context.stack.quic { + let adnl_ip = self.network_context.stack.adnl.ip_address_adnl(); + let quic_port = if let Some(addr) = self.network_context.quic_address { + addr.port() + } else { + adnl_ip.port().saturating_add(adnl::QuicNode::OFFSET_PORT) + }; + let bind_addr = SocketAddr::new(Ipv4Addr::UNSPECIFIED.into(), quic_port); + if let Err(e) = quic.remove_key(&validator_adnl_key_id, bind_addr) { + log::warn!( + "Cannot remove validator ADNL key {} from QUIC: {e}", + validator_adnl_key_id + ); + } + } + // Remove from adnl_key_ids — this causes periodic_store_ip_addr + // to exit its loop, stopping DHT announcements for this key. + let removed = self.validator_context.adnl_key_ids.remove(&election_id).is_some(); + log::info!( + "config event (RemoveValidatorAdnlKey) id: {} finished (removed={removed})", + &validator_adnl_key_id + ); + Ok(removed) + } } } } diff --git a/src/node/src/network/tests/test_node_network_validator_list.rs b/src/node/src/network/tests/test_node_network_validator_list.rs index 466d443..8cf7c8f 100644 --- a/src/node/src/network/tests/test_node_network_validator_list.rs +++ b/src/node/src/network/tests/test_node_network_validator_list.rs @@ -61,6 +61,26 @@ fn test_select_local_validator_candidate_matches_pubkey_when_adnl_missing() { assert!(adnl_missing); } +#[test] +fn test_select_local_validator_candidate_accepts_pubkey_adnl_fallback() { + let validator_key = make_test_key(); + let chain_adnl_key = make_test_key(); + let validator = make_validator_node(validator_key.clone(), chain_adnl_key); + let validator_key_ids = vec![validator_key.id().clone()]; + // C++ parity fallback: allow validator pubkey short-id to serve as ADNL identity. + let validator_adnl_key_ids = vec![validator_key.id().clone()]; + + let (local_validator, adnl_missing) = select_local_validator_candidate( + std::slice::from_ref(&validator), + &validator_key_ids, + &validator_adnl_key_ids, + ); + + let local_validator = local_validator.expect("pubkey membership should select validator"); + assert_eq!(local_validator.public_key.id(), validator_key.id()); + assert!(!adnl_missing); +} + #[test] fn test_select_local_validator_candidate_returns_none_without_pubkey_match() { let validator_key = make_test_key(); diff --git a/src/node/src/validator/consensus.rs b/src/node/src/validator/consensus.rs index fc5bf29..de28f0a 100644 --- a/src/node/src/validator/consensus.rs +++ b/src/node/src/validator/consensus.rs @@ -32,7 +32,7 @@ use std::{ sync::Arc, time::Duration, }; -use ton_block::ShardIdent; +use ton_block::{BlockIdExt, ShardIdent}; // ============================================================================= // Consensus Timing Constants (for accelerated consensus mode only) @@ -67,14 +67,14 @@ pub use consensus_common::{ serialize_tl_bare_object, serialize_tl_boxed_object, utils::{get_elapsed_time, get_hash, get_hash_from_block_payload}, AsyncRequest, AsyncRequestPtr, BlockCandidatePriority, BlockHash, BlockPayloadPtr, - BlockSignature, BlockSourceInfo, CollationParentHint, CommittedBlockProof, - CommittedBlockProofCallback, ConsensusCommonFactory, ConsensusNode, ConsensusOverlay, - ConsensusOverlayListener, ConsensusOverlayListenerPtr, ConsensusOverlayLogReplayListener, - ConsensusOverlayLogReplayListenerPtr, ConsensusOverlayManager, ConsensusOverlayManagerPtr, - ConsensusOverlayPtr, ConsensusReplayListener, ConsensusReplayListenerPtr, LogPlayer, - LogPlayerPtr, LogReplayOptions, OverlayTransportType, PrivateKey, PublicKey, PublicKeyHash, - RawBuffer, Result, Session, SessionId, SessionListener, SessionListenerPtr, SessionNode, - SessionPtr, SessionStats, ValidatorBlockCandidate, ValidatorBlockCandidateCallback, + BlockSignature, BlockSourceInfo, CollationParentHint, ConsensusCommonFactory, ConsensusNode, + ConsensusOverlay, ConsensusOverlayListener, ConsensusOverlayListenerPtr, + ConsensusOverlayLogReplayListener, ConsensusOverlayLogReplayListenerPtr, + ConsensusOverlayManager, ConsensusOverlayManagerPtr, ConsensusOverlayPtr, + ConsensusReplayListener, ConsensusReplayListenerPtr, LogPlayer, LogPlayerPtr, LogReplayOptions, + OverlayTransportType, PrivateKey, PublicKey, PublicKeyHash, RawBuffer, Result, Session, + SessionId, SessionListener, SessionListenerPtr, SessionNode, SessionPtr, SessionStats, + ValidatorBlockCandidate, ValidatorBlockCandidateCallback, ValidatorBlockCandidateDecisionCallback, ValidatorBlockCandidatePtr, ValidatorWeight, }; @@ -260,21 +260,18 @@ impl SessionHolder { } } - /// Notify session about masterchain finalization + /// Notify session about the current applied top for its shard. /// - /// For simplex shard sessions, this updates `last_mc_finalized_seqno` which is - /// used to decide if an empty block should be generated (finalization recovery). + /// For simplex sessions, this updates the session-local applied-top tracking used for + /// empty-block recovery and MC validation ordering. /// /// For catchain sessions, this is a no-op as they don't need MC finalization tracking. /// /// # Arguments - /// * `mc_block_seqno` - The seqno of the finalized masterchain block - pub fn notify_mc_finalized(&self, mc_block_seqno: u32) { + /// * `applied_top` - Current applied top for this session shard + pub fn notify_mc_finalized(&self, applied_top: BlockIdExt) { if let SessionInner::Simplex(s) = &self.inner { - s.notify_mc_finalized(mc_block_seqno); - } else { - // Catchain sessions don't need MC finalization notification - let _ = mc_block_seqno; // Suppress unused warning + s.notify_mc_finalized(applied_top); } } } diff --git a/src/node/src/validator/fabric.rs b/src/node/src/validator/fabric.rs index d5f00a5..e6a93c4 100644 --- a/src/node/src/validator/fabric.rs +++ b/src/node/src/validator/fabric.rs @@ -343,7 +343,6 @@ pub async fn run_collate_query( let labels = [("shard", shard.to_string())]; metrics::counter!("ton_node_collator_failures_total", &labels).increment(1); let test_bundles_config = &engine.test_bundles_config().collator; - let err_str = if test_bundles_config.is_enable() { err.to_string() } else { String::default() }; #[cfg(feature = "telemetry")] diff --git a/src/node/src/validator/tests/test_session_id.rs b/src/node/src/validator/tests/test_session_id.rs index 4e2883a..862b4e0 100644 --- a/src/node/src/validator/tests/test_session_id.rs +++ b/src/node/src/validator/tests/test_session_id.rs @@ -11,12 +11,16 @@ use super::*; use crate::validator::{log_parser::LogParser, validator_utils::GeneralSessionInfo}; use std::{ + collections::{HashMap, HashSet}, fs, io::{self, BufRead}, sync::Arc, time::Duration, }; -use ton_block::{signature::SigPubKey, validators::ValidatorDescr, Ed25519KeyOption}; +use ton_block::{ + signature::SigPubKey, validators::ValidatorDescr, Ed25519KeyOption, FutureSplitMerge, + McStateExtra, ShardDescr, ShardIdent, +}; fn parse_shard_ident(parser: &LogParser, name: &str) -> ShardIdent { ShardIdent::with_tagged_prefix( @@ -328,8 +332,8 @@ fn test_validator_list_status_get_local_key_for_list() { let list_curr = UInt256::from_slice(&[1u8; 32]); let list_next = UInt256::from_slice(&[2u8; 32]); - status.add_list(list_curr.clone(), vec![key_a.clone()], true); - status.add_list(list_next.clone(), vec![key_b.clone()], true); + status.add_list(list_curr.clone(), vec![key_a.clone()]); + status.add_list(list_next.clone(), vec![key_b.clone()]); status.curr = Some(list_curr.clone()); status.next = Some(list_next.clone()); @@ -355,7 +359,7 @@ fn test_validator_list_status_get_local_key_curr_none() { let key = make_test_key(); let list_next = UInt256::from_slice(&[2u8; 32]); - status.add_list(list_next.clone(), vec![key.clone()], true); + status.add_list(list_next.clone(), vec![key.clone()]); status.next = Some(list_next.clone()); // curr is None @@ -375,8 +379,8 @@ fn test_validator_list_status_actual_or_coming() { let list_next = UInt256::from_slice(&[2u8; 32]); let list_old = UInt256::from_slice(&[3u8; 32]); - status.add_list(list_curr.clone(), vec![key.clone()], true); - status.add_list(list_next.clone(), vec![key.clone()], true); + status.add_list(list_curr.clone(), vec![key.clone()]); + status.add_list(list_next.clone(), vec![key.clone()]); status.curr = Some(list_curr.clone()); status.next = Some(list_next.clone()); @@ -386,56 +390,155 @@ fn test_validator_list_status_actual_or_coming() { } #[test] -fn test_validator_list_status_network_readiness() { +fn test_validator_list_status_add_list_replaces_existing_entry() { let mut status = ValidatorListStatus::default(); - let key = make_test_key(); + let key_a = make_test_key(); + let key_b = make_test_key(); let list_id = UInt256::from_slice(&[9u8; 32]); - status.add_list(list_id.clone(), vec![key.clone()], false); - assert!(!status.is_list_network_ready(&list_id)); - assert_eq!(status.get_local_keys_for_list(&list_id).unwrap()[0].id(), key.id()); + status.add_list(list_id.clone(), vec![key_a.clone()]); + assert_eq!(status.get_local_keys_for_list(&list_id).unwrap()[0].id(), key_a.id()); - status.add_list(list_id.clone(), vec![key], true); - assert!(status.is_list_network_ready(&list_id)); + status.add_list(list_id.clone(), vec![key_b.clone()]); + assert_eq!(status.get_local_keys_for_list(&list_id).unwrap()[0].id(), key_b.id()); } #[test] -fn test_validator_list_status_ready_current_list_requires_network_readiness() { +fn test_validator_list_status_next_only_membership_remains_usable_for_future_sessions() { let mut status = ValidatorListStatus::default(); let key = make_test_key(); - let list_id = UInt256::from_slice(&[7u8; 32]); + let next_list = UInt256::from_slice(&[10u8; 32]); + let validators = vec![make_validator_descr_from_key(&key)]; - status.add_list(list_id.clone(), vec![key.clone()], false); - status.curr = Some(list_id.clone()); - assert!(status.get_ready_current_list().is_none()); + status.add_list(next_list.clone(), vec![key.clone()]); + status.next = Some(next_list.clone()); - status.add_list(list_id.clone(), vec![key], true); - assert_eq!(status.get_ready_current_list(), Some(&list_id)); + let selected = + find_local_validator_key(&validators, status.get_local_keys_for_list(&next_list)) + .expect("next-list pubkey membership should stay usable for future sessions"); + assert_eq!(selected.id(), key.id()); } #[test] -fn test_validator_list_status_ready_current_list_ignores_next_only_membership() { +fn test_curr_membership_remains_usable_without_context_gate() { let mut status = ValidatorListStatus::default(); let key = make_test_key(); - let next_list = UInt256::from_slice(&[8u8; 32]); + let curr_list = UInt256::from_slice(&[11u8; 32]); + let validators = vec![make_validator_descr_from_key(&key)]; - status.add_list(next_list.clone(), vec![key], true); - status.next = Some(next_list); + status.add_list(curr_list.clone(), vec![key.clone()]); + status.curr = Some(curr_list); - assert!(status.get_ready_current_list().is_none()); + let selected = find_local_validator_key(&validators, status.get_local_keys()) + .expect("curr-list pubkey membership should stay usable without context gate"); + assert_eq!(selected.id(), key.id()); } #[test] -fn test_validator_list_status_next_only_ready_list_remains_usable_for_future_sessions() { +fn test_next_membership_remains_usable_without_context_gate() { let mut status = ValidatorListStatus::default(); let key = make_test_key(); - let next_list = UInt256::from_slice(&[10u8; 32]); + let next_list = UInt256::from_slice(&[12u8; 32]); + let validators = vec![make_validator_descr_from_key(&key)]; - status.add_list(next_list.clone(), vec![key], true); + status.add_list(next_list.clone(), vec![key.clone()]); status.next = Some(next_list.clone()); - assert!(status.get_ready_current_list().is_none()); - assert!(status.is_list_network_ready(&next_list)); + let selected = + find_local_validator_key(&validators, status.get_local_keys_for_list(&next_list)) + .expect("next-list pubkey membership should stay usable without context gate"); + assert_eq!(selected.id(), key.id()); +} + +#[test] +fn test_select_existing_session_prefers_current_over_future() { + let session_id = UInt256::from_slice(&[13u8; 32]); + let current_sessions = HashMap::from([(session_id.clone(), "current")]); + let mut future_sessions = HashMap::from([(session_id.clone(), "future")]); + + match select_existing_session_for_current_map( + &session_id, + ¤t_sessions, + &mut future_sessions, + ) { + Some(ExistingSessionSource::Current(selected)) => assert_eq!(selected, "current"), + _ => panic!("expected current session to be selected first"), + } + + assert!( + future_sessions.contains_key(&session_id), + "future session should remain untouched when current is selected" + ); +} + +#[test] +fn test_select_existing_session_promotes_future_when_current_absent() { + let session_id = UInt256::from_slice(&[14u8; 32]); + let current_sessions: HashMap = HashMap::new(); + let mut future_sessions = HashMap::from([(session_id.clone(), "future")]); + + match select_existing_session_for_current_map( + &session_id, + ¤t_sessions, + &mut future_sessions, + ) { + Some(ExistingSessionSource::Future(selected)) => assert_eq!(selected, "future"), + _ => panic!("expected future session promotion when current is absent"), + } + + assert!( + !future_sessions.contains_key(&session_id), + "promoted future session must be removed from future map" + ); +} + +#[test] +fn test_current_map_swap_model_leaves_unselected_old_session_for_destroy() { + let keep_current_id = UInt256::from_slice(&[15u8; 32]); + let drop_current_id = UInt256::from_slice(&[16u8; 32]); + let promote_future_id = UInt256::from_slice(&[17u8; 32]); + + let current_sessions = HashMap::from([ + (keep_current_id.clone(), "current-keep"), + (drop_current_id.clone(), "current-drop"), + ]); + let mut future_sessions = HashMap::from([(promote_future_id.clone(), "future-promote")]); + let mut gc_sessions: HashSet = + current_sessions.keys().chain(future_sessions.keys()).cloned().collect(); + let mut new_current_sessions = HashMap::new(); + + for selected_id in [keep_current_id.clone(), promote_future_id.clone()] { + let selected = match select_existing_session_for_current_map( + &selected_id, + ¤t_sessions, + &mut future_sessions, + ) { + Some(ExistingSessionSource::Current(value)) => value, + Some(ExistingSessionSource::Future(value)) => value, + None => panic!("selected session should exist in either current or future maps"), + }; + new_current_sessions.insert(selected_id.clone(), selected); + gc_sessions.remove(&selected_id); + } + + assert_eq!( + new_current_sessions.get(&keep_current_id), + Some(&"current-keep"), + "kept current session should stay in rebuilt current map" + ); + assert_eq!( + new_current_sessions.get(&promote_future_id), + Some(&"future-promote"), + "future session should be promoted into rebuilt current map" + ); + assert!( + !new_current_sessions.contains_key(&drop_current_id), + "unselected old current session should not remain in rebuilt current map" + ); + assert!( + gc_sessions.contains(&drop_current_id), + "unselected old current session must stay in GC set for destruction" + ); } #[test] @@ -582,3 +685,69 @@ fn test_get_session_unsafe_id_skips_patch_when_flag_false() { get_session_unsafe_id(session_info, &[val], true, true, Some(100), &config, false); assert_ne!(id_with_flag_true, plain_id, "flag=true must apply the unsafe rotation patch"); } + +#[test] +fn test_simplex_empty_block_lag_threshold_matches_cpp_policy() { + assert_eq!(simplex_empty_block_lag_threshold(&ShardIdent::masterchain()), None); + + let shard = ShardIdent::with_tagged_prefix(0, 0x8000_0000_0000_0000).unwrap(); + assert_eq!(simplex_empty_block_lag_threshold(&shard), Some(8)); +} + +#[test] +fn test_runtime_simplex_options_use_temporary_strict_collation_mode() { + let catchain_options = CatchainSessionOptions { + proto_version: 4, + max_block_size: 1024, + max_collated_data_size: 2048, + ..Default::default() + }; + + let mc_opts = build_runtime_simplex_session_options( + &ShardIdent::masterchain(), + &SimplexConfig::default(), + &catchain_options, + ); + assert!( + mc_opts.require_notarized_parent_for_collation, + "runtime manager-built simplex sessions must currently stay in strict mode" + ); + assert_eq!(mc_opts.empty_block_mc_lag_threshold, None); + + let shard = ShardIdent::with_tagged_prefix(0, 0x8000_0000_0000_0000).unwrap(); + let shard_opts = + build_runtime_simplex_session_options(&shard, &SimplexConfig::default(), &catchain_options); + assert!( + shard_opts.require_notarized_parent_for_collation, + "runtime shard sessions must currently stay in strict mode" + ); + assert_eq!(shard_opts.empty_block_mc_lag_threshold, Some(8)); + + let default_opts = simplex::SessionOptions::default(); + assert!( + default_opts.require_notarized_parent_for_collation, + "default session options must remain in strict mode for tests/manual sessions" + ); +} + +#[test] +fn test_mc_registered_top_for_shard_returns_session_specific_block_id() { + let mut mc_state_extra = McStateExtra::default(); + let root_hash = UInt256::rand(); + let shard_descr = + ShardDescr::with_params(321, 100, 200, root_hash.clone(), FutureSplitMerge::None); + let shard = mc_state_extra.add_workchain(0, &shard_descr).unwrap(); + + let block = mc_registered_top_for_shard(&mc_state_extra, &shard) + .unwrap() + .expect("registered top block should be present for known shard"); + assert_eq!(block.shard(), &shard); + assert_eq!(block.seq_no, 321); + assert_eq!(block.root_hash(), &root_hash); + + let unknown_shard = ShardIdent::masterchain(); + assert!( + mc_registered_top_for_shard(&mc_state_extra, &unknown_shard).unwrap().is_none(), + "unknown shard must not produce MC-registered top block id" + ); +} diff --git a/src/node/src/validator/tests/test_validator_session_listener.rs b/src/node/src/validator/tests/test_validator_session_listener.rs new file mode 100644 index 0000000..71336e1 --- /dev/null +++ b/src/node/src/validator/tests/test_validator_session_listener.rs @@ -0,0 +1,78 @@ +use super::*; +use ton_block::{BlockSignatures, BlockSignaturesVariant, Ed25519KeyOption, UInt256}; + +#[test] +fn test_on_applied_top_action_preserves_queue_order() { + let (listener, mut receiver) = + ValidatorSessionListener::create(UInt256::default(), ShardIdent::masterchain()); + let applied_top = + BlockIdExt::with_params(ShardIdent::masterchain(), 7, UInt256::rand(), UInt256::rand()); + + listener + .queue_sender() + .send(ValidationAction::OnAppliedTop { applied_top: applied_top.clone() }) + .expect("send applied-top"); + listener + .queue_sender() + .send(ValidationAction::OnBlockSkipped { round: 11 }) + .expect("send skip"); + + match receiver.try_recv().expect("first queued action") { + ValidationAction::OnAppliedTop { applied_top: queued } => { + assert_eq!(queued, applied_top); + } + other => panic!("unexpected first action: {}", other), + } + + match receiver.try_recv().expect("second queued action") { + ValidationAction::OnBlockSkipped { round } => assert_eq!(round, 11), + other => panic!("unexpected second action: {}", other), + } +} + +#[test] +fn test_on_block_finalized_enqueues_explicit_block_identity() { + let (listener, mut receiver) = + ValidatorSessionListener::create(UInt256::default(), ShardIdent::masterchain()); + + let block_id = + BlockIdExt::with_params(ShardIdent::masterchain(), 42, UInt256::rand(), UInt256::rand()); + let source = Ed25519KeyOption::generate().expect("generate key"); + let source_info = BlockSourceInfo { + source, + priority: consensus_common::BlockCandidatePriority { + round: 10, + priority: 0, + first_block_round: 10, + }, + }; + let root_hash = UInt256::rand(); + let file_hash = UInt256::rand(); + let data = consensus_common::ConsensusCommonFactory::create_block_payload(vec![1, 2, 3, 4]); + let signatures = BlockSignaturesVariant::Ordinary(BlockSignatures::default()); + + listener.on_block_finalized( + block_id.clone(), + source_info, + root_hash.clone(), + file_hash.clone(), + data.clone(), + signatures, + Vec::new(), + ); + + match receiver.try_recv().expect("queued finalized action") { + ValidationAction::OnBlockFinalized(finalized) => { + assert_eq!(finalized.block_id, block_id); + assert_eq!(finalized.source_info.priority.round, 10); + assert_eq!(finalized.root_hash, root_hash); + assert_eq!(finalized.file_hash, file_hash); + assert_eq!(finalized.data.data(), data.data()); + match finalized.signatures { + BlockSignaturesVariant::Ordinary(_) => {} + other => panic!("unexpected signatures variant in queued action: {:?}", other), + } + } + other => panic!("unexpected queued action: {}", other), + } +} diff --git a/src/node/src/validator/validator_group.rs b/src/node/src/validator/validator_group.rs index 5b507be..74f1609 100644 --- a/src/node/src/validator/validator_group.rs +++ b/src/node/src/validator/validator_group.rs @@ -12,11 +12,11 @@ use super::{ consensus::{ - get_hash, BlockHash, BlockPayloadPtr, CollationParentHint, CommittedBlockProof, - CommittedBlockProofCallback, ConsensusOptions, ConsensusOverlayManagerPtr, ConsensusType, - PrivateKey, PublicKey, PublicKeyHash, Session, SessionHolderPtr, SessionId, - SessionListener, SessionListenerPtr, SessionNode, ValidatorBlockCandidate, - ValidatorBlockCandidateCallback, ValidatorBlockCandidateDecisionCallback, + get_hash, BlockHash, BlockPayloadPtr, CollationParentHint, ConsensusOptions, + ConsensusOverlayManagerPtr, ConsensusType, PrivateKey, PublicKey, PublicKeyHash, Session, + SessionHolderPtr, SessionId, SessionListener, SessionListenerPtr, SessionNode, + ValidatorBlockCandidate, ValidatorBlockCandidateCallback, + ValidatorBlockCandidateDecisionCallback, }, fabric::*, validator_utils::{GeneralSessionInfo, PrevBlockHistory}, @@ -130,6 +130,16 @@ fn should_reject_stale_mc_candidate( } } +fn sync_last_accepted_mc_seqno_from_applied_top( + group_impl: &mut ValidatorGroupImpl, + applied_top: &BlockIdExt, +) { + if group_impl.shard.is_masterchain() { + let prev = group_impl.last_accepted_mc_seqno.unwrap_or(0); + group_impl.last_accepted_mc_seqno = Some(prev.max(applied_top.seq_no)); + } +} + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy)] pub enum ValidatorGroupStatus { Created, @@ -649,6 +659,7 @@ pub struct ValidatorGroup { allow_unsafe_self_blocks_resync: bool, group_impl: Arc>, + action_queue: tokio::sync::mpsc::UnboundedSender, callback: Arc, receiver: Arc>>>, @@ -689,6 +700,7 @@ impl ValidatorGroup { session_id.clone(), general_session_info.shard.clone(), ); + let action_queue = listener.queue_sender(); log::trace!(target: "validator", "Creating validator group: {}, consensus_type: {}", id, consensus_type); ValidatorGroup { @@ -703,6 +715,7 @@ impl ValidatorGroup { engine, allow_unsafe_self_blocks_resync, group_impl: Arc::new(MutexWrapper::new(group_impl, id)), + action_queue, callback: Arc::new(listener), receiver: Arc::new(Mutex::new(Some(receiver))), last_validation_time: Arc::new(AtomicU64::new(0)), @@ -751,27 +764,34 @@ impl ValidatorGroup { .await } - /// Notify this session about masterchain finalization. + /// Enqueue an applied-top update into this group's ordered action queue. /// - /// For simplex shard sessions, this updates the MC finalization tracking which is - /// used for empty block generation (finalization recovery). + /// For simplex sessions, the queued action eventually updates the session processor: + /// - shard sessions use it for empty-block recovery against MC-registered tops + /// - masterchain sessions use it to mirror the applied MC head /// - /// This should be called by ValidatorManager when a masterchain block is finalized, - /// for all shard validator groups. + /// The manager should not await this on the hot path; queue processing preserves + /// per-group sequencing with other validator-group actions. /// /// # Arguments - /// * `mc_block_seqno` - The seqno of the finalized masterchain block - pub async fn notify_mc_finalized(&self, mc_block_seqno: u32) { - // Only shard sessions need MC finalization notification - if self.shard().is_masterchain() { - return; + /// * `applied_top` - Current applied top for this group shard + pub fn notify_mc_finalized(&self, applied_top: BlockIdExt) { + if let Err(error) = self.action_queue.send(ValidationAction::OnAppliedTop { applied_top }) { + log::warn!( + target: "validator", + "Failed to enqueue applied-top notification for {}: {}", + self.shard, + error + ); } + } - //TODO: lock optimization is required + pub async fn on_applied_top(&self, applied_top: BlockIdExt) { self.group_impl .execute_sync(|group_impl| { + sync_last_accepted_mc_seqno_from_applied_top(group_impl, &applied_top); if let Some(ref session) = group_impl.session { - session.notify_mc_finalized(mc_block_seqno); + session.notify_mc_finalized(applied_top); } }) .await; @@ -1661,6 +1681,15 @@ impl ValidatorGroup { signatures: BlockSignaturesVariant, approve_sig_set: Vec<(PublicKeyHash, BlockPayloadPtr)>, ) { + let is_simplex_group = self + .group_impl + .execute_sync(|group_impl| group_impl.consensus_type == ConsensusType::Simplex) + .await; + assert!( + !is_simplex_group, + "ValidatorGroup::on_block_committed must not be called for simplex sessions" + ); + let next_block_descr = self.get_next_block_descr(Some(&root_hash)).await; let data_vec = data.data().to_vec(); @@ -1755,12 +1784,9 @@ impl ValidatorGroup { err => err, // TODO: retry block commit }; - let committed_seqno = next_block_id.seq_no; - group_impl.prev_block_ids.update_prev(vec![next_block_id]); - - if group_impl.shard.is_masterchain() { - let prev = group_impl.last_accepted_mc_seqno.unwrap_or(0); - group_impl.last_accepted_mc_seqno = Some(prev.max(committed_seqno)); + if full_result.is_ok() { + sync_last_accepted_mc_seqno_from_applied_top(group_impl, &next_block_id); + group_impl.prev_block_ids.update_prev(vec![next_block_id]); } (full_result, group_impl.prev_block_ids.display_prevs()) @@ -1789,6 +1815,184 @@ impl ValidatorGroup { } } + /// Out-of-order finalized block delivery. + /// + /// Called immediately when a finalization certificate is observed, + /// regardless of whether predecessors have been committed. + /// `block_id` carries the full identity (shard, seqno, root_hash, file_hash) + /// so we don't rely on sequential `prev_block_ids` tracking. + /// + /// The engine's `apply_block` is dependency-driven and will recursively + /// fetch predecessors, so out-of-order acceptance is safe at that layer. + #[allow(clippy::too_many_arguments)] + pub async fn on_block_finalized( + &self, + block_id: BlockIdExt, + round: u32, + source: PublicKey, + _root_hash: BlockHash, + _file_hash: BlockHash, + data: BlockPayloadPtr, + signatures: BlockSignaturesVariant, + approve_sig_set: Vec<(PublicKeyHash, BlockPayloadPtr)>, + ) { + let is_simplex_group = self + .group_impl + .execute_sync(|group_impl| group_impl.consensus_type == ConsensusType::Simplex) + .await; + if !is_simplex_group { + log::error!( + target: "validator", + "ValidatorGroup::on_block_finalized: unexpected callback for non-simplex session; \ + ignoring (block_id={block_id}, source={}, round={round})", + source.id() + ); + return; + } + + // Important: do not block ValidationAction queue on out-of-order acceptance. + // This path can involve downloads/apply and must run in detached task. + let engine = self.engine.clone(); + let validator_set = self.validator_set.clone(); + let group_impl = self.group_impl.clone(); + let local_key = self.local_key.clone(); + let source_id = source.id().clone(); + let data_vec = data.data().to_vec(); + let data_opt = if data_vec.is_empty() { None } else { Some(data_vec) }; + let we_generated = source.id() == local_key.id(); + metrics::counter!("ton_node_validator_finalized_received_total", "consensus" => "simplex") + .increment(1); + + // Activate session on first finalized-block receipt because simplex is + // finalized-driven and does not use on_block_committed callbacks. + self.group_impl + .execute_sync(|group_impl| { + if group_impl.status == ValidatorGroupStatus::Sync { + group_impl.status = ValidatorGroupStatus::Active; + let cl = if group_impl.consensus_type == ConsensusType::Simplex { + "simplex" + } else { + "catchain" + }; + metrics::counter!( + "ton_node_validator_session_activated_total", + "consensus" => cl + ) + .increment(1); + log::info!( + target: "validator", + "SESSION_LIFECYCLE: transition shard={} cc_seqno={} \ + session_id={:x} sync -> active \ + (first finalized block received)", + group_impl.shard, + group_impl.cc_seqno, + group_impl.session_id + ); + } + }) + .await; + + log::info!( + target: "validator", + "ValidatorGroup::on_block_finalized: scheduling async accept for \ + block_id={block_id} source={source_id} round={round}" + ); + + tokio::spawn(async move { + let (accept_data, prevs) = + match Self::resolve_prev_for_finalized_block(engine.clone(), &block_id, data_opt) + .await + { + Ok(v) => v, + Err(e) => { + log::error!( + target: "validator", + "ValidatorGroup::on_block_finalized: + failed to resolve prev for {block_id}: {e}" + ); + return; + } + }; + + let result = run_accept_block_query( + block_id.clone(), + accept_data, + prevs, + validator_set, + signatures, + approve_sig_set, + we_generated, + engine, + ) + .await; + + match result { + Ok(()) => { + log::info!( + target: "validator", + "ValidatorGroup::on_block_finalized: \ + accepted block_id={block_id} source={source_id} round={round}" + ); + + group_impl + .execute_sync(|group_impl| { + sync_last_accepted_mc_seqno_from_applied_top(group_impl, &block_id); + }) + .await; + } + Err(err) => { + log::error!( + target: "validator", + "ValidatorGroup::on_block_finalized: accept failed for \ + block_id={block_id} source={source_id} round={round}: {err}" + ); + } + } + }); + } + + fn extract_prev_ids_from_block_data( + block_id: &BlockIdExt, + data: Vec, + ) -> Result> { + let block = crate::block::BlockStuff::deserialize_block(block_id.clone(), Arc::new(data))?; + Self::extract_prev_ids_from_block(&block) + } + + fn extract_prev_ids_from_block(block: &crate::block::BlockStuff) -> Result> { + let (prev1, prev2) = block.construct_prev_id()?; + let mut prev = Vec::with_capacity(if prev2.is_some() { 2 } else { 1 }); + prev.push(prev1); + if let Some(prev2) = prev2 { + prev.push(prev2); + } + Ok(prev) + } + + async fn resolve_prev_for_finalized_block( + engine: Arc, + block_id: &BlockIdExt, + data_opt: Option>, + ) -> Result<(Option>, Vec)> { + if let Some(data) = data_opt { + match Self::extract_prev_ids_from_block_data(block_id, data.clone()) { + Ok(prev) => return Ok((Some(data), prev)), + Err(e) => { + log::warn!( + target: "validator", + "ValidatorGroup::resolve_prev_for_finalized_block: \ + failed to parse payload for {block_id}: {e}. Falling back to download_block" + ); + } + } + } + + let (downloaded_block, _proof) = engine.download_block(block_id, Some(10)).await?; + let prev = Self::extract_prev_ids_from_block(&downloaded_block)?; + let downloaded_data = downloaded_block.data().to_vec(); + Ok((Some(downloaded_data), prev)) + } + pub async fn on_block_skipped(&self, round: u32) { log::info!( target: "validator", @@ -1836,60 +2040,6 @@ impl ValidatorGroup { ); callback(result); } - - /// Download committed block proof from full-node. - /// - /// Spawns an async task (non-blocking) that downloads the block proof via - /// EngineOperations, extracts BlockSignaturesVariant, and invokes the callback. - /// The spawned task does NOT hold up the ValidationAction queue. - pub async fn on_get_committed_candidate( - &self, - block_id: BlockIdExt, - callback: CommittedBlockProofCallback, - ) { - log::info!( - target: "validator", - "ValidatorGroup::on_get_committed_candidate: block_id={}, {}", - block_id, self.info().await - ); - - let engine = self.engine.clone(); - let block_id_clone = block_id.clone(); - tokio::spawn(async move { - let result = Self::fetch_committed_block_proof(engine.as_ref(), &block_id_clone).await; - let result_txt = match &result { - Ok(_) => "Ok".to_string(), - Err(e) => format!("Err: {}", e), - }; - log::info!( - target: "validator", - "ValidatorGroup::on_get_committed_candidate: result={} for {}", - result_txt, block_id_clone, - ); - callback(result); - }); - } - - async fn fetch_committed_block_proof( - engine: &dyn crate::engine_traits::EngineOperations, - block_id: &BlockIdExt, - ) -> Result { - let is_link = !block_id.shard().is_masterchain(); - let proof = engine.download_block_proof(block_id, is_link, false).await?; - - let signatures = proof.drain_signatures()?; - - if block_id.shard().is_masterchain() { - match &signatures { - BlockSignaturesVariant::Simplex(s) if s.is_final => { /* ok */ } - _ => { - fail!("Expected Simplex(is_final=true) for MC block {}", block_id); - } - } - } - - Ok(CommittedBlockProof { block_id: block_id.clone(), signatures }) - } } impl Drop for ValidatorGroup { @@ -1941,6 +2091,38 @@ mod tests { assert!(should_reject_stale_mc_candidate(Some(100), 50)); } + #[test] + fn test_sync_last_accepted_mc_seqno_from_applied_top_is_monotonic() { + let mut group = make_group_impl_for_start_tests(); + + let top10 = BlockIdExt::with_params( + ShardIdent::masterchain(), + 10, + UInt256::from([0x11; 32]), + UInt256::from([0x12; 32]), + ); + sync_last_accepted_mc_seqno_from_applied_top(&mut group, &top10); + assert_eq!(group.last_accepted_mc_seqno, Some(10)); + + let top7 = BlockIdExt::with_params( + ShardIdent::masterchain(), + 7, + UInt256::from([0x21; 32]), + UInt256::from([0x22; 32]), + ); + sync_last_accepted_mc_seqno_from_applied_top(&mut group, &top7); + assert_eq!(group.last_accepted_mc_seqno, Some(10)); + + let top11 = BlockIdExt::with_params( + ShardIdent::masterchain(), + 11, + UInt256::from([0x31; 32]), + UInt256::from([0x32; 32]), + ); + sync_last_accepted_mc_seqno_from_applied_top(&mut group, &top11); + assert_eq!(group.last_accepted_mc_seqno, Some(11)); + } + #[test] fn test_prepare_start_immediate_keeps_created_and_marks_pending() { let mut group = make_group_impl_for_start_tests(); diff --git a/src/node/src/validator/validator_manager.rs b/src/node/src/validator/validator_manager.rs index ca61cfe..72800c3 100644 --- a/src/node/src/validator/validator_manager.rs +++ b/src/node/src/validator/validator_manager.rs @@ -10,7 +10,7 @@ */ use super::consensus::{ serialize_tl_boxed_object, CatchainSessionOptions, ConsensusNode, ConsensusOptions, PublicKey, - RawBuffer, + RawBuffer, SimplexSessionOptions, }; use crate::{ config::ValidatorManagerConfig, @@ -50,6 +50,10 @@ const MC_ACCELERATED_CONSENSUS_ENABLED: bool = true; #[cfg(not(feature = "xp25"))] const MC_ACCELERATED_CONSENSUS_ENABLED: bool = false; +// TODO(simplex-mc-parity): flip this back to `false` once runtime collation is ready to +// use the C++-compatible optimistic whole-window pipelining mode in production. +const SIMPLEX_RUNTIME_REQUIRE_NOTARIZED_PARENT_FOR_COLLATION: bool = true; + fn format_shard_short(shard: &ShardIdent) -> String { if shard.is_masterchain() { "MC".to_string() @@ -79,6 +83,76 @@ fn format_duration_short(d: Duration) -> String { } } +const SHARD_EMPTY_BLOCK_MC_LAG_THRESHOLD: u32 = 8; + +fn simplex_empty_block_lag_threshold(shard: &ShardIdent) -> Option { + if shard.is_masterchain() { + None + } else { + Some(SHARD_EMPTY_BLOCK_MC_LAG_THRESHOLD) + } +} + +fn applied_top_for_session_shard( + mc_state: &ShardStateStuff, + mc_state_extra: &McStateExtra, + shard: &ShardIdent, +) -> Result> { + if shard.is_masterchain() { + Ok(Some(mc_state.block_id().clone())) + } else { + mc_registered_top_for_shard(mc_state_extra, shard) + } +} + +fn mc_registered_top_for_shard( + mc_state_extra: &McStateExtra, + shard: &ShardIdent, +) -> Result> { + mc_state_extra.shards().find_shard(shard).map(|record| record.map(|r| r.block_id().clone())) +} + +fn build_runtime_simplex_session_options( + shard: &ShardIdent, + cfg: &SimplexConfig, + catchain_options: &CatchainSessionOptions, +) -> SimplexSessionOptions { + let np = &cfg.noncritical_params; + SimplexSessionOptions { + proto_version: catchain_options.proto_version as u32, + slots_per_leader_window: cfg.slots_per_leader_window, + target_rate: Duration::from_millis(np.target_rate_ms as u64), + first_block_timeout: Duration::from_millis(np.first_block_timeout_ms as u64), + first_block_timeout_multiplier: f32::from_bits(np.first_block_timeout_multiplier_bits) + as f64, + first_block_timeout_cap: Duration::from_millis(np.first_block_timeout_cap_ms as u64), + candidate_resolve_timeout: Duration::from_millis(np.candidate_resolve_timeout_ms as u64), + candidate_resolve_timeout_multiplier: f32::from_bits( + np.candidate_resolve_timeout_multiplier_bits, + ) as f64, + candidate_resolve_timeout_cap: Duration::from_millis( + np.candidate_resolve_timeout_cap_ms as u64, + ), + candidate_resolve_cooldown: Duration::from_millis(np.candidate_resolve_cooldown_ms as u64), + standstill_timeout: Duration::from_millis(np.standstill_timeout_ms as u64), + standstill_max_egress_bytes_per_s: np.standstill_max_egress_bytes_per_s, + max_leader_window_desync: np.max_leader_window_desync, + bad_signature_ban_duration: Duration::from_millis(np.bad_signature_ban_duration_ms as u64), + candidate_resolve_rate_limit: np.candidate_resolve_rate_limit, + // TODO(simplex-mc-parity): temporary strict runtime mode until optimistic + // whole-window pipelining is enabled for production sessions. + require_notarized_parent_for_collation: + SIMPLEX_RUNTIME_REQUIRE_NOTARIZED_PARENT_FOR_COLLATION, + max_block_size: catchain_options.max_block_size as usize, + max_collated_data_size: catchain_options.max_collated_data_size as usize, + use_quic: cfg.use_quic, + // C++ parity: shard sessions use lag threshold 8 for empty-block recovery. + // MC sessions use internal consensus-finalized tracking and keep this unset. + empty_block_mc_lag_threshold: simplex_empty_block_lag_threshold(shard), + ..Default::default() + } +} + fn validation_state_phase_label(status: ValidatorGroupStatus) -> &'static str { match status { ValidatorGroupStatus::Created | ValidatorGroupStatus::EngineCreated => "pre-start", @@ -226,6 +300,23 @@ fn find_local_validator_key( None } +enum ExistingSessionSource { + Current(T), + Future(T), +} + +fn select_existing_session_for_current_map( + session_id: &UInt256, + current_sessions: &HashMap, + future_sessions: &mut HashMap, +) -> Option> { + if let Some(existing) = current_sessions.get(session_id) { + return Some(ExistingSessionSource::Current(existing.clone())); + } + + future_sessions.remove(session_id).map(ExistingSessionSource::Future) +} + /// Computes session_id and if unsafe rotation is taking place, /// replaces session_id with unsafe rotation session id. /// The `do_unsafe_catchain_rotate` flag mirrors C++ `force_recover`: @@ -527,16 +618,13 @@ impl ValidationStatus { /// Local node's participation record for a single validator list. /// -/// Pairs the node's local validator keys with a `network_ready` flag indicating whether the -/// ADNL/overlay infrastructure was successfully set up for this list. Keys are stored in the -/// same local-key order that C++ uses for `temp_keys_`, so per-shard selection can still pick -/// the first matching local key within a subset. +/// Stores local validator keys in the same local-key order that C++ uses for `temp_keys_`, +/// so per-shard selection can still pick the first matching local key within a subset. struct LocalValidatorListEntry { keys: Vec, - network_ready: bool, } -/// Tracks which validator lists the local node belongs to and their readiness state. +/// Tracks which validator lists the local node belongs to. /// /// Maintains the current and next validator list IDs (mirroring the masterchain state's /// current and next validator sets) along with the local node's keys for each. @@ -553,8 +641,8 @@ struct ValidatorListStatus { } impl ValidatorListStatus { - fn add_list(&mut self, list_id: ValidatorListHash, keys: Vec, network_ready: bool) { - self.known_lists.insert(list_id, LocalValidatorListEntry { keys, network_ready }); + fn add_list(&mut self, list_id: ValidatorListHash, keys: Vec) { + self.known_lists.insert(list_id, LocalValidatorListEntry { keys }); } fn contains_list(&self, list_id: &ValidatorListHash) -> bool { @@ -577,14 +665,6 @@ impl ValidatorListStatus { self.curr.as_ref().and_then(|current_list| self.get_local_keys_for_list(current_list)) } - fn get_ready_current_list(&self) -> Option<&ValidatorListHash> { - self.curr.as_ref().filter(|current_list| self.is_list_network_ready(current_list)) - } - - fn is_list_network_ready(&self, list_id: &ValidatorListHash) -> bool { - self.known_lists.get(list_id).map(|entry| entry.network_ready).unwrap_or(false) - } - fn actual_or_coming(&self, list_id: &ValidatorListHash) -> bool { match &self.curr { Some(curr_id) if list_id == curr_id => return true, @@ -705,9 +785,8 @@ impl ValidatorManagerImpl { /// Register the local node in a validator list and return its hash if matched. /// /// Calls [`EngineOperations::set_validator_list`] which checks local keys against the - /// validator set and sets up ADNL/overlay infrastructure. The result is cached in - /// [`ValidatorListStatus`]: even when `network_ready` is `false`, the list hash is - /// returned so that the caller can track membership without ADNL being fully operational. + /// validator set and attempts to set up ADNL/overlay infrastructure. + /// Membership is cached in [`ValidatorListStatus`] independently from transport readiness. /// /// Returns `Ok(None)` only when the local node is genuinely not in the validator set. async fn update_single_validator_list( @@ -720,7 +799,7 @@ impl ValidatorManagerImpl { Some(l) => l, }; if self.validator_list_status.contains_list(&list_id) - && self.validator_list_status.is_list_network_ready(&list_id) + && self.engine.validator_network().has_validator_list_context(&list_id) { return Ok(Some(list_id)); } @@ -740,13 +819,11 @@ impl ValidatorManagerImpl { } match self.engine.set_validator_list(list_id.clone(), &nodes_res).await? { - ValidatorListOutcome::Selected { key, matching_keys, network_ready } => { - self.validator_list_status.add_list( - list_id.clone(), - matching_keys.clone(), - network_ready, - ); - if network_ready { + ValidatorListOutcome::Selected { key, matching_keys } => { + self.validator_list_status.add_list(list_id.clone(), matching_keys); + let context_ready = + self.engine.validator_network().has_validator_list_context(&list_id); + if context_ready { log::info!(target: "validator_manager", "Local node: pk_id: {} id: {}", hex::encode(key.pub_key().unwrap()), hex::encode(key.id().data()) @@ -754,8 +831,8 @@ impl ValidatorManagerImpl { } else { log::warn!( target: "validator_manager", - "Local node is a {} validator by pubkey (id {:x}, key {}), but ADNL/network \ - context is not ready yet; will retry and keep validator membership", + "Local node is a {} validator by pubkey (id {:x}, key {}), \ + but ADNL/network context is still pending; continuing with membership only", name, list_id, hex::encode(key.id().data()) @@ -788,13 +865,12 @@ impl ValidatorManagerImpl { self.update_single_validator_list(validator_set.list(), "current").await?; self.validator_list_status.curr_utime_since = Some(validator_set.utime_since()); if let Some(id) = self.validator_list_status.curr.as_ref() { - if self.validator_list_status.is_list_network_ready(id) { - self.engine.activate_validator_list(id.clone())?; - } else { + self.engine.activate_validator_list(id.clone())?; + if !self.engine.validator_network().has_validator_list_context(id) { log::warn!( target: "validator_manager", - "Current validator list {:x} is matched by pubkey but network context \ - is not ready; keeping previous active validator list until ready", + "Current validator list {:x} is selected by pubkey but transport context is \ + still pending; session ownership remains active and startup will retry", id ); } @@ -851,42 +927,66 @@ impl ValidatorManagerImpl { } } - /// Notify all shard simplex sessions about masterchain finalization. + /// Notify simplex sessions with the currently applied top for their session shard. /// - /// This should be called when a masterchain block is finalized (committed). - /// For shard simplex sessions, this updates MC finalization tracking which is - /// used for empty block generation (finalization recovery). + /// This should be called after each masterchain state update: + /// - masterchain simplex sessions receive the applied masterchain block id + /// - shard simplex sessions receive the shard top currently registered in masterchain /// - /// For catchain sessions and MC sessions, this is a no-op. - /// - /// Notifications are spawned in parallel without waiting for completion. + /// Delivery is enqueued into each validator group's ordered action queue without + /// blocking the manager's hot path. /// /// # Arguments - /// * `mc_block_seqno` - The seqno of the finalized masterchain block - fn notify_shard_sessions_mc_finalized(&self, mc_block_seqno: u32) { + /// * `mc_state` - Current applied masterchain state + /// * `mc_state_extra` - Current masterchain extra containing shard descriptors + fn notify_simplex_sessions_applied_tops( + &self, + mc_state: &ShardStateStuff, + mc_state_extra: &McStateExtra, + ) { consensus_common::check_execution_time!(5000); // 5ms max for (session_id, group) in self.current_sessions.iter() { - if group.shard().is_masterchain() || !group.is_simplex() { + if !group.is_simplex() { continue; } + let applied_top = + match applied_top_for_session_shard(mc_state, mc_state_extra, group.shard()) { + Ok(Some(block_id)) => block_id, + Ok(None) => { + log::trace!( + target: "validator_manager", + "Skipping applied-top notify for session {:x} (shard {}): \ + no matching top in current MC state", + session_id, + group.shard(), + ); + continue; + } + Err(e) => { + log::warn!( + target: "validator_manager", + "Failed to lookup shard {} in MC state for session {:x}: {}", + group.shard(), + session_id, + e + ); + continue; + } + }; log::trace!( target: "validator_manager", - "Notifying session {:x} (shard {}) about MC finalization: seqno={}", + "Notifying session {:x} (shard {}) about applied top: {}", session_id, group.shard(), - mc_block_seqno + applied_top + ); + group.notify_mc_finalized(applied_top); + log::trace!( + target: "validator_manager", + "Applied-top notification queued for session {:x}", + session_id ); - let group = group.clone(); - let sid = session_id.clone(); - tokio::spawn(async move { - group.notify_mc_finalized(mc_block_seqno).await; - log::trace!( - target: "validator_manager", - "MC finalization notification delivered for session {:x}, seqno={}", - sid, mc_block_seqno - ); - }); } } @@ -1114,8 +1214,6 @@ impl ValidatorManagerImpl { catchain_options: &CatchainSessionOptions, _cc_seqno: u32, ) -> ConsensusOptions { - use super::consensus::SimplexSessionOptions; - // C++ ref: mc-config.cpp — Config::get_new_consensus_config reads ConfigParam 30 // directly without checking global_version. Absence of the param // (or a parse error) falls through to the catchain path below. @@ -1156,42 +1254,7 @@ impl ValidatorManagerImpl { // // TODO: C++ also applies per-shard/cc_seqno overrides here via // get_noncritical_params() in validator-options.hpp. - let np = &cfg.noncritical_params; - let opts = SimplexSessionOptions { - proto_version: catchain_options.proto_version as u32, - slots_per_leader_window: cfg.slots_per_leader_window, - target_rate: Duration::from_millis(np.target_rate_ms as u64), - first_block_timeout: Duration::from_millis(np.first_block_timeout_ms as u64), - first_block_timeout_multiplier: f32::from_bits( - np.first_block_timeout_multiplier_bits, - ) as f64, - first_block_timeout_cap: Duration::from_millis( - np.first_block_timeout_cap_ms as u64, - ), - candidate_resolve_timeout: Duration::from_millis( - np.candidate_resolve_timeout_ms as u64, - ), - candidate_resolve_timeout_multiplier: f32::from_bits( - np.candidate_resolve_timeout_multiplier_bits, - ) as f64, - candidate_resolve_timeout_cap: Duration::from_millis( - np.candidate_resolve_timeout_cap_ms as u64, - ), - candidate_resolve_cooldown: Duration::from_millis( - np.candidate_resolve_cooldown_ms as u64, - ), - standstill_timeout: Duration::from_millis(np.standstill_timeout_ms as u64), - standstill_max_egress_bytes_per_s: np.standstill_max_egress_bytes_per_s, - max_leader_window_desync: np.max_leader_window_desync, - bad_signature_ban_duration: Duration::from_millis( - np.bad_signature_ban_duration_ms as u64, - ), - candidate_resolve_rate_limit: np.candidate_resolve_rate_limit, - max_block_size: catchain_options.max_block_size as usize, - max_collated_data_size: catchain_options.max_collated_data_size as usize, - use_quic: cfg.use_quic, - ..Default::default() - }; + let opts = build_runtime_simplex_session_options(shard, &cfg, catchain_options); return ConsensusOptions::Simplex(opts); } @@ -1355,17 +1418,13 @@ impl ValidatorManagerImpl { master_cc_range: &RangeInclusive, last_masterchain_block: &BlockIdExt, ) -> Result<()> { - let validator_list_id = match self.validator_list_status.get_ready_current_list() { + let validator_list_id = match self.validator_list_status.curr.as_ref() { Some(list_id) => list_id.clone(), None => { - if let Some(list_id) = self.validator_list_status.curr.as_ref() { - log::warn!( - target: "validator_manager", - "Skipping current-session start for validator list {:x}: \ - network context is not ready yet", - list_id - ); - } + log::trace!( + target: "validator_manager", + "Skipping current-session start: local node is not in current validator list" + ); return Ok(()); } }; @@ -1384,6 +1443,10 @@ impl ValidatorManagerImpl { log::trace!(target: "validator_manager", "Starting/updating sessions {}", if do_unsafe_catchain_rotate {"(unsafe rotate)"} else {""} ); + // C++ parity: rebuild current-session ownership in a fresh map each update pass. + // We retain only sessions selected by the current shard iteration and swap atomically + // at the end. Old current entries that are not selected are stopped right after swap. + let mut new_current_sessions: HashMap> = HashMap::new(); for (ident, prev_blocks) in new_shards { let cc_seqno_from_state = if ident.is_masterchain() { @@ -1494,43 +1557,60 @@ impl ValidatorManagerImpl { cc_seqno, ); - let session = if let Some(promoted) = self.future_sessions.remove(&session_id) { - log::info!(target: "validator_manager", - "SESSION_LIFECYCLE: promote shard={} cc_seqno={} session_id={:x} \ - future -> current", - ident, cc_seqno, session_id); - self.current_sessions.entry(session_id.clone()).or_insert(promoted).clone() - } else { - self.current_sessions - .entry(session_id.clone()) - .or_insert_with(|| { - let consensus_name = match &consensus_options { - ConsensusOptions::Simplex(_) => "simplex", - ConsensusOptions::Catchain(_) => "catchain", - }; - log::info!(target: "validator_manager", - "SESSION_LIFECYCLE: create_current shard={} cc_seqno={} \ - session_id={:x} consensus={} local_key={}", - ident, cc_seqno, session_id, consensus_name, - hex::encode(local_id.id().data())); - metrics::counter!( - "ton_node_validator_session_created_total", - "consensus" => consensus_name - ) - .increment(1); - Arc::new(ValidatorGroup::new( - general_session_info.clone(), - local_id.clone(), - session_id.clone(), - validator_list_id.clone(), - vsubset.clone(), - consensus_options.clone(), - engine, - allow_unsafe_self_blocks_resync, - )) - }) - .clone() + let session = match select_existing_session_for_current_map( + &session_id, + &self.current_sessions, + &mut self.future_sessions, + ) { + Some(ExistingSessionSource::Current(existing)) => { + log::trace!( + target: "validator_manager", + "SESSION_LIFECYCLE: keep_current shard={} cc_seqno={} session_id={:x}", + ident, + cc_seqno, + session_id + ); + existing + } + Some(ExistingSessionSource::Future(promoted)) => { + log::info!( + target: "validator_manager", + "SESSION_LIFECYCLE: promote shard={} cc_seqno={} session_id={:x} \ + future -> current", + ident, + cc_seqno, + session_id + ); + promoted + } + None => { + let consensus_name = match &consensus_options { + ConsensusOptions::Simplex(_) => "simplex", + ConsensusOptions::Catchain(_) => "catchain", + }; + log::info!(target: "validator_manager", + "SESSION_LIFECYCLE: create_current shard={} cc_seqno={} \ + session_id={:x} consensus={} local_key={}", + ident, cc_seqno, session_id, consensus_name, + hex::encode(local_id.id().data())); + metrics::counter!( + "ton_node_validator_session_created_total", + "consensus" => consensus_name + ) + .increment(1); + Arc::new(ValidatorGroup::new( + general_session_info.clone(), + local_id.clone(), + session_id.clone(), + validator_list_id.clone(), + vsubset.clone(), + consensus_options.clone(), + engine, + allow_unsafe_self_blocks_resync, + )) + } }; + new_current_sessions.insert(session_id.clone(), session.clone()); let session_status = session.get_status().await; if session.try_prepare_start().await? { @@ -1571,7 +1651,55 @@ impl ValidatorManagerImpl { } log::trace!(target: "validator_manager", "Session {} started (if necessary)", ident); } - log::trace!(target: "validator_manager", "Starting/updating sessions, end of list"); + let stale_old_current_sessions: Vec<(UInt256, Arc)> = self + .current_sessions + .iter() + .filter(|(id, _)| !new_current_sessions.contains_key(*id)) + .map(|(id, group)| (id.clone(), group.clone())) + .collect(); + + let old_current_count = self.current_sessions.len(); + self.current_sessions = new_current_sessions; + + for (session_id, stale_group) in stale_old_current_sessions { + self.destroyed_sessions.insert(session_id.clone()); + let stale_shard = stale_group.shard().clone(); + let status = stale_group.get_status().await; + log::info!( + target: "validator_manager", + "SESSION_LIFECYCLE: gc_stop shard={} session_id={:x} status={} destroy_db=true \ + (obsolete after current-map swap)", + stale_group.shard(), + session_id, + status + ); + let consensus_label = if stale_group.is_simplex() { "simplex" } else { "catchain" }; + metrics::counter!( + "ton_node_validator_session_destroyed_total", + "consensus" => consensus_label + ) + .increment(1); + if let Err(e) = stale_group.clone().stop(self.rt.clone(), true).await { + log::error!( + target: "validator_manager", + "SESSION_LIFECYCLE: gc_stop_failed shard={} session_id={:x}: {}", + stale_group.shard(), + session_id, + e + ); + } + if !self.is_active_shard(&stale_shard).await { + self.engine.remove_last_validation_time(&stale_shard); + self.engine.remove_last_collation_time(&stale_shard); + } + } + + log::trace!( + target: "validator_manager", + "Starting/updating sessions, end of list (current map swapped: old={} new={})", + old_current_count, + self.current_sessions.len() + ); Ok(()) } @@ -1820,17 +1948,6 @@ impl ValidatorManagerImpl { mc_validators.append(&mut wc.validators.clone()); } - if !self.validator_list_status.is_list_network_ready(next_val_list_id) { - log::trace!( - target: "validator_manager", - "Skipping future-session precreation for shard {}: validator list {:x} \ - network context is not ready yet", - ident, - next_val_list_id - ); - continue; - } - if let Some(local_id) = self.find_us_for_list(&wc.validators, next_val_list_id) { let max_vertical_seqno = self.engine.hardforks().len() as u32; let new_session_info = Arc::new(GeneralSessionInfo { @@ -2002,9 +2119,9 @@ impl ValidatorManagerImpl { }); } - // Notify shard simplex sessions about MC finalization - // This is needed for empty block generation (finalization recovery) - self.notify_shard_sessions_mc_finalized(last_masterchain_block.seq_no); + // Notify simplex sessions with the currently applied top for their shard. + // This drives C++-parity empty-block recovery logic and MC validation ordering. + self.notify_simplex_sessions_applied_tops(mc_state.as_ref(), mc_state_extra); log::trace!(target: "validator_manager", "starting stop&remove"); self.stop_and_remove_sessions(&gc_validator_sessions, true).await; @@ -2193,16 +2310,17 @@ impl ValidatorManagerImpl { ] { if let Some(list_id) = list_id_opt { let entry = self.validator_list_status.get_list(list_id); - let net_ready = entry.map_or(false, |e| e.network_ready); + let context_ready = + self.engine.validator_network().has_validator_list_context(list_id); let key_strs: Vec = entry .map(|e| e.keys.iter().map(|k| base64_encode(k.id().data())).collect()) .unwrap_or_default(); lines.push(format!( - " [{}] list_id={:x} election_utime={} net_ready={} keys=[{}]", + " [{}] list_id={:x} election_utime={} context_ready={} keys=[{}]", role, list_id, utime_opt.map_or("-".to_string(), |u| u.to_string()), - net_ready, + context_ready, key_strs.join(", "), )); } else { diff --git a/src/node/src/validator/validator_session_listener.rs b/src/node/src/validator/validator_session_listener.rs index 1fece8b..4a6cb11 100644 --- a/src/node/src/validator/validator_session_listener.rs +++ b/src/node/src/validator/validator_session_listener.rs @@ -10,8 +10,8 @@ */ use super::consensus::{ get_elapsed_time, AsyncRequestPtr, BlockHash, BlockPayloadPtr, BlockSourceInfo, - CollationParentHint, CommittedBlockProofCallback, ConsensusReplayListener, PublicKey, - PublicKeyHash, SessionId, SessionListener, SessionStats, ValidatorBlockCandidateCallback, + CollationParentHint, ConsensusReplayListener, PublicKey, PublicKeyHash, SessionId, + SessionListener, SessionStats, ValidatorBlockCandidateCallback, ValidatorBlockCandidateDecisionCallback, }; use crate::validator::validator_group::{ValidatorGroup, ValidatorGroupStatus}; @@ -31,8 +31,21 @@ pub struct OnBlockCommitted { approve_signatures: Vec<(PublicKeyHash, BlockPayloadPtr)>, } +pub struct OnBlockFinalized { + pub block_id: BlockIdExt, + pub source_info: BlockSourceInfo, + pub root_hash: BlockHash, + pub file_hash: BlockHash, + pub data: BlockPayloadPtr, + pub signatures: BlockSignaturesVariant, + pub approve_signatures: Vec<(PublicKeyHash, BlockPayloadPtr)>, +} + #[allow(clippy::enum_variant_names)] pub enum ValidationAction { + OnAppliedTop { + applied_top: BlockIdExt, + }, OnGenerateSlot { source_info: BlockSourceInfo, request: AsyncRequestPtr, @@ -57,10 +70,7 @@ pub enum ValidationAction { collated_data_hash: BlockHash, callback: ValidatorBlockCandidateCallback, }, - OnGetCommittedCandidate { - block_id: BlockIdExt, - callback: CommittedBlockProofCallback, - }, + OnBlockFinalized(OnBlockFinalized), } impl fmt::Display for OnBlockCommitted { @@ -72,6 +82,9 @@ impl fmt::Display for OnBlockCommitted { impl fmt::Display for ValidationAction { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match *self { + ValidationAction::OnAppliedTop { ref applied_top } => { + write!(f, "OnAppliedTop block_id={}", applied_top) + } ValidationAction::OnGenerateSlot { ref source_info, ref request, .. } => { write!( f, @@ -95,8 +108,12 @@ impl fmt::Display for ValidationAction { ValidationAction::OnGetApprovedCandidate { .. } => write!(f, "OnGetApprovedCandidate"), - ValidationAction::OnGetCommittedCandidate { ref block_id, .. } => { - write!(f, "OnGetCommittedCandidate block_id={}", block_id) + ValidationAction::OnBlockFinalized(ref finalized) => { + write!( + f, + "OnBlockFinalized block_id={} round={}", + finalized.block_id, finalized.source_info.priority.round + ) } } } @@ -121,6 +138,10 @@ impl ValidatorSessionListener { } } + pub(crate) fn queue_sender(&self) -> tokio::sync::mpsc::UnboundedSender { + self.queue.clone() + } + pub fn create( session_id: SessionId, shard: ShardIdent, @@ -229,16 +250,33 @@ impl SessionListener for ValidatorSessionListener { ); } - /// Download committed block proof from full-node - fn get_committed_candidate(&self, block_id: BlockIdExt, callback: CommittedBlockProofCallback) { + fn on_block_finalized( + &self, + block_id: BlockIdExt, + source_info: BlockSourceInfo, + root_hash: BlockHash, + file_hash: BlockHash, + data: BlockPayloadPtr, + signatures: BlockSignaturesVariant, + approve_signatures: Vec<(PublicKeyHash, BlockPayloadPtr)>, + ) { + let round = source_info.priority.round; log::info!( target: "validator", - "SessionListener::get_committed_candidate block_id={} (session_id={:x}, shard={})", - block_id, self.session_id, self.shard + "SessionListener::on_block_finalized: block_id={}, round={} (session_id={:x}, shard={})", + block_id, round, self.session_id, self.shard ); self.do_send_general( - None, - ValidationAction::OnGetCommittedCandidate { block_id, callback }, + Some(round), + ValidationAction::OnBlockFinalized(OnBlockFinalized { + block_id, + source_info, + root_hash, + file_hash, + data, + signatures, + approve_signatures, + }), ); } } @@ -264,6 +302,8 @@ async fn process_validation_action(action: ValidationAction, g: Arc g.on_applied_top(applied_top).await, + ValidationAction::OnGenerateSlot { source_info, request, parent, callback } => { let round = source_info.priority.round; let priority = source_info.priority.priority; @@ -366,8 +406,38 @@ async fn process_validation_action(action: ValidationAction, g: Arc { - g.on_get_committed_candidate(block_id, callback).await + ValidationAction::OnBlockFinalized(OnBlockFinalized { + block_id, + source_info, + root_hash, + file_hash, + data, + signatures, + approve_signatures, + }) => { + let round = source_info.priority.round; + let source = source_info.source; + + log::trace!( + target: "validator", + "({}): OnBlockFinalized: block_id={}, round={}, source={}", + next_block_descr, + block_id, + round, + source.id(), + ); + + g.on_block_finalized( + block_id, + round, + source, + root_hash, + file_hash, + data, + signatures, + approve_signatures, + ) + .await } } } @@ -469,3 +539,7 @@ pub async fn process_validation_queue( g.info().await ); } + +#[cfg(test)] +#[path = "tests/test_validator_session_listener.rs"] +mod tests; diff --git a/src/node/tests/test_run_net/test_run_net_ci.sh b/src/node/tests/test_run_net/test_run_net_ci.sh index 5bc2068..327a47f 100755 --- a/src/node/tests/test_run_net/test_run_net_ci.sh +++ b/src/node/tests/test_run_net/test_run_net_ci.sh @@ -27,6 +27,7 @@ function find_block { echo $LOOP_RES fi } +date echo "Waiting for first master block" counter=0 until [ "$(find_block '-1\:8000000000000000, 1' 'LOOP')" == "$NODES" ] @@ -34,6 +35,7 @@ do sleep 10 counter=$((counter + 1)) if [ $counter -gt 5 ]; then + date find_block "-1\:8000000000000000, 1" echo "Reached timeout limit" bash "$TEST_ROOT/stop_network.sh" @@ -42,12 +44,14 @@ do done find_block "-1\:8000000000000000, 1" +date echo "Waiting for 50th master block" until [ "$(find_block '-1\:8000000000000000, 50' 'LOOP')" == "$NODES" ] do sleep 10 counter=$((counter + 1)) if [ $counter -gt 20 ]; then + date find_block "-1\:8000000000000000, 50" echo "Reached timeout limit" bash "$TEST_ROOT/stop_network.sh" @@ -56,6 +60,7 @@ do done find_block "-1\:8000000000000000, 50" +date echo "Waiting for 50th shard block" counter=0 until [ "$(find_block '0:(.*), 50' 'LOOP')" == "$NODES" ] @@ -63,6 +68,7 @@ do sleep 10 counter=$((counter + 1)) if [ $counter -gt 30 ]; then + date find_block "0:(.*), 50" echo "Reached timeout limit" bash "$TEST_ROOT/stop_network.sh" @@ -71,5 +77,6 @@ do done find_block "0:(.*), 50" +date bash "$TEST_ROOT/stop_network.sh" echo "TEST PASSED" diff --git a/src/node/validator-session/tests/test_accelerated_consensus_session.rs b/src/node/validator-session/tests/test_accelerated_consensus_session.rs index de8d7a2..06b2ad2 100644 --- a/src/node/validator-session/tests/test_accelerated_consensus_session.rs +++ b/src/node/validator-session/tests/test_accelerated_consensus_session.rs @@ -434,15 +434,6 @@ impl SessionListener for SessionInstance { self.source_index ); } - - fn get_committed_candidate( - &self, - block_id: BlockIdExt, - callback: consensus_common::CommittedBlockProofCallback, - ) { - log::info!("get_committed_candidate: STUB for block_id={block_id}"); - callback(Err(error!("get_committed_candidate not implemented in test"))); - } } impl SessionListener for SessionInstanceListener { @@ -523,15 +514,6 @@ impl SessionListener for SessionInstanceListener { ); } } - - fn get_committed_candidate( - &self, - block_id: BlockIdExt, - callback: consensus_common::CommittedBlockProofCallback, - ) { - log::info!("get_committed_candidate: STUB for block_id={block_id}"); - callback(Err(error!("get_committed_candidate not implemented in test"))); - } } /// Generalized test function that runs validator session tests with configurable parameters diff --git a/src/node/validator-session/tests/test_fast_session.rs b/src/node/validator-session/tests/test_fast_session.rs index 2e43b41..8fd0381 100644 --- a/src/node/validator-session/tests/test_fast_session.rs +++ b/src/node/validator-session/tests/test_fast_session.rs @@ -21,7 +21,7 @@ use std::{ }, time::Duration, }; -use ton_block::{error, BlockIdExt, BlockSignaturesVariant, Ed25519KeyOption, ShardIdent, UInt256}; +use ton_block::{BlockIdExt, BlockSignaturesVariant, Ed25519KeyOption, ShardIdent, UInt256}; use validator_session::*; include!("../../../common/src/info.rs"); @@ -148,15 +148,6 @@ impl SessionListener for DummySessionListener { root_hash ); } - - fn get_committed_candidate( - &self, - block_id: BlockIdExt, - callback: consensus_common::CommittedBlockProofCallback, - ) { - log::info!("get_committed_candidate: STUB for block_id={block_id}"); - callback(Err(error!("get_committed_candidate not implemented in test"))); - } } impl CatchainReplayListener for DummySessionListener {