From f1ab5c7200d765cbf8fa95c01fdf5a7a5afabe53 Mon Sep 17 00:00:00 2001 From: Slava Date: Sat, 25 Jan 2025 13:40:11 +0300 Subject: [PATCH 1/2] Fix issues with catchain forks --- CHANGELOG.md | 12 +++ Cargo.toml | 16 +-- catchain/Cargo.toml | 6 +- catchain/src/catchain.rs | 4 +- src/network/catchain_client.rs | 26 +++-- src/validating_utils.rs | 2 +- src/validator/validate_query.rs | 110 +++++++++++++++++++++ storage/Cargo.toml | 6 +- storage/src/dynamic_boc_rc_db.rs | 22 +++-- validator-session/Cargo.toml | 6 +- validator-session/src/session_processor.rs | 4 +- 11 files changed, 174 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe8bedc..73ce31e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ All notable changes to this project will be documented in this file. +## Version 0.60.14 + +- Deliver catchain fork info to node + +## Version 0.60.13 + +- Tuning catchain performance + +## Version 0.60.12 + +- Fixed possible zero division in cells db telemetry + ## Version 0.60.11 - Decreased error severity for wrong REMP blocks enumeration diff --git a/Cargo.toml b/Cargo.toml index 3eb17a1..2e9002a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ build = 'common/build/build.rs' edition = '2021' name = 'ever-node' -version = '0.60.11' +version = '0.60.14' [workspace] @@ -85,16 +85,16 @@ string-builder = '0.2' thiserror = '1.0' tokio = { features = [ 'rt-multi-thread' ], version = '1.40' } tokio-util = '0.7' -adnl = { features = [ 'client', 'node', 'server' ], git = 'https://github.com/everx-labs/ever-adnl.git', tag = '0.11.32' } +adnl = { features = [ 'client', 'node', 'server' ], git = 'https://github.com/everx-labs/ever-adnl.git', tag = '0.11.38' } catchain = { path = 'catchain' } -ever_abi = { git = 'https://github.com/everx-labs/ever-abi.git', tag = '2.8.1' } -ever_block = { git = 'https://github.com/everx-labs/ever-block.git', tag = '1.11.20' } -ever_block_json = { git = 'https://github.com/everx-labs/ever-block-json.git', tag = '0.9.37' } -ever_executor = { git = 'https://github.com/everx-labs/ever-executor.git', tag = '1.18.22' } -ever_vm = { git = 'https://github.com/everx-labs/ever-vm.git', tag = '2.2.21' } +ever_abi = { git = 'https://github.com/everx-labs/ever-abi.git', tag = '2.8.5' } +ever_block = { git = 'https://github.com/everx-labs/ever-block.git', tag = '1.11.22' } +ever_block_json = { git = 'https://github.com/everx-labs/ever-block-json.git', tag = '0.9.41' } +ever_executor = { git = 'https://github.com/everx-labs/ever-executor.git', tag = '1.18.31' } +ever_vm = { git = 'https://github.com/everx-labs/ever-vm.git', tag = '2.2.26' } lockfree = { git = 'https://github.com/everx-labs/lockfree.git' } storage = { path = 'storage' } -ton_api = { git = 'https://github.com/everx-labs/ever-tl.git', package = 'ton_api', tag = '0.4.29' } +ton_api = { git = 'https://github.com/everx-labs/ever-tl.git', package = 'ton_api', tag = '0.4.32' } validator_session = { path = 'validator-session' } [dev-dependencies] diff --git a/catchain/Cargo.toml b/catchain/Cargo.toml index 82765ce..460d42b 100644 --- a/catchain/Cargo.toml +++ b/catchain/Cargo.toml @@ -18,10 +18,10 @@ quanta = '0.11' rand = '0.8' regex = '1.10' tokio = { features = [ 'rt-multi-thread' ], version = '1.40' } -adnl = { features = [ 'node' ], git = 'https://github.com/everx-labs/ever-adnl.git', tag = '0.11.32' } -ever_block = { git = 'https://github.com/everx-labs/ever-block.git', tag = '1.11.20' } +adnl = { features = [ 'node' ], git = 'https://github.com/everx-labs/ever-adnl.git', tag = '0.11.38' } +ever_block = { git = 'https://github.com/everx-labs/ever-block.git', tag = '1.11.22' } storage = { path = '../storage' } -ton_api = { git = 'https://github.com/everx-labs/ever-tl.git', package = 'ton_api', tag = '0.4.29' } +ton_api = { git = 'https://github.com/everx-labs/ever-tl.git', package = 'ton_api', tag = '0.4.32' } [dev-dependencies] env_logger = '0.11' diff --git a/catchain/src/catchain.rs b/catchain/src/catchain.rs index ad1efcd..f743fc9 100644 --- a/catchain/src/catchain.rs +++ b/catchain/src/catchain.rs @@ -1703,7 +1703,9 @@ impl CatchainProcessor { log::trace!("Catchain forcing creation of a new block"); } - if self.active_process { + let is_requested_time_expired = SystemTime::now().duration_since(time).is_ok(); + + if self.active_process && is_requested_time_expired { self.force_process = true; } else { self.set_next_block_generation_time(time); diff --git a/src/network/catchain_client.rs b/src/network/catchain_client.rs index 8909b5c..407f62b 100644 --- a/src/network/catchain_client.rs +++ b/src/network/catchain_client.rs @@ -14,7 +14,7 @@ use crate::network::node_network::NetworkContext; use adnl::{ - declare_counted, OverlayNode, PrivateOverlayShortId, RldpNode, + declare_counted, CatchainData, OverlayNode, PrivateOverlayShortId, RldpNode, common::{ AdnlPeers, Answer, CountedObject, Counter, QueryAnswer, QueryResult, Subscriber, TaggedByteSlice, TaggedTlObject, Wait @@ -342,16 +342,22 @@ impl CatchainClient { }; let message = receiver.wait_for_catchain(overlay_id).await; match message { - Ok(Some((catchain_block_update, validator_session_block_update, source_id))) => { - log::trace!(target: Self::TARGET, "private overlay broadcast ValidatorSession_BlockUpdate (successed)"); - let vs_block_update = validator_session_block_update.into_boxed(); - let block_update = catchain_block_update.into_boxed(); + Ok(Some((catchain_block_update, inner_update, source_id))) => { + log::trace!( + target: Self::TARGET, + "private overlay broadcast ValidatorSession_BlockUpdate (successed)" + ); if let Some(listener) = catchain_listener.upgrade() { - let mut data: catchain::RawBuffer = catchain::RawBuffer::default(); - let mut serializer = ton_api::Serializer::new(&mut data); - serializer.write_boxed(&block_update)?; - serializer.write_boxed(&vs_block_update)?; - let data = catchain::CatchainFactory::create_block_payload(data); + let mut data: catchain::RawBuffer = catchain::RawBuffer::default(); + let mut serializer = ton_api::Serializer::new(&mut data); + serializer.write_boxed(&catchain_block_update.into_boxed())?; + match inner_update { + CatchainData::Catchain(upd) => + serializer.write_boxed(&upd.into_boxed())?, + CatchainData::ValidatorSession(upd) => + serializer.write_boxed(&upd.into_boxed())? + }; + let data = catchain::CatchainFactory::create_block_payload(data); listener .on_message( source_id, diff --git a/src/validating_utils.rs b/src/validating_utils.rs index f676186..5fd82c8 100644 --- a/src/validating_utils.rs +++ b/src/validating_utils.rs @@ -62,7 +62,7 @@ pub fn supported_capabilities() -> u64 { } pub fn supported_version() -> u32 { - 60 + 59 } #[allow(clippy::too_many_arguments)] diff --git a/src/validator/validate_query.rs b/src/validator/validate_query.rs index edb7faa..d621ec6 100644 --- a/src/validator/validate_query.rs +++ b/src/validator/validate_query.rs @@ -1126,6 +1126,9 @@ impl ValidateQuery { ) } + //#[cfg(feature = "true_fast_finality")] + //self.check_shard_validators_stat(base, old.as_ref(), info)?; + base.result.min_shard_ref_mc_seqno.fetch_min(info.descr.min_ref_mc_seqno, Ordering::Relaxed); base.result.max_shard_utime.fetch_max(info.descr.gen_utime, Ordering::Relaxed); base.result.max_shard_lt.fetch_max(info.descr.end_lt, Ordering::Relaxed); @@ -1133,6 +1136,113 @@ impl ValidateQuery { Ok(()) } + /*#[cfg(feature = "true_fast_finality")] + fn check_shard_validators_stat( + &mut self, + base: &ValidateBase, + old: Option<&McShardRecord>, + cur: &McShardRecord + ) -> Result<()> { + + log::debug!("{}: check_shard_validators_stat {}", self.next_block_descr, cur.shard()); + + let ff_config = base.config_params.fast_finality_config().unwrap_or_default(); + let mut calc_stat; + + if let Some(old) = old { + let new_blocks = (cur.descr.seq_no - old.descr.seq_no) as u16; + let collators = cur.descr.collators()?.clone(); + calc_stat = old.descr.collators()?.stat.clone(); + + if base.mc_extra.is_key_block() { + // key block - reset statistic + let len = base.mc_extra.config() + .ok_or_else(|| error!("No config in mc_extra"))? + .validator_set()?.total() as u16; + calc_stat = ValidatorsStat::new(len); + + } else if cur.shard() == old.shard() || old.shard().is_parent_for(cur.shard()) { + // shard updated without split/merge OR has been split + self.apply_unreliability_fading(&ff_config, &collators.current, new_blocks)?; + for i in 0..calc_stat.len() as u16 { + calc_stat.update(i, |familiarity| { + let (new_familiarity, name) = if i == collators.current.collator { + (familiarity.saturating_add(ff_config.familiarity_collator_fine * new_blocks) + .min(ff_config.familiarity_max), "familiarity_collator_fine") + } else if collators.current.mempool.contains(&i) { + (familiarity.saturating_add(ff_config.familiarity_msgpool_fine * new_blocks) + .min(ff_config.familiarity_max), "familiarity_msgpool_fine") + } else { + (familiarity.saturating_sub(ff_config.familiarity_fading * new_blocks), + "familiarity_fading") + }; + log::debug!("{}: {}: shard {} validator {} familiarity {} -> {}", + self.next_block_descr, name, cur.shard(), i, familiarity, new_familiarity); + new_familiarity + })?; + } + + } else if cur.shard().is_parent_for(old.shard()) { + + // shard has been merged - merge statistic + let old2 = self.old_mc_shards.find_shard(&cur.shard().right_ancestor_mask()?)? + .ok_or_else(|| error!("No plus_one shard"))?; + + self.apply_unreliability_fading(&ff_config, &collators.next, new_blocks)?; + + calc_stat = ValidatorsStat::new(base.config_params.validator_set()?.total()); + let old_collators1 = old.descr.collators()?.clone(); + let old_collators2 = old2.descr.collators()?.clone(); + for i in 0..calc_stat.len() as u16 { + let familiarity1 = old_collators1.stat.get(i).unwrap_or(0); + let familiarity2 = old_collators2.stat.get(i).unwrap_or(0); + let familiarity = ((familiarity1 as u32 + familiarity2 as u32) / 2) as u16; + calc_stat.update(i, |_| { + let new_familiarity = if i == collators.current.collator { + min(familiarity.saturating_add(ff_config.familiarity_collator_fine * new_blocks), + ff_config.familiarity_max) + } else if collators.current.mempool.contains(&i) { + min(familiarity.saturating_add(ff_config.familiarity_msgpool_fine * new_blocks), + ff_config.familiarity_max) + } else { + familiarity.saturating_sub(ff_config.familiarity_fading * new_blocks) + }; + log::debug!( + "{}: merge shard validators stat: shard {} validator {} familiarity {}, {} -> {}", + self.next_block_descr, cur.shard(), i, familiarity1, familiarity2, new_familiarity); + new_familiarity + })?; + } + } + } else { + // newly created workchain + let len = base.config_params.validator_set()?.total() as u16; + calc_stat = ValidatorsStat::new(len); + } + + let stat = &cur.descr.collators()?.stat; + if calc_stat.len() != stat.len() { + reject_query!("validators stat length mismatch is shard {} {} != {}", + cur.shard(), calc_stat.len(), stat.len()); + } + let mut err = false; + for i in 0..stat.len() as u16 { + if stat.get(i)? != calc_stat.get(i)? { + log::error!("{}: stat mismatch - validator: {}, calculated: {}, in block: {}", + self.next_block_descr, i, calc_stat.get(i)?, stat.get(i)?); + err = true; + } else { + log::trace!("{}: stat ok - validator: {}, calculated: {}, in block: {}", + self.next_block_descr, i, calc_stat.get(i)?, stat.get(i)?); + } + } + if err { + reject_query!("validators stat mismatch is shard {}", cur.shard()); + } + + Ok(()) + }*/ + // checks old_shard_conf_ -> base.mc_extra.shards() transition using top_shard_descr_dict_ from collated data // similar to Collator::update_shard_config() fn check_shard_layout(&mut self, base: &ValidateBase, mc_data: &McData) -> Result<()> { diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 103c4f5..0a80463 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -27,10 +27,10 @@ strum_macros = '0.18' thiserror = '1.0' tokio = { features = [ 'fs', 'rt-multi-thread' ], version = '1.5' } tokio-util = '0.7' -adnl = { git = 'https://github.com/everx-labs/ever-adnl.git', tag = '0.11.32' } -ever_block = { git = 'https://github.com/everx-labs/ever-block.git', tag = '1.11.20' } +adnl = { git = 'https://github.com/everx-labs/ever-adnl.git', tag = '0.11.38' } +ever_block = { git = 'https://github.com/everx-labs/ever-block.git', tag = '1.11.22' } lockfree = { git = 'https://github.com/everx-labs/lockfree.git' } -ton_api = { git = 'https://github.com/everx-labs/ever-tl.git', package = 'ton_api', tag = '0.4.29' } +ton_api = { git = 'https://github.com/everx-labs/ever-tl.git', package = 'ton_api', tag = '0.4.32' } [build-dependencies] cc = { features = [ 'parallel' ], version = '1.0.61' } diff --git a/storage/src/dynamic_boc_rc_db.rs b/storage/src/dynamic_boc_rc_db.rs index e9bd0ef..24cbe63 100644 --- a/storage/src/dynamic_boc_rc_db.rs +++ b/storage/src/dynamic_boc_rc_db.rs @@ -249,11 +249,14 @@ impl DynamicBocDb { tx.blocking_send((key, value))?; read += 1; if read % 1_000_000 == 0 { - log::info!( - target: TARGET, - "Cells DB migration: read {} items, speed {} items/sec", - read, read / now.elapsed().as_secs() - ); + let sec = now.elapsed().as_secs(); + if sec != 0 { + log::info!( + target: TARGET, + "Cells DB migration: read {} items, speed {} items/sec", + read, read / sec + ); + } } } @@ -399,8 +402,10 @@ impl DynamicBocDb { let now3 = Instant::now(); self.db.write(transaction)?; #[cfg(feature = "telemetry")] - self.telemetry.boc_db_element_write_nanos.update( - now.elapsed().as_nanos() as u64 / (visited.len() as u64 + created as u64)); + if !visited.is_empty() { + self.telemetry.boc_db_element_write_nanos.update( + now.elapsed().as_nanos() as u64 / (visited.len() as u64 + created as u64)); + } log::debug!( target: TARGET, @@ -522,7 +527,8 @@ impl DynamicBocDb { self.db.write(transaction)?; let updated = visited.len() - deleted; - #[cfg(feature = "telemetry")] { + #[cfg(feature = "telemetry")] + if !visited.is_empty() { self.telemetry.deleted_cells_speed.update(deleted as u64); self.telemetry.updated_cells_speed.update(updated as u64); self.telemetry.boc_db_element_write_nanos.update( diff --git a/validator-session/Cargo.toml b/validator-session/Cargo.toml index 8408f57..5b324e1 100644 --- a/validator-session/Cargo.toml +++ b/validator-session/Cargo.toml @@ -15,11 +15,11 @@ metrics = '0.21' num-derive = '0.4' num-traits = '0.2' rand = '0.8' -adnl = { git = 'https://github.com/everx-labs/ever-adnl.git', tag = '0.11.32' } +adnl = { git = 'https://github.com/everx-labs/ever-adnl.git', tag = '0.11.38' } catchain = { path = '../catchain' } -ever_block = { git = 'https://github.com/everx-labs/ever-block.git', tag = '1.11.20' } +ever_block = { git = 'https://github.com/everx-labs/ever-block.git', tag = '1.11.22' } storage = { path = '../storage' } -ton_api = { git = 'https://github.com/everx-labs/ever-tl.git', package = 'ton_api', tag = '0.4.29' } +ton_api = { git = 'https://github.com/everx-labs/ever-tl.git', package = 'ton_api', tag = '0.4.32' } [dev-dependencies] chrono = '0.4' diff --git a/validator-session/src/session_processor.rs b/validator-session/src/session_processor.rs index 4f20c6a..69df064 100644 --- a/validator-session/src/session_processor.rs +++ b/validator-session/src/session_processor.rs @@ -687,9 +687,7 @@ impl SessionProcessor for SessionProcessorImpl { let block_processing_latency = get_elapsed_time(&block.get_creation_time()); let delivery_issue = block_processing_latency < BLOCK_PROCESSING_WARN_LATENCY; - if !delivery_issue { - force_batching_mode = true; //ask catchain to batch blocks in case of overloaded session incoming queues - } + force_batching_mode = true; //ask catchain to batch blocks in case of overloaded session incoming queues let warn_elapsed = get_elapsed_time(&self.last_process_blocks_warn_dump_time); From 1f8373469ae22165df11a94fee9803009a96e6b5 Mon Sep 17 00:00:00 2001 From: ViacheslavB Date: Tue, 25 Feb 2025 06:47:44 +0000 Subject: [PATCH 2/2] Fix block version --- src/validating_utils.rs | 2 +- src/validator/validate_query.rs | 110 -------------------------------- 2 files changed, 1 insertion(+), 111 deletions(-) diff --git a/src/validating_utils.rs b/src/validating_utils.rs index 5fd82c8..f676186 100644 --- a/src/validating_utils.rs +++ b/src/validating_utils.rs @@ -62,7 +62,7 @@ pub fn supported_capabilities() -> u64 { } pub fn supported_version() -> u32 { - 59 + 60 } #[allow(clippy::too_many_arguments)] diff --git a/src/validator/validate_query.rs b/src/validator/validate_query.rs index d621ec6..edb7faa 100644 --- a/src/validator/validate_query.rs +++ b/src/validator/validate_query.rs @@ -1126,9 +1126,6 @@ impl ValidateQuery { ) } - //#[cfg(feature = "true_fast_finality")] - //self.check_shard_validators_stat(base, old.as_ref(), info)?; - base.result.min_shard_ref_mc_seqno.fetch_min(info.descr.min_ref_mc_seqno, Ordering::Relaxed); base.result.max_shard_utime.fetch_max(info.descr.gen_utime, Ordering::Relaxed); base.result.max_shard_lt.fetch_max(info.descr.end_lt, Ordering::Relaxed); @@ -1136,113 +1133,6 @@ impl ValidateQuery { Ok(()) } - /*#[cfg(feature = "true_fast_finality")] - fn check_shard_validators_stat( - &mut self, - base: &ValidateBase, - old: Option<&McShardRecord>, - cur: &McShardRecord - ) -> Result<()> { - - log::debug!("{}: check_shard_validators_stat {}", self.next_block_descr, cur.shard()); - - let ff_config = base.config_params.fast_finality_config().unwrap_or_default(); - let mut calc_stat; - - if let Some(old) = old { - let new_blocks = (cur.descr.seq_no - old.descr.seq_no) as u16; - let collators = cur.descr.collators()?.clone(); - calc_stat = old.descr.collators()?.stat.clone(); - - if base.mc_extra.is_key_block() { - // key block - reset statistic - let len = base.mc_extra.config() - .ok_or_else(|| error!("No config in mc_extra"))? - .validator_set()?.total() as u16; - calc_stat = ValidatorsStat::new(len); - - } else if cur.shard() == old.shard() || old.shard().is_parent_for(cur.shard()) { - // shard updated without split/merge OR has been split - self.apply_unreliability_fading(&ff_config, &collators.current, new_blocks)?; - for i in 0..calc_stat.len() as u16 { - calc_stat.update(i, |familiarity| { - let (new_familiarity, name) = if i == collators.current.collator { - (familiarity.saturating_add(ff_config.familiarity_collator_fine * new_blocks) - .min(ff_config.familiarity_max), "familiarity_collator_fine") - } else if collators.current.mempool.contains(&i) { - (familiarity.saturating_add(ff_config.familiarity_msgpool_fine * new_blocks) - .min(ff_config.familiarity_max), "familiarity_msgpool_fine") - } else { - (familiarity.saturating_sub(ff_config.familiarity_fading * new_blocks), - "familiarity_fading") - }; - log::debug!("{}: {}: shard {} validator {} familiarity {} -> {}", - self.next_block_descr, name, cur.shard(), i, familiarity, new_familiarity); - new_familiarity - })?; - } - - } else if cur.shard().is_parent_for(old.shard()) { - - // shard has been merged - merge statistic - let old2 = self.old_mc_shards.find_shard(&cur.shard().right_ancestor_mask()?)? - .ok_or_else(|| error!("No plus_one shard"))?; - - self.apply_unreliability_fading(&ff_config, &collators.next, new_blocks)?; - - calc_stat = ValidatorsStat::new(base.config_params.validator_set()?.total()); - let old_collators1 = old.descr.collators()?.clone(); - let old_collators2 = old2.descr.collators()?.clone(); - for i in 0..calc_stat.len() as u16 { - let familiarity1 = old_collators1.stat.get(i).unwrap_or(0); - let familiarity2 = old_collators2.stat.get(i).unwrap_or(0); - let familiarity = ((familiarity1 as u32 + familiarity2 as u32) / 2) as u16; - calc_stat.update(i, |_| { - let new_familiarity = if i == collators.current.collator { - min(familiarity.saturating_add(ff_config.familiarity_collator_fine * new_blocks), - ff_config.familiarity_max) - } else if collators.current.mempool.contains(&i) { - min(familiarity.saturating_add(ff_config.familiarity_msgpool_fine * new_blocks), - ff_config.familiarity_max) - } else { - familiarity.saturating_sub(ff_config.familiarity_fading * new_blocks) - }; - log::debug!( - "{}: merge shard validators stat: shard {} validator {} familiarity {}, {} -> {}", - self.next_block_descr, cur.shard(), i, familiarity1, familiarity2, new_familiarity); - new_familiarity - })?; - } - } - } else { - // newly created workchain - let len = base.config_params.validator_set()?.total() as u16; - calc_stat = ValidatorsStat::new(len); - } - - let stat = &cur.descr.collators()?.stat; - if calc_stat.len() != stat.len() { - reject_query!("validators stat length mismatch is shard {} {} != {}", - cur.shard(), calc_stat.len(), stat.len()); - } - let mut err = false; - for i in 0..stat.len() as u16 { - if stat.get(i)? != calc_stat.get(i)? { - log::error!("{}: stat mismatch - validator: {}, calculated: {}, in block: {}", - self.next_block_descr, i, calc_stat.get(i)?, stat.get(i)?); - err = true; - } else { - log::trace!("{}: stat ok - validator: {}, calculated: {}, in block: {}", - self.next_block_descr, i, calc_stat.get(i)?, stat.get(i)?); - } - } - if err { - reject_query!("validators stat mismatch is shard {}", cur.shard()); - } - - Ok(()) - }*/ - // checks old_shard_conf_ -> base.mc_extra.shards() transition using top_shard_descr_dict_ from collated data // similar to Collator::update_shard_config() fn check_shard_layout(&mut self, base: &ValidateBase, mc_data: &McData) -> Result<()> {