From c774ea0d49a5ea5e303127c26922263f4656fb9a Mon Sep 17 00:00:00 2001 From: gregorydemay Date: Thu, 12 Feb 2026 09:56:33 +0100 Subject: [PATCH 1/4] DEFI-2566: new error type ConsistentError::Client for client errors --- .../cketh/minter/src/eth_rpc_client/mod.rs | 45 ++++++++++++++----- .../cketh/minter/src/eth_rpc_client/tests.rs | 11 ++--- 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/rs/ethereum/cketh/minter/src/eth_rpc_client/mod.rs b/rs/ethereum/cketh/minter/src/eth_rpc_client/mod.rs index ad9add3b80a7..0547280ec7ef 100644 --- a/rs/ethereum/cketh/minter/src/eth_rpc_client/mod.rs +++ b/rs/ethereum/cketh/minter/src/eth_rpc_client/mod.rs @@ -5,7 +5,7 @@ use evm_rpc_types::{ RpcError, RpcService as EvmRpcService, RpcServices as EvmRpcServices, }; use ic_canister_log::log; -use ic_canister_runtime::IcRuntime; +use ic_canister_runtime::{IcError, IcRuntime}; use std::{ collections::{BTreeMap, BTreeSet}, fmt::Debug, @@ -140,9 +140,9 @@ impl MultiCallResults { let distinct_errors: BTreeSet<_> = self.errors.values().collect(); match distinct_errors.len() { 0 => panic!("BUG: expect errors should be non-empty"), - 1 => { - MultiCallError::ConsistentError(distinct_errors.into_iter().next().unwrap().clone()) - } + 1 => MultiCallError::ConsistentError(ConsistentError::EvmRpc( + distinct_errors.into_iter().next().unwrap().clone(), + )), _ => MultiCallError::InconsistentResults(self), } } @@ -150,10 +150,24 @@ impl MultiCallResults { #[derive(Eq, PartialEq, Debug)] pub enum MultiCallError { - ConsistentError(RpcError), + ConsistentError(ConsistentError), InconsistentResults(MultiCallResults), } +#[derive(Eq, PartialEq, Debug)] +pub enum ConsistentError { + /// Error coming from the client talking to the EVM RPC canister + Client(IcError), + /// Error coming from the EVM RPC canister itself. + EvmRpc(RpcError), +} + +impl From for ConsistentError { + fn from(error: RpcError) -> Self { + Self::EvmRpc(error) + } +} + pub trait ReductionStrategy { fn reduce(&self, results: EvmMultiRpcResult) -> Result>; } @@ -235,7 +249,9 @@ where F: Fn(MultiCallResults) -> Result>, { match result { - EvmMultiRpcResult::Consistent(result) => result.map_err(MultiCallError::ConsistentError), + EvmMultiRpcResult::Consistent(result) => { + result.map_err(|e| MultiCallError::ConsistentError(e.into())) + } EvmMultiRpcResult::Inconsistent(results) => { reduce(MultiCallResults::from_non_empty_iter(results)) } @@ -264,10 +280,19 @@ impl MultiCallError { predicate: P, ) -> bool { match self { - MultiCallError::ConsistentError(RpcError::HttpOutcallError(error)) => predicate(error), - MultiCallError::ConsistentError(RpcError::JsonRpcError { .. }) => false, - MultiCallError::ConsistentError(RpcError::ProviderError(_)) => false, - MultiCallError::ConsistentError(RpcError::ValidationError(_)) => false, + MultiCallError::ConsistentError(ConsistentError::Client(_)) => false, + MultiCallError::ConsistentError(ConsistentError::EvmRpc( + RpcError::HttpOutcallError(error), + )) => predicate(error), + MultiCallError::ConsistentError(ConsistentError::EvmRpc(RpcError::JsonRpcError { + .. + })) => false, + MultiCallError::ConsistentError(ConsistentError::EvmRpc(RpcError::ProviderError( + _, + ))) => false, + MultiCallError::ConsistentError(ConsistentError::EvmRpc( + RpcError::ValidationError(_), + )) => false, MultiCallError::InconsistentResults(results) => { results .errors diff --git a/rs/ethereum/cketh/minter/src/eth_rpc_client/tests.rs b/rs/ethereum/cketh/minter/src/eth_rpc_client/tests.rs index cc3b72a2b3a6..d732037c28f1 100644 --- a/rs/ethereum/cketh/minter/src/eth_rpc_client/tests.rs +++ b/rs/ethereum/cketh/minter/src/eth_rpc_client/tests.rs @@ -1,7 +1,8 @@ use crate::{ eth_rpc::Hash, eth_rpc_client::{ - MinByKey, MultiCallError, MultiCallResults, StrictMajorityByKey, ToReducedWithStrategy, + ConsistentError, MinByKey, MultiCallError, MultiCallResults, StrictMajorityByKey, + ToReducedWithStrategy, responses::{TransactionReceipt, TransactionStatus}, }, numeric::{BlockNumber, GasAmount, TransactionCount, WeiPerGas}, @@ -278,10 +279,10 @@ mod multi_call_results { proptest! { #[test] fn should_not_match_when_consistent_json_rpc_error(code in any::(), message in ".*") { - let error: MultiCallError = MultiCallError::ConsistentError(RpcError::JsonRpcError(JsonRpcError { + let error: MultiCallError = MultiCallError::ConsistentError(ConsistentError::EvmRpc(RpcError::JsonRpcError(JsonRpcError { code, message, - })); + }))); let always_true = |_outcall_error: &HttpOutcallError| true; assert!(!error.has_http_outcall_error_matching(always_true)); @@ -291,10 +292,10 @@ mod multi_call_results { #[test] fn should_match_when_consistent_http_outcall_error() { let error: MultiCallError = MultiCallError::ConsistentError( - RpcError::HttpOutcallError(HttpOutcallError::IcError { + ConsistentError::EvmRpc(RpcError::HttpOutcallError(HttpOutcallError::IcError { code: LegacyRejectionCode::SysTransient, message: "message".to_string(), - }), + })), ); let always_true = |_outcall_error: &HttpOutcallError| true; let always_false = |_outcall_error: &HttpOutcallError| false; From c329ba1caf7c516a29515bbd38705131be0e909b Mon Sep 17 00:00:00 2001 From: gregorydemay Date: Thu, 12 Feb 2026 14:59:42 +0100 Subject: [PATCH 2/4] DEFI-2566: failing test --- rs/ethereum/cketh/minter/tests/cketh.rs | 24 ++++++++++++++++++++++++ rs/ethereum/cketh/test_utils/src/lib.rs | 24 +++++++++++++++++++++++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/rs/ethereum/cketh/minter/tests/cketh.rs b/rs/ethereum/cketh/minter/tests/cketh.rs index eb8a5d9eaa43..e90a4ff5f364 100644 --- a/rs/ethereum/cketh/minter/tests/cketh.rs +++ b/rs/ethereum/cketh/minter/tests/cketh.rs @@ -1341,4 +1341,28 @@ mod cketh_evm_rpc { .build() .expect_rpc_calls(&cketh); } + + #[test] + fn should_not_panic_when_evm_rpc_canister_is_stopped() { + let cketh = CkEthSetup::default(); + // The minter starts right away by scraping the logs, + // which leads the state machine to panick if we were to stop directly the EVM RPC canister. + // So we first stop the minter to start fresh. + cketh.stop_minter(); + cketh + .env + .stop_canister(cketh.evm_rpc_id) + .expect("Failed to stop EVM RPC canister"); + cketh.start_minter(); + + cketh.env.advance_time(SCRAPING_ETH_LOGS_INTERVAL); + + for _ in 0..10 { + cketh.env.tick(); + let logs = cketh.minter_canister_logs(); + if let Some(panicking_log) = logs.iter().find(|l| l.content.contains("ic0.trap")) { + panic!("Minter panicked: {}", panicking_log.content); + } + } + } } diff --git a/rs/ethereum/cketh/test_utils/src/lib.rs b/rs/ethereum/cketh/test_utils/src/lib.rs index 5f3437cbde05..00bcc788da8e 100644 --- a/rs/ethereum/cketh/test_utils/src/lib.rs +++ b/rs/ethereum/cketh/test_utils/src/lib.rs @@ -145,8 +145,8 @@ impl CkEthSetup { .unwrap(), ) .unwrap(); - let minter_id = install_minter(&env, ledger_id, minter_id, evm_rpc_id); install_evm_rpc(&env, evm_rpc_id); + let minter_id = install_minter(&env, ledger_id, minter_id, evm_rpc_id); let caller = PrincipalId::new_user_test_id(DEFAULT_PRINCIPAL_ID); let cketh = Self { @@ -696,6 +696,21 @@ impl CkEthSetup { ) .unwrap() } + + pub fn minter_canister_logs(&self) -> Vec { + let log = self.env.canister_log(self.minter_id); + + let mut records = log.records().iter().collect::>(); + records.sort_by(|a, b| a.idx.cmp(&b.idx)); + records + .into_iter() + .map(|log| CanisterLog { + timestamp_nanos: log.timestamp_nanos, + idx: log.idx, + content: String::from_utf8_lossy(&log.content).to_string(), + }) + .collect() + } } pub fn format_ethereum_address_to_eip_55(address: &str) -> String { @@ -782,3 +797,10 @@ pub struct LedgerBalance { pub account: Account, pub balance: Nat, } + +#[derive(Debug)] +pub struct CanisterLog { + pub timestamp_nanos: u64, + pub idx: u64, + pub content: String, +} From 93cd107e985559c8ad6b20b64ba1816c5bc87ed1 Mon Sep 17 00:00:00 2001 From: gregorydemay Date: Thu, 12 Feb 2026 13:39:02 +0100 Subject: [PATCH 3/4] DEFI-2566: use try_send instead of send --- rs/ethereum/cketh/minter/src/deposit.rs | 4 ++-- .../cketh/minter/src/eth_rpc_client/mod.rs | 16 ++++++++++++++++ rs/ethereum/cketh/minter/src/tx.rs | 2 +- rs/ethereum/cketh/minter/src/withdraw.rs | 12 ++++++------ 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/rs/ethereum/cketh/minter/src/deposit.rs b/rs/ethereum/cketh/minter/src/deposit.rs index 732a24552345..d848eecc302d 100644 --- a/rs/ethereum/cketh/minter/src/deposit.rs +++ b/rs/ethereum/cketh/minter/src/deposit.rs @@ -163,7 +163,7 @@ pub async fn update_last_observed_block_number() -> Option { match read_state(rpc_client) .get_block_by_number(block_height.clone()) .with_cycles(MIN_ATTACHED_CYCLES) - .send() + .try_send() .await .reduce_with_strategy(NoReduction) { @@ -257,7 +257,7 @@ where .with_response_size_estimate( ETH_GET_LOGS_INITIAL_RESPONSE_SIZE_ESTIMATE + HEADER_SIZE_LIMIT, ) - .send() + .try_send() .await .reduce_with_strategy(NoReduction) .map(::parse_all_logs); diff --git a/rs/ethereum/cketh/minter/src/eth_rpc_client/mod.rs b/rs/ethereum/cketh/minter/src/eth_rpc_client/mod.rs index 0547280ec7ef..520cab902586 100644 --- a/rs/ethereum/cketh/minter/src/eth_rpc_client/mod.rs +++ b/rs/ethereum/cketh/minter/src/eth_rpc_client/mod.rs @@ -274,7 +274,23 @@ impl ToReducedWithStrategy for EvmMultiRpcResult { } } +impl ToReducedWithStrategy for Result, IcError> { + fn reduce_with_strategy( + self, + strategy: impl ReductionStrategy, + ) -> Result> { + match self { + Ok(result) => strategy.reduce(result), + Err(error) => Err(MultiCallError::from_client_error(error)), + } + } +} + impl MultiCallError { + pub fn from_client_error(error: IcError) -> Self { + MultiCallError::ConsistentError(ConsistentError::Client(error)) + } + pub fn has_http_outcall_error_matching bool>( &self, predicate: P, diff --git a/rs/ethereum/cketh/minter/src/tx.rs b/rs/ethereum/cketh/minter/src/tx.rs index 8b7e952fe283..73c1984e4414 100644 --- a/rs/ethereum/cketh/minter/src/tx.rs +++ b/rs/ethereum/cketh/minter/src/tx.rs @@ -662,7 +662,7 @@ pub async fn lazy_refresh_gas_fee_estimate() -> Option { .fee_history((5_u8, BlockTag::Latest)) .with_reward_percentiles(vec![20]) .with_cycles(MIN_ATTACHED_CYCLES) - .send() + .try_send() .await .reduce_with_strategy(StrictMajorityByKey::new(|fee_history: &FeeHistory| { Nat::from(fee_history.oldest_block.clone()) diff --git a/rs/ethereum/cketh/minter/src/withdraw.rs b/rs/ethereum/cketh/minter/src/withdraw.rs index 13e7e194e15c..1ffa91bb7035 100644 --- a/rs/ethereum/cketh/minter/src/withdraw.rs +++ b/rs/ethereum/cketh/minter/src/withdraw.rs @@ -193,9 +193,9 @@ async fn latest_transaction_count() -> Option { match read_state(rpc_client) .get_transaction_count((minter_address().await.into_bytes(), BlockTag::Latest)) .with_cycles(MIN_ATTACHED_CYCLES) - .send() + .try_send() .await - .map(TransactionCount::from) + .map(|res| res.map(TransactionCount::from)) .reduce_with_strategy(MinByKey::new(|count: &TransactionCount| *count)) { Ok(transaction_count) => Some(transaction_count), @@ -354,7 +354,7 @@ async fn send_transactions_batch(latest_transaction_count: Option Result Date: Thu, 12 Feb 2026 16:09:59 +0100 Subject: [PATCH 4/4] DEFI-2566: move old impl to test --- rs/ethereum/cketh/minter/src/eth_rpc_client/mod.rs | 9 --------- rs/ethereum/cketh/minter/src/eth_rpc_client/tests.rs | 10 ++++++++++ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/rs/ethereum/cketh/minter/src/eth_rpc_client/mod.rs b/rs/ethereum/cketh/minter/src/eth_rpc_client/mod.rs index 520cab902586..6ccfde26ac81 100644 --- a/rs/ethereum/cketh/minter/src/eth_rpc_client/mod.rs +++ b/rs/ethereum/cketh/minter/src/eth_rpc_client/mod.rs @@ -265,15 +265,6 @@ pub trait ToReducedWithStrategy { ) -> Result>; } -impl ToReducedWithStrategy for EvmMultiRpcResult { - fn reduce_with_strategy( - self, - strategy: impl ReductionStrategy, - ) -> Result> { - strategy.reduce(self) - } -} - impl ToReducedWithStrategy for Result, IcError> { fn reduce_with_strategy( self, diff --git a/rs/ethereum/cketh/minter/src/eth_rpc_client/tests.rs b/rs/ethereum/cketh/minter/src/eth_rpc_client/tests.rs index d732037c28f1..91656ed1af75 100644 --- a/rs/ethereum/cketh/minter/src/eth_rpc_client/tests.rs +++ b/rs/ethereum/cketh/minter/src/eth_rpc_client/tests.rs @@ -22,6 +22,7 @@ const LLAMA_NODES: EvmRpcService = EvmRpcService::EthMainnet(EthMainnetService:: mod multi_call_results { use super::*; + use crate::eth_rpc_client::ReductionStrategy; mod reduce_with_min_by_key { use super::*; @@ -338,6 +339,15 @@ mod multi_call_results { assert!(error_with_outcall_error.has_http_outcall_error_matching(always_true)); } } + + impl ToReducedWithStrategy for MultiRpcResult { + fn reduce_with_strategy( + self, + strategy: impl ReductionStrategy, + ) -> Result> { + strategy.reduce(self) + } + } } mod eth_get_transaction_receipt {