From 7695c7d18e43bbb5fee79597be91efd88abbe810 Mon Sep 17 00:00:00 2001 From: Timi16 Date: Sat, 24 Jan 2026 19:46:09 -0800 Subject: [PATCH 1/4] Adding proper indexing to wait instead of crash --- .../src/local_cache/finalised_state.rs | 142 +++++++----------- 1 file changed, 56 insertions(+), 86 deletions(-) diff --git a/zaino-state/src/local_cache/finalised_state.rs b/zaino-state/src/local_cache/finalised_state.rs index 85b0ed591..391887719 100644 --- a/zaino-state/src/local_cache/finalised_state.rs +++ b/zaino-state/src/local_cache/finalised_state.rs @@ -375,26 +375,37 @@ impl FinalisedState { /// - Re-populated the database from the NEXT block in the chain (`reorg_height + 1`). async fn sync_db_from_reorg(&self) -> Result<(), FinalisedStateError> { let network = self.config.network.to_zebra_network(); - let mut reorg_height = self.get_db_height().unwrap_or(Height(0)); - // let reorg_height_int = reorg_height.0.saturating_sub(100); - // reorg_height = Height(reorg_height_int); - let mut reorg_hash = self.get_hash(reorg_height.0).unwrap_or(Hash([0u8; 32])); - - let mut check_hash = match self - .fetcher - .get_block(reorg_height.0.to_string(), Some(1)) - .await? - { - zaino_fetch::jsonrpsee::response::GetBlockResponse::Object(block) => block.hash.0, - _ => { - return Err(FinalisedStateError::Custom( - "Unexpected block response type".to_string(), - )) + + // FIXED: Wait for initial block to exist instead of crashing + let mut check_hash = loop { + match self + .fetcher + .get_block(reorg_height.0.to_string(), Some(1)) + .await + { + Ok(zaino_fetch::jsonrpsee::response::GetBlockResponse::Object(block)) => { + break block.hash.0; + } + Ok(_) => { + return Err(FinalisedStateError::Custom( + "Unexpected block response type".to_string(), + )) + } + Err(e) => { + // Block doesn't exist yet - wait for it to be mined + tracing::debug!( + "Block at height {} not found yet, waiting for chain to populate: {}", + reorg_height.0, + e + ); + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + continue; + } } }; - + // Find reorg height. // // Here this is the latest height at which the internal block hash matches the server block hash. @@ -413,31 +424,45 @@ impl FinalisedState { break; } }; - reorg_hash = self.get_hash(reorg_height.0).unwrap_or(Hash([0u8; 32])); - - check_hash = match self - .fetcher - .get_block(reorg_height.0.to_string(), Some(1)) - .await? - { - zaino_fetch::jsonrpsee::response::GetBlockResponse::Object(block) => block.hash.0, - _ => { - return Err(FinalisedStateError::Custom( - "Unexpected block response type".to_string(), - )) + + // FIXED: Wait for block instead of crashing + check_hash = loop { + match self + .fetcher + .get_block(reorg_height.0.to_string(), Some(1)) + .await + { + Ok(zaino_fetch::jsonrpsee::response::GetBlockResponse::Object(block)) => { + break block.hash.0; + } + Ok(_) => { + return Err(FinalisedStateError::Custom( + "Unexpected block response type".to_string(), + )) + } + Err(e) => { + tracing::debug!( + "Block at height {} not found yet, waiting: {}", + reorg_height.0, + e + ); + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + continue; + } } }; } - + // Refill from max(reorg_height[+1], sapling_activation_height) to current server (finalised state) height. - let mut sync_height = self + let sync_height = self .fetcher .get_blockchain_info() .await? .blocks .0 .saturating_sub(99); + for block_height in ((reorg_height.0 + 1).max( self.config .network @@ -474,62 +499,7 @@ impl FinalisedState { } } } - - // Wait for server to sync to with p2p network and sync new blocks. - if !self.config.network.to_zebra_network().is_regtest() { - self.status.store(StatusType::Syncing); - loop { - let blockchain_info = self.fetcher.get_blockchain_info().await?; - let server_height = blockchain_info.blocks.0; - for block_height in (sync_height + 1)..(server_height - 99) { - if self.get_hash(block_height).is_ok() { - self.delete_block(Height(block_height))?; - } - loop { - match fetch_block_from_node( - self.state.as_ref(), - Some(&network), - &self.fetcher, - HashOrHeight::Height(Height(block_height)), - ) - .await - { - Ok((hash, block)) => { - self.insert_block((Height(block_height), hash, block))?; - info!( - "Block at height {} successfully inserted in finalised state.", - block_height - ); - break; - } - Err(e) => { - self.status.store(StatusType::RecoverableError); - warn!("{e}"); - tokio::time::sleep(std::time::Duration::from_millis(500)).await; - } - } - } - } - sync_height = server_height - 99; - if (blockchain_info.blocks.0 as i64 - blockchain_info.estimated_height.0 as i64) - .abs() - <= 10 - { - break; - } else { - info!(" - Validator syncing with network. ZainoDB chain height: {}, Validator chain height: {}, Estimated Network chain height: {}", - &sync_height, - &blockchain_info.blocks.0, - &blockchain_info.estimated_height.0 - ); - tokio::time::sleep(std::time::Duration::from_millis(1000)).await; - continue; - } - } - } - - self.status.store(StatusType::Ready); - + Ok(()) } From 7071c8d3e770eb040f899cacfad530c3e175c866 Mon Sep 17 00:00:00 2001 From: Timi16 Date: Sun, 25 Jan 2026 11:32:27 -0800 Subject: [PATCH 2/4] Removing cached state --- .../src/local_cache/finalised_state.rs | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/zaino-state/src/local_cache/finalised_state.rs b/zaino-state/src/local_cache/finalised_state.rs index 391887719..48e28208c 100644 --- a/zaino-state/src/local_cache/finalised_state.rs +++ b/zaino-state/src/local_cache/finalised_state.rs @@ -455,14 +455,30 @@ impl FinalisedState { } // Refill from max(reorg_height[+1], sapling_activation_height) to current server (finalised state) height. - let sync_height = self + let current_chain_height = self .fetcher .get_blockchain_info() .await? .blocks - .0 - .saturating_sub(99); - + .0; + + let sync_height = current_chain_height.saturating_sub(99); + // FIXED: If cached height is beyond current chain, clear cache (fresh regtest chain) + if reorg_height.0 > current_chain_height { + tracing::warn!( + "Cached height {} is beyond current chain height {}. Clearing cache for fresh chain.", + reorg_height.0, + current_chain_height + ); + { + let mut txn = self.database.begin_rw_txn()?; + txn.clear_db(self.heights_to_hashes)?; + txn.clear_db(self.hashes_to_blocks)?; + txn.commit()?; + } + // Start from sapling activation height on fresh chain + reorg_height = Height(self.config.network.to_zebra_network().sapling_activation_height().0); + } for block_height in ((reorg_height.0 + 1).max( self.config .network From 111cf8b7eca2fd638fdde0e14ac3359cb71801e4 Mon Sep 17 00:00:00 2001 From: Timi16 Date: Sun, 25 Jan 2026 18:27:54 -0800 Subject: [PATCH 3/4] Fixing reorg to sync --- .../src/local_cache/finalised_state.rs | 49 ++++++++++--------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/zaino-state/src/local_cache/finalised_state.rs b/zaino-state/src/local_cache/finalised_state.rs index 48e28208c..b410c9731 100644 --- a/zaino-state/src/local_cache/finalised_state.rs +++ b/zaino-state/src/local_cache/finalised_state.rs @@ -376,6 +376,31 @@ impl FinalisedState { async fn sync_db_from_reorg(&self) -> Result<(), FinalisedStateError> { let network = self.config.network.to_zebra_network(); let mut reorg_height = self.get_db_height().unwrap_or(Height(0)); + + // FIXED: Check if cached height is beyond current chain BEFORE trying to fetch + let current_chain_height = self + .fetcher + .get_blockchain_info() + .await? + .blocks + .0; + + if reorg_height.0 > current_chain_height { + tracing::warn!( + "Cached height {} is beyond current chain height {}. Clearing cache for fresh chain.", + reorg_height.0, + current_chain_height + ); + { + let mut txn = self.database.begin_rw_txn()?; + txn.clear_db(self.heights_to_hashes)?; + txn.clear_db(self.hashes_to_blocks)?; + txn.commit()?; + } + // Reset to height 0 for fresh chain + reorg_height = Height(0); + } + let mut reorg_hash = self.get_hash(reorg_height.0).unwrap_or(Hash([0u8; 32])); // FIXED: Wait for initial block to exist instead of crashing @@ -455,30 +480,8 @@ impl FinalisedState { } // Refill from max(reorg_height[+1], sapling_activation_height) to current server (finalised state) height. - let current_chain_height = self - .fetcher - .get_blockchain_info() - .await? - .blocks - .0; - let sync_height = current_chain_height.saturating_sub(99); - // FIXED: If cached height is beyond current chain, clear cache (fresh regtest chain) - if reorg_height.0 > current_chain_height { - tracing::warn!( - "Cached height {} is beyond current chain height {}. Clearing cache for fresh chain.", - reorg_height.0, - current_chain_height - ); - { - let mut txn = self.database.begin_rw_txn()?; - txn.clear_db(self.heights_to_hashes)?; - txn.clear_db(self.hashes_to_blocks)?; - txn.commit()?; - } - // Start from sapling activation height on fresh chain - reorg_height = Height(self.config.network.to_zebra_network().sapling_activation_height().0); - } + for block_height in ((reorg_height.0 + 1).max( self.config .network From cd2df0db48751287ee760c49662514450f8293ce Mon Sep 17 00:00:00 2001 From: Timi16 Date: Tue, 10 Feb 2026 13:35:59 -0800 Subject: [PATCH 4/4] Addressing Pacu changes --- .../src/local_cache/finalised_state.rs | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/zaino-state/src/local_cache/finalised_state.rs b/zaino-state/src/local_cache/finalised_state.rs index b410c9731..a05e2f715 100644 --- a/zaino-state/src/local_cache/finalised_state.rs +++ b/zaino-state/src/local_cache/finalised_state.rs @@ -377,7 +377,13 @@ impl FinalisedState { let network = self.config.network.to_zebra_network(); let mut reorg_height = self.get_db_height().unwrap_or(Height(0)); - // FIXED: Check if cached height is beyond current chain BEFORE trying to fetch + // Check if cached height exceeds current chain height. + // This can occur when: + // - Switching between networks (mainnet/testnet/regtest) + // - Chain has reset or been pruned + // - Database persisted from a previous chain state + // Without this check, attempting to fetch non-existent blocks would cause a crash. + // If detected, clear the cache and resync from genesis. let current_chain_height = self .fetcher .get_blockchain_info() @@ -403,7 +409,10 @@ impl FinalisedState { let mut reorg_hash = self.get_hash(reorg_height.0).unwrap_or(Hash([0u8; 32])); - // FIXED: Wait for initial block to exist instead of crashing + // Wait for initial block to exist instead of crashing. + // On fresh chains or during initial sync, blocks may not be available immediately. + // This loop polls until the block at reorg_height is available, preventing + // crashes when the indexer starts before the chain has any blocks. let mut check_hash = loop { match self .fetcher @@ -451,7 +460,10 @@ impl FinalisedState { }; reorg_hash = self.get_hash(reorg_height.0).unwrap_or(Hash([0u8; 32])); - // FIXED: Wait for block instead of crashing + // Wait for block to exist instead of crashing. + // During reorg detection, we walk backwards through the chain to find the + // last valid block. If we're near the chain tip and blocks haven't been + // mined yet, this prevents crashes by waiting for blocks to become available. check_hash = loop { match self .fetcher @@ -519,6 +531,7 @@ impl FinalisedState { } } + self.status.store(StatusType::Ready); Ok(()) } @@ -710,4 +723,4 @@ impl FinalisedStateSubscriber { pub fn status(&self) -> StatusType { self.status.load() } -} +} \ No newline at end of file