From bde9e97573b66a5e9adfe32cdb4cab0cb3748b20 Mon Sep 17 00:00:00 2001 From: rustaceanrob Date: Thu, 4 Sep 2025 09:41:43 +0100 Subject: [PATCH 1/3] hintfile: Use `configure_me` for building hints --- README.md | 1 - hintfile/Cargo.toml | 8 ++++++++ hintfile/build.rs | 5 +++++ hintfile/config_spec.toml | 17 +++++++++++++++++ hintfile/src/bin/construct.rs | 35 +++++++++++++++++++++++------------ 5 files changed, 53 insertions(+), 13 deletions(-) create mode 100644 hintfile/build.rs create mode 100644 hintfile/config_spec.toml diff --git a/README.md b/README.md index 4219160..21e9487 100644 --- a/README.md +++ b/README.md @@ -7,4 +7,3 @@ This repository is a collection of crates related to a SwiftSync node implementa - `accumulator`: A hash-based SwiftSync accumulator used to add and subtrack elements from a set. - `hintfile`: Generate a SwiftSync hintfile as the role of a server. - `node`: Perform fast IBD using a SwiftSync hints file. -- `network`: Tools to find Bitcoin peers. diff --git a/hintfile/Cargo.toml b/hintfile/Cargo.toml index b36597f..8163790 100644 --- a/hintfile/Cargo.toml +++ b/hintfile/Cargo.toml @@ -6,5 +6,13 @@ edition = "2021" [dependencies] kernel = { workspace = true } +configure_me = "0.4.0" + +[build-dependencies] +configure_me_codegen = "0.4.0" + +[package.metadata.configure_me] +spec = "config_spec.toml" + [[bin]] name = "construct" diff --git a/hintfile/build.rs b/hintfile/build.rs new file mode 100644 index 0000000..641ae79 --- /dev/null +++ b/hintfile/build.rs @@ -0,0 +1,5 @@ +extern crate configure_me_codegen; + +fn main() -> Result<(), configure_me_codegen::Error> { + configure_me_codegen::build_script_auto() +} diff --git a/hintfile/config_spec.toml b/hintfile/config_spec.toml new file mode 100644 index 0000000..af4c93f --- /dev/null +++ b/hintfile/config_spec.toml @@ -0,0 +1,17 @@ +[[param]] +name = "name" +type = "String" +default = "\"./bitcoin.hints\".into()" +doc = "The path to the .hints file you would like to create. Default is `./bitcoin.hints`" + +[[param]] +name = "bitcoin_dir" +type = "String" +default = "\"~/.bitcoin\".into()" +doc = "The directory of your `bitcoind` data" + +[[param]] +name = "network" +type = "String" +default = "\"bitcoin\".into()" +doc = "The bitcoin network to operate on. Default `bitcoin`. Options are `bitcoin` or `signet`" diff --git a/hintfile/src/bin/construct.rs b/hintfile/src/bin/construct.rs index 3e47fe0..aac96b0 100644 --- a/hintfile/src/bin/construct.rs +++ b/hintfile/src/bin/construct.rs @@ -1,25 +1,36 @@ -use std::{fs::File, io::Write, sync::Arc}; +use std::{fs::File, io::Write, path::PathBuf, str::FromStr}; use hintfile::write_compact_size; use kernel::{ChainType, ChainstateManager, ChainstateManagerOptions, ContextBuilder, KernelError}; -const CHAIN_TYPE: ChainType = ChainType::SIGNET; +configure_me::include_config!(); -fn main() { - let mut file = File::create("./bitcoin.hints").unwrap(); +fn chain_type_from_string(network: String) -> ChainType { + match network.to_lowercase().as_ref() { + "bitcoin" => ChainType::MAINNET, + "signet" => ChainType::SIGNET, + _ => panic!("supported chains are `bitcoin` or `signet`"), + } +} - let mut args = std::env::args(); - let _ = args.next(); - let data_dir = args.next().expect("Usage: "); - let mut blocks_dir = data_dir.clone(); - blocks_dir.push_str("/blocks"); +fn main() { + let (config, _) = Config::including_optional_config_files::<&[&str]>(&[]).unwrap_or_exit(); + let chain_type = chain_type_from_string(config.network); + let hintfile_path = PathBuf::from_str(&config.name).unwrap(); + let bitcoind = PathBuf::from_str(&config.bitcoin_dir).unwrap(); + let blocks_dir = bitcoind.join("blocks"); + let mut file = File::create(hintfile_path).unwrap(); println!("Initializing"); let ctx = ContextBuilder::new() - .chain_type(CHAIN_TYPE) + .chain_type(chain_type) .build() .unwrap(); - let options = ChainstateManagerOptions::new(&ctx, &data_dir, &blocks_dir).unwrap(); - let _context = Arc::new(ctx); + let options = ChainstateManagerOptions::new( + &ctx, + bitcoind.to_str().unwrap(), + blocks_dir.to_str().unwrap(), + ) + .unwrap(); let chainman = ChainstateManager::new(options).unwrap(); println!("Chain state initialized"); // Writing the chain tip allows the client to know where to stop From af730e7128e3d3cbc86d65129a52f6cc2af24f13 Mon Sep 17 00:00:00 2001 From: rustaceanrob Date: Thu, 4 Sep 2025 10:00:24 +0100 Subject: [PATCH 2/3] hintfile: Record the offset from previous coin Instead of encoding the literal indexes in a block, the hintfile can record the delta between the last found coin and the next coin. When querying a height, these can be unpacked by taking a running index and aggregating the offsets. --- hintfile/src/bin/construct.rs | 1 + hintfile/src/lib.rs | 15 ++++++++++++--- node/src/lib.rs | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/hintfile/src/bin/construct.rs b/hintfile/src/bin/construct.rs index aac96b0..74f21ba 100644 --- a/hintfile/src/bin/construct.rs +++ b/hintfile/src/bin/construct.rs @@ -50,6 +50,7 @@ fn main() { if chainman.have_coin(&transaction, vout) { println!("Found coin at offset {curr}"); block_unspents.push(curr); + curr = 0; } curr += 1; } diff --git a/hintfile/src/lib.rs b/hintfile/src/lib.rs index 477e78d..0923672 100644 --- a/hintfile/src/lib.rs +++ b/hintfile/src/lib.rs @@ -79,11 +79,20 @@ impl Hints { /// /// If there are no offset present at that height, aka an overflow, or the entry has already /// been fetched. - pub fn get_block_offsets(&self, height: BlockHeight) -> Vec { - self.map + pub fn get_indexes(&self, height: BlockHeight) -> Vec { + let offsets = self + .map .get(&height) .cloned() - .expect("block height overflow") + .expect("block height overflow"); + let mut indexes = Vec::with_capacity(offsets.len()); + let mut prev = 0; + for offset in offsets { + let next = prev + offset; + indexes.push(next); + prev = next; + } + indexes } } diff --git a/node/src/lib.rs b/node/src/lib.rs index 58c0c3e..f39205b 100644 --- a/node/src/lib.rs +++ b/node/src/lib.rs @@ -220,7 +220,7 @@ pub fn get_blocks_for_range( .expect("header is in best chain."); let block_height = block_index.height().unsigned_abs(); let unspent_indexes: HashSet = - hints.get_block_offsets(block_height).into_iter().collect(); + hints.get_indexes(block_height).into_iter().collect(); // tracing::info!("{task_id} -> {block_height}:{hash}"); let file_path = block_dir.join(format!("{hash}.block")); let file = File::create_new(file_path); From 4e01288692b0193b9a8dba04e111093dee1e0d67 Mon Sep 17 00:00:00 2001 From: rustaceanrob Date: Thu, 4 Sep 2025 10:35:15 +0100 Subject: [PATCH 3/3] node: Use new hintfile format --- node/src/bin/ibd.rs | 2 +- node/src/lib.rs | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/node/src/bin/ibd.rs b/node/src/bin/ibd.rs index 675c308..256716d 100644 --- a/node/src/bin/ibd.rs +++ b/node/src/bin/ibd.rs @@ -74,7 +74,7 @@ fn main() { let acc_task = std::thread::spawn(move || accumulator_state.verify()); let peers = Arc::new(Mutex::new(peers)); let mut tasks = Vec::new(); - let hashes = hashes_from_chain(Arc::clone(&chain), task_num); + let hashes = hashes_from_chain(Arc::clone(&chain), network, task_num); for (task_id, chunk) in hashes.into_iter().enumerate() { let chain = Arc::clone(&chain); let tx = tx.clone(); diff --git a/node/src/lib.rs b/node/src/lib.rs index f39205b..47c1b91 100644 --- a/node/src/lib.rs +++ b/node/src/lib.rs @@ -315,7 +315,11 @@ pub fn get_blocks_for_range( tracing::info!("All block ranges fetched: {task_id}"); } -pub fn hashes_from_chain(chain: Arc, jobs: usize) -> Vec> { +pub fn hashes_from_chain( + chain: Arc, + network: Network, + jobs: usize, +) -> Vec> { let height = chain.best_header().height(); let mut hashes = Vec::with_capacity(height as usize); let mut curr = chain.best_header(); @@ -330,6 +334,9 @@ pub fn hashes_from_chain(chain: Arc, jobs: usize) -> Vec> = first_epoch