Merge pull request #46 from rustaceanrob/9-4-hints

Record offset in hintfile instead of literal index

Author: rustaceanrob

README.md

@@ -7,4 +7,3 @@ This repository is a collection of crates related to a SwiftSync node implementa
 - `accumulator`: A hash-based SwiftSync accumulator used to add and subtrack elements from a set.
 - `hintfile`: Generate a SwiftSync hintfile as the role of a server.
 - `node`: Perform fast IBD using a SwiftSync hints file.
-- `network`: Tools to find Bitcoin peers.

hintfile/Cargo.toml

@@ -6,5 +6,13 @@ edition = "2021"
 [dependencies]
 kernel = { workspace = true }
 
+configure_me = "0.4.0"
+
+[build-dependencies]
+configure_me_codegen = "0.4.0"
+
+[package.metadata.configure_me]
+spec = "config_spec.toml"
+
 [[bin]]
 name = "construct"

hintfile/build.rs

@@ -0,0 +1,5 @@
+extern crate configure_me_codegen;
+
+fn main() -> Result<(), configure_me_codegen::Error> {
+    configure_me_codegen::build_script_auto()
+}

hintfile/config_spec.toml

@@ -0,0 +1,17 @@
+[[param]]
+name = "name"
+type = "String"
+default = "\"./bitcoin.hints\".into()"
+doc = "The path to the .hints file you would like to create. Default is `./bitcoin.hints`"
+
+[[param]]
+name = "bitcoin_dir"
+type = "String"
+default = "\"~/.bitcoin\".into()"
+doc = "The directory of your `bitcoind` data"
+
+[[param]]
+name = "network"
+type = "String"
+default = "\"bitcoin\".into()"
+doc = "The bitcoin network to operate on. Default `bitcoin`. Options are `bitcoin` or `signet`"

hintfile/src/bin/construct.rs

@@ -1,25 +1,36 @@
-use std::{fs::File, io::Write, sync::Arc};
+use std::{fs::File, io::Write, path::PathBuf, str::FromStr};
 
 use hintfile::write_compact_size;
 use kernel::{ChainType, ChainstateManager, ChainstateManagerOptions, ContextBuilder, KernelError};
 
-const CHAIN_TYPE: ChainType = ChainType::SIGNET;
+configure_me::include_config!();
 
-fn main() {
-    let mut file = File::create("./bitcoin.hints").unwrap();
+fn chain_type_from_string(network: String) -> ChainType {
+    match network.to_lowercase().as_ref() {
+        "bitcoin" => ChainType::MAINNET,
+        "signet" => ChainType::SIGNET,
+        _ => panic!("supported chains are `bitcoin` or `signet`"),
+    }
+}
 
-    let mut args = std::env::args();
-    let _ = args.next();
-    let data_dir = args.next().expect("Usage: <path_to_bitcoin_dir>");
-    let mut blocks_dir = data_dir.clone();
-    blocks_dir.push_str("/blocks");
+fn main() {
+    let (config, _) = Config::including_optional_config_files::<&[&str]>(&[]).unwrap_or_exit();
+    let chain_type = chain_type_from_string(config.network);
+    let hintfile_path = PathBuf::from_str(&config.name).unwrap();
+    let bitcoind = PathBuf::from_str(&config.bitcoin_dir).unwrap();
+    let blocks_dir = bitcoind.join("blocks");
+    let mut file = File::create(hintfile_path).unwrap();
     println!("Initializing");
     let ctx = ContextBuilder::new()
-        .chain_type(CHAIN_TYPE)
+        .chain_type(chain_type)
         .build()
         .unwrap();
-    let options = ChainstateManagerOptions::new(&ctx, &data_dir, &blocks_dir).unwrap();
-    let _context = Arc::new(ctx);
+    let options = ChainstateManagerOptions::new(
+        &ctx,
+        bitcoind.to_str().unwrap(),
+        blocks_dir.to_str().unwrap(),
+    )
+    .unwrap();
     let chainman = ChainstateManager::new(options).unwrap();
     println!("Chain state initialized");
     // Writing the chain tip allows the client to know where to stop
@@ -39,6 +50,7 @@ fn main() {
                 if chainman.have_coin(&transaction, vout) {
                     println!("Found coin at offset {curr}");
                     block_unspents.push(curr);
+                    curr = 0;
                 }
                 curr += 1;
             }

hintfile/src/lib.rs

@@ -79,11 +79,20 @@ impl Hints {
     ///
     /// If there are no offset present at that height, aka an overflow, or the entry has already
     /// been fetched.
-    pub fn get_block_offsets(&self, height: BlockHeight) -> Vec<u64> {
-        self.map
+    pub fn get_indexes(&self, height: BlockHeight) -> Vec<u64> {
+        let offsets = self
+            .map
             .get(&height)
             .cloned()
-            .expect("block height overflow")
+            .expect("block height overflow");
+        let mut indexes = Vec::with_capacity(offsets.len());
+        let mut prev = 0;
+        for offset in offsets {
+            let next = prev + offset;
+            indexes.push(next);
+            prev = next;
+        }
+        indexes
     }
 }
 

node/src/bin/ibd.rs

@@ -74,7 +74,7 @@ fn main() {
     let acc_task = std::thread::spawn(move || accumulator_state.verify());
     let peers = Arc::new(Mutex::new(peers));
     let mut tasks = Vec::new();
-    let hashes = hashes_from_chain(Arc::clone(&chain), task_num);
+    let hashes = hashes_from_chain(Arc::clone(&chain), network, task_num);
     for (task_id, chunk) in hashes.into_iter().enumerate() {
         let chain = Arc::clone(&chain);
         let tx = tx.clone();

node/src/lib.rs

@@ -220,7 +220,7 @@ pub fn get_blocks_for_range(
                         .expect("header is in best chain.");
                     let block_height = block_index.height().unsigned_abs();
                     let unspent_indexes: HashSet<u64> =
-                        hints.get_block_offsets(block_height).into_iter().collect();
+                        hints.get_indexes(block_height).into_iter().collect();
                     // tracing::info!("{task_id} -> {block_height}:{hash}");
                     let file_path = block_dir.join(format!("{hash}.block"));
                     let file = File::create_new(file_path);
@@ -315,7 +315,11 @@ pub fn get_blocks_for_range(
     tracing::info!("All block ranges fetched: {task_id}");
 }
 
-pub fn hashes_from_chain(chain: Arc<ChainstateManager>, jobs: usize) -> Vec<Vec<BlockHash>> {
+pub fn hashes_from_chain(
+    chain: Arc<ChainstateManager>,
+    network: Network,
+    jobs: usize,
+) -> Vec<Vec<BlockHash>> {
     let height = chain.best_header().height();
     let mut hashes = Vec::with_capacity(height as usize);
     let mut curr = chain.best_header();
@@ -330,6 +334,9 @@ pub fn hashes_from_chain(chain: Arc<ChainstateManager>, jobs: usize) -> Vec<Vec<
         hashes.push(hash);
         curr = next;
     }
+    if matches!(network, Network::Signet) {
+        return hashes.chunks(20_000).map(|slice| slice.to_vec()).collect();
+    }
     // These blocks are empty. Fetch the maximum amount of blocks.
     let first_epoch = hashes.split_off(hashes.len() - 200_000);
     let first_chunks: Vec<Vec<BlockHash>> = first_epoch