[WIP] Add BIP-352 tweak index

bindex-lib/Cargo.toml

@@ -28,4 +28,7 @@ rust-rocksdb = { version = "0.36.0", default-features = false, features = ["zstd
 hex_lit = "0.1"
 # bitcoind path should be specified via BITCOIND_EXE
 corepc-node = { version = "0.10", default-features = false, features = ["29_0"] }
+serde = "1.0"
+serde_json = "1.0"
 tempfile = "3"
+silentpayments = { git = "https://github.com/cygnet3/spdk", rev = "ce68148d8621e58a33abe251e28b7587b04998dd" }

bindex-lib/src/chain.rs

@@ -252,7 +252,7 @@ impl IndexedChain {
         use rayon::prelude::*;
 
         let mut batches = Vec::with_capacity(headers.len());
-        for chunk in headers.chunks(10) {
+        for chunk in headers.chunks(128) {
             let items: Vec<_> = chunk
                 .par_iter()
                 .map(|header| self.fetch_data(header.block_hash()))
@@ -287,10 +287,8 @@ impl IndexedChain {
         for batch in batches {
             self.headers.add(batch.header);
         }
-
-        stats.elapsed = t.elapsed();
         if stats.indexed_blocks > 0 {
-            self.store.flush()?;
+            stats.elapsed = t.elapsed();
             info!(
                 "block={} height={}: indexed {} blocks, {:.3}[MB], dt = {:.3}[s]: {:.3} [ms/block], {:.3} [MB/block], {:.3} [MB/s]",
                 self.headers.tip_hash(),
@@ -302,6 +300,7 @@ impl IndexedChain {
                 stats.size_read as f64 / (1e6 * stats.indexed_blocks as f64),
                 stats.size_read as f64 / (1e6 * stats.elapsed.as_secs_f64()),
             );
+            self.store.flush()?;
         } else {
             // Start autocompactions when there are no new indexed blocks
             self.store.start_compactions()?;

bindex-lib/src/db.rs

@@ -35,8 +35,9 @@ const HEADERS_CF: &str = "headers";
 const SCRIPT_HASH_CF: &str = "script_hash";
 const TXPOS_CF: &str = "txpos";
 const TXID_CF: &str = "txid";
+const SPTWEAK_CF: &str = "sptweak";
 
-const COLUMN_FAMILIES: &[&str] = &[HEADERS_CF, TXPOS_CF, TXID_CF, SCRIPT_HASH_CF];
+const COLUMN_FAMILIES: &[&str] = &[HEADERS_CF, TXPOS_CF, TXID_CF, SPTWEAK_CF, SCRIPT_HASH_CF];
 
 fn cf_descriptors(
     opts: &rocksdb::Options,
@@ -114,6 +115,18 @@ impl DB {
             .map(index::TxBlockPosRow::serialize)
             .for_each(|(k, v)| f(&mut write_batch, cf, &k, &v));
 
+        // key = next_txnum || txid_prefix || tweak
+        let cf = self.cf(SPTWEAK_CF);
+        // Batch are grouped by `txnum`.
+        for batch in batches {
+            let mut rows: Vec<&index::TxTweakRow> = batch.sptweak_rows.iter().collect();
+            // Sort row group by `txid` prefix.
+            rows.sort_unstable_by_key(|item| &item.txid);
+            for row in rows {
+                f(&mut write_batch, cf, &row.serialize(&batch.header), b"");
+            }
+        }
+
         // key = next_txnum, value = blockhash || blockheader
         let cf = self.cf(HEADERS_CF);
         for batch in batches {

bindex-lib/src/index/mod.rs

@@ -1,5 +1,6 @@
 mod header;
 mod scripthash;
+mod sptweak;
 mod txid;
 mod txpos;
 
@@ -8,6 +9,7 @@ use bitcoin_slices::bsl;
 
 pub use header::IndexedHeader;
 pub use scripthash::ScriptHash;
+pub use sptweak::TxTweakRow;
 pub use txpos::{TxBlockPos, TxBlockPosRow};
 
 #[derive(thiserror::Error, Debug)]
@@ -150,6 +152,14 @@ impl BlockBytes {
     pub fn txs_count(&self) -> u32 {
         parse_txs_count(&self.0[bitcoin::block::Header::SIZE..])
     }
+
+    #[cfg(test)]
+    pub fn txdata(&self) -> Vec<bitcoin::Transaction> {
+        use bitcoin::consensus::encode::deserialize;
+
+        let block: bitcoin::Block = deserialize(&self.0).expect("invalid block");
+        block.txdata
+    }
 }
 
 pub struct SpentBytes(Vec<u8>);
@@ -194,6 +204,7 @@ pub struct Batch {
     pub scripthash_rows: Vec<HashPrefixRow>,
     pub txid_rows: Vec<HashPrefixRow>,
     pub txpos_rows: Vec<txpos::TxBlockPosRow>,
+    pub sptweak_rows: Vec<sptweak::TxTweakRow>,
     pub header: IndexedHeader,
 }
 
@@ -208,16 +219,19 @@ impl Batch {
         let scripthash = scripthash::index(block, spent, first_txnum)?;
         let txpos = txpos::index(block, first_txnum)?;
         let txid = txid::index(block, first_txnum)?;
+        let sptweak = sptweak::index(block, spent, first_txnum)?;
 
         // All must have the same number of transactions
         assert_eq!(txnum_range.next, scripthash.next_txnum);
         assert_eq!(txnum_range.next, txpos.next_txnum);
         assert_eq!(txnum_range.next, txid.next_txnum);
+        assert_eq!(txnum_range.next, sptweak.next_txnum);
 
         Ok(Batch {
             scripthash_rows: scripthash.rows,
             txpos_rows: txpos.rows,
             txid_rows: txid.rows,
+            sptweak_rows: sptweak.rows,
             header: IndexedHeader::new(txnum_range.next, blockhash, block),
         })
     }