more docs

Author: adriangb

parquet/src/bloom_filter/mod.rs

@@ -68,11 +68,45 @@
 //! | 1,000,000 | 0.00001   | 131,072 | 4,096     |
 //! | 1,000,000 | 0.000001  | 262,144 | 8,192     |
 //!
+//! # Structure: Filter → Blocks → Words → Bits
+//!
+//! An SBBF is an array of **blocks**. Each block is 256 bits (32 bytes),
+//! divided into eight 32-bit **words**. A word is just a `u32` — an array of
+//! 32 individual bits that can each be "set" (1) or "not set" (0).
+//!
+//! ```text
+//!   Sbbf (the whole filter)
+//!   ┌──────────┬──────────┬──────────┬─── ─── ──┬──────────┐
+//!   │ Block 0  │ Block 1  │ Block 2  │   ...    │ Block N-1│
+//!   └──────────┴──────────┴──────────┴─── ─── ──┴──────────┘
+//!        │
+//!        ▼
+//!   One Block = 256 bits = 8 words
+//!   ┌────────┬────────┬────────┬────────┬────────┬────────┬────────┬────────┐
+//!   │ word 0 │ word 1 │ word 2 │ word 3 │ word 4 │ word 5 │ word 6 │ word 7 │
+//!   │ (u32)  │ (u32)  │ (u32)  │ (u32)  │ (u32)  │ (u32)  │ (u32)  │ (u32)  │
+//!   └────────┴────────┴────────┴────────┴────────┴────────┴────────┴────────┘
+//!        │
+//!        ▼
+//!   One Word = 32 individual bits
+//!   ┌─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┬─┐
+//!   │0│0│1│0│0│0│0│0│0│0│0│0│0│0│0│0│0│0│0│0│0│0│0│0│0│0│0│0│0│0│0│0│  ← bit 29 is set
+//!   └─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┘
+//! ```
+//!
+//! **Inserting** a value hashes it to a 64-bit number, then:
+//!  1. The upper 32 bits pick which **block** (via `Sbbf::hash_to_block_index`).
+//!  2. The lower 32 bits pick one bit position in each of the 8 **words** (via `Block::mask`).
+//!     So each insert sets exactly **8 bits** (one per word) in a single block.
+//!
+//! **Checking** does the same two steps and returns `true` only if all 8 bits
+//! are already set — meaning the value was *probably* inserted (or is a false
+//! positive).
+//!
 //! # Bloom Filter Folding
 //!
-//! When the NDV is not known ahead of time, bloom filters support a **folding mode** that
-//! eliminates the need to guess NDV upfront. See [`Sbbf::fold_to_target_fpp`] for details
-//! on the algorithm and its mathematical basis.
+//! After inserting all values into a bloom filter it can be "folded" to minimize it's size.
+//! See [`Sbbf::fold_to_target_fpp`] for details  on the algorithm and its mathematical basis.
 //!
 //! [parquet-bf-spec]: https://github.com/apache/parquet-format/blob/master/BloomFilter.md
 //! [sbbf-paper]: https://arxiv.org/pdf/2101.01719
@@ -120,8 +154,22 @@ pub struct BloomFilterHeader {
 }
 );
 
-/// Each block is 256 bits, broken up into eight contiguous "words", each consisting of 32 bits.
-/// Each word is thought of as an array of bits; each bit is either "set" or "not set".
+/// A single 256-bit block, the basic unit of the Split Block Bloom Filter.
+///
+/// A block is eight contiguous 32-bit **words** (`[u32; 8]`).
+/// Each word is an independent bit-array of 32 positions:
+///
+/// ```text
+///   Block (256 bits total)
+///   ┌────────┬────────┬────────┬────────┬────────┬────────┬────────┬────────┐
+///   │ word 0 │ word 1 │ word 2 │ word 3 │ word 4 │ word 5 │ word 6 │ word 7 │
+///   │ 32 bits│ 32 bits│ 32 bits│ 32 bits│ 32 bits│ 32 bits│ 32 bits│ 32 bits│
+///   └────────┴────────┴────────┴────────┴────────┴────────┴────────┴────────┘
+/// ```
+///
+/// When a value is inserted, [`Block::mask`] picks one bit in each word
+/// (8 bits total), and those bits are OR'd in. When checking, we verify
+/// all 8 bits are set.
 #[derive(Debug, Copy, Clone)]
 #[repr(transparent)]
 struct Block([u32; 8]);
@@ -561,11 +609,12 @@ impl Sbbf {
     /// Folding **never introduces false negatives**. Every bit that was set in the original
     /// filter remains set in the folded filter (via bitwise OR). The only effect is a controlled
     /// increase in FPP as set bits from different blocks are merged together.
+    /// This is was originally proven in [Sailhan & Stehr 2012] for standard bloom filters and is empirically
+    /// demonstrated for SBBFs in Lemma 1 and Lemma 2 of the tests.
     ///
     /// ## References
     ///
-    /// - Sailhan, F. & Stehr, M-O. "Folding and Unfolding Bloom Filters",
-    ///   IEEE iThings 2012. <https://doi.org/10.1109/GreenCom.2012.16>
+    /// [Sailhan & Stehr 2012]: https://doi.org/10.1109/GreenCom.2012.16
     pub fn fold_to_target_fpp(&mut self, target_fpp: f64) {
         let num_folds = self.num_folds_for_target_fpp(target_fpp);
         if num_folds > 0 {