From 27f9a3cd5ffe1364b51876682dd91ff5c871588d Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Tue, 27 May 2025 18:04:34 +0200 Subject: [PATCH 01/60] feat: start --- examples/insert.rs | 14 ++++++-------- src/findex.rs | 27 ++++++++++++++++----------- src/test_utils/benches.rs | 1 - 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/examples/insert.rs b/examples/insert.rs index 0a97cadc..a9cc3a6e 100644 --- a/examples/insert.rs +++ b/examples/insert.rs @@ -5,7 +5,6 @@ use cosmian_crypto_core::{ reexport::rand_core::{CryptoRngCore, SeedableRng}, }; use cosmian_findex::{Findex, InMemory, IndexADT, MemoryEncryptionLayer, Op}; -use futures::executor::block_on; use std::collections::{HashMap, HashSet}; /// This function generates a random set of (key, values) couples. Since Findex @@ -85,7 +84,8 @@ fn decoder(words: Vec<[u8; WORD_LENGTH]>) -> Result, String> { Ok(values) } -fn main() { +#[tokio::main] +async fn main() { // For cryptographic applications, it is important to use a secure RNG. In // Rust, those RNG implement the `CryptoRng` trait. let mut rng = CsRng::from_entropy(); @@ -115,17 +115,15 @@ fn main() { // Here we insert all bindings one by one, blocking on each call. A better // way would be to performed all such calls in parallel using tasks. - index - .clone() - .into_iter() - .for_each(|(kw, vs)| block_on(findex.insert(kw, vs)).expect("insert failed")); - + for (kw, vs) in index.clone().into_iter() { + findex.insert(kw, vs).await.expect("insert failed"); + } // In order to verify insertion was correctly performed, we search for all // the indexed keywords... let res = index .keys() .cloned() - .map(|kw| (kw, block_on(findex.search(&kw)).expect("search failed"))) + .map(|kw| (kw, findex.search(&kw).await.expect("search failed"))) .collect::>(); // ... and verify we get the whole index back! diff --git a/src/findex.rs b/src/findex.rs index 418ccfe5..3134f980 100644 --- a/src/findex.rs +++ b/src/findex.rs @@ -141,11 +141,10 @@ mod tests { memory::MemoryEncryptionLayer, }; use cosmian_crypto_core::{CsRng, Secret, define_byte_type, reexport::rand_core::SeedableRng}; - use futures::executor::block_on; use std::collections::HashSet; - #[test] - fn test_insert_search_delete_search() { + #[tokio::test] + async fn test_insert_search_delete_search() { // Define a byte type, and use `Value` as an alias for 8-bytes values of // that type. type Value = Bytes<8>; @@ -178,10 +177,16 @@ mod tests { Value::try_from(2).unwrap(), Value::try_from(4).unwrap(), ]; - block_on(findex.insert("cat".to_string(), cat_bindings.clone())).unwrap(); - block_on(findex.insert("dog".to_string(), dog_bindings.clone())).unwrap(); - let cat_res = block_on(findex.search(&"cat".to_string())).unwrap(); - let dog_res = block_on(findex.search(&"dog".to_string())).unwrap(); + findex + .insert("cat".to_string(), cat_bindings.clone()) + .await + .unwrap(); + findex + .insert("dog".to_string(), dog_bindings.clone()) + .await + .unwrap(); + let cat_res = findex.search(&"cat".to_string()).await.unwrap(); + let dog_res = findex.search(&"dog".to_string()).await.unwrap(); assert_eq!( cat_bindings.iter().cloned().collect::>(), cat_res @@ -191,10 +196,10 @@ mod tests { dog_res ); - block_on(findex.delete("dog", dog_bindings)).unwrap(); - block_on(findex.delete("cat", cat_bindings)).unwrap(); - let cat_res = block_on(findex.search(&"cat".to_string())).unwrap(); - let dog_res = block_on(findex.search(&"dog".to_string())).unwrap(); + findex.delete("dog", dog_bindings).await.unwrap(); + findex.delete("cat", cat_bindings).await.unwrap(); + let cat_res = findex.search(&"cat".to_string()).await.unwrap(); + let dog_res = findex.search(&"dog".to_string()).await.unwrap(); assert_eq!(HashSet::new(), cat_res); assert_eq!(HashSet::new(), dog_res); } diff --git a/src/test_utils/benches.rs b/src/test_utils/benches.rs index c5b01aa7..073ebdcc 100644 --- a/src/test_utils/benches.rs +++ b/src/test_utils/benches.rs @@ -4,7 +4,6 @@ use crate::{ }; use cosmian_crypto_core::{Secret, reexport::rand_core::CryptoRngCore}; use criterion::{BenchmarkId, Criterion}; -use futures::future::join_all; use std::{collections::HashSet, fmt::Debug, sync::Arc}; use tokio::runtime::{Builder, Runtime}; From 887f7ad9210c8503882edc2decc7d0330046d0a2 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Wed, 28 May 2025 12:21:08 +0200 Subject: [PATCH 02/60] feat: finish --- Cargo.toml | 4 +--- examples/insert.rs | 10 +++++----- src/adt.rs | 7 +++---- src/test_utils/benches.rs | 20 +++++++++++++------- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index aec0d3d9..172fb389 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ path = "src/lib.rs" rust-mem = [] redis-mem = ["redis"] sqlite-mem = ["async-sqlite"] -test-utils = ["tokio", "criterion", "futures", "rand", "rand_distr"] +test-utils = ["tokio", "criterion", "rand", "rand_distr"] postgres-mem = ["tokio-postgres", "tokio", "deadpool-postgres"] [dependencies] @@ -37,7 +37,6 @@ xts-mode = "0.5" # Used in benches and tests. criterion = { version = "0.5", optional = true } -futures = { version = "0.3", optional = true } rand = { version = "0.9.0", optional = true } rand_distr = { version = "0.5.1", optional = true } tokio = { version = "1.44", features = ["rt-multi-thread"], optional = true } @@ -57,7 +56,6 @@ tokio-postgres = { version = "0.7.9", optional = true, features = [ [dev-dependencies] -futures = { version = "0.3" } tokio = { version = "1.44", features = ["macros", "rt-multi-thread"] } [[bench]] diff --git a/examples/insert.rs b/examples/insert.rs index a9cc3a6e..d7d4ccd8 100644 --- a/examples/insert.rs +++ b/examples/insert.rs @@ -120,11 +120,11 @@ async fn main() { } // In order to verify insertion was correctly performed, we search for all // the indexed keywords... - let res = index - .keys() - .cloned() - .map(|kw| (kw, findex.search(&kw).await.expect("search failed"))) - .collect::>(); + let mut res = HashMap::new(); + for kw in index.keys().cloned() { + let values = findex.search(&kw).await.expect("search failed"); + res.insert(kw, values); + } // ... and verify we get the whole index back! assert_eq!(res, index); diff --git a/src/adt.rs b/src/adt.rs index cc381716..b5df02b9 100644 --- a/src/adt.rs +++ b/src/adt.rs @@ -85,7 +85,6 @@ pub mod tests { //! This module defines tests any implementation of the VectorADT interface must pass. use crate::adt::VectorADT; - use futures::{executor::block_on, future::join_all}; /// Adding information from different copies of the same vector should be visible by all /// copies. @@ -127,10 +126,10 @@ pub mod tests { }) }) .collect::>(); - for h in join_all(handles).await { - h.unwrap(); + for h in handles { + h.await.unwrap(); } - let mut res = block_on(v.read()).unwrap(); + let mut res = v.read().await.unwrap(); let old = res.clone(); res.sort(); assert_ne!(old, res); diff --git a/src/test_utils/benches.rs b/src/test_utils/benches.rs index 073ebdcc..df5c0ef2 100644 --- a/src/test_utils/benches.rs +++ b/src/test_utils/benches.rs @@ -138,8 +138,8 @@ pub fn bench_memory_search_multiple_keywords< let findex = findex.clone(); handles.push(tokio::spawn(async move { findex.search(&kw).await })) } - for res in join_all(handles).await { - res.unwrap().unwrap(); + for res in handles { + res.await.expect("Search task failed").unwrap(); } }) }, @@ -255,10 +255,14 @@ pub fn bench_memory_contention< }, |iterator| { rt.block_on(async { - join_all(iterator.map(|(findex, (kw, vs))| { - tokio::spawn(async move { findex.insert(kw, vs).await }) - })) - .await + let handles = iterator + .map(|(findex, (kw, vs))| { + tokio::spawn(async move { findex.insert(kw, vs).await }) + }) + .collect::>(); + for h in handles { + h.await.expect("Insert task failed"); + } }) }, criterion::BatchSize::SmallInput, @@ -296,7 +300,9 @@ pub fn bench_memory_contention< let h = tokio::spawn(async move { findex.insert(kw, vs).await }); handles.push(h); } - join_all(handles).await + for h in handles { + h.await.expect("Insert task failed").unwrap(); + } }) }, criterion::BatchSize::SmallInput, From 6e1015c8664f43832571ed1f73d8a725ad7e46f2 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Wed, 28 May 2025 12:28:53 +0200 Subject: [PATCH 03/60] feat: lint --- src/test_utils/benches.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test_utils/benches.rs b/src/test_utils/benches.rs index df5c0ef2..d5143dae 100644 --- a/src/test_utils/benches.rs +++ b/src/test_utils/benches.rs @@ -261,7 +261,7 @@ pub fn bench_memory_contention< }) .collect::>(); for h in handles { - h.await.expect("Insert task failed"); + h.await.expect("Insert task failed").unwrap(); } }) }, From 60cacc49d64d56e1a63129b30ec29d9da29d27e5 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Wed, 28 May 2025 14:16:32 +0200 Subject: [PATCH 04/60] fix: downgrade ci toolchain to 1.85 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 009d7719..c8efde84 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,14 +7,14 @@ jobs: cargo-lint: uses: Cosmian/reusable_workflows/.github/workflows/cargo-nursery.yml@develop with: - toolchain: stable + toolchain: 1.85.0 cargo-publish: needs: - cargo-lint uses: Cosmian/reusable_workflows/.github/workflows/cargo-publish.yml@develop if: startsWith(github.ref, 'refs/tags/') with: - toolchain: stable + toolchain: 1.85.0 secrets: inherit cleanup: needs: From 3f7fb2956624ba898e54955009d8cc2b3ec54fef Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Wed, 28 May 2025 17:06:21 +0200 Subject: [PATCH 05/60] Update src/adt.rs fix: lint --- src/adt.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/adt.rs b/src/adt.rs index b5df02b9..8dc3bc8a 100644 --- a/src/adt.rs +++ b/src/adt.rs @@ -127,7 +127,8 @@ pub mod tests { }) .collect::>(); for h in handles { - h.await.unwrap(); + h.await + .expect("Join handle failed during test_vector_concurrent"); } let mut res = v.read().await.unwrap(); let old = res.clone(); From 7f36f50c5508ff84f6087d30db78f25b3affa7ab Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Fri, 30 May 2025 11:59:10 +0200 Subject: [PATCH 06/60] fix: bump criterion --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 172fb389..d1ef78f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,7 @@ cosmian_crypto_core = { version = "10.1", default-features = false, features = [ xts-mode = "0.5" # Used in benches and tests. -criterion = { version = "0.5", optional = true } +criterion = { version = "0.6", optional = true } rand = { version = "0.9.0", optional = true } rand_distr = { version = "0.5.1", optional = true } tokio = { version = "1.44", features = ["rt-multi-thread"], optional = true } From d6295d594f432330066a3b2779ea02ead11705f1 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Mon, 2 Jun 2025 15:36:20 +0200 Subject: [PATCH 07/60] feat: generic runtime --- Cargo.toml | 11 ++++----- src/memory/encryption_layer.rs | 6 +++-- src/memory/in_memory_store.rs | 7 ++++-- src/memory/postgresql_store/memory.rs | 10 ++++++-- src/memory/redis_store.rs | 7 +++--- src/memory/sqlite_store.rs | 4 ++-- src/test_utils/benches.rs | 3 +++ src/test_utils/memory_tests.rs | 34 ++++++++++++++++++++++++--- 8 files changed, 62 insertions(+), 20 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d1ef78f8..53c45c32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ path = "src/lib.rs" rust-mem = [] redis-mem = ["redis"] sqlite-mem = ["async-sqlite"] -test-utils = ["tokio", "criterion", "rand", "rand_distr"] +test-utils = ["criterion", "rand", "rand_distr"] postgres-mem = ["tokio-postgres", "tokio", "deadpool-postgres"] [dependencies] @@ -35,12 +35,10 @@ cosmian_crypto_core = { version = "10.1", default-features = false, features = [ ] } xts-mode = "0.5" -# Used in benches and tests. +# Used only in benches and tests criterion = { version = "0.6", optional = true } rand = { version = "0.9.0", optional = true } rand_distr = { version = "0.5.1", optional = true } -tokio = { version = "1.44", features = ["rt-multi-thread"], optional = true } - # Memory dependencies async-sqlite = { version = "0.5", optional = true } @@ -50,9 +48,10 @@ redis = { version = "0.28", features = [ "connection-manager", "tokio-comp", ], optional = true } -tokio-postgres = { version = "0.7.9", optional = true, features = [ +tokio = { version = "1.44", features = ["rt-multi-thread"], optional = true } +tokio-postgres = { version = "0.7.9", features = [ "array-impls", -] } +], optional = true } [dev-dependencies] diff --git a/src/memory/encryption_layer.rs b/src/memory/encryption_layer.rs index 57254773..2d4b0396 100644 --- a/src/memory/encryption_layer.rs +++ b/src/memory/encryption_layer.rs @@ -142,7 +142,8 @@ mod tests { address::Address, memory::{MemoryEncryptionLayer, in_memory_store::InMemory}, test_utils::{ - gen_seed, test_guarded_write_concurrent, test_single_write_and_read, test_wrong_guard, + TokioSpawner, gen_seed, test_guarded_write_concurrent, test_single_write_and_read, + test_wrong_guard, }, }; @@ -184,6 +185,7 @@ mod tests { #[tokio::test] async fn test_concurrent_read_write() { let mem = create_memory(&mut CsRng::from_entropy()); - test_guarded_write_concurrent::(&mem, gen_seed(), None).await; + test_guarded_write_concurrent::(&mem, gen_seed(), None, &TokioSpawner) + .await; } } diff --git a/src/memory/in_memory_store.rs b/src/memory/in_memory_store.rs index 78ca9b9b..61b8043b 100644 --- a/src/memory/in_memory_store.rs +++ b/src/memory/in_memory_store.rs @@ -96,7 +96,10 @@ mod tests { use super::InMemory; use crate::{ test_utils::gen_seed, - test_utils::{test_guarded_write_concurrent, test_single_write_and_read, test_wrong_guard}, + test_utils::{ + TokioSpawner, test_guarded_write_concurrent, test_single_write_and_read, + test_wrong_guard, + }, }; #[tokio::test] @@ -114,6 +117,6 @@ mod tests { #[tokio::test] async fn test_concurrent_read_write() { let memory = InMemory::<[u8; 16], [u8; 16]>::default(); - test_guarded_write_concurrent(&memory, gen_seed(), None).await; + test_guarded_write_concurrent(&memory, gen_seed(), None, &TokioSpawner).await; } } diff --git a/src/memory/postgresql_store/memory.rs b/src/memory/postgresql_store/memory.rs index 1ed592ef..1f7577fb 100644 --- a/src/memory/postgresql_store/memory.rs +++ b/src/memory/postgresql_store/memory.rs @@ -242,7 +242,7 @@ mod tests { use super::*; use crate::{ - ADDRESS_LENGTH, Address, WORD_LENGTH, + ADDRESS_LENGTH, Address, TokioSpawner, WORD_LENGTH, test_utils::{ gen_seed, test_guarded_write_concurrent, test_rw_same_address, test_single_write_and_read, test_wrong_guard, @@ -354,7 +354,13 @@ mod tests { #[tokio::test] async fn test_rw_ccr() -> Result<(), PostgresMemoryError> { setup_and_run_test("findex_test_rw_ccr", |m| async move { - test_guarded_write_concurrent::(&m, gen_seed(), Some(100)).await; + test_guarded_write_concurrent::( + &m, + gen_seed(), + Some(100), + &TokioSpawner, + ) + .await; }) .await } diff --git a/src/memory/redis_store.rs b/src/memory/redis_store.rs index 5d4801be..2cc509b4 100644 --- a/src/memory/redis_store.rs +++ b/src/memory/redis_store.rs @@ -149,8 +149,8 @@ mod tests { WORD_LENGTH, test_utils::gen_seed, { - test_guarded_write_concurrent, test_rw_same_address, test_single_write_and_read, - test_wrong_guard, + TokioSpawner, test_guarded_write_concurrent, test_rw_same_address, + test_single_write_and_read, test_wrong_guard, }, }; @@ -182,6 +182,7 @@ mod tests { #[tokio::test] async fn test_rw_ccr() { let m = RedisMemory::connect(&get_redis_url()).await.unwrap(); - test_guarded_write_concurrent::(&m, gen_seed(), None).await + test_guarded_write_concurrent::(&m, gen_seed(), None, &TokioSpawner) + .await } } diff --git a/src/memory/sqlite_store.rs b/src/memory/sqlite_store.rs index 36d4f12e..56e06d52 100644 --- a/src/memory/sqlite_store.rs +++ b/src/memory/sqlite_store.rs @@ -176,7 +176,7 @@ mod tests { use super::*; use crate::{ - WORD_LENGTH, gen_seed, test_guarded_write_concurrent, test_rw_same_address, + TokioSpawner, WORD_LENGTH, gen_seed, test_guarded_write_concurrent, test_rw_same_address, test_single_write_and_read, test_wrong_guard, }; @@ -211,6 +211,6 @@ mod tests { let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH) .await .unwrap(); - test_guarded_write_concurrent(&m, gen_seed(), Some(100)).await + test_guarded_write_concurrent(&m, gen_seed(), Some(100), &TokioSpawner).await } } diff --git a/src/test_utils/benches.rs b/src/test_utils/benches.rs index d5143dae..aecfef49 100644 --- a/src/test_utils/benches.rs +++ b/src/test_utils/benches.rs @@ -1,3 +1,6 @@ +//! This module provides a comprehensive benchmarking suite for testing the performance of Findex memory implementations. +//! These benchmarks are designed to be generic and work with any memory backend that implements the MemoryADT trait. +//! The [findex-memories](https://github.com/Cosmian/findex-memories) crate provides a set of memory backends that have been benched. use crate::{ ADDRESS_LENGTH, Address, Findex, IndexADT, MemoryADT, MemoryEncryptionLayer, WORD_LENGTH, dummy_decode, dummy_encode, diff --git a/src/test_utils/memory_tests.rs b/src/test_utils/memory_tests.rs index 763d3082..d40a9e05 100644 --- a/src/test_utils/memory_tests.rs +++ b/src/test_utils/memory_tests.rs @@ -13,7 +13,7 @@ use cosmian_crypto_core::{ CsRng, reexport::rand_core::{RngCore, SeedableRng}, }; -use std::fmt::Debug; +use std::{fmt::Debug, future::Future}; pub const SEED_LENGTH: usize = 32; @@ -218,6 +218,32 @@ pub async fn test_rw_same_address( ); } +/// Trait for spawning async tasks in a runtime-agnostic way +pub trait TaskSpawner: Send + Sync { + type JoinHandle: Future> + Send; + type JoinError: std::error::Error + Send; + + fn spawn(&self, future: F) -> Self::JoinHandle + where + F: Future + Send + 'static, + F::Output: Send + 'static; +} + +pub struct TokioSpawner; + +impl TaskSpawner for TokioSpawner { + type JoinHandle = tokio::task::JoinHandle; + type JoinError = tokio::task::JoinError; + + fn spawn(&self, future: F) -> Self::JoinHandle + where + F: Future + Send + 'static, + F::Output: Send + 'static, + { + tokio::spawn(future) + } +} + /// Tests concurrent guarded write operations on a Memory ADT implementation. /// /// Spawns multiple threads to perform concurrent counter increments. @@ -230,16 +256,18 @@ pub async fn test_rw_same_address( /// * `memory` - The Memory ADT implementation to test. /// * `seed` - The seed used to initialize the random number generator. /// * `n_threads` - The number of threads to spawn. If None, defaults to 100. -pub async fn test_guarded_write_concurrent( +pub async fn test_guarded_write_concurrent( memory: &Memory, seed: [u8; SEED_LENGTH], n_threads: Option, + spawer: &S, ) where Memory: 'static + Send + Sync + MemoryADT + Clone, Memory::Address: Send + From<[u8; ADDRESS_LENGTH]>, Memory::Word: Send + Debug + PartialEq + From<[u8; WORD_LENGTH]> + Into<[u8; WORD_LENGTH]> + Clone, Memory::Error: Send + std::error::Error, + S: TaskSpawner, { // A worker increment N times the counter m[a]. async fn worker( @@ -290,7 +318,7 @@ pub async fn test_guarded_write_concurrent( let handles = (0..n) .map(|_| { let m = memory.clone(); - tokio::spawn(worker(m, a)) + spawer.spawn(worker(m, a)) }) .collect::>(); From 4b715ef756e3b3c488cc0067777f54526fda9869 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Mon, 2 Jun 2025 15:55:09 +0200 Subject: [PATCH 08/60] feat: restaure tokio in the test --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 53c45c32..d13fe7fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ path = "src/lib.rs" rust-mem = [] redis-mem = ["redis"] sqlite-mem = ["async-sqlite"] -test-utils = ["criterion", "rand", "rand_distr"] +test-utils = ["tokio", "criterion", "rand", "rand_distr"] postgres-mem = ["tokio-postgres", "tokio", "deadpool-postgres"] [dependencies] @@ -39,6 +39,7 @@ xts-mode = "0.5" criterion = { version = "0.6", optional = true } rand = { version = "0.9.0", optional = true } rand_distr = { version = "0.5.1", optional = true } +tokio = { version = "1.44", features = ["rt-multi-thread"], optional = true } # Memory dependencies async-sqlite = { version = "0.5", optional = true } @@ -48,7 +49,6 @@ redis = { version = "0.28", features = [ "connection-manager", "tokio-comp", ], optional = true } -tokio = { version = "1.44", features = ["rt-multi-thread"], optional = true } tokio-postgres = { version = "0.7.9", features = [ "array-impls", ], optional = true } From 7860579c224988277ed86434f5b6189c41ee0108 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Tue, 3 Jun 2025 12:49:22 +0200 Subject: [PATCH 09/60] chore: move files, no cargo toml --- {src => crate/findex/src}/address.rs | 0 {src => crate/findex/src}/adt.rs | 0 {src => crate/findex/src}/encoding.rs | 0 {src => crate/findex/src}/error.rs | 0 {src => crate/findex/src}/findex.rs | 0 {src => crate/findex/src}/lib.rs | 0 {src => crate/findex/src}/memory/encryption_layer.rs | 0 {src => crate/findex/src}/memory/in_memory_store.rs | 0 {src => crate/findex/src}/memory/mod.rs | 0 {src => crate/findex/src}/memory/postgresql_store/error.rs | 0 {src => crate/findex/src}/memory/postgresql_store/memory.rs | 0 {src => crate/findex/src}/memory/postgresql_store/mod.rs | 0 {src => crate/findex/src}/memory/redis_store.rs | 0 {src => crate/findex/src}/memory/sqlite_store.rs | 0 {src => crate/findex/src}/ovec.rs | 0 {src => crate/findex/src}/test_utils/benches.rs | 0 {src => crate/findex/src}/test_utils/memory_tests.rs | 0 {src => crate/findex/src}/test_utils/mod.rs | 0 18 files changed, 0 insertions(+), 0 deletions(-) rename {src => crate/findex/src}/address.rs (100%) rename {src => crate/findex/src}/adt.rs (100%) rename {src => crate/findex/src}/encoding.rs (100%) rename {src => crate/findex/src}/error.rs (100%) rename {src => crate/findex/src}/findex.rs (100%) rename {src => crate/findex/src}/lib.rs (100%) rename {src => crate/findex/src}/memory/encryption_layer.rs (100%) rename {src => crate/findex/src}/memory/in_memory_store.rs (100%) rename {src => crate/findex/src}/memory/mod.rs (100%) rename {src => crate/findex/src}/memory/postgresql_store/error.rs (100%) rename {src => crate/findex/src}/memory/postgresql_store/memory.rs (100%) rename {src => crate/findex/src}/memory/postgresql_store/mod.rs (100%) rename {src => crate/findex/src}/memory/redis_store.rs (100%) rename {src => crate/findex/src}/memory/sqlite_store.rs (100%) rename {src => crate/findex/src}/ovec.rs (100%) rename {src => crate/findex/src}/test_utils/benches.rs (100%) rename {src => crate/findex/src}/test_utils/memory_tests.rs (100%) rename {src => crate/findex/src}/test_utils/mod.rs (100%) diff --git a/src/address.rs b/crate/findex/src/address.rs similarity index 100% rename from src/address.rs rename to crate/findex/src/address.rs diff --git a/src/adt.rs b/crate/findex/src/adt.rs similarity index 100% rename from src/adt.rs rename to crate/findex/src/adt.rs diff --git a/src/encoding.rs b/crate/findex/src/encoding.rs similarity index 100% rename from src/encoding.rs rename to crate/findex/src/encoding.rs diff --git a/src/error.rs b/crate/findex/src/error.rs similarity index 100% rename from src/error.rs rename to crate/findex/src/error.rs diff --git a/src/findex.rs b/crate/findex/src/findex.rs similarity index 100% rename from src/findex.rs rename to crate/findex/src/findex.rs diff --git a/src/lib.rs b/crate/findex/src/lib.rs similarity index 100% rename from src/lib.rs rename to crate/findex/src/lib.rs diff --git a/src/memory/encryption_layer.rs b/crate/findex/src/memory/encryption_layer.rs similarity index 100% rename from src/memory/encryption_layer.rs rename to crate/findex/src/memory/encryption_layer.rs diff --git a/src/memory/in_memory_store.rs b/crate/findex/src/memory/in_memory_store.rs similarity index 100% rename from src/memory/in_memory_store.rs rename to crate/findex/src/memory/in_memory_store.rs diff --git a/src/memory/mod.rs b/crate/findex/src/memory/mod.rs similarity index 100% rename from src/memory/mod.rs rename to crate/findex/src/memory/mod.rs diff --git a/src/memory/postgresql_store/error.rs b/crate/findex/src/memory/postgresql_store/error.rs similarity index 100% rename from src/memory/postgresql_store/error.rs rename to crate/findex/src/memory/postgresql_store/error.rs diff --git a/src/memory/postgresql_store/memory.rs b/crate/findex/src/memory/postgresql_store/memory.rs similarity index 100% rename from src/memory/postgresql_store/memory.rs rename to crate/findex/src/memory/postgresql_store/memory.rs diff --git a/src/memory/postgresql_store/mod.rs b/crate/findex/src/memory/postgresql_store/mod.rs similarity index 100% rename from src/memory/postgresql_store/mod.rs rename to crate/findex/src/memory/postgresql_store/mod.rs diff --git a/src/memory/redis_store.rs b/crate/findex/src/memory/redis_store.rs similarity index 100% rename from src/memory/redis_store.rs rename to crate/findex/src/memory/redis_store.rs diff --git a/src/memory/sqlite_store.rs b/crate/findex/src/memory/sqlite_store.rs similarity index 100% rename from src/memory/sqlite_store.rs rename to crate/findex/src/memory/sqlite_store.rs diff --git a/src/ovec.rs b/crate/findex/src/ovec.rs similarity index 100% rename from src/ovec.rs rename to crate/findex/src/ovec.rs diff --git a/src/test_utils/benches.rs b/crate/findex/src/test_utils/benches.rs similarity index 100% rename from src/test_utils/benches.rs rename to crate/findex/src/test_utils/benches.rs diff --git a/src/test_utils/memory_tests.rs b/crate/findex/src/test_utils/memory_tests.rs similarity index 100% rename from src/test_utils/memory_tests.rs rename to crate/findex/src/test_utils/memory_tests.rs diff --git a/src/test_utils/mod.rs b/crate/findex/src/test_utils/mod.rs similarity index 100% rename from src/test_utils/mod.rs rename to crate/findex/src/test_utils/mod.rs From bd1fc6001f8801a9e0b6d1e66b72e0023ae66965 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Tue, 3 Jun 2025 15:39:35 +0200 Subject: [PATCH 10/60] feat: add the old files --- Cargo.toml | 60 ++- crate/findex/Cargo.toml | 48 +++ crate/findex/src/findex.rs | 1 + crate/findex/src/lib.rs | 9 - crate/findex/src/memory/encryption_layer.rs | 8 +- .../{in_memory_store.rs => in_memory.rs} | 12 +- crate/findex/src/memory/mod.rs | 19 +- .../src/memory/postgresql_store/error.rs | 82 ---- .../src/memory/postgresql_store/memory.rs | 367 ------------------ .../findex/src/memory/postgresql_store/mod.rs | 5 - crate/findex/src/memory/redis_store.rs | 188 --------- crate/findex/src/memory/sqlite_store.rs | 216 ----------- crate/findex/src/test_utils/memory_tests.rs | 34 +- crate/memories/Cargo.toml | 31 ++ gwt | 1 + 15 files changed, 122 insertions(+), 959 deletions(-) create mode 100644 crate/findex/Cargo.toml rename crate/findex/src/memory/{in_memory_store.rs => in_memory.rs} (86%) delete mode 100644 crate/findex/src/memory/postgresql_store/error.rs delete mode 100644 crate/findex/src/memory/postgresql_store/memory.rs delete mode 100644 crate/findex/src/memory/postgresql_store/mod.rs delete mode 100644 crate/findex/src/memory/redis_store.rs delete mode 100644 crate/findex/src/memory/sqlite_store.rs create mode 100644 crate/memories/Cargo.toml create mode 160000 gwt diff --git a/Cargo.toml b/Cargo.toml index d13fe7fc..f9f6d5a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,9 @@ -[package] -name = "cosmian_findex" +[workspace] +default-members = ["crate/findex"] +members = ["crate/findex", "crate/memories"] +resolver = "2" + +[workspace.package] version = "7.1.0" authors = [ "Bruno Grieder ", @@ -16,52 +20,34 @@ license = "BUSL-1.1" repository = "https://github.com/Cosmian/findex/" description = "Symmetric Searchable Encryption" -[lib] -name = "cosmian_findex" -path = "src/lib.rs" - -[features] -rust-mem = [] -redis-mem = ["redis"] -sqlite-mem = ["async-sqlite"] -test-utils = ["tokio", "criterion", "rand", "rand_distr"] -postgres-mem = ["tokio-postgres", "tokio", "deadpool-postgres"] -[dependencies] +[workspace.dependencies] aes = "0.8" cosmian_crypto_core = { version = "10.1", default-features = false, features = [ "macro", "sha3", ] } xts-mode = "0.5" - -# Used only in benches and tests -criterion = { version = "0.6", optional = true } -rand = { version = "0.9.0", optional = true } -rand_distr = { version = "0.5.1", optional = true } -tokio = { version = "1.44", features = ["rt-multi-thread"], optional = true } - -# Memory dependencies -async-sqlite = { version = "0.5", optional = true } -deadpool-postgres = { version = "0.14.1", optional = true } +criterion = { version = "0.6" } +rand = { version = "0.9.0" } +rand_distr = { version = "0.5.1" } +tokio = { version = "1.44" } +async-sqlite = { version = "0.5" } +deadpool-postgres = { version = "0.14.1" } redis = { version = "0.28", features = [ "aio", "connection-manager", "tokio-comp", -], optional = true } -tokio-postgres = { version = "0.7.9", features = [ - "array-impls", -], optional = true } - +] } +tokio-postgres = { version = "0.7.9", features = ["array-impls"] } -[dev-dependencies] -tokio = { version = "1.44", features = ["macros", "rt-multi-thread"] } -[[bench]] -name = "benches" -harness = false -required-features = ["test-utils"] +# TODO: later +# [[bench]] +# name = "benches" +# harness = false +# required-features = ["test-utils"] -[[example]] -name = "insert" -required-features = ["test-utils"] +# [[example]] +# name = "insert" +# required-features = ["test-utils"] diff --git a/crate/findex/Cargo.toml b/crate/findex/Cargo.toml new file mode 100644 index 00000000..0fdc74c8 --- /dev/null +++ b/crate/findex/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "cosmian_findex" +version.workspace = true +authors.workspace = true +categories.workspace = true +edition.workspace = true +keywords.workspace = true +license.workspace = true +repository.workspace = true +description.workspace = true + +[lib] +name = "cosmian_findex" +path = "src/lib.rs" + +[features] +rust-mem = [] +redis-mem = ["redis"] +sqlite-mem = ["async-sqlite"] +postgres-mem = ["tokio-postgres", "tokio", "deadpool-postgres"] +test-utils = ["tokio", "criterion", "rand", "rand_distr"] + +[dependencies] +aes.workspace = true +cosmian_crypto_core.workspace = true +xts-mode.workspace = true + +# Optional dependencies for test-utils +tokio = { workspace = true, features = ["rt-multi-thread"], optional = true } +criterion = { workspace = true, optional = true } +rand = { workspace = true, optional = true } +rand_distr = { workspace = true, optional = true } + +# Memory dependencies +async-sqlite = { version = "0.5", optional = true } +deadpool-postgres = { version = "0.14.1", optional = true } +redis = { version = "0.31", features = [ + "aio", + "connection-manager", + "tokio-comp", +], optional = true } +tokio-postgres = { version = "0.7.9", features = [ + "array-impls", +], optional = true } + + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crate/findex/src/findex.rs b/crate/findex/src/findex.rs index 3134f980..2db627b4 100644 --- a/crate/findex/src/findex.rs +++ b/crate/findex/src/findex.rs @@ -166,6 +166,7 @@ mod tests { &seed, InMemory::, [u8; WORD_LENGTH]>::default(), ); + let findex = Findex::new(memory, dummy_encode::, dummy_decode); let cat_bindings = [ Value::try_from(1).unwrap(), diff --git a/crate/findex/src/lib.rs b/crate/findex/src/lib.rs index b5dc2689..656012a2 100644 --- a/crate/findex/src/lib.rs +++ b/crate/findex/src/lib.rs @@ -29,15 +29,6 @@ pub use memory::{KEY_LENGTH, MemoryEncryptionLayer}; #[cfg(any(test, feature = "test-utils"))] pub use test_utils::*; -#[cfg(feature = "redis-mem")] -pub use memory::{RedisMemory, RedisMemoryError}; - -#[cfg(feature = "sqlite-mem")] -pub use memory::{SqliteMemory, SqliteMemoryError}; - -#[cfg(feature = "postgres-mem")] -pub use memory::{PostgresMemory, PostgresMemoryError}; - #[cfg(any(test, feature = "test-utils"))] pub use encoding::{ dummy_encoding::{WORD_LENGTH, dummy_decode, dummy_encode}, diff --git a/crate/findex/src/memory/encryption_layer.rs b/crate/findex/src/memory/encryption_layer.rs index 2d4b0396..c48025ae 100644 --- a/crate/findex/src/memory/encryption_layer.rs +++ b/crate/findex/src/memory/encryption_layer.rs @@ -140,10 +140,9 @@ mod tests { use crate::{ ADDRESS_LENGTH, address::Address, - memory::{MemoryEncryptionLayer, in_memory_store::InMemory}, + memory::{MemoryEncryptionLayer, in_memory::InMemory}, test_utils::{ - TokioSpawner, gen_seed, test_guarded_write_concurrent, test_single_write_and_read, - test_wrong_guard, + gen_seed, test_guarded_write_concurrent, test_single_write_and_read, test_wrong_guard, }, }; @@ -185,7 +184,6 @@ mod tests { #[tokio::test] async fn test_concurrent_read_write() { let mem = create_memory(&mut CsRng::from_entropy()); - test_guarded_write_concurrent::(&mem, gen_seed(), None, &TokioSpawner) - .await; + test_guarded_write_concurrent::(&mem, gen_seed(), None).await; } } diff --git a/crate/findex/src/memory/in_memory_store.rs b/crate/findex/src/memory/in_memory.rs similarity index 86% rename from crate/findex/src/memory/in_memory_store.rs rename to crate/findex/src/memory/in_memory.rs index 61b8043b..faf5d082 100644 --- a/crate/findex/src/memory/in_memory_store.rs +++ b/crate/findex/src/memory/in_memory.rs @@ -1,3 +1,5 @@ +//! A simple RAM-based implementation of the MemoryADT trait that stores key-value pairs in a thread-safe in-memory HashMap. + use std::{ collections::HashMap, fmt::{Debug, Display}, @@ -97,7 +99,7 @@ mod tests { use crate::{ test_utils::gen_seed, test_utils::{ - TokioSpawner, test_guarded_write_concurrent, test_single_write_and_read, + test_guarded_write_concurrent, test_rw_same_address, test_single_write_and_read, test_wrong_guard, }, }; @@ -114,9 +116,15 @@ mod tests { test_wrong_guard(&memory, gen_seed()).await; } + #[tokio::test] + async fn test_sequential_rw_same_address() { + let memory = InMemory::<[u8; 16], [u8; 16]>::default(); + test_rw_same_address(&memory, gen_seed()).await; + } + #[tokio::test] async fn test_concurrent_read_write() { let memory = InMemory::<[u8; 16], [u8; 16]>::default(); - test_guarded_write_concurrent(&memory, gen_seed(), None, &TokioSpawner).await; + test_guarded_write_concurrent(&memory, gen_seed(), None).await; } } diff --git a/crate/findex/src/memory/mod.rs b/crate/findex/src/memory/mod.rs index 4846f78e..c4efa806 100644 --- a/crate/findex/src/memory/mod.rs +++ b/crate/findex/src/memory/mod.rs @@ -2,21 +2,6 @@ mod encryption_layer; pub use encryption_layer::{KEY_LENGTH, MemoryEncryptionLayer}; #[cfg(any(test, feature = "test-utils"))] -mod in_memory_store; +mod in_memory; #[cfg(any(test, feature = "test-utils"))] -pub use in_memory_store::InMemory; - -#[cfg(feature = "redis-mem")] -mod redis_store; -#[cfg(feature = "redis-mem")] -pub use redis_store::{RedisMemory, RedisMemoryError}; - -#[cfg(feature = "sqlite-mem")] -mod sqlite_store; -#[cfg(feature = "sqlite-mem")] -pub use sqlite_store::{SqliteMemory, SqliteMemoryError}; - -#[cfg(feature = "postgres-mem")] -mod postgresql_store; -#[cfg(feature = "postgres-mem")] -pub use postgresql_store::{PostgresMemory, PostgresMemoryError}; +pub use in_memory::InMemory; diff --git a/crate/findex/src/memory/postgresql_store/error.rs b/crate/findex/src/memory/postgresql_store/error.rs deleted file mode 100644 index 48bd5dd1..00000000 --- a/crate/findex/src/memory/postgresql_store/error.rs +++ /dev/null @@ -1,82 +0,0 @@ -use std::fmt; - -#[derive(Debug)] -pub enum PostgresMemoryError { - TokioPostgresError(deadpool_postgres::tokio_postgres::Error), - TryFromSliceError(std::array::TryFromSliceError), - BuildPoolError(deadpool_postgres::BuildError), - GetConnectionFromPoolError(deadpool_postgres::PoolError), - PoolConfigError(deadpool_postgres::ConfigError), - RetryExhaustedError(usize), - InvalidDataLength(usize), - TableCreationError(u64), -} - -impl std::error::Error for PostgresMemoryError {} - -impl fmt::Display for PostgresMemoryError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::TokioPostgresError(err) => write!(f, "tokio-postgres error: {}", err), - Self::TryFromSliceError(err) => write!(f, "try_from_slice error: {}", err), - Self::InvalidDataLength(len) => { - write!( - f, - "invalid data length: received {} bytes from db instead of WORD_LENGTH bytes", - len - ) - } - Self::PoolConfigError(err) => { - write!(f, "deadpool_postgres error during pool creation: {}", err) - } - Self::BuildPoolError(err) => { - write!(f, "deadpool_postgres error during pool build: {}", err) - } - Self::GetConnectionFromPoolError(err) => write!( - f, - "deadpool_postgres error while trying to get a connection from the pool: {}", - err - ), - Self::RetryExhaustedError(retries) => { - write!(f, "retries exhausted after {} attempts", retries) - } - Self::TableCreationError(err) => { - write!( - f, - "error creating table, {} rows returned while 0 were expected.", - err - ) - } - } - } -} - -impl From for PostgresMemoryError { - fn from(err: deadpool_postgres::tokio_postgres::Error) -> Self { - Self::TokioPostgresError(err) - } -} - -impl From for PostgresMemoryError { - fn from(err: std::array::TryFromSliceError) -> Self { - Self::TryFromSliceError(err) - } -} - -impl From for PostgresMemoryError { - fn from(err: deadpool_postgres::ConfigError) -> Self { - Self::PoolConfigError(err) - } -} - -impl From for PostgresMemoryError { - fn from(err: deadpool_postgres::BuildError) -> Self { - Self::BuildPoolError(err) - } -} - -impl From for PostgresMemoryError { - fn from(err: deadpool_postgres::PoolError) -> Self { - Self::GetConnectionFromPoolError(err) - } -} diff --git a/crate/findex/src/memory/postgresql_store/memory.rs b/crate/findex/src/memory/postgresql_store/memory.rs deleted file mode 100644 index 1f7577fb..00000000 --- a/crate/findex/src/memory/postgresql_store/memory.rs +++ /dev/null @@ -1,367 +0,0 @@ -use super::PostgresMemoryError; -use crate::{Address, MemoryADT}; -use deadpool_postgres::Pool; -use std::marker::PhantomData; -use tokio_postgres::{ - Socket, - tls::{MakeTlsConnect, TlsConnect}, -}; - -#[derive(Clone, Debug)] -pub struct PostgresMemory { - pool: Pool, - table_name: String, - _marker: PhantomData<(Address, Word)>, -} - -impl - PostgresMemory, [u8; WORD_LENGTH]> -{ - /// Connect to a Postgres database and create a table if it doesn't exist - pub async fn initialize_table( - &self, - db_url: String, - table_name: String, - tls: T, - ) -> Result<(), PostgresMemoryError> - where - T: MakeTlsConnect + Send, - T::Stream: Send + 'static, - T::TlsConnect: Send, - >::Future: Send, - { - let (client, connection) = tokio_postgres::connect(&db_url, tls).await?; - - // The connection object performs the actual communication with the database - // `Connection` only resolves when the connection is closed, either because a fatal error has - // occurred, or because its associated `Client` has dropped and all outstanding work has completed. - let conn_handle = tokio::spawn(async move { - if let Err(e) = connection.await { - eprintln!("connection error: {}", e); - } - }); - - let returned = client - .execute( - &format!( - " - CREATE TABLE IF NOT EXISTS {} ( - a BYTEA PRIMARY KEY CHECK (octet_length(a) = {}), - w BYTEA NOT NULL CHECK (octet_length(w) = {}) - );", - table_name, ADDRESS_LENGTH, WORD_LENGTH - ), - &[], - ) - .await?; - if returned != 0 { - return Err(PostgresMemoryError::TableCreationError(returned)); - } - - drop(client); - let _ = conn_handle.await; // ensures that the connection is closed - Ok(()) - } - - /// Connect to a Postgres database and create a table if it doesn't exist - pub async fn connect_with_pool( - pool: Pool, - table_name: String, - ) -> Result { - Ok(Self { - pool, - table_name, - _marker: PhantomData, - }) - } - - /// Deletes all rows from the findex memory table - #[cfg(feature = "test-utils")] - pub async fn clear(&self) -> Result<(), PostgresMemoryError> { - self.pool - .get() - .await? - .execute(&format!("TRUNCATE TABLE {};", self.table_name), &[]) - .await?; - Ok(()) - } -} - -impl MemoryADT - for PostgresMemory, [u8; WORD_LENGTH]> -{ - type Address = Address; - type Word = [u8; WORD_LENGTH]; - type Error = PostgresMemoryError; - - async fn batch_read( - &self, - addresses: Vec, - ) -> Result>, Self::Error> { - let client = self.pool.get().await?; - // statements are cached per connection and not per pool - let stmt = client - .prepare_cached( - format!( - "SELECT f.w - FROM UNNEST($1::bytea[]) WITH ORDINALITY AS params(addr, idx) - LEFT JOIN {} f ON params.addr = f.a - ORDER BY params.idx;", - self.table_name - ) - .as_str(), - ) - .await?; - - client - // the left join is necessary to ensure that the order of the addresses is preserved - // as well as to return None for addresses that don't exist - .query( - &stmt, - &[&addresses - .iter() - .map(|addr| addr.as_slice()) - .collect::>()], - ) - .await? - .iter() - .map(|row| { - let bytes_slice: Option<&[u8]> = row.try_get("w")?; // `row.get(0)` can panic - bytes_slice.map_or(Ok(None), |slice| { - slice - .try_into() - .map(Some) - .map_err(|_| PostgresMemoryError::InvalidDataLength(slice.len())) - }) - }) - .collect() - } - - async fn guarded_write( - &self, - guard: (Self::Address, Option), - bindings: Vec<(Self::Address, Self::Word)>, - ) -> Result, Self::Error> { - let g_write_script = format!( - " - WITH - guard_check AS ( - SELECT w FROM {0} WHERE a = $1::bytea - ), - dedup_input_table AS ( - SELECT DISTINCT ON (a) a, w - FROM UNNEST($3::bytea[], $4::bytea[]) WITH ORDINALITY AS t(a, w, order_idx) - ORDER BY a, order_idx DESC - ), - insert_cte AS ( - INSERT INTO {0} (a, w) - SELECT a, w FROM dedup_input_table AS t(a,w) - WHERE ( - $2::bytea IS NULL AND NOT EXISTS (SELECT 1 FROM guard_check) - ) OR ( - $2::bytea IS NOT NULL AND EXISTS ( - SELECT 1 FROM guard_check WHERE w = $2::bytea - ) - ) - ON CONFLICT (a) DO UPDATE SET w = EXCLUDED.w - ) - SELECT COALESCE((SELECT w FROM guard_check)) AS original_guard_value;", - self.table_name - ); - - let (addresses, words): (Vec<[u8; ADDRESS_LENGTH]>, Vec) = - bindings.into_iter().map(|(a, w)| (*a, w)).unzip(); - const MAX_RETRIES: usize = 10; - - for _ in 0..MAX_RETRIES { - // while counterintuitive, getting a new client on each retry is a better approach - // than trying to reuse the same client since it allows other operations to use the - // connection between retries. - let mut client = self.pool.get().await?; - let stmt = client.prepare_cached(g_write_script.as_str()).await?; - - let result = async { - let tx = client - .build_transaction() - .isolation_level( - deadpool_postgres::tokio_postgres::IsolationLevel::Serializable, - ) - .start() - .await?; - - let res = tx - .query_opt( - &stmt, - &[ - &*guard.0, - &guard.1.as_ref().map(|w| w.as_slice()), - &addresses, - &words, - ], - ) - .await? - .map_or( - Ok::, PostgresMemoryError>(None), - |row| { - row.try_get::<_, Option<&[u8]>>(0)? - .map_or(Ok(None), |r| Ok(Some(r.try_into()?))) - }, - )?; - tx.commit().await?; - Ok(res) - } - .await; - match result { - Ok(value) => return Ok(value), - Err(err) => { - // Retry on serialization failures (error code 40001), otherwise fail and return the error - if let PostgresMemoryError::TokioPostgresError(pg_err) = &err { - if pg_err.code().is_some_and(|code| code.code() == "40001") { - continue; - } - } - return Err(err); - } - } - } - Err(PostgresMemoryError::RetryExhaustedError(MAX_RETRIES)) - } -} -#[cfg(test)] -mod tests { - //! To run the postgresql benchmarks locally, add the following service to your pg_service.conf file - //! (usually under ~/.pg_service.conf): - //! - //! [cosmian_service] - //! host=localhost - //! dbname=cosmian - //! user=cosmian - //! password=cosmian - use deadpool_postgres::Config; - use tokio_postgres::NoTls; - - use super::*; - use crate::{ - ADDRESS_LENGTH, Address, TokioSpawner, WORD_LENGTH, - test_utils::{ - gen_seed, test_guarded_write_concurrent, test_rw_same_address, - test_single_write_and_read, test_wrong_guard, - }, - }; - - const DB_URL: &str = "postgres://cosmian:cosmian@localhost/cosmian"; - - // Template function for pool creation - pub async fn create_testing_pool(db_url: &str) -> Result { - let mut pg_config = Config::new(); - pg_config.url = Some(db_url.to_string()); - let pool = pg_config.builder(NoTls)?.build()?; - Ok(pool) - } - - // Setup function that handles pool creation, memory initialization, test execution, and cleanup - async fn setup_and_run_test( - table_name: &str, - test_fn: F, - ) -> Result<(), PostgresMemoryError> - where - F: FnOnce(PostgresMemory, [u8; WORD_LENGTH]>) -> Fut + Send, - Fut: std::future::Future + Send, - { - let test_pool = create_testing_pool(DB_URL).await.unwrap(); - let m = PostgresMemory::, [u8; WORD_LENGTH]>::connect_with_pool( - test_pool.clone(), - table_name.to_string(), - ) - .await?; - - m.initialize_table(DB_URL.to_string(), table_name.to_string(), NoTls) - .await?; - - test_fn(m).await; - - // Cleanup - drop the table to avoid flacky tests - test_pool - .get() - .await? - .execute(&format!("DROP table {};", table_name), &[]) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn test_initialization() -> Result<(), PostgresMemoryError> { - let table_name: &str = "test_initialization"; - let test_pool = create_testing_pool(DB_URL).await.unwrap(); - let m = PostgresMemory::, [u8; WORD_LENGTH]>::connect_with_pool( - test_pool.clone(), - table_name.to_string(), - ) - .await?; - - m.initialize_table(DB_URL.to_string(), table_name.to_string(), NoTls) - .await?; - - // check that the table actually exists - let client = test_pool.get().await?; - let returned = client - .query( - &format!( - "SELECT COUNT(*) FROM information_schema.tables WHERE table_name = '{}';", - table_name - ), - &[], - ) - .await?; - - assert_eq!(returned[0].get::<_, i64>(0), 1); - - // Cleanup - drop the table to avoid flacky tests - test_pool - .get() - .await? - .execute(&format!("DROP table {table_name};"), &[]) - .await?; - - Ok(()) - } - - #[tokio::test] - async fn test_rw_seq() -> Result<(), PostgresMemoryError> { - setup_and_run_test("findex_test_rw_seq", |m| async move { - test_single_write_and_read::(&m, gen_seed()).await; - }) - .await - } - - #[tokio::test] - async fn test_guard_seq() -> Result<(), PostgresMemoryError> { - setup_and_run_test("findex_test_guard_seq", |m| async move { - test_wrong_guard::(&m, gen_seed()).await; - }) - .await - } - - #[tokio::test] - async fn test_rw_same_address_seq() -> Result<(), PostgresMemoryError> { - setup_and_run_test("findex_test_rw_same_address_seq", |m| async move { - test_rw_same_address::(&m, gen_seed()).await; - }) - .await - } - - #[tokio::test] - async fn test_rw_ccr() -> Result<(), PostgresMemoryError> { - setup_and_run_test("findex_test_rw_ccr", |m| async move { - test_guarded_write_concurrent::( - &m, - gen_seed(), - Some(100), - &TokioSpawner, - ) - .await; - }) - .await - } -} diff --git a/crate/findex/src/memory/postgresql_store/mod.rs b/crate/findex/src/memory/postgresql_store/mod.rs deleted file mode 100644 index aa369c28..00000000 --- a/crate/findex/src/memory/postgresql_store/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -mod error; -mod memory; - -pub use error::PostgresMemoryError; -pub use memory::PostgresMemory; diff --git a/crate/findex/src/memory/redis_store.rs b/crate/findex/src/memory/redis_store.rs deleted file mode 100644 index 2cc509b4..00000000 --- a/crate/findex/src/memory/redis_store.rs +++ /dev/null @@ -1,188 +0,0 @@ -use crate::{Address, MemoryADT}; -use redis::aio::ConnectionManager; -use std::{fmt, marker::PhantomData}; - -// Arguments passed to the LUA script, in order: -// 1. Guard address. -// 2. Guard value. -// 3. Vector length. -// 4+. Vector elements (address, word). -const GUARDED_WRITE_LUA_SCRIPT: &str = " -local guard_address = ARGV[1] -local guard_word = ARGV[2] -local length = ARGV[3] -local current_word = redis.call('GET',guard_address) - --- If no word is found, nil is converted to 'false'. -if guard_word == tostring(current_word) then - for i = 4,(length*2)+3,2 - do - redis.call('SET', ARGV[i], ARGV[i+1]) - end -end -return current_word -"; - -#[derive(Debug, PartialEq)] -pub enum RedisMemoryError { - RedisError(redis::RedisError), -} - -impl fmt::Display for RedisMemoryError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::RedisError(e) => write!(f, "Redis error: {}", e), - } - } -} - -impl std::error::Error for RedisMemoryError {} - -impl From for RedisMemoryError { - fn from(e: redis::RedisError) -> Self { - Self::RedisError(e) - } -} - -#[derive(Clone)] -pub struct RedisMemory { - manager: ConnectionManager, - script_hash: String, - _marker: PhantomData<(Address, Word)>, -} - -impl fmt::Debug for RedisMemory { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("RedisMemory") - .field("connection", &"") - .finish() - } -} - -impl RedisMemory { - /// Connects to a Redis server with a `ConnectionManager`. - pub async fn connect_with_manager( - mut manager: ConnectionManager, - ) -> Result { - let script_hash = redis::cmd("SCRIPT") - .arg("LOAD") - .arg(GUARDED_WRITE_LUA_SCRIPT) - .query_async(&mut manager) - .await?; - - Ok(Self { - manager, - script_hash, - _marker: PhantomData, - }) - } - - /// Connects to a Redis server using the given URL. - pub async fn connect(url: &str) -> Result { - let client = redis::Client::open(url)?; - let manager = client.get_connection_manager().await?; - Self::connect_with_manager(manager).await - } - - #[cfg(feature = "test-utils")] - pub async fn clear(&self) -> Result<(), RedisMemoryError> { - redis::cmd("FLUSHDB") - .query_async(&mut self.manager.clone()) - .await - .map_err(RedisMemoryError::RedisError) - } -} - -impl MemoryADT - for RedisMemory, [u8; WORD_LENGTH]> -{ - type Address = Address; - type Word = [u8; WORD_LENGTH]; - type Error = RedisMemoryError; - - async fn batch_read( - &self, - addresses: Vec, - ) -> Result>, Self::Error> { - let mut cmd = redis::cmd("MGET"); - let cmd = addresses.iter().fold(&mut cmd, |c, a| c.arg(&**a)); - // Cloning the connection manager is cheap since it is an `Arc`. - cmd.query_async(&mut self.manager.clone()) - .await - .map_err(RedisMemoryError::RedisError) - } - - async fn guarded_write( - &self, - guard: (Self::Address, Option), - bindings: Vec<(Self::Address, Self::Word)>, - ) -> Result, Self::Error> { - let (guard_address, guard_value) = guard; - let mut cmd = redis::cmd("EVALSHA"); - let cmd = cmd - .arg(self.script_hash.as_str()) - .arg(0) - .arg(&*guard_address) - .arg( - guard_value - .as_ref() - .map(|bytes| bytes.as_slice()) - .unwrap_or(b"false".as_slice()), - ); - - let cmd = bindings - .iter() - .fold(cmd.arg(bindings.len()), |cmd, (a, w)| cmd.arg(&**a).arg(w)); - - // Cloning the connection manager is cheap since it is an `Arc`. - cmd.query_async(&mut self.manager.clone()) - .await - .map_err(RedisMemoryError::RedisError) - } -} - -#[cfg(test)] -mod tests { - - use super::*; - use crate::{ - WORD_LENGTH, - test_utils::gen_seed, - { - TokioSpawner, test_guarded_write_concurrent, test_rw_same_address, - test_single_write_and_read, test_wrong_guard, - }, - }; - - fn get_redis_url() -> String { - std::env::var("REDIS_HOST").map_or_else( - |_| "redis://localhost:6379".to_owned(), - |var_env| format!("redis://{var_env}:6379"), - ) - } - - #[tokio::test] - async fn test_rw_seq() { - let m = RedisMemory::connect(&get_redis_url()).await.unwrap(); - test_single_write_and_read::(&m, gen_seed()).await - } - - #[tokio::test] - async fn test_guard_seq() { - let m = RedisMemory::connect(&get_redis_url()).await.unwrap(); - test_wrong_guard::(&m, gen_seed()).await - } - - #[tokio::test] - async fn test_collision_seq() { - let m = RedisMemory::connect(&get_redis_url()).await.unwrap(); - test_rw_same_address::(&m, gen_seed()).await - } - - #[tokio::test] - async fn test_rw_ccr() { - let m = RedisMemory::connect(&get_redis_url()).await.unwrap(); - test_guarded_write_concurrent::(&m, gen_seed(), None, &TokioSpawner) - .await - } -} diff --git a/crate/findex/src/memory/sqlite_store.rs b/crate/findex/src/memory/sqlite_store.rs deleted file mode 100644 index 56e06d52..00000000 --- a/crate/findex/src/memory/sqlite_store.rs +++ /dev/null @@ -1,216 +0,0 @@ -use crate::{Address, MemoryADT}; -use async_sqlite::{ - Pool, PoolBuilder, - rusqlite::{OptionalExtension, params_from_iter}, -}; -use std::{ - collections::HashMap, - fmt::{self, Debug}, - marker::PhantomData, - ops::Deref, -}; - -#[derive(Debug)] -pub enum SqliteMemoryError { - AsyncSqliteError(async_sqlite::Error), -} - -impl std::error::Error for SqliteMemoryError {} - -impl fmt::Display for SqliteMemoryError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::AsyncSqliteError(err) => write!(f, "async-sqlite error: {}", err), - } - } -} - -impl From for SqliteMemoryError { - fn from(err: async_sqlite::Error) -> Self { - Self::AsyncSqliteError(err) - } -} - -#[derive(Clone)] -pub struct SqliteMemory { - pool: Pool, - _marker: PhantomData<(Address, Word)>, -} - -impl Debug for SqliteMemory { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("SqliteMemory") - .field("pool", &"") - .field("_marker", &PhantomData::<(Address, Word)>) - .finish() - } -} - -// The following settings are used to improve performance: -// - journal_mode = WAL : WAL journaling is faster than the default DELETE mode. -// - synchronous = NORMAL: Reduces disk I/O by only calling fsync() at critical moments rather -// than after every transaction (FULL mode); this does not compromise data integrity. -const CREATE_TABLE_SCRIPT: &str = " -PRAGMA synchronous = NORMAL; -PRAGMA journal_mode = WAL; -CREATE TABLE IF NOT EXISTS memory ( - a BLOB PRIMARY KEY, - w BLOB NOT NULL -);"; - -impl SqliteMemory { - /// Connects to a known DB using the given path. - /// - /// # Arguments - /// - /// * `path` - The path to the sqlite3 database file. - pub async fn connect(path: &str) -> Result { - // This pool connections number defaults to the number of logical CPUs - // of the current system. - let pool = PoolBuilder::new().path(path).open().await?; - - pool.conn(move |conn| conn.execute_batch(CREATE_TABLE_SCRIPT)) - .await?; - - Ok(Self { - pool, - _marker: PhantomData, - }) - } -} - -impl SqliteMemory { - #[cfg(feature = "test-utils")] - pub async fn clear(&self) -> Result<(), SqliteMemoryError> { - self.pool - .conn(|cnx| cnx.execute("DELETE FROM memory", [])) - .await?; - Ok(()) - } -} - -impl MemoryADT - for SqliteMemory, [u8; WORD_LENGTH]> -{ - type Address = Address; - type Error = SqliteMemoryError; - type Word = [u8; WORD_LENGTH]; - - async fn batch_read( - &self, - addresses: Vec, - ) -> Result>, Self::Error> { - self.pool - .conn(move |conn| { - let results = conn - .prepare(&format!( - "SELECT a, w FROM memory WHERE a IN ({})", - vec!["?"; addresses.len()].join(",") - ))? - .query_map( - params_from_iter(addresses.iter().map(Deref::deref)), - |row| { - let a = Address::from(row.get::<_, [u8; ADDRESS_LENGTH]>(0)?); - let w = row.get(1)?; - Ok((a, w)) - }, - )? - .collect::, _>>()?; - - // Return order of an SQL select statement is undefined, and - // mismatches are ignored. A post-processing is thus needed to - // generate a returned value complying to the batch-read spec. - Ok(addresses - .iter() - // Copying is necessary here since the same word could be - // returned multiple times. - .map(|addr| results.get(addr).copied()) - .collect()) - }) - .await - .map_err(Self::Error::from) - } - - async fn guarded_write( - &self, - guard: (Self::Address, Option), - bindings: Vec<(Self::Address, Self::Word)>, - ) -> Result, Self::Error> { - let (ag, wg) = guard; - - self.pool - .conn_mut(move |conn| { - let tx = conn.transaction_with_behavior( - async_sqlite::rusqlite::TransactionBehavior::Immediate, - )?; - - let current_word = tx - .query_row("SELECT w FROM memory WHERE a = ?", [&*ag], |row| row.get(0)) - .optional()?; - - if current_word == wg { - tx.execute( - &format!( - "INSERT OR REPLACE INTO memory (a, w) VALUES {}", - vec!["(?,?)"; bindings.len()].join(",") - ), - params_from_iter( - bindings - .iter() - // There seems to be no way to avoid cloning here. - .flat_map(|(a, w)| [a.to_vec(), w.to_vec()]), - ), - )?; - tx.commit()?; - } - - Ok(current_word) - }) - .await - .map_err(Self::Error::from) - } -} - -#[cfg(test)] -mod tests { - - use super::*; - use crate::{ - TokioSpawner, WORD_LENGTH, gen_seed, test_guarded_write_concurrent, test_rw_same_address, - test_single_write_and_read, test_wrong_guard, - }; - - const DB_PATH: &str = "./target/debug/sqlite-test.db"; - - #[tokio::test] - async fn test_rw_seq() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH) - .await - .unwrap(); - test_single_write_and_read(&m, gen_seed()).await - } - - #[tokio::test] - async fn test_guard_seq() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH) - .await - .unwrap(); - test_wrong_guard(&m, gen_seed()).await - } - - #[tokio::test] - async fn test_collision_seq() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH) - .await - .unwrap(); - test_rw_same_address(&m, gen_seed()).await - } - - #[tokio::test] - async fn test_rw_ccr() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH) - .await - .unwrap(); - test_guarded_write_concurrent(&m, gen_seed(), Some(100), &TokioSpawner).await - } -} diff --git a/crate/findex/src/test_utils/memory_tests.rs b/crate/findex/src/test_utils/memory_tests.rs index d40a9e05..763d3082 100644 --- a/crate/findex/src/test_utils/memory_tests.rs +++ b/crate/findex/src/test_utils/memory_tests.rs @@ -13,7 +13,7 @@ use cosmian_crypto_core::{ CsRng, reexport::rand_core::{RngCore, SeedableRng}, }; -use std::{fmt::Debug, future::Future}; +use std::fmt::Debug; pub const SEED_LENGTH: usize = 32; @@ -218,32 +218,6 @@ pub async fn test_rw_same_address( ); } -/// Trait for spawning async tasks in a runtime-agnostic way -pub trait TaskSpawner: Send + Sync { - type JoinHandle: Future> + Send; - type JoinError: std::error::Error + Send; - - fn spawn(&self, future: F) -> Self::JoinHandle - where - F: Future + Send + 'static, - F::Output: Send + 'static; -} - -pub struct TokioSpawner; - -impl TaskSpawner for TokioSpawner { - type JoinHandle = tokio::task::JoinHandle; - type JoinError = tokio::task::JoinError; - - fn spawn(&self, future: F) -> Self::JoinHandle - where - F: Future + Send + 'static, - F::Output: Send + 'static, - { - tokio::spawn(future) - } -} - /// Tests concurrent guarded write operations on a Memory ADT implementation. /// /// Spawns multiple threads to perform concurrent counter increments. @@ -256,18 +230,16 @@ impl TaskSpawner for TokioSpawner { /// * `memory` - The Memory ADT implementation to test. /// * `seed` - The seed used to initialize the random number generator. /// * `n_threads` - The number of threads to spawn. If None, defaults to 100. -pub async fn test_guarded_write_concurrent( +pub async fn test_guarded_write_concurrent( memory: &Memory, seed: [u8; SEED_LENGTH], n_threads: Option, - spawer: &S, ) where Memory: 'static + Send + Sync + MemoryADT + Clone, Memory::Address: Send + From<[u8; ADDRESS_LENGTH]>, Memory::Word: Send + Debug + PartialEq + From<[u8; WORD_LENGTH]> + Into<[u8; WORD_LENGTH]> + Clone, Memory::Error: Send + std::error::Error, - S: TaskSpawner, { // A worker increment N times the counter m[a]. async fn worker( @@ -318,7 +290,7 @@ pub async fn test_guarded_write_concurrent( let handles = (0..n) .map(|_| { let m = memory.clone(); - spawer.spawn(worker(m, a)) + tokio::spawn(worker(m, a)) }) .collect::>(); diff --git a/crate/memories/Cargo.toml b/crate/memories/Cargo.toml new file mode 100644 index 00000000..8151c679 --- /dev/null +++ b/crate/memories/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "cosmian_findex_memories" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +# categories.workspace = true +# keywords.workspace = true +# repository.workspace = true +# description.workspace = true + +[lib] +name = "cosmian_findex_memories" +path = "src/lib.rs" + +# [features] +# test-utils = ["tokio", "criterion", "rand", "rand_distr"] + +# [dependencies] +# aes.workspace = true +# cosmian_crypto_core.workspace = true +# xts-mode.workspace = true + +# # Optional dependencies for test-utils +# tokio = { workspace = true, optional = true } +# criterion = { workspace = true, optional = true } +# rand = { workspace = true, optional = true } +# rand_distr = { workspace = true, optional = true } + +# [dev-dependencies] +# tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/gwt b/gwt new file mode 160000 index 00000000..c6f07d1f --- /dev/null +++ b/gwt @@ -0,0 +1 @@ +Subproject commit c6f07d1fba3a72acc0fe3c3b74cb97807f3b62af From 409ce51efc8f01ac03b8b9beadf582dfbcf48193 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Tue, 3 Jun 2025 15:50:13 +0200 Subject: [PATCH 11/60] feat: more cargo --- Cargo.toml | 10 ------- crate/findex/Cargo.toml | 25 ++++------------ crate/memories/Cargo.toml | 62 +++++++++++++++++++++++++++++---------- 3 files changed, 52 insertions(+), 45 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f9f6d5a0..5df50b1c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,24 +22,14 @@ description = "Symmetric Searchable Encryption" [workspace.dependencies] -aes = "0.8" cosmian_crypto_core = { version = "10.1", default-features = false, features = [ "macro", "sha3", ] } -xts-mode = "0.5" criterion = { version = "0.6" } rand = { version = "0.9.0" } rand_distr = { version = "0.5.1" } tokio = { version = "1.44" } -async-sqlite = { version = "0.5" } -deadpool-postgres = { version = "0.14.1" } -redis = { version = "0.28", features = [ - "aio", - "connection-manager", - "tokio-comp", -] } -tokio-postgres = { version = "0.7.9", features = ["array-impls"] } # TODO: later diff --git a/crate/findex/Cargo.toml b/crate/findex/Cargo.toml index 0fdc74c8..05bb5c90 100644 --- a/crate/findex/Cargo.toml +++ b/crate/findex/Cargo.toml @@ -14,35 +14,22 @@ name = "cosmian_findex" path = "src/lib.rs" [features] -rust-mem = [] -redis-mem = ["redis"] -sqlite-mem = ["async-sqlite"] -postgres-mem = ["tokio-postgres", "tokio", "deadpool-postgres"] test-utils = ["tokio", "criterion", "rand", "rand_distr"] [dependencies] -aes.workspace = true cosmian_crypto_core.workspace = true -xts-mode.workspace = true +aes = "0.8" +xts-mode = "0.5" # Optional dependencies for test-utils -tokio = { workspace = true, features = ["rt-multi-thread"], optional = true } +tokio = { workspace = true, features = [ + "rt-multi-thread", + "macros", +], optional = true } criterion = { workspace = true, optional = true } rand = { workspace = true, optional = true } rand_distr = { workspace = true, optional = true } -# Memory dependencies -async-sqlite = { version = "0.5", optional = true } -deadpool-postgres = { version = "0.14.1", optional = true } -redis = { version = "0.31", features = [ - "aio", - "connection-manager", - "tokio-comp", -], optional = true } -tokio-postgres = { version = "0.7.9", features = [ - "array-impls", -], optional = true } - [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crate/memories/Cargo.toml b/crate/memories/Cargo.toml index 8151c679..9f60437f 100644 --- a/crate/memories/Cargo.toml +++ b/crate/memories/Cargo.toml @@ -13,19 +13,49 @@ edition.workspace = true name = "cosmian_findex_memories" path = "src/lib.rs" -# [features] -# test-utils = ["tokio", "criterion", "rand", "rand_distr"] - -# [dependencies] -# aes.workspace = true -# cosmian_crypto_core.workspace = true -# xts-mode.workspace = true - -# # Optional dependencies for test-utils -# tokio = { workspace = true, optional = true } -# criterion = { workspace = true, optional = true } -# rand = { workspace = true, optional = true } -# rand_distr = { workspace = true, optional = true } - -# [dev-dependencies] -# tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +[features] +redis-mem = ["redis"] +sqlite-mem = ["async-sqlite"] +postgres-mem = ["tokio-postgres", "tokio", "deadpool-postgres"] + +[dependencies] +cosmian_crypto_core.workspace = true +cosmian_findex = { path = "../findex" } +async-sqlite = { version = "0.4", optional = true } # `async-sqlite` dependency is pinned at v0.4 due to dependency tree issues with sqlx requiring a lower version of `libsqlite3-sys` +deadpool-postgres = { version = "0.14.1", optional = true } +redis = { version = "0.31", features = [ + "aio", + "connection-manager", + "tokio-comp", +], optional = true } +tokio-postgres = { version = "0.7.9", optional = true, features = [ + "array-impls", +] } +tokio = { workspace = true, features = ["rt-multi-thread"], optional = true } + + +[dev-dependencies] +cosmian_findex = { path = "../findex", features = ["test-utils"] } +futures = { version = "0.3" } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +criterion = { workspace = true } +rand = { workspace = true } +rand_distr = { workspace = true } + +# Optional dependencies for t TOD TODO +# [[bench]] +# name = "benches" +# harness = false +# required-features = ["redis-mem", "sqlite-mem", "postgres-mem"] + +# [[example]] +# name = "redis" +# required-features = ["redis-mem"] + +# [[example]] +# name = "sqlite" +# required-features = ["sqlite-mem"] + +# [[example]] +# name = "postgresql" +# required-features = ["postgres-mem"] From a8988d52f8d8d8923ff45d224f52ff5cdfa6503e Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Tue, 3 Jun 2025 16:02:02 +0200 Subject: [PATCH 12/60] feat: memories are back :) --- .gitignore | 1 + crate/memories/src/lib.rs | 14 + crate/memories/src/postgresql_mem/error.rs | 82 +++++ crate/memories/src/postgresql_mem/memory.rs | 359 ++++++++++++++++++++ crate/memories/src/postgresql_mem/mod.rs | 5 + crate/memories/src/redis_mem.rs | 183 ++++++++++ crate/memories/src/sqlite_mem.rs | 249 ++++++++++++++ 7 files changed, 893 insertions(+) create mode 100644 crate/memories/src/lib.rs create mode 100644 crate/memories/src/postgresql_mem/error.rs create mode 100644 crate/memories/src/postgresql_mem/memory.rs create mode 100644 crate/memories/src/postgresql_mem/mod.rs create mode 100644 crate/memories/src/redis_mem.rs create mode 100644 crate/memories/src/sqlite_mem.rs diff --git a/.gitignore b/.gitignore index aad6b0e6..973da758 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ target/ perf* flamegraph.svg **/*.tgz +*.sqlite.db* diff --git a/crate/memories/src/lib.rs b/crate/memories/src/lib.rs new file mode 100644 index 00000000..59408d97 --- /dev/null +++ b/crate/memories/src/lib.rs @@ -0,0 +1,14 @@ +#[cfg(feature = "redis-mem")] +mod redis_mem; +#[cfg(feature = "redis-mem")] +pub use redis_mem::{RedisMemory, RedisMemoryError}; + +#[cfg(feature = "sqlite-mem")] +mod sqlite_mem; +#[cfg(feature = "sqlite-mem")] +pub use sqlite_mem::{SqliteMemory, SqliteMemoryError, FINDEX_TABLE_NAME}; + +#[cfg(feature = "postgres-mem")] +mod postgresql_mem; +#[cfg(feature = "postgres-mem")] +pub use postgresql_mem::{PostgresMemory, PostgresMemoryError}; diff --git a/crate/memories/src/postgresql_mem/error.rs b/crate/memories/src/postgresql_mem/error.rs new file mode 100644 index 00000000..48bd5dd1 --- /dev/null +++ b/crate/memories/src/postgresql_mem/error.rs @@ -0,0 +1,82 @@ +use std::fmt; + +#[derive(Debug)] +pub enum PostgresMemoryError { + TokioPostgresError(deadpool_postgres::tokio_postgres::Error), + TryFromSliceError(std::array::TryFromSliceError), + BuildPoolError(deadpool_postgres::BuildError), + GetConnectionFromPoolError(deadpool_postgres::PoolError), + PoolConfigError(deadpool_postgres::ConfigError), + RetryExhaustedError(usize), + InvalidDataLength(usize), + TableCreationError(u64), +} + +impl std::error::Error for PostgresMemoryError {} + +impl fmt::Display for PostgresMemoryError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::TokioPostgresError(err) => write!(f, "tokio-postgres error: {}", err), + Self::TryFromSliceError(err) => write!(f, "try_from_slice error: {}", err), + Self::InvalidDataLength(len) => { + write!( + f, + "invalid data length: received {} bytes from db instead of WORD_LENGTH bytes", + len + ) + } + Self::PoolConfigError(err) => { + write!(f, "deadpool_postgres error during pool creation: {}", err) + } + Self::BuildPoolError(err) => { + write!(f, "deadpool_postgres error during pool build: {}", err) + } + Self::GetConnectionFromPoolError(err) => write!( + f, + "deadpool_postgres error while trying to get a connection from the pool: {}", + err + ), + Self::RetryExhaustedError(retries) => { + write!(f, "retries exhausted after {} attempts", retries) + } + Self::TableCreationError(err) => { + write!( + f, + "error creating table, {} rows returned while 0 were expected.", + err + ) + } + } + } +} + +impl From for PostgresMemoryError { + fn from(err: deadpool_postgres::tokio_postgres::Error) -> Self { + Self::TokioPostgresError(err) + } +} + +impl From for PostgresMemoryError { + fn from(err: std::array::TryFromSliceError) -> Self { + Self::TryFromSliceError(err) + } +} + +impl From for PostgresMemoryError { + fn from(err: deadpool_postgres::ConfigError) -> Self { + Self::PoolConfigError(err) + } +} + +impl From for PostgresMemoryError { + fn from(err: deadpool_postgres::BuildError) -> Self { + Self::BuildPoolError(err) + } +} + +impl From for PostgresMemoryError { + fn from(err: deadpool_postgres::PoolError) -> Self { + Self::GetConnectionFromPoolError(err) + } +} diff --git a/crate/memories/src/postgresql_mem/memory.rs b/crate/memories/src/postgresql_mem/memory.rs new file mode 100644 index 00000000..eaeffd4e --- /dev/null +++ b/crate/memories/src/postgresql_mem/memory.rs @@ -0,0 +1,359 @@ +use super::PostgresMemoryError; +use cosmian_findex::{Address, MemoryADT}; +use deadpool_postgres::Pool; +use std::marker::PhantomData; +use tokio_postgres::{ + tls::{MakeTlsConnect, TlsConnect}, + Socket, +}; + +#[derive(Clone, Debug)] +pub struct PostgresMemory { + pool: Pool, + table_name: String, + _marker: PhantomData<(Address, Word)>, +} + +impl + PostgresMemory, [u8; WORD_LENGTH]> +{ + /// Connect to a Postgres database and create a table if it doesn't exist + pub async fn initialize_table( + &self, + db_url: String, + table_name: String, + tls: T, + ) -> Result<(), PostgresMemoryError> + where + T: MakeTlsConnect + Send, + T::Stream: Send + 'static, + T::TlsConnect: Send, + >::Future: Send, + { + let (client, connection) = tokio_postgres::connect(&db_url, tls).await?; + + // The connection object performs the actual communication with the database + // `Connection` only resolves when the connection is closed, either because a fatal error has + // occurred, or because its associated `Client` has dropped and all outstanding work has completed. + let conn_handle = tokio::spawn(async move { + if let Err(e) = connection.await { + eprintln!("connection error: {}", e); + } + }); + + let returned = client + .execute( + &format!( + " + CREATE TABLE IF NOT EXISTS {} ( + a BYTEA PRIMARY KEY CHECK (octet_length(a) = {}), + w BYTEA NOT NULL CHECK (octet_length(w) = {}) + );", + table_name, ADDRESS_LENGTH, WORD_LENGTH + ), + &[], + ) + .await?; + if returned != 0 { + return Err(PostgresMemoryError::TableCreationError(returned)); + } + + drop(client); + let _ = conn_handle.await; // ensures that the connection is closed + Ok(()) + } + + /// Connect to a Postgres database and create a table if it doesn't exist + pub async fn connect_with_pool( + pool: Pool, + table_name: String, + ) -> Result { + Ok(Self { + pool, + table_name, + _marker: PhantomData, + }) + } + + // /// Deletes all rows from the findex memory table + // #[cfg(feature = "test-utils")] + // pub async fn clear(&self) -> Result<(), PostgresMemoryError> { + // self.pool + // .get() + // .await? + // .execute(&format!("TRUNCATE TABLE {};", self.table_name), &[]) + // .await?; + // Ok(()) + // } +} + +impl MemoryADT + for PostgresMemory, [u8; WORD_LENGTH]> +{ + type Address = Address; + type Word = [u8; WORD_LENGTH]; + type Error = PostgresMemoryError; + + async fn batch_read( + &self, + addresses: Vec, + ) -> Result>, Self::Error> { + let client = self.pool.get().await?; + // statements are cached per connection and not per pool + let stmt = client + .prepare_cached( + format!( + "SELECT f.w + FROM UNNEST($1::bytea[]) WITH ORDINALITY AS params(addr, idx) + LEFT JOIN {} f ON params.addr = f.a + ORDER BY params.idx;", + self.table_name + ) + .as_str(), + ) + .await?; + + client + // the left join is necessary to ensure that the order of the addresses is preserved + // as well as to return None for addresses that don't exist + .query( + &stmt, + &[&addresses + .iter() + .map(|addr| addr.as_slice()) + .collect::>()], + ) + .await? + .iter() + .map(|row| { + let bytes_slice: Option<&[u8]> = row.try_get("w")?; // `row.get(0)` can panic + bytes_slice.map_or(Ok(None), |slice| { + slice + .try_into() + .map(Some) + .map_err(|_| PostgresMemoryError::InvalidDataLength(slice.len())) + }) + }) + .collect() + } + + async fn guarded_write( + &self, + guard: (Self::Address, Option), + bindings: Vec<(Self::Address, Self::Word)>, + ) -> Result, Self::Error> { + let g_write_script = format!( + " + WITH + guard_check AS ( + SELECT w FROM {0} WHERE a = $1::bytea + ), + dedup_input_table AS ( + SELECT DISTINCT ON (a) a, w + FROM UNNEST($3::bytea[], $4::bytea[]) WITH ORDINALITY AS t(a, w, order_idx) + ORDER BY a, order_idx DESC + ), + insert_cte AS ( + INSERT INTO {0} (a, w) + SELECT a, w FROM dedup_input_table AS t(a,w) + WHERE ( + $2::bytea IS NULL AND NOT EXISTS (SELECT 1 FROM guard_check) + ) OR ( + $2::bytea IS NOT NULL AND EXISTS ( + SELECT 1 FROM guard_check WHERE w = $2::bytea + ) + ) + ON CONFLICT (a) DO UPDATE SET w = EXCLUDED.w + ) + SELECT COALESCE((SELECT w FROM guard_check)) AS original_guard_value;", + self.table_name + ); + + let (addresses, words): (Vec<[u8; ADDRESS_LENGTH]>, Vec) = + bindings.into_iter().map(|(a, w)| (*a, w)).unzip(); + const MAX_RETRIES: usize = 10; + + for _ in 0..MAX_RETRIES { + // while counterintuitive, getting a new client on each retry is a better approach + // than trying to reuse the same client since it allows other operations to use the + // connection between retries. + let mut client = self.pool.get().await?; + let stmt = client.prepare_cached(g_write_script.as_str()).await?; + + let result = async { + let tx = client + .build_transaction() + .isolation_level( + deadpool_postgres::tokio_postgres::IsolationLevel::Serializable, + ) + .start() + .await?; + + let res = tx + .query_opt( + &stmt, + &[ + &*guard.0, + &guard.1.as_ref().map(|w| w.as_slice()), + &addresses, + &words, + ], + ) + .await? + .map_or( + Ok::, PostgresMemoryError>(None), + |row| { + row.try_get::<_, Option<&[u8]>>(0)? + .map_or(Ok(None), |r| Ok(Some(r.try_into()?))) + }, + )?; + tx.commit().await?; + Ok(res) + } + .await; + match result { + Ok(value) => return Ok(value), + Err(err) => { + // Retry on serialization failures (error code 40001), otherwise fail and return the error + if let PostgresMemoryError::TokioPostgresError(pg_err) = &err { + if pg_err.code().is_some_and(|code| code.code() == "40001") { + continue; + } + } + return Err(err); + } + } + } + Err(PostgresMemoryError::RetryExhaustedError(MAX_RETRIES)) + } +} + +#[cfg(test)] +mod tests { + //! To run the postgresql benchmarks locally, add the following service to your pg_service.conf file + //! (usually under ~/.pg_service.conf): + //! + //! [cosmian_service] + //! host=localhost + //! dbname=cosmian + //! user=cosmian + //! password=cosmian + use deadpool_postgres::Config; + use tokio_postgres::NoTls; + + use super::*; + use cosmian_findex::{ + gen_seed, test_guarded_write_concurrent, test_rw_same_address, test_single_write_and_read, + test_wrong_guard, ADDRESS_LENGTH, WORD_LENGTH, + }; + + const DB_URL: &str = "postgres://cosmian:cosmian@localhost/cosmian"; + + // Template function for pool creation + pub async fn create_testing_pool(db_url: &str) -> Result { + let mut pg_config = Config::new(); + pg_config.url = Some(db_url.to_string()); + let pool = pg_config.builder(NoTls)?.build()?; + Ok(pool) + } + + // Setup function that handles pool creation, memory initialization, test execution, and cleanup + async fn setup_and_run_test( + table_name: &str, + test_fn: F, + ) -> Result<(), PostgresMemoryError> + where + F: FnOnce(PostgresMemory, [u8; WORD_LENGTH]>) -> Fut + Send, + Fut: std::future::Future + Send, + { + let test_pool = create_testing_pool(DB_URL).await.unwrap(); + let m = PostgresMemory::, [u8; WORD_LENGTH]>::connect_with_pool( + test_pool.clone(), + table_name.to_string(), + ) + .await?; + + m.initialize_table(DB_URL.to_string(), table_name.to_string(), NoTls) + .await?; + + test_fn(m).await; + + // Cleanup - drop the table to avoid flacky tests + test_pool + .get() + .await? + .execute(&format!("DROP table {};", table_name), &[]) + .await?; + + Ok(()) + } + + #[tokio::test] + async fn test_initialization() -> Result<(), PostgresMemoryError> { + let table_name: &str = "test_initialization"; + let test_pool = create_testing_pool(DB_URL).await.unwrap(); + let m = PostgresMemory::, [u8; WORD_LENGTH]>::connect_with_pool( + test_pool.clone(), + table_name.to_string(), + ) + .await?; + + m.initialize_table(DB_URL.to_string(), table_name.to_string(), NoTls) + .await?; + + // check that the table actually exists + let client = test_pool.get().await?; + let returned = client + .query( + &format!( + "SELECT COUNT(*) FROM information_schema.tables WHERE table_name = '{}';", + table_name + ), + &[], + ) + .await?; + + assert_eq!(returned[0].get::<_, i64>(0), 1); + + // Cleanup - drop the table to avoid flacky tests + test_pool + .get() + .await? + .execute(&format!("DROP table {table_name};"), &[]) + .await?; + + Ok(()) + } + + #[tokio::test] + async fn test_rw_seq() -> Result<(), PostgresMemoryError> { + setup_and_run_test("findex_test_rw_seq", |m| async move { + test_single_write_and_read::(&m, gen_seed()).await; + }) + .await + } + + #[tokio::test] + async fn test_guard_seq() -> Result<(), PostgresMemoryError> { + setup_and_run_test("findex_test_guard_seq", |m| async move { + test_wrong_guard::(&m, gen_seed()).await; + }) + .await + } + + #[tokio::test] + async fn test_rw_same_address_seq() -> Result<(), PostgresMemoryError> { + setup_and_run_test("findex_test_rw_same_address_seq", |m| async move { + test_rw_same_address::(&m, gen_seed()).await; + }) + .await + } + + #[tokio::test] + async fn test_rw_ccr() -> Result<(), PostgresMemoryError> { + setup_and_run_test("findex_test_rw_ccr", |m| async move { + test_guarded_write_concurrent::(&m, gen_seed(), Some(100)).await; + }) + .await + } +} diff --git a/crate/memories/src/postgresql_mem/mod.rs b/crate/memories/src/postgresql_mem/mod.rs new file mode 100644 index 00000000..aa369c28 --- /dev/null +++ b/crate/memories/src/postgresql_mem/mod.rs @@ -0,0 +1,5 @@ +mod error; +mod memory; + +pub use error::PostgresMemoryError; +pub use memory::PostgresMemory; diff --git a/crate/memories/src/redis_mem.rs b/crate/memories/src/redis_mem.rs new file mode 100644 index 00000000..03155a4e --- /dev/null +++ b/crate/memories/src/redis_mem.rs @@ -0,0 +1,183 @@ +use cosmian_findex::{Address, MemoryADT}; +use redis::aio::ConnectionManager; +use std::{fmt, marker::PhantomData}; + +// Arguments passed to the LUA script, in order: +// 1. Guard address. +// 2. Guard value. +// 3. Vector length. +// 4+. Vector elements (address, word). +const GUARDED_WRITE_LUA_SCRIPT: &str = " +local guard_address = ARGV[1] +local guard_word = ARGV[2] +local length = ARGV[3] +local current_word = redis.call('GET',guard_address) + +-- If no word is found, nil is converted to 'false'. +if guard_word == tostring(current_word) then + for i = 4,(length*2)+3,2 + do + redis.call('SET', ARGV[i], ARGV[i+1]) + end +end +return current_word +"; + +#[derive(Debug, PartialEq)] +pub enum RedisMemoryError { + RedisError(redis::RedisError), +} + +impl fmt::Display for RedisMemoryError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::RedisError(e) => write!(f, "Redis error: {}", e), + } + } +} + +impl std::error::Error for RedisMemoryError {} + +impl From for RedisMemoryError { + fn from(e: redis::RedisError) -> Self { + Self::RedisError(e) + } +} + +#[derive(Clone)] +pub struct RedisMemory { + manager: ConnectionManager, + script_hash: String, + _marker: PhantomData<(Address, Word)>, +} + +impl fmt::Debug for RedisMemory { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RedisMemory") + .field("connection", &"") + .finish() + } +} + +impl RedisMemory { + /// Connects to a Redis server with a `ConnectionManager`. + pub async fn connect_with_manager( + mut manager: ConnectionManager, + ) -> Result { + let script_hash = redis::cmd("SCRIPT") + .arg("LOAD") + .arg(GUARDED_WRITE_LUA_SCRIPT) + .query_async(&mut manager) + .await?; + + Ok(Self { + manager, + script_hash, + _marker: PhantomData, + }) + } + + /// Connects to a Redis server using the given URL. + pub async fn connect(url: &str) -> Result { + let client = redis::Client::open(url)?; + let manager = client.get_connection_manager().await?; + Self::connect_with_manager(manager).await + } + + // #[cfg(feature = "test-utils")] + // pub async fn clear(&self) -> Result<(), RedisMemoryError> { + // redis::cmd("FLUSHDB") + // .query_async(&mut self.manager.clone()) + // .await + // .map_err(RedisMemoryError::RedisError) + // } +} + +impl MemoryADT + for RedisMemory, [u8; WORD_LENGTH]> +{ + type Address = Address; + type Word = [u8; WORD_LENGTH]; + type Error = RedisMemoryError; + + async fn batch_read( + &self, + addresses: Vec, + ) -> Result>, Self::Error> { + let mut cmd = redis::cmd("MGET"); + let cmd = addresses.iter().fold(&mut cmd, |c, a| c.arg(&**a)); + // Cloning the connection manager is cheap since it is an `Arc`. + cmd.query_async(&mut self.manager.clone()) + .await + .map_err(RedisMemoryError::RedisError) + } + + async fn guarded_write( + &self, + guard: (Self::Address, Option), + bindings: Vec<(Self::Address, Self::Word)>, + ) -> Result, Self::Error> { + let (guard_address, guard_value) = guard; + let mut cmd = redis::cmd("EVALSHA"); + let cmd = cmd + .arg(self.script_hash.as_str()) + .arg(0) + .arg(&*guard_address) + .arg( + guard_value + .as_ref() + .map(|bytes| bytes.as_slice()) + .unwrap_or(b"false".as_slice()), + ); + + let cmd = bindings + .iter() + .fold(cmd.arg(bindings.len()), |cmd, (a, w)| cmd.arg(&**a).arg(w)); + + // Cloning the connection manager is cheap since it is an `Arc`. + cmd.query_async(&mut self.manager.clone()) + .await + .map_err(RedisMemoryError::RedisError) + } +} + +#[cfg(test)] +mod tests { + + use super::*; + use cosmian_findex::{ + gen_seed, test_guarded_write_concurrent, test_rw_same_address, test_single_write_and_read, + test_wrong_guard, WORD_LENGTH, + }; + + fn get_redis_url() -> String { + std::env::var("REDIS_HOST").map_or_else( + |_| "redis://localhost:6379".to_owned(), + |var_env| format!("redis://{var_env}:6379"), + ) + } + + #[tokio::test] + async fn test_rw_seq() { + let m = RedisMemory::connect(&get_redis_url()).await.unwrap(); + test_single_write_and_read::(&m, gen_seed()).await + } + + #[tokio::test] + async fn test_guard_seq() { + let m = RedisMemory::connect(&get_redis_url()).await.unwrap(); + test_wrong_guard::(&m, gen_seed()).await + } + + #[tokio::test] + async fn test_collision_seq() { + let m = RedisMemory::connect(&get_redis_url()).await.unwrap(); + test_rw_same_address::(&m, gen_seed()).await + } + + #[tokio::test] + async fn test_rw_ccr() { + let m = RedisMemory::connect(&get_redis_url()).await.unwrap(); + test_guarded_write_concurrent::(&m, gen_seed(), None).await + } +} diff --git a/crate/memories/src/sqlite_mem.rs b/crate/memories/src/sqlite_mem.rs new file mode 100644 index 00000000..424b390b --- /dev/null +++ b/crate/memories/src/sqlite_mem.rs @@ -0,0 +1,249 @@ +use async_sqlite::{ + Pool, PoolBuilder, + rusqlite::{OptionalExtension, params_from_iter}, +}; +use cosmian_findex::{Address, MemoryADT}; +use std::{ + collections::HashMap, + fmt::{self, Debug}, + marker::PhantomData, + ops::Deref, +}; + +#[derive(Debug)] +pub enum SqliteMemoryError { + AsyncSqliteError(async_sqlite::Error), +} + +impl std::error::Error for SqliteMemoryError {} + +impl fmt::Display for SqliteMemoryError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::AsyncSqliteError(err) => write!(f, "async-sqlite error: {}", err), + } + } +} + +impl From for SqliteMemoryError { + fn from(err: async_sqlite::Error) -> Self { + Self::AsyncSqliteError(err) + } +} + +pub const FINDEX_TABLE_NAME: &str = "findex_memory"; +#[derive(Clone)] +pub struct SqliteMemory { + pool: Pool, + _marker: PhantomData<(Address, Word)>, +} + +impl Debug for SqliteMemory { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SqliteMemory") + .field("pool", &"") + .field("_marker", &PhantomData::<(Address, Word)>) + .finish() + } +} + +// The following settings are used to improve performance: +// - journal_mode = WAL : WAL journaling is faster than the default DELETE mode. +// - synchronous = NORMAL: Reduces disk I/O by only calling fsync() at critical +// moments rather than after every transaction (FULL mode); this does not +// compromise data integrity. +fn create_table_script(table_name: &str) -> String { + format!( + " +PRAGMA synchronous = NORMAL; +PRAGMA journal_mode = WAL; +CREATE TABLE IF NOT EXISTS {} ( + a BLOB PRIMARY KEY, + w BLOB NOT NULL +);", + table_name + ) +} + +impl SqliteMemory { + /// Builds a pool connected to a known DB (using the given path) and creates + /// the `findex_memory` table. + /// + /// # Arguments + /// + /// * `path` - The path to the sqlite3 database file. + pub async fn connect(path: &str) -> Result { + // The number of connections in this pools defaults to the number of + // logical CPUs of the system. + let pool = PoolBuilder::new().path(path).open().await?; + + pool.conn(|conn| conn.execute_batch(&create_table_script(FINDEX_TABLE_NAME))) + .await?; + + Ok(Self { + pool, + _marker: PhantomData, + }) + } + + /// Returns an `SqliteMemory` instance storing data in a table with the given name + /// and connecting to the DB using connections from the given pool. + /// + /// # Arguments + /// + /// * `pool` - The pool to use for the memory. + /// * `table_name` - The name of the table to create. + pub async fn connect_with_pool( + pool: Pool, + table_name: String, + ) -> Result { + pool.conn(move |conn| conn.execute_batch(&create_table_script(&table_name))) + .await?; + Ok(Self { + pool, + _marker: PhantomData, + }) + } +} + +// impl SqliteMemory { +// #[cfg(feature = "test-utils")] +// pub async fn clear(&self) -> Result<(), SqliteMemoryError> { +// self.pool +// .conn(|cnx| cnx.execute(&format!("DELETE FROM {FINDEX_TABLE_NAME}"), [])) +// .await?; +// Ok(()) +// } +// } + +impl MemoryADT + for SqliteMemory, [u8; WORD_LENGTH]> +{ + type Address = Address; + type Error = SqliteMemoryError; + type Word = [u8; WORD_LENGTH]; + + async fn batch_read( + &self, + addresses: Vec, + ) -> Result>, Self::Error> { + self.pool + .conn(move |conn| { + let results = conn + .prepare(&format!( + "SELECT a, w FROM {} WHERE a IN ({})", + FINDEX_TABLE_NAME, + vec!["?"; addresses.len()].join(",") + ))? + .query_map( + params_from_iter(addresses.iter().map(Deref::deref)), + |row| { + let a = Address::from(row.get::<_, [u8; ADDRESS_LENGTH]>(0)?); + let w = row.get(1)?; + Ok((a, w)) + }, + )? + .collect::, _>>()?; + + // Return order of an SQL select statement is undefined, and + // mismatches are ignored. A post-processing is thus needed to + // generate a returned value complying to the batch-read spec. + Ok(addresses + .iter() + // Copying is necessary here since the same word could be + // returned multiple times. + .map(|addr| results.get(addr).copied()) + .collect()) + }) + .await + .map_err(Self::Error::from) + } + + async fn guarded_write( + &self, + guard: (Self::Address, Option), + bindings: Vec<(Self::Address, Self::Word)>, + ) -> Result, Self::Error> { + let (ag, wg) = guard; + + self.pool + .conn_mut(move |conn| { + let tx = conn.transaction_with_behavior( + async_sqlite::rusqlite::TransactionBehavior::Immediate, + )?; + + let current_word = tx + .query_row( + &format!("SELECT w FROM {} WHERE a = ?", FINDEX_TABLE_NAME), + [&*ag], + |row| row.get(0), + ) + .optional()?; + + if current_word == wg { + tx.execute( + &format!( + "INSERT OR REPLACE INTO {} (a, w) VALUES {}", + FINDEX_TABLE_NAME, + vec!["(?,?)"; bindings.len()].join(",") + ), + params_from_iter( + bindings + .iter() + // There seems to be no way to avoid cloning here. + .flat_map(|(a, w)| [a.to_vec(), w.to_vec()]), + ), + )?; + tx.commit()?; + } + + Ok(current_word) + }) + .await + .map_err(Self::Error::from) + } +} + +#[cfg(test)] +mod tests { + + use super::*; + use cosmian_findex::{ + WORD_LENGTH, gen_seed, test_guarded_write_concurrent, test_rw_same_address, + test_single_write_and_read, test_wrong_guard, + }; + + const DB_PATH: &str = "../../target/debug/sqlite-test.sqlite.db"; + + #[tokio::test] + async fn test_rw_seq() { + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH) + .await + .unwrap(); + test_single_write_and_read(&m, gen_seed()).await + } + + #[tokio::test] + async fn test_guard_seq() { + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH) + .await + .unwrap(); + test_wrong_guard(&m, gen_seed()).await + } + + #[tokio::test] + async fn test_collision_seq() { + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH) + .await + .unwrap(); + test_rw_same_address(&m, gen_seed()).await + } + + #[tokio::test] + async fn test_rw_ccr() { + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH) + .await + .unwrap(); + test_guarded_write_concurrent(&m, gen_seed(), Some(100)).await + } +} From 7c14a504a70ea289adf909870ac8bb02a59ae809 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Wed, 4 Jun 2025 16:21:22 +0200 Subject: [PATCH 13/60] feat: put tests and benches (untested) --- crate/findex/Cargo.toml | 3 +- .../findex/benches}/data/concurrent.dat | 0 .../findex/benches}/data/insert.dat | 0 .../findex/benches}/data/search.dat | 0 .../findex/benches}/make_figures.tex | 0 {examples => crate/findex/examples}/insert.rs | 0 crate/memories/Cargo.toml | 4 +- .../memories/benches}/benches.rs | 63 ++--------- crate/memories/examples/postgresql.rs | 105 ++++++++++++++++++ crate/memories/examples/redis.rs | 65 +++++++++++ crate/memories/examples/shared_utils.rs | 86 ++++++++++++++ crate/memories/examples/sqlite.rs | 62 +++++++++++ crate/memories/src/postgresql_mem/memory.rs | 26 ++--- crate/memories/src/redis_mem.rs | 18 +-- crate/memories/src/sqlite_mem.rs | 18 +-- gwt | 1 - 16 files changed, 362 insertions(+), 89 deletions(-) rename {benches => crate/findex/benches}/data/concurrent.dat (100%) rename {benches => crate/findex/benches}/data/insert.dat (100%) rename {benches => crate/findex/benches}/data/search.dat (100%) rename {benches => crate/findex/benches}/make_figures.tex (100%) rename {examples => crate/findex/examples}/insert.rs (100%) rename {benches => crate/memories/benches}/benches.rs (88%) create mode 100644 crate/memories/examples/postgresql.rs create mode 100644 crate/memories/examples/redis.rs create mode 100644 crate/memories/examples/shared_utils.rs create mode 100644 crate/memories/examples/sqlite.rs delete mode 160000 gwt diff --git a/crate/findex/Cargo.toml b/crate/findex/Cargo.toml index 05bb5c90..e9021b74 100644 --- a/crate/findex/Cargo.toml +++ b/crate/findex/Cargo.toml @@ -21,7 +21,7 @@ cosmian_crypto_core.workspace = true aes = "0.8" xts-mode = "0.5" -# Optional dependencies for test-utils +# Optional dependencies for testing and benchmarking tokio = { workspace = true, features = [ "rt-multi-thread", "macros", @@ -30,6 +30,5 @@ criterion = { workspace = true, optional = true } rand = { workspace = true, optional = true } rand_distr = { workspace = true, optional = true } - [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/benches/data/concurrent.dat b/crate/findex/benches/data/concurrent.dat similarity index 100% rename from benches/data/concurrent.dat rename to crate/findex/benches/data/concurrent.dat diff --git a/benches/data/insert.dat b/crate/findex/benches/data/insert.dat similarity index 100% rename from benches/data/insert.dat rename to crate/findex/benches/data/insert.dat diff --git a/benches/data/search.dat b/crate/findex/benches/data/search.dat similarity index 100% rename from benches/data/search.dat rename to crate/findex/benches/data/search.dat diff --git a/benches/make_figures.tex b/crate/findex/benches/make_figures.tex similarity index 100% rename from benches/make_figures.tex rename to crate/findex/benches/make_figures.tex diff --git a/examples/insert.rs b/crate/findex/examples/insert.rs similarity index 100% rename from examples/insert.rs rename to crate/findex/examples/insert.rs diff --git a/crate/memories/Cargo.toml b/crate/memories/Cargo.toml index 9f60437f..1c4914b6 100644 --- a/crate/memories/Cargo.toml +++ b/crate/memories/Cargo.toml @@ -17,6 +17,7 @@ path = "src/lib.rs" redis-mem = ["redis"] sqlite-mem = ["async-sqlite"] postgres-mem = ["tokio-postgres", "tokio", "deadpool-postgres"] +test-utils = ["cosmian_findex/test-utils"] [dependencies] cosmian_crypto_core.workspace = true @@ -28,11 +29,10 @@ redis = { version = "0.31", features = [ "connection-manager", "tokio-comp", ], optional = true } +tokio = { workspace = true, features = ["rt-multi-thread"], optional = true } tokio-postgres = { version = "0.7.9", optional = true, features = [ "array-impls", ] } -tokio = { workspace = true, features = ["rt-multi-thread"], optional = true } - [dev-dependencies] cosmian_findex = { path = "../findex", features = ["test-utils"] } diff --git a/benches/benches.rs b/crate/memories/benches/benches.rs similarity index 88% rename from benches/benches.rs rename to crate/memories/benches/benches.rs index 22d5ee16..166efe6b 100644 --- a/benches/benches.rs +++ b/crate/memories/benches/benches.rs @@ -2,23 +2,23 @@ // activated. #![allow(unused_imports, unused_variables, unused_mut, dead_code)] -use cosmian_crypto_core::{CsRng, reexport::rand_core::SeedableRng}; +use cosmian_crypto_core::{ + CsRng, + reexport::rand_core::{RngCore, SeedableRng}, +}; use cosmian_findex::{ bench_memory_contention, bench_memory_insert_multiple_bindings, bench_memory_one_to_many, bench_memory_search_multiple_bindings, bench_memory_search_multiple_keywords, }; use criterion::{Criterion, criterion_group, criterion_main}; -#[cfg(feature = "rust-mem")] -use cosmian_findex::InMemory; - #[cfg(feature = "sqlite-mem")] -use cosmian_findex::SqliteMemory; +use cosmian_findex_memories::SqliteMemory; #[cfg(feature = "sqlite-mem")] const SQLITE_PATH: &str = "./target/benches.sqlite"; #[cfg(feature = "redis-mem")] -use cosmian_findex::RedisMemory; +use cosmian_findex_memories::RedisMemory; #[cfg(feature = "redis-mem")] fn get_redis_url() -> String { @@ -30,7 +30,7 @@ fn get_redis_url() -> String { /// Refer to `src/memory/postgresql_store/memory.rs` for local setup instructions #[cfg(feature = "postgres-mem")] -use cosmian_findex::{PostgresMemory, PostgresMemoryError}; +use cosmian_findex_memories::{PostgresMemory, PostgresMemoryError}; #[cfg(feature = "postgres-mem")] fn get_postgresql_url() -> String { @@ -70,15 +70,6 @@ const N_PTS: usize = 9; fn bench_search_multiple_bindings(c: &mut Criterion) { let mut rng = CsRng::from_entropy(); - #[cfg(feature = "rust-mem")] - bench_memory_search_multiple_bindings( - "in-memory", - N_PTS, - async || InMemory::default(), - c, - &mut rng, - ); - #[cfg(feature = "redis-mem")] bench_memory_search_multiple_bindings( "Redis", @@ -117,15 +108,6 @@ fn bench_search_multiple_bindings(c: &mut Criterion) { fn bench_search_multiple_keywords(c: &mut Criterion) { let mut rng = CsRng::from_entropy(); - #[cfg(feature = "rust-mem")] - bench_memory_search_multiple_keywords( - "in-memory", - N_PTS, - async || InMemory::default(), - c, - &mut rng, - ); - #[cfg(feature = "redis-mem")] bench_memory_search_multiple_keywords( "Redis", @@ -164,19 +146,6 @@ fn bench_search_multiple_keywords(c: &mut Criterion) { fn bench_insert_multiple_bindings(c: &mut Criterion) { let mut rng = CsRng::from_entropy(); - #[cfg(feature = "rust-mem")] - bench_memory_insert_multiple_bindings( - "in-memory", - N_PTS, - async || InMemory::default(), - c, - async |m: &InMemory<_, _>| -> Result<(), String> { - m.clear(); - Ok(()) - }, - &mut rng, - ); - #[cfg(feature = "redis-mem")] bench_memory_insert_multiple_bindings( "Redis", @@ -220,19 +189,6 @@ fn bench_insert_multiple_bindings(c: &mut Criterion) { fn bench_contention(c: &mut Criterion) { let mut rng = CsRng::from_entropy(); - #[cfg(feature = "rust-mem")] - bench_memory_contention( - "in-memory", - N_PTS, - async || InMemory::default(), - c, - async |m: &InMemory<_, _>| -> Result<(), String> { - m.clear(); - Ok(()) - }, - &mut rng, - ); - #[cfg(feature = "redis-mem")] bench_memory_contention( "Redis", @@ -272,8 +228,9 @@ fn bench_contention(c: &mut Criterion) { #[cfg(any(feature = "redis-mem", feature = "postgres-mem"))] mod delayed_memory { - use cosmian_findex::{ - Address, MemoryADT, PostgresMemory, PostgresMemoryError, RedisMemory, RedisMemoryError, + use cosmian_findex::{Address, MemoryADT}; + use cosmian_findex_memories::{ + PostgresMemory, PostgresMemoryError, RedisMemory, RedisMemoryError, }; use rand::Rng; use rand_distr::StandardNormal; diff --git a/crate/memories/examples/postgresql.rs b/crate/memories/examples/postgresql.rs new file mode 100644 index 00000000..3f236d60 --- /dev/null +++ b/crate/memories/examples/postgresql.rs @@ -0,0 +1,105 @@ +//! This example show-cases the use of Findex to securely store a hash-map. +#[path = "shared_utils.rs"] +mod shared_utils; + +use cosmian_crypto_core::{reexport::rand_core::SeedableRng, CsRng, Secret}; +use cosmian_findex::{Address, Findex, IndexADT, MemoryEncryptionLayer, ADDRESS_LENGTH}; +use cosmian_findex_memories::{PostgresMemory, PostgresMemoryError}; +use deadpool_postgres::{Config, Pool}; +use futures::executor::block_on; +use shared_utils::{decoder, encoder, gen_index, WORD_LENGTH}; +use std::collections::HashMap; +use tokio_postgres::NoTls; + +const DB_URL: &str = "postgres://cosmian:cosmian@localhost/cosmian"; +const TABLE_NAME: &str = "findex_example"; + +async fn create_pool(db_url: &str) -> Result { + let mut pg_config = Config::new(); + pg_config.url = Some(db_url.to_string()); + let pool = pg_config.builder(NoTls)?.build()?; + Ok(pool) +} + +#[tokio::main] +async fn main() { + // For cryptographic applications, it is important to use a secure RNG. In + // Rust, those RNG implement the `CryptoRng` trait. + let mut rng = CsRng::from_entropy(); + + // Generate fresh Findex key. In practice only one user is in charge of + // generating the key (the administrator?): all users *must* share the same + // key in order to make the index inter-operable. + let key = Secret::random(&mut rng); + + // Generating the random index. + let index = gen_index(&mut rng); + + let pool = create_pool(DB_URL).await.unwrap(); + let m = PostgresMemory::, [u8; WORD_LENGTH]>::connect_with_pool( + pool.clone(), + TABLE_NAME.to_string(), + ) + .await + .unwrap(); + + // Notice we chose to not enable Tls: it's not needed for this example as we are + // using the encryption layer in top of the memory interface - i.e. the data is + // already encrypted before being sent to the database and Tls would add unnecessary overhead. + m.initialize_table(DB_URL.to_string(), TABLE_NAME.to_string(), NoTls) + .await + .unwrap(); + + // Addd the following service to your pg_service.conf file (usually under ~/.pg_service.conf): + // + // [cosmian_service] + // host=localhost + // dbname=cosmian + // user=cosmian + // password=cosmian + let memory = PostgresMemory::, [u8; WORD_LENGTH]>::connect_with_pool( + pool.clone(), + TABLE_NAME.to_string(), + ) + .await + .unwrap(); + + let encrypted_memory = MemoryEncryptionLayer::new(&key, memory); + + // Instantiating Findex requires passing the key, the memory used and the + // encoder and decoder. Quite simple, after all :) + let findex = Findex::< + WORD_LENGTH, // size of a word + u64, // type of a value + String, // type of an encoding error + _, // type of the memory + >::new(encrypted_memory, encoder, decoder); + + // Here we insert all bindings one by one, blocking on each call. A better + // way would be to performed all such calls in parallel using tasks. + index + .clone() + .into_iter() + .for_each(|(kw, vs)| block_on(findex.insert(kw, vs)).expect("insert failed")); + + // In order to verify insertion was correctly performed, we search for all + // the indexed keywords... + let res = index + .keys() + .cloned() + .map(|kw| (kw, block_on(findex.search(&kw)).expect("search failed"))) + .collect::>(); + + // ... and verify we get the whole index back! + assert_eq!(res, index); + + // Drop the table to avoid problems with subsequent runs + pool.get() + .await + .unwrap() + .execute(&format!("DROP table {};", TABLE_NAME), &[]) + .await + .unwrap(); + + println!("All good !"); +} diff --git a/crate/memories/examples/redis.rs b/crate/memories/examples/redis.rs new file mode 100644 index 00000000..d35209c7 --- /dev/null +++ b/crate/memories/examples/redis.rs @@ -0,0 +1,65 @@ +//! This example show-cases the use of Findex to securely store a hash-map. +#[path = "shared_utils.rs"] +mod shared_utils; + +use cosmian_crypto_core::{reexport::rand_core::SeedableRng, CsRng, Secret}; +use cosmian_findex::{Address, Findex, IndexADT, MemoryEncryptionLayer, ADDRESS_LENGTH}; +use cosmian_findex_memories::RedisMemory; +use futures::executor::block_on; +use shared_utils::{decoder, encoder, gen_index, WORD_LENGTH}; +use std::collections::HashMap; + +const DB_PATH: &str = "redis://localhost:6379"; + +#[tokio::main] +async fn main() { + // For cryptographic applications, it is important to use a secure RNG. In + // Rust, those RNG implement the `CryptoRng` trait. + let mut rng = CsRng::from_entropy(); + + // Generate fresh Findex key. In practice only one user is in charge of + // generating the key (the administrator?): all users *must* share the same + // key in order to make the index inter-operable. + let key = Secret::random(&mut rng); + + // Generating the random index. + let index = gen_index(&mut rng); + + // For this example, we use the `RedisMemory` implementation of the `MemoryADT` + // trait. It connects to a Redis instance and uses it as a key-value store + // for our Findex data structures, which is suitable for production applications. + let memory = RedisMemory::, [u8; WORD_LENGTH]>::connect(DB_PATH) + .await + .unwrap(); + + let encrypted_memory = MemoryEncryptionLayer::new(&key, memory); + + // Instantiating Findex requires passing the key, the memory used and the + // encoder and decoder. Quite simple, after all :) + let findex = Findex::< + WORD_LENGTH, // size of a word + u64, // type of a value + String, // type of an encoding error + _, // type of the memory + >::new(encrypted_memory, encoder, decoder); + + // Here we insert all bindings one by one, blocking on each call. A better + // way would be to performed all such calls in parallel using tasks. + index + .clone() + .into_iter() + .for_each(|(kw, vs)| block_on(findex.insert(kw, vs)).expect("insert failed")); + + // In order to verify insertion was correctly performed, we search for all + // the indexed keywords... + let res = index + .keys() + .cloned() + .map(|kw| (kw, block_on(findex.search(&kw)).expect("search failed"))) + .collect::>(); + + // ... and verify we get the whole index back! + assert_eq!(res, index); + + println!("All good !"); +} diff --git a/crate/memories/examples/shared_utils.rs b/crate/memories/examples/shared_utils.rs new file mode 100644 index 00000000..f2d66e37 --- /dev/null +++ b/crate/memories/examples/shared_utils.rs @@ -0,0 +1,86 @@ +//! This example show-cases the use of Findex to securely store a hash-map. +use cosmian_crypto_core::reexport::rand_core::CryptoRngCore; +use cosmian_findex::Op; +use std::collections::{HashMap, HashSet}; + +/// This function generates a random set of (key, values) couples. Since Findex +/// API is those of an Index which only returns the *set* of values associated +/// to a given key, all random values generated here are stored in a `HashSet`. +/// +/// In Findex's jargon, we say that these sets of values are *bound* to their +/// associated *keyword* of type `[u8; 8]` in this example. Since Findex only +/// requires from the values to be hashable, we could have taken any type that +/// implements `Hash` instead of the `u64`. +pub fn gen_index(rng: &mut impl CryptoRngCore) -> HashMap<[u8; 8], HashSet> { + (0..6) + .map(|i| { + let kw = rng.next_u64().to_be_bytes(); + let vals = (0..10_i64.pow(i) as usize) + .map(|_| rng.next_u64()) + .collect::>(); + (kw, vals) + }) + .collect() +} + +/// The encoder will use 1 bit to encode the operation (insert or delete), and 7 +/// bits to encode the number of values (of type u64) in the word. This allows +/// for words of 2^7 = 128 values, which are serialized into an array of 8 bytes +/// each. The `WORD_LENGTH` is therefore 1 byte of metadata plus 128 * 8 bytes +/// of values. +pub const WORD_LENGTH: usize = 1 + 8 * 128; + +pub fn encoder(op: Op, values: HashSet) -> Result, String> { + let mut words = Vec::new(); // This could be initialized with the correct size. + let mut values = values.into_iter().peekable(); + while values.peek().is_some() { + let chunk = (0..128) + .filter_map(|_| values.next()) + .map(|v| v.to_be_bytes()) + .collect::>(); + + let metadata = + ::try_from(chunk.len() - 1).unwrap() + if let Op::Insert = op { 128 } else { 0 }; + let mut word = [0; WORD_LENGTH]; + word[0] = metadata; + chunk + .into_iter() + .enumerate() + .for_each(|(i, v)| word[1 + i * 8..1 + (i + 1) * 8].copy_from_slice(v.as_slice())); + words.push(word); + } + Ok(words) +} + +pub fn decoder(words: Vec<[u8; WORD_LENGTH]>) -> Result, String> { + let mut values = HashSet::new(); + words.into_iter().for_each(|w| { + let metadata = w[0]; + + // Extract the highest bit to recover the operation. + let op = if metadata < 128 { + Op::Delete + } else { + Op::Insert + }; + + // Remove the highest bit to recover the number of values. + let n = metadata & 127; + + for i in 0..=n as usize { + let v = u64::from_be_bytes(w[1 + i * 8..1 + (i + 1) * 8].try_into().unwrap()); + if let Op::Insert = op { + values.insert(v); + } else { + values.remove(&v); + } + } + }); + + Ok(values) +} + +#[allow(dead_code)] +fn main() { + panic!("This is a utility module and should not be run directly."); +} diff --git a/crate/memories/examples/sqlite.rs b/crate/memories/examples/sqlite.rs new file mode 100644 index 00000000..4d5bce5c --- /dev/null +++ b/crate/memories/examples/sqlite.rs @@ -0,0 +1,62 @@ +//! This example show-cases the use of Findex to securely store a hash-map. +#[path = "shared_utils.rs"] +mod shared_utils; + +use cosmian_crypto_core::{reexport::rand_core::SeedableRng, CsRng, Secret}; +use cosmian_findex::{Findex, IndexADT, MemoryEncryptionLayer}; +use cosmian_findex_memories::SqliteMemory; +use futures::executor::block_on; +use shared_utils::{decoder, encoder, gen_index, WORD_LENGTH}; +use std::collections::HashMap; + +const DB_PATH: &str = "./target/debug/sqlite-test.db"; + +#[tokio::main] +async fn main() { + // For cryptographic applications, it is important to use a secure RNG. In + // Rust, those RNG implement the `CryptoRng` trait. + let mut rng = CsRng::from_entropy(); + + // Generate fresh Findex key. In practice only one user is in charge of + // generating the key (the administrator?): all users *must* share the same + // key in order to make the index inter-operable. + let key = Secret::random(&mut rng); + + // Generating the random index. + let index = gen_index(&mut rng); + + let memory = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH) + .await + .unwrap(); + + let encrypted_memory = MemoryEncryptionLayer::new(&key, memory); + + // Instantiating Findex requires passing the key, the memory used and the + // encoder and decoder. Quite simple, after all :) + let findex = Findex::< + WORD_LENGTH, // size of a word + u64, // type of a value + String, // type of an encoding error + _, // type of the memory + >::new(encrypted_memory, encoder, decoder); + + // Here we insert all bindings one by one, blocking on each call. A better + // way would be to performed all such calls in parallel using tasks. + index + .clone() + .into_iter() + .for_each(|(kw, vs)| block_on(findex.insert(kw, vs)).expect("insert failed")); + + // In order to verify insertion was correctly performed, we search for all + // the indexed keywords... + let res = index + .keys() + .cloned() + .map(|kw| (kw, block_on(findex.search(&kw)).expect("search failed"))) + .collect::>(); + + // ... and verify we get the whole index back! + assert_eq!(res, index); + + println!("All good !"); +} diff --git a/crate/memories/src/postgresql_mem/memory.rs b/crate/memories/src/postgresql_mem/memory.rs index eaeffd4e..eba6b768 100644 --- a/crate/memories/src/postgresql_mem/memory.rs +++ b/crate/memories/src/postgresql_mem/memory.rs @@ -3,8 +3,8 @@ use cosmian_findex::{Address, MemoryADT}; use deadpool_postgres::Pool; use std::marker::PhantomData; use tokio_postgres::{ - tls::{MakeTlsConnect, TlsConnect}, Socket, + tls::{MakeTlsConnect, TlsConnect}, }; #[derive(Clone, Debug)] @@ -75,16 +75,16 @@ impl }) } - // /// Deletes all rows from the findex memory table - // #[cfg(feature = "test-utils")] - // pub async fn clear(&self) -> Result<(), PostgresMemoryError> { - // self.pool - // .get() - // .await? - // .execute(&format!("TRUNCATE TABLE {};", self.table_name), &[]) - // .await?; - // Ok(()) - // } + /// Deletes all rows from the findex memory table + #[cfg(feature = "test-utils")] + pub async fn clear(&self) -> Result<(), PostgresMemoryError> { + self.pool + .get() + .await? + .execute(&format!("TRUNCATE TABLE {};", self.table_name), &[]) + .await?; + Ok(()) + } } impl MemoryADT @@ -243,8 +243,8 @@ mod tests { use super::*; use cosmian_findex::{ - gen_seed, test_guarded_write_concurrent, test_rw_same_address, test_single_write_and_read, - test_wrong_guard, ADDRESS_LENGTH, WORD_LENGTH, + ADDRESS_LENGTH, WORD_LENGTH, gen_seed, test_guarded_write_concurrent, test_rw_same_address, + test_single_write_and_read, test_wrong_guard, }; const DB_URL: &str = "postgres://cosmian:cosmian@localhost/cosmian"; diff --git a/crate/memories/src/redis_mem.rs b/crate/memories/src/redis_mem.rs index 03155a4e..a39747d6 100644 --- a/crate/memories/src/redis_mem.rs +++ b/crate/memories/src/redis_mem.rs @@ -84,13 +84,13 @@ impl RedisMemory { Self::connect_with_manager(manager).await } - // #[cfg(feature = "test-utils")] - // pub async fn clear(&self) -> Result<(), RedisMemoryError> { - // redis::cmd("FLUSHDB") - // .query_async(&mut self.manager.clone()) - // .await - // .map_err(RedisMemoryError::RedisError) - // } + #[cfg(feature = "test-utils")] + pub async fn clear(&self) -> Result<(), RedisMemoryError> { + redis::cmd("FLUSHDB") + .query_async(&mut self.manager.clone()) + .await + .map_err(RedisMemoryError::RedisError) + } } impl MemoryADT @@ -146,8 +146,8 @@ mod tests { use super::*; use cosmian_findex::{ - gen_seed, test_guarded_write_concurrent, test_rw_same_address, test_single_write_and_read, - test_wrong_guard, WORD_LENGTH, + WORD_LENGTH, gen_seed, test_guarded_write_concurrent, test_rw_same_address, + test_single_write_and_read, test_wrong_guard, }; fn get_redis_url() -> String { diff --git a/crate/memories/src/sqlite_mem.rs b/crate/memories/src/sqlite_mem.rs index 424b390b..fd5940b1 100644 --- a/crate/memories/src/sqlite_mem.rs +++ b/crate/memories/src/sqlite_mem.rs @@ -106,15 +106,15 @@ impl SqliteMemory { } } -// impl SqliteMemory { -// #[cfg(feature = "test-utils")] -// pub async fn clear(&self) -> Result<(), SqliteMemoryError> { -// self.pool -// .conn(|cnx| cnx.execute(&format!("DELETE FROM {FINDEX_TABLE_NAME}"), [])) -// .await?; -// Ok(()) -// } -// } +impl SqliteMemory { + #[cfg(feature = "test-utils")] + pub async fn clear(&self) -> Result<(), SqliteMemoryError> { + self.pool + .conn(|cnx| cnx.execute(&format!("DELETE FROM {FINDEX_TABLE_NAME}"), [])) + .await?; + Ok(()) + } +} impl MemoryADT for SqliteMemory, [u8; WORD_LENGTH]> diff --git a/gwt b/gwt deleted file mode 160000 index c6f07d1f..00000000 --- a/gwt +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c6f07d1fba3a72acc0fe3c3b74cb97807f3b62af From c59099472c3cbd248df9f38715512433d0121b82 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Wed, 4 Jun 2025 17:26:19 +0200 Subject: [PATCH 14/60] feat: almosy finish benches --- Cargo.toml | 13 +--------- crate/findex/Cargo.toml | 10 ++++++++ crate/memories/Cargo.toml | 36 +++++++++++++-------------- crate/memories/examples/postgresql.rs | 8 +++--- crate/memories/examples/redis.rs | 8 +++--- crate/memories/examples/sqlite.rs | 6 ++--- 6 files changed, 40 insertions(+), 41 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5df50b1c..30a45c2e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,4 @@ [workspace] -default-members = ["crate/findex"] members = ["crate/findex", "crate/memories"] resolver = "2" @@ -19,6 +18,7 @@ keywords = ["SSE"] license = "BUSL-1.1" repository = "https://github.com/Cosmian/findex/" description = "Symmetric Searchable Encryption" +readme = "README.md" [workspace.dependencies] @@ -30,14 +30,3 @@ criterion = { version = "0.6" } rand = { version = "0.9.0" } rand_distr = { version = "0.5.1" } tokio = { version = "1.44" } - - -# TODO: later -# [[bench]] -# name = "benches" -# harness = false -# required-features = ["test-utils"] - -# [[example]] -# name = "insert" -# required-features = ["test-utils"] diff --git a/crate/findex/Cargo.toml b/crate/findex/Cargo.toml index e9021b74..7ac6a29a 100644 --- a/crate/findex/Cargo.toml +++ b/crate/findex/Cargo.toml @@ -8,6 +8,7 @@ keywords.workspace = true license.workspace = true repository.workspace = true description.workspace = true +readme = "README.md" [lib] name = "cosmian_findex" @@ -32,3 +33,12 @@ rand_distr = { workspace = true, optional = true } [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } + +[[bench]] +name = "benches" +harness = false +required-features = ["test-utils"] + +[[example]] +name = "insert" +required-features = ["test-utils"] diff --git a/crate/memories/Cargo.toml b/crate/memories/Cargo.toml index 1c4914b6..8fac7f9d 100644 --- a/crate/memories/Cargo.toml +++ b/crate/memories/Cargo.toml @@ -4,10 +4,11 @@ version.workspace = true authors.workspace = true license.workspace = true edition.workspace = true -# categories.workspace = true -# keywords.workspace = true -# repository.workspace = true -# description.workspace = true +categories.workspace = true +keywords.workspace = true +repository.workspace = true +description.workspace = true +readme.workspace = true [lib] name = "cosmian_findex_memories" @@ -42,20 +43,19 @@ criterion = { workspace = true } rand = { workspace = true } rand_distr = { workspace = true } -# Optional dependencies for t TOD TODO -# [[bench]] -# name = "benches" -# harness = false -# required-features = ["redis-mem", "sqlite-mem", "postgres-mem"] +[[bench]] +name = "benches" +harness = false +required-features = ["redis-mem", "sqlite-mem", "postgres-mem", "test-utils"] -# [[example]] -# name = "redis" -# required-features = ["redis-mem"] +[[example]] +name = "redis" +required-features = ["redis-mem"] -# [[example]] -# name = "sqlite" -# required-features = ["sqlite-mem"] +[[example]] +name = "sqlite" +required-features = ["sqlite-mem"] -# [[example]] -# name = "postgresql" -# required-features = ["postgres-mem"] +[[example]] +name = "postgresql" +required-features = ["postgres-mem"] diff --git a/crate/memories/examples/postgresql.rs b/crate/memories/examples/postgresql.rs index 3f236d60..46502ed0 100644 --- a/crate/memories/examples/postgresql.rs +++ b/crate/memories/examples/postgresql.rs @@ -1,13 +1,13 @@ -//! This example show-cases the use of Findex to securely store a hash-map. +//! This example show-cases the use of Findex to securely store a hash-map with postgresql. #[path = "shared_utils.rs"] mod shared_utils; -use cosmian_crypto_core::{reexport::rand_core::SeedableRng, CsRng, Secret}; -use cosmian_findex::{Address, Findex, IndexADT, MemoryEncryptionLayer, ADDRESS_LENGTH}; +use cosmian_crypto_core::{CsRng, Secret, reexport::rand_core::SeedableRng}; +use cosmian_findex::{ADDRESS_LENGTH, Address, Findex, IndexADT, MemoryEncryptionLayer}; use cosmian_findex_memories::{PostgresMemory, PostgresMemoryError}; use deadpool_postgres::{Config, Pool}; use futures::executor::block_on; -use shared_utils::{decoder, encoder, gen_index, WORD_LENGTH}; +use shared_utils::{WORD_LENGTH, decoder, encoder, gen_index}; use std::collections::HashMap; use tokio_postgres::NoTls; diff --git a/crate/memories/examples/redis.rs b/crate/memories/examples/redis.rs index d35209c7..c0142bdf 100644 --- a/crate/memories/examples/redis.rs +++ b/crate/memories/examples/redis.rs @@ -1,12 +1,12 @@ -//! This example show-cases the use of Findex to securely store a hash-map. +//! This example show-cases the use of Findex to securely store a hash-map with redis. #[path = "shared_utils.rs"] mod shared_utils; -use cosmian_crypto_core::{reexport::rand_core::SeedableRng, CsRng, Secret}; -use cosmian_findex::{Address, Findex, IndexADT, MemoryEncryptionLayer, ADDRESS_LENGTH}; +use cosmian_crypto_core::{CsRng, Secret, reexport::rand_core::SeedableRng}; +use cosmian_findex::{ADDRESS_LENGTH, Address, Findex, IndexADT, MemoryEncryptionLayer}; use cosmian_findex_memories::RedisMemory; use futures::executor::block_on; -use shared_utils::{decoder, encoder, gen_index, WORD_LENGTH}; +use shared_utils::{WORD_LENGTH, decoder, encoder, gen_index}; use std::collections::HashMap; const DB_PATH: &str = "redis://localhost:6379"; diff --git a/crate/memories/examples/sqlite.rs b/crate/memories/examples/sqlite.rs index 4d5bce5c..2f57108d 100644 --- a/crate/memories/examples/sqlite.rs +++ b/crate/memories/examples/sqlite.rs @@ -1,12 +1,12 @@ -//! This example show-cases the use of Findex to securely store a hash-map. +//! This example show-cases the use of Findex to securely store a hash-map with sqlite. #[path = "shared_utils.rs"] mod shared_utils; -use cosmian_crypto_core::{reexport::rand_core::SeedableRng, CsRng, Secret}; +use cosmian_crypto_core::{CsRng, Secret, reexport::rand_core::SeedableRng}; use cosmian_findex::{Findex, IndexADT, MemoryEncryptionLayer}; use cosmian_findex_memories::SqliteMemory; use futures::executor::block_on; -use shared_utils::{decoder, encoder, gen_index, WORD_LENGTH}; +use shared_utils::{WORD_LENGTH, decoder, encoder, gen_index}; use std::collections::HashMap; const DB_PATH: &str = "./target/debug/sqlite-test.db"; From f5ed4f5d97c153eaaae7385893225ed29c550df0 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Wed, 4 Jun 2025 17:30:51 +0200 Subject: [PATCH 15/60] feat: well documented README.md files --- README.md | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 45d055ac..792113a2 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,33 @@ -# Findex +# Findex: Symmetric Searchable Encryption -To build Findex simply run: +[![Crates.io](https://img.shields.io/crates/v/cosmian_findex.svg)](https://crates.io/crates/cosmian_findex) +[![Documentation](https://docs.rs/cosmian_findex/badge.svg)](https://docs.rs/cosmian_findex) +[![License](https://img.shields.io/badge/License-BUSL--1.1-blue.svg)](LICENSE) -```bash -cargo build --release -``` +Findex is a Symmetric Searchable Encryption (SSE) library that enables encrypted search over encrypted data. It allows you to securely index and search encrypted data without compromising privacy or security. -To test, run: +## Architecture -```bash -cargo test --release --all-features -``` +This repository is organized as a Rust workspace with two crates: + +- `cosmian_findex`: Core library implementing the SSE algorithms +- `cosmian_findex_memories`: Storage backend implementations for different databases + +## Installation -To launch the benchmarks, run: +Add `cosmian_findex` to your Cargo.toml: -```bash -cargo bench --all-features +```toml +[dependencies] +cosmian_findex = "7.1.0" +# Optional - include backend implementations +cosmian_findex_memories = { version = "7.1.0", features = ["redis-mem", "sqlite-mem", "postgres-mem"] } ``` -Note that benches are quite involving and require *several hours* for a full -run. Once all benchmarks are run, you will find detailed reports under `target/criterion`. +## Related Projects +[Findex Server](github.com/cosmian/findex-server) - A production-ready Findex server implementation + +## License + +This project is licensed under the Business Source License 1.1 (BUSL-1.1). + From 769a244cf64c5b50c3bcd2c5446188e51cbf3c29 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Wed, 4 Jun 2025 17:30:59 +0200 Subject: [PATCH 16/60] feat: well documented README.md files2 --- crate/findex/README.md | 23 +++++++++ crate/findex/benches/benches.rs | 82 +++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 crate/findex/README.md create mode 100644 crate/findex/benches/benches.rs diff --git a/crate/findex/README.md b/crate/findex/README.md new file mode 100644 index 00000000..32ec1964 --- /dev/null +++ b/crate/findex/README.md @@ -0,0 +1,23 @@ +# Findex + +To build Findex simply run: + +```bash +cargo build --release +``` + +To test, run: + +```bash +cargo test --release --all-features +``` + +To launch the benchmarks, run: + +```bash +cargo bench --all-features +``` + +Note that benches are quite involving and require *several hours* for a full +run. Once all benchmarks are run, you will find detailed reports under `target/criterion`. +findex diff --git a/crate/findex/benches/benches.rs b/crate/findex/benches/benches.rs new file mode 100644 index 00000000..6ac23f71 --- /dev/null +++ b/crate/findex/benches/benches.rs @@ -0,0 +1,82 @@ +// It is too much trouble to deactivate everything if none of the features are +// activated. +#![allow(unused_imports, unused_variables, unused_mut, dead_code)] + +use cosmian_crypto_core::{CsRng, reexport::rand_core::SeedableRng}; +use cosmian_findex::{ADDRESS_LENGTH, Address, InMemory, WORD_LENGTH}; +use cosmian_findex::{ + bench_memory_contention, bench_memory_insert_multiple_bindings, bench_memory_one_to_many, + bench_memory_search_multiple_bindings, bench_memory_search_multiple_keywords, +}; +use criterion::{Criterion, criterion_group, criterion_main}; + +// Number of points in each graph. +const N_PTS: usize = 9; + +fn bench_search_multiple_bindings(c: &mut Criterion) { + let mut rng = CsRng::from_entropy(); + + bench_memory_search_multiple_bindings( + "in-memory", + N_PTS, + async || InMemory::default(), + c, + &mut rng, + ); +} + +fn bench_search_multiple_keywords(c: &mut Criterion) { + let mut rng = CsRng::from_entropy(); + + bench_memory_search_multiple_keywords( + "in-memory", + N_PTS, + async || InMemory::default(), + c, + &mut rng, + ); +} + +fn bench_insert_multiple_bindings(c: &mut Criterion) { + let mut rng = CsRng::from_entropy(); + + bench_memory_insert_multiple_bindings( + "in-memory", + N_PTS, + async || InMemory::default(), + c, + async |m: &InMemory<_, _>| -> Result<(), String> { + m.clear(); + Ok(()) + }, + &mut rng, + ); +} + +fn bench_contention(c: &mut Criterion) { + let mut rng = CsRng::from_entropy(); + + bench_memory_contention( + "in-memory", + N_PTS, + async || InMemory::default(), + c, + async |m: &InMemory<_, _>| -> Result<(), String> { + m.clear(); + Ok(()) + }, + &mut rng, + ); +} + +criterion_group!( + name = benches; + config = Criterion::default(); + targets = + bench_search_multiple_bindings, + bench_search_multiple_keywords, + bench_insert_multiple_bindings, + bench_contention, +); + +criterion_main!(benches); From 5ebeaeb3f81b4c26a3c0bf8fbcd53376cf55b2a4 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Wed, 4 Jun 2025 17:31:17 +0200 Subject: [PATCH 17/60] feat: well documented README.md files3 --- crate/memories/README.md | 70 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 crate/memories/README.md diff --git a/crate/memories/README.md b/crate/memories/README.md new file mode 100644 index 00000000..483cd8e0 --- /dev/null +++ b/crate/memories/README.md @@ -0,0 +1,70 @@ +# Findex Memories + +A collection of memory implementations for Findex, a concurrent and database-agnostic Searchable Encryption scheme. + +## Overview + +Findex Memories provides "pluggable" storage backends for Findex, allowing the core Findex library to remain database-agnostic while supporting various storage systems. This separation enables users to integrate findex to their prefered database sytem. + +## Available Storage Backends + +This library provides implementations for the following storage systems: + + +| Feature | Database | Dependencies | +|---------|----------|--------------| +| `redis-mem` | Redis | [redis](https://crates.io/crates/redis) v0.31 | +| `sqlite-mem` | SQLite | [async-sqlite](https://crates.io/crates/async-sqlite) v0.4 * | +| `postgres-mem` | PostgreSQL | [tokio-postgres](https://crates.io/crates/tokio-postgres) v0.7.9
[tokio](https://crates.io/crates/tokio) v1.44
[deadpool-postgres](https://crates.io/crates/deadpool-postgres) v0.14.1 | + + +## Usage + +First, add `cosmian_findex_memories` as dependency to your project : + +```bash +cargo add cosmian_findex_memories # do not forget to enable the adequate feature for the backend you want to use ! +``` + +If you don't have a running `Redis` or `Postgres` instance running, you can use the one provided on the root by running `docker-compose up`, then on your application's code : + +```rust +// For Redis +use cosmian_findex_memories::postgres::RedisMemory; +use cosmian_findex::{ADDRESS_LENGTH, Findex, Address, dummy_decode, dummy_encode,WORD_LENGTH}; + +let memory = RedisMemory::, [u8; WORD_LENGTH]>::connect( + "redis://localhost:6379", +).await.unwrap(); + +// optionally, add the encryption layer (recommended) +// let memory = MemoryEncryptionLayer::new(&key, InMemory::default()); + +let findex = Findex::new(memory, dummy_encode::, dummy_decode); + +let cat_bindings = [ + Value::try_from(1).unwrap(), + Value::try_from(3).unwrap(), + Value::try_from(5).unwrap(), +]; + +findex.insert("cat".to_string(), cat_bindings.clone()).await.unwrap(); + +let cat_res = findex.search(&"cat".to_string()).unwrap(); + +assert_eq!( + cat_bindings.iter().cloned().collect::>(), + cat_res +); +``` + +More detailed examples can be found under the [examples folder](examples). + +## Related Projects +- [Findex](github.com/cosmian/findex) - The core Findex library +- [Findex Server](github.com/cosmian/findex-server) - A production-ready Findex server implementation + +## License + +This project is licensed under the Business Source License 1.1 (BUSL-1.1). + From e504d37d6cfdf54b641c0a7ae3162cbd45a54d3c Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Wed, 4 Jun 2025 18:01:06 +0200 Subject: [PATCH 18/60] chore: cargo fmt --- crate/memories/benches/benches.rs | 2 +- crate/memories/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crate/memories/benches/benches.rs b/crate/memories/benches/benches.rs index 166efe6b..1bd1d299 100644 --- a/crate/memories/benches/benches.rs +++ b/crate/memories/benches/benches.rs @@ -15,7 +15,7 @@ use criterion::{Criterion, criterion_group, criterion_main}; #[cfg(feature = "sqlite-mem")] use cosmian_findex_memories::SqliteMemory; #[cfg(feature = "sqlite-mem")] -const SQLITE_PATH: &str = "./target/benches.sqlite"; +const SQLITE_PATH: &str = "benches.sqlite.db"; #[cfg(feature = "redis-mem")] use cosmian_findex_memories::RedisMemory; diff --git a/crate/memories/src/lib.rs b/crate/memories/src/lib.rs index 59408d97..9e373419 100644 --- a/crate/memories/src/lib.rs +++ b/crate/memories/src/lib.rs @@ -6,7 +6,7 @@ pub use redis_mem::{RedisMemory, RedisMemoryError}; #[cfg(feature = "sqlite-mem")] mod sqlite_mem; #[cfg(feature = "sqlite-mem")] -pub use sqlite_mem::{SqliteMemory, SqliteMemoryError, FINDEX_TABLE_NAME}; +pub use sqlite_mem::{FINDEX_TABLE_NAME, SqliteMemory, SqliteMemoryError}; #[cfg(feature = "postgres-mem")] mod postgresql_mem; From 0b3ef595a9f270a2b858c8925a5c2e70290add0b Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Thu, 5 Jun 2025 17:14:53 +0200 Subject: [PATCH 19/60] feat: fix ci ? --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8efde84..d64ad93d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,13 +5,13 @@ on: push jobs: cargo-lint: - uses: Cosmian/reusable_workflows/.github/workflows/cargo-nursery.yml@develop + uses: Cosmian/reusable_workflows/.github/workflows/cargo-nursery.yml@publish_workspace_fix with: toolchain: 1.85.0 cargo-publish: needs: - cargo-lint - uses: Cosmian/reusable_workflows/.github/workflows/cargo-publish.yml@develop + uses: Cosmian/reusable_workflows/.github/workflows/cargo-publish.yml@publish_workspace_fix if: startsWith(github.ref, 'refs/tags/') with: toolchain: 1.85.0 @@ -19,5 +19,5 @@ jobs: cleanup: needs: - cargo-lint - uses: Cosmian/reusable_workflows/.github/workflows/cleanup_cache.yml@develop + uses: Cosmian/reusable_workflows/.github/workflows/cleanup_cache.yml@publish_workspace_fix secrets: inherit From 504a7d50c0314e6314cb71c65635f90055ef7299 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Thu, 5 Jun 2025 17:16:54 +0200 Subject: [PATCH 20/60] feat: fix ci 2 --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d64ad93d..2f0c5b5c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,13 +5,13 @@ on: push jobs: cargo-lint: - uses: Cosmian/reusable_workflows/.github/workflows/cargo-nursery.yml@publish_workspace_fix + uses: Cosmian/reusable_workflows/.github/workflows/cargo-nursery.yml@feat/publish_workspace_fix with: toolchain: 1.85.0 cargo-publish: needs: - cargo-lint - uses: Cosmian/reusable_workflows/.github/workflows/cargo-publish.yml@publish_workspace_fix + uses: Cosmian/reusable_workflows/.github/workflows/cargo-publish.yml@feat/publish_workspace_fix if: startsWith(github.ref, 'refs/tags/') with: toolchain: 1.85.0 @@ -19,5 +19,5 @@ jobs: cleanup: needs: - cargo-lint - uses: Cosmian/reusable_workflows/.github/workflows/cleanup_cache.yml@publish_workspace_fix + uses: Cosmian/reusable_workflows/.github/workflows/cleanup_cache.yml@feat/publish_workspace_fix secrets: inherit From 630ccccdca4c2301a53fb3146d7716b1052d7c33 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Thu, 5 Jun 2025 17:18:15 +0200 Subject: [PATCH 21/60] feat: fix ci 3 --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f0c5b5c..d785a02d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,7 @@ jobs: uses: Cosmian/reusable_workflows/.github/workflows/cargo-nursery.yml@feat/publish_workspace_fix with: toolchain: 1.85.0 + workspace: true cargo-publish: needs: - cargo-lint From 67e2a28e288144bdf4376b07306336626af08525 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Thu, 5 Jun 2025 17:21:51 +0200 Subject: [PATCH 22/60] feat: fix ci 4 --- crate/findex/Cargo.toml | 4 ++-- crate/memories/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crate/findex/Cargo.toml b/crate/findex/Cargo.toml index 7ac6a29a..11836580 100644 --- a/crate/findex/Cargo.toml +++ b/crate/findex/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cosmian_findex" -version.workspace = true +version = "7.2.0" authors.workspace = true categories.workspace = true edition.workspace = true @@ -8,7 +8,7 @@ keywords.workspace = true license.workspace = true repository.workspace = true description.workspace = true -readme = "README.md" +readme.workspace = true [lib] name = "cosmian_findex" diff --git a/crate/memories/Cargo.toml b/crate/memories/Cargo.toml index 8fac7f9d..5117cd09 100644 --- a/crate/memories/Cargo.toml +++ b/crate/memories/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cosmian_findex_memories" -version.workspace = true +version = "1.0.0" authors.workspace = true license.workspace = true edition.workspace = true From c3bec15f3a3469b8855ea6e5fbf77d0a2c107aa6 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Thu, 5 Jun 2025 17:26:20 +0200 Subject: [PATCH 23/60] feat: fix ci 5 --- Cargo.toml | 2 +- crate/findex/Cargo.toml | 2 +- crate/memories/Cargo.toml | 6 ++++-- crate/memories/README.md | 9 --------- 4 files changed, 6 insertions(+), 13 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 30a45c2e..39e422f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["crate/findex", "crate/memories"] resolver = "2" [workspace.package] -version = "7.1.0" +version = "1.0.1" authors = [ "Bruno Grieder ", "CĂ©lia Corsin ", diff --git a/crate/findex/Cargo.toml b/crate/findex/Cargo.toml index 11836580..7d2c32ef 100644 --- a/crate/findex/Cargo.toml +++ b/crate/findex/Cargo.toml @@ -8,7 +8,7 @@ keywords.workspace = true license.workspace = true repository.workspace = true description.workspace = true -readme.workspace = true +readme = "README.md" [lib] name = "cosmian_findex" diff --git a/crate/memories/Cargo.toml b/crate/memories/Cargo.toml index 5117cd09..00a2e586 100644 --- a/crate/memories/Cargo.toml +++ b/crate/memories/Cargo.toml @@ -8,7 +8,7 @@ categories.workspace = true keywords.workspace = true repository.workspace = true description.workspace = true -readme.workspace = true +readme = "README.md" [lib] name = "cosmian_findex_memories" @@ -22,7 +22,9 @@ test-utils = ["cosmian_findex/test-utils"] [dependencies] cosmian_crypto_core.workspace = true -cosmian_findex = { path = "../findex" } +cosmian_findex = { path = "../findex", version = "7.2.0", features = [ + "test-utils", +] } async-sqlite = { version = "0.4", optional = true } # `async-sqlite` dependency is pinned at v0.4 due to dependency tree issues with sqlx requiring a lower version of `libsqlite3-sys` deadpool-postgres = { version = "0.14.1", optional = true } redis = { version = "0.31", features = [ diff --git a/crate/memories/README.md b/crate/memories/README.md index 483cd8e0..5cd5dd7a 100644 --- a/crate/memories/README.md +++ b/crate/memories/README.md @@ -59,12 +59,3 @@ assert_eq!( ``` More detailed examples can be found under the [examples folder](examples). - -## Related Projects -- [Findex](github.com/cosmian/findex) - The core Findex library -- [Findex Server](github.com/cosmian/findex-server) - A production-ready Findex server implementation - -## License - -This project is licensed under the Business Source License 1.1 (BUSL-1.1). - From 54637077f2f1ce63c06a5cb85ac92fcf1dedd2ac Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Thu, 5 Jun 2025 17:29:15 +0200 Subject: [PATCH 24/60] feat: fix ci 6 --- Cargo.toml | 2 +- crate/memories/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 39e422f8..3da8f512 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["crate/findex", "crate/memories"] resolver = "2" [workspace.package] -version = "1.0.1" +version = "7.2.0" authors = [ "Bruno Grieder ", "CĂ©lia Corsin ", diff --git a/crate/memories/Cargo.toml b/crate/memories/Cargo.toml index 00a2e586..38257504 100644 --- a/crate/memories/Cargo.toml +++ b/crate/memories/Cargo.toml @@ -22,7 +22,7 @@ test-utils = ["cosmian_findex/test-utils"] [dependencies] cosmian_crypto_core.workspace = true -cosmian_findex = { path = "../findex", version = "7.2.0", features = [ +cosmian_findex = { path = "../findex", version = "7.1", features = [ "test-utils", ] } async-sqlite = { version = "0.4", optional = true } # `async-sqlite` dependency is pinned at v0.4 due to dependency tree issues with sqlx requiring a lower version of `libsqlite3-sys` From 3dcfd092e42d594014459a0ae15f8fe52eba96cf Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Thu, 5 Jun 2025 17:31:17 +0200 Subject: [PATCH 25/60] feat: fix ci 7 --- crate/memories/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crate/memories/Cargo.toml b/crate/memories/Cargo.toml index 38257504..6956ecd1 100644 --- a/crate/memories/Cargo.toml +++ b/crate/memories/Cargo.toml @@ -22,7 +22,7 @@ test-utils = ["cosmian_findex/test-utils"] [dependencies] cosmian_crypto_core.workspace = true -cosmian_findex = { path = "../findex", version = "7.1", features = [ +cosmian_findex = { path = "../findex", version = "7.2", features = [ "test-utils", ] } async-sqlite = { version = "0.4", optional = true } # `async-sqlite` dependency is pinned at v0.4 due to dependency tree issues with sqlx requiring a lower version of `libsqlite3-sys` From c43a1e09183c57257d3e0978087d50ea77c6d549 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Thu, 5 Jun 2025 17:40:28 +0200 Subject: [PATCH 26/60] feat: fix ci 8 (final???) --- crate/memories/Cargo.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crate/memories/Cargo.toml b/crate/memories/Cargo.toml index 6956ecd1..dc587a27 100644 --- a/crate/memories/Cargo.toml +++ b/crate/memories/Cargo.toml @@ -22,9 +22,7 @@ test-utils = ["cosmian_findex/test-utils"] [dependencies] cosmian_crypto_core.workspace = true -cosmian_findex = { path = "../findex", version = "7.2", features = [ - "test-utils", -] } +cosmian_findex = { path = "../findex", version = "7.2" } async-sqlite = { version = "0.4", optional = true } # `async-sqlite` dependency is pinned at v0.4 due to dependency tree issues with sqlx requiring a lower version of `libsqlite3-sys` deadpool-postgres = { version = "0.14.1", optional = true } redis = { version = "0.31", features = [ @@ -38,7 +36,9 @@ tokio-postgres = { version = "0.7.9", optional = true, features = [ ] } [dev-dependencies] -cosmian_findex = { path = "../findex", features = ["test-utils"] } +cosmian_findex = { path = "../findex", version = "7.2", features = [ + "test-utils", +] } futures = { version = "0.3" } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } criterion = { workspace = true } From a6b1589480c56fb2b5637d281102781d538d1fc5 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Fri, 6 Jun 2025 14:47:47 +0200 Subject: [PATCH 27/60] feat: reexoports --- crate/memories/examples/postgresql.rs | 12 ++++++++---- crate/memories/examples/redis.rs | 6 ++++-- crate/memories/examples/sqlite.rs | 6 ++++-- crate/memories/src/lib.rs | 14 ++++++++++++++ 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/crate/memories/examples/postgresql.rs b/crate/memories/examples/postgresql.rs index 46502ed0..ca9b5a72 100644 --- a/crate/memories/examples/postgresql.rs +++ b/crate/memories/examples/postgresql.rs @@ -3,13 +3,17 @@ mod shared_utils; use cosmian_crypto_core::{CsRng, Secret, reexport::rand_core::SeedableRng}; -use cosmian_findex::{ADDRESS_LENGTH, Address, Findex, IndexADT, MemoryEncryptionLayer}; -use cosmian_findex_memories::{PostgresMemory, PostgresMemoryError}; -use deadpool_postgres::{Config, Pool}; +use cosmian_findex_memories::{ + PostgresMemory, PostgresMemoryError, + reexport::{ + cosmian_findex::{ADDRESS_LENGTH, Address, Findex, IndexADT, MemoryEncryptionLayer}, + deadpool_postgres::{Config, Pool}, + tokio_postgres::NoTls, + }, +}; use futures::executor::block_on; use shared_utils::{WORD_LENGTH, decoder, encoder, gen_index}; use std::collections::HashMap; -use tokio_postgres::NoTls; const DB_URL: &str = "postgres://cosmian:cosmian@localhost/cosmian"; const TABLE_NAME: &str = "findex_example"; diff --git a/crate/memories/examples/redis.rs b/crate/memories/examples/redis.rs index c0142bdf..f06f8265 100644 --- a/crate/memories/examples/redis.rs +++ b/crate/memories/examples/redis.rs @@ -3,8 +3,10 @@ mod shared_utils; use cosmian_crypto_core::{CsRng, Secret, reexport::rand_core::SeedableRng}; -use cosmian_findex::{ADDRESS_LENGTH, Address, Findex, IndexADT, MemoryEncryptionLayer}; -use cosmian_findex_memories::RedisMemory; +use cosmian_findex_memories::{ + RedisMemory, + reexport::cosmian_findex::{ADDRESS_LENGTH, Address, Findex, IndexADT, MemoryEncryptionLayer}, +}; use futures::executor::block_on; use shared_utils::{WORD_LENGTH, decoder, encoder, gen_index}; use std::collections::HashMap; diff --git a/crate/memories/examples/sqlite.rs b/crate/memories/examples/sqlite.rs index 2f57108d..028f5ebb 100644 --- a/crate/memories/examples/sqlite.rs +++ b/crate/memories/examples/sqlite.rs @@ -3,8 +3,10 @@ mod shared_utils; use cosmian_crypto_core::{CsRng, Secret, reexport::rand_core::SeedableRng}; -use cosmian_findex::{Findex, IndexADT, MemoryEncryptionLayer}; -use cosmian_findex_memories::SqliteMemory; +use cosmian_findex_memories::{ + SqliteMemory, + reexport::cosmian_findex::{Findex, IndexADT, MemoryEncryptionLayer}, +}; use futures::executor::block_on; use shared_utils::{WORD_LENGTH, decoder, encoder, gen_index}; use std::collections::HashMap; diff --git a/crate/memories/src/lib.rs b/crate/memories/src/lib.rs index 9e373419..6e4d1503 100644 --- a/crate/memories/src/lib.rs +++ b/crate/memories/src/lib.rs @@ -12,3 +12,17 @@ pub use sqlite_mem::{FINDEX_TABLE_NAME, SqliteMemory, SqliteMemoryError}; mod postgresql_mem; #[cfg(feature = "postgres-mem")] pub use postgresql_mem::{PostgresMemory, PostgresMemoryError}; + +pub mod reexport { + #[cfg(feature = "sqlite-mem")] + pub use async_sqlite; + pub use cosmian_findex; + #[cfg(feature = "postgres-mem")] + pub use deadpool_postgres; + #[cfg(feature = "redis-mem")] + pub use redis; + #[cfg(feature = "postgres-mem")] + pub use tokio; + #[cfg(feature = "postgres-mem")] + pub use tokio_postgres; +} From bd565adbbeaea3a62d1afadac14d3a43121be85a Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Thu, 12 Jun 2025 13:09:28 +0200 Subject: [PATCH 28/60] fix: first rev fixes fix: bump, bye 7.x ... Hello findex 8 fix: pre name change fixes fix: final fixes fix: final fixes2 fix: udpate bench and example fix: add ci pachete fix: ci fixes --- .github/workflows/ci.yml | 25 ++++++++++-- Cargo.toml | 4 +- README.md | 6 +-- crate/findex/Cargo.toml | 6 +-- crate/findex/README.md | 2 +- crate/findex/src/memory/in_memory.rs | 2 +- crate/findex/src/test_utils/benches.rs | 8 ++-- crate/memories/Cargo.toml | 8 ++-- crate/memories/README.md | 44 +++------------------ crate/memories/benches/benches.rs | 42 +++++++++++++++++--- crate/memories/docker-compose.yml | 25 ++++++++++++ crate/memories/examples/postgresql.rs | 35 +++++++--------- crate/memories/examples/redis.rs | 4 +- crate/memories/examples/sqlite.rs | 3 +- crate/memories/src/lib.rs | 2 +- crate/memories/src/postgresql_mem/memory.rs | 16 ++++---- crate/memories/src/sqlite_mem.rs | 36 +++++++++++------ 17 files changed, 158 insertions(+), 110 deletions(-) create mode 100644 crate/memories/docker-compose.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d785a02d..63b1e174 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,14 +5,33 @@ on: push jobs: cargo-lint: - uses: Cosmian/reusable_workflows/.github/workflows/cargo-nursery.yml@feat/publish_workspace_fix + uses: Cosmian/reusable_workflows/.github/workflows/cargo-nursery.yml@develop with: toolchain: 1.85.0 workspace: true + cargo-machete: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: 1.85.0 + - name: Install cargo-machete + run: cargo install cargo-machete + - name: Check unused dependencies in findex crate + run: | + cd crate/findex + cargo machete --with-metadata + - name: Check unused dependencies in memories crate + run: | + cd crate/memories + cargo machete --with-metadata cargo-publish: needs: - cargo-lint - uses: Cosmian/reusable_workflows/.github/workflows/cargo-publish.yml@feat/publish_workspace_fix + uses: Cosmian/reusable_workflows/.github/workflows/cargo-publish.yml@develop if: startsWith(github.ref, 'refs/tags/') with: toolchain: 1.85.0 @@ -20,5 +39,5 @@ jobs: cleanup: needs: - cargo-lint - uses: Cosmian/reusable_workflows/.github/workflows/cleanup_cache.yml@feat/publish_workspace_fix + uses: Cosmian/reusable_workflows/.github/workflows/cleanup_cache.yml@develop secrets: inherit diff --git a/Cargo.toml b/Cargo.toml index 3da8f512..c31b2fab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["crate/findex", "crate/memories"] resolver = "2" [workspace.package] -version = "7.2.0" +version = "1.0.0" authors = [ "Bruno Grieder ", "CĂ©lia Corsin ", @@ -27,6 +27,4 @@ cosmian_crypto_core = { version = "10.1", default-features = false, features = [ "sha3", ] } criterion = { version = "0.6" } -rand = { version = "0.9.0" } -rand_distr = { version = "0.5.1" } tokio = { version = "1.44" } diff --git a/README.md b/README.md index 792113a2..4929d3a6 100644 --- a/README.md +++ b/README.md @@ -19,15 +19,15 @@ Add `cosmian_findex` to your Cargo.toml: ```toml [dependencies] -cosmian_findex = "7.1.0" +cosmian_findex = "8.0.0" # Optional - include backend implementations -cosmian_findex_memories = { version = "7.1.0", features = ["redis-mem", "sqlite-mem", "postgres-mem"] } +cosmian_findex_memories = { version = "8.0.0", features = ["redis-mem", "sqlite-mem", "postgres-mem"] } ``` ## Related Projects + [Findex Server](github.com/cosmian/findex-server) - A production-ready Findex server implementation ## License This project is licensed under the Business Source License 1.1 (BUSL-1.1). - diff --git a/crate/findex/Cargo.toml b/crate/findex/Cargo.toml index 7d2c32ef..d0119b1d 100644 --- a/crate/findex/Cargo.toml +++ b/crate/findex/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cosmian_findex" -version = "7.2.0" +version = "8.0.0" authors.workspace = true categories.workspace = true edition.workspace = true @@ -15,7 +15,7 @@ name = "cosmian_findex" path = "src/lib.rs" [features] -test-utils = ["tokio", "criterion", "rand", "rand_distr"] +test-utils = ["tokio", "criterion"] [dependencies] cosmian_crypto_core.workspace = true @@ -28,8 +28,6 @@ tokio = { workspace = true, features = [ "macros", ], optional = true } criterion = { workspace = true, optional = true } -rand = { workspace = true, optional = true } -rand_distr = { workspace = true, optional = true } [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crate/findex/README.md b/crate/findex/README.md index 32ec1964..051ef356 100644 --- a/crate/findex/README.md +++ b/crate/findex/README.md @@ -18,6 +18,6 @@ To launch the benchmarks, run: cargo bench --all-features ``` -Note that benches are quite involving and require *several hours* for a full +Note that benches are quite involving and require _several hours_ for a full run. Once all benchmarks are run, you will find detailed reports under `target/criterion`. findex diff --git a/crate/findex/src/memory/in_memory.rs b/crate/findex/src/memory/in_memory.rs index faf5d082..c25fe0de 100644 --- a/crate/findex/src/memory/in_memory.rs +++ b/crate/findex/src/memory/in_memory.rs @@ -1,4 +1,4 @@ -//! A simple RAM-based implementation of the MemoryADT trait that stores key-value pairs in a thread-safe in-memory HashMap. +//! A thread-safe implementation of the `MemoryADT` trait based on a `HashMap`. use std::{ collections::HashMap, diff --git a/crate/findex/src/test_utils/benches.rs b/crate/findex/src/test_utils/benches.rs index aecfef49..c30351cf 100644 --- a/crate/findex/src/test_utils/benches.rs +++ b/crate/findex/src/test_utils/benches.rs @@ -1,6 +1,8 @@ -//! This module provides a comprehensive benchmarking suite for testing the performance of Findex memory implementations. -//! These benchmarks are designed to be generic and work with any memory backend that implements the MemoryADT trait. -//! The [findex-memories](https://github.com/Cosmian/findex-memories) crate provides a set of memory backends that have been benched. +//! This module provides a comprehensive benchmarking suite for testing the +//! performance of Findex memory implementations. These benchmarks are designed +//! to be generic and work with any memory backend that implements the MemoryADT +//! trait. + use crate::{ ADDRESS_LENGTH, Address, Findex, IndexADT, MemoryADT, MemoryEncryptionLayer, WORD_LENGTH, dummy_decode, dummy_encode, diff --git a/crate/memories/Cargo.toml b/crate/memories/Cargo.toml index dc587a27..94a6130b 100644 --- a/crate/memories/Cargo.toml +++ b/crate/memories/Cargo.toml @@ -22,7 +22,7 @@ test-utils = ["cosmian_findex/test-utils"] [dependencies] cosmian_crypto_core.workspace = true -cosmian_findex = { path = "../findex", version = "7.2" } +cosmian_findex = { path = "../findex", version = "8.0" } async-sqlite = { version = "0.4", optional = true } # `async-sqlite` dependency is pinned at v0.4 due to dependency tree issues with sqlx requiring a lower version of `libsqlite3-sys` deadpool-postgres = { version = "0.14.1", optional = true } redis = { version = "0.31", features = [ @@ -36,14 +36,14 @@ tokio-postgres = { version = "0.7.9", optional = true, features = [ ] } [dev-dependencies] -cosmian_findex = { path = "../findex", version = "7.2", features = [ +cosmian_findex = { path = "../findex", version = "8.0", features = [ "test-utils", ] } futures = { version = "0.3" } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } criterion = { workspace = true } -rand = { workspace = true } -rand_distr = { workspace = true } +rand = { version = "0.9.0" } +rand_distr = { version = "0.5.1" } [[bench]] name = "benches" diff --git a/crate/memories/README.md b/crate/memories/README.md index 5cd5dd7a..d00b9328 100644 --- a/crate/memories/README.md +++ b/crate/memories/README.md @@ -10,13 +10,11 @@ Findex Memories provides "pluggable" storage backends for Findex, allowing the c This library provides implementations for the following storage systems: - -| Feature | Database | Dependencies | -|---------|----------|--------------| -| `redis-mem` | Redis | [redis](https://crates.io/crates/redis) v0.31 | -| `sqlite-mem` | SQLite | [async-sqlite](https://crates.io/crates/async-sqlite) v0.4 * | +| Feature | Database | Dependencies | +| -------------- | ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `redis-mem` | Redis | [redis](https://crates.io/crates/redis) v0.31 | +| `sqlite-mem` | SQLite | [async-sqlite](https://crates.io/crates/async-sqlite) v0.4 \* | | `postgres-mem` | PostgreSQL | [tokio-postgres](https://crates.io/crates/tokio-postgres) v0.7.9
[tokio](https://crates.io/crates/tokio) v1.44
[deadpool-postgres](https://crates.io/crates/deadpool-postgres) v0.14.1 | - ## Usage @@ -26,36 +24,6 @@ First, add `cosmian_findex_memories` as dependency to your project : cargo add cosmian_findex_memories # do not forget to enable the adequate feature for the backend you want to use ! ``` -If you don't have a running `Redis` or `Postgres` instance running, you can use the one provided on the root by running `docker-compose up`, then on your application's code : - -```rust -// For Redis -use cosmian_findex_memories::postgres::RedisMemory; -use cosmian_findex::{ADDRESS_LENGTH, Findex, Address, dummy_decode, dummy_encode,WORD_LENGTH}; - -let memory = RedisMemory::, [u8; WORD_LENGTH]>::connect( - "redis://localhost:6379", -).await.unwrap(); - -// optionally, add the encryption layer (recommended) -// let memory = MemoryEncryptionLayer::new(&key, InMemory::default()); - -let findex = Findex::new(memory, dummy_encode::, dummy_decode); - -let cat_bindings = [ - Value::try_from(1).unwrap(), - Value::try_from(3).unwrap(), - Value::try_from(5).unwrap(), -]; - -findex.insert("cat".to_string(), cat_bindings.clone()).await.unwrap(); - -let cat_res = findex.search(&"cat".to_string()).unwrap(); - -assert_eq!( - cat_bindings.iter().cloned().collect::>(), - cat_res -); -``` +If you don't have a running `Redis` or `Postgres` instance running, you can use the provided [docker-compose.yml](./docker-compose.yml) file provided with this repository by running `docker-compose up`. -More detailed examples can be found under the [examples folder](examples). +For detailed usage examples, refer to the [examples folder](examples). diff --git a/crate/memories/benches/benches.rs b/crate/memories/benches/benches.rs index 1bd1d299..266947f1 100644 --- a/crate/memories/benches/benches.rs +++ b/crate/memories/benches/benches.rs @@ -28,10 +28,17 @@ fn get_redis_url() -> String { ) } -/// Refer to `src/memory/postgresql_store/memory.rs` for local setup instructions #[cfg(feature = "postgres-mem")] use cosmian_findex_memories::{PostgresMemory, PostgresMemoryError}; +// To run the postgresql benchmarks locally, add the following service to your pg_service.conf file +// (usually under ~/.pg_service.conf): +// +// [cosmian_service] +// host=localhost +// dbname=cosmian +// user=cosmian +// password=cosmian #[cfg(feature = "postgres-mem")] fn get_postgresql_url() -> String { std::env::var("POSTGRES_HOST").map_or_else( @@ -83,7 +90,14 @@ fn bench_search_multiple_bindings(c: &mut Criterion) { bench_memory_search_multiple_bindings( "SQLite", N_PTS, - async || SqliteMemory::connect(SQLITE_PATH).await.unwrap(), + async || { + SqliteMemory::connect( + SQLITE_PATH, + "bench_memory_search_multiple_bindings".to_string(), + ) + .await + .unwrap() + }, c, &mut rng, ); @@ -121,7 +135,14 @@ fn bench_search_multiple_keywords(c: &mut Criterion) { bench_memory_search_multiple_keywords( "SQLite", N_PTS, - async || SqliteMemory::connect(SQLITE_PATH).await.unwrap(), + async || { + SqliteMemory::connect( + SQLITE_PATH, + "bench_memory_search_multiple_keywords".to_string(), + ) + .await + .unwrap() + }, c, &mut rng, ); @@ -160,7 +181,14 @@ fn bench_insert_multiple_bindings(c: &mut Criterion) { bench_memory_insert_multiple_bindings( "SQLite", N_PTS, - async || SqliteMemory::connect(SQLITE_PATH).await.unwrap(), + async || { + SqliteMemory::connect( + SQLITE_PATH, + "bench_memory_insert_multiple_bindings".to_string(), + ) + .await + .unwrap() + }, c, SqliteMemory::clear, &mut rng, @@ -203,7 +231,11 @@ fn bench_contention(c: &mut Criterion) { bench_memory_contention( "SQLite", N_PTS, - async || SqliteMemory::connect(SQLITE_PATH).await.unwrap(), + async || { + SqliteMemory::connect(SQLITE_PATH, "bench_memory_contention".to_string()) + .await + .unwrap() + }, c, SqliteMemory::clear, &mut rng, diff --git a/crate/memories/docker-compose.yml b/crate/memories/docker-compose.yml new file mode 100644 index 00000000..4a58a2fc --- /dev/null +++ b/crate/memories/docker-compose.yml @@ -0,0 +1,25 @@ +--- +services: + redis: + image: redis:latest + ports: + - 6379:6379 + healthcheck: + test: [CMD, redis-cli, ping] + interval: 10s + timeout: 5s + retries: 5 + postgres: + image: postgres + ports: + - 5432:5432 + environment: + - POSTGRES_USER=cosmian + - POSTGRES_DB=cosmian + - POSTGRES_PASSWORD=cosmian + - PGDATA=/tmp/postgres3 + healthcheck: + test: [CMD, pg_isready, -U, cosmian] + interval: 10s + timeout: 5s + retries: 5 diff --git a/crate/memories/examples/postgresql.rs b/crate/memories/examples/postgresql.rs index ca9b5a72..0fd3b061 100644 --- a/crate/memories/examples/postgresql.rs +++ b/crate/memories/examples/postgresql.rs @@ -1,4 +1,5 @@ -//! This example show-cases the use of Findex to securely store a hash-map with postgresql. +//! This example show-cases the use of Findex to securely store a hash-map with +//! PostgreSQL. #[path = "shared_utils.rs"] mod shared_utils; @@ -36,24 +37,9 @@ async fn main() { // key in order to make the index inter-operable. let key = Secret::random(&mut rng); - // Generating the random index. + // Generating a random index. let index = gen_index(&mut rng); - let pool = create_pool(DB_URL).await.unwrap(); - let m = PostgresMemory::, [u8; WORD_LENGTH]>::connect_with_pool( - pool.clone(), - TABLE_NAME.to_string(), - ) - .await - .unwrap(); - - // Notice we chose to not enable Tls: it's not needed for this example as we are - // using the encryption layer in top of the memory interface - i.e. the data is - // already encrypted before being sent to the database and Tls would add unnecessary overhead. - m.initialize_table(DB_URL.to_string(), TABLE_NAME.to_string(), NoTls) - .await - .unwrap(); - // Addd the following service to your pg_service.conf file (usually under ~/.pg_service.conf): // // [cosmian_service] @@ -61,14 +47,23 @@ async fn main() { // dbname=cosmian // user=cosmian // password=cosmian - let memory = PostgresMemory::, [u8; WORD_LENGTH]>::connect_with_pool( + let pool = create_pool(DB_URL).await.unwrap(); + let m = PostgresMemory::, [u8; WORD_LENGTH]>::connect_with_pool( pool.clone(), TABLE_NAME.to_string(), ) .await .unwrap(); - let encrypted_memory = MemoryEncryptionLayer::new(&key, memory); + // Notice we chose to not enable TLS: it's not needed for this example as we + // are using the encryption layer in top of the memory interface - i.e. the + // data is already encrypted before being sent to the database and TLS would + // add unnecessary overhead. + m.initialize_table(DB_URL.to_string(), TABLE_NAME.to_string(), NoTls) + .await + .unwrap(); + + let encrypted_memory = MemoryEncryptionLayer::new(&key, m); // Instantiating Findex requires passing the key, the memory used and the // encoder and decoder. Quite simple, after all :) @@ -97,7 +92,7 @@ async fn main() { // ... and verify we get the whole index back! assert_eq!(res, index); - // Drop the table to avoid problems with subsequent runs + // Drop the table to avoid problems with subsequent runs. pool.get() .await .unwrap() diff --git a/crate/memories/examples/redis.rs b/crate/memories/examples/redis.rs index f06f8265..ee52e122 100644 --- a/crate/memories/examples/redis.rs +++ b/crate/memories/examples/redis.rs @@ -27,9 +27,7 @@ async fn main() { // Generating the random index. let index = gen_index(&mut rng); - // For this example, we use the `RedisMemory` implementation of the `MemoryADT` - // trait. It connects to a Redis instance and uses it as a key-value store - // for our Findex data structures, which is suitable for production applications. + // This example uses our Redis-based implementation of `MemoryADT`. let memory = RedisMemory::, [u8; WORD_LENGTH]>::connect(DB_PATH) .await .unwrap(); diff --git a/crate/memories/examples/sqlite.rs b/crate/memories/examples/sqlite.rs index 028f5ebb..e38146f5 100644 --- a/crate/memories/examples/sqlite.rs +++ b/crate/memories/examples/sqlite.rs @@ -12,6 +12,7 @@ use shared_utils::{WORD_LENGTH, decoder, encoder, gen_index}; use std::collections::HashMap; const DB_PATH: &str = "./target/debug/sqlite-test.db"; +const TABLE_NAME: &str = "findex_memory"; #[tokio::main] async fn main() { @@ -27,7 +28,7 @@ async fn main() { // Generating the random index. let index = gen_index(&mut rng); - let memory = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH) + let memory = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); diff --git a/crate/memories/src/lib.rs b/crate/memories/src/lib.rs index 6e4d1503..3ffc8213 100644 --- a/crate/memories/src/lib.rs +++ b/crate/memories/src/lib.rs @@ -6,7 +6,7 @@ pub use redis_mem::{RedisMemory, RedisMemoryError}; #[cfg(feature = "sqlite-mem")] mod sqlite_mem; #[cfg(feature = "sqlite-mem")] -pub use sqlite_mem::{FINDEX_TABLE_NAME, SqliteMemory, SqliteMemoryError}; +pub use sqlite_mem::{SqliteMemory, SqliteMemoryError}; #[cfg(feature = "postgres-mem")] mod postgresql_mem; diff --git a/crate/memories/src/postgresql_mem/memory.rs b/crate/memories/src/postgresql_mem/memory.rs index eba6b768..a14603f4 100644 --- a/crate/memories/src/postgresql_mem/memory.rs +++ b/crate/memories/src/postgresql_mem/memory.rs @@ -230,14 +230,14 @@ impl MemoryADT #[cfg(test)] mod tests { - //! To run the postgresql benchmarks locally, add the following service to your pg_service.conf file - //! (usually under ~/.pg_service.conf): - //! - //! [cosmian_service] - //! host=localhost - //! dbname=cosmian - //! user=cosmian - //! password=cosmian + // To run the postgresql tests locally, add the following service to your pg_service.conf file + // (usually under ~/.pg_service.conf): + // + // [cosmian_service] + // host=localhost + // dbname=cosmian + // user=cosmian + // password=cosmian use deadpool_postgres::Config; use tokio_postgres::NoTls; diff --git a/crate/memories/src/sqlite_mem.rs b/crate/memories/src/sqlite_mem.rs index fd5940b1..03c0a317 100644 --- a/crate/memories/src/sqlite_mem.rs +++ b/crate/memories/src/sqlite_mem.rs @@ -31,10 +31,10 @@ impl From for SqliteMemoryError { } } -pub const FINDEX_TABLE_NAME: &str = "findex_memory"; #[derive(Clone)] pub struct SqliteMemory { pool: Pool, + table_name: String, _marker: PhantomData<(Address, Word)>, } @@ -72,16 +72,22 @@ impl SqliteMemory { /// # Arguments /// /// * `path` - The path to the sqlite3 database file. - pub async fn connect(path: &str) -> Result { + pub async fn connect>( + path: P, + table_name: String, + ) -> Result { // The number of connections in this pools defaults to the number of // logical CPUs of the system. let pool = PoolBuilder::new().path(path).open().await?; + // The closure must satisfy 'static bounds, there is no escape from cloning the table name. + let table_name_clone = table_name.clone(); - pool.conn(|conn| conn.execute_batch(&create_table_script(FINDEX_TABLE_NAME))) + pool.conn(move |conn| conn.execute_batch(&create_table_script(&table_name_clone))) .await?; Ok(Self { pool, + table_name, _marker: PhantomData, }) } @@ -97,10 +103,12 @@ impl SqliteMemory { pool: Pool, table_name: String, ) -> Result { - pool.conn(move |conn| conn.execute_batch(&create_table_script(&table_name))) + let table_name_clone = table_name.clone(); + pool.conn(move |conn| conn.execute_batch(&create_table_script(&table_name_clone))) .await?; Ok(Self { pool, + table_name, _marker: PhantomData, }) } @@ -109,8 +117,9 @@ impl SqliteMemory { impl SqliteMemory { #[cfg(feature = "test-utils")] pub async fn clear(&self) -> Result<(), SqliteMemoryError> { + let findex_table_name = self.table_name.clone(); self.pool - .conn(|cnx| cnx.execute(&format!("DELETE FROM {FINDEX_TABLE_NAME}"), [])) + .conn(move |conn| conn.execute(&format!("DELETE FROM {findex_table_name}"), [])) .await?; Ok(()) } @@ -127,12 +136,13 @@ impl MemoryADT &self, addresses: Vec, ) -> Result>, Self::Error> { + let findex_table_name = self.table_name.clone(); self.pool .conn(move |conn| { let results = conn .prepare(&format!( "SELECT a, w FROM {} WHERE a IN ({})", - FINDEX_TABLE_NAME, + findex_table_name, vec!["?"; addresses.len()].join(",") ))? .query_map( @@ -164,6 +174,7 @@ impl MemoryADT guard: (Self::Address, Option), bindings: Vec<(Self::Address, Self::Word)>, ) -> Result, Self::Error> { + let findex_table_name = self.table_name.clone(); let (ag, wg) = guard; self.pool @@ -174,7 +185,7 @@ impl MemoryADT let current_word = tx .query_row( - &format!("SELECT w FROM {} WHERE a = ?", FINDEX_TABLE_NAME), + &format!("SELECT w FROM {} WHERE a = ?", findex_table_name), [&*ag], |row| row.get(0), ) @@ -184,7 +195,7 @@ impl MemoryADT tx.execute( &format!( "INSERT OR REPLACE INTO {} (a, w) VALUES {}", - FINDEX_TABLE_NAME, + findex_table_name, vec!["(?,?)"; bindings.len()].join(",") ), params_from_iter( @@ -214,10 +225,11 @@ mod tests { }; const DB_PATH: &str = "../../target/debug/sqlite-test.sqlite.db"; + const TABLE_NAME: &str = "findex_memory"; #[tokio::test] async fn test_rw_seq() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH) + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); test_single_write_and_read(&m, gen_seed()).await @@ -225,7 +237,7 @@ mod tests { #[tokio::test] async fn test_guard_seq() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH) + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); test_wrong_guard(&m, gen_seed()).await @@ -233,7 +245,7 @@ mod tests { #[tokio::test] async fn test_collision_seq() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH) + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); test_rw_same_address(&m, gen_seed()).await @@ -241,7 +253,7 @@ mod tests { #[tokio::test] async fn test_rw_ccr() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH) + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); test_guarded_write_concurrent(&m, gen_seed(), Some(100)).await From d62f233e03f65fd0a95b6233bc05b78aea582cad Mon Sep 17 00:00:00 2001 From: Hatem M'naouer <19950216+HatemMn@users.noreply.github.com> Date: Fri, 13 Jun 2025 17:28:46 +0200 Subject: [PATCH 29/60] Update crate/memories/Cargo.toml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophile BRÉZOT --- crate/memories/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crate/memories/Cargo.toml b/crate/memories/Cargo.toml index 94a6130b..6c87a574 100644 --- a/crate/memories/Cargo.toml +++ b/crate/memories/Cargo.toml @@ -25,7 +25,7 @@ cosmian_crypto_core.workspace = true cosmian_findex = { path = "../findex", version = "8.0" } async-sqlite = { version = "0.4", optional = true } # `async-sqlite` dependency is pinned at v0.4 due to dependency tree issues with sqlx requiring a lower version of `libsqlite3-sys` deadpool-postgres = { version = "0.14.1", optional = true } -redis = { version = "0.31", features = [ +redis = { version = "0.32", default-features = false, features = [ "aio", "connection-manager", "tokio-comp", From ac5af208804e08ffa18b3309b31e47db0d5628da Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Tue, 24 Jun 2025 12:12:46 +0200 Subject: [PATCH 30/60] feat: version unify --- Cargo.toml | 2 +- crate/findex/Cargo.toml | 2 +- crate/memories/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c31b2fab..3e7908ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["crate/findex", "crate/memories"] resolver = "2" [workspace.package] -version = "1.0.0" +version = "8.0.0" authors = [ "Bruno Grieder ", "Célia Corsin ", diff --git a/crate/findex/Cargo.toml b/crate/findex/Cargo.toml index d0119b1d..71a79b2e 100644 --- a/crate/findex/Cargo.toml +++ b/crate/findex/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cosmian_findex" -version = "8.0.0" +version.workspace = true authors.workspace = true categories.workspace = true edition.workspace = true diff --git a/crate/memories/Cargo.toml b/crate/memories/Cargo.toml index 6c87a574..1579f329 100644 --- a/crate/memories/Cargo.toml +++ b/crate/memories/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cosmian_findex_memories" -version = "1.0.0" +version.workspace = true authors.workspace = true license.workspace = true edition.workspace = true From 893a7a7163f10b93e1678988844a4cf37a7bc552 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Thu, 12 Jun 2025 17:55:00 +0200 Subject: [PATCH 31/60] upgrade toolchain --- .github/workflows/benches.yml | 2 +- .github/workflows/ci.yml | 6 +++--- .github/workflows/hack.yml | 2 +- crate/findex/src/lib.rs | 1 + rust-toolchain.toml | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml index a4e6a877..8762c04a 100644 --- a/.github/workflows/benches.yml +++ b/.github/workflows/benches.yml @@ -8,6 +8,6 @@ jobs: bench: uses: Cosmian/reusable_workflows/.github/workflows/cargo-bench.yml@develop with: - toolchain: stable + toolchain: 1.87.0 features: test-utils,redis-mem,sqlite-mem,rust-mem,postgres-mem force: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63b1e174..b6f777c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ jobs: cargo-lint: uses: Cosmian/reusable_workflows/.github/workflows/cargo-nursery.yml@develop with: - toolchain: 1.85.0 + toolchain: 1.87.0 workspace: true cargo-machete: runs-on: ubuntu-latest @@ -17,7 +17,7 @@ jobs: - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: - toolchain: 1.85.0 + toolchain: 1.87.0 - name: Install cargo-machete run: cargo install cargo-machete - name: Check unused dependencies in findex crate @@ -34,7 +34,7 @@ jobs: uses: Cosmian/reusable_workflows/.github/workflows/cargo-publish.yml@develop if: startsWith(github.ref, 'refs/tags/') with: - toolchain: 1.85.0 + toolchain: 1.87.0 secrets: inherit cleanup: needs: diff --git a/.github/workflows/hack.yml b/.github/workflows/hack.yml index 78db6cb2..c8e6fa9c 100644 --- a/.github/workflows/hack.yml +++ b/.github/workflows/hack.yml @@ -35,7 +35,7 @@ jobs: - uses: actions/checkout@v1 - uses: actions-rs/toolchain@v1 with: - toolchain: stable + toolchain: 1.87.0 override: true - name: Install cargo-hack run: cargo install --locked cargo-hack || true diff --git a/crate/findex/src/lib.rs b/crate/findex/src/lib.rs index 656012a2..d0d407d6 100644 --- a/crate/findex/src/lib.rs +++ b/crate/findex/src/lib.rs @@ -12,6 +12,7 @@ mod error; mod findex; mod memory; mod ovec; + #[cfg(any(test, feature = "test-utils"))] mod test_utils; diff --git a/rust-toolchain.toml b/rust-toolchain.toml index c8181991..45a64fa6 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.85.0" +channel = "1.87.0" components = ["rustfmt"] From e76093a061d3284af15143a0f4e623d0fd1d136e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Fri, 13 Jun 2025 12:43:40 +0200 Subject: [PATCH 32/60] refacto of the PostgreSQL memory implementation --- Cargo.toml | 2 +- crate/memories/Cargo.toml | 19 +- crate/memories/benches/benches.rs | 38 +-- crate/memories/examples/postgresql.rs | 12 +- crate/memories/examples/sqlite.rs | 2 +- crate/memories/src/lib.rs | 2 - crate/memories/src/postgresql_mem/memory.rs | 244 +++++++++----------- crate/memories/src/sqlite_mem.rs | 100 ++++---- 8 files changed, 191 insertions(+), 228 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3e7908ec..8658cf3e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,4 +27,4 @@ cosmian_crypto_core = { version = "10.1", default-features = false, features = [ "sha3", ] } criterion = { version = "0.6" } -tokio = { version = "1.44" } +tokio = { version = "1.45" } diff --git a/crate/memories/Cargo.toml b/crate/memories/Cargo.toml index 1579f329..d1ba1a6e 100644 --- a/crate/memories/Cargo.toml +++ b/crate/memories/Cargo.toml @@ -17,33 +17,32 @@ path = "src/lib.rs" [features] redis-mem = ["redis"] sqlite-mem = ["async-sqlite"] -postgres-mem = ["tokio-postgres", "tokio", "deadpool-postgres"] -test-utils = ["cosmian_findex/test-utils"] +postgres-mem = ["deadpool-postgres", "tokio", "tokio-postgres"] +test-utils = ["cosmian_findex/test-utils", "redis/tokio-comp"] [dependencies] -cosmian_crypto_core.workspace = true +async-sqlite = { version = "0.5", optional = true } cosmian_findex = { path = "../findex", version = "8.0" } -async-sqlite = { version = "0.4", optional = true } # `async-sqlite` dependency is pinned at v0.4 due to dependency tree issues with sqlx requiring a lower version of `libsqlite3-sys` -deadpool-postgres = { version = "0.14.1", optional = true } +deadpool-postgres = { version = "0.14", optional = true } redis = { version = "0.32", default-features = false, features = [ - "aio", "connection-manager", "tokio-comp", ], optional = true } tokio = { workspace = true, features = ["rt-multi-thread"], optional = true } -tokio-postgres = { version = "0.7.9", optional = true, features = [ +tokio-postgres = { version = "0.7", optional = true, features = [ "array-impls", ] } [dev-dependencies] +cosmian_crypto_core.workspace = true cosmian_findex = { path = "../findex", version = "8.0", features = [ "test-utils", ] } -futures = { version = "0.3" } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } criterion = { workspace = true } -rand = { version = "0.9.0" } -rand_distr = { version = "0.5.1" } +futures = { version = "0.3" } +rand = "0.9" +rand_distr = "0.5" [[bench]] name = "benches" diff --git a/crate/memories/benches/benches.rs b/crate/memories/benches/benches.rs index 266947f1..ba17c61f 100644 --- a/crate/memories/benches/benches.rs +++ b/crate/memories/benches/benches.rs @@ -61,14 +61,14 @@ async fn connect_and_init_table( pg_config.url = Some(db_url.to_string()); let test_pool = pg_config.builder(NoTls)?.build()?; - let m = PostgresMemory::, [u8; WORD_LENGTH]>::connect_with_pool( + let m = PostgresMemory::, [u8; WORD_LENGTH]>::new_with_pool( test_pool, table_name.to_string(), ) - .await?; + .await; + + m.initialize().await?; - m.initialize_table(db_url.to_string(), table_name, NoTls) - .await?; Ok(m) } // Number of points in each graph. @@ -91,12 +91,14 @@ fn bench_search_multiple_bindings(c: &mut Criterion) { "SQLite", N_PTS, async || { - SqliteMemory::connect( + let m = SqliteMemory::new( SQLITE_PATH, "bench_memory_search_multiple_bindings".to_string(), ) .await - .unwrap() + .unwrap(); + m.initialize().await.unwrap(); + m }, c, &mut rng, @@ -107,12 +109,14 @@ fn bench_search_multiple_bindings(c: &mut Criterion) { "Postgres", N_PTS, async || { - connect_and_init_table( + let m = connect_and_init_table( get_postgresql_url(), "bench_memory_search_multiple_bindings".to_string(), ) .await - .unwrap() + .unwrap(); + m.initialize().await.unwrap(); + m }, c, &mut rng, @@ -136,12 +140,14 @@ fn bench_search_multiple_keywords(c: &mut Criterion) { "SQLite", N_PTS, async || { - SqliteMemory::connect( + let m = SqliteMemory::new( SQLITE_PATH, "bench_memory_search_multiple_keywords".to_string(), ) .await - .unwrap() + .unwrap(); + m.initialize().await.unwrap(); + m }, c, &mut rng, @@ -182,12 +188,14 @@ fn bench_insert_multiple_bindings(c: &mut Criterion) { "SQLite", N_PTS, async || { - SqliteMemory::connect( + let m = SqliteMemory::new( SQLITE_PATH, "bench_memory_insert_multiple_bindings".to_string(), ) .await - .unwrap() + .unwrap(); + m.initialize().await.unwrap(); + m }, c, SqliteMemory::clear, @@ -232,9 +240,11 @@ fn bench_contention(c: &mut Criterion) { "SQLite", N_PTS, async || { - SqliteMemory::connect(SQLITE_PATH, "bench_memory_contention".to_string()) + let m = SqliteMemory::new(SQLITE_PATH, "bench_memory_contention".to_string()) .await - .unwrap() + .unwrap(); + m.initialize().await.unwrap(); + m }, c, SqliteMemory::clear, diff --git a/crate/memories/examples/postgresql.rs b/crate/memories/examples/postgresql.rs index 0fd3b061..5160a162 100644 --- a/crate/memories/examples/postgresql.rs +++ b/crate/memories/examples/postgresql.rs @@ -40,7 +40,8 @@ async fn main() { // Generating a random index. let index = gen_index(&mut rng); - // Addd the following service to your pg_service.conf file (usually under ~/.pg_service.conf): + // Addd the following service to your pg_service.conf file (usually under + // `~/.pg_service.conf`): // // [cosmian_service] // host=localhost @@ -48,20 +49,17 @@ async fn main() { // user=cosmian // password=cosmian let pool = create_pool(DB_URL).await.unwrap(); - let m = PostgresMemory::, [u8; WORD_LENGTH]>::connect_with_pool( + let m = PostgresMemory::, [u8; WORD_LENGTH]>::new_with_pool( pool.clone(), TABLE_NAME.to_string(), ) - .await - .unwrap(); + .await; // Notice we chose to not enable TLS: it's not needed for this example as we // are using the encryption layer in top of the memory interface - i.e. the // data is already encrypted before being sent to the database and TLS would // add unnecessary overhead. - m.initialize_table(DB_URL.to_string(), TABLE_NAME.to_string(), NoTls) - .await - .unwrap(); + m.initialize().await.unwrap(); let encrypted_memory = MemoryEncryptionLayer::new(&key, m); diff --git a/crate/memories/examples/sqlite.rs b/crate/memories/examples/sqlite.rs index e38146f5..40d93653 100644 --- a/crate/memories/examples/sqlite.rs +++ b/crate/memories/examples/sqlite.rs @@ -28,7 +28,7 @@ async fn main() { // Generating the random index. let index = gen_index(&mut rng); - let memory = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH, TABLE_NAME.to_owned()) + let memory = SqliteMemory::<_, [u8; WORD_LENGTH]>::new(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); diff --git a/crate/memories/src/lib.rs b/crate/memories/src/lib.rs index 3ffc8213..25f593c8 100644 --- a/crate/memories/src/lib.rs +++ b/crate/memories/src/lib.rs @@ -22,7 +22,5 @@ pub mod reexport { #[cfg(feature = "redis-mem")] pub use redis; #[cfg(feature = "postgres-mem")] - pub use tokio; - #[cfg(feature = "postgres-mem")] pub use tokio_postgres; } diff --git a/crate/memories/src/postgresql_mem/memory.rs b/crate/memories/src/postgresql_mem/memory.rs index a14603f4..59b0023c 100644 --- a/crate/memories/src/postgresql_mem/memory.rs +++ b/crate/memories/src/postgresql_mem/memory.rs @@ -2,10 +2,6 @@ use super::PostgresMemoryError; use cosmian_findex::{Address, MemoryADT}; use deadpool_postgres::Pool; use std::marker::PhantomData; -use tokio_postgres::{ - Socket, - tls::{MakeTlsConnect, TlsConnect}, -}; #[derive(Clone, Debug)] pub struct PostgresMemory { @@ -17,65 +13,38 @@ pub struct PostgresMemory { impl PostgresMemory, [u8; WORD_LENGTH]> { - /// Connect to a Postgres database and create a table if it doesn't exist - pub async fn initialize_table( - &self, - db_url: String, - table_name: String, - tls: T, - ) -> Result<(), PostgresMemoryError> - where - T: MakeTlsConnect + Send, - T::Stream: Send + 'static, - T::TlsConnect: Send, - >::Future: Send, - { - let (client, connection) = tokio_postgres::connect(&db_url, tls).await?; - - // The connection object performs the actual communication with the database - // `Connection` only resolves when the connection is closed, either because a fatal error has - // occurred, or because its associated `Client` has dropped and all outstanding work has completed. - let conn_handle = tokio::spawn(async move { - if let Err(e) = connection.await { - eprintln!("connection error: {}", e); - } - }); + /// Returns a new memory instance from the given connection pool to a + /// PostgreSQL database. + pub async fn new_with_pool(pool: Pool, table_name: String) -> Self { + Self { + pool, + table_name, + _marker: PhantomData, + } + } - let returned = client + /// Connects to a PostgreSQL database and creates a table if it doesn't + /// exist. + pub async fn initialize(&self) -> Result<(), PostgresMemoryError> { + self.pool + .get() + .await? .execute( &format!( - " - CREATE TABLE IF NOT EXISTS {} ( + "CREATE TABLE IF NOT EXISTS {} ( a BYTEA PRIMARY KEY CHECK (octet_length(a) = {}), w BYTEA NOT NULL CHECK (octet_length(w) = {}) );", - table_name, ADDRESS_LENGTH, WORD_LENGTH + self.table_name, ADDRESS_LENGTH, WORD_LENGTH ), &[], ) .await?; - if returned != 0 { - return Err(PostgresMemoryError::TableCreationError(returned)); - } - drop(client); - let _ = conn_handle.await; // ensures that the connection is closed Ok(()) } - /// Connect to a Postgres database and create a table if it doesn't exist - pub async fn connect_with_pool( - pool: Pool, - table_name: String, - ) -> Result { - Ok(Self { - pool, - table_name, - _marker: PhantomData, - }) - } - - /// Deletes all rows from the findex memory table + /// Clears all bindings from this memory. #[cfg(feature = "test-utils")] pub async fn clear(&self) -> Result<(), PostgresMemoryError> { self.pool @@ -83,8 +52,38 @@ impl .await? .execute(&format!("TRUNCATE TABLE {};", self.table_name), &[]) .await?; + Ok(()) } + + fn gwrite_script(&self) -> String { + format!( + " + WITH + guard_check AS ( + SELECT w FROM {0} WHERE a = $1::bytea + ), + dedup_input_table AS ( + SELECT DISTINCT ON (a) a, w + FROM UNNEST($3::bytea[], $4::bytea[]) WITH ORDINALITY AS t(a, w, order_idx) + ORDER BY a, order_idx DESC + ), + insert_cte AS ( + INSERT INTO {0} (a, w) + SELECT a, w FROM dedup_input_table AS t(a,w) + WHERE ( + $2::bytea IS NULL AND NOT EXISTS (SELECT 1 FROM guard_check) + ) OR ( + $2::bytea IS NOT NULL AND EXISTS ( + SELECT 1 FROM guard_check WHERE w = $2::bytea + ) + ) + ON CONFLICT (a) DO UPDATE SET w = EXCLUDED.w + ) + SELECT COALESCE((SELECT w FROM guard_check)) AS original_guard_value;", + self.table_name + ) + } } impl MemoryADT @@ -99,23 +98,22 @@ impl MemoryADT addresses: Vec, ) -> Result>, Self::Error> { let client = self.pool.get().await?; - // statements are cached per connection and not per pool + + // Statements are cached per connection and not per pool. let stmt = client - .prepare_cached( - format!( - "SELECT f.w + .prepare_cached(&format!( + // The left join is necessary to ensure that the order of the + // addresses is preserved as well as to return None for addresses + // that don't exist. + "SELECT f.w FROM UNNEST($1::bytea[]) WITH ORDINALITY AS params(addr, idx) LEFT JOIN {} f ON params.addr = f.a ORDER BY params.idx;", - self.table_name - ) - .as_str(), - ) + self.table_name + )) .await?; client - // the left join is necessary to ensure that the order of the addresses is preserved - // as well as to return None for addresses that don't exist .query( &stmt, &[&addresses @@ -126,15 +124,12 @@ impl MemoryADT .await? .iter() .map(|row| { - let bytes_slice: Option<&[u8]> = row.try_get("w")?; // `row.get(0)` can panic - bytes_slice.map_or(Ok(None), |slice| { - slice - .try_into() - .map(Some) - .map_err(|_| PostgresMemoryError::InvalidDataLength(slice.len())) - }) + row.try_get::<_, Option<&[u8]>>("w")? + .map(Self::Word::try_from) + .transpose() + .map_err(PostgresMemoryError::TryFromSliceError) }) - .collect() + .collect::>() } async fn guarded_write( @@ -142,45 +137,19 @@ impl MemoryADT guard: (Self::Address, Option), bindings: Vec<(Self::Address, Self::Word)>, ) -> Result, Self::Error> { - let g_write_script = format!( - " - WITH - guard_check AS ( - SELECT w FROM {0} WHERE a = $1::bytea - ), - dedup_input_table AS ( - SELECT DISTINCT ON (a) a, w - FROM UNNEST($3::bytea[], $4::bytea[]) WITH ORDINALITY AS t(a, w, order_idx) - ORDER BY a, order_idx DESC - ), - insert_cte AS ( - INSERT INTO {0} (a, w) - SELECT a, w FROM dedup_input_table AS t(a,w) - WHERE ( - $2::bytea IS NULL AND NOT EXISTS (SELECT 1 FROM guard_check) - ) OR ( - $2::bytea IS NOT NULL AND EXISTS ( - SELECT 1 FROM guard_check WHERE w = $2::bytea - ) - ) - ON CONFLICT (a) DO UPDATE SET w = EXCLUDED.w - ) - SELECT COALESCE((SELECT w FROM guard_check)) AS original_guard_value;", - self.table_name - ); - let (addresses, words): (Vec<[u8; ADDRESS_LENGTH]>, Vec) = bindings.into_iter().map(|(a, w)| (*a, w)).unzip(); - const MAX_RETRIES: usize = 10; - for _ in 0..MAX_RETRIES { - // while counterintuitive, getting a new client on each retry is a better approach - // than trying to reuse the same client since it allows other operations to use the - // connection between retries. + // Since a guarded write operation is lock-free, this loop is guaranteed + // to terminate. + loop { + // Do not lock a resource for a potentially long loop, instead + // request a new one at each iteration. let mut client = self.pool.get().await?; - let stmt = client.prepare_cached(g_write_script.as_str()).await?; - let result = async { + let stmt = client.prepare_cached(&self.gwrite_script()).await?; + + let res = async { let tx = client .build_transaction() .isolation_level( @@ -200,21 +169,26 @@ impl MemoryADT ], ) .await? - .map_or( - Ok::, PostgresMemoryError>(None), - |row| { - row.try_get::<_, Option<&[u8]>>(0)? - .map_or(Ok(None), |r| Ok(Some(r.try_into()?))) - }, - )?; + .map(|row| { + row.try_get::<_, Option<&[u8]>>(0)? + .map(Self::Word::try_from) + .transpose() + .map_err(PostgresMemoryError::TryFromSliceError) + }) + .transpose()? + .flatten(); + tx.commit().await?; + Ok(res) } .await; - match result { + + match res { Ok(value) => return Ok(value), Err(err) => { - // Retry on serialization failures (error code 40001), otherwise fail and return the error + // Retry on serialization failures (error code 40001), + // otherwise fail and return the error if let PostgresMemoryError::TokioPostgresError(pg_err) = &err { if pg_err.code().is_some_and(|code| code.code() == "40001") { continue; @@ -224,28 +198,29 @@ impl MemoryADT } } } - Err(PostgresMemoryError::RetryExhaustedError(MAX_RETRIES)) } } #[cfg(test)] mod tests { - // To run the postgresql tests locally, add the following service to your pg_service.conf file - // (usually under ~/.pg_service.conf): - // - // [cosmian_service] - // host=localhost - // dbname=cosmian - // user=cosmian - // password=cosmian - use deadpool_postgres::Config; - use tokio_postgres::NoTls; + //! To run the postgresql tests locally, add the following service to your + //! pg_service.conf file (usually under `~/.pg_service.conf`): + //! + //! ``` + //! [cosmian_service] + //! host=localhost + //! dbname=cosmian + //! user=cosmian + //! password=cosmian + //! ``` use super::*; use cosmian_findex::{ ADDRESS_LENGTH, WORD_LENGTH, gen_seed, test_guarded_write_concurrent, test_rw_same_address, test_single_write_and_read, test_wrong_guard, }; + use deadpool_postgres::Config; + use tokio_postgres::NoTls; const DB_URL: &str = "postgres://cosmian:cosmian@localhost/cosmian"; @@ -257,7 +232,8 @@ mod tests { Ok(pool) } - // Setup function that handles pool creation, memory initialization, test execution, and cleanup + // Setup function that handles pool creation, memory initialization, test + // execution, and cleanup async fn setup_and_run_test( table_name: &str, test_fn: F, @@ -267,14 +243,9 @@ mod tests { Fut: std::future::Future + Send, { let test_pool = create_testing_pool(DB_URL).await.unwrap(); - let m = PostgresMemory::, [u8; WORD_LENGTH]>::connect_with_pool( - test_pool.clone(), - table_name.to_string(), - ) - .await?; + let m = PostgresMemory::new_with_pool(test_pool.clone(), table_name.to_string()).await; - m.initialize_table(DB_URL.to_string(), table_name.to_string(), NoTls) - .await?; + m.initialize().await?; test_fn(m).await; @@ -292,14 +263,13 @@ mod tests { async fn test_initialization() -> Result<(), PostgresMemoryError> { let table_name: &str = "test_initialization"; let test_pool = create_testing_pool(DB_URL).await.unwrap(); - let m = PostgresMemory::, [u8; WORD_LENGTH]>::connect_with_pool( + let m = PostgresMemory::, [u8; WORD_LENGTH]>::new_with_pool( test_pool.clone(), table_name.to_string(), ) - .await?; + .await; - m.initialize_table(DB_URL.to_string(), table_name.to_string(), NoTls) - .await?; + m.initialize().await?; // check that the table actually exists let client = test_pool.get().await?; @@ -328,7 +298,7 @@ mod tests { #[tokio::test] async fn test_rw_seq() -> Result<(), PostgresMemoryError> { setup_and_run_test("findex_test_rw_seq", |m| async move { - test_single_write_and_read::(&m, gen_seed()).await; + test_single_write_and_read(&m, gen_seed()).await; }) .await } @@ -336,7 +306,7 @@ mod tests { #[tokio::test] async fn test_guard_seq() -> Result<(), PostgresMemoryError> { setup_and_run_test("findex_test_guard_seq", |m| async move { - test_wrong_guard::(&m, gen_seed()).await; + test_wrong_guard(&m, gen_seed()).await; }) .await } @@ -344,7 +314,7 @@ mod tests { #[tokio::test] async fn test_rw_same_address_seq() -> Result<(), PostgresMemoryError> { setup_and_run_test("findex_test_rw_same_address_seq", |m| async move { - test_rw_same_address::(&m, gen_seed()).await; + test_rw_same_address(&m, gen_seed()).await; }) .await } @@ -352,7 +322,7 @@ mod tests { #[tokio::test] async fn test_rw_ccr() -> Result<(), PostgresMemoryError> { setup_and_run_test("findex_test_rw_ccr", |m| async move { - test_guarded_write_concurrent::(&m, gen_seed(), Some(100)).await; + test_guarded_write_concurrent(&m, gen_seed(), Some(100)).await; }) .await } diff --git a/crate/memories/src/sqlite_mem.rs b/crate/memories/src/sqlite_mem.rs index 03c0a317..2ea5ac3f 100644 --- a/crate/memories/src/sqlite_mem.rs +++ b/crate/memories/src/sqlite_mem.rs @@ -47,43 +47,16 @@ impl Debug for SqliteMemory { } } -// The following settings are used to improve performance: -// - journal_mode = WAL : WAL journaling is faster than the default DELETE mode. -// - synchronous = NORMAL: Reduces disk I/O by only calling fsync() at critical -// moments rather than after every transaction (FULL mode); this does not -// compromise data integrity. -fn create_table_script(table_name: &str) -> String { - format!( - " -PRAGMA synchronous = NORMAL; -PRAGMA journal_mode = WAL; -CREATE TABLE IF NOT EXISTS {} ( - a BLOB PRIMARY KEY, - w BLOB NOT NULL -);", - table_name - ) -} - impl SqliteMemory { - /// Builds a pool connected to a known DB (using the given path) and creates - /// the `findex_memory` table. - /// - /// # Arguments - /// - /// * `path` - The path to the sqlite3 database file. - pub async fn connect>( - path: P, + /// Returns a new memory instance using a pool of connections to the SQLite + /// database at the given path. + pub async fn new( + path: impl AsRef, table_name: String, ) -> Result { // The number of connections in this pools defaults to the number of // logical CPUs of the system. let pool = PoolBuilder::new().path(path).open().await?; - // The closure must satisfy 'static bounds, there is no escape from cloning the table name. - let table_name_clone = table_name.clone(); - - pool.conn(move |conn| conn.execute_batch(&create_table_script(&table_name_clone))) - .await?; Ok(Self { pool, @@ -92,34 +65,45 @@ impl SqliteMemory { }) } - /// Returns an `SqliteMemory` instance storing data in a table with the given name - /// and connecting to the DB using connections from the given pool. - /// - /// # Arguments - /// - /// * `pool` - The pool to use for the memory. - /// * `table_name` - The name of the table to create. - pub async fn connect_with_pool( - pool: Pool, - table_name: String, - ) -> Result { - let table_name_clone = table_name.clone(); - pool.conn(move |conn| conn.execute_batch(&create_table_script(&table_name_clone))) - .await?; - Ok(Self { + /// Returns a new memory instance using a pool of connections to an SQLite + /// database. + pub async fn new_with_pool(pool: Pool, table_name: String) -> Self { + Self { pool, table_name, _marker: PhantomData, - }) + } } -} -impl SqliteMemory { + /// Makes sure the correct table exist in the associated database. + pub async fn initialize(&self) -> Result<(), SqliteMemoryError> { + // The following settings are used to improve performance: + // - journal_mode = WAL : WAL journaling is faster than the default DELETE mode. + // - synchronous = NORMAL: Reduces disk I/O by only calling fsync() at critical + // moments rather than after every transaction (FULL mode); this does not + // compromise data integrity. + let table_name = format!( + "PRAGMA synchronous = NORMAL; + PRAGMA journal_mode = WAL; + CREATE TABLE IF NOT EXISTS {} ( + a BLOB PRIMARY KEY, + w BLOB NOT NULL + );", + self.table_name + ); + + self.pool + .conn(move |conn| conn.execute_batch(&table_name)) + .await?; + Ok(()) + } + + /// Clears all bindings from this memory. #[cfg(feature = "test-utils")] pub async fn clear(&self) -> Result<(), SqliteMemoryError> { - let findex_table_name = self.table_name.clone(); + let script = format!("DELETE FROM {}", self.table_name); self.pool - .conn(move |conn| conn.execute(&format!("DELETE FROM {findex_table_name}"), [])) + .conn(move |conn| conn.execute(&script, [])) .await?; Ok(()) } @@ -160,7 +144,7 @@ impl MemoryADT // generate a returned value complying to the batch-read spec. Ok(addresses .iter() - // Copying is necessary here since the same word could be + // Copying is necessary since the same word could be // returned multiple times. .map(|addr| results.get(addr).copied()) .collect()) @@ -229,33 +213,37 @@ mod tests { #[tokio::test] async fn test_rw_seq() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH, TABLE_NAME.to_owned()) + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); + m.initialize().await.unwrap(); test_single_write_and_read(&m, gen_seed()).await } #[tokio::test] async fn test_guard_seq() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH, TABLE_NAME.to_owned()) + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); + m.initialize().await.unwrap(); test_wrong_guard(&m, gen_seed()).await } #[tokio::test] async fn test_collision_seq() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH, TABLE_NAME.to_owned()) + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); + m.initialize().await.unwrap(); test_rw_same_address(&m, gen_seed()).await } #[tokio::test] async fn test_rw_ccr() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH, TABLE_NAME.to_owned()) + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); + m.initialize().await.unwrap(); test_guarded_write_concurrent(&m, gen_seed(), Some(100)).await } } From 61573a3476e5e7bc9f919ce12d35b673cc3325d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Fri, 13 Jun 2025 14:15:31 +0200 Subject: [PATCH 33/60] enforce uniform naming across memories --- crate/memories/benches/benches.rs | 18 +++++++++--------- crate/memories/examples/redis.rs | 2 +- crate/memories/examples/sqlite.rs | 7 ++++--- crate/memories/src/redis_mem.rs | 24 +++++++++++------------- crate/memories/src/sqlite_mem.rs | 10 +++++----- 5 files changed, 30 insertions(+), 31 deletions(-) diff --git a/crate/memories/benches/benches.rs b/crate/memories/benches/benches.rs index ba17c61f..f596d5b8 100644 --- a/crate/memories/benches/benches.rs +++ b/crate/memories/benches/benches.rs @@ -81,7 +81,7 @@ fn bench_search_multiple_bindings(c: &mut Criterion) { bench_memory_search_multiple_bindings( "Redis", N_PTS, - async || RedisMemory::connect(&get_redis_url()).await.unwrap(), + async || RedisMemory::new_with_url(&get_redis_url()).await.unwrap(), c, &mut rng, ); @@ -91,7 +91,7 @@ fn bench_search_multiple_bindings(c: &mut Criterion) { "SQLite", N_PTS, async || { - let m = SqliteMemory::new( + let m = SqliteMemory::new_with_path( SQLITE_PATH, "bench_memory_search_multiple_bindings".to_string(), ) @@ -130,7 +130,7 @@ fn bench_search_multiple_keywords(c: &mut Criterion) { bench_memory_search_multiple_keywords( "Redis", N_PTS, - async || RedisMemory::connect(&get_redis_url()).await.unwrap(), + async || RedisMemory::new_with_url(&get_redis_url()).await.unwrap(), c, &mut rng, ); @@ -140,7 +140,7 @@ fn bench_search_multiple_keywords(c: &mut Criterion) { "SQLite", N_PTS, async || { - let m = SqliteMemory::new( + let m = SqliteMemory::new_with_path( SQLITE_PATH, "bench_memory_search_multiple_keywords".to_string(), ) @@ -177,7 +177,7 @@ fn bench_insert_multiple_bindings(c: &mut Criterion) { bench_memory_insert_multiple_bindings( "Redis", N_PTS, - async || RedisMemory::connect(&get_redis_url()).await.unwrap(), + async || RedisMemory::new_with_url(&get_redis_url()).await.unwrap(), c, RedisMemory::clear, &mut rng, @@ -188,7 +188,7 @@ fn bench_insert_multiple_bindings(c: &mut Criterion) { "SQLite", N_PTS, async || { - let m = SqliteMemory::new( + let m = SqliteMemory::new_with_path( SQLITE_PATH, "bench_memory_insert_multiple_bindings".to_string(), ) @@ -229,7 +229,7 @@ fn bench_contention(c: &mut Criterion) { bench_memory_contention( "Redis", N_PTS, - async || RedisMemory::connect(&get_redis_url()).await.unwrap(), + async || RedisMemory::new_with_url(&get_redis_url()).await.unwrap(), c, RedisMemory::clear, &mut rng, @@ -240,7 +240,7 @@ fn bench_contention(c: &mut Criterion) { "SQLite", N_PTS, async || { - let m = SqliteMemory::new(SQLITE_PATH, "bench_memory_contention".to_string()) + let m = SqliteMemory::new_with_path(SQLITE_PATH, "bench_memory_contention".to_string()) .await .unwrap(); m.initialize().await.unwrap(); @@ -368,7 +368,7 @@ fn bench_one_to_many(c: &mut Criterion) { N_PTS, async || { DelayedMemory::new( - RedisMemory::connect(&get_redis_url()).await.unwrap(), + RedisMemory::new_with_url(&get_redis_url()).await.unwrap(), *mean, *variance, ) diff --git a/crate/memories/examples/redis.rs b/crate/memories/examples/redis.rs index ee52e122..b7066888 100644 --- a/crate/memories/examples/redis.rs +++ b/crate/memories/examples/redis.rs @@ -28,7 +28,7 @@ async fn main() { let index = gen_index(&mut rng); // This example uses our Redis-based implementation of `MemoryADT`. - let memory = RedisMemory::, [u8; WORD_LENGTH]>::connect(DB_PATH) + let memory = RedisMemory::, [u8; WORD_LENGTH]>::new_with_url(DB_PATH) .await .unwrap(); diff --git a/crate/memories/examples/sqlite.rs b/crate/memories/examples/sqlite.rs index 40d93653..6b6503e2 100644 --- a/crate/memories/examples/sqlite.rs +++ b/crate/memories/examples/sqlite.rs @@ -28,9 +28,10 @@ async fn main() { // Generating the random index. let index = gen_index(&mut rng); - let memory = SqliteMemory::<_, [u8; WORD_LENGTH]>::new(DB_PATH, TABLE_NAME.to_owned()) - .await - .unwrap(); + let memory = + SqliteMemory::<_, [u8; WORD_LENGTH]>::new_with_path(DB_PATH, TABLE_NAME.to_owned()) + .await + .unwrap(); let encrypted_memory = MemoryEncryptionLayer::new(&key, memory); diff --git a/crate/memories/src/redis_mem.rs b/crate/memories/src/redis_mem.rs index a39747d6..8825459b 100644 --- a/crate/memories/src/redis_mem.rs +++ b/crate/memories/src/redis_mem.rs @@ -60,8 +60,8 @@ impl fmt::Debug for RedisMemory { } impl RedisMemory { - /// Connects to a Redis server with a `ConnectionManager`. - pub async fn connect_with_manager( + /// Returns a new instance using this connection manager. + pub async fn new_with_manager( mut manager: ConnectionManager, ) -> Result { let script_hash = redis::cmd("SCRIPT") @@ -77,13 +77,14 @@ impl RedisMemory { }) } - /// Connects to a Redis server using the given URL. - pub async fn connect(url: &str) -> Result { + /// Returns a new instance using the Redis instance at the given URL. + pub async fn new_with_url(url: &str) -> Result { let client = redis::Client::open(url)?; let manager = client.get_connection_manager().await?; - Self::connect_with_manager(manager).await + Self::new_with_manager(manager).await } + /// Clears all bindings from this memory. #[cfg(feature = "test-utils")] pub async fn clear(&self) -> Result<(), RedisMemoryError> { redis::cmd("FLUSHDB") @@ -106,7 +107,6 @@ impl MemoryADT ) -> Result>, Self::Error> { let mut cmd = redis::cmd("MGET"); let cmd = addresses.iter().fold(&mut cmd, |c, a| c.arg(&**a)); - // Cloning the connection manager is cheap since it is an `Arc`. cmd.query_async(&mut self.manager.clone()) .await .map_err(RedisMemoryError::RedisError) @@ -118,6 +118,7 @@ impl MemoryADT bindings: Vec<(Self::Address, Self::Word)>, ) -> Result, Self::Error> { let (guard_address, guard_value) = guard; + let mut cmd = redis::cmd("EVALSHA"); let cmd = cmd .arg(self.script_hash.as_str()) @@ -129,12 +130,9 @@ impl MemoryADT .map(|bytes| bytes.as_slice()) .unwrap_or(b"false".as_slice()), ); - let cmd = bindings .iter() .fold(cmd.arg(bindings.len()), |cmd, (a, w)| cmd.arg(&**a).arg(w)); - - // Cloning the connection manager is cheap since it is an `Arc`. cmd.query_async(&mut self.manager.clone()) .await .map_err(RedisMemoryError::RedisError) @@ -159,25 +157,25 @@ mod tests { #[tokio::test] async fn test_rw_seq() { - let m = RedisMemory::connect(&get_redis_url()).await.unwrap(); + let m = RedisMemory::new_with_url(&get_redis_url()).await.unwrap(); test_single_write_and_read::(&m, gen_seed()).await } #[tokio::test] async fn test_guard_seq() { - let m = RedisMemory::connect(&get_redis_url()).await.unwrap(); + let m = RedisMemory::new_with_url(&get_redis_url()).await.unwrap(); test_wrong_guard::(&m, gen_seed()).await } #[tokio::test] async fn test_collision_seq() { - let m = RedisMemory::connect(&get_redis_url()).await.unwrap(); + let m = RedisMemory::new_with_url(&get_redis_url()).await.unwrap(); test_rw_same_address::(&m, gen_seed()).await } #[tokio::test] async fn test_rw_ccr() { - let m = RedisMemory::connect(&get_redis_url()).await.unwrap(); + let m = RedisMemory::new_with_url(&get_redis_url()).await.unwrap(); test_guarded_write_concurrent::(&m, gen_seed(), None).await } } diff --git a/crate/memories/src/sqlite_mem.rs b/crate/memories/src/sqlite_mem.rs index 2ea5ac3f..1c0c058b 100644 --- a/crate/memories/src/sqlite_mem.rs +++ b/crate/memories/src/sqlite_mem.rs @@ -50,7 +50,7 @@ impl Debug for SqliteMemory { impl SqliteMemory { /// Returns a new memory instance using a pool of connections to the SQLite /// database at the given path. - pub async fn new( + pub async fn new_with_path( path: impl AsRef, table_name: String, ) -> Result { @@ -213,7 +213,7 @@ mod tests { #[tokio::test] async fn test_rw_seq() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new(DB_PATH, TABLE_NAME.to_owned()) + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new_with_path(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); m.initialize().await.unwrap(); @@ -222,7 +222,7 @@ mod tests { #[tokio::test] async fn test_guard_seq() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new(DB_PATH, TABLE_NAME.to_owned()) + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new_with_path(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); m.initialize().await.unwrap(); @@ -231,7 +231,7 @@ mod tests { #[tokio::test] async fn test_collision_seq() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new(DB_PATH, TABLE_NAME.to_owned()) + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new_with_path(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); m.initialize().await.unwrap(); @@ -240,7 +240,7 @@ mod tests { #[tokio::test] async fn test_rw_ccr() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new(DB_PATH, TABLE_NAME.to_owned()) + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new_with_path(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); m.initialize().await.unwrap(); From 63eb2123d801446e3693fe225dcf86d9266adfcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Fri, 13 Jun 2025 14:21:03 +0200 Subject: [PATCH 34/60] remove redundant dependency specification --- crate/memories/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crate/memories/Cargo.toml b/crate/memories/Cargo.toml index d1ba1a6e..a3333c6e 100644 --- a/crate/memories/Cargo.toml +++ b/crate/memories/Cargo.toml @@ -18,7 +18,7 @@ path = "src/lib.rs" redis-mem = ["redis"] sqlite-mem = ["async-sqlite"] postgres-mem = ["deadpool-postgres", "tokio", "tokio-postgres"] -test-utils = ["cosmian_findex/test-utils", "redis/tokio-comp"] +test-utils = ["cosmian_findex/test-utils"] [dependencies] async-sqlite = { version = "0.5", optional = true } From fb02fe9236eb9563af88a1b7ea6147ec295dcda1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Fri, 13 Jun 2025 14:25:28 +0200 Subject: [PATCH 35/60] remove unnecessary type hint --- crate/memories/src/postgresql_mem/memory.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crate/memories/src/postgresql_mem/memory.rs b/crate/memories/src/postgresql_mem/memory.rs index 59b0023c..a22a366d 100644 --- a/crate/memories/src/postgresql_mem/memory.rs +++ b/crate/memories/src/postgresql_mem/memory.rs @@ -129,7 +129,7 @@ impl MemoryADT .transpose() .map_err(PostgresMemoryError::TryFromSliceError) }) - .collect::>() + .collect() } async fn guarded_write( From 58e4fc1cd555a4b11e62c2b4d077dfb026979806 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Thu, 26 Jun 2025 16:31:44 +0200 Subject: [PATCH 36/60] fix: ci --- .github/workflows/benches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml index 8762c04a..807555f5 100644 --- a/.github/workflows/benches.yml +++ b/.github/workflows/benches.yml @@ -9,5 +9,5 @@ jobs: uses: Cosmian/reusable_workflows/.github/workflows/cargo-bench.yml@develop with: toolchain: 1.87.0 - features: test-utils,redis-mem,sqlite-mem,rust-mem,postgres-mem + features: test-utils,redis-mem,sqlite-mem,postgres-mem force: true From c1552ce6cfabfa9a858c67a5504324a66dbefaa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Thu, 26 Jun 2025 14:37:33 +0000 Subject: [PATCH 37/60] Update crate/memories/src/sqlite_mem.rs Co-authored-by: Hatem M'naouer <19950216+HatemMn@users.noreply.github.com> --- crate/memories/src/sqlite_mem.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crate/memories/src/sqlite_mem.rs b/crate/memories/src/sqlite_mem.rs index 1c0c058b..36eebfff 100644 --- a/crate/memories/src/sqlite_mem.rs +++ b/crate/memories/src/sqlite_mem.rs @@ -82,7 +82,7 @@ impl SqliteMemory { // - synchronous = NORMAL: Reduces disk I/O by only calling fsync() at critical // moments rather than after every transaction (FULL mode); this does not // compromise data integrity. - let table_name = format!( + let initialization_script = format!( "PRAGMA synchronous = NORMAL; PRAGMA journal_mode = WAL; CREATE TABLE IF NOT EXISTS {} ( @@ -93,7 +93,7 @@ impl SqliteMemory { ); self.pool - .conn(move |conn| conn.execute_batch(&table_name)) + .conn(move |conn| conn.execute_batch(&initialization_script)) .await?; Ok(()) } From c851ec99a255255ee8d12d4e39ca90f00f58b64e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Thu, 12 Jun 2025 17:55:00 +0200 Subject: [PATCH 38/60] upgrade toolchain --- .github/workflows/benches.yml | 2 +- .github/workflows/ci.yml | 6 +++--- .github/workflows/hack.yml | 2 +- crate/findex/src/lib.rs | 1 + rust-toolchain.toml | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml index a4e6a877..8762c04a 100644 --- a/.github/workflows/benches.yml +++ b/.github/workflows/benches.yml @@ -8,6 +8,6 @@ jobs: bench: uses: Cosmian/reusable_workflows/.github/workflows/cargo-bench.yml@develop with: - toolchain: stable + toolchain: 1.87.0 features: test-utils,redis-mem,sqlite-mem,rust-mem,postgres-mem force: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63b1e174..b6f777c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ jobs: cargo-lint: uses: Cosmian/reusable_workflows/.github/workflows/cargo-nursery.yml@develop with: - toolchain: 1.85.0 + toolchain: 1.87.0 workspace: true cargo-machete: runs-on: ubuntu-latest @@ -17,7 +17,7 @@ jobs: - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: - toolchain: 1.85.0 + toolchain: 1.87.0 - name: Install cargo-machete run: cargo install cargo-machete - name: Check unused dependencies in findex crate @@ -34,7 +34,7 @@ jobs: uses: Cosmian/reusable_workflows/.github/workflows/cargo-publish.yml@develop if: startsWith(github.ref, 'refs/tags/') with: - toolchain: 1.85.0 + toolchain: 1.87.0 secrets: inherit cleanup: needs: diff --git a/.github/workflows/hack.yml b/.github/workflows/hack.yml index 78db6cb2..c8e6fa9c 100644 --- a/.github/workflows/hack.yml +++ b/.github/workflows/hack.yml @@ -35,7 +35,7 @@ jobs: - uses: actions/checkout@v1 - uses: actions-rs/toolchain@v1 with: - toolchain: stable + toolchain: 1.87.0 override: true - name: Install cargo-hack run: cargo install --locked cargo-hack || true diff --git a/crate/findex/src/lib.rs b/crate/findex/src/lib.rs index 656012a2..d0d407d6 100644 --- a/crate/findex/src/lib.rs +++ b/crate/findex/src/lib.rs @@ -12,6 +12,7 @@ mod error; mod findex; mod memory; mod ovec; + #[cfg(any(test, feature = "test-utils"))] mod test_utils; diff --git a/rust-toolchain.toml b/rust-toolchain.toml index c8181991..45a64fa6 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.85.0" +channel = "1.87.0" components = ["rustfmt"] From fc98a2b64fb6cfe277eb3084f96a3c5841544478 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Fri, 13 Jun 2025 12:43:40 +0200 Subject: [PATCH 39/60] refacto of the PostgreSQL memory implementation --- Cargo.toml | 2 +- crate/memories/Cargo.toml | 19 +- crate/memories/benches/benches.rs | 38 +-- crate/memories/examples/postgresql.rs | 12 +- crate/memories/examples/sqlite.rs | 2 +- crate/memories/src/lib.rs | 2 - crate/memories/src/postgresql_mem/memory.rs | 244 +++++++++----------- crate/memories/src/sqlite_mem.rs | 100 ++++---- 8 files changed, 191 insertions(+), 228 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3e7908ec..8658cf3e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,4 +27,4 @@ cosmian_crypto_core = { version = "10.1", default-features = false, features = [ "sha3", ] } criterion = { version = "0.6" } -tokio = { version = "1.44" } +tokio = { version = "1.45" } diff --git a/crate/memories/Cargo.toml b/crate/memories/Cargo.toml index 1579f329..d1ba1a6e 100644 --- a/crate/memories/Cargo.toml +++ b/crate/memories/Cargo.toml @@ -17,33 +17,32 @@ path = "src/lib.rs" [features] redis-mem = ["redis"] sqlite-mem = ["async-sqlite"] -postgres-mem = ["tokio-postgres", "tokio", "deadpool-postgres"] -test-utils = ["cosmian_findex/test-utils"] +postgres-mem = ["deadpool-postgres", "tokio", "tokio-postgres"] +test-utils = ["cosmian_findex/test-utils", "redis/tokio-comp"] [dependencies] -cosmian_crypto_core.workspace = true +async-sqlite = { version = "0.5", optional = true } cosmian_findex = { path = "../findex", version = "8.0" } -async-sqlite = { version = "0.4", optional = true } # `async-sqlite` dependency is pinned at v0.4 due to dependency tree issues with sqlx requiring a lower version of `libsqlite3-sys` -deadpool-postgres = { version = "0.14.1", optional = true } +deadpool-postgres = { version = "0.14", optional = true } redis = { version = "0.32", default-features = false, features = [ - "aio", "connection-manager", "tokio-comp", ], optional = true } tokio = { workspace = true, features = ["rt-multi-thread"], optional = true } -tokio-postgres = { version = "0.7.9", optional = true, features = [ +tokio-postgres = { version = "0.7", optional = true, features = [ "array-impls", ] } [dev-dependencies] +cosmian_crypto_core.workspace = true cosmian_findex = { path = "../findex", version = "8.0", features = [ "test-utils", ] } -futures = { version = "0.3" } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } criterion = { workspace = true } -rand = { version = "0.9.0" } -rand_distr = { version = "0.5.1" } +futures = { version = "0.3" } +rand = "0.9" +rand_distr = "0.5" [[bench]] name = "benches" diff --git a/crate/memories/benches/benches.rs b/crate/memories/benches/benches.rs index 266947f1..ba17c61f 100644 --- a/crate/memories/benches/benches.rs +++ b/crate/memories/benches/benches.rs @@ -61,14 +61,14 @@ async fn connect_and_init_table( pg_config.url = Some(db_url.to_string()); let test_pool = pg_config.builder(NoTls)?.build()?; - let m = PostgresMemory::, [u8; WORD_LENGTH]>::connect_with_pool( + let m = PostgresMemory::, [u8; WORD_LENGTH]>::new_with_pool( test_pool, table_name.to_string(), ) - .await?; + .await; + + m.initialize().await?; - m.initialize_table(db_url.to_string(), table_name, NoTls) - .await?; Ok(m) } // Number of points in each graph. @@ -91,12 +91,14 @@ fn bench_search_multiple_bindings(c: &mut Criterion) { "SQLite", N_PTS, async || { - SqliteMemory::connect( + let m = SqliteMemory::new( SQLITE_PATH, "bench_memory_search_multiple_bindings".to_string(), ) .await - .unwrap() + .unwrap(); + m.initialize().await.unwrap(); + m }, c, &mut rng, @@ -107,12 +109,14 @@ fn bench_search_multiple_bindings(c: &mut Criterion) { "Postgres", N_PTS, async || { - connect_and_init_table( + let m = connect_and_init_table( get_postgresql_url(), "bench_memory_search_multiple_bindings".to_string(), ) .await - .unwrap() + .unwrap(); + m.initialize().await.unwrap(); + m }, c, &mut rng, @@ -136,12 +140,14 @@ fn bench_search_multiple_keywords(c: &mut Criterion) { "SQLite", N_PTS, async || { - SqliteMemory::connect( + let m = SqliteMemory::new( SQLITE_PATH, "bench_memory_search_multiple_keywords".to_string(), ) .await - .unwrap() + .unwrap(); + m.initialize().await.unwrap(); + m }, c, &mut rng, @@ -182,12 +188,14 @@ fn bench_insert_multiple_bindings(c: &mut Criterion) { "SQLite", N_PTS, async || { - SqliteMemory::connect( + let m = SqliteMemory::new( SQLITE_PATH, "bench_memory_insert_multiple_bindings".to_string(), ) .await - .unwrap() + .unwrap(); + m.initialize().await.unwrap(); + m }, c, SqliteMemory::clear, @@ -232,9 +240,11 @@ fn bench_contention(c: &mut Criterion) { "SQLite", N_PTS, async || { - SqliteMemory::connect(SQLITE_PATH, "bench_memory_contention".to_string()) + let m = SqliteMemory::new(SQLITE_PATH, "bench_memory_contention".to_string()) .await - .unwrap() + .unwrap(); + m.initialize().await.unwrap(); + m }, c, SqliteMemory::clear, diff --git a/crate/memories/examples/postgresql.rs b/crate/memories/examples/postgresql.rs index 0fd3b061..5160a162 100644 --- a/crate/memories/examples/postgresql.rs +++ b/crate/memories/examples/postgresql.rs @@ -40,7 +40,8 @@ async fn main() { // Generating a random index. let index = gen_index(&mut rng); - // Addd the following service to your pg_service.conf file (usually under ~/.pg_service.conf): + // Addd the following service to your pg_service.conf file (usually under + // `~/.pg_service.conf`): // // [cosmian_service] // host=localhost @@ -48,20 +49,17 @@ async fn main() { // user=cosmian // password=cosmian let pool = create_pool(DB_URL).await.unwrap(); - let m = PostgresMemory::, [u8; WORD_LENGTH]>::connect_with_pool( + let m = PostgresMemory::, [u8; WORD_LENGTH]>::new_with_pool( pool.clone(), TABLE_NAME.to_string(), ) - .await - .unwrap(); + .await; // Notice we chose to not enable TLS: it's not needed for this example as we // are using the encryption layer in top of the memory interface - i.e. the // data is already encrypted before being sent to the database and TLS would // add unnecessary overhead. - m.initialize_table(DB_URL.to_string(), TABLE_NAME.to_string(), NoTls) - .await - .unwrap(); + m.initialize().await.unwrap(); let encrypted_memory = MemoryEncryptionLayer::new(&key, m); diff --git a/crate/memories/examples/sqlite.rs b/crate/memories/examples/sqlite.rs index e38146f5..40d93653 100644 --- a/crate/memories/examples/sqlite.rs +++ b/crate/memories/examples/sqlite.rs @@ -28,7 +28,7 @@ async fn main() { // Generating the random index. let index = gen_index(&mut rng); - let memory = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH, TABLE_NAME.to_owned()) + let memory = SqliteMemory::<_, [u8; WORD_LENGTH]>::new(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); diff --git a/crate/memories/src/lib.rs b/crate/memories/src/lib.rs index 3ffc8213..25f593c8 100644 --- a/crate/memories/src/lib.rs +++ b/crate/memories/src/lib.rs @@ -22,7 +22,5 @@ pub mod reexport { #[cfg(feature = "redis-mem")] pub use redis; #[cfg(feature = "postgres-mem")] - pub use tokio; - #[cfg(feature = "postgres-mem")] pub use tokio_postgres; } diff --git a/crate/memories/src/postgresql_mem/memory.rs b/crate/memories/src/postgresql_mem/memory.rs index a14603f4..59b0023c 100644 --- a/crate/memories/src/postgresql_mem/memory.rs +++ b/crate/memories/src/postgresql_mem/memory.rs @@ -2,10 +2,6 @@ use super::PostgresMemoryError; use cosmian_findex::{Address, MemoryADT}; use deadpool_postgres::Pool; use std::marker::PhantomData; -use tokio_postgres::{ - Socket, - tls::{MakeTlsConnect, TlsConnect}, -}; #[derive(Clone, Debug)] pub struct PostgresMemory { @@ -17,65 +13,38 @@ pub struct PostgresMemory { impl PostgresMemory, [u8; WORD_LENGTH]> { - /// Connect to a Postgres database and create a table if it doesn't exist - pub async fn initialize_table( - &self, - db_url: String, - table_name: String, - tls: T, - ) -> Result<(), PostgresMemoryError> - where - T: MakeTlsConnect + Send, - T::Stream: Send + 'static, - T::TlsConnect: Send, - >::Future: Send, - { - let (client, connection) = tokio_postgres::connect(&db_url, tls).await?; - - // The connection object performs the actual communication with the database - // `Connection` only resolves when the connection is closed, either because a fatal error has - // occurred, or because its associated `Client` has dropped and all outstanding work has completed. - let conn_handle = tokio::spawn(async move { - if let Err(e) = connection.await { - eprintln!("connection error: {}", e); - } - }); + /// Returns a new memory instance from the given connection pool to a + /// PostgreSQL database. + pub async fn new_with_pool(pool: Pool, table_name: String) -> Self { + Self { + pool, + table_name, + _marker: PhantomData, + } + } - let returned = client + /// Connects to a PostgreSQL database and creates a table if it doesn't + /// exist. + pub async fn initialize(&self) -> Result<(), PostgresMemoryError> { + self.pool + .get() + .await? .execute( &format!( - " - CREATE TABLE IF NOT EXISTS {} ( + "CREATE TABLE IF NOT EXISTS {} ( a BYTEA PRIMARY KEY CHECK (octet_length(a) = {}), w BYTEA NOT NULL CHECK (octet_length(w) = {}) );", - table_name, ADDRESS_LENGTH, WORD_LENGTH + self.table_name, ADDRESS_LENGTH, WORD_LENGTH ), &[], ) .await?; - if returned != 0 { - return Err(PostgresMemoryError::TableCreationError(returned)); - } - drop(client); - let _ = conn_handle.await; // ensures that the connection is closed Ok(()) } - /// Connect to a Postgres database and create a table if it doesn't exist - pub async fn connect_with_pool( - pool: Pool, - table_name: String, - ) -> Result { - Ok(Self { - pool, - table_name, - _marker: PhantomData, - }) - } - - /// Deletes all rows from the findex memory table + /// Clears all bindings from this memory. #[cfg(feature = "test-utils")] pub async fn clear(&self) -> Result<(), PostgresMemoryError> { self.pool @@ -83,8 +52,38 @@ impl .await? .execute(&format!("TRUNCATE TABLE {};", self.table_name), &[]) .await?; + Ok(()) } + + fn gwrite_script(&self) -> String { + format!( + " + WITH + guard_check AS ( + SELECT w FROM {0} WHERE a = $1::bytea + ), + dedup_input_table AS ( + SELECT DISTINCT ON (a) a, w + FROM UNNEST($3::bytea[], $4::bytea[]) WITH ORDINALITY AS t(a, w, order_idx) + ORDER BY a, order_idx DESC + ), + insert_cte AS ( + INSERT INTO {0} (a, w) + SELECT a, w FROM dedup_input_table AS t(a,w) + WHERE ( + $2::bytea IS NULL AND NOT EXISTS (SELECT 1 FROM guard_check) + ) OR ( + $2::bytea IS NOT NULL AND EXISTS ( + SELECT 1 FROM guard_check WHERE w = $2::bytea + ) + ) + ON CONFLICT (a) DO UPDATE SET w = EXCLUDED.w + ) + SELECT COALESCE((SELECT w FROM guard_check)) AS original_guard_value;", + self.table_name + ) + } } impl MemoryADT @@ -99,23 +98,22 @@ impl MemoryADT addresses: Vec, ) -> Result>, Self::Error> { let client = self.pool.get().await?; - // statements are cached per connection and not per pool + + // Statements are cached per connection and not per pool. let stmt = client - .prepare_cached( - format!( - "SELECT f.w + .prepare_cached(&format!( + // The left join is necessary to ensure that the order of the + // addresses is preserved as well as to return None for addresses + // that don't exist. + "SELECT f.w FROM UNNEST($1::bytea[]) WITH ORDINALITY AS params(addr, idx) LEFT JOIN {} f ON params.addr = f.a ORDER BY params.idx;", - self.table_name - ) - .as_str(), - ) + self.table_name + )) .await?; client - // the left join is necessary to ensure that the order of the addresses is preserved - // as well as to return None for addresses that don't exist .query( &stmt, &[&addresses @@ -126,15 +124,12 @@ impl MemoryADT .await? .iter() .map(|row| { - let bytes_slice: Option<&[u8]> = row.try_get("w")?; // `row.get(0)` can panic - bytes_slice.map_or(Ok(None), |slice| { - slice - .try_into() - .map(Some) - .map_err(|_| PostgresMemoryError::InvalidDataLength(slice.len())) - }) + row.try_get::<_, Option<&[u8]>>("w")? + .map(Self::Word::try_from) + .transpose() + .map_err(PostgresMemoryError::TryFromSliceError) }) - .collect() + .collect::>() } async fn guarded_write( @@ -142,45 +137,19 @@ impl MemoryADT guard: (Self::Address, Option), bindings: Vec<(Self::Address, Self::Word)>, ) -> Result, Self::Error> { - let g_write_script = format!( - " - WITH - guard_check AS ( - SELECT w FROM {0} WHERE a = $1::bytea - ), - dedup_input_table AS ( - SELECT DISTINCT ON (a) a, w - FROM UNNEST($3::bytea[], $4::bytea[]) WITH ORDINALITY AS t(a, w, order_idx) - ORDER BY a, order_idx DESC - ), - insert_cte AS ( - INSERT INTO {0} (a, w) - SELECT a, w FROM dedup_input_table AS t(a,w) - WHERE ( - $2::bytea IS NULL AND NOT EXISTS (SELECT 1 FROM guard_check) - ) OR ( - $2::bytea IS NOT NULL AND EXISTS ( - SELECT 1 FROM guard_check WHERE w = $2::bytea - ) - ) - ON CONFLICT (a) DO UPDATE SET w = EXCLUDED.w - ) - SELECT COALESCE((SELECT w FROM guard_check)) AS original_guard_value;", - self.table_name - ); - let (addresses, words): (Vec<[u8; ADDRESS_LENGTH]>, Vec) = bindings.into_iter().map(|(a, w)| (*a, w)).unzip(); - const MAX_RETRIES: usize = 10; - for _ in 0..MAX_RETRIES { - // while counterintuitive, getting a new client on each retry is a better approach - // than trying to reuse the same client since it allows other operations to use the - // connection between retries. + // Since a guarded write operation is lock-free, this loop is guaranteed + // to terminate. + loop { + // Do not lock a resource for a potentially long loop, instead + // request a new one at each iteration. let mut client = self.pool.get().await?; - let stmt = client.prepare_cached(g_write_script.as_str()).await?; - let result = async { + let stmt = client.prepare_cached(&self.gwrite_script()).await?; + + let res = async { let tx = client .build_transaction() .isolation_level( @@ -200,21 +169,26 @@ impl MemoryADT ], ) .await? - .map_or( - Ok::, PostgresMemoryError>(None), - |row| { - row.try_get::<_, Option<&[u8]>>(0)? - .map_or(Ok(None), |r| Ok(Some(r.try_into()?))) - }, - )?; + .map(|row| { + row.try_get::<_, Option<&[u8]>>(0)? + .map(Self::Word::try_from) + .transpose() + .map_err(PostgresMemoryError::TryFromSliceError) + }) + .transpose()? + .flatten(); + tx.commit().await?; + Ok(res) } .await; - match result { + + match res { Ok(value) => return Ok(value), Err(err) => { - // Retry on serialization failures (error code 40001), otherwise fail and return the error + // Retry on serialization failures (error code 40001), + // otherwise fail and return the error if let PostgresMemoryError::TokioPostgresError(pg_err) = &err { if pg_err.code().is_some_and(|code| code.code() == "40001") { continue; @@ -224,28 +198,29 @@ impl MemoryADT } } } - Err(PostgresMemoryError::RetryExhaustedError(MAX_RETRIES)) } } #[cfg(test)] mod tests { - // To run the postgresql tests locally, add the following service to your pg_service.conf file - // (usually under ~/.pg_service.conf): - // - // [cosmian_service] - // host=localhost - // dbname=cosmian - // user=cosmian - // password=cosmian - use deadpool_postgres::Config; - use tokio_postgres::NoTls; + //! To run the postgresql tests locally, add the following service to your + //! pg_service.conf file (usually under `~/.pg_service.conf`): + //! + //! ``` + //! [cosmian_service] + //! host=localhost + //! dbname=cosmian + //! user=cosmian + //! password=cosmian + //! ``` use super::*; use cosmian_findex::{ ADDRESS_LENGTH, WORD_LENGTH, gen_seed, test_guarded_write_concurrent, test_rw_same_address, test_single_write_and_read, test_wrong_guard, }; + use deadpool_postgres::Config; + use tokio_postgres::NoTls; const DB_URL: &str = "postgres://cosmian:cosmian@localhost/cosmian"; @@ -257,7 +232,8 @@ mod tests { Ok(pool) } - // Setup function that handles pool creation, memory initialization, test execution, and cleanup + // Setup function that handles pool creation, memory initialization, test + // execution, and cleanup async fn setup_and_run_test( table_name: &str, test_fn: F, @@ -267,14 +243,9 @@ mod tests { Fut: std::future::Future + Send, { let test_pool = create_testing_pool(DB_URL).await.unwrap(); - let m = PostgresMemory::, [u8; WORD_LENGTH]>::connect_with_pool( - test_pool.clone(), - table_name.to_string(), - ) - .await?; + let m = PostgresMemory::new_with_pool(test_pool.clone(), table_name.to_string()).await; - m.initialize_table(DB_URL.to_string(), table_name.to_string(), NoTls) - .await?; + m.initialize().await?; test_fn(m).await; @@ -292,14 +263,13 @@ mod tests { async fn test_initialization() -> Result<(), PostgresMemoryError> { let table_name: &str = "test_initialization"; let test_pool = create_testing_pool(DB_URL).await.unwrap(); - let m = PostgresMemory::, [u8; WORD_LENGTH]>::connect_with_pool( + let m = PostgresMemory::, [u8; WORD_LENGTH]>::new_with_pool( test_pool.clone(), table_name.to_string(), ) - .await?; + .await; - m.initialize_table(DB_URL.to_string(), table_name.to_string(), NoTls) - .await?; + m.initialize().await?; // check that the table actually exists let client = test_pool.get().await?; @@ -328,7 +298,7 @@ mod tests { #[tokio::test] async fn test_rw_seq() -> Result<(), PostgresMemoryError> { setup_and_run_test("findex_test_rw_seq", |m| async move { - test_single_write_and_read::(&m, gen_seed()).await; + test_single_write_and_read(&m, gen_seed()).await; }) .await } @@ -336,7 +306,7 @@ mod tests { #[tokio::test] async fn test_guard_seq() -> Result<(), PostgresMemoryError> { setup_and_run_test("findex_test_guard_seq", |m| async move { - test_wrong_guard::(&m, gen_seed()).await; + test_wrong_guard(&m, gen_seed()).await; }) .await } @@ -344,7 +314,7 @@ mod tests { #[tokio::test] async fn test_rw_same_address_seq() -> Result<(), PostgresMemoryError> { setup_and_run_test("findex_test_rw_same_address_seq", |m| async move { - test_rw_same_address::(&m, gen_seed()).await; + test_rw_same_address(&m, gen_seed()).await; }) .await } @@ -352,7 +322,7 @@ mod tests { #[tokio::test] async fn test_rw_ccr() -> Result<(), PostgresMemoryError> { setup_and_run_test("findex_test_rw_ccr", |m| async move { - test_guarded_write_concurrent::(&m, gen_seed(), Some(100)).await; + test_guarded_write_concurrent(&m, gen_seed(), Some(100)).await; }) .await } diff --git a/crate/memories/src/sqlite_mem.rs b/crate/memories/src/sqlite_mem.rs index 03c0a317..2ea5ac3f 100644 --- a/crate/memories/src/sqlite_mem.rs +++ b/crate/memories/src/sqlite_mem.rs @@ -47,43 +47,16 @@ impl Debug for SqliteMemory { } } -// The following settings are used to improve performance: -// - journal_mode = WAL : WAL journaling is faster than the default DELETE mode. -// - synchronous = NORMAL: Reduces disk I/O by only calling fsync() at critical -// moments rather than after every transaction (FULL mode); this does not -// compromise data integrity. -fn create_table_script(table_name: &str) -> String { - format!( - " -PRAGMA synchronous = NORMAL; -PRAGMA journal_mode = WAL; -CREATE TABLE IF NOT EXISTS {} ( - a BLOB PRIMARY KEY, - w BLOB NOT NULL -);", - table_name - ) -} - impl SqliteMemory { - /// Builds a pool connected to a known DB (using the given path) and creates - /// the `findex_memory` table. - /// - /// # Arguments - /// - /// * `path` - The path to the sqlite3 database file. - pub async fn connect>( - path: P, + /// Returns a new memory instance using a pool of connections to the SQLite + /// database at the given path. + pub async fn new( + path: impl AsRef, table_name: String, ) -> Result { // The number of connections in this pools defaults to the number of // logical CPUs of the system. let pool = PoolBuilder::new().path(path).open().await?; - // The closure must satisfy 'static bounds, there is no escape from cloning the table name. - let table_name_clone = table_name.clone(); - - pool.conn(move |conn| conn.execute_batch(&create_table_script(&table_name_clone))) - .await?; Ok(Self { pool, @@ -92,34 +65,45 @@ impl SqliteMemory { }) } - /// Returns an `SqliteMemory` instance storing data in a table with the given name - /// and connecting to the DB using connections from the given pool. - /// - /// # Arguments - /// - /// * `pool` - The pool to use for the memory. - /// * `table_name` - The name of the table to create. - pub async fn connect_with_pool( - pool: Pool, - table_name: String, - ) -> Result { - let table_name_clone = table_name.clone(); - pool.conn(move |conn| conn.execute_batch(&create_table_script(&table_name_clone))) - .await?; - Ok(Self { + /// Returns a new memory instance using a pool of connections to an SQLite + /// database. + pub async fn new_with_pool(pool: Pool, table_name: String) -> Self { + Self { pool, table_name, _marker: PhantomData, - }) + } } -} -impl SqliteMemory { + /// Makes sure the correct table exist in the associated database. + pub async fn initialize(&self) -> Result<(), SqliteMemoryError> { + // The following settings are used to improve performance: + // - journal_mode = WAL : WAL journaling is faster than the default DELETE mode. + // - synchronous = NORMAL: Reduces disk I/O by only calling fsync() at critical + // moments rather than after every transaction (FULL mode); this does not + // compromise data integrity. + let table_name = format!( + "PRAGMA synchronous = NORMAL; + PRAGMA journal_mode = WAL; + CREATE TABLE IF NOT EXISTS {} ( + a BLOB PRIMARY KEY, + w BLOB NOT NULL + );", + self.table_name + ); + + self.pool + .conn(move |conn| conn.execute_batch(&table_name)) + .await?; + Ok(()) + } + + /// Clears all bindings from this memory. #[cfg(feature = "test-utils")] pub async fn clear(&self) -> Result<(), SqliteMemoryError> { - let findex_table_name = self.table_name.clone(); + let script = format!("DELETE FROM {}", self.table_name); self.pool - .conn(move |conn| conn.execute(&format!("DELETE FROM {findex_table_name}"), [])) + .conn(move |conn| conn.execute(&script, [])) .await?; Ok(()) } @@ -160,7 +144,7 @@ impl MemoryADT // generate a returned value complying to the batch-read spec. Ok(addresses .iter() - // Copying is necessary here since the same word could be + // Copying is necessary since the same word could be // returned multiple times. .map(|addr| results.get(addr).copied()) .collect()) @@ -229,33 +213,37 @@ mod tests { #[tokio::test] async fn test_rw_seq() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH, TABLE_NAME.to_owned()) + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); + m.initialize().await.unwrap(); test_single_write_and_read(&m, gen_seed()).await } #[tokio::test] async fn test_guard_seq() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH, TABLE_NAME.to_owned()) + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); + m.initialize().await.unwrap(); test_wrong_guard(&m, gen_seed()).await } #[tokio::test] async fn test_collision_seq() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH, TABLE_NAME.to_owned()) + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); + m.initialize().await.unwrap(); test_rw_same_address(&m, gen_seed()).await } #[tokio::test] async fn test_rw_ccr() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::connect(DB_PATH, TABLE_NAME.to_owned()) + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); + m.initialize().await.unwrap(); test_guarded_write_concurrent(&m, gen_seed(), Some(100)).await } } From ba2a3c2797c1cbd1183b1654eb18e68f9f181815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Fri, 13 Jun 2025 14:15:31 +0200 Subject: [PATCH 40/60] enforce uniform naming across memories --- crate/memories/benches/benches.rs | 18 +++++++++--------- crate/memories/examples/redis.rs | 2 +- crate/memories/examples/sqlite.rs | 7 ++++--- crate/memories/src/redis_mem.rs | 24 +++++++++++------------- crate/memories/src/sqlite_mem.rs | 10 +++++----- 5 files changed, 30 insertions(+), 31 deletions(-) diff --git a/crate/memories/benches/benches.rs b/crate/memories/benches/benches.rs index ba17c61f..f596d5b8 100644 --- a/crate/memories/benches/benches.rs +++ b/crate/memories/benches/benches.rs @@ -81,7 +81,7 @@ fn bench_search_multiple_bindings(c: &mut Criterion) { bench_memory_search_multiple_bindings( "Redis", N_PTS, - async || RedisMemory::connect(&get_redis_url()).await.unwrap(), + async || RedisMemory::new_with_url(&get_redis_url()).await.unwrap(), c, &mut rng, ); @@ -91,7 +91,7 @@ fn bench_search_multiple_bindings(c: &mut Criterion) { "SQLite", N_PTS, async || { - let m = SqliteMemory::new( + let m = SqliteMemory::new_with_path( SQLITE_PATH, "bench_memory_search_multiple_bindings".to_string(), ) @@ -130,7 +130,7 @@ fn bench_search_multiple_keywords(c: &mut Criterion) { bench_memory_search_multiple_keywords( "Redis", N_PTS, - async || RedisMemory::connect(&get_redis_url()).await.unwrap(), + async || RedisMemory::new_with_url(&get_redis_url()).await.unwrap(), c, &mut rng, ); @@ -140,7 +140,7 @@ fn bench_search_multiple_keywords(c: &mut Criterion) { "SQLite", N_PTS, async || { - let m = SqliteMemory::new( + let m = SqliteMemory::new_with_path( SQLITE_PATH, "bench_memory_search_multiple_keywords".to_string(), ) @@ -177,7 +177,7 @@ fn bench_insert_multiple_bindings(c: &mut Criterion) { bench_memory_insert_multiple_bindings( "Redis", N_PTS, - async || RedisMemory::connect(&get_redis_url()).await.unwrap(), + async || RedisMemory::new_with_url(&get_redis_url()).await.unwrap(), c, RedisMemory::clear, &mut rng, @@ -188,7 +188,7 @@ fn bench_insert_multiple_bindings(c: &mut Criterion) { "SQLite", N_PTS, async || { - let m = SqliteMemory::new( + let m = SqliteMemory::new_with_path( SQLITE_PATH, "bench_memory_insert_multiple_bindings".to_string(), ) @@ -229,7 +229,7 @@ fn bench_contention(c: &mut Criterion) { bench_memory_contention( "Redis", N_PTS, - async || RedisMemory::connect(&get_redis_url()).await.unwrap(), + async || RedisMemory::new_with_url(&get_redis_url()).await.unwrap(), c, RedisMemory::clear, &mut rng, @@ -240,7 +240,7 @@ fn bench_contention(c: &mut Criterion) { "SQLite", N_PTS, async || { - let m = SqliteMemory::new(SQLITE_PATH, "bench_memory_contention".to_string()) + let m = SqliteMemory::new_with_path(SQLITE_PATH, "bench_memory_contention".to_string()) .await .unwrap(); m.initialize().await.unwrap(); @@ -368,7 +368,7 @@ fn bench_one_to_many(c: &mut Criterion) { N_PTS, async || { DelayedMemory::new( - RedisMemory::connect(&get_redis_url()).await.unwrap(), + RedisMemory::new_with_url(&get_redis_url()).await.unwrap(), *mean, *variance, ) diff --git a/crate/memories/examples/redis.rs b/crate/memories/examples/redis.rs index ee52e122..b7066888 100644 --- a/crate/memories/examples/redis.rs +++ b/crate/memories/examples/redis.rs @@ -28,7 +28,7 @@ async fn main() { let index = gen_index(&mut rng); // This example uses our Redis-based implementation of `MemoryADT`. - let memory = RedisMemory::, [u8; WORD_LENGTH]>::connect(DB_PATH) + let memory = RedisMemory::, [u8; WORD_LENGTH]>::new_with_url(DB_PATH) .await .unwrap(); diff --git a/crate/memories/examples/sqlite.rs b/crate/memories/examples/sqlite.rs index 40d93653..6b6503e2 100644 --- a/crate/memories/examples/sqlite.rs +++ b/crate/memories/examples/sqlite.rs @@ -28,9 +28,10 @@ async fn main() { // Generating the random index. let index = gen_index(&mut rng); - let memory = SqliteMemory::<_, [u8; WORD_LENGTH]>::new(DB_PATH, TABLE_NAME.to_owned()) - .await - .unwrap(); + let memory = + SqliteMemory::<_, [u8; WORD_LENGTH]>::new_with_path(DB_PATH, TABLE_NAME.to_owned()) + .await + .unwrap(); let encrypted_memory = MemoryEncryptionLayer::new(&key, memory); diff --git a/crate/memories/src/redis_mem.rs b/crate/memories/src/redis_mem.rs index a39747d6..8825459b 100644 --- a/crate/memories/src/redis_mem.rs +++ b/crate/memories/src/redis_mem.rs @@ -60,8 +60,8 @@ impl fmt::Debug for RedisMemory { } impl RedisMemory { - /// Connects to a Redis server with a `ConnectionManager`. - pub async fn connect_with_manager( + /// Returns a new instance using this connection manager. + pub async fn new_with_manager( mut manager: ConnectionManager, ) -> Result { let script_hash = redis::cmd("SCRIPT") @@ -77,13 +77,14 @@ impl RedisMemory { }) } - /// Connects to a Redis server using the given URL. - pub async fn connect(url: &str) -> Result { + /// Returns a new instance using the Redis instance at the given URL. + pub async fn new_with_url(url: &str) -> Result { let client = redis::Client::open(url)?; let manager = client.get_connection_manager().await?; - Self::connect_with_manager(manager).await + Self::new_with_manager(manager).await } + /// Clears all bindings from this memory. #[cfg(feature = "test-utils")] pub async fn clear(&self) -> Result<(), RedisMemoryError> { redis::cmd("FLUSHDB") @@ -106,7 +107,6 @@ impl MemoryADT ) -> Result>, Self::Error> { let mut cmd = redis::cmd("MGET"); let cmd = addresses.iter().fold(&mut cmd, |c, a| c.arg(&**a)); - // Cloning the connection manager is cheap since it is an `Arc`. cmd.query_async(&mut self.manager.clone()) .await .map_err(RedisMemoryError::RedisError) @@ -118,6 +118,7 @@ impl MemoryADT bindings: Vec<(Self::Address, Self::Word)>, ) -> Result, Self::Error> { let (guard_address, guard_value) = guard; + let mut cmd = redis::cmd("EVALSHA"); let cmd = cmd .arg(self.script_hash.as_str()) @@ -129,12 +130,9 @@ impl MemoryADT .map(|bytes| bytes.as_slice()) .unwrap_or(b"false".as_slice()), ); - let cmd = bindings .iter() .fold(cmd.arg(bindings.len()), |cmd, (a, w)| cmd.arg(&**a).arg(w)); - - // Cloning the connection manager is cheap since it is an `Arc`. cmd.query_async(&mut self.manager.clone()) .await .map_err(RedisMemoryError::RedisError) @@ -159,25 +157,25 @@ mod tests { #[tokio::test] async fn test_rw_seq() { - let m = RedisMemory::connect(&get_redis_url()).await.unwrap(); + let m = RedisMemory::new_with_url(&get_redis_url()).await.unwrap(); test_single_write_and_read::(&m, gen_seed()).await } #[tokio::test] async fn test_guard_seq() { - let m = RedisMemory::connect(&get_redis_url()).await.unwrap(); + let m = RedisMemory::new_with_url(&get_redis_url()).await.unwrap(); test_wrong_guard::(&m, gen_seed()).await } #[tokio::test] async fn test_collision_seq() { - let m = RedisMemory::connect(&get_redis_url()).await.unwrap(); + let m = RedisMemory::new_with_url(&get_redis_url()).await.unwrap(); test_rw_same_address::(&m, gen_seed()).await } #[tokio::test] async fn test_rw_ccr() { - let m = RedisMemory::connect(&get_redis_url()).await.unwrap(); + let m = RedisMemory::new_with_url(&get_redis_url()).await.unwrap(); test_guarded_write_concurrent::(&m, gen_seed(), None).await } } diff --git a/crate/memories/src/sqlite_mem.rs b/crate/memories/src/sqlite_mem.rs index 2ea5ac3f..1c0c058b 100644 --- a/crate/memories/src/sqlite_mem.rs +++ b/crate/memories/src/sqlite_mem.rs @@ -50,7 +50,7 @@ impl Debug for SqliteMemory { impl SqliteMemory { /// Returns a new memory instance using a pool of connections to the SQLite /// database at the given path. - pub async fn new( + pub async fn new_with_path( path: impl AsRef, table_name: String, ) -> Result { @@ -213,7 +213,7 @@ mod tests { #[tokio::test] async fn test_rw_seq() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new(DB_PATH, TABLE_NAME.to_owned()) + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new_with_path(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); m.initialize().await.unwrap(); @@ -222,7 +222,7 @@ mod tests { #[tokio::test] async fn test_guard_seq() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new(DB_PATH, TABLE_NAME.to_owned()) + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new_with_path(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); m.initialize().await.unwrap(); @@ -231,7 +231,7 @@ mod tests { #[tokio::test] async fn test_collision_seq() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new(DB_PATH, TABLE_NAME.to_owned()) + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new_with_path(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); m.initialize().await.unwrap(); @@ -240,7 +240,7 @@ mod tests { #[tokio::test] async fn test_rw_ccr() { - let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new(DB_PATH, TABLE_NAME.to_owned()) + let m = SqliteMemory::<_, [u8; WORD_LENGTH]>::new_with_path(DB_PATH, TABLE_NAME.to_owned()) .await .unwrap(); m.initialize().await.unwrap(); From da1891d5e8910ffb22de8baef5bd60ed996864a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Fri, 13 Jun 2025 14:21:03 +0200 Subject: [PATCH 41/60] remove redundant dependency specification --- crate/memories/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crate/memories/Cargo.toml b/crate/memories/Cargo.toml index d1ba1a6e..a3333c6e 100644 --- a/crate/memories/Cargo.toml +++ b/crate/memories/Cargo.toml @@ -18,7 +18,7 @@ path = "src/lib.rs" redis-mem = ["redis"] sqlite-mem = ["async-sqlite"] postgres-mem = ["deadpool-postgres", "tokio", "tokio-postgres"] -test-utils = ["cosmian_findex/test-utils", "redis/tokio-comp"] +test-utils = ["cosmian_findex/test-utils"] [dependencies] async-sqlite = { version = "0.5", optional = true } From e1e6bc747b0a4009571237337606221eb246ebe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Fri, 13 Jun 2025 14:25:28 +0200 Subject: [PATCH 42/60] remove unnecessary type hint --- crate/memories/src/postgresql_mem/memory.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crate/memories/src/postgresql_mem/memory.rs b/crate/memories/src/postgresql_mem/memory.rs index 59b0023c..a22a366d 100644 --- a/crate/memories/src/postgresql_mem/memory.rs +++ b/crate/memories/src/postgresql_mem/memory.rs @@ -129,7 +129,7 @@ impl MemoryADT .transpose() .map_err(PostgresMemoryError::TryFromSliceError) }) - .collect::>() + .collect() } async fn guarded_write( From 3e27338efcda7aec1307c2000fc6261df5e70234 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Thu, 26 Jun 2025 16:31:44 +0200 Subject: [PATCH 43/60] fix: ci --- .github/workflows/benches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml index 8762c04a..807555f5 100644 --- a/.github/workflows/benches.yml +++ b/.github/workflows/benches.yml @@ -9,5 +9,5 @@ jobs: uses: Cosmian/reusable_workflows/.github/workflows/cargo-bench.yml@develop with: toolchain: 1.87.0 - features: test-utils,redis-mem,sqlite-mem,rust-mem,postgres-mem + features: test-utils,redis-mem,sqlite-mem,postgres-mem force: true From 61a805d8c5cfe3f00bb315352eb01ad7325224ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Thu, 26 Jun 2025 14:37:33 +0000 Subject: [PATCH 44/60] Update crate/memories/src/sqlite_mem.rs Co-authored-by: Hatem M'naouer <19950216+HatemMn@users.noreply.github.com> --- crate/memories/src/sqlite_mem.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crate/memories/src/sqlite_mem.rs b/crate/memories/src/sqlite_mem.rs index 1c0c058b..36eebfff 100644 --- a/crate/memories/src/sqlite_mem.rs +++ b/crate/memories/src/sqlite_mem.rs @@ -82,7 +82,7 @@ impl SqliteMemory { // - synchronous = NORMAL: Reduces disk I/O by only calling fsync() at critical // moments rather than after every transaction (FULL mode); this does not // compromise data integrity. - let table_name = format!( + let initialization_script = format!( "PRAGMA synchronous = NORMAL; PRAGMA journal_mode = WAL; CREATE TABLE IF NOT EXISTS {} ( @@ -93,7 +93,7 @@ impl SqliteMemory { ); self.pool - .conn(move |conn| conn.execute_batch(&table_name)) + .conn(move |conn| conn.execute_batch(&initialization_script)) .await?; Ok(()) } From fd47c7c886278baba51a4f1ad1c85a83f4e30552 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20BR=C3=89ZOT?= Date: Thu, 26 Jun 2025 17:22:23 +0200 Subject: [PATCH 45/60] fix rebase --- crate/memories/Cargo.toml | 20 ++++++++++++-------- crate/memories/src/sqlite_mem.rs | 11 ++++++----- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/crate/memories/Cargo.toml b/crate/memories/Cargo.toml index a3333c6e..0abdb5aa 100644 --- a/crate/memories/Cargo.toml +++ b/crate/memories/Cargo.toml @@ -21,23 +21,27 @@ postgres-mem = ["deadpool-postgres", "tokio", "tokio-postgres"] test-utils = ["cosmian_findex/test-utils"] [dependencies] +cosmian_findex = { path = "../findex" } + +# Needed for SQLite memory. async-sqlite = { version = "0.5", optional = true } -cosmian_findex = { path = "../findex", version = "8.0" } + +# Needed for PostgreSQL memory. deadpool-postgres = { version = "0.14", optional = true } -redis = { version = "0.32", default-features = false, features = [ - "connection-manager", - "tokio-comp", -], optional = true } tokio = { workspace = true, features = ["rt-multi-thread"], optional = true } tokio-postgres = { version = "0.7", optional = true, features = [ "array-impls", ] } +# Needed for Redis memory. +redis = { version = "0.32", default-features = false, features = [ + "connection-manager", + "tokio-comp" +], optional = true } + [dev-dependencies] cosmian_crypto_core.workspace = true -cosmian_findex = { path = "../findex", version = "8.0", features = [ - "test-utils", -] } +cosmian_findex = { path = "../findex", features = ["test-utils"] } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } criterion = { workspace = true } futures = { version = "0.3" } diff --git a/crate/memories/src/sqlite_mem.rs b/crate/memories/src/sqlite_mem.rs index 36eebfff..2057d6dd 100644 --- a/crate/memories/src/sqlite_mem.rs +++ b/crate/memories/src/sqlite_mem.rs @@ -75,13 +75,14 @@ impl SqliteMemory { } } - /// Makes sure the correct table exist in the associated database. + /// Creates the correct table in the associated database if it does not exist. pub async fn initialize(&self) -> Result<(), SqliteMemoryError> { // The following settings are used to improve performance: - // - journal_mode = WAL : WAL journaling is faster than the default DELETE mode. - // - synchronous = NORMAL: Reduces disk I/O by only calling fsync() at critical - // moments rather than after every transaction (FULL mode); this does not - // compromise data integrity. + // - journal_mode = WAL : WAL journaling is faster than the default + // DELETE mode. + // - synchronous = NORMAL: Reduces disk I/O by only calling fsync() at + // critical moments rather than after every transaction (FULL mode); + // this does not compromise data integrity. let initialization_script = format!( "PRAGMA synchronous = NORMAL; PRAGMA journal_mode = WAL; From ceafd4542499a7097b7213d1ecd280188666e174 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Fri, 27 Jun 2025 16:58:46 +0200 Subject: [PATCH 46/60] fix: review fixes fix: review fixes part 2 --- README.md | 13 +--- crate/findex/Cargo.toml | 2 +- crate/findex/README.md | 23 ++----- crate/findex/src/test_utils/benches.rs | 19 +---- crate/memories/Cargo.toml | 17 +---- crate/memories/README.md | 33 +++++---- crate/memories/benches/benches.rs | 57 ++++++--------- .../examples/{postgresql.rs => example.rs} | 69 +++++++++---------- crate/memories/examples/redis.rs | 65 ----------------- crate/memories/examples/shared_utils.rs | 1 - crate/memories/examples/sqlite.rs | 66 ------------------ crate/memories/src/postgresql_mem/memory.rs | 11 --- 12 files changed, 86 insertions(+), 290 deletions(-) rename crate/memories/examples/{postgresql.rs => example.rs} (61%) delete mode 100644 crate/memories/examples/redis.rs delete mode 100644 crate/memories/examples/sqlite.rs diff --git a/README.md b/README.md index 4929d3a6..a67ac6e2 100644 --- a/README.md +++ b/README.md @@ -11,18 +11,7 @@ Findex is a Symmetric Searchable Encryption (SSE) library that enables encrypted This repository is organized as a Rust workspace with two crates: - `cosmian_findex`: Core library implementing the SSE algorithms -- `cosmian_findex_memories`: Storage backend implementations for different databases - -## Installation - -Add `cosmian_findex` to your Cargo.toml: - -```toml -[dependencies] -cosmian_findex = "8.0.0" -# Optional - include backend implementations -cosmian_findex_memories = { version = "8.0.0", features = ["redis-mem", "sqlite-mem", "postgres-mem"] } -``` +- `cosmian_findex_memories`: Storage back-end implementations for different databases ## Related Projects diff --git a/crate/findex/Cargo.toml b/crate/findex/Cargo.toml index 71a79b2e..8bf97a0f 100644 --- a/crate/findex/Cargo.toml +++ b/crate/findex/Cargo.toml @@ -22,7 +22,7 @@ cosmian_crypto_core.workspace = true aes = "0.8" xts-mode = "0.5" -# Optional dependencies for testing and benchmarking +# Optional dependencies for testing and benchmarking. tokio = { workspace = true, features = [ "rt-multi-thread", "macros", diff --git a/crate/findex/README.md b/crate/findex/README.md index 051ef356..0bd5ec05 100644 --- a/crate/findex/README.md +++ b/crate/findex/README.md @@ -1,23 +1,14 @@ # Findex -To build Findex simply run: +This crate provides the core functionality of Findex, defining the abstract data types, cryptographic operations, and encoding algorithms. -```bash -cargo build --release -``` - -To test, run: - -```bash -cargo test --release --all-features -``` +## Setup -To launch the benchmarks, run: +Add `cosmian_findex` as dependency to your project : -```bash -cargo bench --all-features +```toml +[dependencies] +cosmian_findex = "8.0.0" ``` -Note that benches are quite involving and require _several hours_ for a full -run. Once all benchmarks are run, you will find detailed reports under `target/criterion`. -findex +An usage example is available in the [examples folder](./examples). diff --git a/crate/findex/src/test_utils/benches.rs b/crate/findex/src/test_utils/benches.rs index c30351cf..7c039077 100644 --- a/crate/findex/src/test_utils/benches.rs +++ b/crate/findex/src/test_utils/benches.rs @@ -1,6 +1,6 @@ //! This module provides a comprehensive benchmarking suite for testing the -//! performance of Findex memory implementations. These benchmarks are designed -//! to be generic and work with any memory backend that implements the MemoryADT +//! performance of Findex memory implementations. These benchmarks are designed +//! to be generic and work with any memory back end that implements the MemoryADT //! trait. use crate::{ @@ -61,9 +61,6 @@ pub fn bench_memory_search_multiple_bindings< c: &mut Criterion, rng: &mut impl CryptoRngCore, ) { - // Redis memory backend requires a tokio runtime, and all operations to - // happen in the same runtime, otherwise the connection returns a broken - // pipe error. let rt = Runtime::new().unwrap(); let findex = Findex::new( @@ -102,9 +99,6 @@ pub fn bench_memory_search_multiple_keywords< c: &mut Criterion, rng: &mut impl CryptoRngCore, ) { - // Redis memory backend requires a tokio runtime, and all operations to - // happen in the same runtime, otherwise the connection returns a broken - // pipe error. let rt = Runtime::new().unwrap(); let findex = Arc::new(Findex::new( @@ -171,9 +165,6 @@ pub fn bench_memory_insert_multiple_bindings< clear: impl AsyncFn(&Memory) -> Result<(), E>, rng: &mut impl CryptoRngCore, ) { - // Redis memory backend requires a tokio runtime, and all operations to - // happen in the same runtime, otherwise the connection returns a broken - // pipe error. let rt = Runtime::new().unwrap(); let mut m = rt.block_on(m()); @@ -226,9 +217,6 @@ pub fn bench_memory_contention< ) { const N_CLIENTS: usize = 10; - // Redis memory backend requires a tokio runtime, and all operations to - // happen in the same runtime, otherwise the connection returns a broken - // pipe error. let rt = Builder::new_multi_thread().enable_all().build().unwrap(); let m = rt.block_on(m()); @@ -336,9 +324,6 @@ pub fn bench_memory_one_to_many< ) { const MAX_VAL: usize = 100; - // Redis memory backend requires a tokio runtime, and all operations to - // happen in the same runtime, otherwise the connection returns a broken - // pipe error. let rt = Builder::new_multi_thread().enable_all().build().unwrap(); let m = rt.block_on(m()); diff --git a/crate/memories/Cargo.toml b/crate/memories/Cargo.toml index 6b2248f1..a7dbf7a6 100644 --- a/crate/memories/Cargo.toml +++ b/crate/memories/Cargo.toml @@ -33,11 +33,6 @@ tokio-postgres = { version = "0.7", optional = true, features = [ "array-impls", ] } -# Needed for Redis memory. -redis = { version = "0.32", default-features = false, features = [ - "connection-manager", - "tokio-comp" -], optional = true } [dev-dependencies] cosmian_crypto_core.workspace = true @@ -56,13 +51,5 @@ harness = false required-features = ["redis-mem", "sqlite-mem", "postgres-mem", "test-utils"] [[example]] -name = "redis" -required-features = ["redis-mem"] - -[[example]] -name = "sqlite" -required-features = ["sqlite-mem"] - -[[example]] -name = "postgresql" -required-features = ["postgres-mem"] +name = "example" +required-features = ["redis-mem", "sqlite-mem", "postgres-mem"] diff --git a/crate/memories/README.md b/crate/memories/README.md index d00b9328..a35af3bf 100644 --- a/crate/memories/README.md +++ b/crate/memories/README.md @@ -1,29 +1,36 @@ # Findex Memories A collection of memory implementations for Findex, a concurrent and database-agnostic Searchable Encryption scheme. +Findex memories provide compatibility with concrete databases, allowing the core Findex library to remain database-agnostic. This separation enables users to integrate Findex with their preferred database system. -## Overview +## Setup -Findex Memories provides "pluggable" storage backends for Findex, allowing the core Findex library to remain database-agnostic while supporting various storage systems. This separation enables users to integrate findex to their prefered database sytem. +First, add `cosmian_findex_memories` as dependency to your project : + +```bash +cargo add cosmian_findex_memories # do not forget to enable the adequate feature for the back end you want to use ! +``` + +If you don't have a running `Redis` or `Postgres` instance running, you can use the [`docker-compose.yml`](./docker-compose.yml) file provided with this repository by running `docker-compose up`. + +For detailed usage examples, refer to the [examples folder](examples). -## Available Storage Backends +## Available Storage Back-ends This library provides implementations for the following storage systems: | Feature | Database | Dependencies | | -------------- | ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `redis-mem` | Redis | [redis](https://crates.io/crates/redis) v0.31 | -| `sqlite-mem` | SQLite | [async-sqlite](https://crates.io/crates/async-sqlite) v0.4 \* | +| `sqlite-mem` | SQLite | [async-sqlite](https://crates.io/crates/async-sqlite) v0.5 | | `postgres-mem` | PostgreSQL | [tokio-postgres](https://crates.io/crates/tokio-postgres) v0.7.9
[tokio](https://crates.io/crates/tokio) v1.44
[deadpool-postgres](https://crates.io/crates/deadpool-postgres) v0.14.1 | -## Usage +To execute the PostgreSQL tests and run the benches locally with your postgres installation, the easiest way would be to add the following service to your pg_service.conf file (usually under `~/.pg_service.conf`): -First, add `cosmian_findex_memories` as dependency to your project : - -```bash -cargo add cosmian_findex_memories # do not forget to enable the adequate feature for the backend you want to use ! +```toml +[cosmian_service] +host=localhost +dbname=cosmian +user=cosmian +password=cosmian ``` - -If you don't have a running `Redis` or `Postgres` instance running, you can use the provided [docker-compose.yml](./docker-compose.yml) file provided with this repository by running `docker-compose up`. - -For detailed usage examples, refer to the [examples folder](examples). diff --git a/crate/memories/benches/benches.rs b/crate/memories/benches/benches.rs index f596d5b8..2b39845f 100644 --- a/crate/memories/benches/benches.rs +++ b/crate/memories/benches/benches.rs @@ -17,6 +17,9 @@ use cosmian_findex_memories::SqliteMemory; #[cfg(feature = "sqlite-mem")] const SQLITE_PATH: &str = "benches.sqlite.db"; +// Redis memory back-end requires a tokio runtime, and all operations to +// happen in the same runtime, otherwise the connection returns a broken +// pipe error. #[cfg(feature = "redis-mem")] use cosmian_findex_memories::RedisMemory; @@ -91,12 +94,9 @@ fn bench_search_multiple_bindings(c: &mut Criterion) { "SQLite", N_PTS, async || { - let m = SqliteMemory::new_with_path( - SQLITE_PATH, - "bench_memory_search_multiple_bindings".to_string(), - ) - .await - .unwrap(); + let m = SqliteMemory::new_with_path(SQLITE_PATH, "bench_memory_smd".to_string()) + .await + .unwrap(); m.initialize().await.unwrap(); m }, @@ -109,12 +109,9 @@ fn bench_search_multiple_bindings(c: &mut Criterion) { "Postgres", N_PTS, async || { - let m = connect_and_init_table( - get_postgresql_url(), - "bench_memory_search_multiple_bindings".to_string(), - ) - .await - .unwrap(); + let m = connect_and_init_table(get_postgresql_url(), "bench_memory_smd".to_string()) + .await + .unwrap(); m.initialize().await.unwrap(); m }, @@ -140,12 +137,9 @@ fn bench_search_multiple_keywords(c: &mut Criterion) { "SQLite", N_PTS, async || { - let m = SqliteMemory::new_with_path( - SQLITE_PATH, - "bench_memory_search_multiple_keywords".to_string(), - ) - .await - .unwrap(); + let m = SqliteMemory::new_with_path(SQLITE_PATH, "bench_memory_smk".to_string()) + .await + .unwrap(); m.initialize().await.unwrap(); m }, @@ -158,12 +152,9 @@ fn bench_search_multiple_keywords(c: &mut Criterion) { "Postgres", N_PTS, async || { - connect_and_init_table( - get_postgresql_url(), - "bench_memory_search_multiple_keywords".to_string(), - ) - .await - .unwrap() + connect_and_init_table(get_postgresql_url(), "bench_memory_smk".to_string()) + .await + .unwrap() }, c, &mut rng, @@ -188,12 +179,9 @@ fn bench_insert_multiple_bindings(c: &mut Criterion) { "SQLite", N_PTS, async || { - let m = SqliteMemory::new_with_path( - SQLITE_PATH, - "bench_memory_insert_multiple_bindings".to_string(), - ) - .await - .unwrap(); + let m = SqliteMemory::new_with_path(SQLITE_PATH, "bench_memory_imd".to_string()) + .await + .unwrap(); m.initialize().await.unwrap(); m }, @@ -207,12 +195,9 @@ fn bench_insert_multiple_bindings(c: &mut Criterion) { "Postgres", N_PTS, async || { - connect_and_init_table( - get_postgresql_url(), - "bench_memory_insert_multiple_bindings".to_string(), - ) - .await - .unwrap() + connect_and_init_table(get_postgresql_url(), "bench_memory_imd".to_string()) + .await + .unwrap() }, c, async |m: &PostgresMemory<_, _>| -> Result<(), String> { diff --git a/crate/memories/examples/postgresql.rs b/crate/memories/examples/example.rs similarity index 61% rename from crate/memories/examples/postgresql.rs rename to crate/memories/examples/example.rs index 5160a162..e3174bd6 100644 --- a/crate/memories/examples/postgresql.rs +++ b/crate/memories/examples/example.rs @@ -1,23 +1,22 @@ -//! This example show-cases the use of Findex to securely store a hash-map with -//! PostgreSQL. +//! This example show-cases the use of Findex to securely store a hash-map with different back ends. #[path = "shared_utils.rs"] mod shared_utils; use cosmian_crypto_core::{CsRng, Secret, reexport::rand_core::SeedableRng}; use cosmian_findex_memories::{ - PostgresMemory, PostgresMemoryError, - reexport::{ - cosmian_findex::{ADDRESS_LENGTH, Address, Findex, IndexADT, MemoryEncryptionLayer}, - deadpool_postgres::{Config, Pool}, - tokio_postgres::NoTls, - }, + PostgresMemory, PostgresMemoryError, RedisMemory, SqliteMemory, + reexport::cosmian_findex::{ADDRESS_LENGTH, Address, Findex, IndexADT, MemoryEncryptionLayer}, }; +use deadpool_postgres::{Config, Pool}; use futures::executor::block_on; use shared_utils::{WORD_LENGTH, decoder, encoder, gen_index}; use std::collections::HashMap; +use tokio_postgres::NoTls; const DB_URL: &str = "postgres://cosmian:cosmian@localhost/cosmian"; -const TABLE_NAME: &str = "findex_example"; +const DB_PATH: &str = "redis://localhost:6379"; +const DB_PATH2: &str = "./target/debug/sqlite-test.db"; +const TABLE_NAME: &str = "findex_memory"; async fn create_pool(db_url: &str) -> Result { let mut pg_config = Config::new(); @@ -37,31 +36,35 @@ async fn main() { // key in order to make the index inter-operable. let key = Secret::random(&mut rng); - // Generating a random index. + // Generating the random index. let index = gen_index(&mut rng); - // Addd the following service to your pg_service.conf file (usually under - // `~/.pg_service.conf`): - // - // [cosmian_service] - // host=localhost - // dbname=cosmian - // user=cosmian - // password=cosmian - let pool = create_pool(DB_URL).await.unwrap(); - let m = PostgresMemory::, [u8; WORD_LENGTH]>::new_with_pool( - pool.clone(), - TABLE_NAME.to_string(), + // This example uses our Redis-based implementation of `MemoryADT`. + let redis_memory = + RedisMemory::, [u8; WORD_LENGTH]>::new_with_url(DB_PATH) + .await + .unwrap(); + + // You can also use our Sqlite-based implementation of `MemoryADT`. + let _sqlite_memory = SqliteMemory::, [u8; WORD_LENGTH]>::new_with_path( + DB_PATH2, + TABLE_NAME.to_owned(), ) - .await; + .await + .unwrap(); - // Notice we chose to not enable TLS: it's not needed for this example as we - // are using the encryption layer in top of the memory interface - i.e. the - // data is already encrypted before being sent to the database and TLS would - // add unnecessary overhead. - m.initialize().await.unwrap(); + // Or else, the Postgres-based implementation of `MemoryADT`. Refer to README.md for details on how to setup + // the database to use this example. + let pool = create_pool(DB_URL).await.unwrap(); + let _postgres_memory = + PostgresMemory::, [u8; WORD_LENGTH]>::new_with_pool( + pool.clone(), + TABLE_NAME.to_string(), + ) + .await; - let encrypted_memory = MemoryEncryptionLayer::new(&key, m); + // Adding an encryption layer to the chosen memory + let encrypted_memory = MemoryEncryptionLayer::new(&key, redis_memory); // Instantiating Findex requires passing the key, the memory used and the // encoder and decoder. Quite simple, after all :) @@ -90,13 +93,5 @@ async fn main() { // ... and verify we get the whole index back! assert_eq!(res, index); - // Drop the table to avoid problems with subsequent runs. - pool.get() - .await - .unwrap() - .execute(&format!("DROP table {};", TABLE_NAME), &[]) - .await - .unwrap(); - println!("All good !"); } diff --git a/crate/memories/examples/redis.rs b/crate/memories/examples/redis.rs deleted file mode 100644 index b7066888..00000000 --- a/crate/memories/examples/redis.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! This example show-cases the use of Findex to securely store a hash-map with redis. -#[path = "shared_utils.rs"] -mod shared_utils; - -use cosmian_crypto_core::{CsRng, Secret, reexport::rand_core::SeedableRng}; -use cosmian_findex_memories::{ - RedisMemory, - reexport::cosmian_findex::{ADDRESS_LENGTH, Address, Findex, IndexADT, MemoryEncryptionLayer}, -}; -use futures::executor::block_on; -use shared_utils::{WORD_LENGTH, decoder, encoder, gen_index}; -use std::collections::HashMap; - -const DB_PATH: &str = "redis://localhost:6379"; - -#[tokio::main] -async fn main() { - // For cryptographic applications, it is important to use a secure RNG. In - // Rust, those RNG implement the `CryptoRng` trait. - let mut rng = CsRng::from_entropy(); - - // Generate fresh Findex key. In practice only one user is in charge of - // generating the key (the administrator?): all users *must* share the same - // key in order to make the index inter-operable. - let key = Secret::random(&mut rng); - - // Generating the random index. - let index = gen_index(&mut rng); - - // This example uses our Redis-based implementation of `MemoryADT`. - let memory = RedisMemory::, [u8; WORD_LENGTH]>::new_with_url(DB_PATH) - .await - .unwrap(); - - let encrypted_memory = MemoryEncryptionLayer::new(&key, memory); - - // Instantiating Findex requires passing the key, the memory used and the - // encoder and decoder. Quite simple, after all :) - let findex = Findex::< - WORD_LENGTH, // size of a word - u64, // type of a value - String, // type of an encoding error - _, // type of the memory - >::new(encrypted_memory, encoder, decoder); - - // Here we insert all bindings one by one, blocking on each call. A better - // way would be to performed all such calls in parallel using tasks. - index - .clone() - .into_iter() - .for_each(|(kw, vs)| block_on(findex.insert(kw, vs)).expect("insert failed")); - - // In order to verify insertion was correctly performed, we search for all - // the indexed keywords... - let res = index - .keys() - .cloned() - .map(|kw| (kw, block_on(findex.search(&kw)).expect("search failed"))) - .collect::>(); - - // ... and verify we get the whole index back! - assert_eq!(res, index); - - println!("All good !"); -} diff --git a/crate/memories/examples/shared_utils.rs b/crate/memories/examples/shared_utils.rs index f2d66e37..af828b8a 100644 --- a/crate/memories/examples/shared_utils.rs +++ b/crate/memories/examples/shared_utils.rs @@ -1,4 +1,3 @@ -//! This example show-cases the use of Findex to securely store a hash-map. use cosmian_crypto_core::reexport::rand_core::CryptoRngCore; use cosmian_findex::Op; use std::collections::{HashMap, HashSet}; diff --git a/crate/memories/examples/sqlite.rs b/crate/memories/examples/sqlite.rs deleted file mode 100644 index 6b6503e2..00000000 --- a/crate/memories/examples/sqlite.rs +++ /dev/null @@ -1,66 +0,0 @@ -//! This example show-cases the use of Findex to securely store a hash-map with sqlite. -#[path = "shared_utils.rs"] -mod shared_utils; - -use cosmian_crypto_core::{CsRng, Secret, reexport::rand_core::SeedableRng}; -use cosmian_findex_memories::{ - SqliteMemory, - reexport::cosmian_findex::{Findex, IndexADT, MemoryEncryptionLayer}, -}; -use futures::executor::block_on; -use shared_utils::{WORD_LENGTH, decoder, encoder, gen_index}; -use std::collections::HashMap; - -const DB_PATH: &str = "./target/debug/sqlite-test.db"; -const TABLE_NAME: &str = "findex_memory"; - -#[tokio::main] -async fn main() { - // For cryptographic applications, it is important to use a secure RNG. In - // Rust, those RNG implement the `CryptoRng` trait. - let mut rng = CsRng::from_entropy(); - - // Generate fresh Findex key. In practice only one user is in charge of - // generating the key (the administrator?): all users *must* share the same - // key in order to make the index inter-operable. - let key = Secret::random(&mut rng); - - // Generating the random index. - let index = gen_index(&mut rng); - - let memory = - SqliteMemory::<_, [u8; WORD_LENGTH]>::new_with_path(DB_PATH, TABLE_NAME.to_owned()) - .await - .unwrap(); - - let encrypted_memory = MemoryEncryptionLayer::new(&key, memory); - - // Instantiating Findex requires passing the key, the memory used and the - // encoder and decoder. Quite simple, after all :) - let findex = Findex::< - WORD_LENGTH, // size of a word - u64, // type of a value - String, // type of an encoding error - _, // type of the memory - >::new(encrypted_memory, encoder, decoder); - - // Here we insert all bindings one by one, blocking on each call. A better - // way would be to performed all such calls in parallel using tasks. - index - .clone() - .into_iter() - .for_each(|(kw, vs)| block_on(findex.insert(kw, vs)).expect("insert failed")); - - // In order to verify insertion was correctly performed, we search for all - // the indexed keywords... - let res = index - .keys() - .cloned() - .map(|kw| (kw, block_on(findex.search(&kw)).expect("search failed"))) - .collect::>(); - - // ... and verify we get the whole index back! - assert_eq!(res, index); - - println!("All good !"); -} diff --git a/crate/memories/src/postgresql_mem/memory.rs b/crate/memories/src/postgresql_mem/memory.rs index a22a366d..9d60d3f0 100644 --- a/crate/memories/src/postgresql_mem/memory.rs +++ b/crate/memories/src/postgresql_mem/memory.rs @@ -203,17 +203,6 @@ impl MemoryADT #[cfg(test)] mod tests { - //! To run the postgresql tests locally, add the following service to your - //! pg_service.conf file (usually under `~/.pg_service.conf`): - //! - //! ``` - //! [cosmian_service] - //! host=localhost - //! dbname=cosmian - //! user=cosmian - //! password=cosmian - //! ``` - use super::*; use cosmian_findex::{ ADDRESS_LENGTH, WORD_LENGTH, gen_seed, test_guarded_write_concurrent, test_rw_same_address, From 6ae61c16476543c4f3ba0e3de86e1c9a91cabc3a Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Mon, 30 Jun 2025 12:31:31 +0200 Subject: [PATCH 47/60] fix: benches --- crate/memories/README.md | 6 ++++++ crate/memories/benches/benches.rs | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/crate/memories/README.md b/crate/memories/README.md index a35af3bf..7ba43630 100644 --- a/crate/memories/README.md +++ b/crate/memories/README.md @@ -34,3 +34,9 @@ dbname=cosmian user=cosmian password=cosmian ``` + +You also need to set the adequate environment variables for the benches : + +```shell +export POSTGRES_HOST=localhost REDIS_HOST=localhost && cargo bench --all-features +``` diff --git a/crate/memories/benches/benches.rs b/crate/memories/benches/benches.rs index 2b39845f..4afd67a1 100644 --- a/crate/memories/benches/benches.rs +++ b/crate/memories/benches/benches.rs @@ -26,7 +26,12 @@ use cosmian_findex_memories::RedisMemory; #[cfg(feature = "redis-mem")] fn get_redis_url() -> String { std::env::var("REDIS_HOST").map_or_else( - |_| "redis://localhost:6379".to_owned(), + |_| { + eprintln!( + "⚠️ WARNING: REDIS_HOST environment variable is not set, using default value, set it to localhost to run the benchmarks locally." + ); + "redis://redis:6379".to_owned() + }, |var_env| format!("redis://{var_env}:6379"), ) } @@ -34,18 +39,13 @@ fn get_redis_url() -> String { #[cfg(feature = "postgres-mem")] use cosmian_findex_memories::{PostgresMemory, PostgresMemoryError}; -// To run the postgresql benchmarks locally, add the following service to your pg_service.conf file -// (usually under ~/.pg_service.conf): -// -// [cosmian_service] -// host=localhost -// dbname=cosmian -// user=cosmian -// password=cosmian #[cfg(feature = "postgres-mem")] fn get_postgresql_url() -> String { std::env::var("POSTGRES_HOST").map_or_else( - |_| "postgres://cosmian:cosmian@localhost/cosmian".to_string(), + |_| { + eprintln!("⚠️ WARNING: POSTGRES_HOST environment variable is not set, set it to localhost to run the benchmarks locally."); + "postgres://cosmian:cosmian@postgres/cosmian".to_string() + }, |var_env| format!("postgres://cosmian:cosmian@{var_env}/cosmian"), ) } From 89a04d44537d7fa7401b8d01d460c0ab54f557c4 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Mon, 30 Jun 2025 14:00:30 +0200 Subject: [PATCH 48/60] fix: benches V2 --- .github/workflows/benches.yml | 4 ++++ crate/memories/README.md | 6 ------ crate/memories/benches/benches.rs | 12 ++---------- 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml index 807555f5..1aed26e2 100644 --- a/.github/workflows/benches.yml +++ b/.github/workflows/benches.yml @@ -4,6 +4,10 @@ run-name: Benchmark on ${{ github.ref_name }} - ${{ github.sha }} on: workflow_dispatch +env: + REDIS_HOST: "redis" + POSTGRES_HOST: "postgres" + jobs: bench: uses: Cosmian/reusable_workflows/.github/workflows/cargo-bench.yml@develop diff --git a/crate/memories/README.md b/crate/memories/README.md index 7ba43630..a35af3bf 100644 --- a/crate/memories/README.md +++ b/crate/memories/README.md @@ -34,9 +34,3 @@ dbname=cosmian user=cosmian password=cosmian ``` - -You also need to set the adequate environment variables for the benches : - -```shell -export POSTGRES_HOST=localhost REDIS_HOST=localhost && cargo bench --all-features -``` diff --git a/crate/memories/benches/benches.rs b/crate/memories/benches/benches.rs index 4afd67a1..fe1f15d6 100644 --- a/crate/memories/benches/benches.rs +++ b/crate/memories/benches/benches.rs @@ -26,12 +26,7 @@ use cosmian_findex_memories::RedisMemory; #[cfg(feature = "redis-mem")] fn get_redis_url() -> String { std::env::var("REDIS_HOST").map_or_else( - |_| { - eprintln!( - "⚠️ WARNING: REDIS_HOST environment variable is not set, using default value, set it to localhost to run the benchmarks locally." - ); - "redis://redis:6379".to_owned() - }, + |_| "redis://localhost:6379".to_owned(), |var_env| format!("redis://{var_env}:6379"), ) } @@ -42,10 +37,7 @@ use cosmian_findex_memories::{PostgresMemory, PostgresMemoryError}; #[cfg(feature = "postgres-mem")] fn get_postgresql_url() -> String { std::env::var("POSTGRES_HOST").map_or_else( - |_| { - eprintln!("⚠️ WARNING: POSTGRES_HOST environment variable is not set, set it to localhost to run the benchmarks locally."); - "postgres://cosmian:cosmian@postgres/cosmian".to_string() - }, + |_| "postgres://cosmian:cosmian@localhost/cosmian".to_string(), |var_env| format!("postgres://cosmian:cosmian@{var_env}/cosmian"), ) } From dc9d36f760b5e7e11b225f26243ce353d1485a42 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Mon, 30 Jun 2025 15:08:42 +0200 Subject: [PATCH 49/60] fix: benches final fix --- .github/workflows/benches.yml | 4 +- crate/findex/Cargo.toml | 5 -- crate/findex/benches/benches.rs | 82 ------------------------ crate/findex/benches/data/concurrent.dat | 8 --- crate/findex/benches/data/insert.dat | 16 ----- crate/findex/benches/data/search.dat | 16 ----- crate/findex/benches/make_figures.tex | 76 ---------------------- 7 files changed, 3 insertions(+), 204 deletions(-) delete mode 100644 crate/findex/benches/benches.rs delete mode 100644 crate/findex/benches/data/concurrent.dat delete mode 100644 crate/findex/benches/data/insert.dat delete mode 100644 crate/findex/benches/data/search.dat delete mode 100644 crate/findex/benches/make_figures.tex diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml index 1aed26e2..ba603acb 100644 --- a/.github/workflows/benches.yml +++ b/.github/workflows/benches.yml @@ -10,8 +10,10 @@ env: jobs: bench: - uses: Cosmian/reusable_workflows/.github/workflows/cargo-bench.yml@develop + uses: Cosmian/reusable_workflows/.github/workflows/cargo-bench.yml@fix/add_hosts_arguments with: toolchain: 1.87.0 features: test-utils,redis-mem,sqlite-mem,postgres-mem force: true + redis-host: "redis" + postgres-host: "postgres" diff --git a/crate/findex/Cargo.toml b/crate/findex/Cargo.toml index 8bf97a0f..4c668027 100644 --- a/crate/findex/Cargo.toml +++ b/crate/findex/Cargo.toml @@ -32,11 +32,6 @@ criterion = { workspace = true, optional = true } [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } -[[bench]] -name = "benches" -harness = false -required-features = ["test-utils"] - [[example]] name = "insert" required-features = ["test-utils"] diff --git a/crate/findex/benches/benches.rs b/crate/findex/benches/benches.rs deleted file mode 100644 index 6ac23f71..00000000 --- a/crate/findex/benches/benches.rs +++ /dev/null @@ -1,82 +0,0 @@ -// It is too much trouble to deactivate everything if none of the features are -// activated. -#![allow(unused_imports, unused_variables, unused_mut, dead_code)] - -use cosmian_crypto_core::{CsRng, reexport::rand_core::SeedableRng}; -use cosmian_findex::{ADDRESS_LENGTH, Address, InMemory, WORD_LENGTH}; -use cosmian_findex::{ - bench_memory_contention, bench_memory_insert_multiple_bindings, bench_memory_one_to_many, - bench_memory_search_multiple_bindings, bench_memory_search_multiple_keywords, -}; -use criterion::{Criterion, criterion_group, criterion_main}; - -// Number of points in each graph. -const N_PTS: usize = 9; - -fn bench_search_multiple_bindings(c: &mut Criterion) { - let mut rng = CsRng::from_entropy(); - - bench_memory_search_multiple_bindings( - "in-memory", - N_PTS, - async || InMemory::default(), - c, - &mut rng, - ); -} - -fn bench_search_multiple_keywords(c: &mut Criterion) { - let mut rng = CsRng::from_entropy(); - - bench_memory_search_multiple_keywords( - "in-memory", - N_PTS, - async || InMemory::default(), - c, - &mut rng, - ); -} - -fn bench_insert_multiple_bindings(c: &mut Criterion) { - let mut rng = CsRng::from_entropy(); - - bench_memory_insert_multiple_bindings( - "in-memory", - N_PTS, - async || InMemory::default(), - c, - async |m: &InMemory<_, _>| -> Result<(), String> { - m.clear(); - Ok(()) - }, - &mut rng, - ); -} - -fn bench_contention(c: &mut Criterion) { - let mut rng = CsRng::from_entropy(); - - bench_memory_contention( - "in-memory", - N_PTS, - async || InMemory::default(), - c, - async |m: &InMemory<_, _>| -> Result<(), String> { - m.clear(); - Ok(()) - }, - &mut rng, - ); -} - -criterion_group!( - name = benches; - config = Criterion::default(); - targets = - bench_search_multiple_bindings, - bench_search_multiple_keywords, - bench_insert_multiple_bindings, - bench_contention, -); - -criterion_main!(benches); diff --git a/crate/findex/benches/data/concurrent.dat b/crate/findex/benches/data/concurrent.dat deleted file mode 100644 index acd3ed33..00000000 --- a/crate/findex/benches/data/concurrent.dat +++ /dev/null @@ -1,8 +0,0 @@ -1 0 -2 56.00 -3 103.00 -4 155.00 -5 211.00 -6 260.00 -7 307.00 -8 360.00 diff --git a/crate/findex/benches/data/insert.dat b/crate/findex/benches/data/insert.dat deleted file mode 100644 index 7c073cdf..00000000 --- a/crate/findex/benches/data/insert.dat +++ /dev/null @@ -1,16 +0,0 @@ -10 8.33 -16 11.62 -26 17.01 -40 24.88 -64 36.90 -100 57.03 -159 89.30 -252 139.04 -399 222.42 -631 352.70 -1000 545.50 -1585 911.40 -2512 1323.00 -3982 2230.00 -6310 3661.00 -10000 5600.00 diff --git a/crate/findex/benches/data/search.dat b/crate/findex/benches/data/search.dat deleted file mode 100644 index 149a60f9..00000000 --- a/crate/findex/benches/data/search.dat +++ /dev/null @@ -1,16 +0,0 @@ -10 6.96 -16 10.00 -26 15.16 -40 22.45 -64 34.38 -100 52.34 -159 82.67 -252 128.00 -399 234.20 -631 323.40 -1000 513.40 -1585 844.50 -2512 1385.00 -3982 2356.00 -6310 3985.00 -10000 6745.00 diff --git a/crate/findex/benches/make_figures.tex b/crate/findex/benches/make_figures.tex deleted file mode 100644 index cffbd0c0..00000000 --- a/crate/findex/benches/make_figures.tex +++ /dev/null @@ -1,76 +0,0 @@ -\documentclass{article} -\usepackage{pgfplotstable} - -\begin{document} - -\begin{figure} - \centering - \begin{tikzpicture} - \begin{axis}[ - legend pos=north west, - xlabel={\#bindings}, - ylabel={time ($\mu$s)}, - xmin=1, xmax=10000, - grid=both - ] - \addplot[color=blue, mark=x] table {./data/search.dat}; - \addplot[color=red] table [y={create col/linear regression}] {./data/search.dat}; - \addlegendentry{search-time(\#bindings)} - \addlegendentry{ - $ y = - \pgfmathprintnumber{\pgfplotstableregressiona} - \cdot b - \pgfmathprintnumber[print sign]{\pgfplotstableregressionb}$ - } - \end{axis} - \end{tikzpicture} - \begin{tikzpicture} - \begin{axis}[ - legend pos=north west, - xlabel={\#bindings}, - ylabel={time ($\mu$s)}, - xmin=1, xmax=10000, - grid=both - ] - \addplot[color=blue, mark=x] table {./data/insert.dat}; - \addplot[color=red] table [y={create col/linear regression}] {./data/insert.dat}; - \addlegendentry{insertion-time(\#bindings)} - \addlegendentry{ - $ y = - \pgfmathprintnumber{\pgfplotstableregressiona} - \cdot b - \pgfmathprintnumber[print sign]{\pgfplotstableregressionb}$ - } - \end{axis} - \end{tikzpicture} - \caption[multi-binding search]{(Left) Client-side computation time (in $\mu$s) for a single-keyword search, given the number of bound one-word values. (Right) Client-side computation time (in $\mu$s) for a single-keyword insert, given the number of bound one-word values.} - \label{fig:multi-binding-search} - \label{fig:multi-binding-insert} -\end{figure} - -\begin{figure} - \centering - \begin{tikzpicture} - \begin{axis}[ - legend pos=north west, - xlabel={\#clients}, - ylabel={time ($\mu$s)}, - grid=both - ] - \addplot[color=blue, mark=x] table {./data/concurrent.dat}; - \addlegendentry{insertion-time(\#clients)} - \addplot[color=red] table [y={create col/linear regression}] {./data/concurrent.dat}; - \addlegendentry{ - $ y = - \pgfmathprintnumber{\pgfplotstableregressiona} - \cdot c - \pgfmathprintnumber[print sign]{\pgfplotstableregressionb}$ - } - \end{axis} - \end{tikzpicture} - \caption[concurrent insert]{Concurrency overhead (in $\mu$s) for adding 100 bindings on the same keyword, given the number of concurrent clients.} - \label{fig:concurrent-insert} -\end{figure} - - -\end{document} From 0960144db4262390db0bdcd24936499505b02612 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Tue, 1 Jul 2025 16:50:56 +0200 Subject: [PATCH 50/60] feat: start of model --- crate/findex/src/batcher_findex.rs | 99 +++++++++++++++++++++++ crate/findex/src/lib.rs | 3 + crate/findex/src/memory/batching_layer.rs | 76 +++++++++++++++++ crate/findex/src/memory/mod.rs | 2 + 4 files changed, 180 insertions(+) create mode 100644 crate/findex/src/batcher_findex.rs create mode 100644 crate/findex/src/memory/batching_layer.rs diff --git a/crate/findex/src/batcher_findex.rs b/crate/findex/src/batcher_findex.rs new file mode 100644 index 00000000..d8ced4c6 --- /dev/null +++ b/crate/findex/src/batcher_findex.rs @@ -0,0 +1,99 @@ +use std::{collections::HashSet, fmt::Debug, future::Future, hash::Hash, sync::Arc}; + +use crate::{ADDRESS_LENGTH, Address, Decoder, Encoder, Findex, IndexADT, MemoryADT}; + +pub trait BatcherSSEADT { + // TODO : maybe add the findex functions as trait + type Findex: IndexADT; // need those ? + Send + Sync; + type BatcherMemory: BatchingLayerADT
; + type Error: Send + Sync + std::error::Error; + + /// Search the index for the values bound to the given keywords. + fn batch_search( + &self, + keywords: Vec<&Keyword>, + ) -> impl Future>, Self::Error>>; + + /// Adds the given values to the index. + fn batch_insert( + &self, + entries: Vec<(Keyword, impl Sync + Send + IntoIterator)>, + ) -> impl Send + Future>; + + /// Removes the given values from the index. + fn batch_delete( + &self, + entries: Vec<(Keyword, impl Sync + Send + IntoIterator)>, + ) -> impl Send + Future>; +} + +// Define BatchingLayerADT as a supertrait of MemoryADT +pub trait BatchingLayerADT: MemoryADT { + // You only need to declare the NEW methods here + // The associated types and existing methods from MemoryADT are inherited + + /// Writes a batch of guarded write operations with bindings. + fn batch_guarded_write( + &self, + operations: Vec<( + (Self::Address, Option), + Vec<(Self::Address, Self::Word)>, + )>, + ) -> impl Send + Future>, Self::Error>>; + + // This is the function that will create the N channel ... ? +} + +impl MemoryADT for Arc { + type Address = M::Address; + type Word = M::Word; + type Error = M::Error; + + fn batch_read( + &self, + addresses: Vec, + ) -> impl Send + Future>, Self::Error>> { + (**self).batch_read(addresses) + } + + fn guarded_write( + &self, + guard: (Self::Address, Option), + bindings: Vec<(Self::Address, Self::Word)>, + ) -> impl Send + Future, Self::Error>> { + (**self).guarded_write(guard, bindings) + } +} + +#[derive(Debug)] +pub struct BatcherFindex< + const WORD_LENGTH: usize, + Value: Send + Sync + Hash + Eq, + EncodingError: Send + Sync + Debug, + BatcherMemory: Send + Sync + BatchingLayerADT
, Word = [u8; WORD_LENGTH]>, +> { + encode: Arc>, + decode: Arc>, +} +// batching_layer: Arc, +// findex: Findex>, +impl< + const WORD_LENGTH: usize, + Value: Send + Sync + Hash + Eq, + BatcherMemory: Send + + Sync + + Clone + + BatchingLayerADT
, Word = [u8; WORD_LENGTH]>, + EncodingError: Send + Sync + Debug, +> BatcherFindex +{ + pub fn new( + encode: Encoder, + decode: Decoder, + ) -> Self { + Self { + encode: Arc::new(encode), + decode: Arc::new(decode), + } + } +} diff --git a/crate/findex/src/lib.rs b/crate/findex/src/lib.rs index d0d407d6..84c63895 100644 --- a/crate/findex/src/lib.rs +++ b/crate/findex/src/lib.rs @@ -42,3 +42,6 @@ pub use memory::InMemory; /// 16-byte addresses ensure a high collision resistance that poses virtually no /// limitation on the index. pub const ADDRESS_LENGTH: usize = 16; + +// TODO: clean this later +mod batcher_findex; diff --git a/crate/findex/src/memory/batching_layer.rs b/crate/findex/src/memory/batching_layer.rs new file mode 100644 index 00000000..9008da7c --- /dev/null +++ b/crate/findex/src/memory/batching_layer.rs @@ -0,0 +1,76 @@ +// entre la encryption layer et la storage layer +// reçoit : "batch" de données de la encryption layer +// envoie : UNE seule requete a la memory adt sous jacente + +use crate::{ADDRESS_LENGTH, Address, MemoryADT, WORD_LENGTH}; + +pub trait BatchingLayerADT { + type Address; + type Word; + type Error; + + /// Reads a batch of addresses and returns their corresponding words. + fn batch_read( + &self, + addresses: Vec, + ) -> impl Send + std::prelude::rust_2024::Future>, Self::Error>>; + + /// Writes a guarded write operation with bindings. + fn guarded_write( + &self, + guard: (Self::Address, Option), + bindings: Vec<(Self::Address, Self::Word)>, + ) -> impl Send + std::prelude::rust_2024::Future, Self::Error>>; + + /// Writes a batch of guarded write operations with bindings. + fn batch_guarded_write( + &self, + guard: (Self::Address, Option), + bindings: Vec<(Self::Address, Self::Word)>, + ) -> impl Send + std::prelude::rust_2024::Future, Self::Error>>; +} + +pub struct BatchingLayer { + memory_adt: M, + // INFO: autogen garbage + + // pending_reads: HashMap>, + // pending_writes: HashMap, + // batch_size: usize, + // flush_timeout: Duration, + // last_flush: Instant, +} + +impl, Word = [u8; WORD_LENGTH]>> + BatchingLayerADT for BatchingLayer +{ + type Address; + type Word; + type Error; + + fn batch_read( + &self, + addresses: Vec, + ) -> impl Send + std::prelude::rust_2024::Future>, Self::Error>> + { + todo!() + } + + fn guarded_write( + &self, + guard: (Self::Address, Option), + bindings: Vec<(Self::Address, Self::Word)>, + ) -> impl Send + std::prelude::rust_2024::Future, Self::Error>> + { + todo!() + } + + fn batch_guarded_write( + &self, + guard: (Self::Address, Option), + bindings: Vec<(Self::Address, Self::Word)>, + ) -> impl Send + std::prelude::rust_2024::Future, Self::Error>> + { + todo!() + } +} diff --git a/crate/findex/src/memory/mod.rs b/crate/findex/src/memory/mod.rs index c4efa806..4b83e62a 100644 --- a/crate/findex/src/memory/mod.rs +++ b/crate/findex/src/memory/mod.rs @@ -5,3 +5,5 @@ pub use encryption_layer::{KEY_LENGTH, MemoryEncryptionLayer}; mod in_memory; #[cfg(any(test, feature = "test-utils"))] pub use in_memory::InMemory; + +pub mod batching_layer; From f5eb794ea770df552f0c0cf28ac90458fc3e9622 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Tue, 1 Jul 2025 20:54:37 +0200 Subject: [PATCH 51/60] feat: batch_search looks fine, at least for now --- crate/findex/Cargo.toml | 1 + crate/findex/src/batcher_findex.rs | 214 +++++++++++++++++++++++++++-- 2 files changed, 200 insertions(+), 15 deletions(-) diff --git a/crate/findex/Cargo.toml b/crate/findex/Cargo.toml index 4c668027..0bfcb25c 100644 --- a/crate/findex/Cargo.toml +++ b/crate/findex/Cargo.toml @@ -28,6 +28,7 @@ tokio = { workspace = true, features = [ "macros", ], optional = true } criterion = { workspace = true, optional = true } +futures = "0.3.31" [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crate/findex/src/batcher_findex.rs b/crate/findex/src/batcher_findex.rs index d8ced4c6..6e2d06af 100644 --- a/crate/findex/src/batcher_findex.rs +++ b/crate/findex/src/batcher_findex.rs @@ -1,11 +1,18 @@ -use std::{collections::HashSet, fmt::Debug, future::Future, hash::Hash, sync::Arc}; - use crate::{ADDRESS_LENGTH, Address, Decoder, Encoder, Findex, IndexADT, MemoryADT}; - +use std::{ + collections::{HashMap, HashSet}, + fmt::Debug, + future::Future, + hash::Hash, + ops::Add, + sync::{Arc, Mutex}, +}; +// TODO : should all of these be sync ? +use futures::channel::oneshot; pub trait BatcherSSEADT { // TODO : maybe add the findex functions as trait - type Findex: IndexADT; // need those ? + Send + Sync; - type BatcherMemory: BatchingLayerADT
; + // type Findex: IndexADT + Send + Sync; // need those ? + Send + Sync; + // type BatcherMemory: BatchingLayerADT
; type Error: Send + Sync + std::error::Error; /// Search the index for the values bound to the given keywords. @@ -40,38 +47,137 @@ pub trait BatchingLayerADT: MemoryADT { Vec<(Self::Address, Self::Word)>, )>, ) -> impl Send + Future>, Self::Error>>; +} - // This is the function that will create the N channel ... ? +struct BufferedMemory +where + M::Address: Clone, +{ + inner: M, // the actual memory layer that implements the actual network / memory call + buffer_size: usize, + pending_batches: Mutex< + Vec<( + Vec, + oneshot::Sender>, M::Error>>, + )>, + >, } -impl MemoryADT for Arc { +impl BufferedMemory +where + ::Address: Clone, +{ + fn new(inner: M, buffer_size: usize) -> Self { + Self { + inner, + buffer_size, + pending_batches: Mutex::new(Vec::new()), + } + } + + async fn flush(&self) -> Result<(), M::Error> { + // maybe add a check that the capacities are correct + let batches: Vec<( + Vec, + oneshot::Sender>, M::Error>>, + )> = { + let mut pending = self.pending_batches.lock().unwrap(); + if pending.is_empty() { + return Ok(()); + } + std::mem::take(&mut *pending) + }; + + // Build combined address list while tracking which addresses belong to which batch + let mut all_addresses = Vec::new(); + let mut batch_indices = Vec::new(); + + for (individual_address_batch, _) in &batches { + // Record the starting index for this batch + // will be of the form (start_index, batch_length) + batch_indices.push((all_addresses.len(), individual_address_batch.len())); + // Add this batch's addresses to the combined list + all_addresses.extend_from_slice(individual_address_batch); + } + + // Execute the combined batch_read + let mut all_results = self.inner.batch_read(all_addresses).await?; + + // Distribute results to each batch's sender + // TODO: this is the most readable approach but we could optimize it ig ? + for ((_, batch_len), (_, sender)) in batch_indices.into_iter().zip(batches) { + // Always drain from index 0 + let batch_results = all_results.drain(0..batch_len).collect(); + let _ = sender.send(Ok(batch_results)); + } + + Ok(()) + } +} + +impl MemoryADT for BufferedMemory +where + M::Address: Clone + Send, + M::Word: Send, +{ type Address = M::Address; type Word = M::Word; type Error = M::Error; - fn batch_read( + async fn batch_read( &self, addresses: Vec, - ) -> impl Send + Future>, Self::Error>> { - (**self).batch_read(addresses) + ) -> Result>, Self::Error> { + if addresses.is_empty() { + return Ok(Vec::new()); + } + + // Create a channel for this batch + let (sender, receiver) = oneshot::channel(); + let should_flush; + + // Add to pending batches + { + let mut pending = self.pending_batches.lock().unwrap(); + pending.push((addresses, sender)); + + // Determine if we should flush + should_flush = pending.len() >= self.buffer_size; + } + + // Flush if buffer is full + // only 1 thread will have this equal to true + if should_flush { + self.flush().await?; + } + + // Wait for results + receiver.await.map_err(|_| { + // This is a placeholder that will need to be replaced later + panic!("Channel closed unexpectedly ?") + })? } fn guarded_write( &self, - guard: (Self::Address, Option), - bindings: Vec<(Self::Address, Self::Word)>, + _guard: (Self::Address, Option), + _bindings: Vec<(Self::Address, Self::Word)>, ) -> impl Send + Future, Self::Error>> { - (**self).guarded_write(guard, bindings) + // shuts down the compiler warning for now + async move { todo!("Implement guarded_write for BufferedMemory") } } } - #[derive(Debug)] pub struct BatcherFindex< const WORD_LENGTH: usize, Value: Send + Sync + Hash + Eq, EncodingError: Send + Sync + Debug, - BatcherMemory: Send + Sync + BatchingLayerADT
, Word = [u8; WORD_LENGTH]>, + BatcherMemory: Clone + + Send + + Sync + + BatchingLayerADT
, Word = [u8; WORD_LENGTH]>, > { + memory: BatcherMemory, encode: Arc>, decode: Arc>, } @@ -88,12 +194,90 @@ impl< > BatcherFindex { pub fn new( + memory: BatcherMemory, encode: Encoder, decode: Decoder, ) -> Self { Self { + memory, encode: Arc::new(encode), decode: Arc::new(decode), } } } + +// impl< +// const WORD_LENGTH: usize, +// Value: Send + Sync + Hash + Eq, +// EncodingError: Send + Sync + Debug + std::error::Error, +// BatcherMemory: Send +// + Sync +// + Clone +// + BatchingLayerADT
, Word = [u8; WORD_LENGTH]>, +// > BatcherSSEADT, Value> +// for BatcherFindex +// { +// // type Findex = Findex; +// // type BatcherMemory = BatcherMemory; +// type Error = EncodingError; + +// async fn batch_search( +// &self, +// keywords: Vec<&Address>, +// ) -> Result>, Self::Error> { +// let mut results = Vec::>::with_capacity(keywords.len()); +// let search_futures = Vec::with_capacity(keywords.len()); +// let batching_layer = self.memory.clone(); + +// for keyword in keywords { +// // Create a future that executes the search for this keyword +// let future = async move { +// // Create a temporary Findex instance using the shared batching layer +// let findex = Findex::>::new( +// batching_layer, +// // arc clones, no worries +// self.encode.clone(), +// self.decode.clone(), +// ); + +// // Execute the search +// findex.search(keyword).await +// }; + +// search_futures.push(future); +// } +// // at this point nothing is polled yet + +// // Execute all futures concurrently and collect results +// let results = futures::future::join_all(search_futures).await; + +// // Process results +// let mut output = Vec::with_capacity(results.len()); +// for result in results { +// output.push(result?); +// } + +// Ok(output) +// todo!("Implement batch_search for BatcherFindex"); +// } + +// async fn batch_insert( +// &self, +// entries: Vec<( +// Address, +// impl Sync + Send + IntoIterator, +// )>, +// ) -> Result<(), Self::Error> { +// todo!() +// } + +// async fn batch_delete( +// &self, +// entries: Vec<( +// Address, +// impl Sync + Send + IntoIterator, +// )>, +// ) -> Result<(), Self::Error> { +// todo!() +// } +// } From da9cc1266293c507a72091c0ba8d36285a706a31 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Wed, 2 Jul 2025 15:37:46 +0200 Subject: [PATCH 52/60] feat: 1 2 3 soleil --- crate/findex/src/batcher_findex.rs | 230 +++++++++++++++++++---------- 1 file changed, 154 insertions(+), 76 deletions(-) diff --git a/crate/findex/src/batcher_findex.rs b/crate/findex/src/batcher_findex.rs index 6e2d06af..2f2f23c1 100644 --- a/crate/findex/src/batcher_findex.rs +++ b/crate/findex/src/batcher_findex.rs @@ -1,4 +1,6 @@ -use crate::{ADDRESS_LENGTH, Address, Decoder, Encoder, Findex, IndexADT, MemoryADT}; +use crate::{ + ADDRESS_LENGTH, Address, Decoder, Encoder, Findex, InMemory, IndexADT, MemoryADT, memory, +}; use std::{ collections::{HashMap, HashSet}, fmt::Debug, @@ -7,6 +9,9 @@ use std::{ ops::Add, sync::{Arc, Mutex}, }; + +// ---------------------------- THE NEW ADT TYPES ----------------------------- + // TODO : should all of these be sync ? use futures::channel::oneshot; pub trait BatcherSSEADT { @@ -49,6 +54,10 @@ pub trait BatchingLayerADT: MemoryADT { ) -> impl Send + Future>, Self::Error>>; } +// ---------------------------------- BufferedMemory Structure ---------------------------------- +// It takes as inner memory any memory that implements the batcher ADT +// which is basically, having MemoryADT + The function batch_guarded_write + struct BufferedMemory where M::Address: Clone, @@ -167,6 +176,71 @@ where async move { todo!("Implement guarded_write for BufferedMemory") } } } + +// Also implement BatchingLayerADT for BufferedMemory +impl BatchingLayerADT for BufferedMemory +where + M::Address: Clone + Send + Sync, + M::Word: Send + Sync, +{ + fn batch_guarded_write( + &self, + _operations: Vec<( + (Self::Address, Option), + Vec<(Self::Address, Self::Word)>, + )>, + ) -> impl Send + Future>, Self::Error>> { + async move { todo!("Implement batch_guarded_write for BufferedMemory") } + } +} + +// neded for findex constraints +impl Clone for BufferedMemory +where + M::Address: Clone + Send + Sync, + M::Word: Send + Sync, +{ + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + buffer_size: self.buffer_size, + pending_batches: Mutex::new(Vec::new()), + } + } +} + +// This simply forward the BR/GW calls to the inner memory +// when findex instances (below) call the batcher's operations +impl MemoryADT for Arc +where + M::Address: Send, + M::Word: Send, +{ + type Address = M::Address; + type Word = M::Word; + type Error = M::Error; + + async fn batch_read( + &self, + addresses: Vec, + ) -> Result>, Self::Error> { + (**self).batch_read(addresses).await + } + + async fn guarded_write( + &self, + guard: (Self::Address, Option), + bindings: Vec<(Self::Address, Self::Word)>, + ) -> Result, Self::Error> { + (**self).guarded_write(guard, bindings).await + } + + // Implement other required methods similarly +} + +// ---------------------------- BatcherFindex Structure ----------------------------- +// He is a bigger findex that does findex operations but in batches lol + #[derive(Debug)] pub struct BatcherFindex< const WORD_LENGTH: usize, @@ -206,78 +280,82 @@ impl< } } -// impl< -// const WORD_LENGTH: usize, -// Value: Send + Sync + Hash + Eq, -// EncodingError: Send + Sync + Debug + std::error::Error, -// BatcherMemory: Send -// + Sync -// + Clone -// + BatchingLayerADT
, Word = [u8; WORD_LENGTH]>, -// > BatcherSSEADT, Value> -// for BatcherFindex -// { -// // type Findex = Findex; -// // type BatcherMemory = BatcherMemory; -// type Error = EncodingError; - -// async fn batch_search( -// &self, -// keywords: Vec<&Address>, -// ) -> Result>, Self::Error> { -// let mut results = Vec::>::with_capacity(keywords.len()); -// let search_futures = Vec::with_capacity(keywords.len()); -// let batching_layer = self.memory.clone(); - -// for keyword in keywords { -// // Create a future that executes the search for this keyword -// let future = async move { -// // Create a temporary Findex instance using the shared batching layer -// let findex = Findex::>::new( -// batching_layer, -// // arc clones, no worries -// self.encode.clone(), -// self.decode.clone(), -// ); - -// // Execute the search -// findex.search(keyword).await -// }; - -// search_futures.push(future); -// } -// // at this point nothing is polled yet - -// // Execute all futures concurrently and collect results -// let results = futures::future::join_all(search_futures).await; - -// // Process results -// let mut output = Vec::with_capacity(results.len()); -// for result in results { -// output.push(result?); -// } - -// Ok(output) -// todo!("Implement batch_search for BatcherFindex"); -// } - -// async fn batch_insert( -// &self, -// entries: Vec<( -// Address, -// impl Sync + Send + IntoIterator, -// )>, -// ) -> Result<(), Self::Error> { -// todo!() -// } - -// async fn batch_delete( -// &self, -// entries: Vec<( -// Address, -// impl Sync + Send + IntoIterator, -// )>, -// ) -> Result<(), Self::Error> { -// todo!() -// } -// } +impl< + const WORD_LENGTH: usize, + Keyword: Send + Sync + Hash + Eq, + Value: Send + Sync + Hash + Eq, + EncodingError: Send + Sync + Debug + std::error::Error, + BatcherMemory: Send + + Sync + + Clone + + BatchingLayerADT
, Word = [u8; WORD_LENGTH]>, +> BatcherSSEADT + for BatcherFindex +{ + // type Findex = Findex; + // type BatcherMemory = BatcherMemory; + type Error = EncodingError; + + async fn batch_search( + &self, + keywords: Vec<&Keyword>, + ) -> Result>, Self::Error> { + let mut search_futures = Vec::new(); + let n = keywords.len(); + let buffered_memory = Arc::new(BufferedMemory::new(self.memory.clone(), n)); + + for keyword in keywords { + let buffered_memory_clone = buffered_memory.clone(); + let future = async move { + // Create a temporary Findex instance using the shared batching layer + let findex: Findex< + WORD_LENGTH, + Value, + EncodingError, + Arc>, + > = Findex::::new( + buffered_memory_clone, + *self.encode, + *self.decode, + ); + + // Execute the search + findex.search(keyword).await + }; + + search_futures.push(future); + } + // at this point nothing is polled yet + + // Execute all futures concurrently and collect results + let results = futures::future::join_all(search_futures).await; + + // Process results + let mut output = Vec::with_capacity(results.len()); + for result in results { + output.push(result.unwrap()); + } + + Ok(output) + } + + async fn batch_insert( + &self, + entries: Vec<( + Address, + impl Sync + Send + IntoIterator, + )>, + ) -> Result<(), Self::Error> { + todo!() + } + + async fn batch_delete( + &self, + entries: Vec<( + Address, + impl Sync + Send + IntoIterator, + )>, + ) -> Result<(), Self::Error> { + todo!() + } +} From 359b249ea49c67bcfdcf67835eeb077bd675f116 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Wed, 2 Jul 2025 17:17:59 +0200 Subject: [PATCH 53/60] feat: idk looks like this works --- crate/findex/src/batcher_findex.rs | 169 ++++++++++++++++++---- crate/findex/src/memory/batching_layer.rs | 130 ++++++++--------- 2 files changed, 205 insertions(+), 94 deletions(-) diff --git a/crate/findex/src/batcher_findex.rs b/crate/findex/src/batcher_findex.rs index 2f2f23c1..e55e99c6 100644 --- a/crate/findex/src/batcher_findex.rs +++ b/crate/findex/src/batcher_findex.rs @@ -1,6 +1,7 @@ use crate::{ ADDRESS_LENGTH, Address, Decoder, Encoder, Findex, InMemory, IndexADT, MemoryADT, memory, }; +use std::fmt::Display; use std::{ collections::{HashMap, HashSet}, fmt::Debug, @@ -194,26 +195,27 @@ where } } -// neded for findex constraints -impl Clone for BufferedMemory -where - M::Address: Clone + Send + Sync, - M::Word: Send + Sync, -{ - fn clone(&self) -> Self { - Self { - inner: self.inner.clone(), - buffer_size: self.buffer_size, - pending_batches: Mutex::new(Vec::new()), - } - } -} +// // neded for findex constraints +// update lol no it's not needed +// impl Clone for BufferedMemory +// where +// M::Address: Clone + Send + Sync, +// M::Word: Send + Sync, +// { +// fn clone(&self) -> Self { +// Self { +// inner: self.inner.clone(), +// buffer_size: self.buffer_size, +// pending_batches: Mutex::new(Vec::new()), +// } +// } +// } // This simply forward the BR/GW calls to the inner memory // when findex instances (below) call the batcher's operations -impl MemoryADT for Arc +impl MemoryADT for Arc> where - M::Address: Send, + M::Address: Send + Clone, M::Word: Send, { type Address = M::Address; @@ -264,7 +266,7 @@ impl< + Sync + Clone + BatchingLayerADT
, Word = [u8; WORD_LENGTH]>, - EncodingError: Send + Sync + Debug, + EncodingError: Send + Sync + Debug + std::error::Error, > BatcherFindex { pub fn new( @@ -280,11 +282,28 @@ impl< } } +// err type + +#[derive(Debug)] +pub enum TemporaryError { + DefaultGenericErrorForBatcher(String), +} + +impl Display for TemporaryError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:?}") + } +} + +impl std::error::Error for TemporaryError {} + +// the new error type + impl< const WORD_LENGTH: usize, Keyword: Send + Sync + Hash + Eq, Value: Send + Sync + Hash + Eq, - EncodingError: Send + Sync + Debug + std::error::Error, + EncodingError: Send + Sync + Debug, BatcherMemory: Send + Sync + Clone @@ -294,7 +313,7 @@ impl< { // type Findex = Findex; // type BatcherMemory = BatcherMemory; - type Error = EncodingError; + type Error = TemporaryError; async fn batch_search( &self, @@ -341,21 +360,113 @@ impl< async fn batch_insert( &self, - entries: Vec<( - Address, - impl Sync + Send + IntoIterator, - )>, + _entries: Vec<(Keyword, impl Sync + Send + IntoIterator)>, ) -> Result<(), Self::Error> { - todo!() + todo!("hello do not call me pls"); } async fn batch_delete( &self, - entries: Vec<( - Address, - impl Sync + Send + IntoIterator, - )>, + _entries: Vec<(Keyword, impl Sync + Send + IntoIterator)>, ) -> Result<(), Self::Error> { - todo!() + todo!("I eat cement"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + ADDRESS_LENGTH, Findex, InMemory, IndexADT, address::Address, dummy_decode, dummy_encode, + }; + use cosmian_crypto_core::{CsRng, Secret, define_byte_type, reexport::rand_core::SeedableRng}; + use std::collections::HashSet; + + impl BatchingLayerADT for InMemory, [u8; 16]> { + fn batch_guarded_write( + &self, + _operations: Vec<( + (Address, Option<[u8; 16]>), + Vec<(Address, [u8; 16])>, + )>, + ) -> impl Send + Future>, Self::Error>> { + async move { todo!("call me and you will regret it") } + } + } + + #[tokio::test] + async fn test_insert_search_delete_search() { + // Define a byte type, and use `Value` as an alias for 8-bytes values of + // that type. + type Value = Bytes<8>; + + define_byte_type!(Bytes); + + impl TryFrom for Bytes { + type Error = String; + fn try_from(value: usize) -> Result { + Self::try_from(value.to_be_bytes().as_slice()).map_err(|e| e.to_string()) + } + } + + const WORD_LENGTH: usize = 16; + + let mut rng = CsRng::from_entropy(); + let garbage_memory = InMemory::, [u8; WORD_LENGTH]>::default(); + + let findex = Findex::new( + garbage_memory.clone(), + dummy_encode::, + dummy_decode, + ); + let cat_bindings = [ + Value::try_from(1).unwrap(), + Value::try_from(3).unwrap(), + Value::try_from(5).unwrap(), + ]; + let dog_bindings = [ + Value::try_from(0).unwrap(), + Value::try_from(2).unwrap(), + Value::try_from(4).unwrap(), + ]; + findex + .insert("cat".to_string(), cat_bindings.clone()) + .await + .unwrap(); + findex + .insert("dog".to_string(), dog_bindings.clone()) + .await + .unwrap(); + let cat_res = findex.search(&"cat".to_string()).await.unwrap(); + let dog_res = findex.search(&"dog".to_string()).await.unwrap(); + assert_eq!( + cat_bindings.iter().cloned().collect::>(), + cat_res + ); + assert_eq!( + dog_bindings.iter().cloned().collect::>(), + dog_res + ); + + // all of the previous garbage is the classic findex tests, now we will try to retrieve the same values using butcher findex + let key1 = "cat".to_string(); + let key2 = "dog".to_string(); + let cat_dog_input = vec![&key1, &key2]; + + let batcher_findex = BatcherFindex::::new( + garbage_memory, + |op, values| { + dummy_encode::(op, values) + .map_err(|e| TemporaryError::DefaultGenericErrorForBatcher(e)) + }, + |words| { + dummy_decode(words).map_err(|e| TemporaryError::DefaultGenericErrorForBatcher(e)) + }, + ); + + let res = batcher_findex.batch_search(cat_dog_input).await.unwrap(); + println!("cat bindings: {cat_res:?}\n"); + println!("dog bindings: {dog_res:?}\n"); + println!("results of a batch_search performed on the vector Vec![cat, dog]: \n {res:?}\n"); } } diff --git a/crate/findex/src/memory/batching_layer.rs b/crate/findex/src/memory/batching_layer.rs index 9008da7c..0cf45130 100644 --- a/crate/findex/src/memory/batching_layer.rs +++ b/crate/findex/src/memory/batching_layer.rs @@ -1,76 +1,76 @@ -// entre la encryption layer et la storage layer -// reçoit : "batch" de données de la encryption layer -// envoie : UNE seule requete a la memory adt sous jacente +// // entre la encryption layer et la storage layer +// // reçoit : "batch" de données de la encryption layer +// // envoie : UNE seule requete a la memory adt sous jacente -use crate::{ADDRESS_LENGTH, Address, MemoryADT, WORD_LENGTH}; +// use crate::{ADDRESS_LENGTH, Address, MemoryADT, WORD_LENGTH}; -pub trait BatchingLayerADT { - type Address; - type Word; - type Error; +// pub trait BatchingLayerADT { +// type Address; +// type Word; +// type Error; - /// Reads a batch of addresses and returns their corresponding words. - fn batch_read( - &self, - addresses: Vec, - ) -> impl Send + std::prelude::rust_2024::Future>, Self::Error>>; +// /// Reads a batch of addresses and returns their corresponding words. +// fn batch_read( +// &self, +// addresses: Vec, +// ) -> impl Send + std::prelude::rust_2024::Future>, Self::Error>>; - /// Writes a guarded write operation with bindings. - fn guarded_write( - &self, - guard: (Self::Address, Option), - bindings: Vec<(Self::Address, Self::Word)>, - ) -> impl Send + std::prelude::rust_2024::Future, Self::Error>>; +// /// Writes a guarded write operation with bindings. +// fn guarded_write( +// &self, +// guard: (Self::Address, Option), +// bindings: Vec<(Self::Address, Self::Word)>, +// ) -> impl Send + std::prelude::rust_2024::Future, Self::Error>>; - /// Writes a batch of guarded write operations with bindings. - fn batch_guarded_write( - &self, - guard: (Self::Address, Option), - bindings: Vec<(Self::Address, Self::Word)>, - ) -> impl Send + std::prelude::rust_2024::Future, Self::Error>>; -} +// /// Writes a batch of guarded write operations with bindings. +// fn batch_guarded_write( +// &self, +// guard: (Self::Address, Option), +// bindings: Vec<(Self::Address, Self::Word)>, +// ) -> impl Send + std::prelude::rust_2024::Future, Self::Error>>; +// } -pub struct BatchingLayer { - memory_adt: M, - // INFO: autogen garbage +// pub struct BatchingLayer { +// memory_adt: M, +// // INFO: autogen garbage - // pending_reads: HashMap>, - // pending_writes: HashMap, - // batch_size: usize, - // flush_timeout: Duration, - // last_flush: Instant, -} +// // pending_reads: HashMap>, +// // pending_writes: HashMap, +// // batch_size: usize, +// // flush_timeout: Duration, +// // last_flush: Instant, +// } -impl, Word = [u8; WORD_LENGTH]>> - BatchingLayerADT for BatchingLayer -{ - type Address; - type Word; - type Error; +// impl, Word = [u8; WORD_LENGTH]>> +// BatchingLayerADT for BatchingLayer +// { +// type Address; +// type Word; +// type Error; - fn batch_read( - &self, - addresses: Vec, - ) -> impl Send + std::prelude::rust_2024::Future>, Self::Error>> - { - todo!() - } +// fn batch_read( +// &self, +// addresses: Vec, +// ) -> impl Send + std::prelude::rust_2024::Future>, Self::Error>> +// { +// todo!() +// } - fn guarded_write( - &self, - guard: (Self::Address, Option), - bindings: Vec<(Self::Address, Self::Word)>, - ) -> impl Send + std::prelude::rust_2024::Future, Self::Error>> - { - todo!() - } +// fn guarded_write( +// &self, +// guard: (Self::Address, Option), +// bindings: Vec<(Self::Address, Self::Word)>, +// ) -> impl Send + std::prelude::rust_2024::Future, Self::Error>> +// { +// todo!() +// } - fn batch_guarded_write( - &self, - guard: (Self::Address, Option), - bindings: Vec<(Self::Address, Self::Word)>, - ) -> impl Send + std::prelude::rust_2024::Future, Self::Error>> - { - todo!() - } -} +// fn batch_guarded_write( +// &self, +// guard: (Self::Address, Option), +// bindings: Vec<(Self::Address, Self::Word)>, +// ) -> impl Send + std::prelude::rust_2024::Future, Self::Error>> +// { +// todo!() +// } +// } From 3fba3aa21334531cc9ab48df3547921d02097ecf Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Wed, 2 Jul 2025 17:35:28 +0200 Subject: [PATCH 54/60] feat: work in progress --- crate/findex/src/adt.rs | 46 +++++++++++++++++++++++++++ crate/findex/src/batcher_findex.rs | 51 +++--------------------------- 2 files changed, 50 insertions(+), 47 deletions(-) diff --git a/crate/findex/src/adt.rs b/crate/findex/src/adt.rs index 8dc3bc8a..27912e2c 100644 --- a/crate/findex/src/adt.rs +++ b/crate/findex/src/adt.rs @@ -33,6 +33,37 @@ pub trait IndexADT { ) -> impl Send + Future>; } +/// BatcherSSEADT (Batched Searchable Symmetric Encryption Abstract Data Type) +/// +/// This trait defines batch operations for encrypted searchable indices. It extends +/// the functionality of the standard `IndexADT` by providing methods that operate on +/// multiple keywords or entries simultaneously to cut network's overhead and improve performance. +pub trait BatcherSSEADT: + IndexADT +{ + // TODO : maybe add the findex functions as trait + // type Findex: IndexADT + Send + Sync; // need those ? + Send + Sync; + // type BatcherMemory: BatchingLayerADT
; + + /// Search the index for the values bound to the given keywords. + fn batch_search( + &self, + keywords: Vec<&Keyword>, + ) -> impl Future>, Self::Error>>; + + /// Adds the given values to the index. + fn batch_insert( + &self, + entries: Vec<(Keyword, impl Sync + Send + IntoIterator)>, + ) -> impl Send + Future>; + + /// Removes the given values from the index. + fn batch_delete( + &self, + entries: Vec<(Keyword, impl Sync + Send + IntoIterator)>, + ) -> impl Send + Future>; +} + pub trait VectorADT: Send + Sync { /// Vectors are homogeneous. type Value: Send + Sync; @@ -76,6 +107,21 @@ pub trait MemoryADT { ) -> impl Send + Future, Self::Error>>; } +// Same as MemoryADT but also batches writes +pub trait BatchingMemoryADT: MemoryADT { + // You only need to declare the NEW methods here + // The associated types and existing methods from MemoryADT are inherited + + /// Writes a batch of guarded write operations with bindings. + fn batch_guarded_write( + &self, + operations: Vec<( + (Self::Address, Option), + Vec<(Self::Address, Self::Word)>, + )>, + ) -> impl Send + Future>, Self::Error>>; +} + #[cfg(test)] pub mod tests { diff --git a/crate/findex/src/batcher_findex.rs b/crate/findex/src/batcher_findex.rs index e55e99c6..942ac140 100644 --- a/crate/findex/src/batcher_findex.rs +++ b/crate/findex/src/batcher_findex.rs @@ -1,65 +1,22 @@ -use crate::{ - ADDRESS_LENGTH, Address, Decoder, Encoder, Findex, InMemory, IndexADT, MemoryADT, memory, -}; +use crate::{ADDRESS_LENGTH, Address, Decoder, Encoder, Findex, IndexADT, MemoryADT}; use std::fmt::Display; use std::{ - collections::{HashMap, HashSet}, + collections::HashSet, fmt::Debug, future::Future, hash::Hash, - ops::Add, sync::{Arc, Mutex}, }; - -// ---------------------------- THE NEW ADT TYPES ----------------------------- - // TODO : should all of these be sync ? use futures::channel::oneshot; -pub trait BatcherSSEADT { - // TODO : maybe add the findex functions as trait - // type Findex: IndexADT + Send + Sync; // need those ? + Send + Sync; - // type BatcherMemory: BatchingLayerADT
; - type Error: Send + Sync + std::error::Error; - - /// Search the index for the values bound to the given keywords. - fn batch_search( - &self, - keywords: Vec<&Keyword>, - ) -> impl Future>, Self::Error>>; - - /// Adds the given values to the index. - fn batch_insert( - &self, - entries: Vec<(Keyword, impl Sync + Send + IntoIterator)>, - ) -> impl Send + Future>; - /// Removes the given values from the index. - fn batch_delete( - &self, - entries: Vec<(Keyword, impl Sync + Send + IntoIterator)>, - ) -> impl Send + Future>; -} - -// Define BatchingLayerADT as a supertrait of MemoryADT -pub trait BatchingLayerADT: MemoryADT { - // You only need to declare the NEW methods here - // The associated types and existing methods from MemoryADT are inherited - - /// Writes a batch of guarded write operations with bindings. - fn batch_guarded_write( - &self, - operations: Vec<( - (Self::Address, Option), - Vec<(Self::Address, Self::Word)>, - )>, - ) -> impl Send + Future>, Self::Error>>; -} +// ---------------------------- THE NEW ADT TYPES ----------------------------- // ---------------------------------- BufferedMemory Structure ---------------------------------- // It takes as inner memory any memory that implements the batcher ADT // which is basically, having MemoryADT + The function batch_guarded_write -struct BufferedMemory +struct BufferedMemory where M::Address: Clone, { From 97e658d3bca54b7d599a3e9784fff4e5cbeb0aee Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Mon, 7 Jul 2025 19:06:29 +0200 Subject: [PATCH 55/60] feat: work in progress2 --- crate/findex/src/adt.rs | 9 +- crate/findex/src/batcher_findex.rs | 308 +++++--------------- crate/findex/src/error.rs | 1 + crate/findex/src/memory/batching_layer.rs | 330 +++++++++++++++++----- crate/findex/src/memory/in_memory.rs | 25 +- crate/findex/src/memory/mod.rs | 1 + 6 files changed, 354 insertions(+), 320 deletions(-) diff --git a/crate/findex/src/adt.rs b/crate/findex/src/adt.rs index 27912e2c..3341400e 100644 --- a/crate/findex/src/adt.rs +++ b/crate/findex/src/adt.rs @@ -38,17 +38,16 @@ pub trait IndexADT { /// This trait defines batch operations for encrypted searchable indices. It extends /// the functionality of the standard `IndexADT` by providing methods that operate on /// multiple keywords or entries simultaneously to cut network's overhead and improve performance. -pub trait BatcherSSEADT: - IndexADT -{ +pub trait BatcherSSEADT { // TODO : maybe add the findex functions as trait // type Findex: IndexADT + Send + Sync; // need those ? + Send + Sync; - // type BatcherMemory: BatchingLayerADT
; + // type BatcherMemory: BatchingMemoryADT
; + type Error: Send + Sync + std::error::Error; /// Search the index for the values bound to the given keywords. fn batch_search( &self, - keywords: Vec<&Keyword>, + keywords: Vec<&Keyword>, // n --> n fois barch read --> n+1 ) -> impl Future>, Self::Error>>; /// Adds the given values to the index. diff --git a/crate/findex/src/batcher_findex.rs b/crate/findex/src/batcher_findex.rs index 942ac140..14bc0297 100644 --- a/crate/findex/src/batcher_findex.rs +++ b/crate/findex/src/batcher_findex.rs @@ -1,201 +1,10 @@ -use crate::{ADDRESS_LENGTH, Address, Decoder, Encoder, Findex, IndexADT, MemoryADT}; +use crate::adt::{BatcherSSEADT, BatchingMemoryADT}; +use crate::memory::MemoryBatcher; +use crate::{ADDRESS_LENGTH, Address, Decoder, Encoder, Error, Findex, IndexADT}; use std::fmt::Display; -use std::{ - collections::HashSet, - fmt::Debug, - future::Future, - hash::Hash, - sync::{Arc, Mutex}, -}; +use std::sync::atomic::AtomicUsize; +use std::{collections::HashSet, fmt::Debug, hash::Hash, sync::Arc}; // TODO : should all of these be sync ? -use futures::channel::oneshot; - -// ---------------------------- THE NEW ADT TYPES ----------------------------- - -// ---------------------------------- BufferedMemory Structure ---------------------------------- -// It takes as inner memory any memory that implements the batcher ADT -// which is basically, having MemoryADT + The function batch_guarded_write - -struct BufferedMemory -where - M::Address: Clone, -{ - inner: M, // the actual memory layer that implements the actual network / memory call - buffer_size: usize, - pending_batches: Mutex< - Vec<( - Vec, - oneshot::Sender>, M::Error>>, - )>, - >, -} - -impl BufferedMemory -where - ::Address: Clone, -{ - fn new(inner: M, buffer_size: usize) -> Self { - Self { - inner, - buffer_size, - pending_batches: Mutex::new(Vec::new()), - } - } - - async fn flush(&self) -> Result<(), M::Error> { - // maybe add a check that the capacities are correct - let batches: Vec<( - Vec, - oneshot::Sender>, M::Error>>, - )> = { - let mut pending = self.pending_batches.lock().unwrap(); - if pending.is_empty() { - return Ok(()); - } - std::mem::take(&mut *pending) - }; - - // Build combined address list while tracking which addresses belong to which batch - let mut all_addresses = Vec::new(); - let mut batch_indices = Vec::new(); - - for (individual_address_batch, _) in &batches { - // Record the starting index for this batch - // will be of the form (start_index, batch_length) - batch_indices.push((all_addresses.len(), individual_address_batch.len())); - // Add this batch's addresses to the combined list - all_addresses.extend_from_slice(individual_address_batch); - } - - // Execute the combined batch_read - let mut all_results = self.inner.batch_read(all_addresses).await?; - - // Distribute results to each batch's sender - // TODO: this is the most readable approach but we could optimize it ig ? - for ((_, batch_len), (_, sender)) in batch_indices.into_iter().zip(batches) { - // Always drain from index 0 - let batch_results = all_results.drain(0..batch_len).collect(); - let _ = sender.send(Ok(batch_results)); - } - - Ok(()) - } -} - -impl MemoryADT for BufferedMemory -where - M::Address: Clone + Send, - M::Word: Send, -{ - type Address = M::Address; - type Word = M::Word; - type Error = M::Error; - - async fn batch_read( - &self, - addresses: Vec, - ) -> Result>, Self::Error> { - if addresses.is_empty() { - return Ok(Vec::new()); - } - - // Create a channel for this batch - let (sender, receiver) = oneshot::channel(); - let should_flush; - - // Add to pending batches - { - let mut pending = self.pending_batches.lock().unwrap(); - pending.push((addresses, sender)); - - // Determine if we should flush - should_flush = pending.len() >= self.buffer_size; - } - - // Flush if buffer is full - // only 1 thread will have this equal to true - if should_flush { - self.flush().await?; - } - - // Wait for results - receiver.await.map_err(|_| { - // This is a placeholder that will need to be replaced later - panic!("Channel closed unexpectedly ?") - })? - } - - fn guarded_write( - &self, - _guard: (Self::Address, Option), - _bindings: Vec<(Self::Address, Self::Word)>, - ) -> impl Send + Future, Self::Error>> { - // shuts down the compiler warning for now - async move { todo!("Implement guarded_write for BufferedMemory") } - } -} - -// Also implement BatchingLayerADT for BufferedMemory -impl BatchingLayerADT for BufferedMemory -where - M::Address: Clone + Send + Sync, - M::Word: Send + Sync, -{ - fn batch_guarded_write( - &self, - _operations: Vec<( - (Self::Address, Option), - Vec<(Self::Address, Self::Word)>, - )>, - ) -> impl Send + Future>, Self::Error>> { - async move { todo!("Implement batch_guarded_write for BufferedMemory") } - } -} - -// // neded for findex constraints -// update lol no it's not needed -// impl Clone for BufferedMemory -// where -// M::Address: Clone + Send + Sync, -// M::Word: Send + Sync, -// { -// fn clone(&self) -> Self { -// Self { -// inner: self.inner.clone(), -// buffer_size: self.buffer_size, -// pending_batches: Mutex::new(Vec::new()), -// } -// } -// } - -// This simply forward the BR/GW calls to the inner memory -// when findex instances (below) call the batcher's operations -impl MemoryADT for Arc> -where - M::Address: Send + Clone, - M::Word: Send, -{ - type Address = M::Address; - type Word = M::Word; - type Error = M::Error; - - async fn batch_read( - &self, - addresses: Vec, - ) -> Result>, Self::Error> { - (**self).batch_read(addresses).await - } - - async fn guarded_write( - &self, - guard: (Self::Address, Option), - bindings: Vec<(Self::Address, Self::Word)>, - ) -> Result, Self::Error> { - (**self).guarded_write(guard, bindings).await - } - - // Implement other required methods similarly -} // ---------------------------- BatcherFindex Structure ----------------------------- // He is a bigger findex that does findex operations but in batches lol @@ -208,7 +17,7 @@ pub struct BatcherFindex< BatcherMemory: Clone + Send + Sync - + BatchingLayerADT
, Word = [u8; WORD_LENGTH]>, + + BatchingMemoryADT
, Word = [u8; WORD_LENGTH]>, > { memory: BatcherMemory, encode: Arc>, @@ -222,7 +31,7 @@ impl< BatcherMemory: Send + Sync + Clone - + BatchingLayerADT
, Word = [u8; WORD_LENGTH]>, + + BatchingMemoryADT
, Word = [u8; WORD_LENGTH]>, EncodingError: Send + Sync + Debug + std::error::Error, > BatcherFindex { @@ -237,40 +46,71 @@ impl< decode: Arc::new(decode), } } -} -// err type + // Insert or delete are both an unbounded number of calls to `guarded_write` on the memory layer. + async fn batch_insert_or_delete( + &self, + entries: Vec<(Keyword, impl Sync + Send + IntoIterator)>, + is_insert: bool, + ) -> Result<(), Error>> + where + Keyword: Send + Sync + Hash + Eq, + { + let mut search_futures = Vec::new(); + let n = entries.len(); + let buffered_memory = Arc::new(MemoryBatcher::new_writer( + self.memory.clone(), + AtomicUsize::new(n), + )); + + for (guard_keyword, bindings) in entries { + let memory_arc = buffered_memory.clone(); + let future = async move { + // Create a temporary Findex instance using the shared batching layer + let findex: Findex< + WORD_LENGTH, + Value, + EncodingError, + Arc>, + > = Findex::::new( + // this (cheap) Arc cline is necessary because `decrement_capacity` is called + // below and needs to be able to access the Arc + memory_arc.clone(), + *self.encode, + *self.decode, + ); -#[derive(Debug)] -pub enum TemporaryError { - DefaultGenericErrorForBatcher(String), -} + if is_insert { + findex.insert(guard_keyword, bindings).await.unwrap(); // TODO: add to errors + } else { + findex.delete(guard_keyword, bindings).await.unwrap(); // TODO: add to errors + } + // once one of the operations succeeds, we should make the buffer smaller + memory_arc.decrement_capacity(); + }; + + search_futures.push(future); + } -impl Display for TemporaryError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{self:?}") + Ok(()) } } -impl std::error::Error for TemporaryError {} - -// the new error type - impl< const WORD_LENGTH: usize, Keyword: Send + Sync + Hash + Eq, Value: Send + Sync + Hash + Eq, - EncodingError: Send + Sync + Debug, + EncodingError: Send + Sync + Debug + std::error::Error, BatcherMemory: Send + Sync + Clone - + BatchingLayerADT
, Word = [u8; WORD_LENGTH]>, + + BatchingMemoryADT
, Word = [u8; WORD_LENGTH]>, > BatcherSSEADT for BatcherFindex { // type Findex = Findex; // type BatcherMemory = BatcherMemory; - type Error = TemporaryError; + type Error = Error>; async fn batch_search( &self, @@ -278,7 +118,10 @@ impl< ) -> Result>, Self::Error> { let mut search_futures = Vec::new(); let n = keywords.len(); - let buffered_memory = Arc::new(BufferedMemory::new(self.memory.clone(), n)); + let buffered_memory = Arc::new(MemoryBatcher::new_reader( + self.memory.clone(), + AtomicUsize::new(n), + )); for keyword in keywords { let buffered_memory_clone = buffered_memory.clone(); @@ -288,7 +131,7 @@ impl< WORD_LENGTH, Value, EncodingError, - Arc>, + Arc>, > = Findex::::new( buffered_memory_clone, *self.encode, @@ -317,16 +160,16 @@ impl< async fn batch_insert( &self, - _entries: Vec<(Keyword, impl Sync + Send + IntoIterator)>, + entries: Vec<(Keyword, impl Sync + Send + IntoIterator)>, ) -> Result<(), Self::Error> { - todo!("hello do not call me pls"); + self.batch_insert_or_delete(entries, true).await } async fn batch_delete( &self, - _entries: Vec<(Keyword, impl Sync + Send + IntoIterator)>, + entries: Vec<(Keyword, impl Sync + Send + IntoIterator)>, ) -> Result<(), Self::Error> { - todo!("I eat cement"); + self.batch_insert_or_delete(entries, true).await } } @@ -334,25 +177,14 @@ impl< mod tests { use super::*; use crate::{ - ADDRESS_LENGTH, Findex, InMemory, IndexADT, address::Address, dummy_decode, dummy_encode, + ADDRESS_LENGTH, Error, Findex, InMemory, IndexADT, address::Address, dummy_decode, + dummy_encode, }; - use cosmian_crypto_core::{CsRng, Secret, define_byte_type, reexport::rand_core::SeedableRng}; + use cosmian_crypto_core::define_byte_type; use std::collections::HashSet; - impl BatchingLayerADT for InMemory, [u8; 16]> { - fn batch_guarded_write( - &self, - _operations: Vec<( - (Address, Option<[u8; 16]>), - Vec<(Address, [u8; 16])>, - )>, - ) -> impl Send + Future>, Self::Error>> { - async move { todo!("call me and you will regret it") } - } - } - #[tokio::test] - async fn test_insert_search_delete_search() { + async fn test_search_lol() { // Define a byte type, and use `Value` as an alias for 8-bytes values of // that type. type Value = Bytes<8>; @@ -368,7 +200,6 @@ mod tests { const WORD_LENGTH: usize = 16; - let mut rng = CsRng::from_entropy(); let garbage_memory = InMemory::, [u8; WORD_LENGTH]>::default(); let findex = Findex::new( @@ -414,10 +245,11 @@ mod tests { garbage_memory, |op, values| { dummy_encode::(op, values) - .map_err(|e| TemporaryError::DefaultGenericErrorForBatcher(e)) + .map_err(|e| Error::>::DefaultGenericErrorForBatcher(e)) }, |words| { - dummy_decode(words).map_err(|e| TemporaryError::DefaultGenericErrorForBatcher(e)) + dummy_decode(words) + .map_err(|e| Error::>::DefaultGenericErrorForBatcher(e)) }, ); diff --git a/crate/findex/src/error.rs b/crate/findex/src/error.rs index 8954f877..73fd4094 100644 --- a/crate/findex/src/error.rs +++ b/crate/findex/src/error.rs @@ -7,6 +7,7 @@ pub enum Error
{ Conversion(String), MissingValue(Address, usize), CorruptedMemoryCache, + DefaultGenericErrorForBatcher(String), // TODO: redirect batcher errors to this for now } impl Display for Error
{ diff --git a/crate/findex/src/memory/batching_layer.rs b/crate/findex/src/memory/batching_layer.rs index 0cf45130..c781bf1f 100644 --- a/crate/findex/src/memory/batching_layer.rs +++ b/crate/findex/src/memory/batching_layer.rs @@ -1,76 +1,254 @@ -// // entre la encryption layer et la storage layer -// // reçoit : "batch" de données de la encryption layer -// // envoie : UNE seule requete a la memory adt sous jacente - -// use crate::{ADDRESS_LENGTH, Address, MemoryADT, WORD_LENGTH}; - -// pub trait BatchingLayerADT { -// type Address; -// type Word; -// type Error; - -// /// Reads a batch of addresses and returns their corresponding words. -// fn batch_read( -// &self, -// addresses: Vec, -// ) -> impl Send + std::prelude::rust_2024::Future>, Self::Error>>; - -// /// Writes a guarded write operation with bindings. -// fn guarded_write( -// &self, -// guard: (Self::Address, Option), -// bindings: Vec<(Self::Address, Self::Word)>, -// ) -> impl Send + std::prelude::rust_2024::Future, Self::Error>>; - -// /// Writes a batch of guarded write operations with bindings. -// fn batch_guarded_write( -// &self, -// guard: (Self::Address, Option), -// bindings: Vec<(Self::Address, Self::Word)>, -// ) -> impl Send + std::prelude::rust_2024::Future, Self::Error>>; -// } - -// pub struct BatchingLayer { -// memory_adt: M, -// // INFO: autogen garbage - -// // pending_reads: HashMap>, -// // pending_writes: HashMap, -// // batch_size: usize, -// // flush_timeout: Duration, -// // last_flush: Instant, -// } - -// impl, Word = [u8; WORD_LENGTH]>> -// BatchingLayerADT for BatchingLayer -// { -// type Address; -// type Word; -// type Error; - -// fn batch_read( -// &self, -// addresses: Vec, -// ) -> impl Send + std::prelude::rust_2024::Future>, Self::Error>> -// { -// todo!() -// } - -// fn guarded_write( -// &self, -// guard: (Self::Address, Option), -// bindings: Vec<(Self::Address, Self::Word)>, -// ) -> impl Send + std::prelude::rust_2024::Future, Self::Error>> -// { -// todo!() -// } - -// fn batch_guarded_write( -// &self, -// guard: (Self::Address, Option), -// bindings: Vec<(Self::Address, Self::Word)>, -// ) -> impl Send + std::prelude::rust_2024::Future, Self::Error>> -// { -// todo!() -// } -// } +// ---------------------------------- BufferedMemory Structure ---------------------------------- +// It takes as inner memory any memory that implements the batcher ADT +// which is basically, having MemoryADT + The function batch_guarded_write + +use std::sync::{ + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, +}; + +use futures::channel::{mpsc, oneshot}; + +use crate::{MemoryADT, adt::BatchingMemoryADT}; + +enum PendingBatchOps +where + M::Address: Clone, +{ + PendingReads( + Mutex< + Vec<( + Vec, + oneshot::Sender>, M::Error>>, + )>, + >, + ), + PendingWrites( + Mutex< + Vec<( + ((M::Address, Option), Vec<(M::Address, M::Word)>), + oneshot::Sender, M::Error>>, + )>, + >, + ), +} + +pub struct MemoryBatcher +// memory batcher +where + M::Address: Clone, +{ + inner: M, // the actual memory layer that implements the actual network / memory call + capacity: AtomicUsize, // n + pending_ops: PendingBatchOps, // ceci doit etre appelé buffer +} + +impl MemoryBatcher +where + ::Address: Clone, +{ + pub fn new_reader(inner: M, buffer_size: AtomicUsize) -> Self { + Self { + inner, + capacity: buffer_size, + pending_ops: PendingBatchOps::PendingReads(Mutex::new(Vec::new())), + } + } + + pub fn new_writer(inner: M, buffer_size: AtomicUsize) -> Self { + Self { + inner, + capacity: buffer_size, + pending_ops: PendingBatchOps::PendingWrites(Mutex::new(Vec::new())), + } + } + + // atomically decrement the buffer size, needed on inserts/deletes + pub(crate) fn decrement_capacity(&self) -> () { + // `fetch_sub` returns the previous value, so if it was 1, it means the buffer's job is done + let _ = self.capacity.fetch_sub(1, Ordering::SeqCst); + } + + async fn flush(&self) -> Result<(), M::Error> { + match &self.pending_ops { + PendingBatchOps::PendingReads(read_ops) => { + // maybe add a check that the capacities are correct + let batches: Vec<( + Vec, + oneshot::Sender>, M::Error>>, + )> = { + let mut pending = read_ops.lock().unwrap(); + if pending.is_empty() { + return Ok(()); + } + std::mem::take(&mut *pending) + }; + + // Build combined address list while tracking which addresses belong to which batch + let mut all_addresses = Vec::new(); + let mut batch_indices = Vec::new(); + + for (individual_address_batch, _) in &batches { + // Record the starting index for this batch + // will be of the form (start_index, batch_length) + batch_indices.push((all_addresses.len(), individual_address_batch.len())); + // Add this batch's addresses to the combined list + all_addresses.extend_from_slice(individual_address_batch); + } + + // Execute the combined batch_read + let mut all_results = self.inner.batch_read(all_addresses).await?; + + // Distribute results to each batch's sender + // TODO: this is the most readable approach but we could optimize it ig ? + for ((_, batch_len), (_, sender)) in batch_indices.into_iter().zip(batches) { + // Always drain from index 0 + let batch_results = all_results.drain(0..batch_len).collect(); + let _ = sender.send(Ok(batch_results)); + } + + Ok(()) + } + PendingBatchOps::PendingWrites(write_ops) => { + // maybe add a check that the capacities are correct + let batches = { + let mut pending = write_ops.lock().unwrap(); + if pending.is_empty() { + return Ok(()); + } + std::mem::take(&mut *pending) + }; + + let (bindings, senders): (Vec<_>, Vec<_>) = batches.into_iter().unzip(); + + let res = self.inner.batch_guarded_write(bindings).await?; + // Distribute results to each batch's sender + for (res, sender) in res.into_iter().zip(senders) { + let _ = sender.send(Ok(res)); + } + Ok(()) + } + }?; + Ok(()) + } +} + +impl MemoryADT for MemoryBatcher +where + M::Address: Clone + Send, + M::Word: Send, +{ + type Address = M::Address; + type Word = M::Word; + type Error = M::Error; + + async fn batch_read( + &self, + addresses: Vec, + ) -> Result>, Self::Error> { + match &self.pending_ops { + PendingBatchOps::PendingWrites(_) => panic!( + "`batch_read` is called on a writer MemoryBatcher, make sure to use `new_reader` during initialization." + ), + PendingBatchOps::PendingReads(read_ops) => { + // Create a channel for this batch + let (sender, receiver) = oneshot::channel(); + let should_flush; + + // Add to pending batches + { + let mut pending = read_ops.lock().unwrap(); + pending.push((addresses, sender)); + + // Determine if we should flush + + should_flush = pending.len() == self.capacity.load(Ordering::SeqCst); + if pending.len() > self.capacity.load(Ordering::SeqCst) { + panic!( + "this isn't supposed to happen, by design, change this to an error case later" + ) + } + } + // Flush if buffer is full + // only 1 thread will have this equal to true + if should_flush { + self.flush().await?; + } + + // Wait for results + receiver.await.map_err(|_| { + // TODO This is a placeholder that will need to be replaced later + panic!("Channel closed unexpectedly ?") + })? + } + } + } + + async fn guarded_write( + &self, + guard: (Self::Address, Option), + bindings: Vec<(Self::Address, Self::Word)>, + ) -> Result, Self::Error> { + match &self.pending_ops { + PendingBatchOps::PendingReads(_) => panic!("what's happenning ?"), + PendingBatchOps::PendingWrites(write_ops) => { + let (sender, receiver) = oneshot::channel(); + let should_flush; + + // Add to pending batches + { + let mut pending = write_ops.lock().unwrap(); + pending.push(((guard, bindings), sender)); + + let capacity = self.capacity.load(Ordering::SeqCst); + if pending.len() > capacity { + // TODO: determin if this should be kept + panic!( + "this isn't supposed to happen, by design, change this to an error case later" + ) + } + should_flush = pending.len() == capacity; + } + + // Flush if buffer is full + // only caller thread will have this equal to true + if should_flush { + self.flush().await?; + } + + // Wait for results + receiver.await.map_err(|_| { + // TODO This is a placeholder that will need to be replaced later + panic!("Channel closed unexpectedly ?") + })? + } + } + } +} + +// This simply forwards the BR/GW calls to the inner memory +// when findex instances (below) call the batcher's operations +impl MemoryADT for Arc> +where + M::Address: Send + Clone, + M::Word: Send, +{ + type Address = M::Address; + type Word = M::Word; + type Error = M::Error; + + async fn batch_read( + &self, + addresses: Vec, + ) -> Result>, Self::Error> { + (**self).batch_read(addresses).await + } + + async fn guarded_write( + &self, + guard: (Self::Address, Option), + bindings: Vec<(Self::Address, Self::Word)>, + ) -> Result, Self::Error> { + (**self).guarded_write(guard, bindings).await + } +} diff --git a/crate/findex/src/memory/in_memory.rs b/crate/findex/src/memory/in_memory.rs index c25fe0de..324b83ff 100644 --- a/crate/findex/src/memory/in_memory.rs +++ b/crate/findex/src/memory/in_memory.rs @@ -7,7 +7,7 @@ use std::{ sync::{Arc, Mutex}, }; -use crate::MemoryADT; +use crate::{MemoryADT, adt::BatchingMemoryADT}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct MemoryError; @@ -76,6 +76,29 @@ impl + BatchingMemoryADT for InMemory +{ + async fn batch_guarded_write( + &self, + operations: Vec<((Address, Option), Vec<(Address, Value)>)>, + ) -> Result>, Self::Error> { + let store = &mut *self.inner.lock().expect("poisoned lock"); + let mut res = Vec::with_capacity(operations.len()); + for (guard, bindings) in operations { + let (a, old) = guard; + let cur = store.get(&a).cloned(); + if old == cur { + for (k, v) in bindings { + store.insert(k, v); + } + } + res.push(cur); + } + Ok(res) + } +} + impl IntoIterator for InMemory { diff --git a/crate/findex/src/memory/mod.rs b/crate/findex/src/memory/mod.rs index 4b83e62a..f9900d29 100644 --- a/crate/findex/src/memory/mod.rs +++ b/crate/findex/src/memory/mod.rs @@ -7,3 +7,4 @@ mod in_memory; pub use in_memory::InMemory; pub mod batching_layer; +pub use batching_layer::*; From b722d4394f9aa8cab731f9f2031be878f0dff603 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Tue, 8 Jul 2025 13:54:29 +0200 Subject: [PATCH 56/60] chore: working first essay, needs clean --- crate/findex/src/batcher_findex.rs | 245 +++++++++++++++++++++- crate/findex/src/memory/batching_layer.rs | 8 +- 2 files changed, 246 insertions(+), 7 deletions(-) diff --git a/crate/findex/src/batcher_findex.rs b/crate/findex/src/batcher_findex.rs index 14bc0297..e3d8ac9c 100644 --- a/crate/findex/src/batcher_findex.rs +++ b/crate/findex/src/batcher_findex.rs @@ -92,6 +92,9 @@ impl< search_futures.push(future); } + // Execute all futures concurrently and collect results + futures::future::join_all(search_futures).await; + Ok(()) } } @@ -169,7 +172,7 @@ impl< &self, entries: Vec<(Keyword, impl Sync + Send + IntoIterator)>, ) -> Result<(), Self::Error> { - self.batch_insert_or_delete(entries, true).await + self.batch_insert_or_delete(entries, false).await } } @@ -183,6 +186,246 @@ mod tests { use cosmian_crypto_core::define_byte_type; use std::collections::HashSet; + #[tokio::test] + async fn test_batch_insert() { + type Value = Bytes<8>; + define_byte_type!(Bytes); + + impl TryFrom for Bytes { + type Error = String; + fn try_from(value: usize) -> Result { + Self::try_from(value.to_be_bytes().as_slice()).map_err(|e| e.to_string()) + } + } + + const WORD_LENGTH: usize = 16; + + let garbage_memory = InMemory::, [u8; WORD_LENGTH]>::default(); + + let batcher_findex = BatcherFindex::::new( + garbage_memory.clone(), + |op, values| { + dummy_encode::(op, values) + .map_err(|e| Error::>::DefaultGenericErrorForBatcher(e)) + }, + |words| { + dummy_decode(words) + .map_err(|e| Error::>::DefaultGenericErrorForBatcher(e)) + }, + ); + + // Prepare test data + let cat_bindings = vec![ + Value::try_from(1).unwrap(), + Value::try_from(3).unwrap(), + Value::try_from(5).unwrap(), + ]; + let dog_bindings = vec![ + Value::try_from(0).unwrap(), + Value::try_from(2).unwrap(), + Value::try_from(4).unwrap(), + ]; + let fish_bindings = vec![Value::try_from(7).unwrap(), Value::try_from(9).unwrap()]; + + // Batch insert multiple entries + let entries = vec![ + ("cat".to_string(), cat_bindings.clone()), + ("dog".to_string(), dog_bindings.clone()), + ("fish".to_string(), fish_bindings.clone()), + ]; + + batcher_findex.batch_insert(entries).await.unwrap(); + + // Verify insertions by searching + let key1 = "cat".to_string(); + let key2 = "dog".to_string(); + let key3 = "fish".to_string(); + let findex = Findex::new( + garbage_memory.clone(), + dummy_encode::, + dummy_decode, + ); + + let cat_result = findex.search(&"cat".to_string()).await.unwrap(); + let dog_result = findex.search(&"dog".to_string()).await.unwrap(); + let fish_result = findex.search(&"fish".to_string()).await.unwrap(); + + // assert_eq!(cat_result, cat_bindings.into_iter().collect::>()); + // assert_eq!(dog_result, dog_bindings.into_iter().collect::>()); + // assert_eq!( + // fish_result, + // fish_bindings.into_iter().collect::>() + // ); + + println!("Cat result: {:?}", cat_result); + println!("Dog result: {:?}", dog_result); + println!("Fish result: {:?}", fish_result); + // assert_eq!(results[0], cat_bindings.into_iter().collect::>()); + // assert_eq!(results[1], dog_bindings.into_iter().collect::>()); + // assert_eq!( + // results[2], + // fish_bindings.into_iter().collect::>() + // ); + } + + #[tokio::test] + // TODO!: I didn't write this one test it's autogen - to not delete this comment before rewriting it !!! + async fn test_batch_delete() { + type Value = Bytes<8>; + define_byte_type!(Bytes); + + impl TryFrom for Bytes { + type Error = String; + fn try_from(value: usize) -> Result { + Self::try_from(value.to_be_bytes().as_slice()).map_err(|e| e.to_string()) + } + } + + const WORD_LENGTH: usize = 16; + + let garbage_memory = InMemory::, [u8; WORD_LENGTH]>::default(); + + // First, populate the memory with initial data using regular Findex + let findex = Findex::new( + garbage_memory.clone(), + dummy_encode::, + dummy_decode, + ); + + // Prepare initial test data + let cat_bindings = vec![ + Value::try_from(1).unwrap(), + Value::try_from(3).unwrap(), + Value::try_from(5).unwrap(), + Value::try_from(7).unwrap(), + ]; + let dog_bindings = vec![ + Value::try_from(0).unwrap(), + Value::try_from(2).unwrap(), + Value::try_from(4).unwrap(), + Value::try_from(6).unwrap(), + ]; + let fish_bindings = vec![ + Value::try_from(8).unwrap(), + Value::try_from(9).unwrap(), + Value::try_from(10).unwrap(), + ]; + + // Insert initial data + findex + .insert("cat".to_string(), cat_bindings.clone()) + .await + .unwrap(); + findex + .insert("dog".to_string(), dog_bindings.clone()) + .await + .unwrap(); + findex + .insert("fish".to_string(), fish_bindings.clone()) + .await + .unwrap(); + + // Create BatcherFindex for deletion operations + let batcher_findex = BatcherFindex::::new( + garbage_memory.clone(), + |op, values| { + dummy_encode::(op, values) + .map_err(|e| Error::>::DefaultGenericErrorForBatcher(e)) + }, + |words| { + dummy_decode(words) + .map_err(|e| Error::>::DefaultGenericErrorForBatcher(e)) + }, + ); + + // Verify initial state + let initial_cat = findex.search(&"cat".to_string()).await.unwrap(); + let initial_dog = findex.search(&"dog".to_string()).await.unwrap(); + let initial_fish = findex.search(&"fish".to_string()).await.unwrap(); + + println!("Initial cat result: {:?}", initial_cat); + println!("Initial dog result: {:?}", initial_dog); + println!("Initial fish result: {:?}", initial_fish); + + // Prepare deletion entries - delete some values from each keyword + let delete_entries = vec![ + ( + "cat".to_string(), + vec![ + Value::try_from(1).unwrap(), // Remove 1 + Value::try_from(5).unwrap(), // Remove 5 + ], + ), + ( + "dog".to_string(), + vec![ + Value::try_from(0).unwrap(), // Remove 0 + Value::try_from(4).unwrap(), // Remove 4 + ], + ), + ( + "fish".to_string(), + vec![ + Value::try_from(9).unwrap(), // Remove 9 + ], + ), + ]; + + // Perform batch delete + batcher_findex.batch_delete(delete_entries).await.unwrap(); + + // Verify deletions by searching with fresh Findex instance + let verification_findex = Findex::new( + garbage_memory.clone(), + dummy_encode::, + dummy_decode, + ); + + let cat_after_delete = verification_findex + .search(&"cat".to_string()) + .await + .unwrap(); + let dog_after_delete = verification_findex + .search(&"dog".to_string()) + .await + .unwrap(); + let fish_after_delete = verification_findex + .search(&"fish".to_string()) + .await + .unwrap(); + + println!("Cat after delete: {:?}", cat_after_delete); + println!("Dog after delete: {:?}", dog_after_delete); + println!("Fish after delete: {:?}", fish_after_delete); + + // Expected results after deletion + let expected_cat = vec![ + Value::try_from(3).unwrap(), // 1 and 5 removed, 3 and 7 remain + Value::try_from(7).unwrap(), + ] + .into_iter() + .collect::>(); + + let expected_dog = vec![ + Value::try_from(2).unwrap(), // 0 and 4 removed, 2 and 6 remain + Value::try_from(6).unwrap(), + ] + .into_iter() + .collect::>(); + + let expected_fish = vec![ + Value::try_from(8).unwrap(), // 9 removed, 8 and 10 remain + Value::try_from(10).unwrap(), + ] + .into_iter() + .collect::>(); + + // Verify the deletions worked correctly + assert_eq!(cat_after_delete, expected_cat); + assert_eq!(dog_after_delete, expected_dog); + assert_eq!(fish_after_delete, expected_fish); + } + #[tokio::test] async fn test_search_lol() { // Define a byte type, and use `Value` as an alias for 8-bytes values of diff --git a/crate/findex/src/memory/batching_layer.rs b/crate/findex/src/memory/batching_layer.rs index c781bf1f..bdc23d9e 100644 --- a/crate/findex/src/memory/batching_layer.rs +++ b/crate/findex/src/memory/batching_layer.rs @@ -86,12 +86,8 @@ where // Build combined address list while tracking which addresses belong to which batch let mut all_addresses = Vec::new(); - let mut batch_indices = Vec::new(); for (individual_address_batch, _) in &batches { - // Record the starting index for this batch - // will be of the form (start_index, batch_length) - batch_indices.push((all_addresses.len(), individual_address_batch.len())); // Add this batch's addresses to the combined list all_addresses.extend_from_slice(individual_address_batch); } @@ -101,9 +97,9 @@ where // Distribute results to each batch's sender // TODO: this is the most readable approach but we could optimize it ig ? - for ((_, batch_len), (_, sender)) in batch_indices.into_iter().zip(batches) { + for (input, sender) in batches { // Always drain from index 0 - let batch_results = all_results.drain(0..batch_len).collect(); + let batch_results = all_results.drain(0..input.len()).collect(); let _ = sender.send(Ok(batch_results)); } From 20b0c985310c39f8e073401465c867ea9b8b1e05 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Tue, 8 Jul 2025 16:22:49 +0200 Subject: [PATCH 57/60] feat: working on error types --- crate/findex/src/batcher_findex.rs | 1 - crate/findex/src/memory/batching_layer.rs | 111 +++++++++++++++------- 2 files changed, 75 insertions(+), 37 deletions(-) diff --git a/crate/findex/src/batcher_findex.rs b/crate/findex/src/batcher_findex.rs index e3d8ac9c..9f7dd733 100644 --- a/crate/findex/src/batcher_findex.rs +++ b/crate/findex/src/batcher_findex.rs @@ -1,7 +1,6 @@ use crate::adt::{BatcherSSEADT, BatchingMemoryADT}; use crate::memory::MemoryBatcher; use crate::{ADDRESS_LENGTH, Address, Decoder, Encoder, Error, Findex, IndexADT}; -use std::fmt::Display; use std::sync::atomic::AtomicUsize; use std::{collections::HashSet, fmt::Debug, hash::Hash, sync::Arc}; // TODO : should all of these be sync ? diff --git a/crate/findex/src/memory/batching_layer.rs b/crate/findex/src/memory/batching_layer.rs index bdc23d9e..d4ac7afa 100644 --- a/crate/findex/src/memory/batching_layer.rs +++ b/crate/findex/src/memory/batching_layer.rs @@ -2,16 +2,14 @@ // It takes as inner memory any memory that implements the batcher ADT // which is basically, having MemoryADT + The function batch_guarded_write +use crate::{MemoryADT, adt::BatchingMemoryADT}; +use futures::channel::oneshot; use std::sync::{ Arc, Mutex, atomic::{AtomicUsize, Ordering}, }; -use futures::channel::{mpsc, oneshot}; - -use crate::{MemoryADT, adt::BatchingMemoryADT}; - -enum PendingBatchOps +enum PendingOperations where M::Address: Clone, { @@ -34,44 +32,85 @@ where } pub struct MemoryBatcher -// memory batcher where M::Address: Clone, { inner: M, // the actual memory layer that implements the actual network / memory call - capacity: AtomicUsize, // n - pending_ops: PendingBatchOps, // ceci doit etre appelé buffer + capacity: AtomicUsize, // capacity at which the operation should be executed + buffer: PendingOperations, +} + +#[derive(Debug)] +pub enum BatchingLayerError { + MemoryError(MemError), + ChannelClosed, +} + +impl std::fmt::Display for BatchingLayerError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BatchingLayerError::MemoryError(err) => write!(f, "Memory error: {}", err), + BatchingLayerError::ChannelClosed => write!(f, "Channel closed unexpectedly"), + } + } +} + +impl std::error::Error for BatchingLayerError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + BatchingLayerError::MemoryError(err) => Some(err), + BatchingLayerError::ChannelClosed => None, + } + } +} + +impl From for BatchingLayerError { + fn from(err: MemError) -> Self { + BatchingLayerError::MemoryError(err) + } } impl MemoryBatcher where ::Address: Clone, { - pub fn new_reader(inner: M, buffer_size: AtomicUsize) -> Self { + pub fn new_reader(inner: M, capacity: AtomicUsize) -> Self { Self { inner, - capacity: buffer_size, - pending_ops: PendingBatchOps::PendingReads(Mutex::new(Vec::new())), + capacity, + buffer: PendingOperations::PendingReads(Mutex::new(Vec::new())), } } - pub fn new_writer(inner: M, buffer_size: AtomicUsize) -> Self { + pub fn new_writer(inner: M, capacity: AtomicUsize) -> Self { Self { inner, - capacity: buffer_size, - pending_ops: PendingBatchOps::PendingWrites(Mutex::new(Vec::new())), + capacity, + buffer: PendingOperations::PendingWrites(Mutex::new(Vec::new())), } } // atomically decrement the buffer size, needed on inserts/deletes pub(crate) fn decrement_capacity(&self) -> () { // `fetch_sub` returns the previous value, so if it was 1, it means the buffer's job is done - let _ = self.capacity.fetch_sub(1, Ordering::SeqCst); + let previous = self.capacity.fetch_sub(1, Ordering::SeqCst); + match &self.buffer { + PendingOperations::PendingReads(read_ops) => { + if previous <= read_ops.lock().unwrap().len() { + let _ = self.flush(); + } + } + PendingOperations::PendingWrites(write_ops) => { + if previous <= write_ops.lock().unwrap().len() { + let _ = self.flush(); + } + } + } } - async fn flush(&self) -> Result<(), M::Error> { - match &self.pending_ops { - PendingBatchOps::PendingReads(read_ops) => { + async fn flush(&self) -> Result<(), BatchingLayerError> { + match &self.buffer { + PendingOperations::PendingReads(read_ops) => { // maybe add a check that the capacities are correct let batches: Vec<( Vec, @@ -85,27 +124,27 @@ where }; // Build combined address list while tracking which addresses belong to which batch - let mut all_addresses = Vec::new(); - - for (individual_address_batch, _) in &batches { - // Add this batch's addresses to the combined list - all_addresses.extend_from_slice(individual_address_batch); - } + let all_addresses: Vec<_> = batches + .iter() + .flat_map(|(addresses, _)| addresses.iter()) + .cloned() + .collect(); // Execute the combined batch_read let mut all_results = self.inner.batch_read(all_addresses).await?; // Distribute results to each batch's sender - // TODO: this is the most readable approach but we could optimize it ig ? - for (input, sender) in batches { - // Always drain from index 0 - let batch_results = all_results.drain(0..input.len()).collect(); - let _ = sender.send(Ok(batch_results)); + for (input, sender) in batches.into_iter().rev() { + let split_point = all_results.len() - input.len(); + // After this call, all_results will be left containing the elements [0, split_point) + // that's why we need to reverse the batches + let batch_results = all_results.split_off(split_point); + sender.send(Ok(batch_results))?; } Ok(()) } - PendingBatchOps::PendingWrites(write_ops) => { + PendingOperations::PendingWrites(write_ops) => { // maybe add a check that the capacities are correct let batches = { let mut pending = write_ops.lock().unwrap(); @@ -142,11 +181,11 @@ where &self, addresses: Vec, ) -> Result>, Self::Error> { - match &self.pending_ops { - PendingBatchOps::PendingWrites(_) => panic!( + match &self.buffer { + PendingOperations::PendingWrites(_) => panic!( "`batch_read` is called on a writer MemoryBatcher, make sure to use `new_reader` during initialization." ), - PendingBatchOps::PendingReads(read_ops) => { + PendingOperations::PendingReads(read_ops) => { // Create a channel for this batch let (sender, receiver) = oneshot::channel(); let should_flush; @@ -185,9 +224,9 @@ where guard: (Self::Address, Option), bindings: Vec<(Self::Address, Self::Word)>, ) -> Result, Self::Error> { - match &self.pending_ops { - PendingBatchOps::PendingReads(_) => panic!("what's happenning ?"), - PendingBatchOps::PendingWrites(write_ops) => { + match &self.buffer { + PendingOperations::PendingReads(_) => panic!("what's happenning ?"), + PendingOperations::PendingWrites(write_ops) => { let (sender, receiver) = oneshot::channel(); let should_flush; From 7f772ce96e1e928e6c176adc672a3e2dcd9ec895 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Tue, 8 Jul 2025 20:22:32 +0200 Subject: [PATCH 58/60] feat: working on error types, keeping on --- crate/findex/src/memory/batching_layer.rs | 127 +++++++++++++++------- 1 file changed, 85 insertions(+), 42 deletions(-) diff --git a/crate/findex/src/memory/batching_layer.rs b/crate/findex/src/memory/batching_layer.rs index d4ac7afa..5bd14ba2 100644 --- a/crate/findex/src/memory/batching_layer.rs +++ b/crate/findex/src/memory/batching_layer.rs @@ -3,12 +3,16 @@ // which is basically, having MemoryADT + The function batch_guarded_write use crate::{MemoryADT, adt::BatchingMemoryADT}; -use futures::channel::oneshot; -use std::sync::{ - Arc, Mutex, - atomic::{AtomicUsize, Ordering}, +use futures::channel::oneshot::{self, Canceled}; +use std::fmt::{Debug, format}; +use std::{ + fmt::Display, + marker::PhantomData, + sync::{ + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, + }, }; - enum PendingOperations where M::Address: Clone, @@ -41,35 +45,59 @@ where } #[derive(Debug)] -pub enum BatchingLayerError { - MemoryError(MemError), - ChannelClosed, +pub enum BatchingLayerError +where + M::Error: Debug, +{ + MemoryError(M::Error), + MutexError(String), + ChannelError(String), + _Phantom(PhantomData), } -impl std::fmt::Display for BatchingLayerError { +impl Display for BatchingLayerError +where + M::Error: Debug, +{ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - BatchingLayerError::MemoryError(err) => write!(f, "Memory error: {}", err), - BatchingLayerError::ChannelClosed => write!(f, "Channel closed unexpectedly"), + BatchingLayerError::MemoryError(err) => write!(f, "Memory error: {:?}", err), + BatchingLayerError::MutexError(msg) => write!(f, "Mutex error: {}", msg), + BatchingLayerError::ChannelError(_) => { + write!(f, "Channel closed unexpectedly.") + } + BatchingLayerError::_Phantom(_) => panic!("This variant should never be constructed"), } } } - -impl std::error::Error for BatchingLayerError { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match self { - BatchingLayerError::MemoryError(err) => Some(err), - BatchingLayerError::ChannelClosed => None, - } +impl From for BatchingLayerError +where + M::Error: Debug, +{ + fn from(_: Canceled) -> Self { + BatchingLayerError::ChannelError( + "The sender was dropped before sending its results with the `send` function." + .to_string(), + ) } } -impl From for BatchingLayerError { - fn from(err: MemError) -> Self { - BatchingLayerError::MemoryError(err) +impl From> for BatchingLayerError +where + M::Error: Debug, +{ + fn from(e: std::sync::PoisonError) -> Self { + BatchingLayerError::MutexError(format!("Mutex lock poisoned: {e}")) } } +impl std::error::Error for BatchingLayerError +where + M: Debug, + M::Error: Debug, +{ +} + impl MemoryBatcher where ::Address: Clone, @@ -91,24 +119,25 @@ where } // atomically decrement the buffer size, needed on inserts/deletes - pub(crate) fn decrement_capacity(&self) -> () { + pub(crate) fn decrement_capacity(&self) -> Result<(), BatchingLayerError> { // `fetch_sub` returns the previous value, so if it was 1, it means the buffer's job is done let previous = self.capacity.fetch_sub(1, Ordering::SeqCst); match &self.buffer { PendingOperations::PendingReads(read_ops) => { - if previous <= read_ops.lock().unwrap().len() { + if previous <= read_ops.lock()?.len() { let _ = self.flush(); } } PendingOperations::PendingWrites(write_ops) => { - if previous <= write_ops.lock().unwrap().len() { + if previous <= write_ops.lock()?.len() { let _ = self.flush(); } } } + Ok(()) } - async fn flush(&self) -> Result<(), BatchingLayerError> { + async fn flush(&self) -> Result<(), BatchingLayerError> { match &self.buffer { PendingOperations::PendingReads(read_ops) => { // maybe add a check that the capacities are correct @@ -130,8 +159,13 @@ where .cloned() .collect(); - // Execute the combined batch_read - let mut all_results = self.inner.batch_read(all_addresses).await?; + let mut all_results = self + .inner + .batch_read(all_addresses) + .await + // Implementing the adequate from trait for this error seems impossible due to + // conflicting implementation in crate `core` + .map_err(BatchingLayerError::::MemoryError)?; // Distribute results to each batch's sender for (input, sender) in batches.into_iter().rev() { @@ -139,10 +173,13 @@ where // After this call, all_results will be left containing the elements [0, split_point) // that's why we need to reverse the batches let batch_results = all_results.split_off(split_point); - sender.send(Ok(batch_results))?; + sender.send(Ok(batch_results)).map_err(|_| { + BatchingLayerError::::ChannelError( + "The receiver end of this read operation was dropped before the `send` function could be called." + .to_owned(), + ) + })?; } - - Ok(()) } PendingOperations::PendingWrites(write_ops) => { // maybe add a check that the capacities are correct @@ -156,26 +193,34 @@ where let (bindings, senders): (Vec<_>, Vec<_>) = batches.into_iter().unzip(); - let res = self.inner.batch_guarded_write(bindings).await?; + let res = self + .inner + .batch_guarded_write(bindings) + .await + .map_err(BatchingLayerError::::MemoryError)?; // Distribute results to each batch's sender for (res, sender) in res.into_iter().zip(senders) { - let _ = sender.send(Ok(res)); + sender.send(Ok(res)).map_err(|_| { + BatchingLayerError::::ChannelError( + "The receiver end of this write operation was dropped before the `send` function could be called." + .to_owned(), + ) + })?; } - Ok(()) } - }?; + }; Ok(()) } } -impl MemoryADT for MemoryBatcher +impl MemoryADT for MemoryBatcher where M::Address: Clone + Send, M::Word: Send, { type Address = M::Address; type Word = M::Word; - type Error = M::Error; + type Error = BatchingLayerError; async fn batch_read( &self, @@ -211,10 +256,8 @@ where } // Wait for results - receiver.await.map_err(|_| { - // TODO This is a placeholder that will need to be replaced later - panic!("Channel closed unexpectedly ?") - })? + let a = receiver.await??; + a } } } @@ -263,14 +306,14 @@ where // This simply forwards the BR/GW calls to the inner memory // when findex instances (below) call the batcher's operations -impl MemoryADT for Arc> +impl MemoryADT for Arc> where M::Address: Send + Clone, M::Word: Send, { type Address = M::Address; type Word = M::Word; - type Error = M::Error; + type Error = BatchingLayerError; async fn batch_read( &self, From 12f1ca11e3f8fee57e520556fcd59ab605201ae2 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Tue, 8 Jul 2025 21:06:22 +0200 Subject: [PATCH 59/60] test: write good unit tests --- crate/findex/src/batcher_findex.rs | 236 ++++++---------------- crate/findex/src/memory/batching_layer.rs | 13 +- 2 files changed, 65 insertions(+), 184 deletions(-) diff --git a/crate/findex/src/batcher_findex.rs b/crate/findex/src/batcher_findex.rs index 9f7dd733..3e7a041e 100644 --- a/crate/findex/src/batcher_findex.rs +++ b/crate/findex/src/batcher_findex.rs @@ -27,7 +27,8 @@ pub struct BatcherFindex< impl< const WORD_LENGTH: usize, Value: Send + Sync + Hash + Eq, - BatcherMemory: Send + BatcherMemory: Debug + + Send + Sync + Clone + BatchingMemoryADT
, Word = [u8; WORD_LENGTH]>, @@ -103,7 +104,8 @@ impl< Keyword: Send + Sync + Hash + Eq, Value: Send + Sync + Hash + Eq, EncodingError: Send + Sync + Debug + std::error::Error, - BatcherMemory: Send + BatcherMemory: Debug + + Send + Sync + Clone + BatchingMemoryADT
, Word = [u8; WORD_LENGTH]>, @@ -175,6 +177,9 @@ impl< } } +// The underlying tests assume the existence of a `Findex` implementation that is correct +// The testing strategy for each function is to use the `Findex` implementation to perform the same operations +// and compare the results with the `BatcherFindex` implementation. #[cfg(test)] mod tests { use super::*; @@ -185,24 +190,24 @@ mod tests { use cosmian_crypto_core::define_byte_type; use std::collections::HashSet; - #[tokio::test] - async fn test_batch_insert() { - type Value = Bytes<8>; - define_byte_type!(Bytes); - - impl TryFrom for Bytes { - type Error = String; - fn try_from(value: usize) -> Result { - Self::try_from(value.to_be_bytes().as_slice()).map_err(|e| e.to_string()) - } + type Value = Bytes<8>; + define_byte_type!(Bytes); + + impl TryFrom for Bytes { + type Error = String; + fn try_from(value: usize) -> Result { + Self::try_from(value.to_be_bytes().as_slice()).map_err(|e| e.to_string()) } + } - const WORD_LENGTH: usize = 16; + const WORD_LENGTH: usize = 16; - let garbage_memory = InMemory::, [u8; WORD_LENGTH]>::default(); + #[tokio::test] + async fn test_batch_insert() { + let trivial_memory = InMemory::, [u8; WORD_LENGTH]>::default(); let batcher_findex = BatcherFindex::::new( - garbage_memory.clone(), + trivial_memory.clone(), |op, values| { dummy_encode::(op, values) .map_err(|e| Error::>::DefaultGenericErrorForBatcher(e)) @@ -213,85 +218,50 @@ mod tests { }, ); - // Prepare test data let cat_bindings = vec![ Value::try_from(1).unwrap(), + Value::try_from(2).unwrap(), Value::try_from(3).unwrap(), - Value::try_from(5).unwrap(), ]; let dog_bindings = vec![ - Value::try_from(0).unwrap(), - Value::try_from(2).unwrap(), Value::try_from(4).unwrap(), + Value::try_from(5).unwrap(), + Value::try_from(6).unwrap(), ]; - let fish_bindings = vec![Value::try_from(7).unwrap(), Value::try_from(9).unwrap()]; // Batch insert multiple entries let entries = vec![ ("cat".to_string(), cat_bindings.clone()), ("dog".to_string(), dog_bindings.clone()), - ("fish".to_string(), fish_bindings.clone()), ]; batcher_findex.batch_insert(entries).await.unwrap(); - // Verify insertions by searching - let key1 = "cat".to_string(); - let key2 = "dog".to_string(); - let key3 = "fish".to_string(); + // instantiate a (non batched) Findex to verify the results let findex = Findex::new( - garbage_memory.clone(), + trivial_memory.clone(), dummy_encode::, dummy_decode, ); let cat_result = findex.search(&"cat".to_string()).await.unwrap(); + assert_eq!(cat_result, cat_bindings.into_iter().collect::>()); + let dog_result = findex.search(&"dog".to_string()).await.unwrap(); - let fish_result = findex.search(&"fish".to_string()).await.unwrap(); - - // assert_eq!(cat_result, cat_bindings.into_iter().collect::>()); - // assert_eq!(dog_result, dog_bindings.into_iter().collect::>()); - // assert_eq!( - // fish_result, - // fish_bindings.into_iter().collect::>() - // ); - - println!("Cat result: {:?}", cat_result); - println!("Dog result: {:?}", dog_result); - println!("Fish result: {:?}", fish_result); - // assert_eq!(results[0], cat_bindings.into_iter().collect::>()); - // assert_eq!(results[1], dog_bindings.into_iter().collect::>()); - // assert_eq!( - // results[2], - // fish_bindings.into_iter().collect::>() - // ); + assert_eq!(dog_result, dog_bindings.into_iter().collect::>()); } #[tokio::test] - // TODO!: I didn't write this one test it's autogen - to not delete this comment before rewriting it !!! async fn test_batch_delete() { - type Value = Bytes<8>; - define_byte_type!(Bytes); - - impl TryFrom for Bytes { - type Error = String; - fn try_from(value: usize) -> Result { - Self::try_from(value.to_be_bytes().as_slice()).map_err(|e| e.to_string()) - } - } - - const WORD_LENGTH: usize = 16; - - let garbage_memory = InMemory::, [u8; WORD_LENGTH]>::default(); + let trivial_memory = InMemory::, [u8; WORD_LENGTH]>::default(); // First, populate the memory with initial data using regular Findex let findex = Findex::new( - garbage_memory.clone(), + trivial_memory.clone(), dummy_encode::, dummy_decode, ); - // Prepare initial test data let cat_bindings = vec![ Value::try_from(1).unwrap(), Value::try_from(3).unwrap(), @@ -304,13 +274,7 @@ mod tests { Value::try_from(4).unwrap(), Value::try_from(6).unwrap(), ]; - let fish_bindings = vec![ - Value::try_from(8).unwrap(), - Value::try_from(9).unwrap(), - Value::try_from(10).unwrap(), - ]; - // Insert initial data findex .insert("cat".to_string(), cat_bindings.clone()) .await @@ -319,14 +283,10 @@ mod tests { .insert("dog".to_string(), dog_bindings.clone()) .await .unwrap(); - findex - .insert("fish".to_string(), fish_bindings.clone()) - .await - .unwrap(); // Create BatcherFindex for deletion operations let batcher_findex = BatcherFindex::::new( - garbage_memory.clone(), + trivial_memory.clone(), |op, values| { dummy_encode::(op, values) .map_err(|e| Error::>::DefaultGenericErrorForBatcher(e)) @@ -337,115 +297,39 @@ mod tests { }, ); - // Verify initial state - let initial_cat = findex.search(&"cat".to_string()).await.unwrap(); - let initial_dog = findex.search(&"dog".to_string()).await.unwrap(); - let initial_fish = findex.search(&"fish".to_string()).await.unwrap(); - - println!("Initial cat result: {:?}", initial_cat); - println!("Initial dog result: {:?}", initial_dog); - println!("Initial fish result: {:?}", initial_fish); - - // Prepare deletion entries - delete some values from each keyword let delete_entries = vec![ ( "cat".to_string(), - vec![ - Value::try_from(1).unwrap(), // Remove 1 - Value::try_from(5).unwrap(), // Remove 5 - ], - ), - ( - "dog".to_string(), - vec![ - Value::try_from(0).unwrap(), // Remove 0 - Value::try_from(4).unwrap(), // Remove 4 - ], - ), - ( - "fish".to_string(), - vec![ - Value::try_from(9).unwrap(), // Remove 9 - ], + vec![Value::try_from(1).unwrap(), Value::try_from(5).unwrap()], ), + ("dog".to_string(), dog_bindings), // Remove all dog bindings ]; // Perform batch delete batcher_findex.batch_delete(delete_entries).await.unwrap(); - // Verify deletions by searching with fresh Findex instance - let verification_findex = Findex::new( - garbage_memory.clone(), - dummy_encode::, - dummy_decode, - ); - - let cat_after_delete = verification_findex - .search(&"cat".to_string()) - .await - .unwrap(); - let dog_after_delete = verification_findex - .search(&"dog".to_string()) - .await - .unwrap(); - let fish_after_delete = verification_findex - .search(&"fish".to_string()) - .await - .unwrap(); - - println!("Cat after delete: {:?}", cat_after_delete); - println!("Dog after delete: {:?}", dog_after_delete); - println!("Fish after delete: {:?}", fish_after_delete); + // Verify deletions were performed using a regular findex instance + let cat_result = findex.search(&"cat".to_string()).await.unwrap(); + let dog_result = findex.search(&"dog".to_string()).await.unwrap(); - // Expected results after deletion let expected_cat = vec![ Value::try_from(3).unwrap(), // 1 and 5 removed, 3 and 7 remain Value::try_from(7).unwrap(), ] .into_iter() .collect::>(); + let expected_dog = HashSet::new(); // all of the dog bindings are removed - let expected_dog = vec![ - Value::try_from(2).unwrap(), // 0 and 4 removed, 2 and 6 remain - Value::try_from(6).unwrap(), - ] - .into_iter() - .collect::>(); - - let expected_fish = vec![ - Value::try_from(8).unwrap(), // 9 removed, 8 and 10 remain - Value::try_from(10).unwrap(), - ] - .into_iter() - .collect::>(); - - // Verify the deletions worked correctly - assert_eq!(cat_after_delete, expected_cat); - assert_eq!(dog_after_delete, expected_dog); - assert_eq!(fish_after_delete, expected_fish); + assert_eq!(cat_result, expected_cat); + assert_eq!(dog_result, expected_dog); } #[tokio::test] - async fn test_search_lol() { - // Define a byte type, and use `Value` as an alias for 8-bytes values of - // that type. - type Value = Bytes<8>; - - define_byte_type!(Bytes); - - impl TryFrom for Bytes { - type Error = String; - fn try_from(value: usize) -> Result { - Self::try_from(value.to_be_bytes().as_slice()).map_err(|e| e.to_string()) - } - } - - const WORD_LENGTH: usize = 16; - - let garbage_memory = InMemory::, [u8; WORD_LENGTH]>::default(); + async fn test_batch_search() { + let trivial_memory = InMemory::, [u8; WORD_LENGTH]>::default(); let findex = Findex::new( - garbage_memory.clone(), + trivial_memory.clone(), dummy_encode::, dummy_decode, ); @@ -467,24 +351,9 @@ mod tests { .insert("dog".to_string(), dog_bindings.clone()) .await .unwrap(); - let cat_res = findex.search(&"cat".to_string()).await.unwrap(); - let dog_res = findex.search(&"dog".to_string()).await.unwrap(); - assert_eq!( - cat_bindings.iter().cloned().collect::>(), - cat_res - ); - assert_eq!( - dog_bindings.iter().cloned().collect::>(), - dog_res - ); - - // all of the previous garbage is the classic findex tests, now we will try to retrieve the same values using butcher findex - let key1 = "cat".to_string(); - let key2 = "dog".to_string(); - let cat_dog_input = vec![&key1, &key2]; let batcher_findex = BatcherFindex::::new( - garbage_memory, + trivial_memory, |op, values| { dummy_encode::(op, values) .map_err(|e| Error::>::DefaultGenericErrorForBatcher(e)) @@ -495,9 +364,20 @@ mod tests { }, ); - let res = batcher_findex.batch_search(cat_dog_input).await.unwrap(); - println!("cat bindings: {cat_res:?}\n"); - println!("dog bindings: {dog_res:?}\n"); - println!("results of a batch_search performed on the vector Vec![cat, dog]: \n {res:?}\n"); + let key1 = "cat".to_string(); + let key2 = "dog".to_string(); + // Perform batch search + let batch_search_results = batcher_findex + .batch_search(vec![&key1, &key2]) + .await + .unwrap(); + + assert_eq!( + batch_search_results, + vec![ + cat_bindings.iter().cloned().collect::>(), + dog_bindings.iter().cloned().collect::>() + ] + ); } } diff --git a/crate/findex/src/memory/batching_layer.rs b/crate/findex/src/memory/batching_layer.rs index 5bd14ba2..306bd345 100644 --- a/crate/findex/src/memory/batching_layer.rs +++ b/crate/findex/src/memory/batching_layer.rs @@ -256,8 +256,9 @@ where } // Wait for results - let a = receiver.await??; - a + receiver + .await? + .map_err(|e| BatchingLayerError::::MemoryError(e)) } } } @@ -295,10 +296,10 @@ where } // Wait for results - receiver.await.map_err(|_| { - // TODO This is a placeholder that will need to be replaced later - panic!("Channel closed unexpectedly ?") - })? + // Wait for results + receiver + .await? + .map_err(|e| BatchingLayerError::::MemoryError(e)) } } } From 68739690655118364cd90ab436ac9cbd622c39e8 Mon Sep 17 00:00:00 2001 From: HatemMn <19950216+HatemMn@users.noreply.github.com> Date: Wed, 9 Jul 2025 11:53:18 +0200 Subject: [PATCH 60/60] feat: even more err management --- crate/findex/src/batcher_findex.rs | 435 +++++++++++++++-------------- crate/findex/src/error.rs | 38 ++- 2 files changed, 256 insertions(+), 217 deletions(-) diff --git a/crate/findex/src/batcher_findex.rs b/crate/findex/src/batcher_findex.rs index 3e7a041e..d1455c2d 100644 --- a/crate/findex/src/batcher_findex.rs +++ b/crate/findex/src/batcher_findex.rs @@ -1,13 +1,11 @@ use crate::adt::{BatcherSSEADT, BatchingMemoryADT}; +use crate::error::BatchFindexError; use crate::memory::MemoryBatcher; use crate::{ADDRESS_LENGTH, Address, Decoder, Encoder, Error, Findex, IndexADT}; use std::sync::atomic::AtomicUsize; use std::{collections::HashSet, fmt::Debug, hash::Hash, sync::Arc}; // TODO : should all of these be sync ? -// ---------------------------- BatcherFindex Structure ----------------------------- -// He is a bigger findex that does findex operations but in batches lol - #[derive(Debug)] pub struct BatcherFindex< const WORD_LENGTH: usize, @@ -22,8 +20,7 @@ pub struct BatcherFindex< encode: Arc>, decode: Arc>, } -// batching_layer: Arc, -// findex: Findex>, + impl< const WORD_LENGTH: usize, Value: Send + Sync + Hash + Eq, @@ -52,7 +49,7 @@ impl< &self, entries: Vec<(Keyword, impl Sync + Send + IntoIterator)>, is_insert: bool, - ) -> Result<(), Error>> + ) -> Result<(), BatchFindexError> where Keyword: Send + Sync + Hash + Eq, { @@ -80,20 +77,26 @@ impl< *self.decode, ); - if is_insert { - findex.insert(guard_keyword, bindings).await.unwrap(); // TODO: add to errors + let result = if is_insert { + findex.insert(guard_keyword, bindings).await } else { - findex.delete(guard_keyword, bindings).await.unwrap(); // TODO: add to errors + findex.delete(guard_keyword, bindings).await + }; + + // Convert Findex error to BatchingLayerError manually if needed + if let Err(findex_err) = result { + return Err(BatchFindexError::FindexError(findex_err)); } // once one of the operations succeeds, we should make the buffer smaller - memory_arc.decrement_capacity(); + memory_arc.decrement_capacity()?; + Ok(()) }; search_futures.push(future); } // Execute all futures concurrently and collect results - futures::future::join_all(search_futures).await; + futures::future::try_join_all(search_futures).await?; Ok(()) } @@ -114,7 +117,7 @@ impl< { // type Findex = Findex; // type BatcherMemory = BatcherMemory; - type Error = Error>; + type Error = BatchFindexError; async fn batch_search( &self, @@ -177,207 +180,207 @@ impl< } } -// The underlying tests assume the existence of a `Findex` implementation that is correct -// The testing strategy for each function is to use the `Findex` implementation to perform the same operations -// and compare the results with the `BatcherFindex` implementation. -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - ADDRESS_LENGTH, Error, Findex, InMemory, IndexADT, address::Address, dummy_decode, - dummy_encode, - }; - use cosmian_crypto_core::define_byte_type; - use std::collections::HashSet; - - type Value = Bytes<8>; - define_byte_type!(Bytes); - - impl TryFrom for Bytes { - type Error = String; - fn try_from(value: usize) -> Result { - Self::try_from(value.to_be_bytes().as_slice()).map_err(|e| e.to_string()) - } - } - - const WORD_LENGTH: usize = 16; - - #[tokio::test] - async fn test_batch_insert() { - let trivial_memory = InMemory::, [u8; WORD_LENGTH]>::default(); - - let batcher_findex = BatcherFindex::::new( - trivial_memory.clone(), - |op, values| { - dummy_encode::(op, values) - .map_err(|e| Error::>::DefaultGenericErrorForBatcher(e)) - }, - |words| { - dummy_decode(words) - .map_err(|e| Error::>::DefaultGenericErrorForBatcher(e)) - }, - ); - - let cat_bindings = vec![ - Value::try_from(1).unwrap(), - Value::try_from(2).unwrap(), - Value::try_from(3).unwrap(), - ]; - let dog_bindings = vec![ - Value::try_from(4).unwrap(), - Value::try_from(5).unwrap(), - Value::try_from(6).unwrap(), - ]; - - // Batch insert multiple entries - let entries = vec![ - ("cat".to_string(), cat_bindings.clone()), - ("dog".to_string(), dog_bindings.clone()), - ]; - - batcher_findex.batch_insert(entries).await.unwrap(); - - // instantiate a (non batched) Findex to verify the results - let findex = Findex::new( - trivial_memory.clone(), - dummy_encode::, - dummy_decode, - ); - - let cat_result = findex.search(&"cat".to_string()).await.unwrap(); - assert_eq!(cat_result, cat_bindings.into_iter().collect::>()); - - let dog_result = findex.search(&"dog".to_string()).await.unwrap(); - assert_eq!(dog_result, dog_bindings.into_iter().collect::>()); - } - - #[tokio::test] - async fn test_batch_delete() { - let trivial_memory = InMemory::, [u8; WORD_LENGTH]>::default(); - - // First, populate the memory with initial data using regular Findex - let findex = Findex::new( - trivial_memory.clone(), - dummy_encode::, - dummy_decode, - ); - - let cat_bindings = vec![ - Value::try_from(1).unwrap(), - Value::try_from(3).unwrap(), - Value::try_from(5).unwrap(), - Value::try_from(7).unwrap(), - ]; - let dog_bindings = vec![ - Value::try_from(0).unwrap(), - Value::try_from(2).unwrap(), - Value::try_from(4).unwrap(), - Value::try_from(6).unwrap(), - ]; - - findex - .insert("cat".to_string(), cat_bindings.clone()) - .await - .unwrap(); - findex - .insert("dog".to_string(), dog_bindings.clone()) - .await - .unwrap(); - - // Create BatcherFindex for deletion operations - let batcher_findex = BatcherFindex::::new( - trivial_memory.clone(), - |op, values| { - dummy_encode::(op, values) - .map_err(|e| Error::>::DefaultGenericErrorForBatcher(e)) - }, - |words| { - dummy_decode(words) - .map_err(|e| Error::>::DefaultGenericErrorForBatcher(e)) - }, - ); - - let delete_entries = vec![ - ( - "cat".to_string(), - vec![Value::try_from(1).unwrap(), Value::try_from(5).unwrap()], - ), - ("dog".to_string(), dog_bindings), // Remove all dog bindings - ]; - - // Perform batch delete - batcher_findex.batch_delete(delete_entries).await.unwrap(); - - // Verify deletions were performed using a regular findex instance - let cat_result = findex.search(&"cat".to_string()).await.unwrap(); - let dog_result = findex.search(&"dog".to_string()).await.unwrap(); - - let expected_cat = vec![ - Value::try_from(3).unwrap(), // 1 and 5 removed, 3 and 7 remain - Value::try_from(7).unwrap(), - ] - .into_iter() - .collect::>(); - let expected_dog = HashSet::new(); // all of the dog bindings are removed - - assert_eq!(cat_result, expected_cat); - assert_eq!(dog_result, expected_dog); - } - - #[tokio::test] - async fn test_batch_search() { - let trivial_memory = InMemory::, [u8; WORD_LENGTH]>::default(); - - let findex = Findex::new( - trivial_memory.clone(), - dummy_encode::, - dummy_decode, - ); - let cat_bindings = [ - Value::try_from(1).unwrap(), - Value::try_from(3).unwrap(), - Value::try_from(5).unwrap(), - ]; - let dog_bindings = [ - Value::try_from(0).unwrap(), - Value::try_from(2).unwrap(), - Value::try_from(4).unwrap(), - ]; - findex - .insert("cat".to_string(), cat_bindings.clone()) - .await - .unwrap(); - findex - .insert("dog".to_string(), dog_bindings.clone()) - .await - .unwrap(); - - let batcher_findex = BatcherFindex::::new( - trivial_memory, - |op, values| { - dummy_encode::(op, values) - .map_err(|e| Error::>::DefaultGenericErrorForBatcher(e)) - }, - |words| { - dummy_decode(words) - .map_err(|e| Error::>::DefaultGenericErrorForBatcher(e)) - }, - ); - - let key1 = "cat".to_string(); - let key2 = "dog".to_string(); - // Perform batch search - let batch_search_results = batcher_findex - .batch_search(vec![&key1, &key2]) - .await - .unwrap(); - - assert_eq!( - batch_search_results, - vec![ - cat_bindings.iter().cloned().collect::>(), - dog_bindings.iter().cloned().collect::>() - ] - ); - } -} +// // The underlying tests assume the existence of a `Findex` implementation that is correct +// // The testing strategy for each function is to use the `Findex` implementation to perform the same operations +// // and compare the results with the `BatcherFindex` implementation. +// #[cfg(test)] +// mod tests { +// use super::*; +// use crate::{ +// ADDRESS_LENGTH, Error, Findex, InMemory, IndexADT, address::Address, dummy_decode, +// dummy_encode, +// }; +// use cosmian_crypto_core::define_byte_type; +// use std::collections::HashSet; + +// type Value = Bytes<8>; +// define_byte_type!(Bytes); + +// impl TryFrom for Bytes { +// type Error = String; +// fn try_from(value: usize) -> Result { +// Self::try_from(value.to_be_bytes().as_slice()).map_err(|e| e.to_string()) +// } +// } + +// const WORD_LENGTH: usize = 16; + +// #[tokio::test] +// async fn test_batch_insert() { +// let trivial_memory = InMemory::, [u8; WORD_LENGTH]>::default(); + +// let batcher_findex = BatcherFindex::::new( +// trivial_memory.clone(), +// |op, values| { +// dummy_encode::(op, values) +// .map_err(|e| Error::>::BatchedOperationError(e)) +// }, +// |words| { +// dummy_decode(words) +// .map_err(|e| Error::>::BatchedOperationError(e)) +// }, +// ); + +// let cat_bindings = vec![ +// Value::try_from(1).unwrap(), +// Value::try_from(2).unwrap(), +// Value::try_from(3).unwrap(), +// ]; +// let dog_bindings = vec![ +// Value::try_from(4).unwrap(), +// Value::try_from(5).unwrap(), +// Value::try_from(6).unwrap(), +// ]; + +// // Batch insert multiple entries +// let entries = vec![ +// ("cat".to_string(), cat_bindings.clone()), +// ("dog".to_string(), dog_bindings.clone()), +// ]; + +// batcher_findex.batch_insert(entries).await.unwrap(); + +// // instantiate a (non batched) Findex to verify the results +// let findex = Findex::new( +// trivial_memory.clone(), +// dummy_encode::, +// dummy_decode, +// ); + +// let cat_result = findex.search(&"cat".to_string()).await.unwrap(); +// assert_eq!(cat_result, cat_bindings.into_iter().collect::>()); + +// let dog_result = findex.search(&"dog".to_string()).await.unwrap(); +// assert_eq!(dog_result, dog_bindings.into_iter().collect::>()); +// } + +// #[tokio::test] +// async fn test_batch_delete() { +// let trivial_memory = InMemory::, [u8; WORD_LENGTH]>::default(); + +// // First, populate the memory with initial data using regular Findex +// let findex = Findex::new( +// trivial_memory.clone(), +// dummy_encode::, +// dummy_decode, +// ); + +// let cat_bindings = vec![ +// Value::try_from(1).unwrap(), +// Value::try_from(3).unwrap(), +// Value::try_from(5).unwrap(), +// Value::try_from(7).unwrap(), +// ]; +// let dog_bindings = vec![ +// Value::try_from(0).unwrap(), +// Value::try_from(2).unwrap(), +// Value::try_from(4).unwrap(), +// Value::try_from(6).unwrap(), +// ]; + +// findex +// .insert("cat".to_string(), cat_bindings.clone()) +// .await +// .unwrap(); +// findex +// .insert("dog".to_string(), dog_bindings.clone()) +// .await +// .unwrap(); + +// // Create BatcherFindex for deletion operations +// let batcher_findex = BatcherFindex::::new( +// trivial_memory.clone(), +// |op, values| { +// dummy_encode::(op, values) +// .map_err(|e| Error::>::BatchedOperationError(e)) +// }, +// |words| { +// dummy_decode(words) +// .map_err(|e| Error::>::BatchedOperationError(e)) +// }, +// ); + +// let delete_entries = vec![ +// ( +// "cat".to_string(), +// vec![Value::try_from(1).unwrap(), Value::try_from(5).unwrap()], +// ), +// ("dog".to_string(), dog_bindings), // Remove all dog bindings +// ]; + +// // Perform batch delete +// batcher_findex.batch_delete(delete_entries).await.unwrap(); + +// // Verify deletions were performed using a regular findex instance +// let cat_result = findex.search(&"cat".to_string()).await.unwrap(); +// let dog_result = findex.search(&"dog".to_string()).await.unwrap(); + +// let expected_cat = vec![ +// Value::try_from(3).unwrap(), // 1 and 5 removed, 3 and 7 remain +// Value::try_from(7).unwrap(), +// ] +// .into_iter() +// .collect::>(); +// let expected_dog = HashSet::new(); // all of the dog bindings are removed + +// assert_eq!(cat_result, expected_cat); +// assert_eq!(dog_result, expected_dog); +// } + +// #[tokio::test] +// async fn test_batch_search() { +// let trivial_memory = InMemory::, [u8; WORD_LENGTH]>::default(); + +// let findex = Findex::new( +// trivial_memory.clone(), +// dummy_encode::, +// dummy_decode, +// ); +// let cat_bindings = [ +// Value::try_from(1).unwrap(), +// Value::try_from(3).unwrap(), +// Value::try_from(5).unwrap(), +// ]; +// let dog_bindings = [ +// Value::try_from(0).unwrap(), +// Value::try_from(2).unwrap(), +// Value::try_from(4).unwrap(), +// ]; +// findex +// .insert("cat".to_string(), cat_bindings.clone()) +// .await +// .unwrap(); +// findex +// .insert("dog".to_string(), dog_bindings.clone()) +// .await +// .unwrap(); + +// let batcher_findex = BatcherFindex::::new( +// trivial_memory, +// |op, values| { +// dummy_encode::(op, values) +// .map_err(|e| Error::>::BatchedOperationError(e)) +// }, +// |words| { +// dummy_decode(words) +// .map_err(|e| Error::>::BatchedOperationError(e)) +// }, +// ); + +// let key1 = "cat".to_string(); +// let key2 = "dog".to_string(); +// // Perform batch search +// let batch_search_results = batcher_findex +// .batch_search(vec![&key1, &key2]) +// .await +// .unwrap(); + +// assert_eq!( +// batch_search_results, +// vec![ +// cat_bindings.iter().cloned().collect::>(), +// dog_bindings.iter().cloned().collect::>() +// ] +// ); +// } +// } diff --git a/crate/findex/src/error.rs b/crate/findex/src/error.rs index 73fd4094..38f672b2 100644 --- a/crate/findex/src/error.rs +++ b/crate/findex/src/error.rs @@ -1,5 +1,42 @@ use std::fmt::{Debug, Display}; +use crate::{MemoryADT, memory::BatchingLayerError}; + +#[derive(Debug)] +pub enum BatchFindexError { + BatchingLayerError(BatchingLayerError), + FindexError(Error), +} + +impl Display for BatchFindexError +where + ::Address: Debug, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BatchFindexError::BatchingLayerError(e) => write!(f, "Batching layer error: {e}"), + BatchFindexError::FindexError(error) => write!(f, "Findex error: {error:?}"), + } + } +} + +impl From> for BatchFindexError { + fn from(e: Error) -> Self { + BatchFindexError::FindexError(e) + } +} + +impl From> for BatchFindexError { + fn from(e: BatchingLayerError) -> Self { + BatchFindexError::BatchingLayerError(e) + } +} + +impl std::error::Error for BatchFindexError where + ::Address: Debug +{ +} + #[derive(Debug)] pub enum Error
{ Parsing(String), @@ -7,7 +44,6 @@ pub enum Error
{ Conversion(String), MissingValue(Address, usize), CorruptedMemoryCache, - DefaultGenericErrorForBatcher(String), // TODO: redirect batcher errors to this for now } impl Display for Error
{