From b476885b18d9798fd96851222dfa16aae63ad5c2 Mon Sep 17 00:00:00 2001 From: jverdicc Date: Fri, 20 Feb 2026 06:20:45 -0500 Subject: [PATCH] Add Experiment 3 simulation harness, CLI command, and tests --- artifacts/exp3/default/exp3_results.json | 8 + artifacts/exp3/default/exp3_results.md | 6 + crates/discos-cli/src/main.rs | 93 ++++++++++ crates/discos-core/src/experiments/exp3.rs | 200 +++++++++++++++++++++ crates/discos-core/src/experiments/mod.rs | 1 + crates/discos-core/tests/exp3_tests.rs | 106 +++++++++++ 6 files changed, 414 insertions(+) create mode 100644 artifacts/exp3/default/exp3_results.json create mode 100644 artifacts/exp3/default/exp3_results.md create mode 100644 crates/discos-core/src/experiments/exp3.rs create mode 100644 crates/discos-core/tests/exp3_tests.rs diff --git a/artifacts/exp3/default/exp3_results.json b/artifacts/exp3/default/exp3_results.json new file mode 100644 index 0000000..167eff8 --- /dev/null +++ b/artifacts/exp3/default/exp3_results.json @@ -0,0 +1,8 @@ +{ + "acc_standard": 1.0, + "mi_standard_bits": 0.9702529760879881, + "acc_dlc": 0.6044, + "mi_dlc_bits": 0.04930428074708419, + "acc_pln": 0.4992, + "mi_pln_bits": 0.003846607139582803 +} \ No newline at end of file diff --git a/artifacts/exp3/default/exp3_results.md b/artifacts/exp3/default/exp3_results.md new file mode 100644 index 0000000..30c830f --- /dev/null +++ b/artifacts/exp3/default/exp3_results.md @@ -0,0 +1,6 @@ +# Experiment 3 Results + +| Metric | Standard | DLC | PLN | +|---|---:|---:|---:| +| Accuracy | 1.000000 | 0.604400 | 0.499200 | +| MI (bits) | 0.970253 | 0.049304 | 0.003847 | diff --git a/crates/discos-cli/src/main.rs b/crates/discos-cli/src/main.rs index 6559dfd..2a8c453 100644 --- a/crates/discos-cli/src/main.rs +++ b/crates/discos-cli/src/main.rs @@ -31,6 +31,8 @@ use discos_client::{ pb, verify_consistency, verify_inclusion, verify_sth_signature, ConsistencyProof, DiscosClient, InclusionProof, SignedTreeHead, }; +#[cfg(feature = "sim")] +use discos_core::experiments::exp3::{run_exp3, Exp3Config}; use discos_core::{ structured_claims::{ canonicalize_cbrn_claim, parse_cbrn_claim_json, validate_cbrn_claim, CbrnStructuredClaim, @@ -94,6 +96,11 @@ enum Command { #[command(subcommand)] cmd: ScenarioCommand, }, + #[cfg(feature = "sim")] + Sim { + #[command(subcommand)] + cmd: SimCommand, + }, } #[derive(Debug, Subcommand)] @@ -130,6 +137,38 @@ enum ScenarioCommand { }, } +#[cfg(feature = "sim")] +#[derive(Debug, Subcommand)] +enum SimCommand { + Run { + #[command(subcommand)] + cmd: SimRunCommand, + }, +} + +#[cfg(feature = "sim")] +#[derive(Debug, Subcommand)] +enum SimRunCommand { + Exp3 { + #[arg(long, default_value_t = 42)] + seed: u64, + #[arg(long, default_value_t = 5000)] + n_trials: usize, + #[arg(long, default_value_t = 10.0)] + intensity: f64, + #[arg(long, default_value_t = 1.0)] + noise_sigma: f64, + #[arg(long, default_value_t = 0.05)] + residual_frac_dlc: f64, + #[arg(long, default_value_t = 0.003)] + residual_frac_pln: f64, + #[arg(long, default_value_t = 32)] + num_bins_mi: usize, + #[arg(long)] + out: PathBuf, + }, +} + #[derive(Debug, Subcommand)] enum ClaimCommand { Create { @@ -476,6 +515,60 @@ async fn main() -> anyhow::Result<()> { println!("{}", result); } }, + #[cfg(feature = "sim")] + Command::Sim { cmd } => match cmd { + SimCommand::Run { cmd } => match cmd { + SimRunCommand::Exp3 { + seed, + n_trials, + intensity, + noise_sigma, + residual_frac_dlc, + residual_frac_pln, + num_bins_mi, + out, + } => { + let cfg = Exp3Config { + seed, + n_trials, + intensity, + noise_sigma, + residual_frac_dlc, + residual_frac_pln, + num_bins_mi, + }; + + let result = run_exp3(&cfg).await?; + fs::create_dir_all(&out)?; + let json_path = out.join("exp3_results.json"); + write_json_file(&json_path, &result)?; + + let md_path = out.join("exp3_results.md"); + let md = format!( + "# Experiment 3 Results\n\n| Metric | Standard | DLC | PLN |\n|---|---:|---:|---:|\n| Accuracy | {:.6} | {:.6} | {:.6} |\n| MI (bits) | {:.6} | {:.6} | {:.6} |\n", + result.acc_standard, + result.acc_dlc, + result.acc_pln, + result.mi_standard_bits, + result.mi_dlc_bits, + result.mi_pln_bits + ); + fs::write(&md_path, md)?; + + println!( + "{}", + serde_json::json!({ + "ok": true, + "experiment": "exp3", + "out_dir": out, + "result_json": json_path, + "result_md": md_path, + "result": result + }) + ); + } + }, + }, Command::Claim { cmd } => match cmd { ClaimCommand::Create { claim_name, diff --git a/crates/discos-core/src/experiments/exp3.rs b/crates/discos-core/src/experiments/exp3.rs new file mode 100644 index 0000000..7964ece --- /dev/null +++ b/crates/discos-core/src/experiments/exp3.rs @@ -0,0 +1,200 @@ +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaCha20Rng; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Exp3Config { + pub seed: u64, + pub n_trials: usize, + pub intensity: f64, + pub noise_sigma: f64, + pub residual_frac_dlc: f64, + pub residual_frac_pln: f64, + pub num_bins_mi: usize, +} + +impl Default for Exp3Config { + fn default() -> Self { + Self { + seed: 42, + n_trials: 5000, + intensity: 10.0, + noise_sigma: 1.0, + residual_frac_dlc: 0.05, + residual_frac_pln: 0.003, + num_bins_mi: 32, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Exp3Result { + pub acc_standard: f64, + pub mi_standard_bits: f64, + pub acc_dlc: f64, + pub mi_dlc_bits: f64, + pub acc_pln: f64, + pub mi_pln_bits: f64, +} + +const BASE_TIME: f64 = 100.0; + +pub async fn run_exp3(cfg: &Exp3Config) -> anyhow::Result { + anyhow::ensure!(cfg.n_trials >= 10, "n_trials must be at least 10"); + anyhow::ensure!(cfg.intensity.is_finite(), "intensity must be finite"); + anyhow::ensure!(cfg.noise_sigma.is_finite(), "noise_sigma must be finite"); + anyhow::ensure!(cfg.noise_sigma >= 0.0, "noise_sigma must be non-negative"); + anyhow::ensure!( + cfg.residual_frac_dlc.is_finite() && cfg.residual_frac_dlc >= 0.0, + "residual_frac_dlc must be finite and non-negative" + ); + anyhow::ensure!( + cfg.residual_frac_pln.is_finite() && cfg.residual_frac_pln >= 0.0, + "residual_frac_pln must be finite and non-negative" + ); + anyhow::ensure!(cfg.num_bins_mi >= 2, "num_bins_mi must be at least 2"); + + let mut rng = ChaCha20Rng::seed_from_u64(cfg.seed); + let mut bits = Vec::with_capacity(cfg.n_trials); + let mut standard_times = Vec::with_capacity(cfg.n_trials); + let mut dlc_times = Vec::with_capacity(cfg.n_trials); + let mut pln_times = Vec::with_capacity(cfg.n_trials); + + for _ in 0..cfg.n_trials { + let b = if rng.gen_bool(0.5) { 1u8 } else { 0u8 }; + let b_term = b as f64; + + let n_standard = sample_standard_normal(&mut rng) * cfg.noise_sigma; + let n_dlc = sample_standard_normal(&mut rng) * cfg.noise_sigma; + let n_pln = sample_standard_normal(&mut rng) * cfg.noise_sigma; + + bits.push(b); + standard_times.push(BASE_TIME + b_term * cfg.intensity + n_standard); + dlc_times.push(BASE_TIME + b_term * (cfg.residual_frac_dlc * cfg.intensity) + n_dlc); + pln_times.push(BASE_TIME + b_term * (cfg.residual_frac_pln * cfg.intensity) + n_pln); + } + + let acc_standard = train_then_eval_threshold_accuracy(&bits, &standard_times)?; + let acc_dlc = train_then_eval_threshold_accuracy(&bits, &dlc_times)?; + let acc_pln = train_then_eval_threshold_accuracy(&bits, &pln_times)?; + + let mi_standard_bits = + estimate_mutual_information_bits(&bits, &standard_times, cfg.num_bins_mi)?; + let mi_dlc_bits = estimate_mutual_information_bits(&bits, &dlc_times, cfg.num_bins_mi)?; + let mi_pln_bits = estimate_mutual_information_bits(&bits, &pln_times, cfg.num_bins_mi)?; + + Ok(Exp3Result { + acc_standard, + mi_standard_bits, + acc_dlc, + mi_dlc_bits, + acc_pln, + mi_pln_bits, + }) +} + +fn sample_standard_normal(rng: &mut ChaCha20Rng) -> f64 { + let u1 = rng.gen::().max(f64::MIN_POSITIVE); + let u2 = rng.gen::(); + (-2.0 * u1.ln()).sqrt() * (2.0 * std::f64::consts::PI * u2).cos() +} + +fn train_then_eval_threshold_accuracy(bits: &[u8], times: &[f64]) -> anyhow::Result { + anyhow::ensure!(bits.len() == times.len(), "bits/times length mismatch"); + anyhow::ensure!(bits.len() >= 2, "need at least two samples"); + + let split = bits.len() / 2; + anyhow::ensure!(split > 0 && split < bits.len(), "invalid train/eval split"); + + let train_bits = &bits[..split]; + let train_times = ×[..split]; + let eval_bits = &bits[split..]; + let eval_times = ×[split..]; + + let threshold = best_threshold(train_bits, train_times)?; + Ok(accuracy_with_threshold(eval_bits, eval_times, threshold)) +} + +fn best_threshold(bits: &[u8], times: &[f64]) -> anyhow::Result { + anyhow::ensure!(bits.len() == times.len(), "bits/times length mismatch"); + anyhow::ensure!(!times.is_empty(), "no training samples"); + + let mut sorted = times.to_vec(); + sorted.sort_by(|a, b| a.total_cmp(b)); + + let mut candidates = Vec::with_capacity(sorted.len() + 1); + candidates.push(sorted[0] - 1.0); + for window in sorted.windows(2) { + candidates.push((window[0] + window[1]) / 2.0); + } + candidates.push(sorted[sorted.len() - 1] + 1.0); + + let mut best = candidates[0]; + let mut best_acc = -1.0f64; + + for threshold in candidates { + let acc = accuracy_with_threshold(bits, times, threshold); + if acc > best_acc { + best_acc = acc; + best = threshold; + } + } + + Ok(best) +} + +fn accuracy_with_threshold(bits: &[u8], times: &[f64], threshold: f64) -> f64 { + let mut correct = 0usize; + for (b, t) in bits.iter().zip(times) { + let pred = if *t >= threshold { 1u8 } else { 0u8 }; + if pred == *b { + correct += 1; + } + } + correct as f64 / (bits.len() as f64) +} + +fn estimate_mutual_information_bits( + bits: &[u8], + times: &[f64], + bins: usize, +) -> anyhow::Result { + anyhow::ensure!(bits.len() == times.len(), "bits/times length mismatch"); + anyhow::ensure!(!times.is_empty(), "no samples"); + anyhow::ensure!(bins > 0, "bins must be positive"); + + let n = bits.len(); + let mut indices = (0..n).collect::>(); + indices.sort_by(|&i, &j| times[i].total_cmp(×[j])); + + let mut assigned_bins = vec![0usize; n]; + for (rank, idx) in indices.into_iter().enumerate() { + assigned_bins[idx] = (rank * bins) / n; + } + + let mut counts = vec![vec![0usize; bins]; 2]; + let mut count_b = [0usize; 2]; + let mut count_t = vec![0usize; bins]; + + for (b, bin) in bits.iter().zip(assigned_bins) { + let b_idx = (*b as usize).min(1); + counts[b_idx][bin] += 1; + count_b[b_idx] += 1; + count_t[bin] += 1; + } + + let n_f = n as f64; + let mut mi = 0.0f64; + for b in 0..2 { + for (bin, &joint_count) in counts[b].iter().enumerate() { + if joint_count == 0 { + continue; + } + let p_bt = (joint_count as f64) / n_f; + let p_b = (count_b[b] as f64) / n_f; + let p_t = (count_t[bin] as f64) / n_f; + mi += p_bt * (p_bt / (p_b * p_t)).log2(); + } + } + Ok(mi) +} diff --git a/crates/discos-core/src/experiments/mod.rs b/crates/discos-core/src/experiments/mod.rs index aeb4cad..3d54080 100644 --- a/crates/discos-core/src/experiments/mod.rs +++ b/crates/discos-core/src/experiments/mod.rs @@ -17,3 +17,4 @@ pub mod exp1; pub mod exp11; pub mod exp12; pub mod exp2; +pub mod exp3; diff --git a/crates/discos-core/tests/exp3_tests.rs b/crates/discos-core/tests/exp3_tests.rs new file mode 100644 index 0000000..1eb663c --- /dev/null +++ b/crates/discos-core/tests/exp3_tests.rs @@ -0,0 +1,106 @@ +#![cfg(feature = "sim")] + +use discos_core::experiments::exp3::{run_exp3, Exp3Config}; +use proptest::prelude::*; + +#[tokio::test] +async fn exp3_ordering_holds() { + let result = run_exp3(&Exp3Config { + seed: 123, + ..Exp3Config::default() + }) + .await + .unwrap_or_else(|e| panic!("exp3 should run: {e}")); + + assert!(result.acc_standard > result.acc_dlc); + assert!(result.acc_dlc > result.acc_pln); +} + +#[tokio::test] +async fn exp3_pln_near_chance() { + let result = run_exp3(&Exp3Config { + seed: 123, + ..Exp3Config::default() + }) + .await + .unwrap_or_else(|e| panic!("exp3 should run: {e}")); + + assert!((0.45..=0.55).contains(&result.acc_pln)); +} + +#[tokio::test] +async fn exp3_mi_decreases() { + let result = run_exp3(&Exp3Config { + seed: 123, + ..Exp3Config::default() + }) + .await + .unwrap_or_else(|e| panic!("exp3 should run: {e}")); + + assert!(result.mi_standard_bits > result.mi_dlc_bits); + assert!(result.mi_dlc_bits > result.mi_pln_bits); +} + +#[tokio::test] +async fn exp3_fixed_seed_golden_values() { + let result = run_exp3(&Exp3Config { + seed: 7, + n_trials: 6000, + ..Exp3Config::default() + }) + .await + .unwrap_or_else(|e| panic!("exp3 should run: {e}")); + + assert!((result.acc_standard - 1.0).abs() <= 1e-12); + assert!((result.acc_dlc - 0.6136666666666667).abs() <= 1e-12); + assert!((result.acc_pln - 0.5076666666666667).abs() <= 1e-12); + assert!((result.mi_standard_bits - 0.9785881470241701).abs() <= 1e-12); + assert!((result.mi_dlc_bits - 0.05021746994008992).abs() <= 1e-12); + assert!((result.mi_pln_bits - 0.005644267583712665).abs() <= 1e-12); +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(32))] + #[test] + fn intensity_zero_means_chance(seed in any::()) { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|e| TestCaseError::fail(format!("runtime build failed: {e}")))?; + + let result = rt + .block_on(run_exp3(&Exp3Config { + seed, + intensity: 0.0, + n_trials: 6000, + ..Exp3Config::default() + })) + .map_err(|e| TestCaseError::fail(format!("exp3 failed: {e}")))?; + + prop_assert!((result.acc_standard - 0.5).abs() <= 0.04); + prop_assert!((result.acc_dlc - 0.5).abs() <= 0.04); + prop_assert!((result.acc_pln - 0.5).abs() <= 0.04); + } + + #[test] + fn residual_frac_bounds(seed in any::(), residual_frac_dlc in 0.01f64..0.5f64, residual_frac_pln in 0.0f64..0.01f64) { + prop_assume!(residual_frac_pln <= residual_frac_dlc); + + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|e| TestCaseError::fail(format!("runtime build failed: {e}")))?; + + let result = rt + .block_on(run_exp3(&Exp3Config { + seed, + n_trials: 6000, + residual_frac_dlc, + residual_frac_pln, + ..Exp3Config::default() + })) + .map_err(|e| TestCaseError::fail(format!("exp3 failed: {e}")))?; + + prop_assert!(result.mi_pln_bits <= result.mi_dlc_bits + 0.03); + } +}