From 64c5cd9c6f4403cb313277f1936d9e9f0fcbcd2d Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Mon, 23 Mar 2026 12:33:40 +0100 Subject: [PATCH 1/8] refactor: make exchange-rate cache helpers public Move cache_key, cache_path and load_disk_cache from KrakenProvider methods to free public functions. Add save_disk_cache as a standalone helper. Make CACHE_DIR and CACHE_FILE constants public. This prepares for reuse by the upcoming cache-rates command. --- src/exchange_rate.rs | 51 ++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/src/exchange_rate.rs b/src/exchange_rate.rs index 542d427..53ef4d4 100644 --- a/src/exchange_rate.rs +++ b/src/exchange_rate.rs @@ -10,8 +10,8 @@ use reqwest::blocking::Client; use crate::common::{AppConfig, Candle, RateLimitedError, build_http_client, fetch_candle_for_timestamp}; -const CACHE_DIR: &str = ".cache"; -const CACHE_FILE: &str = "rates.json"; +pub const CACHE_DIR: &str = ".cache"; +pub const CACHE_FILE: &str = "rates.json"; const REQUEST_DELAY: Duration = Duration::from_millis(1500); /// Provides VWAP exchange rates for a given timestamp. @@ -37,7 +37,7 @@ impl KrakenProvider { .transpose()?; let clearnet_client = build_http_client("clearnet Kraken", None)?; - let disk_cache = Self::load_disk_cache(); + let disk_cache = load_disk_cache(); let initial_cache_size = disk_cache.len(); @@ -51,27 +51,9 @@ impl KrakenProvider { }) } - fn cache_key(pair: &str, interval_minutes: u32, candle_start: i64) -> String { - format!("{pair}:{interval_minutes}:{candle_start}") - } - - fn cache_path() -> PathBuf { - PathBuf::from(CACHE_DIR).join(CACHE_FILE) - } - - fn load_disk_cache() -> HashMap { - let path = Self::cache_path(); - fs::read_to_string(&path) - .ok() - .and_then(|s| serde_json::from_str(&s).ok()) - .unwrap_or_default() - } - fn save_disk_cache(&self) { let Ok(cache) = self.cache.lock() else { return }; - let Ok(json) = serde_json::to_string_pretty(&*cache) else { return }; - let _ = fs::create_dir_all(CACHE_DIR); - let _ = fs::write(Self::cache_path(), json); + let _ = save_disk_cache(&cache); } /// Returns true if new entries were written to the disk cache. @@ -120,7 +102,7 @@ impl ExchangeRateProvider for KrakenProvider { fn get_vwap(&self, timestamp: i64, interval_minutes: u32) -> Result { let interval_seconds = i64::from(interval_minutes) * 60; let candle_start = (timestamp / interval_seconds) * interval_seconds; - let key = Self::cache_key(&self.config.kraken_pair, interval_minutes, candle_start); + let key = cache_key(&self.config.kraken_pair, interval_minutes, candle_start); { let cache = self.cache.lock().map_err(|e| anyhow!("cache lock: {e}"))?; @@ -142,6 +124,29 @@ impl ExchangeRateProvider for KrakenProvider { } } +pub fn cache_key(pair: &str, interval_minutes: u32, candle_start: i64) -> String { + format!("{pair}:{interval_minutes}:{candle_start}") +} + +pub fn cache_path() -> PathBuf { + PathBuf::from(CACHE_DIR).join(CACHE_FILE) +} + +pub fn load_disk_cache() -> HashMap { + let path = cache_path(); + fs::read_to_string(&path) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default() +} + +pub fn save_disk_cache(cache: &HashMap) -> Result<()> { + let json = serde_json::to_string_pretty(cache)?; + fs::create_dir_all(CACHE_DIR)?; + fs::write(cache_path(), json)?; + Ok(()) +} + #[cfg(test)] mod tests { use super::*; From 08547ad1c3a196603b816aaf5877ad0b166e273b Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Mon, 23 Mar 2026 12:34:30 +0100 Subject: [PATCH 2/8] refactor: extract Kraken OHLC response parser Factor out parse_kraken_ohlc_response from fetch_candle_for_timestamp so the HTTP-status and JSON-decoding logic can be shared. Add fetch_candles_since, which returns all candles from a given timestamp instead of searching for a single one. --- src/common.rs | 68 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 14 deletions(-) diff --git a/src/common.rs b/src/common.rs index 820b864..fbe0f10 100644 --- a/src/common.rs +++ b/src/common.rs @@ -434,6 +434,56 @@ pub fn fetch_candle_for_timestamp( return Err(anyhow::Error::new(RateLimitedError { retry_after_secs })); } + let candle_rows = parse_kraken_ohlc_response(response, config, &url)?; + + candle_rows + .iter() + .filter_map(parse_candle_row) + .find(|candle| timestamp >= candle.time && timestamp < candle.time + interval_seconds) + .ok_or_else(|| { + anyhow!( + "Kraken did not return the {} minute candle covering {}", + interval_minutes, + format_local_timestamp(timestamp) + ) + }) +} + +pub fn fetch_candles_since( + client: &Client, + config: &AppConfig, + interval_minutes: u32, + since: i64, +) -> Result> { + let url = format!( + "{}/0/public/OHLC?pair={}&interval={interval_minutes}&since={since}", + KRAKEN_BASE_URL, config.kraken_pair + ); + + let response = client + .get(&url) + .send() + .with_context(|| format!("failed to query Kraken at {url}"))?; + + let candle_rows = parse_kraken_ohlc_response(response, config, &url)?; + + Ok(candle_rows.iter().filter_map(parse_candle_row).collect()) +} + +fn parse_kraken_ohlc_response( + response: reqwest::blocking::Response, + config: &AppConfig, + url: &str, +) -> Result> { + if response.status() == StatusCode::TOO_MANY_REQUESTS { + let retry_after_secs = response + .headers() + .get(reqwest::header::RETRY_AFTER) + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.parse::().ok()); + return Err(anyhow::Error::new(RateLimitedError { retry_after_secs })); + } + let response = response .error_for_status() .with_context(|| format!("Kraken returned an error for {url}"))?; @@ -452,22 +502,12 @@ pub fn fetch_candle_for_timestamp( let result = payload .result .context("Kraken response is missing a result field")?; - let candle_rows = result + + result .get(&config.kraken_pair) .and_then(Value::as_array) - .ok_or_else(|| anyhow!("Kraken response does not include pair {}", config.kraken_pair))?; - - candle_rows - .iter() - .filter_map(parse_candle_row) - .find(|candle| timestamp >= candle.time && timestamp < candle.time + interval_seconds) - .ok_or_else(|| { - anyhow!( - "Kraken did not return the {} minute candle covering {}", - interval_minutes, - format_local_timestamp(timestamp) - ) - }) + .cloned() + .ok_or_else(|| anyhow!("Kraken response does not include pair {}", config.kraken_pair)) } fn parse_candle_row(row: &Value) -> Option { From d744cb0c6b4911fa3f504f45eb0cc0bfb66831df Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Sun, 22 Mar 2026 18:58:02 +0100 Subject: [PATCH 3/8] export: respect DEFAULT_CANDLE_MINUTES The export command hard-coded a 1440-minute fallback and ignored the DEFAULT_CANDLE_MINUTES environment variable. Change candle_minutes to candle_override_minutes (Option) and resolve the effective interval through resolve_candle_minutes, which checks the override, then the env default, then falls back to 1440. Add unit tests for argument parsing and candle-minutes resolution. --- README.md | 1 + src/commands/export.rs | 99 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 92 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d85cc91..4d9411b 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ Key options: - `--mark-to-market` — add year-end reconciliation entries (default on in fiat mode, can be combined with `--fifo`) - `--output ` — output file path (appends if file exists) - `--start-date ` — only include transactions from this date +- `--candle ` — Kraken candle interval (`DEFAULT_CANDLE_MINUTES` or `1440` by default) - `--datadir ` — Bitcoin Core data directory (for cookie auth) - `--chain ` — chain: main, testnet3, testnet4, signet, regtest (default: main) diff --git a/src/commands/export.rs b/src/commands/export.rs index 3a6f8e6..9879ea6 100644 --- a/src/commands/export.rs +++ b/src/commands/export.rs @@ -32,10 +32,10 @@ options: --fifo Use FIFO lot tracking for realized gains (env: FIFO) --start-date Start date YYYY-MM-DD --bank-name Bank/institution name (default: Bitcoin Core - ) - --candle Kraken candle interval + --candle Kraken candle interval (default: DEFAULT_CANDLE_MINUTES or 1440) --ignore-balance-mismatch Warn instead of error on forward/backward balance mismatch"; -#[derive(Debug)] +#[derive(Debug, PartialEq, Eq)] pub struct ExportArgs { pub wallet: Option, pub country: String, @@ -47,12 +47,12 @@ pub struct ExportArgs { pub fifo: bool, pub start_date: Option, pub output: PathBuf, - pub candle_minutes: u32, + pub candle_override_minutes: Option, pub bank_name: Option, pub ignore_balance_mismatch: bool, } -#[derive(Debug)] +#[derive(Debug, PartialEq, Eq)] pub enum ExportFormat { Camt053, } @@ -91,6 +91,10 @@ pub fn run(args: ExportArgs) -> Result<()> { let descriptors = rpc.get_receive_descriptors(&receive_addresses)?; let app_config = AppConfig::from_env()?; + let candle_minutes = resolve_candle_minutes( + args.candle_override_minutes, + app_config.default_candle_minutes, + ); let provider = KrakenProvider::new(&app_config)?; let currency = if args.fiat_mode { @@ -141,7 +145,7 @@ pub fn run(args: ExportArgs) -> Result<()> { fifo: args.fifo, currency: currency.clone(), account_iban: iban, - candle_interval_minutes: args.candle_minutes, + candle_interval_minutes: candle_minutes, start_date, opening_balance_cents, bank_name, @@ -227,6 +231,12 @@ fn quote_currency_from_pair(pair: &str) -> String { } } +fn resolve_candle_minutes(candle_override_minutes: Option, default_candle_minutes: Option) -> u32 { + candle_override_minutes + .or(default_candle_minutes) + .unwrap_or(1440) +} + pub fn parse_args_from(args: I, usage: &str) -> Result where I: IntoIterator, @@ -346,8 +356,6 @@ where .map(|v| v.eq_ignore_ascii_case("true")) .unwrap_or(false); - let candle_minutes = candle_minutes.unwrap_or(1440); - Ok(ExportArgs { wallet, country, @@ -359,8 +367,83 @@ where fifo: fifo || fifo_env, start_date, output, - candle_minutes, + candle_override_minutes: candle_minutes, bank_name, ignore_balance_mismatch, }) } + +#[cfg(test)] +mod tests { + use super::{ExportFormat, ExportArgs, USAGE, parse_args_from, resolve_candle_minutes}; + + #[test] + fn parses_export_args_without_candle_override() { + let args = parse_args_from( + vec![ + "--country".to_owned(), + "NL".to_owned(), + "--wallet".to_owned(), + "test_wallet".to_owned(), + "--output".to_owned(), + "my-wallet.xml".to_owned(), + ], + USAGE, + ) + .expect("args"); + + assert_eq!( + args, + ExportArgs { + wallet: Some("test_wallet".to_owned()), + country: "NL".to_owned(), + datadir: crate::common::default_bitcoin_datadir(), + chain: "main".to_owned(), + format: ExportFormat::Camt053, + fiat_mode: false, + mark_to_market: None, + fifo: false, + start_date: None, + output: "my-wallet.xml".into(), + candle_override_minutes: None, + bank_name: None, + ignore_balance_mismatch: false, + } + ); + } + + #[test] + fn parses_export_args_with_candle_override() { + let args = parse_args_from( + vec![ + "--country".to_owned(), + "NL".to_owned(), + "--wallet".to_owned(), + "test_wallet".to_owned(), + "--output".to_owned(), + "my-wallet.xml".to_owned(), + "--candle".to_owned(), + "60".to_owned(), + ], + USAGE, + ) + .expect("args"); + + assert_eq!(args.candle_override_minutes, Some(60)); + } + + #[test] + fn export_uses_default_candle_minutes_when_present() { + assert_eq!(resolve_candle_minutes(None, Some(60)), 60); + } + + #[test] + fn export_candle_override_beats_default_candle_minutes() { + assert_eq!(resolve_candle_minutes(Some(240), Some(60)), 240); + } + + #[test] + fn export_defaults_to_daily_when_no_override_or_env_default_exists() { + assert_eq!(resolve_candle_minutes(None, None), 1440); + } +} From 20a6fac9bb25cdee55c919d20e1d151f591cd9d3 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Mon, 23 Mar 2026 13:10:59 +0100 Subject: [PATCH 4/8] Add cache-rates subcommand with Kraken OHLCVT backfill --- Cargo.lock | 141 ++++ Cargo.toml | 2 + README.md | 28 +- src/commands/cache_rates.rs | 1534 +++++++++++++++++++++++++++++++++++ src/commands/mod.rs | 1 + src/main.rs | 21 +- 6 files changed, 1725 insertions(+), 2 deletions(-) create mode 100644 src/commands/cache_rates.rs diff --git a/Cargo.lock b/Cargo.lock index b70688a..277b91d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -17,6 +23,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -59,6 +74,7 @@ dependencies = [ "capnp", "capnp-rpc", "chrono", + "csv", "dotenvy", "fixed_decimal", "futures", @@ -72,6 +88,7 @@ dependencies = [ "tempfile", "tokio", "tokio-util", + "zip", ] [[package]] @@ -165,6 +182,53 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -188,6 +252,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9eb1aa714776b75c7e67e1da744b81a129b3ff919c8712b5e1b32252c1f07cc7" +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "errno" version = "0.3.14" @@ -221,6 +291,16 @@ dependencies = [ "writeable", ] +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -345,6 +425,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + [[package]] name = "http" version = "1.4.0" @@ -616,6 +702,16 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -684,6 +780,16 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.1.1" @@ -1046,6 +1152,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + [[package]] name = "slab" version = "0.4.12" @@ -1763,8 +1875,37 @@ dependencies = [ "syn", ] +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror", + "zopfli", +] + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index 625133e..2536685 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ license = "MIT" [dependencies] anyhow = "1.0.97" chrono = { version = "0.4.40", default-features = false, features = ["clock", "std"] } +csv = "1.3.1" dotenvy = "0.15.7" fixed_decimal = "0.7.1" icu_decimal = "2.1.1" @@ -15,6 +16,7 @@ quick-xml = "0.37" reqwest = { version = "0.12.15", default-features = false, features = ["blocking", "json", "rustls-tls", "socks"] } serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.140" +zip = { version = "2.2.3", default-features = false, features = ["deflate"] } [dev-dependencies] bitcoin-capnp-types = { git = "https://github.com/2140-dev/bitcoin-capnp-types" } diff --git a/README.md b/README.md index 4d9411b..6d932af 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Rust CLI for Bitcoin accounting tasks. -Current commands: `received-value`, `export`, `reconstruct`. +Current commands: `received-value`, `cache-rates`, `export`, `reconstruct`. ## Available commands @@ -18,6 +18,29 @@ The command works as follows: 4. Uses the smallest Kraken `OHLC` candle interval that can still cover the transaction confirmation time (override with `--candle `). 5. Estimates the value of the received BTC in the quote currency of the chosen Kraken pair. +### `cache-rates` + +Populate `.cache/rates.json` for one UTC calendar year of Bitcoin rates. + +```bash +cargo run -- cache-rates 2024 +``` + +The command works as follows: + +1. Fetches the daily (`1440`) Kraken `OHLC` data that is still available through the public API. +2. Detects which closed UTC daily candles in the requested year are still missing because they are older than Kraken's 720-candle retention window. +3. Backfills those missing days into `.cache/rates.json` from Kraken's downloadable OHLCVT archive. + +The cache keys are written as `1440`-minute entries. + +Trade-off: + +- Recent days use Kraken's daily `OHLC` API `vwap`, just like the normal live lookup path. +- Older archive-backed days use the daily `(open + close) / 2` midpoint derived from Kraken's OHLCVT CSV, because the downloadable OHLCVT archive does not include `vwap`. +- The command only fills missing cache entries for that year and interval; it does not overwrite existing entries with midpoint-derived values. +- Downloaded Kraken archive ZIPs are kept under `.cache/kraken/` and reused across later runs; the command does not keep expanded CSV files on disk. + ### `export`

@@ -169,6 +192,8 @@ transaction_age <= 720 * interval_minutes * 60 If the transaction is too old to fit inside Kraken's `1d` candle retention window, the tool exits with an error instead of silently switching to a coarser interval. +`cache-rates` is the explicit opt-in workaround for older values: it backfills `.cache/rates.json` from Kraken's OHLCVT archive as daily `(open + close) / 2` midpoint prices. + ## Development See [DEVELOP.md](DEVELOP.md) for Bitcoin Core build instructions, running @@ -192,6 +217,7 @@ Licensed under the MIT License. See `LICENSE` for details. - `src/main.rs` — top-level command dispatcher - `src/lib.rs` — library crate root - `src/commands/received_value.rs` — `received-value` subcommand +- `src/commands/cache_rates.rs` — `cache-rates` subcommand - `src/commands/export.rs` — `export` subcommand - `src/commands/reconstruct.rs` — `reconstruct` subcommand - `src/common.rs` — shared config, mempool, Kraken, Tor, candle, and formatting logic diff --git a/src/commands/cache_rates.rs b/src/commands/cache_rates.rs new file mode 100644 index 0000000..95c6147 --- /dev/null +++ b/src/commands/cache_rates.rs @@ -0,0 +1,1534 @@ +use std::collections::{BTreeSet, HashMap}; +use std::fs::{self, File}; +use std::io; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, anyhow, bail}; +use chrono::{Datelike, NaiveDate, Utc}; +use csv::StringRecord; +use reqwest::blocking::{Client, Response}; +use zip::ZipArchive; + +use crate::common::{AppConfig, build_http_client, fetch_candles_since}; +use crate::exchange_rate::{CACHE_DIR, cache_key, cache_path, load_disk_cache, save_disk_cache}; + +pub const SUBCOMMAND_NAME: &str = "cache-rates"; +pub const USAGE: &str = "usage: btc_fiat_value cache-rates "; + +const KRAKEN_DAILY_INTERVAL_MINUTES: u32 = 1_440; +const QUARTERLY_ARCHIVE_FIRST_YEAR: i32 = 2023; +// Kraken archive references: +// - OHLC REST retention and daily 1440 candles: https://docs.kraken.com/api/docs/rest-api/get-ohlc-data/ +// - Downloadable OHLCVT archive landing page: https://support.kraken.com/articles/360047124832-downloadable-historical-ohlcvt-open-high-low-close-volume-trades-data +// If Kraken changes the Google Drive ids or stops publishing them there, update these constants. +const COMPLETE_OHLCVT_ARCHIVE_FILE_ID: &str = "1ptNqWYidLkhb2VAKuLCxmp2OXEfGO-AP"; +const QUARTERLY_OHLCVT_ARCHIVE_FOLDER_ID: &str = "15RSlNuW_h0kVM8or8McOGOMfHeBFvFGI"; +#[derive(Debug, Eq, PartialEq)] +pub struct CacheRatesArgs { + pub year: i32, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct DriveFile { + id: String, + name: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct PreparedArchive { + archive_path: PathBuf, + archive_file_name: String, + extracted_path: PathBuf, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct ArchiveCoverage { + first: i64, + last: i64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ArchiveBackfillMode { + Midpoint, +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +struct CacheWriteStats { + inserted: usize, + replaced: usize, + skipped: usize, +} + +impl CacheWriteStats { + fn record(&mut self, outcome: CacheWriteOutcome) { + match outcome { + CacheWriteOutcome::Inserted => self.inserted += 1, + CacheWriteOutcome::Replaced => self.replaced += 1, + CacheWriteOutcome::Skipped => self.skipped += 1, + } + } + + fn absorb(&mut self, other: CacheWriteStats) { + self.inserted += other.inserted; + self.replaced += other.replaced; + self.skipped += other.skipped; + } + + fn written(&self) -> usize { + self.inserted + self.replaced + } + + fn total(&self) -> usize { + self.inserted + self.replaced + self.skipped + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum CacheWriteOutcome { + Inserted, + Replaced, + Skipped, +} + +pub fn run(args: CacheRatesArgs) -> Result<()> { + let _temp_cache = TempCacheGuard::new()?; + let config = AppConfig::from_env()?; + let archive_pair = archive_pair_name(&config.kraken_pair)?; + let archive_mode = ArchiveBackfillMode::Midpoint; + let target_interval_minutes = target_interval_minutes(&config)?; + let now = Utc::now(); + let (start_ts, end_ts) = closed_interval_year_bounds(args.year, now, target_interval_minutes)?; + + let mut missing_starts = + expected_interval_starts(start_ts, end_ts, target_interval_minutes); + let mut cache = load_disk_cache(); + let existing_cache_count = count_cached_starts( + &missing_starts, + &cache, + &config.kraken_pair, + target_interval_minutes, + ); + if archive_mode == ArchiveBackfillMode::Midpoint { + drop_cached_starts( + &mut missing_starts, + &cache, + &config.kraken_pair, + target_interval_minutes, + ); + } + let quarterly_files = if !missing_starts.is_empty() && args.year >= QUARTERLY_ARCHIVE_FIRST_YEAR { + let archive_client = build_http_client("Kraken archive", None)?; + eprintln!( + "Fetching Kraken {} archive listing for {}...", + archive_mode.archive_label(), + args.year, + ); + Some(fetch_quarterly_archive_files( + &archive_client, + archive_mode.quarterly_archive_folder_id(), + archive_mode.quarterly_archive_prefix(), + archive_mode.archive_label(), + )?) + } else { + None + }; + let should_fetch_api = true; + + let mut recent_stats = CacheWriteStats::default(); + if should_fetch_api { + let kraken_client = build_http_client("clearnet Kraken", None)?; + eprintln!( + "Fetching Kraken OHLC API rows at {}-minute resolution for {}...", + target_interval_minutes, + args.year, + ); + let candles = fetch_candles_since( + &kraken_client, + &config, + target_interval_minutes, + start_ts, + )?; + + for candle in candles { + if candle.time < start_ts || candle.time >= end_ts { + continue; + } + if !should_store_api_candle(archive_mode, &missing_starts, candle.time) { + continue; + } + + recent_stats.record(store_cache_value( + &mut cache, + &config.kraken_pair, + target_interval_minutes, + candle.time, + candle.vwap, + )); + missing_starts.remove(&candle.time); + } + } + + let mut archive_stats = CacheWriteStats::default(); + let mut used_complete_archive_fallback = false; + if !missing_starts.is_empty() { + let archive_client = build_http_client("Kraken archive", None)?; + + let needed_quarters = missing_quarters(&missing_starts); + let quarterly_backfill_files = quarterly_files + .as_ref() + .and_then(|files| { + resolve_quarterly_archive_files( + files, + archive_mode.quarterly_archive_prefix(), + args.year, + &needed_quarters, + ) + }); + + if let Some(files) = quarterly_backfill_files { + for file in files { + let prepared_archive = prepare_archive_for_backfill( + &archive_client, + &file, + &archive_pair, + end_ts, + target_interval_minutes, + archive_mode, + )?; + archive_stats.absorb(read_prepared_archive( + &prepared_archive, + &file, + &config.kraken_pair, + &archive_pair, + start_ts, + end_ts, + target_interval_minutes, + None, + &mut missing_starts, + &mut cache, + archive_mode, + )?); + } + } else { + used_complete_archive_fallback = true; + let file = DriveFile { + id: archive_mode.complete_archive_file_id().to_owned(), + name: archive_mode.complete_archive_name().to_owned(), + }; + let prepared_archive = prepare_archive_for_backfill( + &archive_client, + &file, + &archive_pair, + end_ts, + target_interval_minutes, + archive_mode, + )?; + archive_stats.absorb(read_prepared_archive( + &prepared_archive, + &file, + &config.kraken_pair, + &archive_pair, + start_ts, + end_ts, + target_interval_minutes, + None, + &mut missing_starts, + &mut cache, + archive_mode, + )?); + } + } + + if !missing_starts.is_empty() { + let first_missing = missing_starts + .iter() + .next() + .copied() + .context("missing intervals should not be empty")?; + bail!( + "cache is incomplete for {}: {} {}-minute candle(s) still missing, starting at {}", + args.year, + missing_starts.len(), + target_interval_minutes, + format_interval_start(first_missing, target_interval_minutes)? + ); + } + + save_disk_cache(&cache)?; + + let inserted_count = recent_stats.inserted + archive_stats.inserted; + let replaced_count = recent_stats.replaced + archive_stats.replaced; + let skipped_count = existing_cache_count; + let total_count = inserted_count + replaced_count + skipped_count; + eprintln!( + "Cached {total_count} {}-minute rate(s) for {} in {}.", + target_interval_minutes, + args.year, + cache_path().display() + ); + eprintln!("Inserted cache entries: {inserted_count}"); + eprintln!("Replaced existing cache entries: {replaced_count}"); + eprintln!("Skipped existing cache entries: {skipped_count}"); + eprintln!("Kraken OHLC API rows: {}", recent_stats.written()); + eprintln!("OHLCVT archive midpoint rows: {}", archive_stats.written()); + if used_complete_archive_fallback && archive_stats.total() > 0 { + eprintln!( + "Quarterly {} archives were unavailable for {}, so the command fell back to the complete {} archive.", + archive_mode.archive_label(), + args.year, + archive_mode.archive_label(), + ); + } + if archive_stats.total() > 0 { + eprintln!( + "Archive-backed rows use the daily (open + close) / 2 midpoint because Kraken's OHLCVT CSV does not include VWAP." + ); + } + + Ok(()) +} + +pub fn parse_args_from(args: I, usage: &str) -> Result +where + I: IntoIterator, +{ + let mut year = None; + let mut args = args.into_iter(); + + while let Some(arg) = args.next() { + if year.is_some() { + bail!("{usage}"); + } + year = Some(arg); + } + + let year = year.ok_or_else(|| anyhow!("{usage}"))?; + let year = year + .parse::() + .with_context(|| format!("invalid year: {year}"))?; + + Ok(CacheRatesArgs { + year, + }) +} + +fn target_interval_minutes(_config: &AppConfig) -> Result { + Ok(KRAKEN_DAILY_INTERVAL_MINUTES) +} + +fn closed_interval_year_bounds( + year: i32, + now: chrono::DateTime, + interval_minutes: u32, +) -> Result<(i64, i64)> { + let year_start = NaiveDate::from_ymd_opt(year, 1, 1) + .ok_or_else(|| anyhow!("invalid year: {year}"))?; + let next_year_start = NaiveDate::from_ymd_opt(year + 1, 1, 1) + .ok_or_else(|| anyhow!("invalid year: {year}"))?; + let year_start_ts = midnight_utc_timestamp(year_start); + let next_year_start_ts = midnight_utc_timestamp(next_year_start); + let interval_seconds = i64::from(interval_minutes) * 60; + + if year > now.year() { + bail!("year {year} is in the future"); + } + + let year_end_exclusive = if year == now.year() { + now.timestamp().div_euclid(interval_seconds) * interval_seconds + } else { + next_year_start_ts + }; + + if year_start_ts >= year_end_exclusive { + bail!( + "year {year} has no closed UTC {}-minute candles yet", + interval_minutes + ); + } + + Ok((year_start_ts, year_end_exclusive.min(next_year_start_ts))) +} + +fn expected_interval_starts(start_ts: i64, end_ts: i64, interval_minutes: u32) -> BTreeSet { + let mut starts = BTreeSet::new(); + let interval_seconds = i64::from(interval_minutes) * 60; + let mut ts = start_ts; + while ts < end_ts { + starts.insert(ts); + ts += interval_seconds; + } + starts +} + +fn count_cached_starts( + starts: &BTreeSet, + cache: &HashMap, + kraken_pair: &str, + interval_minutes: u32, +) -> usize { + starts + .iter() + .filter(|start| cache.contains_key(&cache_key(kraken_pair, interval_minutes, **start))) + .count() +} + +fn should_store_api_candle( + _archive_mode: ArchiveBackfillMode, + missing_starts: &BTreeSet, + candle_start: i64, +) -> bool { + missing_starts.contains(&candle_start) +} + +fn drop_cached_starts( + missing_starts: &mut BTreeSet, + cache: &HashMap, + kraken_pair: &str, + interval_minutes: u32, +) { + missing_starts.retain(|start| { + !cache.contains_key(&cache_key(kraken_pair, interval_minutes, *start)) + }); +} + +fn store_cache_value( + cache: &mut HashMap, + kraken_pair: &str, + interval_minutes: u32, + timestamp: i64, + value: f64, +) -> CacheWriteOutcome { + let key = cache_key(kraken_pair, interval_minutes, timestamp); + let normalized = normalize_fiat_rate(value); + match cache.get(&key) { + Some(existing) if same_fiat_cent(*existing, value) => { + if *existing != normalized { + cache.insert(key, normalized); + } + CacheWriteOutcome::Skipped + } + Some(_) => { + cache.insert(key, normalized); + CacheWriteOutcome::Replaced + } + None => { + cache.insert(key, normalized); + CacheWriteOutcome::Inserted + } + } +} + +fn same_fiat_cent(left: f64, right: f64) -> bool { + round_fiat_cents(left) == round_fiat_cents(right) +} + +fn normalize_fiat_rate(value: f64) -> f64 { + round_fiat_cents(value) as f64 / 100.0 +} + +fn round_fiat_cents(value: f64) -> i64 { + (value * 100.0).round() as i64 +} + +fn missing_quarters(missing_days: &BTreeSet) -> BTreeSet { + let mut quarters = BTreeSet::new(); + for ts in missing_days { + let date = chrono::DateTime::from_timestamp(*ts, 0) + .expect("daily candle timestamp should be valid") + .date_naive(); + quarters.insert((date.month0() / 3) + 1); + } + quarters +} + +fn resolve_quarterly_archive_files( + quarterly_files: &HashMap, + file_prefix: &str, + year: i32, + needed_quarters: &BTreeSet, +) -> Option> { + let mut files = Vec::with_capacity(needed_quarters.len()); + + for quarter in needed_quarters { + let name = format!("{file_prefix}Q{quarter}_{year}.zip"); + files.push(quarterly_files.get(&name)?.clone()); + } + + Some(files) +} + +fn prepare_archive_for_backfill( + client: &Client, + file: &DriveFile, + archive_pair: &str, + end_ts: i64, + target_interval_minutes: u32, + archive_mode: ArchiveBackfillMode, +) -> Result { + let archive_path = archive_download_path(&file.name); + if let Some(parent) = archive_path.parent() { + fs::create_dir_all(parent)?; + } + + ensure_archive_downloaded( + client, + file, + &archive_path, + archive_mode, + archive_pair, + end_ts, + target_interval_minutes, + ) +} + +fn read_prepared_archive( + prepared_archive: &PreparedArchive, + _file: &DriveFile, + kraken_pair: &str, + _archive_pair: &str, + start_ts: i64, + end_ts: i64, + target_interval_minutes: u32, + trade_source_interval_minutes: Option, + missing_starts: &mut BTreeSet, + cache: &mut HashMap, + archive_mode: ArchiveBackfillMode, +) -> Result { + eprintln!( + "Reading extracted {} data from {}...", + archive_mode.archive_label(), + prepared_archive.extracted_path.display(), + ); + read_archive_zip( + &prepared_archive.extracted_path, + kraken_pair, + start_ts, + end_ts, + target_interval_minutes, + trade_source_interval_minutes, + missing_starts, + cache, + archive_mode, + ) +} + +fn read_archive_zip( + path: &Path, + kraken_pair: &str, + start_ts: i64, + end_ts: i64, + target_interval_minutes: u32, + _trade_source_interval_minutes: Option, + missing_starts: &mut BTreeSet, + cache: &mut HashMap, + archive_mode: ArchiveBackfillMode, +) -> Result { + let csv_file = File::open(path) + .with_context(|| format!("failed to open extracted archive data {}", path.display()))?; + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(csv_file); + let entry_name = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("archive data"); + + match archive_mode { + ArchiveBackfillMode::Midpoint => read_ohlcvt_daily_csv( + &mut reader, + &entry_name, + kraken_pair, + start_ts, + end_ts, + target_interval_minutes, + missing_starts, + cache, + ), + } +} + +fn ensure_archive_entry_extracted( + archive_path: &Path, + archive_file_name: &str, + archive_pair: &str, + archive_mode: ArchiveBackfillMode, +) -> Result { + let expected_entry_name = archive_mode.entry_name(archive_pair); + let extracted_path = extracted_archive_entry_path(archive_file_name, &expected_entry_name); + if extracted_path.exists() { + return Ok(extracted_path); + } + + if let Some(parent) = extracted_path.parent() { + fs::create_dir_all(parent)?; + } + + let archive_file = File::open(archive_path) + .with_context(|| format!("failed to open archive {}", archive_path.display()))?; + let mut archive = ZipArchive::new(archive_file).with_context(|| { + format!("failed to read ZIP archive {}", archive_path.display()) + })?; + let entry_name = resolve_archive_entry_name(&mut archive, &expected_entry_name)?; + eprintln!( + "Extracting {} from {} into {}...", + entry_name, + archive_path.display(), + temp_cache_dir().display(), + ); + let mut csv_file = archive + .by_name(&entry_name) + .with_context(|| format!("archive does not contain {entry_name}"))?; + let temp_path = temp_extracted_archive_entry_path(archive_file_name, &expected_entry_name); + if let Some(parent) = temp_path.parent() { + fs::create_dir_all(parent)?; + } + let mut output = File::create(&temp_path) + .with_context(|| format!("failed to create {}", temp_path.display()))?; + io::copy(&mut csv_file, &mut output) + .with_context(|| format!("failed to write {}", temp_path.display()))?; + fs::rename(&temp_path, &extracted_path) + .with_context(|| format!("failed to move extracted data into {}", extracted_path.display()))?; + Ok(extracted_path) +} + +fn extracted_csv_timestamp_bounds(path: &Path, entry_name: &str) -> Result> { + let csv_file = File::open(path) + .with_context(|| format!("failed to open extracted archive data {}", path.display()))?; + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(csv_file); + let mut first = None; + let mut last = None; + + for record in reader.records() { + let record = record.with_context(|| format!("failed to parse a row in {entry_name}"))?; + let timestamp = parse_archive_timestamp(&record, &entry_name)?; + first.get_or_insert(timestamp); + last = Some(timestamp); + } + + Ok(match (first, last) { + (Some(first), Some(last)) => Some(ArchiveCoverage { first, last }), + _ => None, + }) +} + +fn resolve_archive_entry_name( + archive: &mut ZipArchive, + expected_entry_name: &str, +) -> Result { + if archive.by_name(expected_entry_name).is_ok() { + return Ok(expected_entry_name.to_owned()); + } + + let suffix = format!("/{expected_entry_name}"); + let mut matches = Vec::new(); + for i in 0..archive.len() { + let name = archive + .by_index(i) + .with_context(|| format!("failed to inspect ZIP entry #{i}"))? + .name() + .to_owned(); + if name.ends_with(&suffix) { + matches.push(name); + } + } + + match matches.len() { + 1 => Ok(matches.remove(0)), + 0 => bail!("archive does not contain {expected_entry_name}"), + _ => bail!( + "archive contains multiple entries matching {expected_entry_name}: {}", + matches.join(", ") + ), + } +} + +fn read_ohlcvt_daily_csv( + reader: &mut csv::Reader, + entry_name: &str, + kraken_pair: &str, + start_ts: i64, + end_ts: i64, + target_interval_minutes: u32, + missing_starts: &mut BTreeSet, + cache: &mut HashMap, +) -> Result { + let mut stats = CacheWriteStats::default(); + let mut last_logged_year = None; + + for record in reader.records() { + let record = record.with_context(|| format!("failed to parse a row in {entry_name}"))?; + let timestamp = parse_archive_timestamp(&record, entry_name)?; + if timestamp < start_ts || timestamp >= end_ts || !missing_starts.contains(×tamp) { + continue; + } + log_archive_year_progress( + timestamp, + &mut last_logged_year, + "OHLCVT archive", + KRAKEN_DAILY_INTERVAL_MINUTES, + )?; + + let midpoint = parse_archive_midpoint(&record, entry_name)?; + // The OHLCVT archive lacks VWAP, so we store the UTC daily (open + close) / 2 midpoint + // under the normal 1440-minute cache key as a simple per-day price proxy. + stats.record(store_cache_value( + cache, + kraken_pair, + target_interval_minutes, + timestamp, + midpoint, + )); + missing_starts.remove(×tamp); + } + + Ok(stats) +} + +fn log_archive_year_progress( + timestamp: i64, + last_logged_year: &mut Option, + label: &str, + interval_minutes: u32, +) -> Result<()> { + let datetime = chrono::DateTime::from_timestamp(timestamp, 0) + .context("invalid archive progress timestamp")?; + let year = datetime.year(); + if *last_logged_year == Some(year) { + return Ok(()); + } + + *last_logged_year = Some(year); + if interval_minutes == KRAKEN_DAILY_INTERVAL_MINUTES { + eprintln!("Processing {label} rows for {year}..."); + } else { + eprintln!( + "Processing {label} rows for {year} at {}-minute resolution...", + interval_minutes + ); + } + Ok(()) +} + +fn parse_archive_timestamp(record: &StringRecord, entry_name: &str) -> Result { + let value = record + .get(0) + .ok_or_else(|| anyhow!("{entry_name} row is missing the timestamp column"))?; + value + .parse::() + .with_context(|| format!("invalid timestamp {value} in {entry_name}")) +} + +fn parse_archive_midpoint(record: &StringRecord, entry_name: &str) -> Result { + let open = parse_archive_number(record, 1, "open", entry_name)?; + let close = parse_archive_number(record, 4, "close", entry_name)?; + Ok((open + close) / 2.0) +} + +fn parse_archive_number( + record: &StringRecord, + column: usize, + label: &str, + entry_name: &str, +) -> Result { + let value = record + .get(column) + .ok_or_else(|| anyhow!("{entry_name} row is missing the {label} column"))?; + value + .parse::() + .with_context(|| format!("invalid {label} value {value} in {entry_name}")) +} + +fn fetch_quarterly_archive_files( + client: &Client, + folder_id: &str, + file_prefix: &str, + archive_label: &str, +) -> Result> { + let url = format!("https://drive.google.com/drive/folders/{folder_id}?usp=sharing"); + let html = client + .get(&url) + .send() + .with_context(|| format!("failed to fetch {url}"))? + .error_for_status() + .with_context(|| format!("Google Drive returned an error for {url}"))? + .text() + .with_context(|| format!("failed to decode Google Drive folder page at {url}"))?; + + extract_quarterly_drive_files(&html, file_prefix, archive_label) +} + +fn extract_quarterly_drive_files( + html: &str, + file_prefix: &str, + archive_label: &str, +) -> Result> { + let blob = extract_drive_ivd_blob(html) + .context("failed to locate the Google Drive file listing blob")?; + let mut files = HashMap::new(); + let mut cursor = 0usize; + + while let Some(relative_idx) = blob[cursor..].find(file_prefix) { + let name_start = cursor + relative_idx; + let name_end = blob[name_start..] + .find(".zip") + .map(|idx| name_start + idx + 4) + .ok_or_else(|| anyhow!("failed to parse a quarterly {archive_label} archive filename from Google Drive"))?; + let name = blob[name_start..name_end].to_owned(); + + let suffix = &blob[name_end..]; + let prefix = &blob[..name_start]; + let id = extract_id_between( + suffix, + "https:\\/\\/drive.google.com\\/file\\/d\\/", + "\\/view", + ) + .or_else(|| { + extract_last_id_between( + prefix, + "https:\\/\\/drive.google.com\\/file\\/d\\/", + "\\/view", + ) + }) + .with_context(|| format!("failed to find the Google Drive file id for {name}"))?; + + files.insert(name.clone(), DriveFile { id, name }); + cursor = name_end; + } + + if files.is_empty() { + bail!("failed to parse quarterly {archive_label} archive files from Google Drive"); + } + + Ok(files) +} + +fn extract_drive_ivd_blob(html: &str) -> Option<&str> { + let start_marker = "window['_DRIVE_ivd'] = '"; + let end_marker = "';if (window['_DRIVE_ivdc'])"; + let start = html.find(start_marker)? + start_marker.len(); + let rest = &html[start..]; + let end = rest.find(end_marker)?; + Some(&rest[..end]) +} + +fn extract_id_between(haystack: &str, marker: &str, terminator: &str) -> Option { + let start = haystack.find(marker)? + marker.len(); + let rest = &haystack[start..]; + let end = rest.find(terminator)?; + Some(rest[..end].to_owned()) +} + +fn extract_last_id_between(haystack: &str, marker: &str, terminator: &str) -> Option { + let start = haystack.rfind(marker)? + marker.len(); + let rest = &haystack[start..]; + let end = rest.find(terminator)?; + Some(rest[..end].to_owned()) +} + +fn download_google_drive_file(client: &Client, file_id: &str, destination: &Path) -> Result<()> { + let initial_url = format!("https://drive.google.com/uc?export=download&id={file_id}&confirm=t"); + let response = client + .get(&initial_url) + .send() + .with_context(|| format!("failed to download Google Drive file {file_id}"))?; + + if is_html_response(&response) { + let html = response + .text() + .context("failed to decode the Google Drive confirmation page")?; + let action = extract_html_attribute(&html, "

Result { + let mut existing_coverage = None; + if destination.exists() { + if file.name == archive_mode.complete_archive_name() { + let extracted_path = + ensure_archive_entry_extracted(destination, &file.name, archive_pair, archive_mode)?; + let coverage = inspect_archive_coverage( + &extracted_path, + &format!("cached extracted {archive_pair} data"), + KRAKEN_DAILY_INTERVAL_MINUTES, + )?; + existing_coverage = Some(coverage); + let needed_start = needed_archive_start(end_ts, target_interval_minutes); + if coverage.last >= needed_start { + let size = archive_file_size_label(destination); + eprintln!( + "Reusing cached {} at {}{}.", + archive_file_description(file, archive_mode), + destination.display(), + size.as_deref().unwrap_or(""), + ); + return Ok(PreparedArchive { + archive_path: destination.to_path_buf(), + archive_file_name: file.name.clone(), + extracted_path, + }); + } + eprintln!( + "Cached {} at {} does not appear to reach the requested {} data. Redownloading...", + archive_file_description(file, archive_mode), + destination.display(), + format_interval_start(needed_start, target_interval_minutes)?, + ); + clear_temp_cache_dir()?; + } else { + let size = archive_file_size_label(destination); + eprintln!( + "Reusing cached {} at {}{}.", + archive_file_description(file, archive_mode), + destination.display(), + size.as_deref().unwrap_or(""), + ); + return Ok(PreparedArchive { + archive_path: destination.to_path_buf(), + archive_file_name: file.name.clone(), + extracted_path: ensure_archive_entry_extracted( + destination, + &file.name, + archive_pair, + archive_mode, + )?, + }); + } + } + + let temp_path = temp_download_path(&file.name); + if let Some(parent) = temp_path.parent() { + fs::create_dir_all(parent)?; + } + + let result = (|| -> Result { + let size_hint = archive_file_size_label(destination); + eprintln!( + "Downloading {} to {}{}...", + archive_file_description(file, archive_mode), + destination.display(), + size_hint.as_deref().unwrap_or(""), + ); + download_google_drive_file(client, &file.id, &temp_path)?; + if file.name == archive_mode.complete_archive_name() { + let extracted_path = + ensure_archive_entry_extracted(&temp_path, &file.name, archive_pair, archive_mode)?; + let downloaded_coverage = inspect_archive_coverage( + &extracted_path, + &format!("downloaded extracted {archive_pair} data"), + KRAKEN_DAILY_INTERVAL_MINUTES, + )?; + let needed_start = needed_archive_start(end_ts, target_interval_minutes); + if downloaded_coverage.last < needed_start { + bail!( + "downloaded complete archive does not reach the requested {} data; keeping existing cached archive at {}", + format_interval_start(needed_start, target_interval_minutes)?, + destination.display() + ); + } + if !accept_complete_archive_replacement(existing_coverage, downloaded_coverage) { + eprintln!( + "Using downloaded complete archive for this run and keeping it at {}, while preserving cached {} at {} because the new archive would shrink prior coverage.", + temp_path.display(), + archive_mode.archive_label(), + destination.display(), + ); + return Ok(PreparedArchive { + archive_path: temp_path.clone(), + archive_file_name: file.name.clone(), + extracted_path, + }); + } + } + fs::rename(&temp_path, destination).with_context(|| { + format!( + "failed to move downloaded archive into cache at {}", + destination.display() + ) + })?; + eprintln!( + "Saved {} to {}{}.", + archive_file_description(file, archive_mode), + destination.display(), + archive_file_size_label(destination).as_deref().unwrap_or(""), + ); + let extracted_path = + ensure_archive_entry_extracted(destination, &file.name, archive_pair, archive_mode)?; + Ok(PreparedArchive { + archive_path: destination.to_path_buf(), + archive_file_name: file.name.clone(), + extracted_path, + }) + })(); + + if result.is_err() { + let _ = fs::remove_file(&temp_path); + } + + result +} + +fn inspect_archive_coverage( + path: &Path, + label: &str, + target_interval_minutes: u32, +) -> Result { + eprintln!("Scanning coverage from {} at {}...", label, path.display()); + let coverage = extracted_csv_timestamp_bounds(path, label)? + .context("archive pair CSV is empty")?; + eprintln!( + "{} appears to cover {} through {}.", + label, + format_interval_start(coverage.first, target_interval_minutes)?, + format_interval_start(coverage.last, target_interval_minutes)?, + ); + Ok(coverage) +} + +fn accept_complete_archive_replacement( + existing_coverage: Option, + downloaded_coverage: ArchiveCoverage, +) -> bool { + match existing_coverage { + Some(existing) => { + downloaded_coverage.first <= existing.first + && downloaded_coverage.last >= existing.last + } + None => true, + } +} + +fn needed_archive_start(end_ts: i64, target_interval_minutes: u32) -> i64 { + end_ts - i64::from(target_interval_minutes) * 60 +} + +fn archive_file_description(file: &DriveFile, archive_mode: ArchiveBackfillMode) -> String { + if file.name == archive_mode.complete_archive_name() { + format!( + "Kraken complete {} archive (all years)", + archive_mode.archive_label() + ) + } else { + format!("Kraken {} archive {}", archive_mode.archive_label(), file.name) + } +} + +fn archive_file_size_label(path: &Path) -> Option { + let bytes = fs::metadata(path).ok()?.len(); + Some(format!(" ({})", format_byte_size(bytes))) +} + +fn format_byte_size(bytes: u64) -> String { + const UNITS: [&str; 5] = ["B", "KB", "MB", "GB", "TB"]; + let mut value = bytes as f64; + let mut unit_index = 0usize; + + while value >= 1024.0 && unit_index < UNITS.len() - 1 { + value /= 1024.0; + unit_index += 1; + } + + if unit_index == 0 { + format!("{} {}", bytes, UNITS[unit_index]) + } else { + format!("{value:.1} {}", UNITS[unit_index]) + } +} + +fn is_html_response(response: &Response) -> bool { + response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(|value| value.starts_with("text/html")) + .unwrap_or(false) +} + +fn write_response_to_file(response: Response, destination: &Path) -> Result<()> { + let mut response = response.error_for_status()?; + let mut file = File::create(destination) + .with_context(|| format!("failed to create {}", destination.display()))?; + io::copy(&mut response, &mut file) + .with_context(|| format!("failed to write {}", destination.display()))?; + Ok(()) +} + +fn extract_hidden_input_value(html: &str, name: &str) -> Option { + let marker = format!("name=\"{name}\" value=\""); + let start = html.find(&marker)? + marker.len(); + let rest = &html[start..]; + let end = rest.find('"')?; + Some(rest[..end].to_owned()) +} + +fn extract_html_attribute(html: &str, element_prefix: &str, attribute: &str) -> Option { + let element_start = html.find(element_prefix)?; + let element = &html[element_start..]; + let marker = format!("{attribute}=\""); + let attr_start = element.find(&marker)? + marker.len(); + let rest = &element[attr_start..]; + let attr_end = rest.find('"')?; + Some(rest[..attr_end].to_owned()) +} + +fn archive_pair_name(kraken_pair: &str) -> Result { + if let Some(quote) = kraken_pair.strip_prefix("XXBTZ") { + return Ok(format!("XBT{quote}")); + } + + if let Some(quote) = kraken_pair.strip_prefix("XXBT") { + return Ok(format!("XBT{}", quote.trim_start_matches('Z'))); + } + + if kraken_pair.starts_with("XBT") { + return Ok(kraken_pair.to_owned()); + } + + bail!("cache-rates only supports Kraken XBT quote pairs such as XXBTZUSD or XXBTZEUR") +} + +fn midnight_utc_timestamp(date: NaiveDate) -> i64 { + date.and_hms_opt(0, 0, 0) + .expect("midnight should always be a valid time") + .and_utc() + .timestamp() +} + +fn format_interval_start(timestamp: i64, interval_minutes: u32) -> Result { + let datetime = chrono::DateTime::from_timestamp(timestamp, 0) + .context("invalid interval start timestamp")?; + if interval_minutes == KRAKEN_DAILY_INTERVAL_MINUTES { + Ok(datetime.date_naive().to_string()) + } else { + Ok(datetime.to_rfc3339()) + } +} + +fn archive_download_path(file_name: &str) -> PathBuf { + PathBuf::from(CACHE_DIR).join("kraken").join(file_name) +} + +fn temp_download_path(file_name: &str) -> PathBuf { + temp_cache_dir().join(format!("download-{}-{}", std::process::id(), file_name)) +} + +fn extracted_archive_entry_path(archive_file_name: &str, entry_name: &str) -> PathBuf { + temp_cache_dir().join(format!( + "{}--{}", + sanitize_temp_component(archive_file_name), + sanitize_temp_component(entry_name) + )) +} + +fn temp_extracted_archive_entry_path(archive_file_name: &str, entry_name: &str) -> PathBuf { + temp_cache_dir().join(format!( + "extract-{}-{}--{}", + std::process::id(), + sanitize_temp_component(archive_file_name), + sanitize_temp_component(entry_name) + )) +} + +fn temp_cache_dir() -> PathBuf { + PathBuf::from(CACHE_DIR).join("tmp") +} + +fn clear_temp_cache_dir() -> Result<()> { + let path = temp_cache_dir(); + if path.exists() { + fs::remove_dir_all(&path) + .with_context(|| format!("failed to remove {}", path.display()))?; + } + fs::create_dir_all(&path).with_context(|| format!("failed to create {}", path.display()))?; + Ok(()) +} + +fn sanitize_temp_component(value: &str) -> String { + value + .chars() + .map(|ch| match ch { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '.' | '-' | '_' => ch, + _ => '_', + }) + .collect() +} + +struct TempCacheGuard { + path: PathBuf, +} + +impl TempCacheGuard { + fn new() -> Result { + let path = temp_cache_dir(); + clear_temp_cache_dir()?; + Ok(Self { path }) + } +} + +impl Drop for TempCacheGuard { + fn drop(&mut self) { + if self.path.exists() { + let _ = fs::remove_dir_all(&self.path); + eprintln!("Deleted temporary cache directory {}.", self.path.display()); + } + } +} + +impl ArchiveBackfillMode { + fn archive_label(self) -> &'static str { + match self { + Self::Midpoint => "OHLCVT", + } + } + + fn quarterly_archive_prefix(self) -> &'static str { + match self { + Self::Midpoint => "Kraken_OHLCVT_", + } + } + + fn quarterly_archive_folder_id(self) -> &'static str { + match self { + Self::Midpoint => QUARTERLY_OHLCVT_ARCHIVE_FOLDER_ID, + } + } + + fn complete_archive_file_id(self) -> &'static str { + match self { + Self::Midpoint => COMPLETE_OHLCVT_ARCHIVE_FILE_ID, + } + } + + fn complete_archive_name(self) -> &'static str { + match self { + Self::Midpoint => "Kraken_OHLCVT.zip", + } + } + + fn entry_name(self, archive_pair: &str) -> String { + match self { + Self::Midpoint => format!("{archive_pair}_1440.csv"), + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::{BTreeSet, HashMap}; + use std::io::Cursor; + use std::path::PathBuf; + + use super::{ + ArchiveCoverage, CacheRatesArgs, CacheWriteOutcome, + accept_complete_archive_replacement, + archive_download_path, archive_pair_name, + extract_quarterly_drive_files, parse_args_from, + read_ohlcvt_daily_csv, resolve_archive_entry_name, + should_store_api_candle, store_cache_value, + }; + + const REAL_XBTEUR_1440_SAMPLE: &str = "\ +1672531200,15423.8,15524.8,15388.5,15512.9,532.29189029,11775 +1672617600,15513.0,15706.6,15455.0,15600.4,1161.60737247,21237 +1672704000,15599.1,15892.9,15580.5,15795.1,1465.78622353,24222 +"; + const REAL_XBTEUR_1440_OVERLAP_ROW: &str = + "1711929600,66130.0,66130.0,63461.9,64902.9,528.8081404,23715"; + + #[test] + fn parses_cache_rates_args() { + let args = parse_args_from( + vec!["2024".to_owned()], + "usage: btc_fiat_value cache-rates ", + ) + .expect("args"); + assert_eq!(args, CacheRatesArgs { year: 2024 }); + } + + #[test] + fn rejects_extra_cache_rates_args() { + let err = parse_args_from( + vec!["2024".to_owned(), "extra".to_owned()], + "usage: btc_fiat_value cache-rates ", + ) + .expect_err("should fail"); + + assert!(err + .to_string() + .contains("usage: btc_fiat_value cache-rates ")); + } + + #[test] + fn stores_archives_under_cache_kraken_directory() { + assert_eq!( + archive_download_path("Kraken_Trading_History.zip"), + PathBuf::from(".cache/kraken/Kraken_Trading_History.zip") + ); + } + + #[test] + fn midpoint_mode_skips_existing_cache_entries() { + let mut missing_days = BTreeSet::from([1672531200_i64, 1672617600_i64]); + let cache = HashMap::from([("XXBTZEUR:1440:1672531200".to_owned(), 12345.0_f64)]); + + super::drop_cached_starts(&mut missing_days, &cache, "XXBTZEUR", 1440); + + assert_eq!(missing_days, BTreeSet::from([1672617600_i64])); + } + + #[test] + fn midpoint_mode_skips_api_candles_for_existing_cache_entries() { + let missing_days = BTreeSet::from([1672617600_i64]); + + assert!(!should_store_api_candle( + super::ArchiveBackfillMode::Midpoint, + &missing_days, + 1672531200, + )); + assert!(should_store_api_candle( + super::ArchiveBackfillMode::Midpoint, + &missing_days, + 1672617600, + )); + } + + #[test] + fn store_cache_value_skips_unchanged_entries() { + let mut cache = HashMap::from([("XXBTZEUR:60:1735689600".to_owned(), 107.5_f64)]); + + let outcome = store_cache_value(&mut cache, "XXBTZEUR", 60, 1735689600, 107.5); + + assert_eq!(outcome, CacheWriteOutcome::Skipped); + assert_eq!(cache["XXBTZEUR:60:1735689600"], 107.5); + } + + #[test] + fn store_cache_value_skips_same_cent_drift() { + let mut cache = HashMap::from([("XXBTZEUR:60:1735689600".to_owned(), 107.504_f64)]); + + let outcome = store_cache_value(&mut cache, "XXBTZEUR", 60, 1735689600, 107.495); + + assert_eq!(outcome, CacheWriteOutcome::Skipped); + assert_eq!(cache["XXBTZEUR:60:1735689600"], 107.5); + } + + #[test] + fn normalizes_xbt_archive_pair_names() { + assert_eq!(archive_pair_name("XXBTZUSD").unwrap(), "XBTUSD"); + assert_eq!(archive_pair_name("XXBTZEUR").unwrap(), "XBTEUR"); + assert_eq!(archive_pair_name("XBTUSDT").unwrap(), "XBTUSDT"); + } + + #[test] + fn extracts_quarterly_file_ids_from_drive_html() { + let html = concat!( + "window['_DRIVE_ivd'] = '", + "Kraken_OHLCVT_Q1_2024.zip ", + "https:\\/\\/drive.google.com\\/file\\/d\\/abc123\\/view ", + "Kraken_OHLCVT_Q2_2024.zip ", + "https:\\/\\/drive.google.com\\/file\\/d\\/def456\\/view", + "';if (window['_DRIVE_ivdc'])" + ); + + let files = extract_quarterly_drive_files(html, "Kraken_OHLCVT_", "OHLCVT") + .expect("files"); + assert_eq!(files["Kraken_OHLCVT_Q1_2024.zip"].id, "abc123"); + assert_eq!(files["Kraken_OHLCVT_Q2_2024.zip"].id, "def456"); + } + + #[test] + fn reads_real_archive_daily_csv_rows_into_cache() { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new(REAL_XBTEUR_1440_SAMPLE)); + let mut missing_days = BTreeSet::from([1672531200_i64, 1672617600_i64]); + let mut cache = HashMap::new(); + + let stats = read_ohlcvt_daily_csv( + &mut reader, + "XBTEUR_1440.csv", + "XXBTZEUR", + 1672531200, + 1672704000, + 1440, + &mut missing_days, + &mut cache, + ) + .expect("csv should parse"); + + assert_eq!(stats.inserted, 2); + assert_eq!(stats.replaced, 0); + assert_eq!(stats.skipped, 0); + assert_eq!( + cache["XXBTZEUR:1440:1672531200"], + super::normalize_fiat_rate((15423.8 + 15512.9) / 2.0) + ); + assert_eq!( + cache["XXBTZEUR:1440:1672617600"], + super::normalize_fiat_rate((15513.0 + 15600.4) / 2.0) + ); + assert!(missing_days.is_empty()); + } + + #[test] + fn archive_ohlc_matches_live_overlap_day_fields() { + let record = csv::StringRecord::from( + REAL_XBTEUR_1440_OVERLAP_ROW + .split(',') + .collect::>(), + ); + + // Sanity check against the live Kraken 1440 OHLC API on 2024-04-01 UTC: + // the daily candle boundaries and OHLC fields match the archive row. + assert_eq!(super::parse_archive_timestamp(&record, "XBTEUR_1440.csv").unwrap(), 1711929600); + assert_eq!( + super::parse_archive_number(&record, 1, "open", "XBTEUR_1440.csv").unwrap(), + 66130.0 + ); + assert_eq!( + super::parse_archive_number(&record, 2, "high", "XBTEUR_1440.csv").unwrap(), + 66130.0 + ); + assert_eq!( + super::parse_archive_number(&record, 3, "low", "XBTEUR_1440.csv").unwrap(), + 63461.9 + ); + assert_eq!( + super::parse_archive_number(&record, 4, "close", "XBTEUR_1440.csv").unwrap(), + 64902.9 + ); + } + + #[test] + fn reports_missing_close_column_clearly() { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new("1672531200,15423.8,15524.8,15388.5\n")); + let mut missing_days = BTreeSet::from([1672531200_i64]); + let mut cache = HashMap::new(); + + let err = read_ohlcvt_daily_csv( + &mut reader, + "XBTEUR_1440.csv", + "XXBTZEUR", + 1672531200, + 1672617600, + 1440, + &mut missing_days, + &mut cache, + ) + .expect_err("row should fail"); + + assert!(err.to_string().contains("XBTEUR_1440.csv row is missing the close column")); + } + + #[test] + fn reports_invalid_close_price_clearly() { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new("1672531200,15423.8,15524.8,15388.5,nope,532.29189029,11775\n")); + let mut missing_days = BTreeSet::from([1672531200_i64]); + let mut cache = HashMap::new(); + + let err = read_ohlcvt_daily_csv( + &mut reader, + "XBTEUR_1440.csv", + "XXBTZEUR", + 1672531200, + 1672617600, + 1440, + &mut missing_days, + &mut cache, + ) + .expect_err("row should fail"); + + assert!(err + .to_string() + .contains("invalid close value nope in XBTEUR_1440.csv")); + } + + #[test] + fn rejects_complete_archive_replacement_that_drops_old_coverage() { + let existing = ArchiveCoverage { + first: 1_672_531_200, // 2023-01-01 + last: 1_735_689_600, // 2025-01-01 + }; + let downloaded = ArchiveCoverage { + first: 1_704_067_200, // 2024-01-01 + last: 1_767_225_600, // 2026-01-01 + }; + + assert!(!accept_complete_archive_replacement( + Some(existing), + downloaded, + )); + } + + #[test] + fn accepts_complete_archive_for_current_run_when_it_reaches_needed_range() { + let existing = ArchiveCoverage { + first: 1_672_531_200, // 2023-01-01 + last: 1_704_067_200, // 2024-01-01 + }; + let downloaded = ArchiveCoverage { + first: 1_704_067_200, // 2024-01-01 + last: 1_735_689_600, // 2025-01-01 + }; + + assert!(!accept_complete_archive_replacement( + Some(existing), + downloaded, + )); + assert!(downloaded.last >= 1_735_689_600); + } + + #[test] + fn resolves_wrapped_archive_entry_by_suffix() { + let cursor = Cursor::new(Vec::::new()); + let mut writer = zip::ZipWriter::new(cursor); + let options: zip::write::SimpleFileOptions = zip::write::SimpleFileOptions::default(); + writer + .start_file("TimeAndSales_Combined/XBTEUR.csv", options) + .expect("start file"); + std::io::Write::write_all(&mut writer, b"1735689600,90167.3,0.1\n").expect("write file"); + let cursor = writer.finish().expect("finish zip"); + + let mut archive = zip::ZipArchive::new(Cursor::new(cursor.into_inner())).expect("archive"); + let resolved = resolve_archive_entry_name(&mut archive, "XBTEUR.csv").expect("resolve"); + + assert_eq!(resolved, "TimeAndSales_Combined/XBTEUR.csv"); + } +} diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 4d2b414..26b3343 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -1,3 +1,4 @@ +pub mod cache_rates; pub mod export; pub mod received_value; pub mod reconstruct; diff --git a/src/main.rs b/src/main.rs index 58338e8..f64bf6c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,11 +3,12 @@ use std::env; use anyhow::{Result, anyhow, bail}; use dotenvy::dotenv; +use btc_fiat_value::commands::cache_rates::{self as cache_rates_cmd, CacheRatesArgs}; use btc_fiat_value::commands::export::{self as export_cmd, ExportArgs}; use btc_fiat_value::commands::received_value::{self, ReceivedValueArgs}; use btc_fiat_value::commands::reconstruct::{self as reconstruct_cmd, ReconstructArgs}; -const ROOT_USAGE: &str = "usage: btc_fiat_value [options]\n\nsubcommands:\n received-value find the quote-currency value when BTC was received\n export export wallet transactions to accounting format\n reconstruct verify an export by reconstructing the wallet"; +const ROOT_USAGE: &str = "usage: btc_fiat_value [options]\n\nsubcommands:\n received-value find the quote-currency value when BTC was received\n cache-rates populate .cache/rates.json for one year of rates\n export export wallet transactions to accounting format\n reconstruct verify an export by reconstructing the wallet"; fn main() { if let Err(err) = run() { @@ -20,6 +21,7 @@ fn run() -> Result<()> { let _ = dotenv(); match parse_command()? { Command::ReceivedValue(args) => received_value::run(args), + Command::CacheRates(args) => cache_rates_cmd::run(args), Command::Export(args) => export_cmd::run(args), Command::Reconstruct(args) => reconstruct_cmd::run(args), } @@ -28,6 +30,7 @@ fn run() -> Result<()> { #[derive(Debug)] enum Command { ReceivedValue(ReceivedValueArgs), + CacheRates(CacheRatesArgs), Export(ExportArgs), Reconstruct(ReconstructArgs), } @@ -47,6 +50,9 @@ where received_value::SUBCOMMAND_NAME => Ok(Command::ReceivedValue( received_value::parse_args_from(args, received_value::USAGE)?, )), + cache_rates_cmd::SUBCOMMAND_NAME => Ok(Command::CacheRates( + cache_rates_cmd::parse_args_from(args, cache_rates_cmd::USAGE)?, + )), export_cmd::SUBCOMMAND_NAME => Ok(Command::Export( export_cmd::parse_args_from(args, export_cmd::USAGE)?, )), @@ -107,6 +113,19 @@ mod tests { } } + #[test] + fn parses_cache_rates_subcommand() { + let command = parse_command_from(vec!["cache-rates".to_owned(), "2024".to_owned()]) + .expect("command"); + + match command { + Command::CacheRates(args) => { + assert_eq!(args.year, 2024); + } + _ => panic!("expected CacheRates"), + } + } + #[test] fn parses_reconstruct_subcommand() { let command = parse_command_from(vec![ From 6e04d5cdced468bca1020cb44c1b49a040b58271 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Mon, 23 Mar 2026 13:11:21 +0100 Subject: [PATCH 5/8] Add --vwap trade archive backfill to cache-rates --- README.md | 25 +- src/commands/cache_rates.rs | 853 +++++++++++++++++- src/main.rs | 42 + .../kraken_xbteur_2026-03-21_trades.csv | 767 ++++++++++++++++ 4 files changed, 1652 insertions(+), 35 deletions(-) create mode 100644 tests/fixtures/kraken_xbteur_2026-03-21_trades.csv diff --git a/README.md b/README.md index 6d932af..1c3c210 100644 --- a/README.md +++ b/README.md @@ -24,22 +24,33 @@ Populate `.cache/rates.json` for one UTC calendar year of Bitcoin rates. ```bash cargo run -- cache-rates 2024 +cargo run -- cache-rates --vwap 2024 +cargo run -- cache-rates --vwap --candle 60 2024 ``` The command works as follows: -1. Fetches the daily (`1440`) Kraken `OHLC` data that is still available through the public API. -2. Detects which closed UTC daily candles in the requested year are still missing because they are older than Kraken's 720-candle retention window. -3. Backfills those missing days into `.cache/rates.json` from Kraken's downloadable OHLCVT archive. +1. Without `--vwap`, fetches the daily (`1440`) Kraken `OHLC` data that is still available through the public API. +2. Detects which closed UTC candles in the requested year are still missing because they are older than Kraken's 720-candle retention window. +3. Backfills those missing candles into `.cache/rates.json` from one of Kraken's downloadable archives: + the default OHLCVT archive, or the larger time-and-sales archive when `--vwap` is set. -The cache keys are written as `1440`-minute entries. +By default, the cache keys are written as normal `1440`-minute entries. With `--vwap`, the command writes entries at `DEFAULT_CANDLE_MINUTES` if configured, or `1440` otherwise; `--candle ` overrides that in the normal way. Trade-off: - Recent days use Kraken's daily `OHLC` API `vwap`, just like the normal live lookup path. -- Older archive-backed days use the daily `(open + close) / 2` midpoint derived from Kraken's OHLCVT CSV, because the downloadable OHLCVT archive does not include `vwap`. -- The command only fills missing cache entries for that year and interval; it does not overwrite existing entries with midpoint-derived values. +- Without `--vwap`, older archive-backed days use the daily `(open + close) / 2` midpoint derived from Kraken's OHLCVT CSV, because the downloadable OHLCVT archive does not include `vwap`. +- Without `--vwap`, the command only fills missing cache entries for that year and interval; it does not overwrite existing entries with midpoint-derived values. +- With `--vwap`, the command computes exact Kraken VWAP candles at the chosen interval from the downloadable time-and-sales trade archive (`timestamp,price,volume`) and overwrites any existing cache entries for that year and interval. +- With `--vwap`, the command first checks whether the public API still covers part of the requested year at the chosen interval. If quarterly trade archives are available for the rest, it keeps the API slice and downloads only the missing quarters. If the complete trade archive is inevitable anyway, it skips the API and computes the whole year from the archive. +- With `--vwap`, the command tries quarterly trade archives first and falls back to Kraken's complete trade archive automatically if that year's quarterly trade ZIPs are not published. - Downloaded Kraken archive ZIPs are kept under `.cache/kraken/` and reused across later runs; the command does not keep expanded CSV files on disk. +- `--vwap` is substantially heavier because Kraken's trade archives are much larger than the OHLCVT archives, especially when the complete trade archive ZIP is needed instead of quarterly updates. + +For years whose quarterly trade ZIPs are not published, `cache-rates --vwap` +falls back to Kraken's complete trade archive, which requires a ~12G download. +But it does mean VWAP values can be constructed as far back as late 2013. ### `export` @@ -192,7 +203,7 @@ transaction_age <= 720 * interval_minutes * 60 If the transaction is too old to fit inside Kraken's `1d` candle retention window, the tool exits with an error instead of silently switching to a coarser interval. -`cache-rates` is the explicit opt-in workaround for older values: it backfills `.cache/rates.json` from Kraken's OHLCVT archive as daily `(open + close) / 2` midpoint prices. +`cache-rates` is the explicit opt-in workaround for older values: by default it backfills `.cache/rates.json` from Kraken's OHLCVT archive as daily `(open + close) / 2` midpoint prices, or with `--vwap` it computes exact Kraken VWAP candles at the chosen interval from the larger trade archive. ## Development diff --git a/src/commands/cache_rates.rs b/src/commands/cache_rates.rs index 95c6147..583a887 100644 --- a/src/commands/cache_rates.rs +++ b/src/commands/cache_rates.rs @@ -9,11 +9,14 @@ use csv::StringRecord; use reqwest::blocking::{Client, Response}; use zip::ZipArchive; -use crate::common::{AppConfig, build_http_client, fetch_candles_since}; +use crate::common::{ + AppConfig, KRAKEN_INTERVALS_MINUTES, build_http_client, fetch_candles_since, + parse_candle_interval_minutes, +}; use crate::exchange_rate::{CACHE_DIR, cache_key, cache_path, load_disk_cache, save_disk_cache}; pub const SUBCOMMAND_NAME: &str = "cache-rates"; -pub const USAGE: &str = "usage: btc_fiat_value cache-rates "; +pub const USAGE: &str = "usage: btc_fiat_value cache-rates [--vwap] [--candle ] "; const KRAKEN_DAILY_INTERVAL_MINUTES: u32 = 1_440; const QUARTERLY_ARCHIVE_FIRST_YEAR: i32 = 2023; @@ -23,9 +26,18 @@ const QUARTERLY_ARCHIVE_FIRST_YEAR: i32 = 2023; // If Kraken changes the Google Drive ids or stops publishing them there, update these constants. const COMPLETE_OHLCVT_ARCHIVE_FILE_ID: &str = "1ptNqWYidLkhb2VAKuLCxmp2OXEfGO-AP"; const QUARTERLY_OHLCVT_ARCHIVE_FOLDER_ID: &str = "15RSlNuW_h0kVM8or8McOGOMfHeBFvFGI"; +// Kraken time-and-sales archive references: +// - Downloadable trade archive landing page: https://support.kraken.com/articles/360047543791-downloadable-historical-market-data-time-and-sales- +// - OHLC REST docs for the live daily VWAP path: https://docs.kraken.com/api/docs/rest-api/get-ohlc-data/ +// If Kraken changes the Google Drive ids or stops publishing them there, update these constants. +const COMPLETE_TRADE_ARCHIVE_FILE_ID: &str = "10zh3tDpqANYvVtYVgczwVz3UZFRUb1el"; +const QUARTERLY_TRADE_ARCHIVE_FOLDER_ID: &str = "188O9xQjZTythjyLNes_5zfMEFaMbTT22"; + #[derive(Debug, Eq, PartialEq)] pub struct CacheRatesArgs { pub year: i32, + pub use_vwap_archive: bool, + pub candle_override_minutes: Option, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -41,6 +53,12 @@ struct PreparedArchive { extracted_path: PathBuf, } +#[derive(Clone, Debug, Eq, PartialEq)] +struct PreparedArchiveFile { + file: DriveFile, + archive: PreparedArchive, +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct ArchiveCoverage { first: i64, @@ -50,6 +68,7 @@ struct ArchiveCoverage { #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum ArchiveBackfillMode { Midpoint, + Vwap, } #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] @@ -94,8 +113,12 @@ pub fn run(args: CacheRatesArgs) -> Result<()> { let _temp_cache = TempCacheGuard::new()?; let config = AppConfig::from_env()?; let archive_pair = archive_pair_name(&config.kraken_pair)?; - let archive_mode = ArchiveBackfillMode::Midpoint; - let target_interval_minutes = target_interval_minutes(&config)?; + let archive_mode = if args.use_vwap_archive { + ArchiveBackfillMode::Vwap + } else { + ArchiveBackfillMode::Midpoint + }; + let target_interval_minutes = target_interval_minutes(&args, &config)?; let now = Utc::now(); let (start_ts, end_ts) = closed_interval_year_bounds(args.year, now, target_interval_minutes)?; @@ -116,6 +139,7 @@ pub fn run(args: CacheRatesArgs) -> Result<()> { target_interval_minutes, ); } + let full_span_quarters = missing_quarters(&missing_starts); let quarterly_files = if !missing_starts.is_empty() && args.year >= QUARTERLY_ARCHIVE_FIRST_YEAR { let archive_client = build_http_client("Kraken archive", None)?; eprintln!( @@ -132,7 +156,22 @@ pub fn run(args: CacheRatesArgs) -> Result<()> { } else { None }; - let should_fetch_api = true; + let has_full_quarterly_coverage = quarterly_files + .as_ref() + .and_then(|files| { + resolve_quarterly_archive_files( + files, + archive_mode.quarterly_archive_prefix(), + args.year, + &full_span_quarters, + ) + }) + .is_some(); + let api_can_cover_entire_span = missing_starts.len() <= 720; + let should_fetch_api = match archive_mode { + ArchiveBackfillMode::Midpoint => true, + ArchiveBackfillMode::Vwap => api_can_cover_entire_span || has_full_quarterly_coverage, + }; let mut recent_stats = CacheWriteStats::default(); if should_fetch_api { @@ -169,7 +208,9 @@ pub fn run(args: CacheRatesArgs) -> Result<()> { } let mut archive_stats = CacheWriteStats::default(); + let mut fallback_stats = CacheWriteStats::default(); let mut used_complete_archive_fallback = false; + let mut prepared_archives = Vec::new(); if !missing_starts.is_empty() { let archive_client = build_http_client("Kraken archive", None)?; @@ -208,6 +249,10 @@ pub fn run(args: CacheRatesArgs) -> Result<()> { &mut cache, archive_mode, )?); + prepared_archives.push(PreparedArchiveFile { + file, + archive: prepared_archive, + }); } } else { used_complete_archive_fallback = true; @@ -236,6 +281,50 @@ pub fn run(args: CacheRatesArgs) -> Result<()> { &mut cache, archive_mode, )?); + prepared_archives.push(PreparedArchiveFile { + file, + archive: prepared_archive, + }); + } + } + + if archive_mode == ArchiveBackfillMode::Vwap && !missing_starts.is_empty() { + for fallback_interval_minutes in fallback_trade_intervals(target_interval_minutes) { + let missing_before = missing_starts.len(); + eprintln!( + "Warning: exact {}-minute trade VWAP was unavailable for {} interval(s); falling back to {}-minute trade VWAP for those gaps.", + target_interval_minutes, + missing_before, + fallback_interval_minutes, + ); + let mut level_stats = CacheWriteStats::default(); + for prepared_archive in &prepared_archives { + level_stats.absorb(read_prepared_archive( + &prepared_archive.archive, + &prepared_archive.file, + &config.kraken_pair, + &archive_pair, + start_ts, + end_ts, + target_interval_minutes, + Some(fallback_interval_minutes), + &mut missing_starts, + &mut cache, + archive_mode, + )?); + } + fallback_stats.absorb(level_stats); + if level_stats.written() > 0 { + eprintln!( + "Filled {} {}-minute gap(s) using {}-minute trade VWAP.", + level_stats.written(), + target_interval_minutes, + fallback_interval_minutes, + ); + } + if missing_starts.is_empty() { + break; + } } } @@ -256,9 +345,13 @@ pub fn run(args: CacheRatesArgs) -> Result<()> { save_disk_cache(&cache)?; - let inserted_count = recent_stats.inserted + archive_stats.inserted; - let replaced_count = recent_stats.replaced + archive_stats.replaced; - let skipped_count = existing_cache_count; + let inserted_count = recent_stats.inserted + archive_stats.inserted + fallback_stats.inserted; + let replaced_count = recent_stats.replaced + archive_stats.replaced + fallback_stats.replaced; + let skipped_count = if archive_mode == ArchiveBackfillMode::Midpoint { + existing_cache_count + } else { + recent_stats.skipped + archive_stats.skipped + fallback_stats.skipped + }; let total_count = inserted_count + replaced_count + skipped_count; eprintln!( "Cached {total_count} {}-minute rate(s) for {} in {}.", @@ -270,8 +363,16 @@ pub fn run(args: CacheRatesArgs) -> Result<()> { eprintln!("Replaced existing cache entries: {replaced_count}"); eprintln!("Skipped existing cache entries: {skipped_count}"); eprintln!("Kraken OHLC API rows: {}", recent_stats.written()); - eprintln!("OHLCVT archive midpoint rows: {}", archive_stats.written()); - if used_complete_archive_fallback && archive_stats.total() > 0 { + eprintln!("{}: {}", archive_mode.summary_label(), archive_stats.written()); + if fallback_stats.written() > 0 { + eprintln!( + "Trade archive larger-candle fallback rows: {}", + fallback_stats.written() + ); + } + if used_complete_archive_fallback + && (archive_stats.total() > 0 || fallback_stats.total() > 0) + { eprintln!( "Quarterly {} archives were unavailable for {}, so the command fell back to the complete {} archive.", archive_mode.archive_label(), @@ -279,10 +380,16 @@ pub fn run(args: CacheRatesArgs) -> Result<()> { archive_mode.archive_label(), ); } - if archive_stats.total() > 0 { - eprintln!( - "Archive-backed rows use the daily (open + close) / 2 midpoint because Kraken's OHLCVT CSV does not include VWAP." - ); + if archive_stats.total() > 0 || fallback_stats.total() > 0 { + match archive_mode { + ArchiveBackfillMode::Midpoint => eprintln!( + "Archive-backed rows use the daily (open + close) / 2 midpoint because Kraken's OHLCVT CSV does not include VWAP. Re-run with --vwap for exact trade-derived VWAP." + ), + ArchiveBackfillMode::Vwap => eprintln!( + "Archive-backed rows use exact {}-minute VWAP computed from Kraken's time-and-sales trade archive, overwriting any existing cache entries for that year and interval.", + target_interval_minutes + ), + } } Ok(()) @@ -293,15 +400,35 @@ where I: IntoIterator, { let mut year = None; + let mut use_vwap_archive = false; + let mut candle_override_minutes = None; let mut args = args.into_iter(); while let Some(arg) = args.next() { + if arg == "--vwap" { + use_vwap_archive = true; + continue; + } + if arg == "--candle" { + let value = args.next().ok_or_else(|| anyhow!("{usage}"))?; + candle_override_minutes = Some(parse_candle_interval_minutes(&value, "--candle")?); + continue; + } + if let Some(value) = arg.strip_prefix("--candle=") { + candle_override_minutes = Some(parse_candle_interval_minutes(value, "--candle")?); + continue; + } + if year.is_some() { bail!("{usage}"); } year = Some(arg); } + if candle_override_minutes.is_some() && !use_vwap_archive { + bail!("--candle requires --vwap\n\n{usage}"); + } + let year = year.ok_or_else(|| anyhow!("{usage}"))?; let year = year .parse::() @@ -309,11 +436,20 @@ where Ok(CacheRatesArgs { year, + use_vwap_archive, + candle_override_minutes, }) } -fn target_interval_minutes(_config: &AppConfig) -> Result { - Ok(KRAKEN_DAILY_INTERVAL_MINUTES) +fn target_interval_minutes(args: &CacheRatesArgs, config: &AppConfig) -> Result { + if !args.use_vwap_archive { + return Ok(KRAKEN_DAILY_INTERVAL_MINUTES); + } + + Ok(args + .candle_override_minutes + .or(config.default_candle_minutes) + .unwrap_or(KRAKEN_DAILY_INTERVAL_MINUTES)) } fn closed_interval_year_bounds( @@ -373,11 +509,11 @@ fn count_cached_starts( } fn should_store_api_candle( - _archive_mode: ArchiveBackfillMode, + archive_mode: ArchiveBackfillMode, missing_starts: &BTreeSet, candle_start: i64, ) -> bool { - missing_starts.contains(&candle_start) + archive_mode == ArchiveBackfillMode::Vwap || missing_starts.contains(&candle_start) } fn drop_cached_starts( @@ -518,7 +654,7 @@ fn read_archive_zip( start_ts: i64, end_ts: i64, target_interval_minutes: u32, - _trade_source_interval_minutes: Option, + trade_source_interval_minutes: Option, missing_starts: &mut BTreeSet, cache: &mut HashMap, archive_mode: ArchiveBackfillMode, @@ -544,6 +680,17 @@ fn read_archive_zip( missing_starts, cache, ), + ArchiveBackfillMode::Vwap => read_trade_interval_csv( + &mut reader, + &entry_name, + kraken_pair, + start_ts, + end_ts, + trade_source_interval_minutes.unwrap_or(target_interval_minutes), + target_interval_minutes, + missing_starts, + cache, + ), } } @@ -686,6 +833,88 @@ fn read_ohlcvt_daily_csv( Ok(stats) } +fn read_trade_interval_csv( + reader: &mut csv::Reader, + entry_name: &str, + kraken_pair: &str, + start_ts: i64, + end_ts: i64, + source_interval_minutes: u32, + target_interval_minutes: u32, + missing_starts: &mut BTreeSet, + cache: &mut HashMap, +) -> Result { + let mut stats = CacheWriteStats::default(); + let interval_seconds = i64::from(source_interval_minutes) * 60; + let mut current_start = None; + let mut price_volume_sum = 0.0f64; + let mut volume_sum = 0.0f64; + let mut last_logged_year = None; + + for record in reader.records() { + let record = record.with_context(|| format!("failed to parse a row in {entry_name}"))?; + let timestamp = parse_archive_timestamp(&record, entry_name)?; + if timestamp < start_ts { + continue; + } + if timestamp < end_ts { + log_archive_year_progress( + timestamp, + &mut last_logged_year, + "trade archive", + source_interval_minutes, + )?; + } + + let slot_start = timestamp - timestamp.rem_euclid(interval_seconds); + if current_start != Some(slot_start) { + stats.absorb(flush_trade_interval( + current_start, + price_volume_sum, + volume_sum, + kraken_pair, + source_interval_minutes, + target_interval_minutes, + missing_starts, + cache, + )?); + current_start = Some(slot_start); + price_volume_sum = 0.0; + volume_sum = 0.0; + } + + if timestamp >= end_ts { + break; + } + if !bucket_has_missing_targets( + missing_starts, + slot_start, + source_interval_minutes, + target_interval_minutes, + ) { + continue; + } + + let price = parse_archive_number(&record, 1, "price", entry_name)?; + let volume = parse_archive_number(&record, 2, "volume", entry_name)?; + price_volume_sum += price * volume; + volume_sum += volume; + } + + stats.absorb(flush_trade_interval( + current_start, + price_volume_sum, + volume_sum, + kraken_pair, + source_interval_minutes, + target_interval_minutes, + missing_starts, + cache, + )?); + + Ok(stats) +} + fn log_archive_year_progress( timestamp: i64, last_logged_year: &mut Option, @@ -1035,6 +1264,13 @@ fn needed_archive_start(end_ts: i64, target_interval_minutes: u32) -> i64 { end_ts - i64::from(target_interval_minutes) * 60 } +fn fallback_trade_intervals(target_interval_minutes: u32) -> Vec { + KRAKEN_INTERVALS_MINUTES + .into_iter() + .filter(|interval| *interval > target_interval_minutes) + .collect() +} + fn archive_file_description(file: &DriveFile, archive_mode: ArchiveBackfillMode) -> String { if file.name == archive_mode.complete_archive_name() { format!( @@ -1120,6 +1356,70 @@ fn archive_pair_name(kraken_pair: &str) -> Result { bail!("cache-rates only supports Kraken XBT quote pairs such as XXBTZUSD or XXBTZEUR") } +fn flush_trade_interval( + candle_start: Option, + price_volume_sum: f64, + volume_sum: f64, + kraken_pair: &str, + source_interval_minutes: u32, + target_interval_minutes: u32, + missing_starts: &mut BTreeSet, + cache: &mut HashMap, +) -> Result { + let Some(candle_start) = candle_start else { + return Ok(CacheWriteStats::default()); + }; + let source_interval_seconds = i64::from(source_interval_minutes) * 60; + let target_interval_seconds = i64::from(target_interval_minutes) * 60; + let source_end = candle_start + source_interval_seconds; + let bucket_missing = missing_starts + .range(candle_start..source_end) + .copied() + .collect::>(); + if bucket_missing.is_empty() { + return Ok(CacheWriteStats::default()); + } + if volume_sum <= 0.0 { + return Ok(CacheWriteStats::default()); + } + + let vwap = price_volume_sum / volume_sum; + let mut stats = CacheWriteStats::default(); + let mut slot = candle_start; + while slot < source_end { + if missing_starts.remove(&slot) { + stats.record(store_cache_value( + cache, + kraken_pair, + target_interval_minutes, + slot, + vwap, + )); + } + slot += target_interval_seconds; + } + Ok(stats) +} + +fn bucket_has_missing_targets( + missing_starts: &BTreeSet, + candle_start: i64, + source_interval_minutes: u32, + target_interval_minutes: u32, +) -> bool { + let source_interval_seconds = i64::from(source_interval_minutes) * 60; + let target_interval_seconds = i64::from(target_interval_minutes) * 60; + let source_end = candle_start + source_interval_seconds; + let mut slot = candle_start; + while slot < source_end { + if missing_starts.contains(&slot) { + return true; + } + slot += target_interval_seconds; + } + false +} + fn midnight_utc_timestamp(date: NaiveDate) -> i64 { date.and_hms_opt(0, 0, 0) .expect("midnight should always be a valid time") @@ -1211,36 +1511,49 @@ impl ArchiveBackfillMode { fn archive_label(self) -> &'static str { match self { Self::Midpoint => "OHLCVT", + Self::Vwap => "trade", } } fn quarterly_archive_prefix(self) -> &'static str { match self { Self::Midpoint => "Kraken_OHLCVT_", + Self::Vwap => "Kraken_Trading_History_", } } fn quarterly_archive_folder_id(self) -> &'static str { match self { Self::Midpoint => QUARTERLY_OHLCVT_ARCHIVE_FOLDER_ID, + Self::Vwap => QUARTERLY_TRADE_ARCHIVE_FOLDER_ID, } } fn complete_archive_file_id(self) -> &'static str { match self { Self::Midpoint => COMPLETE_OHLCVT_ARCHIVE_FILE_ID, + Self::Vwap => COMPLETE_TRADE_ARCHIVE_FILE_ID, } } fn complete_archive_name(self) -> &'static str { match self { Self::Midpoint => "Kraken_OHLCVT.zip", + Self::Vwap => "Kraken_Trading_History.zip", } } fn entry_name(self, archive_pair: &str) -> String { match self { Self::Midpoint => format!("{archive_pair}_1440.csv"), + Self::Vwap => format!("{archive_pair}.csv"), + } + } + + fn summary_label(self) -> &'static str { + match self { + Self::Midpoint => "OHLCVT archive midpoint rows", + Self::Vwap => "Trade archive VWAP candles", } } } @@ -1250,14 +1563,18 @@ mod tests { use std::collections::{BTreeSet, HashMap}; use std::io::Cursor; use std::path::PathBuf; + use std::time::{SystemTime, UNIX_EPOCH}; + + use crate::common::AppConfig; use super::{ - ArchiveCoverage, CacheRatesArgs, CacheWriteOutcome, + ArchiveCoverage, CacheRatesArgs, CacheWriteOutcome, DriveFile, accept_complete_archive_replacement, - archive_download_path, archive_pair_name, - extract_quarterly_drive_files, parse_args_from, - read_ohlcvt_daily_csv, resolve_archive_entry_name, - should_store_api_candle, store_cache_value, + PreparedArchive, archive_download_path, archive_pair_name, ensure_archive_downloaded, + extracted_archive_entry_path, extract_quarterly_drive_files, parse_args_from, + read_ohlcvt_daily_csv, read_trade_interval_csv, resolve_archive_entry_name, + resolve_quarterly_archive_files, should_store_api_candle, store_cache_value, + target_interval_minutes, }; const REAL_XBTEUR_1440_SAMPLE: &str = "\ @@ -1267,28 +1584,139 @@ mod tests { "; const REAL_XBTEUR_1440_OVERLAP_ROW: &str = "1711929600,66130.0,66130.0,63461.9,64902.9,528.8081404,23715"; + const REAL_XBTEUR_2026_03_21_TRADES: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/kraken_xbteur_2026-03-21_trades.csv" + )); + const REAL_XBTEUR_TRADES_SAMPLE: &str = "\ +1735689600,90167.30000,0.00054633 +1735689600,90167.30000,0.00005463 +1735689600,90167.10000,0.00002185 +1735776000,91190.30000,0.00002161 +1735776000,91190.30000,0.00001080 +1735776000,91190.20000,0.00060000 +"; #[test] fn parses_cache_rates_args() { let args = parse_args_from( vec!["2024".to_owned()], - "usage: btc_fiat_value cache-rates ", + "usage: btc_fiat_value cache-rates [--vwap] [--candle ] ", ) .expect("args"); - assert_eq!(args, CacheRatesArgs { year: 2024 }); + assert_eq!( + args, + CacheRatesArgs { + year: 2024, + use_vwap_archive: false, + candle_override_minutes: None, + } + ); + } + + #[test] + fn parses_cache_rates_vwap_flag() { + let args = parse_args_from( + vec!["--vwap".to_owned(), "2024".to_owned()], + "usage: btc_fiat_value cache-rates [--vwap] [--candle ] ", + ) + .expect("args"); + assert_eq!( + args, + CacheRatesArgs { + year: 2024, + use_vwap_archive: true, + candle_override_minutes: None, + } + ); + } + + #[test] + fn parses_cache_rates_vwap_candle_override() { + let args = parse_args_from( + vec![ + "--vwap".to_owned(), + "--candle".to_owned(), + "60".to_owned(), + "2024".to_owned(), + ], + "usage: btc_fiat_value cache-rates [--vwap] [--candle ] ", + ) + .expect("args"); + assert_eq!( + args, + CacheRatesArgs { + year: 2024, + use_vwap_archive: true, + candle_override_minutes: Some(60), + } + ); } #[test] fn rejects_extra_cache_rates_args() { let err = parse_args_from( vec!["2024".to_owned(), "extra".to_owned()], - "usage: btc_fiat_value cache-rates ", + "usage: btc_fiat_value cache-rates [--vwap] [--candle ] ", ) .expect_err("should fail"); assert!(err .to_string() - .contains("usage: btc_fiat_value cache-rates ")); + .contains("usage: btc_fiat_value cache-rates [--vwap] [--candle ] ")); + } + + #[test] + fn rejects_cache_rates_candle_override_without_vwap() { + let err = parse_args_from( + vec!["--candle".to_owned(), "60".to_owned(), "2024".to_owned()], + "usage: btc_fiat_value cache-rates [--vwap] [--candle ] ", + ) + .expect_err("should fail"); + + assert!(err.to_string().contains("--candle requires --vwap")); + } + + #[test] + fn vwap_uses_default_candle_minutes_when_present() { + let config = AppConfig::from_env_values(|name| match name { + "DEFAULT_CANDLE_MINUTES" => Some("60".to_owned()), + _ => None, + }) + .expect("config"); + + let interval = target_interval_minutes( + &CacheRatesArgs { + year: 2024, + use_vwap_archive: true, + candle_override_minutes: None, + }, + &config, + ) + .expect("interval"); + + assert_eq!(interval, 60); + } + + #[test] + fn vwap_candle_override_beats_default_candle_minutes() { + let config = AppConfig::from_env_values(|name| match name { + "DEFAULT_CANDLE_MINUTES" => Some("60".to_owned()), + _ => None, + }) + .expect("config"); + + let interval = target_interval_minutes( + &CacheRatesArgs { + year: 2024, + use_vwap_archive: true, + candle_override_minutes: Some(240), + }, + &config, + ) + .expect("interval"); + + assert_eq!(interval, 240); } #[test] @@ -1323,6 +1751,11 @@ mod tests { &missing_days, 1672617600, )); + assert!(should_store_api_candle( + super::ArchiveBackfillMode::Vwap, + &missing_days, + 1672531200, + )); } #[test] @@ -1369,6 +1802,44 @@ mod tests { assert_eq!(files["Kraken_OHLCVT_Q2_2024.zip"].id, "def456"); } + #[test] + fn extracts_quarterly_trade_file_ids_from_drive_html() { + let html = concat!( + "window['_DRIVE_ivd'] = '", + "Kraken_Trading_History_Q1_2024.zip ", + "https:\\/\\/drive.google.com\\/file\\/d\\/abc123\\/view ", + "Kraken_Trading_History_Q2_2024.zip ", + "https:\\/\\/drive.google.com\\/file\\/d\\/def456\\/view", + "';if (window['_DRIVE_ivdc'])" + ); + + let files = extract_quarterly_drive_files(html, "Kraken_Trading_History_", "trade") + .expect("files"); + assert_eq!(files["Kraken_Trading_History_Q1_2024.zip"].id, "abc123"); + assert_eq!(files["Kraken_Trading_History_Q2_2024.zip"].id, "def456"); + } + + #[test] + fn falls_back_when_needed_quarterly_trade_archive_is_missing() { + let quarterly_files = HashMap::from([( + "Kraken_Trading_History_Q1_2025.zip".to_owned(), + DriveFile { + id: "abc123".to_owned(), + name: "Kraken_Trading_History_Q1_2025.zip".to_owned(), + }, + )]); + let needed_quarters = BTreeSet::from([1_u32, 2_u32]); + + let resolved = resolve_quarterly_archive_files( + &quarterly_files, + "Kraken_Trading_History_", + 2025, + &needed_quarters, + ); + + assert!(resolved.is_none()); + } + #[test] fn reads_real_archive_daily_csv_rows_into_cache() { let mut reader = csv::ReaderBuilder::new() @@ -1480,6 +1951,332 @@ mod tests { .contains("invalid close value nope in XBTEUR_1440.csv")); } + #[test] + fn reads_real_trade_csv_rows_into_daily_vwap_cache() { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new(REAL_XBTEUR_TRADES_SAMPLE)); + let mut missing_days = BTreeSet::from([1735689600_i64, 1735776000_i64]); + let mut cache = HashMap::new(); + + let stats = read_trade_interval_csv( + &mut reader, + "XBTEUR.csv", + "XXBTZEUR", + 1735689600, + 1735862400, + 1440, + 1440, + &mut missing_days, + &mut cache, + ) + .expect("csv should parse"); + + let day1 = ((90167.3 * 0.00054633) + (90167.3 * 0.00005463) + (90167.1 * 0.00002185)) + / (0.00054633 + 0.00005463 + 0.00002185); + let day2 = ((91190.3 * 0.00002161) + (91190.3 * 0.00001080) + (91190.2 * 0.00060000)) + / (0.00002161 + 0.00001080 + 0.00060000); + + assert_eq!(stats.inserted, 2); + assert_eq!(stats.replaced, 0); + assert_eq!(stats.skipped, 0); + assert_eq!(cache["XXBTZEUR:1440:1735689600"], super::normalize_fiat_rate(day1)); + assert_eq!(cache["XXBTZEUR:1440:1735776000"], super::normalize_fiat_rate(day2)); + assert!(missing_days.is_empty()); + } + + #[test] + fn reads_trade_csv_rows_into_hourly_vwap_cache() { + let trades = "\ +1735689600,100.0,1.0 +1735691400,110.0,3.0 +1735693200,120.0,2.0 +1735695000,90.0,2.0 +"; + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new(trades)); + let mut missing_hours = BTreeSet::from([1735689600_i64, 1735693200_i64]); + let mut cache = HashMap::new(); + + let stats = read_trade_interval_csv( + &mut reader, + "XBTEUR.csv", + "XXBTZEUR", + 1735689600, + 1735696800, + 60, + 60, + &mut missing_hours, + &mut cache, + ) + .expect("csv should parse"); + + assert_eq!(stats.inserted, 2); + assert_eq!(stats.replaced, 0); + assert_eq!(stats.skipped, 0); + assert_eq!(cache["XXBTZEUR:60:1735689600"], super::normalize_fiat_rate(107.5)); + assert_eq!(cache["XXBTZEUR:60:1735693200"], super::normalize_fiat_rate(105.0)); + assert!(missing_hours.is_empty()); + } + + #[test] + fn larger_trade_candle_fills_missing_smaller_intervals() { + let trades = "\ +1735689900,100.0,1.0 +1735697100,140.0,3.0 +"; + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new(trades)); + let mut missing_hours = + BTreeSet::from([1735689600_i64, 1735693200_i64, 1735696800_i64, 1735700400_i64]); + let mut cache = HashMap::new(); + + let stats = read_trade_interval_csv( + &mut reader, + "XBTEUR.csv", + "XXBTZEUR", + 1735689600, + 1735704000, + 240, + 60, + &mut missing_hours, + &mut cache, + ) + .expect("csv should parse"); + + let vwap = (100.0 * 1.0 + 140.0 * 3.0) / 4.0; + assert_eq!(stats.inserted, 4); + assert_eq!(stats.replaced, 0); + assert_eq!(stats.skipped, 0); + assert_eq!(cache["XXBTZEUR:60:1735689600"], super::normalize_fiat_rate(vwap)); + assert_eq!(cache["XXBTZEUR:60:1735693200"], super::normalize_fiat_rate(vwap)); + assert_eq!(cache["XXBTZEUR:60:1735696800"], super::normalize_fiat_rate(vwap)); + assert_eq!(cache["XXBTZEUR:60:1735700400"], super::normalize_fiat_rate(vwap)); + assert!(missing_hours.is_empty()); + } + + #[test] + fn larger_trade_candle_fills_missing_interior_smaller_intervals() { + let trades = "\ +1735689900,100.0,1.0 +1735697100,140.0,3.0 +"; + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new(trades)); + let mut missing_hours = BTreeSet::from([1735693200_i64, 1735696800_i64]); + let mut cache = HashMap::new(); + + let stats = read_trade_interval_csv( + &mut reader, + "XBTEUR.csv", + "XXBTZEUR", + 1735689600, + 1735704000, + 240, + 60, + &mut missing_hours, + &mut cache, + ) + .expect("csv should parse"); + + let vwap = (100.0 * 1.0 + 140.0 * 3.0) / 4.0; + assert_eq!(stats.inserted, 2); + assert_eq!(stats.replaced, 0); + assert_eq!(stats.skipped, 0); + assert_eq!(cache["XXBTZEUR:60:1735693200"], super::normalize_fiat_rate(vwap)); + assert_eq!(cache["XXBTZEUR:60:1735696800"], super::normalize_fiat_rate(vwap)); + assert!(missing_hours.is_empty()); + } + + #[test] + fn trade_vwap_replaces_existing_cache_entry() { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new(REAL_XBTEUR_TRADES_SAMPLE)); + let mut missing_days = BTreeSet::from([1735689600_i64]); + let mut cache = HashMap::from([("XXBTZEUR:1440:1735689600".to_owned(), 1.23_f64)]); + + let stats = read_trade_interval_csv( + &mut reader, + "XBTEUR.csv", + "XXBTZEUR", + 1735689600, + 1735776000, + 1440, + 1440, + &mut missing_days, + &mut cache, + ) + .expect("csv should parse"); + + assert_eq!(stats.inserted, 0); + assert_eq!(stats.replaced, 1); + assert_eq!(stats.skipped, 0); + assert_ne!(cache["XXBTZEUR:1440:1735689600"], 1.23); + assert!(missing_days.is_empty()); + } + + #[test] + fn trade_vwap_skips_existing_entry_when_value_is_unchanged() { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new(REAL_XBTEUR_TRADES_SAMPLE)); + let day1 = ((90167.3 * 0.00054633) + (90167.3 * 0.00005463) + (90167.1 * 0.00002185)) + / (0.00054633 + 0.00005463 + 0.00002185); + let mut missing_days = BTreeSet::from([1735689600_i64]); + let mut cache = HashMap::from([("XXBTZEUR:1440:1735689600".to_owned(), day1)]); + + let stats = read_trade_interval_csv( + &mut reader, + "XBTEUR.csv", + "XXBTZEUR", + 1735689600, + 1735776000, + 1440, + 1440, + &mut missing_days, + &mut cache, + ) + .expect("csv should parse"); + + assert_eq!(stats.inserted, 0); + assert_eq!(stats.replaced, 0); + assert_eq!(stats.skipped, 1); + assert_eq!(cache["XXBTZEUR:1440:1735689600"], super::normalize_fiat_rate(day1)); + assert!(missing_days.is_empty()); + } + + #[test] + fn trade_archive_fixture_produces_expected_hourly_vwap_candles() { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new(REAL_XBTEUR_2026_03_21_TRADES)); + // 2026-03-21 02:00–04:59 UTC — three quiet hours from real Kraken trades. + // API 60-min OHLC VWAPs for this window: 60995.3, 61074.2, 61145.7 + let mut missing_hours = + BTreeSet::from([1774058400_i64, 1774062000_i64, 1774065600_i64]); + let mut cache = HashMap::new(); + + let stats = read_trade_interval_csv( + &mut reader, + "XBTEUR.csv", + "XXBTZEUR", + 1774058400, + 1774069200, + 60, + 60, + &mut missing_hours, + &mut cache, + ) + .expect("csv should parse"); + + assert_eq!(stats.inserted, 3); + assert_eq!(stats.replaced, 0); + assert_eq!(stats.skipped, 0); + assert_eq!(cache["XXBTZEUR:60:1774058400"], super::normalize_fiat_rate(60995.38)); + assert_eq!(cache["XXBTZEUR:60:1774062000"], super::normalize_fiat_rate(61074.23)); + assert_eq!(cache["XXBTZEUR:60:1774065600"], super::normalize_fiat_rate(61145.71)); + assert!(missing_hours.is_empty()); + } + + #[test] + fn reports_missing_trade_volume_clearly() { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new("1735689600,90167.3\n")); + let mut missing_days = BTreeSet::from([1735689600_i64]); + let mut cache = HashMap::new(); + + let err = read_trade_interval_csv( + &mut reader, + "XBTEUR.csv", + "XXBTZEUR", + 1735689600, + 1735776000, + 1440, + 1440, + &mut missing_days, + &mut cache, + ) + .expect_err("row should fail"); + + assert!(err + .to_string() + .contains("XBTEUR.csv row is missing the volume column")); + } + + #[test] + fn reports_invalid_trade_volume_clearly() { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new("1735689600,90167.3,nope\n")); + let mut missing_days = BTreeSet::from([1735689600_i64]); + let mut cache = HashMap::new(); + + let err = read_trade_interval_csv( + &mut reader, + "XBTEUR.csv", + "XXBTZEUR", + 1735689600, + 1735776000, + 1440, + 1440, + &mut missing_days, + &mut cache, + ) + .expect_err("row should fail"); + + assert!(err + .to_string() + .contains("invalid volume value nope in XBTEUR.csv")); + } + + #[test] + fn reuses_existing_archive_download_without_redownloading() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time") + .as_nanos(); + let path = std::env::temp_dir().join(format!("btc-accounting-archive-{unique}.zip")); + let cursor = Cursor::new(Vec::::new()); + let mut writer = zip::ZipWriter::new(cursor); + let options: zip::write::SimpleFileOptions = zip::write::SimpleFileOptions::default(); + writer + .start_file("XBTEUR.csv", options) + .expect("start file"); + std::io::Write::write_all(&mut writer, b"1735689600,90167.3,0.1\n").expect("write file"); + let cursor = writer.finish().expect("finish zip"); + std::fs::write(&path, cursor.into_inner()).expect("fixture"); + let extracted_path = extracted_archive_entry_path("unused.zip", "XBTEUR.csv"); + + let result = ensure_archive_downloaded( + &reqwest::blocking::Client::new(), + &DriveFile { + id: "unused".to_owned(), + name: "unused.zip".to_owned(), + }, + &path, + super::ArchiveBackfillMode::Vwap, + "XBTEUR", + 1735776000, + 1440, + ); + + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&extracted_path); + assert_eq!( + result.expect("result"), + PreparedArchive { + archive_path: path, + archive_file_name: "unused.zip".to_owned(), + extracted_path, + } + ); + } + #[test] fn rejects_complete_archive_replacement_that_drops_old_coverage() { let existing = ArchiveCoverage { diff --git a/src/main.rs b/src/main.rs index f64bf6c..d9675de 100644 --- a/src/main.rs +++ b/src/main.rs @@ -121,6 +121,48 @@ mod tests { match command { Command::CacheRates(args) => { assert_eq!(args.year, 2024); + assert!(!args.use_vwap_archive); + assert_eq!(args.candle_override_minutes, None); + } + _ => panic!("expected CacheRates"), + } + } + + #[test] + fn parses_cache_rates_vwap_subcommand() { + let command = parse_command_from(vec![ + "cache-rates".to_owned(), + "--vwap".to_owned(), + "2024".to_owned(), + ]) + .expect("command"); + + match command { + Command::CacheRates(args) => { + assert_eq!(args.year, 2024); + assert!(args.use_vwap_archive); + assert_eq!(args.candle_override_minutes, None); + } + _ => panic!("expected CacheRates"), + } + } + + #[test] + fn parses_cache_rates_vwap_candle_override_subcommand() { + let command = parse_command_from(vec![ + "cache-rates".to_owned(), + "--vwap".to_owned(), + "--candle".to_owned(), + "60".to_owned(), + "2024".to_owned(), + ]) + .expect("command"); + + match command { + Command::CacheRates(args) => { + assert_eq!(args.year, 2024); + assert!(args.use_vwap_archive); + assert_eq!(args.candle_override_minutes, Some(60)); } _ => panic!("expected CacheRates"), } diff --git a/tests/fixtures/kraken_xbteur_2026-03-21_trades.csv b/tests/fixtures/kraken_xbteur_2026-03-21_trades.csv new file mode 100644 index 0000000..55b6726 --- /dev/null +++ b/tests/fixtures/kraken_xbteur_2026-03-21_trades.csv @@ -0,0 +1,767 @@ +1774058400,60912.70000,0.00008127 +1774058400,60912.70000,0.00026138 +1774058400,60912.70000,0.00001634 +1774058400,60912.70000,0.00016174 +1774058400,60912.70000,0.00243816 +1774058400,60912.70000,0.00016335 +1774058402,60912.70000,0.00024600 +1774058402,60912.70000,0.00013081 +1774058402,60912.70000,0.00058864 +1774058403,60912.70000,0.00022656 +1774058405,60912.60000,0.00016400 +1774058405,60912.60000,0.00009850 +1774058409,60914.80000,0.00057171 +1774058410,60914.80000,0.00019776 +1774058410,60914.80000,0.00016480 +1774058411,60914.80000,0.00010877 +1774058411,60914.80000,0.00013184 +1774058411,60914.80000,0.00013184 +1774058411,60914.80000,0.00009888 +1774058411,60914.80000,0.00023553 +1774058411,60914.80000,0.00019776 +1774058411,60914.80000,0.00008240 +1774058412,60914.80000,0.00049921 +1774058412,60914.80000,0.00024294 +1774058412,60914.80000,0.00013184 +1774058412,60914.80000,0.00019776 +1774058412,60914.80000,0.00013184 +1774058413,60914.80000,0.00033441 +1774058413,60914.80000,0.00019776 +1774058413,60914.80000,0.00026368 +1774058413,60914.80000,0.00016480 +1774058413,60914.80000,0.00016196 +1774058413,60914.80000,0.00010921 +1774058413,60914.80000,0.00026368 +1774058413,60914.80000,0.00026368 +1774058413,60914.80000,0.00008000 +1774058414,60914.80000,0.00049921 +1774058414,60914.80000,0.00008000 +1774058417,60914.80000,0.00001617 +1774058419,60914.80000,0.00016000 +1774058419,60914.80000,0.00016000 +1774058419,60914.80000,0.00016000 +1774058419,60914.80000,0.00016000 +1774058419,60914.80000,0.00049249 +1774058422,60914.80000,0.00010836 +1774058427,60921.30000,0.00016093 +1774058438,60929.10000,0.00016169 +1774058450,60922.80000,0.00035998 +1774058459,60929.10000,0.00161700 +1774058495,60939.10000,0.00243686 +1774058496,60938.80000,0.00160012 +1774058517,60943.30000,0.00016162 +1774058553,60905.80000,0.00032273 +1774058555,60905.80000,0.00324000 +1774058585,60896.20000,0.00240313 +1774058607,60883.20000,0.00122225 +1774058610,60883.20000,0.00048787 +1774058625,60884.10000,0.00040437 +1774058632,60886.40000,0.00023634 +1774058639,60877.30000,0.00016860 +1774058641,60900.80000,0.01550000 +1774058644,60922.40000,0.00162830 +1774058653,60903.80000,0.00247122 +1774058699,60899.40000,0.00169502 +1774058700,60901.70000,0.00003220 +1774058705,60909.30000,0.00036748 +1774058709,60914.10000,0.00015994 +1774058826,60907.90000,0.01492875 +1774058892,60941.20000,0.00180780 +1774058905,60926.90000,0.04584000 +1774058933,60918.30000,0.00061970 +1774058966,60918.30000,0.00003219 +1774058984,60918.30000,0.01332554 +1774058984,60931.00000,0.00427676 +1774059001,60927.30000,0.00197034 +1774059005,60927.20000,0.00134915 +1774059012,60927.30000,0.01480000 +1774059021,60927.20000,0.00079193 +1774059021,60927.30000,0.00519508 +1774059023,60927.20000,0.00033734 +1774059038,60925.40000,0.00538293 +1774059039,60925.40000,0.00001616 +1774059076,60925.30000,0.00080696 +1774059076,60922.20000,0.00020844 +1774059103,60913.10000,0.00002566 +1774059133,60913.00000,0.00740000 +1774059152,60913.10000,0.00023966 +1774059181,60913.10000,0.00259212 +1774059194,60913.10000,0.00080871 +1774059220,60905.30000,0.00079117 +1774059244,60913.00000,0.00051618 +1774059293,60913.00000,0.00006902 +1774059302,60913.10000,0.01260595 +1774059305,60941.60000,0.00023027 +1774059305,60941.60000,0.00019777 +1774059312,60941.60000,0.00016087 +1774059328,60943.00000,0.00044969 +1774059331,60942.90000,0.00161665 +1774059331,60942.90000,0.00283096 +1774059347,60942.90000,0.00027348 +1774059363,60943.00000,0.00015555 +1774059377,60954.70000,0.00151900 +1774059394,60958.30000,0.00014964 +1774059410,60941.60000,0.00024614 +1774059422,60951.10000,0.00013176 +1774059481,60972.60000,0.00010000 +1774059483,60977.80000,0.00010000 +1774059536,60970.50000,0.00001581 +1774059548,60983.40000,0.00010000 +1774059549,60984.10000,0.00023535 +1774059601,60974.20000,0.00015100 +1774059605,60974.90000,0.00009878 +1774059608,60977.00000,0.00010000 +1774059613,60977.40000,0.00120000 +1774059642,60979.80000,0.00047142 +1774059662,60980.70000,0.00500000 +1774059667,60976.90000,0.00395570 +1774059671,60978.50000,0.00038926 +1774059681,60981.30000,0.00362781 +1774059695,60977.60000,0.01550000 +1774059695,60976.50000,0.03202137 +1774059698,60988.40000,0.00010000 +1774059737,60990.00000,0.00003231 +1774059737,60990.00000,0.00010000 +1774059747,60989.70000,0.00038912 +1774059756,60990.80000,0.00031383 +1774059796,60999.00000,0.00010000 +1774059802,60999.50000,0.00290277 +1774059812,60999.30000,0.00038906 +1774059822,60995.40000,0.00014950 +1774059827,60995.30000,0.00033773 +1774059838,60998.20000,0.00176500 +1774059845,60995.30000,0.00075051 +1774059853,60998.50000,0.00010000 +1774059905,61000.00000,0.00010000 +1774059925,61000.00000,0.00002403 +1774059925,61008.90000,0.00002597 +1774059946,61008.90000,0.00044382 +1774059965,61021.30000,0.00010000 +1774060020,61055.00000,0.00008149 +1774060024,61054.00000,0.00010000 +1774060040,61054.90000,0.00079703 +1774060073,61064.30000,0.00296660 +1774060085,61070.30000,0.00012156 +1774060113,61073.20000,0.00032105 +1774060127,61073.00000,0.00008106 +1774060135,61073.00000,0.00006630 +1774060163,61098.70000,0.00010000 +1774060200,61055.60000,0.00008029 +1774060201,61045.70000,0.00008181 +1774060203,61048.30000,0.00019730 +1774060203,61048.00000,0.00026307 +1774060210,61042.60000,0.00010000 +1774060215,61061.90000,0.00066845 +1774060219,61044.90000,0.00053665 +1774060228,61045.90000,0.00016060 +1774060254,61047.40000,0.00014833 +1774060261,61045.40000,0.00034900 +1774060262,61046.10000,0.00050870 +1774060270,61046.10000,0.00010000 +1774060278,61045.30000,0.00469873 +1774060312,61041.20000,0.00015945 +1774060326,61040.00000,0.00010000 +1774060342,61025.10000,0.00006686 +1774060353,61023.00000,0.00024099 +1774060386,61003.30000,0.00013165 +1774060386,61003.30000,0.00292532 +1774060389,61003.30000,0.00016150 +1774060391,61003.30000,0.00001615 +1774060391,61003.30000,0.00057133 +1774060396,61003.70000,0.00015999 +1774060397,61003.40000,0.00010000 +1774060402,61003.20000,0.00086232 +1774060403,61003.30000,0.00049178 +1774060406,61003.20000,0.00037001 +1774060408,61004.40000,0.00041074 +1774060430,61014.40000,0.00116744 +1774060442,61015.50000,0.00819421 +1774060444,61027.10000,0.00080720 +1774060469,61030.60000,0.00010000 +1774060511,61039.10000,0.00005000 +1774060520,61036.90000,0.00063854 +1774060523,61038.70000,0.00010000 +1774060574,61039.50000,0.00001626 +1774060576,61034.80000,0.00010000 +1774060623,61045.50000,0.00003558 +1774060635,61043.30000,0.00010000 +1774060661,61028.20000,0.00508900 +1774060687,61047.60000,0.00010000 +1774060727,61056.00000,0.00046564 +1774060778,61065.80000,0.00010000 +1774060796,61068.10000,0.00086176 +1774060814,61109.90000,0.01209700 +1774060815,61132.90000,0.00607358 +1774060815,61139.60000,0.00474177 +1774060816,61142.70000,0.00007369 +1774060818,61138.20000,0.00014777 +1774060819,61129.80000,0.00035786 +1774060830,61134.10000,0.00010000 +1774060848,61132.70000,0.00008485 +1774060878,61114.60000,0.00010000 +1774060884,61117.80000,0.00007729 +1774060886,61116.30000,0.00047826 +1774060888,61116.20000,0.00023941 +1774060913,61120.20000,0.00015467 +1774060921,61115.70000,0.00013141 +1774060928,61117.40000,0.00021874 +1774060952,61115.50000,0.00016477 +1774060981,61112.70000,0.00788493 +1774060986,61117.90000,0.00040787 +1774060991,61122.50000,0.00010000 +1774061007,61114.30000,0.00001701 +1774061015,61117.80000,0.00635440 +1774061016,61116.60000,0.00040792 +1774061049,61117.40000,0.00010000 +1774061081,61109.20000,0.00128847 +1774061083,61114.00000,0.00076431 +1774061086,61115.30000,0.00061350 +1774061092,61112.40000,0.00001684 +1774061103,61116.60000,0.00017181 +1774061104,61117.20000,0.00015999 +1774061115,61091.90000,0.00036902 +1774061148,61101.50000,0.00010000 +1774061152,61096.60000,0.00310066 +1774061171,61099.90000,0.00011576 +1774061177,61098.60000,0.00040070 +1774061201,61117.50000,0.00036895 +1774061220,61116.50000,0.00001612 +1774061245,61126.60000,0.00015693 +1774061253,61128.70000,0.00014142 +1774061253,61128.80000,0.00022674 +1774061289,61135.70000,0.00036813 +1774061302,61133.60000,0.00010000 +1774061316,61137.80000,0.00045451 +1774061330,61138.20000,0.00003239 +1774061346,61114.50000,0.00003224 +1774061358,61111.90000,0.00028900 +1774061398,61135.00000,0.00769274 +1774061401,61138.00000,0.00100833 +1774061455,61142.70000,0.00147426 +1774061472,61146.20000,0.00010000 +1774061506,61141.00000,0.00013275 +1774061510,61141.60000,0.00313404 +1774061517,61145.50000,0.00036807 +1774061538,61145.90000,0.00079920 +1774061540,61148.90000,0.01241250 +1774061541,61151.50000,0.00041344 +1774061553,61145.70000,0.00036807 +1774061573,61148.10000,0.00036805 +1774061574,61145.80000,0.00106679 +1774061609,61162.80000,0.00034388 +1774061624,61169.40000,0.00014872 +1774061628,61168.80000,0.00033353 +1774061647,61170.30000,0.00015091 +1774061690,61179.70000,0.00040749 +1774061696,61168.60000,0.00686297 +1774061736,61138.00000,0.00006804 +1774061736,61133.30000,0.00033945 +1774061782,61134.00000,0.00040778 +1774061787,61132.90000,0.01337524 +1774061808,61140.60000,0.00040778 +1774061835,61112.10000,0.00014145 +1774061835,61107.60000,0.00067873 +1774061837,61112.20000,0.00040794 +1774061839,61113.20000,0.00014143 +1774061839,61113.30000,0.00022688 +1774061865,61124.90000,0.00197090 +1774061879,61123.30000,0.00032723 +1774061896,61113.20000,0.00008071 +1774061924,61133.50000,0.00014504 +1774061930,61126.30000,0.00037046 +1774061940,61132.50000,0.00001612 +1774062000,61131.40000,0.00008058 +1774062000,61131.40000,0.00080187 +1774062000,61136.00000,0.00320716 +1774062000,61136.00000,0.00001612 +1774062000,61136.00000,0.00001612 +1774062000,61136.00000,0.00008177 +1774062000,61136.00000,0.00080975 +1774062000,61136.00000,0.00016036 +1774062000,61136.00000,0.00161152 +1774062000,61136.00000,0.00001604 +1774062000,61136.00000,0.00107972 +1774062001,61136.00000,0.00027154 +1774062002,61152.50000,0.00015026 +1774062002,61151.60000,0.00016359 +1774062008,61152.60000,0.00016271 +1774062010,61153.40000,0.00016415 +1774062011,61153.40000,0.00009521 +1774062011,61153.40000,0.00019699 +1774062011,61153.40000,0.00049663 +1774062011,61153.40000,0.00009849 +1774062011,61153.40000,0.00016416 +1774062011,61153.40000,0.00016416 +1774062011,61153.40000,0.00013132 +1774062011,61153.40000,0.00013132 +1774062011,61153.40000,0.00009849 +1774062012,61153.40000,0.00026265 +1774062012,61153.40000,0.00013132 +1774062012,61153.40000,0.00026265 +1774062012,61153.40000,0.00009849 +1774062013,61153.40000,0.00023442 +1774062014,61151.60000,0.00809545 +1774062016,61151.60000,0.00016002 +1774062025,61153.30000,0.00117851 +1774062026,61153.30000,0.00046940 +1774062027,61152.00000,0.00016352 +1774062030,61151.60000,0.00004857 +1774062052,61151.60000,0.00040477 +1774062058,61151.50000,0.00179881 +1774062059,61129.30000,0.00012088 +1774062130,61115.70000,0.00040792 +1774062181,61103.50000,0.00064172 +1774062185,61107.00000,0.00030633 +1774062234,61089.90000,0.00138861 +1774062305,61091.70000,0.00096762 +1774062337,61104.50000,0.00008102 +1774062339,61104.20000,0.00326310 +1774062345,61104.10000,0.00037750 +1774062381,61116.70000,0.00188167 +1774062387,61104.80000,0.00040792 +1774062401,61114.10000,0.00100340 +1774062441,61140.20000,0.00029686 +1774062441,61140.50000,0.00014144 +1774062441,61140.60000,0.00131861 +1774062470,61136.00000,0.00003207 +1774062487,61137.50000,0.00794011 +1774062495,61137.40000,0.00039912 +1774062509,61154.80000,0.00005000 +1774062536,61137.20000,0.00001550 +1774062553,61144.70000,0.00039908 +1774062581,61141.40000,0.00033897 +1774062582,61147.60000,0.00039617 +1774062590,61147.10000,0.00040461 +1774062644,61150.40000,0.00327562 +1774062674,61173.60000,0.00007058 +1774062686,61177.20000,0.00003099 +1774062709,61175.80000,0.00024398 +1774062711,61177.20000,0.00025610 +1774062729,61162.60000,0.01635084 +1774062732,61141.90000,0.00473035 +1774062761,61113.20000,0.00010000 +1774062763,61113.20000,0.00016201 +1774062765,61108.60000,0.00055490 +1774062799,61084.60000,0.00159898 +1774062801,61085.80000,0.00102668 +1774062861,61102.70000,0.00128829 +1774062889,61101.60000,0.00053500 +1774062900,61104.60000,0.00019714 +1774062902,61105.20000,0.00016365 +1774062906,61106.90000,0.00080219 +1774062919,61104.20000,0.00036467 +1774062930,61103.00000,0.00004885 +1774062969,61078.40000,0.00030888 +1774062971,61078.40000,0.00030888 +1774062971,61078.00000,0.00030888 +1774062974,61078.60000,0.00030888 +1774062975,61078.20000,0.00030888 +1774062977,61077.10000,0.00030887 +1774062978,61076.10000,0.00030889 +1774062980,61075.90000,0.00038302 +1774062982,61071.60000,0.00061783 +1774062986,61080.20000,0.00061774 +1774062999,61094.00000,0.00155235 +1774063007,61089.00000,0.00033232 +1774063020,61095.00000,0.00001613 +1774063026,61084.50000,0.00030885 +1774063027,61084.50000,0.00037837 +1774063028,61084.50000,0.00030885 +1774063031,61084.50000,0.00061770 +1774063037,61086.60000,0.00014151 +1774063037,61080.40000,0.00387312 +1774063055,61099.30000,0.00473562 +1774063073,61096.50000,0.00152161 +1774063078,61094.80000,0.00042899 +1774063080,61092.10000,0.00049596 +1774063086,61098.70000,0.00016003 +1774063105,61092.80000,0.00005000 +1774063141,61096.70000,0.00069889 +1774063144,61097.50000,0.00003209 +1774063161,61106.50000,0.00042899 +1774063167,61105.70000,0.00071497 +1774063185,61085.10000,0.00061985 +1774063200,61087.30000,0.00008025 +1774063206,61082.70000,0.00049737 +1774063206,61082.70000,0.00013147 +1774063207,61083.20000,0.00016003 +1774063218,61078.80000,0.00110753 +1774063248,61074.10000,0.00057434 +1774063295,61099.30000,0.00005000 +1774063312,61099.00000,0.00049002 +1774063321,61094.20000,0.00008074 +1774063327,61093.20000,0.00042885 +1774063401,61102.20000,0.00723045 +1774063532,61092.20000,0.00016369 +1774063539,61073.20000,0.00023472 +1774063590,61101.80000,0.00008102 +1774063596,61101.80000,0.00042863 +1774063642,61101.80000,0.00087618 +1774063647,61101.80000,0.00002161 +1774063741,61101.80000,0.00042847 +1774063792,61101.70000,0.00039783 +1774063800,61101.80000,0.00040712 +1774063800,61096.60000,0.16611649 +1774063802,61094.80000,0.00009201 +1774063802,61094.80000,0.00009859 +1774063803,61094.80000,0.00031688 +1774063820,61083.00000,0.00038917 +1774063855,61086.60000,0.00003210 +1774063883,61076.70000,0.00010000 +1774063905,61069.70000,0.00106199 +1774063910,61071.40000,0.00050216 +1774063931,61075.80000,0.00010000 +1774063949,61067.60000,0.00121261 +1774063950,61073.10000,0.00080787 +1774063951,61067.50000,0.00040312 +1774063952,61067.50000,0.00010361 +1774063954,61065.10000,0.00016028 +1774063954,61062.10000,0.00040312 +1774063955,61062.10000,0.00032218 +1774063957,61060.60000,0.00161736 +1774063958,61060.60000,0.00161736 +1774063959,61060.60000,0.00016028 +1774063960,61060.60000,0.00020075 +1774063964,61060.60000,0.00080787 +1774063965,61060.60000,0.00080793 +1774063967,61060.60000,0.00080793 +1774063968,61060.60000,0.00040316 +1774063969,61060.60000,0.00040316 +1774063971,61060.60000,0.00040316 +1774063972,61060.60000,0.00080793 +1774063974,61058.80000,0.00080793 +1774063975,61058.80000,0.00048411 +1774063977,61058.80000,0.00024125 +1774063978,61058.80000,0.00040317 +1774063979,61058.80000,0.00129370 +1774063980,61058.80000,0.00010746 +1774063980,61058.80000,0.00080795 +1774063981,61058.80000,0.00080795 +1774063982,61056.50000,0.00249450 +1774063982,61058.80000,0.00161753 +1774063983,61058.80000,0.00032221 +1774063984,61058.80000,0.00161753 +1774063985,61058.80000,0.00040317 +1774063986,61058.80000,0.00024125 +1774063987,61058.80000,0.00161753 +1774063988,61058.80000,0.00080795 +1774063988,61058.80000,0.00161753 +1774063989,61058.80000,0.00080795 +1774063992,61058.80000,0.00161753 +1774063992,61058.80000,0.00080795 +1774063993,61058.80000,0.00040317 +1774063994,61076.90000,0.00040317 +1774063995,61072.00000,0.00024125 +1774064011,61079.70000,0.00042867 +1774064014,61074.60000,0.00010000 +1774064050,61060.30000,0.00010000 +1774064074,61065.90000,0.00046661 +1774064124,61050.10000,0.00032257 +1774064132,61050.30000,0.00022612 +1774064155,61050.00000,0.00000607 +1774064155,61050.00000,0.00092332 +1774064155,61049.80000,0.00009859 +1774064183,61045.50000,0.00010000 +1774064193,61053.80000,0.00031056 +1774064200,61050.90000,0.00968064 +1774064200,61050.80000,0.00023816 +1774064208,61051.10000,0.00057452 +1774064234,61060.40000,0.00014156 +1774064234,61060.40000,0.00131838 +1774064252,61055.10000,0.00010000 +1774064296,61048.60000,0.00010387 +1774064321,61047.30000,0.00602784 +1774064326,61043.20000,0.15000000 +1774064347,61044.20000,0.00026357 +1774064410,61051.40000,0.00011627 +1774064420,61050.60000,0.00014369 +1774064463,61057.80000,0.00093037 +1774064467,61063.50000,0.00012574 +1774064494,61063.00000,0.00003180 +1774064524,61067.20000,0.00318977 +1774064525,61067.20000,0.00010000 +1774064598,61057.60000,0.00010000 +1774064612,61060.30000,0.01550000 +1774064612,61061.20000,0.05372800 +1774064643,61064.70000,0.00019728 +1774064644,61063.90000,0.00029321 +1774064647,61059.20000,0.00060602 +1774064703,61055.80000,0.00009865 +1774064708,61059.40000,0.00020862 +1774064739,61045.50000,0.01550000 +1774064739,61045.40000,0.04991085 +1774064748,61043.30000,0.00036823 +1774064760,61046.90000,0.00032437 +1774064764,61051.30000,0.00010000 +1774064783,61052.60000,0.00347290 +1774064795,61065.20000,0.00013230 +1774064804,61062.70000,0.00067060 +1774064825,61059.80000,0.00010000 +1774064834,61059.70000,0.00014157 +1774064834,61059.10000,0.00009255 +1774064856,61059.90000,0.04786331 +1774064863,61060.10000,0.00015900 +1774064898,61063.50000,0.00010000 +1774064912,61063.50000,0.03138530 +1774064921,61062.20000,0.00821015 +1774064946,61061.40000,0.00016568 +1774064946,61061.40000,0.00010000 +1774064972,61063.60000,0.00008188 +1774065005,61047.90000,0.00010000 +1774065033,61045.80000,0.00016060 +1774065066,61061.10000,0.00010000 +1774065077,61080.50000,0.00036274 +1774065077,61092.30000,0.00050654 +1774065107,61082.30000,0.02288700 +1774065108,61084.10000,0.00054024 +1774065124,61076.60000,0.00010000 +1774065128,61076.60000,0.01316750 +1774065135,61067.40000,0.00204973 +1774065191,61071.80000,0.00015116 +1774065218,61065.40000,0.00372131 +1774065221,61063.80000,0.00706130 +1774065243,61070.00000,0.00311209 +1774065245,61074.90000,0.00010000 +1774065262,61077.20000,0.01552236 +1774065300,61073.00000,0.00008147 +1774065314,61081.50000,0.00018352 +1774065358,61076.90000,0.01506150 +1774065360,61078.50000,0.00031791 +1774065385,61082.20000,0.00010000 +1774065417,61087.00000,0.00029999 +1774065423,61087.00000,0.00010000 +1774065458,61087.00000,0.00015187 +1774065500,61087.00000,0.00010000 +1774065534,61087.00000,0.00093640 +1774065544,61087.00000,0.00010000 +1774065594,61087.00000,0.00007985 +1774065596,61087.00000,0.00637279 +1774065600,61087.00000,0.00080246 +1774065600,61087.00000,0.00001613 +1774065600,61087.00000,0.00012903 +1774065600,61087.00000,0.00029031 +1774065600,61087.00000,0.00016049 +1774065600,61087.00000,0.00040722 +1774065600,61087.00000,0.00001613 +1774065600,61087.00000,0.00016208 +1774065600,61087.00000,0.00008144 +1774065600,61087.00000,0.00001613 +1774065600,61087.00000,0.00016128 +1774065600,61087.00000,0.00012746 +1774065604,61087.00000,0.00084547 +1774065609,61087.00000,0.00081443 +1774065610,61087.00000,0.00008217 +1774065610,61087.00000,0.00009860 +1774065610,61087.00000,0.00019720 +1774065610,61087.00000,0.00019720 +1774065610,61087.00000,0.00016301 +1774065610,61087.00000,0.00008545 +1774065611,61087.00000,0.00008217 +1774065611,61087.00000,0.00016150 +1774065611,61087.00000,0.00016868 +1774065611,61087.00000,0.00016868 +1774065611,61087.00000,0.00024376 +1774065611,61087.00000,0.00008874 +1774065611,61087.00000,0.00018182 +1774065611,61087.00000,0.00016868 +1774065616,61087.00000,0.00016000 +1774065616,61087.00000,0.00016000 +1774065616,61087.00000,0.00016000 +1774065616,61087.00000,0.00016000 +1774065617,61087.00000,0.00016000 +1774065617,61087.00000,0.00016000 +1774065618,61087.00000,0.00032251 +1774065619,61087.00000,0.00032251 +1774065620,61087.00000,0.00019171 +1774065629,61100.30000,0.00010372 +1774065637,61101.80000,0.00016045 +1774065639,61101.80000,0.00016301 +1774065643,61101.80000,0.00000299 +1774065643,61111.50000,0.00004586 +1774065646,61114.80000,0.00010000 +1774065652,61107.50000,0.00004071 +1774065683,61117.10000,0.00016211 +1774065688,61116.50000,0.00024243 +1774065688,61102.70000,0.00055757 +1774065694,61103.70000,0.00005000 +1774065700,61112.40000,0.00014780 +1774065709,61106.50000,0.00010000 +1774065757,61135.00000,0.01477774 +1774065757,61136.20000,0.01088047 +1774065780,61114.20000,0.00016042 +1774065784,61112.40000,0.00010000 +1774065808,61114.10000,0.00025476 +1774065868,61128.20000,0.00033877 +1774065875,61135.00000,0.00010000 +1774065902,61143.00000,0.00477521 +1774065905,61143.00000,0.00080567 +1774065905,61143.00000,0.00064454 +1774065911,61143.00000,0.00798876 +1774066028,61167.50000,0.00014133 +1774066028,61167.60000,0.00461405 +1774066110,61146.50000,0.00049070 +1774066110,61146.40000,0.00045436 +1774066120,61146.20000,0.00245314 +1774066158,61120.80000,0.00060895 +1774066161,61120.80000,0.02630890 +1774066227,61140.80000,0.00013444 +1774066270,61141.10000,0.00008138 +1774066335,61156.60000,0.00036799 +1774066440,61153.40000,0.00076744 +1774066455,61141.30000,0.00081393 +1774066494,61142.00000,0.00059960 +1774066495,61142.00000,0.21203009 +1774066495,61144.60000,0.25444800 +1774066495,61144.60000,0.02000000 +1774066502,61150.60000,0.00009850 +1774066503,61150.20000,0.00013133 +1774066503,61150.20000,0.00008000 +1774066521,61142.00000,0.00370652 +1774066561,61146.60000,0.00130026 +1774066585,61149.30000,0.00127595 +1774066586,61149.70000,0.00032223 +1774066620,61151.40000,0.00001611 +1774066631,61153.70000,0.00326797 +1774066645,61152.40000,0.00168023 +1774066664,61165.70000,0.00005000 +1774066706,61153.20000,0.00044807 +1774066766,61179.60000,0.00234990 +1774066772,61179.50000,0.00040863 +1774066789,61175.00000,0.00010000 +1774066789,61175.00000,0.55975148 +1774066791,61175.00000,0.00672925 +1774066791,61175.00000,0.00672925 +1774066791,61175.00000,0.00332841 +1774066791,61175.00000,0.00332841 +1774066795,61175.00000,0.00333184 +1774066795,61175.00000,0.00333184 +1774066797,61175.00000,0.06569000 +1774066797,61175.00000,0.00000673 +1774066804,61135.60000,0.00242927 +1774066804,61139.90000,0.02023794 +1774066804,61141.90000,0.32710792 +1774066819,61144.40000,0.00006803 +1774066819,61122.20000,0.00048793 +1774066819,61130.70000,0.00009625 +1774066819,61130.30000,0.00960527 +1774066828,61130.30000,0.00213550 +1774066838,61130.20000,0.00119019 +1774066841,61130.10000,0.00167588 +1774066843,61130.30000,0.00007849 +1774066845,61123.60000,0.00250308 +1774066886,61138.70000,0.00010000 +1774066906,61137.10000,0.00003312 +1774066920,61137.10000,0.00001604 +1774067031,61172.80000,0.00037194 +1774067035,61180.80000,0.00032049 +1774067049,61177.30000,0.00010000 +1774067136,61184.80000,0.00010000 +1774067160,61190.20000,0.00639873 +1774067184,61193.10000,0.00139495 +1774067184,61194.10000,0.13559505 +1774067186,61181.30000,0.00300996 +1774067189,61180.20000,0.00010000 +1774067190,61180.20000,0.02235360 +1774067221,61166.30000,0.00159101 +1774067237,61180.10000,0.00014129 +1774067237,61174.40000,0.00158466 +1774067277,61196.40000,0.00010000 +1774067280,61193.30000,0.00001610 +1774067309,61197.50000,0.00081708 +1774067319,61187.70000,0.00010000 +1774067349,61186.00000,0.00347646 +1774067351,61184.50000,0.00011059 +1774067353,61181.80000,0.00061100 +1774067395,61172.00000,0.00033348 +1774067400,61147.80000,0.00016272 +1774067401,61143.90000,0.00030229 +1774067403,61124.30000,0.00013134 +1774067403,61134.20000,0.00036538 +1774067404,61130.20000,0.00033255 +1774067404,61123.50000,0.00016008 +1774067404,61123.50000,0.00016007 +1774067412,61127.70000,0.01603843 +1774067427,61125.60000,0.00021276 +1774067449,61119.60000,0.00080997 +1774067467,61110.30000,0.00015106 +1774067500,61110.20000,0.00014148 +1774067500,61110.20000,0.00014148 +1774067504,61110.20000,0.00004431 +1774067520,61081.00000,0.00001613 +1774067556,61090.40000,0.00001613 +1774067601,61087.30000,0.00014930 +1774067656,61081.00000,0.00008025 +1774067700,61081.00000,0.00040324 +1774067701,61081.00000,0.00005000 +1774067760,61080.90000,0.00009823 +1774067760,61060.20000,0.00267206 +1774067760,61060.00000,0.05576580 +1774067763,61066.20000,0.00010000 +1774067773,61061.60000,0.00036866 +1774067802,61061.50000,0.00025556 +1774067808,61061.30000,0.00015118 +1774067842,61053.00000,0.00014157 +1774067842,61053.00000,0.00009332 +1774067844,61053.00000,0.00015675 +1774067847,61053.50000,0.00498917 +1774067861,61058.50000,0.00014157 +1774067861,61058.50000,0.00649463 +1774067881,61056.80000,0.00147188 +1774067914,61072.80000,0.00410310 +1774067914,61072.90000,0.00444615 +1774067937,61081.50000,0.00016051 +1774067949,61081.10000,0.00116718 +1774067985,61073.00000,0.00008494 +1774067996,61073.00000,0.00115947 +1774067996,61073.00000,0.02107735 +1774067996,61073.00000,0.00302318 +1774067999,61073.00000,0.00047932 +1774067999,61072.90000,0.00005563 +1774068000,61073.00000,0.00017337 +1774068002,61072.90000,0.00023162 +1774068027,61072.90000,0.00071155 +1774068027,61055.10000,0.06647903 +1774068109,61063.60000,0.00402424 +1774068113,61063.80000,0.00080485 +1774068129,61065.60000,0.00032109 +1774068138,61060.10000,0.00016132 +1774068181,61060.10000,0.01307698 +1774068199,61060.00000,0.00005733 +1774068248,61055.00000,0.00010000 +1774068283,61070.00000,0.00005000 +1774068288,61066.50000,0.00033254 +1774068306,61060.40000,0.00010000 +1774068352,61067.60000,0.00158982 +1774068364,61059.90000,0.00010000 +1774068369,61060.30000,0.00023161 +1774068424,61064.60000,0.00010000 +1774068506,61068.70000,0.00065786 +1774068508,61068.80000,0.00081064 +1774068511,61065.80000,0.00010000 +1774068524,61068.40000,0.02634990 +1774068545,61068.80000,0.00010000 +1774068546,61071.30000,0.00054952 +1774068616,61076.70000,0.00004839 +1774068618,61070.40000,0.00010000 +1774068678,61053.30000,0.00006455 +1774068720,61053.30000,0.00026901 +1774068731,61053.30000,0.00077248 +1774068736,61053.20000,0.00010000 +1774068755,61054.20000,0.00154154 +1774068791,61053.20000,0.00010000 +1774068801,61053.30000,0.00036927 +1774068857,61053.20000,0.00039888 +1774068858,61053.20000,0.00010000 +1774068906,61053.70000,0.00033012 +1774068971,61059.10000,0.03244264 +1774068978,61053.30000,0.00037078 +1774068987,61053.20000,0.00010000 +1774069037,61062.20000,0.00014157 +1774069037,61064.60000,0.00002724 +1774069042,61062.20000,0.00014157 +1774069042,61064.60000,0.00001843 +1774069101,61060.10000,0.00193802 +1774069113,61054.90000,0.00337396 +1774069140,61060.00000,0.00004817 +1774069141,61054.80000,0.00058209 +1774069160,61053.70000,0.00010000 From 9cce893e804bfd5d603e1baa9d6fb1221d13c1e2 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Mon, 23 Mar 2026 12:37:48 +0100 Subject: [PATCH 6/8] export: parse opening balance from existing CAMT.053 Extract the OPBD (opening) balance amount, sign, and date from an existing CAMT.053 file during the append-mode parse. Add opening_balance_cents and opening_date fields to Camt053ParseResult and extend the round-trip test to cover them. This information is needed for the upcoming prepend mode, which must preserve the original opening balance when inserting entries before the existing statement. --- src/export/camt053.rs | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/export/camt053.rs b/src/export/camt053.rs index ff6a65a..e61ab43 100644 --- a/src/export/camt053.rs +++ b/src/export/camt053.rs @@ -213,6 +213,10 @@ fn xml_escape_comment(s: &str) -> String { /// Data extracted from an existing CAMT.053 file for append mode. pub struct Camt053ParseResult { + /// Opening balance of the existing file. + pub opening_balance_cents: i64, + /// Opening balance date of the existing file. + pub opening_date: Option, /// Set of NtryRef values already in the file (for dedup). pub existing_entry_refs: HashSet, /// All existing entries (preserved in output). @@ -238,6 +242,8 @@ pub fn parse_camt053(xml: &str) -> Result { let mut account_iban = String::new(); let mut currency = String::new(); + let mut opening_balance_cents: i64 = 0; + let mut opening_date: Option = None; let mut closing_balance_cents: i64 = 0; let mut existing_entry_refs = HashSet::new(); let mut existing_entries = Vec::new(); @@ -262,6 +268,7 @@ pub fn parse_camt053(xml: &str) -> Result { let mut bal_amount_cents: i64 = 0; let mut bal_is_credit = true; let mut bal_is_btc = false; + let mut bal_date = String::new(); loop { match reader.read_event() { @@ -286,6 +293,7 @@ pub fn parse_camt053(xml: &str) -> Result { bal_amount_cents = 0; bal_is_credit = true; bal_is_btc = false; + bal_date.clear(); } "Amt" if in_bal => { // Skip BTC-denominated balances @@ -316,12 +324,19 @@ pub fn parse_camt053(xml: &str) -> Result { if name == "Bal" && in_bal { in_bal = false; + let signed_balance = if bal_is_credit { + bal_amount_cents + } else { + -bal_amount_cents + }; + if bal_code == "OPBD" { + opening_balance_cents = signed_balance; + if !bal_date.is_empty() { + opening_date = Some(bal_date.clone()); + } + } if bal_code == "CLBD" { - closing_balance_cents = if bal_is_credit { - bal_amount_cents - } else { - -bal_amount_cents - }; + closing_balance_cents = signed_balance; } } @@ -364,6 +379,12 @@ pub fn parse_camt053(xml: &str) -> Result { bal_amount_cents = parse_amount_cents(&text)?; } "CdtDbtInd" => bal_is_credit = text == "CRDT", + "Dt" | "DtTm" if path_contains(&path, "Bal") && path_contains(&path, "Dt") => { + let trimmed = text.trim(); + if bal_date.is_empty() && !trimmed.is_empty() { + bal_date = trimmed.to_owned(); + } + } _ => {} } } else { @@ -405,6 +426,8 @@ pub fn parse_camt053(xml: &str) -> Result { .map(|e| booking_date_to_date(&e.booking_date).to_owned()); Ok(Camt053ParseResult { + opening_balance_cents, + opening_date, existing_entry_refs, existing_entries, closing_balance_cents, @@ -557,6 +580,8 @@ mod tests { assert_eq!(parsed.account_iban, "NL00XBTC0000000000"); assert_eq!(parsed.currency, "EUR"); + assert_eq!(parsed.opening_balance_cents, 0); + assert_eq!(parsed.opening_date, Some("2025-01-02".to_owned())); assert_eq!(parsed.closing_balance_cents, 499_985); assert_eq!(parsed.existing_entry_refs.len(), 2); assert!(parsed.existing_entry_refs.contains("100:abcdef01234567890abc:0")); From 29ee03a3eae8cd6bc2a7b827783ecdb849a36051 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Mon, 23 Mar 2026 12:38:23 +0100 Subject: [PATCH 7/8] export: handle prepend semantics when start-date predates existing file When the output file already exists and --start-date is earlier than the file's opening balance date, switch to prepend mode: build entries only for the gap period, then splice them before the existing entries and recalculate statement totals. Reject attempts to move the start date forward past the existing opening date. When prepending, omit the wallet balance check (the known-good closing balance is in the existing file). Add ExistingExportPlan, ExistingMergeMode, merge_with_existing_statement, refresh_statement_totals and supporting helpers with unit tests. --- src/commands/export.rs | 353 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 327 insertions(+), 26 deletions(-) diff --git a/src/commands/export.rs b/src/commands/export.rs index 9879ea6..0ccd7f1 100644 --- a/src/commands/export.rs +++ b/src/commands/export.rs @@ -1,17 +1,22 @@ use std::env; +use std::collections::HashSet; use std::fs::File; use std::io::BufWriter; use std::path::PathBuf; use anyhow::{Context, Result, bail}; -use chrono::NaiveDate; +use chrono::{Duration, NaiveDate}; use crate::accounting::{AccountingConfig, build_statement}; use crate::common::{AppConfig, default_bitcoin_datadir}; +use crate::export::{Entry, Statement}; +use crate::export::booking_date_to_date; +use crate::export::camt053::Camt053ParseResult; use crate::exchange_rate::KrakenProvider; use crate::export::camt053::Camt053Exporter; use crate::export::AccountingExporter; use crate::iban::iban_from_fingerprint; +use crate::import::WalletTransaction; use crate::import::TransactionSource; use crate::import::bitcoin_core_rpc::BitcoinCoreRpc; @@ -57,6 +62,24 @@ pub enum ExportFormat { Camt053, } +#[derive(Debug)] +struct ExistingExportPlan { + merge_mode: ExistingMergeMode, + build_opening_balance_cents: i64, + build_start_date: Option, + build_end_exclusive: Option, + existing_opening_balance_cents: i64, + existing_opening_date: String, + existing_entries: Vec, + existing_entry_refs: HashSet, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ExistingMergeMode { + Append, + Prepend, +} + pub fn run(args: ExportArgs) -> Result<()> { // Resolve wallet name: use provided, or auto-detect the single loaded wallet let wallet = match args.wallet { @@ -80,7 +103,7 @@ pub fn run(args: ExportArgs) -> Result<()> { let iban = iban_from_fingerprint(&fingerprint, &args.country, &args.chain)?; eprintln!("Virtual IBAN: {iban}"); - let transactions = rpc.list_transactions()?; + let mut transactions = rpc.list_transactions()?; let wallet_balance_sats = rpc.get_balance()?; // Collect receive addresses and fetch matching watch-only descriptors @@ -108,7 +131,7 @@ pub fn run(args: ExportArgs) -> Result<()> { // Auto-detect append mode: if output file exists, parse it for dedup and continuation let append = args.output.exists(); - let (opening_balance_cents, start_date, existing_entries, existing_entry_refs) = if append { + let existing_plan = if append { let output_path = &args.output; let existing_xml = std::fs::read_to_string(output_path) @@ -125,18 +148,38 @@ pub fn run(args: ExportArgs) -> Result<()> { bail!("currency mismatch: file has {} but current config uses {}", parsed.currency, currency); } - // Use last booking date + 1 day as implicit start date (unless explicitly set) - let implicit_start = parsed.last_booking_date - .and_then(|d| chrono::NaiveDate::parse_from_str(&d, "%Y-%m-%d").ok()) - .map(|d| d + chrono::Duration::days(1)); - - let start = args.start_date.or(implicit_start); - - (parsed.closing_balance_cents, start, parsed.existing_entries, parsed.existing_entry_refs) + let plan = existing_export_plan_from_existing_export(args.start_date, parsed)?; + if plan.merge_mode == ExistingMergeMode::Prepend { + eprintln!( + "Requested start date predates the existing export; prepending entries before {} and replacing the opening balance in {}.", + plan.existing_opening_date, + output_path.display(), + ); + } + Some(plan) } else { - (0, args.start_date, Vec::new(), std::collections::HashSet::new()) + None }; + if let Some(plan) = existing_plan.as_ref() { + if !plan.existing_entry_refs.is_empty() { + transactions.retain(|tx| !plan.existing_entry_refs.contains(&transaction_entry_ref(tx))); + } + if let Some(end_exclusive) = plan.build_end_exclusive { + let end_exclusive_ts = end_exclusive + .and_hms_opt(0, 0, 0) + .expect("midnight should be valid") + .and_utc() + .timestamp(); + transactions.retain(|tx| tx.block_time < end_exclusive_ts); + } + } + + let existing_entry_refs = existing_plan + .as_ref() + .map(|plan| plan.existing_entry_refs.clone()) + .unwrap_or_default(); + let bank_name = Some(args.bank_name.unwrap_or_else(|| format!("Bitcoin Core - {wallet}"))); let config = AccountingConfig { @@ -146,26 +189,36 @@ pub fn run(args: ExportArgs) -> Result<()> { currency: currency.clone(), account_iban: iban, candle_interval_minutes: candle_minutes, - start_date, - opening_balance_cents, + start_date: existing_plan + .as_ref() + .map(|plan| plan.build_start_date) + .unwrap_or(args.start_date), + opening_balance_cents: existing_plan + .as_ref() + .map(|plan| plan.build_opening_balance_cents) + .unwrap_or(0), bank_name, - wallet_balance_sats: Some(wallet_balance_sats), + wallet_balance_sats: if existing_plan + .as_ref() + .is_some_and(|plan| plan.merge_mode == ExistingMergeMode::Prepend) + { + None + } else { + Some(wallet_balance_sats) + }, ignore_balance_mismatch: args.ignore_balance_mismatch, }; let mut statement = build_statement(&transactions, &provider, &config)?; statement.descriptors = descriptors; - // Dedup: remove entries that already exist in the parsed file - if !existing_entry_refs.is_empty() { - statement.entries.retain(|e| !existing_entry_refs.contains(&e.entry_ref)); - } - - // Prepend existing entries - if !existing_entries.is_empty() { - let mut all_entries = existing_entries; - all_entries.append(&mut statement.entries); - statement.entries = all_entries; + if let Some(plan) = existing_plan { + if !existing_entry_refs.is_empty() { + statement + .entries + .retain(|e| !existing_entry_refs.contains(&e.entry_ref)); + } + merge_with_existing_statement(&mut statement, plan); } // Count new entries (excluding those from the existing file) @@ -237,6 +290,117 @@ fn resolve_candle_minutes(candle_override_minutes: Option, default_candle_m .unwrap_or(1440) } +fn existing_export_plan_from_existing_export( + requested_start_date: Option, + parsed: Camt053ParseResult, +) -> Result { + let existing_opening_date = parse_existing_opening_date(parsed.opening_date.as_deref())?; + let last_booking_date = parse_last_booking_date(parsed.last_booking_date.as_deref())?; + + if let Some(requested_start_date) = requested_start_date { + if requested_start_date > existing_opening_date { + bail!( + "output file already starts at {existing_opening_date}; cannot move --start-date forward to {requested_start_date}" + ); + } + + if requested_start_date < existing_opening_date { + return Ok(ExistingExportPlan { + merge_mode: ExistingMergeMode::Prepend, + build_opening_balance_cents: 0, + build_start_date: Some(requested_start_date), + build_end_exclusive: Some(existing_opening_date), + existing_opening_balance_cents: parsed.opening_balance_cents, + existing_opening_date: existing_opening_date.to_string(), + existing_entries: parsed.existing_entries, + existing_entry_refs: parsed.existing_entry_refs, + }); + } + + return Ok(ExistingExportPlan { + merge_mode: ExistingMergeMode::Append, + build_opening_balance_cents: parsed.closing_balance_cents, + build_start_date: last_booking_date.map(|date| date + Duration::days(1)), + build_end_exclusive: None, + existing_opening_balance_cents: parsed.opening_balance_cents, + existing_opening_date: existing_opening_date.to_string(), + existing_entries: parsed.existing_entries, + existing_entry_refs: parsed.existing_entry_refs, + }); + } + + Ok(ExistingExportPlan { + merge_mode: ExistingMergeMode::Append, + build_opening_balance_cents: parsed.closing_balance_cents, + build_start_date: last_booking_date.map(|date| date + Duration::days(1)), + build_end_exclusive: None, + existing_opening_balance_cents: parsed.opening_balance_cents, + existing_opening_date: existing_opening_date.to_string(), + existing_entries: parsed.existing_entries, + existing_entry_refs: parsed.existing_entry_refs, + }) +} + +fn parse_existing_opening_date(opening_date: Option<&str>) -> Result { + let opening_date = opening_date.context("existing CAMT.053 file is missing its opening balance date")?; + NaiveDate::parse_from_str(opening_date, "%Y-%m-%d") + .with_context(|| format!("invalid opening balance date in existing CAMT.053 file: {opening_date}")) +} + +fn parse_last_booking_date(last_booking_date: Option<&str>) -> Result> { + last_booking_date + .map(|date| { + NaiveDate::parse_from_str(date, "%Y-%m-%d") + .with_context(|| format!("invalid booking date in existing CAMT.053 file: {date}")) + }) + .transpose() +} + +fn transaction_entry_ref(tx: &WalletTransaction) -> String { + format!("{}:{}:{}", tx.block_height, &tx.txid[..20.min(tx.txid.len())], tx.vout) +} + +fn merge_with_existing_statement(statement: &mut Statement, plan: ExistingExportPlan) { + let mut combined_entries = match plan.merge_mode { + ExistingMergeMode::Append => { + let mut combined = plan.existing_entries; + combined.append(&mut statement.entries); + statement.opening_balance_cents = plan.existing_opening_balance_cents; + statement.opening_date = plan.existing_opening_date; + statement.opening_balance_sats = 0; + statement.opening_rate = None; + combined + } + ExistingMergeMode::Prepend => { + let mut combined = statement.entries.clone(); + combined.extend(plan.existing_entries); + combined + } + }; + statement.entries.clear(); + statement.entries.append(&mut combined_entries); + refresh_statement_totals(statement); +} + +fn refresh_statement_totals(statement: &mut Statement) { + let net_entries = statement.entries.iter().map(signed_entry_amount).sum::(); + statement.closing_balance_cents = statement.opening_balance_cents + net_entries; + statement.statement_date = statement + .entries + .last() + .map(|entry| booking_date_to_date(&entry.booking_date).to_owned()) + .unwrap_or_else(|| statement.opening_date.clone()); + statement.statement_id = format!("STMT-{}", statement.statement_date); +} + +fn signed_entry_amount(entry: &Entry) -> i64 { + if entry.is_credit { + entry.amount_cents + } else { + -entry.amount_cents + } +} + pub fn parse_args_from(args: I, usage: &str) -> Result where I: IntoIterator, @@ -375,7 +539,18 @@ where #[cfg(test)] mod tests { - use super::{ExportFormat, ExportArgs, USAGE, parse_args_from, resolve_candle_minutes}; + use std::collections::HashSet; + + use chrono::NaiveDate; + + use crate::export::Entry; + use crate::import::{TxCategory, WalletTransaction}; + + use super::{ + Camt053ParseResult, ExistingMergeMode, ExportArgs, ExportFormat, USAGE, + existing_export_plan_from_existing_export, parse_args_from, resolve_candle_minutes, + transaction_entry_ref, + }; #[test] fn parses_export_args_without_candle_override() { @@ -446,4 +621,130 @@ mod tests { fn export_defaults_to_daily_when_no_override_or_env_default_exists() { assert_eq!(resolve_candle_minutes(None, None), 1440); } + + #[test] + fn append_plan_uses_day_after_last_booking_when_no_start_date_is_provided() { + let parsed = Camt053ParseResult { + opening_balance_cents: 100, + opening_date: Some("2025-01-01".to_owned()), + existing_entry_refs: HashSet::from(["100:abcd:0".to_owned()]), + existing_entries: vec![Entry { + entry_ref: "100:abcd:0".to_owned(), + full_ref: "full".to_owned(), + booking_date: "2025-01-01T00:00:00".to_owned(), + amount_cents: 100, + is_credit: true, + description: "test".to_owned(), + is_fee: false, + }], + closing_balance_cents: 123, + account_iban: "NL00XBTC0000000000".to_owned(), + currency: "EUR".to_owned(), + last_booking_date: Some("2025-01-01".to_owned()), + descriptors: vec![], + }; + + let plan = existing_export_plan_from_existing_export(None, parsed).expect("plan"); + + assert_eq!(plan.build_opening_balance_cents, 123); + assert_eq!( + plan.build_start_date, + Some(NaiveDate::from_ymd_opt(2025, 1, 2).expect("date")) + ); + assert_eq!(plan.existing_entries.len(), 1); + assert!(plan.existing_entry_refs.contains("100:abcd:0")); + assert_eq!(plan.merge_mode, ExistingMergeMode::Append); + } + + #[test] + fn earlier_start_date_triggers_prepend_mode() { + let parsed = Camt053ParseResult { + opening_balance_cents: 100, + opening_date: Some("2025-01-01".to_owned()), + existing_entry_refs: HashSet::from(["100:abcd:0".to_owned()]), + existing_entries: vec![Entry { + entry_ref: "100:abcd:0".to_owned(), + full_ref: "full".to_owned(), + booking_date: "2025-01-01T00:00:00".to_owned(), + amount_cents: 100, + is_credit: true, + description: "test".to_owned(), + is_fee: false, + }], + closing_balance_cents: 123, + account_iban: "NL00XBTC0000000000".to_owned(), + currency: "EUR".to_owned(), + last_booking_date: Some("2025-12-31".to_owned()), + descriptors: vec![], + }; + + let plan = existing_export_plan_from_existing_export( + Some(NaiveDate::from_ymd_opt(2024, 1, 1).expect("date")), + parsed, + ) + .expect("plan"); + + assert_eq!(plan.build_opening_balance_cents, 0); + assert_eq!( + plan.build_start_date, + Some(NaiveDate::from_ymd_opt(2024, 1, 1).expect("date")) + ); + assert_eq!( + plan.build_end_exclusive, + Some(NaiveDate::from_ymd_opt(2025, 1, 1).expect("date")) + ); + assert_eq!(plan.existing_entries.len(), 1); + assert!(plan.existing_entry_refs.contains("100:abcd:0")); + assert_eq!(plan.merge_mode, ExistingMergeMode::Prepend); + } + + #[test] + fn moving_start_date_forward_is_rejected() { + let parsed = Camt053ParseResult { + opening_balance_cents: 100, + opening_date: Some("2025-01-01".to_owned()), + existing_entry_refs: HashSet::from(["100:abcd:0".to_owned()]), + existing_entries: vec![Entry { + entry_ref: "100:abcd:0".to_owned(), + full_ref: "full".to_owned(), + booking_date: "2025-01-01T00:00:00".to_owned(), + amount_cents: 100, + is_credit: true, + description: "test".to_owned(), + is_fee: false, + }], + closing_balance_cents: 123, + account_iban: "NL00XBTC0000000000".to_owned(), + currency: "EUR".to_owned(), + last_booking_date: Some("2025-12-31".to_owned()), + descriptors: vec![], + }; + + let err = existing_export_plan_from_existing_export( + Some(NaiveDate::from_ymd_opt(2025, 6, 1).expect("date")), + parsed, + ) + .expect_err("moving start date forward should fail"); + + assert!(err.to_string().contains("already starts at 2025-01-01")); + assert!(err.to_string().contains("--start-date forward to 2025-06-01")); + } + + #[test] + fn transaction_entry_ref_matches_export_entry_reference_format() { + let tx = WalletTransaction { + txid: "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789".to_owned(), + vout: 7, + amount_sats: 1, + fee_sats: None, + category: TxCategory::Receive, + block_time: 1_735_700_000, + block_height: 123, + block_hash: "bb".repeat(32), + address: "bc1qtest".to_owned(), + label: String::new(), + }; + + assert_eq!(transaction_entry_ref(&tx), "123:abcdef0123456789abcd:7"); + } } From eb94c10d15363df95b21f2e60d9072da73025884 Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Mon, 23 Mar 2026 13:54:49 +0100 Subject: [PATCH 8/8] Add .claude to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 228e4ac..eb59da9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .cache/ target/ AGENTS.md +.claude # Symlink to Bitcoin Core bitcoin-core