diff --git a/.gitignore b/.gitignore index 228e4ac..eb59da9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .cache/ target/ AGENTS.md +.claude # Symlink to Bitcoin Core bitcoin-core diff --git a/Cargo.lock b/Cargo.lock index b70688a..277b91d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -17,6 +23,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -59,6 +74,7 @@ dependencies = [ "capnp", "capnp-rpc", "chrono", + "csv", "dotenvy", "fixed_decimal", "futures", @@ -72,6 +88,7 @@ dependencies = [ "tempfile", "tokio", "tokio-util", + "zip", ] [[package]] @@ -165,6 +182,53 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -188,6 +252,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9eb1aa714776b75c7e67e1da744b81a129b3ff919c8712b5e1b32252c1f07cc7" +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "errno" version = "0.3.14" @@ -221,6 +291,16 @@ dependencies = [ "writeable", ] +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -345,6 +425,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + [[package]] name = "http" version = "1.4.0" @@ -616,6 +702,16 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -684,6 +780,16 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.1.1" @@ -1046,6 +1152,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + [[package]] name = "slab" version = "0.4.12" @@ -1763,8 +1875,37 @@ dependencies = [ "syn", ] +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror", + "zopfli", +] + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index 625133e..2536685 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ license = "MIT" [dependencies] anyhow = "1.0.97" chrono = { version = "0.4.40", default-features = false, features = ["clock", "std"] } +csv = "1.3.1" dotenvy = "0.15.7" fixed_decimal = "0.7.1" icu_decimal = "2.1.1" @@ -15,6 +16,7 @@ quick-xml = "0.37" reqwest = { version = "0.12.15", default-features = false, features = ["blocking", "json", "rustls-tls", "socks"] } serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.140" +zip = { version = "2.2.3", default-features = false, features = ["deflate"] } [dev-dependencies] bitcoin-capnp-types = { git = "https://github.com/2140-dev/bitcoin-capnp-types" } diff --git a/README.md b/README.md index d85cc91..1c3c210 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Rust CLI for Bitcoin accounting tasks. -Current commands: `received-value`, `export`, `reconstruct`. +Current commands: `received-value`, `cache-rates`, `export`, `reconstruct`. ## Available commands @@ -18,6 +18,40 @@ The command works as follows: 4. Uses the smallest Kraken `OHLC` candle interval that can still cover the transaction confirmation time (override with `--candle `). 5. Estimates the value of the received BTC in the quote currency of the chosen Kraken pair. +### `cache-rates` + +Populate `.cache/rates.json` for one UTC calendar year of Bitcoin rates. + +```bash +cargo run -- cache-rates 2024 +cargo run -- cache-rates --vwap 2024 +cargo run -- cache-rates --vwap --candle 60 2024 +``` + +The command works as follows: + +1. Without `--vwap`, fetches the daily (`1440`) Kraken `OHLC` data that is still available through the public API. +2. Detects which closed UTC candles in the requested year are still missing because they are older than Kraken's 720-candle retention window. +3. Backfills those missing candles into `.cache/rates.json` from one of Kraken's downloadable archives: + the default OHLCVT archive, or the larger time-and-sales archive when `--vwap` is set. + +By default, the cache keys are written as normal `1440`-minute entries. With `--vwap`, the command writes entries at `DEFAULT_CANDLE_MINUTES` if configured, or `1440` otherwise; `--candle ` overrides that in the normal way. + +Trade-off: + +- Recent days use Kraken's daily `OHLC` API `vwap`, just like the normal live lookup path. +- Without `--vwap`, older archive-backed days use the daily `(open + close) / 2` midpoint derived from Kraken's OHLCVT CSV, because the downloadable OHLCVT archive does not include `vwap`. +- Without `--vwap`, the command only fills missing cache entries for that year and interval; it does not overwrite existing entries with midpoint-derived values. +- With `--vwap`, the command computes exact Kraken VWAP candles at the chosen interval from the downloadable time-and-sales trade archive (`timestamp,price,volume`) and overwrites any existing cache entries for that year and interval. +- With `--vwap`, the command first checks whether the public API still covers part of the requested year at the chosen interval. If quarterly trade archives are available for the rest, it keeps the API slice and downloads only the missing quarters. If the complete trade archive is inevitable anyway, it skips the API and computes the whole year from the archive. +- With `--vwap`, the command tries quarterly trade archives first and falls back to Kraken's complete trade archive automatically if that year's quarterly trade ZIPs are not published. +- Downloaded Kraken archive ZIPs are kept under `.cache/kraken/` and reused across later runs; the command does not keep expanded CSV files on disk. +- `--vwap` is substantially heavier because Kraken's trade archives are much larger than the OHLCVT archives, especially when the complete trade archive ZIP is needed instead of quarterly updates. + +For years whose quarterly trade ZIPs are not published, `cache-rates --vwap` +falls back to Kraken's complete trade archive, which requires a ~12G download. +But it does mean VWAP values can be constructed as far back as late 2013. + ### `export`

@@ -46,6 +80,7 @@ Key options: - `--mark-to-market` — add year-end reconciliation entries (default on in fiat mode, can be combined with `--fifo`) - `--output ` — output file path (appends if file exists) - `--start-date ` — only include transactions from this date +- `--candle ` — Kraken candle interval (`DEFAULT_CANDLE_MINUTES` or `1440` by default) - `--datadir ` — Bitcoin Core data directory (for cookie auth) - `--chain ` — chain: main, testnet3, testnet4, signet, regtest (default: main) @@ -168,6 +203,8 @@ transaction_age <= 720 * interval_minutes * 60 If the transaction is too old to fit inside Kraken's `1d` candle retention window, the tool exits with an error instead of silently switching to a coarser interval. +`cache-rates` is the explicit opt-in workaround for older values: by default it backfills `.cache/rates.json` from Kraken's OHLCVT archive as daily `(open + close) / 2` midpoint prices, or with `--vwap` it computes exact Kraken VWAP candles at the chosen interval from the larger trade archive. + ## Development See [DEVELOP.md](DEVELOP.md) for Bitcoin Core build instructions, running @@ -191,6 +228,7 @@ Licensed under the MIT License. See `LICENSE` for details. - `src/main.rs` — top-level command dispatcher - `src/lib.rs` — library crate root - `src/commands/received_value.rs` — `received-value` subcommand +- `src/commands/cache_rates.rs` — `cache-rates` subcommand - `src/commands/export.rs` — `export` subcommand - `src/commands/reconstruct.rs` — `reconstruct` subcommand - `src/common.rs` — shared config, mempool, Kraken, Tor, candle, and formatting logic diff --git a/src/commands/cache_rates.rs b/src/commands/cache_rates.rs new file mode 100644 index 0000000..583a887 --- /dev/null +++ b/src/commands/cache_rates.rs @@ -0,0 +1,2331 @@ +use std::collections::{BTreeSet, HashMap}; +use std::fs::{self, File}; +use std::io; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, anyhow, bail}; +use chrono::{Datelike, NaiveDate, Utc}; +use csv::StringRecord; +use reqwest::blocking::{Client, Response}; +use zip::ZipArchive; + +use crate::common::{ + AppConfig, KRAKEN_INTERVALS_MINUTES, build_http_client, fetch_candles_since, + parse_candle_interval_minutes, +}; +use crate::exchange_rate::{CACHE_DIR, cache_key, cache_path, load_disk_cache, save_disk_cache}; + +pub const SUBCOMMAND_NAME: &str = "cache-rates"; +pub const USAGE: &str = "usage: btc_fiat_value cache-rates [--vwap] [--candle ] "; + +const KRAKEN_DAILY_INTERVAL_MINUTES: u32 = 1_440; +const QUARTERLY_ARCHIVE_FIRST_YEAR: i32 = 2023; +// Kraken archive references: +// - OHLC REST retention and daily 1440 candles: https://docs.kraken.com/api/docs/rest-api/get-ohlc-data/ +// - Downloadable OHLCVT archive landing page: https://support.kraken.com/articles/360047124832-downloadable-historical-ohlcvt-open-high-low-close-volume-trades-data +// If Kraken changes the Google Drive ids or stops publishing them there, update these constants. +const COMPLETE_OHLCVT_ARCHIVE_FILE_ID: &str = "1ptNqWYidLkhb2VAKuLCxmp2OXEfGO-AP"; +const QUARTERLY_OHLCVT_ARCHIVE_FOLDER_ID: &str = "15RSlNuW_h0kVM8or8McOGOMfHeBFvFGI"; +// Kraken time-and-sales archive references: +// - Downloadable trade archive landing page: https://support.kraken.com/articles/360047543791-downloadable-historical-market-data-time-and-sales- +// - OHLC REST docs for the live daily VWAP path: https://docs.kraken.com/api/docs/rest-api/get-ohlc-data/ +// If Kraken changes the Google Drive ids or stops publishing them there, update these constants. +const COMPLETE_TRADE_ARCHIVE_FILE_ID: &str = "10zh3tDpqANYvVtYVgczwVz3UZFRUb1el"; +const QUARTERLY_TRADE_ARCHIVE_FOLDER_ID: &str = "188O9xQjZTythjyLNes_5zfMEFaMbTT22"; + +#[derive(Debug, Eq, PartialEq)] +pub struct CacheRatesArgs { + pub year: i32, + pub use_vwap_archive: bool, + pub candle_override_minutes: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct DriveFile { + id: String, + name: String, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct PreparedArchive { + archive_path: PathBuf, + archive_file_name: String, + extracted_path: PathBuf, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct PreparedArchiveFile { + file: DriveFile, + archive: PreparedArchive, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct ArchiveCoverage { + first: i64, + last: i64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ArchiveBackfillMode { + Midpoint, + Vwap, +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +struct CacheWriteStats { + inserted: usize, + replaced: usize, + skipped: usize, +} + +impl CacheWriteStats { + fn record(&mut self, outcome: CacheWriteOutcome) { + match outcome { + CacheWriteOutcome::Inserted => self.inserted += 1, + CacheWriteOutcome::Replaced => self.replaced += 1, + CacheWriteOutcome::Skipped => self.skipped += 1, + } + } + + fn absorb(&mut self, other: CacheWriteStats) { + self.inserted += other.inserted; + self.replaced += other.replaced; + self.skipped += other.skipped; + } + + fn written(&self) -> usize { + self.inserted + self.replaced + } + + fn total(&self) -> usize { + self.inserted + self.replaced + self.skipped + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum CacheWriteOutcome { + Inserted, + Replaced, + Skipped, +} + +pub fn run(args: CacheRatesArgs) -> Result<()> { + let _temp_cache = TempCacheGuard::new()?; + let config = AppConfig::from_env()?; + let archive_pair = archive_pair_name(&config.kraken_pair)?; + let archive_mode = if args.use_vwap_archive { + ArchiveBackfillMode::Vwap + } else { + ArchiveBackfillMode::Midpoint + }; + let target_interval_minutes = target_interval_minutes(&args, &config)?; + let now = Utc::now(); + let (start_ts, end_ts) = closed_interval_year_bounds(args.year, now, target_interval_minutes)?; + + let mut missing_starts = + expected_interval_starts(start_ts, end_ts, target_interval_minutes); + let mut cache = load_disk_cache(); + let existing_cache_count = count_cached_starts( + &missing_starts, + &cache, + &config.kraken_pair, + target_interval_minutes, + ); + if archive_mode == ArchiveBackfillMode::Midpoint { + drop_cached_starts( + &mut missing_starts, + &cache, + &config.kraken_pair, + target_interval_minutes, + ); + } + let full_span_quarters = missing_quarters(&missing_starts); + let quarterly_files = if !missing_starts.is_empty() && args.year >= QUARTERLY_ARCHIVE_FIRST_YEAR { + let archive_client = build_http_client("Kraken archive", None)?; + eprintln!( + "Fetching Kraken {} archive listing for {}...", + archive_mode.archive_label(), + args.year, + ); + Some(fetch_quarterly_archive_files( + &archive_client, + archive_mode.quarterly_archive_folder_id(), + archive_mode.quarterly_archive_prefix(), + archive_mode.archive_label(), + )?) + } else { + None + }; + let has_full_quarterly_coverage = quarterly_files + .as_ref() + .and_then(|files| { + resolve_quarterly_archive_files( + files, + archive_mode.quarterly_archive_prefix(), + args.year, + &full_span_quarters, + ) + }) + .is_some(); + let api_can_cover_entire_span = missing_starts.len() <= 720; + let should_fetch_api = match archive_mode { + ArchiveBackfillMode::Midpoint => true, + ArchiveBackfillMode::Vwap => api_can_cover_entire_span || has_full_quarterly_coverage, + }; + + let mut recent_stats = CacheWriteStats::default(); + if should_fetch_api { + let kraken_client = build_http_client("clearnet Kraken", None)?; + eprintln!( + "Fetching Kraken OHLC API rows at {}-minute resolution for {}...", + target_interval_minutes, + args.year, + ); + let candles = fetch_candles_since( + &kraken_client, + &config, + target_interval_minutes, + start_ts, + )?; + + for candle in candles { + if candle.time < start_ts || candle.time >= end_ts { + continue; + } + if !should_store_api_candle(archive_mode, &missing_starts, candle.time) { + continue; + } + + recent_stats.record(store_cache_value( + &mut cache, + &config.kraken_pair, + target_interval_minutes, + candle.time, + candle.vwap, + )); + missing_starts.remove(&candle.time); + } + } + + let mut archive_stats = CacheWriteStats::default(); + let mut fallback_stats = CacheWriteStats::default(); + let mut used_complete_archive_fallback = false; + let mut prepared_archives = Vec::new(); + if !missing_starts.is_empty() { + let archive_client = build_http_client("Kraken archive", None)?; + + let needed_quarters = missing_quarters(&missing_starts); + let quarterly_backfill_files = quarterly_files + .as_ref() + .and_then(|files| { + resolve_quarterly_archive_files( + files, + archive_mode.quarterly_archive_prefix(), + args.year, + &needed_quarters, + ) + }); + + if let Some(files) = quarterly_backfill_files { + for file in files { + let prepared_archive = prepare_archive_for_backfill( + &archive_client, + &file, + &archive_pair, + end_ts, + target_interval_minutes, + archive_mode, + )?; + archive_stats.absorb(read_prepared_archive( + &prepared_archive, + &file, + &config.kraken_pair, + &archive_pair, + start_ts, + end_ts, + target_interval_minutes, + None, + &mut missing_starts, + &mut cache, + archive_mode, + )?); + prepared_archives.push(PreparedArchiveFile { + file, + archive: prepared_archive, + }); + } + } else { + used_complete_archive_fallback = true; + let file = DriveFile { + id: archive_mode.complete_archive_file_id().to_owned(), + name: archive_mode.complete_archive_name().to_owned(), + }; + let prepared_archive = prepare_archive_for_backfill( + &archive_client, + &file, + &archive_pair, + end_ts, + target_interval_minutes, + archive_mode, + )?; + archive_stats.absorb(read_prepared_archive( + &prepared_archive, + &file, + &config.kraken_pair, + &archive_pair, + start_ts, + end_ts, + target_interval_minutes, + None, + &mut missing_starts, + &mut cache, + archive_mode, + )?); + prepared_archives.push(PreparedArchiveFile { + file, + archive: prepared_archive, + }); + } + } + + if archive_mode == ArchiveBackfillMode::Vwap && !missing_starts.is_empty() { + for fallback_interval_minutes in fallback_trade_intervals(target_interval_minutes) { + let missing_before = missing_starts.len(); + eprintln!( + "Warning: exact {}-minute trade VWAP was unavailable for {} interval(s); falling back to {}-minute trade VWAP for those gaps.", + target_interval_minutes, + missing_before, + fallback_interval_minutes, + ); + let mut level_stats = CacheWriteStats::default(); + for prepared_archive in &prepared_archives { + level_stats.absorb(read_prepared_archive( + &prepared_archive.archive, + &prepared_archive.file, + &config.kraken_pair, + &archive_pair, + start_ts, + end_ts, + target_interval_minutes, + Some(fallback_interval_minutes), + &mut missing_starts, + &mut cache, + archive_mode, + )?); + } + fallback_stats.absorb(level_stats); + if level_stats.written() > 0 { + eprintln!( + "Filled {} {}-minute gap(s) using {}-minute trade VWAP.", + level_stats.written(), + target_interval_minutes, + fallback_interval_minutes, + ); + } + if missing_starts.is_empty() { + break; + } + } + } + + if !missing_starts.is_empty() { + let first_missing = missing_starts + .iter() + .next() + .copied() + .context("missing intervals should not be empty")?; + bail!( + "cache is incomplete for {}: {} {}-minute candle(s) still missing, starting at {}", + args.year, + missing_starts.len(), + target_interval_minutes, + format_interval_start(first_missing, target_interval_minutes)? + ); + } + + save_disk_cache(&cache)?; + + let inserted_count = recent_stats.inserted + archive_stats.inserted + fallback_stats.inserted; + let replaced_count = recent_stats.replaced + archive_stats.replaced + fallback_stats.replaced; + let skipped_count = if archive_mode == ArchiveBackfillMode::Midpoint { + existing_cache_count + } else { + recent_stats.skipped + archive_stats.skipped + fallback_stats.skipped + }; + let total_count = inserted_count + replaced_count + skipped_count; + eprintln!( + "Cached {total_count} {}-minute rate(s) for {} in {}.", + target_interval_minutes, + args.year, + cache_path().display() + ); + eprintln!("Inserted cache entries: {inserted_count}"); + eprintln!("Replaced existing cache entries: {replaced_count}"); + eprintln!("Skipped existing cache entries: {skipped_count}"); + eprintln!("Kraken OHLC API rows: {}", recent_stats.written()); + eprintln!("{}: {}", archive_mode.summary_label(), archive_stats.written()); + if fallback_stats.written() > 0 { + eprintln!( + "Trade archive larger-candle fallback rows: {}", + fallback_stats.written() + ); + } + if used_complete_archive_fallback + && (archive_stats.total() > 0 || fallback_stats.total() > 0) + { + eprintln!( + "Quarterly {} archives were unavailable for {}, so the command fell back to the complete {} archive.", + archive_mode.archive_label(), + args.year, + archive_mode.archive_label(), + ); + } + if archive_stats.total() > 0 || fallback_stats.total() > 0 { + match archive_mode { + ArchiveBackfillMode::Midpoint => eprintln!( + "Archive-backed rows use the daily (open + close) / 2 midpoint because Kraken's OHLCVT CSV does not include VWAP. Re-run with --vwap for exact trade-derived VWAP." + ), + ArchiveBackfillMode::Vwap => eprintln!( + "Archive-backed rows use exact {}-minute VWAP computed from Kraken's time-and-sales trade archive, overwriting any existing cache entries for that year and interval.", + target_interval_minutes + ), + } + } + + Ok(()) +} + +pub fn parse_args_from(args: I, usage: &str) -> Result +where + I: IntoIterator, +{ + let mut year = None; + let mut use_vwap_archive = false; + let mut candle_override_minutes = None; + let mut args = args.into_iter(); + + while let Some(arg) = args.next() { + if arg == "--vwap" { + use_vwap_archive = true; + continue; + } + if arg == "--candle" { + let value = args.next().ok_or_else(|| anyhow!("{usage}"))?; + candle_override_minutes = Some(parse_candle_interval_minutes(&value, "--candle")?); + continue; + } + if let Some(value) = arg.strip_prefix("--candle=") { + candle_override_minutes = Some(parse_candle_interval_minutes(value, "--candle")?); + continue; + } + + if year.is_some() { + bail!("{usage}"); + } + year = Some(arg); + } + + if candle_override_minutes.is_some() && !use_vwap_archive { + bail!("--candle requires --vwap\n\n{usage}"); + } + + let year = year.ok_or_else(|| anyhow!("{usage}"))?; + let year = year + .parse::() + .with_context(|| format!("invalid year: {year}"))?; + + Ok(CacheRatesArgs { + year, + use_vwap_archive, + candle_override_minutes, + }) +} + +fn target_interval_minutes(args: &CacheRatesArgs, config: &AppConfig) -> Result { + if !args.use_vwap_archive { + return Ok(KRAKEN_DAILY_INTERVAL_MINUTES); + } + + Ok(args + .candle_override_minutes + .or(config.default_candle_minutes) + .unwrap_or(KRAKEN_DAILY_INTERVAL_MINUTES)) +} + +fn closed_interval_year_bounds( + year: i32, + now: chrono::DateTime, + interval_minutes: u32, +) -> Result<(i64, i64)> { + let year_start = NaiveDate::from_ymd_opt(year, 1, 1) + .ok_or_else(|| anyhow!("invalid year: {year}"))?; + let next_year_start = NaiveDate::from_ymd_opt(year + 1, 1, 1) + .ok_or_else(|| anyhow!("invalid year: {year}"))?; + let year_start_ts = midnight_utc_timestamp(year_start); + let next_year_start_ts = midnight_utc_timestamp(next_year_start); + let interval_seconds = i64::from(interval_minutes) * 60; + + if year > now.year() { + bail!("year {year} is in the future"); + } + + let year_end_exclusive = if year == now.year() { + now.timestamp().div_euclid(interval_seconds) * interval_seconds + } else { + next_year_start_ts + }; + + if year_start_ts >= year_end_exclusive { + bail!( + "year {year} has no closed UTC {}-minute candles yet", + interval_minutes + ); + } + + Ok((year_start_ts, year_end_exclusive.min(next_year_start_ts))) +} + +fn expected_interval_starts(start_ts: i64, end_ts: i64, interval_minutes: u32) -> BTreeSet { + let mut starts = BTreeSet::new(); + let interval_seconds = i64::from(interval_minutes) * 60; + let mut ts = start_ts; + while ts < end_ts { + starts.insert(ts); + ts += interval_seconds; + } + starts +} + +fn count_cached_starts( + starts: &BTreeSet, + cache: &HashMap, + kraken_pair: &str, + interval_minutes: u32, +) -> usize { + starts + .iter() + .filter(|start| cache.contains_key(&cache_key(kraken_pair, interval_minutes, **start))) + .count() +} + +fn should_store_api_candle( + archive_mode: ArchiveBackfillMode, + missing_starts: &BTreeSet, + candle_start: i64, +) -> bool { + archive_mode == ArchiveBackfillMode::Vwap || missing_starts.contains(&candle_start) +} + +fn drop_cached_starts( + missing_starts: &mut BTreeSet, + cache: &HashMap, + kraken_pair: &str, + interval_minutes: u32, +) { + missing_starts.retain(|start| { + !cache.contains_key(&cache_key(kraken_pair, interval_minutes, *start)) + }); +} + +fn store_cache_value( + cache: &mut HashMap, + kraken_pair: &str, + interval_minutes: u32, + timestamp: i64, + value: f64, +) -> CacheWriteOutcome { + let key = cache_key(kraken_pair, interval_minutes, timestamp); + let normalized = normalize_fiat_rate(value); + match cache.get(&key) { + Some(existing) if same_fiat_cent(*existing, value) => { + if *existing != normalized { + cache.insert(key, normalized); + } + CacheWriteOutcome::Skipped + } + Some(_) => { + cache.insert(key, normalized); + CacheWriteOutcome::Replaced + } + None => { + cache.insert(key, normalized); + CacheWriteOutcome::Inserted + } + } +} + +fn same_fiat_cent(left: f64, right: f64) -> bool { + round_fiat_cents(left) == round_fiat_cents(right) +} + +fn normalize_fiat_rate(value: f64) -> f64 { + round_fiat_cents(value) as f64 / 100.0 +} + +fn round_fiat_cents(value: f64) -> i64 { + (value * 100.0).round() as i64 +} + +fn missing_quarters(missing_days: &BTreeSet) -> BTreeSet { + let mut quarters = BTreeSet::new(); + for ts in missing_days { + let date = chrono::DateTime::from_timestamp(*ts, 0) + .expect("daily candle timestamp should be valid") + .date_naive(); + quarters.insert((date.month0() / 3) + 1); + } + quarters +} + +fn resolve_quarterly_archive_files( + quarterly_files: &HashMap, + file_prefix: &str, + year: i32, + needed_quarters: &BTreeSet, +) -> Option> { + let mut files = Vec::with_capacity(needed_quarters.len()); + + for quarter in needed_quarters { + let name = format!("{file_prefix}Q{quarter}_{year}.zip"); + files.push(quarterly_files.get(&name)?.clone()); + } + + Some(files) +} + +fn prepare_archive_for_backfill( + client: &Client, + file: &DriveFile, + archive_pair: &str, + end_ts: i64, + target_interval_minutes: u32, + archive_mode: ArchiveBackfillMode, +) -> Result { + let archive_path = archive_download_path(&file.name); + if let Some(parent) = archive_path.parent() { + fs::create_dir_all(parent)?; + } + + ensure_archive_downloaded( + client, + file, + &archive_path, + archive_mode, + archive_pair, + end_ts, + target_interval_minutes, + ) +} + +fn read_prepared_archive( + prepared_archive: &PreparedArchive, + _file: &DriveFile, + kraken_pair: &str, + _archive_pair: &str, + start_ts: i64, + end_ts: i64, + target_interval_minutes: u32, + trade_source_interval_minutes: Option, + missing_starts: &mut BTreeSet, + cache: &mut HashMap, + archive_mode: ArchiveBackfillMode, +) -> Result { + eprintln!( + "Reading extracted {} data from {}...", + archive_mode.archive_label(), + prepared_archive.extracted_path.display(), + ); + read_archive_zip( + &prepared_archive.extracted_path, + kraken_pair, + start_ts, + end_ts, + target_interval_minutes, + trade_source_interval_minutes, + missing_starts, + cache, + archive_mode, + ) +} + +fn read_archive_zip( + path: &Path, + kraken_pair: &str, + start_ts: i64, + end_ts: i64, + target_interval_minutes: u32, + trade_source_interval_minutes: Option, + missing_starts: &mut BTreeSet, + cache: &mut HashMap, + archive_mode: ArchiveBackfillMode, +) -> Result { + let csv_file = File::open(path) + .with_context(|| format!("failed to open extracted archive data {}", path.display()))?; + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(csv_file); + let entry_name = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("archive data"); + + match archive_mode { + ArchiveBackfillMode::Midpoint => read_ohlcvt_daily_csv( + &mut reader, + &entry_name, + kraken_pair, + start_ts, + end_ts, + target_interval_minutes, + missing_starts, + cache, + ), + ArchiveBackfillMode::Vwap => read_trade_interval_csv( + &mut reader, + &entry_name, + kraken_pair, + start_ts, + end_ts, + trade_source_interval_minutes.unwrap_or(target_interval_minutes), + target_interval_minutes, + missing_starts, + cache, + ), + } +} + +fn ensure_archive_entry_extracted( + archive_path: &Path, + archive_file_name: &str, + archive_pair: &str, + archive_mode: ArchiveBackfillMode, +) -> Result { + let expected_entry_name = archive_mode.entry_name(archive_pair); + let extracted_path = extracted_archive_entry_path(archive_file_name, &expected_entry_name); + if extracted_path.exists() { + return Ok(extracted_path); + } + + if let Some(parent) = extracted_path.parent() { + fs::create_dir_all(parent)?; + } + + let archive_file = File::open(archive_path) + .with_context(|| format!("failed to open archive {}", archive_path.display()))?; + let mut archive = ZipArchive::new(archive_file).with_context(|| { + format!("failed to read ZIP archive {}", archive_path.display()) + })?; + let entry_name = resolve_archive_entry_name(&mut archive, &expected_entry_name)?; + eprintln!( + "Extracting {} from {} into {}...", + entry_name, + archive_path.display(), + temp_cache_dir().display(), + ); + let mut csv_file = archive + .by_name(&entry_name) + .with_context(|| format!("archive does not contain {entry_name}"))?; + let temp_path = temp_extracted_archive_entry_path(archive_file_name, &expected_entry_name); + if let Some(parent) = temp_path.parent() { + fs::create_dir_all(parent)?; + } + let mut output = File::create(&temp_path) + .with_context(|| format!("failed to create {}", temp_path.display()))?; + io::copy(&mut csv_file, &mut output) + .with_context(|| format!("failed to write {}", temp_path.display()))?; + fs::rename(&temp_path, &extracted_path) + .with_context(|| format!("failed to move extracted data into {}", extracted_path.display()))?; + Ok(extracted_path) +} + +fn extracted_csv_timestamp_bounds(path: &Path, entry_name: &str) -> Result> { + let csv_file = File::open(path) + .with_context(|| format!("failed to open extracted archive data {}", path.display()))?; + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(csv_file); + let mut first = None; + let mut last = None; + + for record in reader.records() { + let record = record.with_context(|| format!("failed to parse a row in {entry_name}"))?; + let timestamp = parse_archive_timestamp(&record, &entry_name)?; + first.get_or_insert(timestamp); + last = Some(timestamp); + } + + Ok(match (first, last) { + (Some(first), Some(last)) => Some(ArchiveCoverage { first, last }), + _ => None, + }) +} + +fn resolve_archive_entry_name( + archive: &mut ZipArchive, + expected_entry_name: &str, +) -> Result { + if archive.by_name(expected_entry_name).is_ok() { + return Ok(expected_entry_name.to_owned()); + } + + let suffix = format!("/{expected_entry_name}"); + let mut matches = Vec::new(); + for i in 0..archive.len() { + let name = archive + .by_index(i) + .with_context(|| format!("failed to inspect ZIP entry #{i}"))? + .name() + .to_owned(); + if name.ends_with(&suffix) { + matches.push(name); + } + } + + match matches.len() { + 1 => Ok(matches.remove(0)), + 0 => bail!("archive does not contain {expected_entry_name}"), + _ => bail!( + "archive contains multiple entries matching {expected_entry_name}: {}", + matches.join(", ") + ), + } +} + +fn read_ohlcvt_daily_csv( + reader: &mut csv::Reader, + entry_name: &str, + kraken_pair: &str, + start_ts: i64, + end_ts: i64, + target_interval_minutes: u32, + missing_starts: &mut BTreeSet, + cache: &mut HashMap, +) -> Result { + let mut stats = CacheWriteStats::default(); + let mut last_logged_year = None; + + for record in reader.records() { + let record = record.with_context(|| format!("failed to parse a row in {entry_name}"))?; + let timestamp = parse_archive_timestamp(&record, entry_name)?; + if timestamp < start_ts || timestamp >= end_ts || !missing_starts.contains(×tamp) { + continue; + } + log_archive_year_progress( + timestamp, + &mut last_logged_year, + "OHLCVT archive", + KRAKEN_DAILY_INTERVAL_MINUTES, + )?; + + let midpoint = parse_archive_midpoint(&record, entry_name)?; + // The OHLCVT archive lacks VWAP, so we store the UTC daily (open + close) / 2 midpoint + // under the normal 1440-minute cache key as a simple per-day price proxy. + stats.record(store_cache_value( + cache, + kraken_pair, + target_interval_minutes, + timestamp, + midpoint, + )); + missing_starts.remove(×tamp); + } + + Ok(stats) +} + +fn read_trade_interval_csv( + reader: &mut csv::Reader, + entry_name: &str, + kraken_pair: &str, + start_ts: i64, + end_ts: i64, + source_interval_minutes: u32, + target_interval_minutes: u32, + missing_starts: &mut BTreeSet, + cache: &mut HashMap, +) -> Result { + let mut stats = CacheWriteStats::default(); + let interval_seconds = i64::from(source_interval_minutes) * 60; + let mut current_start = None; + let mut price_volume_sum = 0.0f64; + let mut volume_sum = 0.0f64; + let mut last_logged_year = None; + + for record in reader.records() { + let record = record.with_context(|| format!("failed to parse a row in {entry_name}"))?; + let timestamp = parse_archive_timestamp(&record, entry_name)?; + if timestamp < start_ts { + continue; + } + if timestamp < end_ts { + log_archive_year_progress( + timestamp, + &mut last_logged_year, + "trade archive", + source_interval_minutes, + )?; + } + + let slot_start = timestamp - timestamp.rem_euclid(interval_seconds); + if current_start != Some(slot_start) { + stats.absorb(flush_trade_interval( + current_start, + price_volume_sum, + volume_sum, + kraken_pair, + source_interval_minutes, + target_interval_minutes, + missing_starts, + cache, + )?); + current_start = Some(slot_start); + price_volume_sum = 0.0; + volume_sum = 0.0; + } + + if timestamp >= end_ts { + break; + } + if !bucket_has_missing_targets( + missing_starts, + slot_start, + source_interval_minutes, + target_interval_minutes, + ) { + continue; + } + + let price = parse_archive_number(&record, 1, "price", entry_name)?; + let volume = parse_archive_number(&record, 2, "volume", entry_name)?; + price_volume_sum += price * volume; + volume_sum += volume; + } + + stats.absorb(flush_trade_interval( + current_start, + price_volume_sum, + volume_sum, + kraken_pair, + source_interval_minutes, + target_interval_minutes, + missing_starts, + cache, + )?); + + Ok(stats) +} + +fn log_archive_year_progress( + timestamp: i64, + last_logged_year: &mut Option, + label: &str, + interval_minutes: u32, +) -> Result<()> { + let datetime = chrono::DateTime::from_timestamp(timestamp, 0) + .context("invalid archive progress timestamp")?; + let year = datetime.year(); + if *last_logged_year == Some(year) { + return Ok(()); + } + + *last_logged_year = Some(year); + if interval_minutes == KRAKEN_DAILY_INTERVAL_MINUTES { + eprintln!("Processing {label} rows for {year}..."); + } else { + eprintln!( + "Processing {label} rows for {year} at {}-minute resolution...", + interval_minutes + ); + } + Ok(()) +} + +fn parse_archive_timestamp(record: &StringRecord, entry_name: &str) -> Result { + let value = record + .get(0) + .ok_or_else(|| anyhow!("{entry_name} row is missing the timestamp column"))?; + value + .parse::() + .with_context(|| format!("invalid timestamp {value} in {entry_name}")) +} + +fn parse_archive_midpoint(record: &StringRecord, entry_name: &str) -> Result { + let open = parse_archive_number(record, 1, "open", entry_name)?; + let close = parse_archive_number(record, 4, "close", entry_name)?; + Ok((open + close) / 2.0) +} + +fn parse_archive_number( + record: &StringRecord, + column: usize, + label: &str, + entry_name: &str, +) -> Result { + let value = record + .get(column) + .ok_or_else(|| anyhow!("{entry_name} row is missing the {label} column"))?; + value + .parse::() + .with_context(|| format!("invalid {label} value {value} in {entry_name}")) +} + +fn fetch_quarterly_archive_files( + client: &Client, + folder_id: &str, + file_prefix: &str, + archive_label: &str, +) -> Result> { + let url = format!("https://drive.google.com/drive/folders/{folder_id}?usp=sharing"); + let html = client + .get(&url) + .send() + .with_context(|| format!("failed to fetch {url}"))? + .error_for_status() + .with_context(|| format!("Google Drive returned an error for {url}"))? + .text() + .with_context(|| format!("failed to decode Google Drive folder page at {url}"))?; + + extract_quarterly_drive_files(&html, file_prefix, archive_label) +} + +fn extract_quarterly_drive_files( + html: &str, + file_prefix: &str, + archive_label: &str, +) -> Result> { + let blob = extract_drive_ivd_blob(html) + .context("failed to locate the Google Drive file listing blob")?; + let mut files = HashMap::new(); + let mut cursor = 0usize; + + while let Some(relative_idx) = blob[cursor..].find(file_prefix) { + let name_start = cursor + relative_idx; + let name_end = blob[name_start..] + .find(".zip") + .map(|idx| name_start + idx + 4) + .ok_or_else(|| anyhow!("failed to parse a quarterly {archive_label} archive filename from Google Drive"))?; + let name = blob[name_start..name_end].to_owned(); + + let suffix = &blob[name_end..]; + let prefix = &blob[..name_start]; + let id = extract_id_between( + suffix, + "https:\\/\\/drive.google.com\\/file\\/d\\/", + "\\/view", + ) + .or_else(|| { + extract_last_id_between( + prefix, + "https:\\/\\/drive.google.com\\/file\\/d\\/", + "\\/view", + ) + }) + .with_context(|| format!("failed to find the Google Drive file id for {name}"))?; + + files.insert(name.clone(), DriveFile { id, name }); + cursor = name_end; + } + + if files.is_empty() { + bail!("failed to parse quarterly {archive_label} archive files from Google Drive"); + } + + Ok(files) +} + +fn extract_drive_ivd_blob(html: &str) -> Option<&str> { + let start_marker = "window['_DRIVE_ivd'] = '"; + let end_marker = "';if (window['_DRIVE_ivdc'])"; + let start = html.find(start_marker)? + start_marker.len(); + let rest = &html[start..]; + let end = rest.find(end_marker)?; + Some(&rest[..end]) +} + +fn extract_id_between(haystack: &str, marker: &str, terminator: &str) -> Option { + let start = haystack.find(marker)? + marker.len(); + let rest = &haystack[start..]; + let end = rest.find(terminator)?; + Some(rest[..end].to_owned()) +} + +fn extract_last_id_between(haystack: &str, marker: &str, terminator: &str) -> Option { + let start = haystack.rfind(marker)? + marker.len(); + let rest = &haystack[start..]; + let end = rest.find(terminator)?; + Some(rest[..end].to_owned()) +} + +fn download_google_drive_file(client: &Client, file_id: &str, destination: &Path) -> Result<()> { + let initial_url = format!("https://drive.google.com/uc?export=download&id={file_id}&confirm=t"); + let response = client + .get(&initial_url) + .send() + .with_context(|| format!("failed to download Google Drive file {file_id}"))?; + + if is_html_response(&response) { + let html = response + .text() + .context("failed to decode the Google Drive confirmation page")?; + let action = extract_html_attribute(&html, "

Result { + let mut existing_coverage = None; + if destination.exists() { + if file.name == archive_mode.complete_archive_name() { + let extracted_path = + ensure_archive_entry_extracted(destination, &file.name, archive_pair, archive_mode)?; + let coverage = inspect_archive_coverage( + &extracted_path, + &format!("cached extracted {archive_pair} data"), + KRAKEN_DAILY_INTERVAL_MINUTES, + )?; + existing_coverage = Some(coverage); + let needed_start = needed_archive_start(end_ts, target_interval_minutes); + if coverage.last >= needed_start { + let size = archive_file_size_label(destination); + eprintln!( + "Reusing cached {} at {}{}.", + archive_file_description(file, archive_mode), + destination.display(), + size.as_deref().unwrap_or(""), + ); + return Ok(PreparedArchive { + archive_path: destination.to_path_buf(), + archive_file_name: file.name.clone(), + extracted_path, + }); + } + eprintln!( + "Cached {} at {} does not appear to reach the requested {} data. Redownloading...", + archive_file_description(file, archive_mode), + destination.display(), + format_interval_start(needed_start, target_interval_minutes)?, + ); + clear_temp_cache_dir()?; + } else { + let size = archive_file_size_label(destination); + eprintln!( + "Reusing cached {} at {}{}.", + archive_file_description(file, archive_mode), + destination.display(), + size.as_deref().unwrap_or(""), + ); + return Ok(PreparedArchive { + archive_path: destination.to_path_buf(), + archive_file_name: file.name.clone(), + extracted_path: ensure_archive_entry_extracted( + destination, + &file.name, + archive_pair, + archive_mode, + )?, + }); + } + } + + let temp_path = temp_download_path(&file.name); + if let Some(parent) = temp_path.parent() { + fs::create_dir_all(parent)?; + } + + let result = (|| -> Result { + let size_hint = archive_file_size_label(destination); + eprintln!( + "Downloading {} to {}{}...", + archive_file_description(file, archive_mode), + destination.display(), + size_hint.as_deref().unwrap_or(""), + ); + download_google_drive_file(client, &file.id, &temp_path)?; + if file.name == archive_mode.complete_archive_name() { + let extracted_path = + ensure_archive_entry_extracted(&temp_path, &file.name, archive_pair, archive_mode)?; + let downloaded_coverage = inspect_archive_coverage( + &extracted_path, + &format!("downloaded extracted {archive_pair} data"), + KRAKEN_DAILY_INTERVAL_MINUTES, + )?; + let needed_start = needed_archive_start(end_ts, target_interval_minutes); + if downloaded_coverage.last < needed_start { + bail!( + "downloaded complete archive does not reach the requested {} data; keeping existing cached archive at {}", + format_interval_start(needed_start, target_interval_minutes)?, + destination.display() + ); + } + if !accept_complete_archive_replacement(existing_coverage, downloaded_coverage) { + eprintln!( + "Using downloaded complete archive for this run and keeping it at {}, while preserving cached {} at {} because the new archive would shrink prior coverage.", + temp_path.display(), + archive_mode.archive_label(), + destination.display(), + ); + return Ok(PreparedArchive { + archive_path: temp_path.clone(), + archive_file_name: file.name.clone(), + extracted_path, + }); + } + } + fs::rename(&temp_path, destination).with_context(|| { + format!( + "failed to move downloaded archive into cache at {}", + destination.display() + ) + })?; + eprintln!( + "Saved {} to {}{}.", + archive_file_description(file, archive_mode), + destination.display(), + archive_file_size_label(destination).as_deref().unwrap_or(""), + ); + let extracted_path = + ensure_archive_entry_extracted(destination, &file.name, archive_pair, archive_mode)?; + Ok(PreparedArchive { + archive_path: destination.to_path_buf(), + archive_file_name: file.name.clone(), + extracted_path, + }) + })(); + + if result.is_err() { + let _ = fs::remove_file(&temp_path); + } + + result +} + +fn inspect_archive_coverage( + path: &Path, + label: &str, + target_interval_minutes: u32, +) -> Result { + eprintln!("Scanning coverage from {} at {}...", label, path.display()); + let coverage = extracted_csv_timestamp_bounds(path, label)? + .context("archive pair CSV is empty")?; + eprintln!( + "{} appears to cover {} through {}.", + label, + format_interval_start(coverage.first, target_interval_minutes)?, + format_interval_start(coverage.last, target_interval_minutes)?, + ); + Ok(coverage) +} + +fn accept_complete_archive_replacement( + existing_coverage: Option, + downloaded_coverage: ArchiveCoverage, +) -> bool { + match existing_coverage { + Some(existing) => { + downloaded_coverage.first <= existing.first + && downloaded_coverage.last >= existing.last + } + None => true, + } +} + +fn needed_archive_start(end_ts: i64, target_interval_minutes: u32) -> i64 { + end_ts - i64::from(target_interval_minutes) * 60 +} + +fn fallback_trade_intervals(target_interval_minutes: u32) -> Vec { + KRAKEN_INTERVALS_MINUTES + .into_iter() + .filter(|interval| *interval > target_interval_minutes) + .collect() +} + +fn archive_file_description(file: &DriveFile, archive_mode: ArchiveBackfillMode) -> String { + if file.name == archive_mode.complete_archive_name() { + format!( + "Kraken complete {} archive (all years)", + archive_mode.archive_label() + ) + } else { + format!("Kraken {} archive {}", archive_mode.archive_label(), file.name) + } +} + +fn archive_file_size_label(path: &Path) -> Option { + let bytes = fs::metadata(path).ok()?.len(); + Some(format!(" ({})", format_byte_size(bytes))) +} + +fn format_byte_size(bytes: u64) -> String { + const UNITS: [&str; 5] = ["B", "KB", "MB", "GB", "TB"]; + let mut value = bytes as f64; + let mut unit_index = 0usize; + + while value >= 1024.0 && unit_index < UNITS.len() - 1 { + value /= 1024.0; + unit_index += 1; + } + + if unit_index == 0 { + format!("{} {}", bytes, UNITS[unit_index]) + } else { + format!("{value:.1} {}", UNITS[unit_index]) + } +} + +fn is_html_response(response: &Response) -> bool { + response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(|value| value.starts_with("text/html")) + .unwrap_or(false) +} + +fn write_response_to_file(response: Response, destination: &Path) -> Result<()> { + let mut response = response.error_for_status()?; + let mut file = File::create(destination) + .with_context(|| format!("failed to create {}", destination.display()))?; + io::copy(&mut response, &mut file) + .with_context(|| format!("failed to write {}", destination.display()))?; + Ok(()) +} + +fn extract_hidden_input_value(html: &str, name: &str) -> Option { + let marker = format!("name=\"{name}\" value=\""); + let start = html.find(&marker)? + marker.len(); + let rest = &html[start..]; + let end = rest.find('"')?; + Some(rest[..end].to_owned()) +} + +fn extract_html_attribute(html: &str, element_prefix: &str, attribute: &str) -> Option { + let element_start = html.find(element_prefix)?; + let element = &html[element_start..]; + let marker = format!("{attribute}=\""); + let attr_start = element.find(&marker)? + marker.len(); + let rest = &element[attr_start..]; + let attr_end = rest.find('"')?; + Some(rest[..attr_end].to_owned()) +} + +fn archive_pair_name(kraken_pair: &str) -> Result { + if let Some(quote) = kraken_pair.strip_prefix("XXBTZ") { + return Ok(format!("XBT{quote}")); + } + + if let Some(quote) = kraken_pair.strip_prefix("XXBT") { + return Ok(format!("XBT{}", quote.trim_start_matches('Z'))); + } + + if kraken_pair.starts_with("XBT") { + return Ok(kraken_pair.to_owned()); + } + + bail!("cache-rates only supports Kraken XBT quote pairs such as XXBTZUSD or XXBTZEUR") +} + +fn flush_trade_interval( + candle_start: Option, + price_volume_sum: f64, + volume_sum: f64, + kraken_pair: &str, + source_interval_minutes: u32, + target_interval_minutes: u32, + missing_starts: &mut BTreeSet, + cache: &mut HashMap, +) -> Result { + let Some(candle_start) = candle_start else { + return Ok(CacheWriteStats::default()); + }; + let source_interval_seconds = i64::from(source_interval_minutes) * 60; + let target_interval_seconds = i64::from(target_interval_minutes) * 60; + let source_end = candle_start + source_interval_seconds; + let bucket_missing = missing_starts + .range(candle_start..source_end) + .copied() + .collect::>(); + if bucket_missing.is_empty() { + return Ok(CacheWriteStats::default()); + } + if volume_sum <= 0.0 { + return Ok(CacheWriteStats::default()); + } + + let vwap = price_volume_sum / volume_sum; + let mut stats = CacheWriteStats::default(); + let mut slot = candle_start; + while slot < source_end { + if missing_starts.remove(&slot) { + stats.record(store_cache_value( + cache, + kraken_pair, + target_interval_minutes, + slot, + vwap, + )); + } + slot += target_interval_seconds; + } + Ok(stats) +} + +fn bucket_has_missing_targets( + missing_starts: &BTreeSet, + candle_start: i64, + source_interval_minutes: u32, + target_interval_minutes: u32, +) -> bool { + let source_interval_seconds = i64::from(source_interval_minutes) * 60; + let target_interval_seconds = i64::from(target_interval_minutes) * 60; + let source_end = candle_start + source_interval_seconds; + let mut slot = candle_start; + while slot < source_end { + if missing_starts.contains(&slot) { + return true; + } + slot += target_interval_seconds; + } + false +} + +fn midnight_utc_timestamp(date: NaiveDate) -> i64 { + date.and_hms_opt(0, 0, 0) + .expect("midnight should always be a valid time") + .and_utc() + .timestamp() +} + +fn format_interval_start(timestamp: i64, interval_minutes: u32) -> Result { + let datetime = chrono::DateTime::from_timestamp(timestamp, 0) + .context("invalid interval start timestamp")?; + if interval_minutes == KRAKEN_DAILY_INTERVAL_MINUTES { + Ok(datetime.date_naive().to_string()) + } else { + Ok(datetime.to_rfc3339()) + } +} + +fn archive_download_path(file_name: &str) -> PathBuf { + PathBuf::from(CACHE_DIR).join("kraken").join(file_name) +} + +fn temp_download_path(file_name: &str) -> PathBuf { + temp_cache_dir().join(format!("download-{}-{}", std::process::id(), file_name)) +} + +fn extracted_archive_entry_path(archive_file_name: &str, entry_name: &str) -> PathBuf { + temp_cache_dir().join(format!( + "{}--{}", + sanitize_temp_component(archive_file_name), + sanitize_temp_component(entry_name) + )) +} + +fn temp_extracted_archive_entry_path(archive_file_name: &str, entry_name: &str) -> PathBuf { + temp_cache_dir().join(format!( + "extract-{}-{}--{}", + std::process::id(), + sanitize_temp_component(archive_file_name), + sanitize_temp_component(entry_name) + )) +} + +fn temp_cache_dir() -> PathBuf { + PathBuf::from(CACHE_DIR).join("tmp") +} + +fn clear_temp_cache_dir() -> Result<()> { + let path = temp_cache_dir(); + if path.exists() { + fs::remove_dir_all(&path) + .with_context(|| format!("failed to remove {}", path.display()))?; + } + fs::create_dir_all(&path).with_context(|| format!("failed to create {}", path.display()))?; + Ok(()) +} + +fn sanitize_temp_component(value: &str) -> String { + value + .chars() + .map(|ch| match ch { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '.' | '-' | '_' => ch, + _ => '_', + }) + .collect() +} + +struct TempCacheGuard { + path: PathBuf, +} + +impl TempCacheGuard { + fn new() -> Result { + let path = temp_cache_dir(); + clear_temp_cache_dir()?; + Ok(Self { path }) + } +} + +impl Drop for TempCacheGuard { + fn drop(&mut self) { + if self.path.exists() { + let _ = fs::remove_dir_all(&self.path); + eprintln!("Deleted temporary cache directory {}.", self.path.display()); + } + } +} + +impl ArchiveBackfillMode { + fn archive_label(self) -> &'static str { + match self { + Self::Midpoint => "OHLCVT", + Self::Vwap => "trade", + } + } + + fn quarterly_archive_prefix(self) -> &'static str { + match self { + Self::Midpoint => "Kraken_OHLCVT_", + Self::Vwap => "Kraken_Trading_History_", + } + } + + fn quarterly_archive_folder_id(self) -> &'static str { + match self { + Self::Midpoint => QUARTERLY_OHLCVT_ARCHIVE_FOLDER_ID, + Self::Vwap => QUARTERLY_TRADE_ARCHIVE_FOLDER_ID, + } + } + + fn complete_archive_file_id(self) -> &'static str { + match self { + Self::Midpoint => COMPLETE_OHLCVT_ARCHIVE_FILE_ID, + Self::Vwap => COMPLETE_TRADE_ARCHIVE_FILE_ID, + } + } + + fn complete_archive_name(self) -> &'static str { + match self { + Self::Midpoint => "Kraken_OHLCVT.zip", + Self::Vwap => "Kraken_Trading_History.zip", + } + } + + fn entry_name(self, archive_pair: &str) -> String { + match self { + Self::Midpoint => format!("{archive_pair}_1440.csv"), + Self::Vwap => format!("{archive_pair}.csv"), + } + } + + fn summary_label(self) -> &'static str { + match self { + Self::Midpoint => "OHLCVT archive midpoint rows", + Self::Vwap => "Trade archive VWAP candles", + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::{BTreeSet, HashMap}; + use std::io::Cursor; + use std::path::PathBuf; + use std::time::{SystemTime, UNIX_EPOCH}; + + use crate::common::AppConfig; + + use super::{ + ArchiveCoverage, CacheRatesArgs, CacheWriteOutcome, DriveFile, + accept_complete_archive_replacement, + PreparedArchive, archive_download_path, archive_pair_name, ensure_archive_downloaded, + extracted_archive_entry_path, extract_quarterly_drive_files, parse_args_from, + read_ohlcvt_daily_csv, read_trade_interval_csv, resolve_archive_entry_name, + resolve_quarterly_archive_files, should_store_api_candle, store_cache_value, + target_interval_minutes, + }; + + const REAL_XBTEUR_1440_SAMPLE: &str = "\ +1672531200,15423.8,15524.8,15388.5,15512.9,532.29189029,11775 +1672617600,15513.0,15706.6,15455.0,15600.4,1161.60737247,21237 +1672704000,15599.1,15892.9,15580.5,15795.1,1465.78622353,24222 +"; + const REAL_XBTEUR_1440_OVERLAP_ROW: &str = + "1711929600,66130.0,66130.0,63461.9,64902.9,528.8081404,23715"; + const REAL_XBTEUR_2026_03_21_TRADES: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/kraken_xbteur_2026-03-21_trades.csv" + )); + const REAL_XBTEUR_TRADES_SAMPLE: &str = "\ +1735689600,90167.30000,0.00054633 +1735689600,90167.30000,0.00005463 +1735689600,90167.10000,0.00002185 +1735776000,91190.30000,0.00002161 +1735776000,91190.30000,0.00001080 +1735776000,91190.20000,0.00060000 +"; + + #[test] + fn parses_cache_rates_args() { + let args = parse_args_from( + vec!["2024".to_owned()], + "usage: btc_fiat_value cache-rates [--vwap] [--candle ] ", + ) + .expect("args"); + assert_eq!( + args, + CacheRatesArgs { + year: 2024, + use_vwap_archive: false, + candle_override_minutes: None, + } + ); + } + + #[test] + fn parses_cache_rates_vwap_flag() { + let args = parse_args_from( + vec!["--vwap".to_owned(), "2024".to_owned()], + "usage: btc_fiat_value cache-rates [--vwap] [--candle ] ", + ) + .expect("args"); + assert_eq!( + args, + CacheRatesArgs { + year: 2024, + use_vwap_archive: true, + candle_override_minutes: None, + } + ); + } + + #[test] + fn parses_cache_rates_vwap_candle_override() { + let args = parse_args_from( + vec![ + "--vwap".to_owned(), + "--candle".to_owned(), + "60".to_owned(), + "2024".to_owned(), + ], + "usage: btc_fiat_value cache-rates [--vwap] [--candle ] ", + ) + .expect("args"); + assert_eq!( + args, + CacheRatesArgs { + year: 2024, + use_vwap_archive: true, + candle_override_minutes: Some(60), + } + ); + } + + #[test] + fn rejects_extra_cache_rates_args() { + let err = parse_args_from( + vec!["2024".to_owned(), "extra".to_owned()], + "usage: btc_fiat_value cache-rates [--vwap] [--candle ] ", + ) + .expect_err("should fail"); + + assert!(err + .to_string() + .contains("usage: btc_fiat_value cache-rates [--vwap] [--candle ] ")); + } + + #[test] + fn rejects_cache_rates_candle_override_without_vwap() { + let err = parse_args_from( + vec!["--candle".to_owned(), "60".to_owned(), "2024".to_owned()], + "usage: btc_fiat_value cache-rates [--vwap] [--candle ] ", + ) + .expect_err("should fail"); + + assert!(err.to_string().contains("--candle requires --vwap")); + } + + #[test] + fn vwap_uses_default_candle_minutes_when_present() { + let config = AppConfig::from_env_values(|name| match name { + "DEFAULT_CANDLE_MINUTES" => Some("60".to_owned()), + _ => None, + }) + .expect("config"); + + let interval = target_interval_minutes( + &CacheRatesArgs { + year: 2024, + use_vwap_archive: true, + candle_override_minutes: None, + }, + &config, + ) + .expect("interval"); + + assert_eq!(interval, 60); + } + + #[test] + fn vwap_candle_override_beats_default_candle_minutes() { + let config = AppConfig::from_env_values(|name| match name { + "DEFAULT_CANDLE_MINUTES" => Some("60".to_owned()), + _ => None, + }) + .expect("config"); + + let interval = target_interval_minutes( + &CacheRatesArgs { + year: 2024, + use_vwap_archive: true, + candle_override_minutes: Some(240), + }, + &config, + ) + .expect("interval"); + + assert_eq!(interval, 240); + } + + #[test] + fn stores_archives_under_cache_kraken_directory() { + assert_eq!( + archive_download_path("Kraken_Trading_History.zip"), + PathBuf::from(".cache/kraken/Kraken_Trading_History.zip") + ); + } + + #[test] + fn midpoint_mode_skips_existing_cache_entries() { + let mut missing_days = BTreeSet::from([1672531200_i64, 1672617600_i64]); + let cache = HashMap::from([("XXBTZEUR:1440:1672531200".to_owned(), 12345.0_f64)]); + + super::drop_cached_starts(&mut missing_days, &cache, "XXBTZEUR", 1440); + + assert_eq!(missing_days, BTreeSet::from([1672617600_i64])); + } + + #[test] + fn midpoint_mode_skips_api_candles_for_existing_cache_entries() { + let missing_days = BTreeSet::from([1672617600_i64]); + + assert!(!should_store_api_candle( + super::ArchiveBackfillMode::Midpoint, + &missing_days, + 1672531200, + )); + assert!(should_store_api_candle( + super::ArchiveBackfillMode::Midpoint, + &missing_days, + 1672617600, + )); + assert!(should_store_api_candle( + super::ArchiveBackfillMode::Vwap, + &missing_days, + 1672531200, + )); + } + + #[test] + fn store_cache_value_skips_unchanged_entries() { + let mut cache = HashMap::from([("XXBTZEUR:60:1735689600".to_owned(), 107.5_f64)]); + + let outcome = store_cache_value(&mut cache, "XXBTZEUR", 60, 1735689600, 107.5); + + assert_eq!(outcome, CacheWriteOutcome::Skipped); + assert_eq!(cache["XXBTZEUR:60:1735689600"], 107.5); + } + + #[test] + fn store_cache_value_skips_same_cent_drift() { + let mut cache = HashMap::from([("XXBTZEUR:60:1735689600".to_owned(), 107.504_f64)]); + + let outcome = store_cache_value(&mut cache, "XXBTZEUR", 60, 1735689600, 107.495); + + assert_eq!(outcome, CacheWriteOutcome::Skipped); + assert_eq!(cache["XXBTZEUR:60:1735689600"], 107.5); + } + + #[test] + fn normalizes_xbt_archive_pair_names() { + assert_eq!(archive_pair_name("XXBTZUSD").unwrap(), "XBTUSD"); + assert_eq!(archive_pair_name("XXBTZEUR").unwrap(), "XBTEUR"); + assert_eq!(archive_pair_name("XBTUSDT").unwrap(), "XBTUSDT"); + } + + #[test] + fn extracts_quarterly_file_ids_from_drive_html() { + let html = concat!( + "window['_DRIVE_ivd'] = '", + "Kraken_OHLCVT_Q1_2024.zip ", + "https:\\/\\/drive.google.com\\/file\\/d\\/abc123\\/view ", + "Kraken_OHLCVT_Q2_2024.zip ", + "https:\\/\\/drive.google.com\\/file\\/d\\/def456\\/view", + "';if (window['_DRIVE_ivdc'])" + ); + + let files = extract_quarterly_drive_files(html, "Kraken_OHLCVT_", "OHLCVT") + .expect("files"); + assert_eq!(files["Kraken_OHLCVT_Q1_2024.zip"].id, "abc123"); + assert_eq!(files["Kraken_OHLCVT_Q2_2024.zip"].id, "def456"); + } + + #[test] + fn extracts_quarterly_trade_file_ids_from_drive_html() { + let html = concat!( + "window['_DRIVE_ivd'] = '", + "Kraken_Trading_History_Q1_2024.zip ", + "https:\\/\\/drive.google.com\\/file\\/d\\/abc123\\/view ", + "Kraken_Trading_History_Q2_2024.zip ", + "https:\\/\\/drive.google.com\\/file\\/d\\/def456\\/view", + "';if (window['_DRIVE_ivdc'])" + ); + + let files = extract_quarterly_drive_files(html, "Kraken_Trading_History_", "trade") + .expect("files"); + assert_eq!(files["Kraken_Trading_History_Q1_2024.zip"].id, "abc123"); + assert_eq!(files["Kraken_Trading_History_Q2_2024.zip"].id, "def456"); + } + + #[test] + fn falls_back_when_needed_quarterly_trade_archive_is_missing() { + let quarterly_files = HashMap::from([( + "Kraken_Trading_History_Q1_2025.zip".to_owned(), + DriveFile { + id: "abc123".to_owned(), + name: "Kraken_Trading_History_Q1_2025.zip".to_owned(), + }, + )]); + let needed_quarters = BTreeSet::from([1_u32, 2_u32]); + + let resolved = resolve_quarterly_archive_files( + &quarterly_files, + "Kraken_Trading_History_", + 2025, + &needed_quarters, + ); + + assert!(resolved.is_none()); + } + + #[test] + fn reads_real_archive_daily_csv_rows_into_cache() { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new(REAL_XBTEUR_1440_SAMPLE)); + let mut missing_days = BTreeSet::from([1672531200_i64, 1672617600_i64]); + let mut cache = HashMap::new(); + + let stats = read_ohlcvt_daily_csv( + &mut reader, + "XBTEUR_1440.csv", + "XXBTZEUR", + 1672531200, + 1672704000, + 1440, + &mut missing_days, + &mut cache, + ) + .expect("csv should parse"); + + assert_eq!(stats.inserted, 2); + assert_eq!(stats.replaced, 0); + assert_eq!(stats.skipped, 0); + assert_eq!( + cache["XXBTZEUR:1440:1672531200"], + super::normalize_fiat_rate((15423.8 + 15512.9) / 2.0) + ); + assert_eq!( + cache["XXBTZEUR:1440:1672617600"], + super::normalize_fiat_rate((15513.0 + 15600.4) / 2.0) + ); + assert!(missing_days.is_empty()); + } + + #[test] + fn archive_ohlc_matches_live_overlap_day_fields() { + let record = csv::StringRecord::from( + REAL_XBTEUR_1440_OVERLAP_ROW + .split(',') + .collect::>(), + ); + + // Sanity check against the live Kraken 1440 OHLC API on 2024-04-01 UTC: + // the daily candle boundaries and OHLC fields match the archive row. + assert_eq!(super::parse_archive_timestamp(&record, "XBTEUR_1440.csv").unwrap(), 1711929600); + assert_eq!( + super::parse_archive_number(&record, 1, "open", "XBTEUR_1440.csv").unwrap(), + 66130.0 + ); + assert_eq!( + super::parse_archive_number(&record, 2, "high", "XBTEUR_1440.csv").unwrap(), + 66130.0 + ); + assert_eq!( + super::parse_archive_number(&record, 3, "low", "XBTEUR_1440.csv").unwrap(), + 63461.9 + ); + assert_eq!( + super::parse_archive_number(&record, 4, "close", "XBTEUR_1440.csv").unwrap(), + 64902.9 + ); + } + + #[test] + fn reports_missing_close_column_clearly() { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new("1672531200,15423.8,15524.8,15388.5\n")); + let mut missing_days = BTreeSet::from([1672531200_i64]); + let mut cache = HashMap::new(); + + let err = read_ohlcvt_daily_csv( + &mut reader, + "XBTEUR_1440.csv", + "XXBTZEUR", + 1672531200, + 1672617600, + 1440, + &mut missing_days, + &mut cache, + ) + .expect_err("row should fail"); + + assert!(err.to_string().contains("XBTEUR_1440.csv row is missing the close column")); + } + + #[test] + fn reports_invalid_close_price_clearly() { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new("1672531200,15423.8,15524.8,15388.5,nope,532.29189029,11775\n")); + let mut missing_days = BTreeSet::from([1672531200_i64]); + let mut cache = HashMap::new(); + + let err = read_ohlcvt_daily_csv( + &mut reader, + "XBTEUR_1440.csv", + "XXBTZEUR", + 1672531200, + 1672617600, + 1440, + &mut missing_days, + &mut cache, + ) + .expect_err("row should fail"); + + assert!(err + .to_string() + .contains("invalid close value nope in XBTEUR_1440.csv")); + } + + #[test] + fn reads_real_trade_csv_rows_into_daily_vwap_cache() { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new(REAL_XBTEUR_TRADES_SAMPLE)); + let mut missing_days = BTreeSet::from([1735689600_i64, 1735776000_i64]); + let mut cache = HashMap::new(); + + let stats = read_trade_interval_csv( + &mut reader, + "XBTEUR.csv", + "XXBTZEUR", + 1735689600, + 1735862400, + 1440, + 1440, + &mut missing_days, + &mut cache, + ) + .expect("csv should parse"); + + let day1 = ((90167.3 * 0.00054633) + (90167.3 * 0.00005463) + (90167.1 * 0.00002185)) + / (0.00054633 + 0.00005463 + 0.00002185); + let day2 = ((91190.3 * 0.00002161) + (91190.3 * 0.00001080) + (91190.2 * 0.00060000)) + / (0.00002161 + 0.00001080 + 0.00060000); + + assert_eq!(stats.inserted, 2); + assert_eq!(stats.replaced, 0); + assert_eq!(stats.skipped, 0); + assert_eq!(cache["XXBTZEUR:1440:1735689600"], super::normalize_fiat_rate(day1)); + assert_eq!(cache["XXBTZEUR:1440:1735776000"], super::normalize_fiat_rate(day2)); + assert!(missing_days.is_empty()); + } + + #[test] + fn reads_trade_csv_rows_into_hourly_vwap_cache() { + let trades = "\ +1735689600,100.0,1.0 +1735691400,110.0,3.0 +1735693200,120.0,2.0 +1735695000,90.0,2.0 +"; + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new(trades)); + let mut missing_hours = BTreeSet::from([1735689600_i64, 1735693200_i64]); + let mut cache = HashMap::new(); + + let stats = read_trade_interval_csv( + &mut reader, + "XBTEUR.csv", + "XXBTZEUR", + 1735689600, + 1735696800, + 60, + 60, + &mut missing_hours, + &mut cache, + ) + .expect("csv should parse"); + + assert_eq!(stats.inserted, 2); + assert_eq!(stats.replaced, 0); + assert_eq!(stats.skipped, 0); + assert_eq!(cache["XXBTZEUR:60:1735689600"], super::normalize_fiat_rate(107.5)); + assert_eq!(cache["XXBTZEUR:60:1735693200"], super::normalize_fiat_rate(105.0)); + assert!(missing_hours.is_empty()); + } + + #[test] + fn larger_trade_candle_fills_missing_smaller_intervals() { + let trades = "\ +1735689900,100.0,1.0 +1735697100,140.0,3.0 +"; + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new(trades)); + let mut missing_hours = + BTreeSet::from([1735689600_i64, 1735693200_i64, 1735696800_i64, 1735700400_i64]); + let mut cache = HashMap::new(); + + let stats = read_trade_interval_csv( + &mut reader, + "XBTEUR.csv", + "XXBTZEUR", + 1735689600, + 1735704000, + 240, + 60, + &mut missing_hours, + &mut cache, + ) + .expect("csv should parse"); + + let vwap = (100.0 * 1.0 + 140.0 * 3.0) / 4.0; + assert_eq!(stats.inserted, 4); + assert_eq!(stats.replaced, 0); + assert_eq!(stats.skipped, 0); + assert_eq!(cache["XXBTZEUR:60:1735689600"], super::normalize_fiat_rate(vwap)); + assert_eq!(cache["XXBTZEUR:60:1735693200"], super::normalize_fiat_rate(vwap)); + assert_eq!(cache["XXBTZEUR:60:1735696800"], super::normalize_fiat_rate(vwap)); + assert_eq!(cache["XXBTZEUR:60:1735700400"], super::normalize_fiat_rate(vwap)); + assert!(missing_hours.is_empty()); + } + + #[test] + fn larger_trade_candle_fills_missing_interior_smaller_intervals() { + let trades = "\ +1735689900,100.0,1.0 +1735697100,140.0,3.0 +"; + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new(trades)); + let mut missing_hours = BTreeSet::from([1735693200_i64, 1735696800_i64]); + let mut cache = HashMap::new(); + + let stats = read_trade_interval_csv( + &mut reader, + "XBTEUR.csv", + "XXBTZEUR", + 1735689600, + 1735704000, + 240, + 60, + &mut missing_hours, + &mut cache, + ) + .expect("csv should parse"); + + let vwap = (100.0 * 1.0 + 140.0 * 3.0) / 4.0; + assert_eq!(stats.inserted, 2); + assert_eq!(stats.replaced, 0); + assert_eq!(stats.skipped, 0); + assert_eq!(cache["XXBTZEUR:60:1735693200"], super::normalize_fiat_rate(vwap)); + assert_eq!(cache["XXBTZEUR:60:1735696800"], super::normalize_fiat_rate(vwap)); + assert!(missing_hours.is_empty()); + } + + #[test] + fn trade_vwap_replaces_existing_cache_entry() { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new(REAL_XBTEUR_TRADES_SAMPLE)); + let mut missing_days = BTreeSet::from([1735689600_i64]); + let mut cache = HashMap::from([("XXBTZEUR:1440:1735689600".to_owned(), 1.23_f64)]); + + let stats = read_trade_interval_csv( + &mut reader, + "XBTEUR.csv", + "XXBTZEUR", + 1735689600, + 1735776000, + 1440, + 1440, + &mut missing_days, + &mut cache, + ) + .expect("csv should parse"); + + assert_eq!(stats.inserted, 0); + assert_eq!(stats.replaced, 1); + assert_eq!(stats.skipped, 0); + assert_ne!(cache["XXBTZEUR:1440:1735689600"], 1.23); + assert!(missing_days.is_empty()); + } + + #[test] + fn trade_vwap_skips_existing_entry_when_value_is_unchanged() { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new(REAL_XBTEUR_TRADES_SAMPLE)); + let day1 = ((90167.3 * 0.00054633) + (90167.3 * 0.00005463) + (90167.1 * 0.00002185)) + / (0.00054633 + 0.00005463 + 0.00002185); + let mut missing_days = BTreeSet::from([1735689600_i64]); + let mut cache = HashMap::from([("XXBTZEUR:1440:1735689600".to_owned(), day1)]); + + let stats = read_trade_interval_csv( + &mut reader, + "XBTEUR.csv", + "XXBTZEUR", + 1735689600, + 1735776000, + 1440, + 1440, + &mut missing_days, + &mut cache, + ) + .expect("csv should parse"); + + assert_eq!(stats.inserted, 0); + assert_eq!(stats.replaced, 0); + assert_eq!(stats.skipped, 1); + assert_eq!(cache["XXBTZEUR:1440:1735689600"], super::normalize_fiat_rate(day1)); + assert!(missing_days.is_empty()); + } + + #[test] + fn trade_archive_fixture_produces_expected_hourly_vwap_candles() { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new(REAL_XBTEUR_2026_03_21_TRADES)); + // 2026-03-21 02:00–04:59 UTC — three quiet hours from real Kraken trades. + // API 60-min OHLC VWAPs for this window: 60995.3, 61074.2, 61145.7 + let mut missing_hours = + BTreeSet::from([1774058400_i64, 1774062000_i64, 1774065600_i64]); + let mut cache = HashMap::new(); + + let stats = read_trade_interval_csv( + &mut reader, + "XBTEUR.csv", + "XXBTZEUR", + 1774058400, + 1774069200, + 60, + 60, + &mut missing_hours, + &mut cache, + ) + .expect("csv should parse"); + + assert_eq!(stats.inserted, 3); + assert_eq!(stats.replaced, 0); + assert_eq!(stats.skipped, 0); + assert_eq!(cache["XXBTZEUR:60:1774058400"], super::normalize_fiat_rate(60995.38)); + assert_eq!(cache["XXBTZEUR:60:1774062000"], super::normalize_fiat_rate(61074.23)); + assert_eq!(cache["XXBTZEUR:60:1774065600"], super::normalize_fiat_rate(61145.71)); + assert!(missing_hours.is_empty()); + } + + #[test] + fn reports_missing_trade_volume_clearly() { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new("1735689600,90167.3\n")); + let mut missing_days = BTreeSet::from([1735689600_i64]); + let mut cache = HashMap::new(); + + let err = read_trade_interval_csv( + &mut reader, + "XBTEUR.csv", + "XXBTZEUR", + 1735689600, + 1735776000, + 1440, + 1440, + &mut missing_days, + &mut cache, + ) + .expect_err("row should fail"); + + assert!(err + .to_string() + .contains("XBTEUR.csv row is missing the volume column")); + } + + #[test] + fn reports_invalid_trade_volume_clearly() { + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_reader(Cursor::new("1735689600,90167.3,nope\n")); + let mut missing_days = BTreeSet::from([1735689600_i64]); + let mut cache = HashMap::new(); + + let err = read_trade_interval_csv( + &mut reader, + "XBTEUR.csv", + "XXBTZEUR", + 1735689600, + 1735776000, + 1440, + 1440, + &mut missing_days, + &mut cache, + ) + .expect_err("row should fail"); + + assert!(err + .to_string() + .contains("invalid volume value nope in XBTEUR.csv")); + } + + #[test] + fn reuses_existing_archive_download_without_redownloading() { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time") + .as_nanos(); + let path = std::env::temp_dir().join(format!("btc-accounting-archive-{unique}.zip")); + let cursor = Cursor::new(Vec::::new()); + let mut writer = zip::ZipWriter::new(cursor); + let options: zip::write::SimpleFileOptions = zip::write::SimpleFileOptions::default(); + writer + .start_file("XBTEUR.csv", options) + .expect("start file"); + std::io::Write::write_all(&mut writer, b"1735689600,90167.3,0.1\n").expect("write file"); + let cursor = writer.finish().expect("finish zip"); + std::fs::write(&path, cursor.into_inner()).expect("fixture"); + let extracted_path = extracted_archive_entry_path("unused.zip", "XBTEUR.csv"); + + let result = ensure_archive_downloaded( + &reqwest::blocking::Client::new(), + &DriveFile { + id: "unused".to_owned(), + name: "unused.zip".to_owned(), + }, + &path, + super::ArchiveBackfillMode::Vwap, + "XBTEUR", + 1735776000, + 1440, + ); + + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&extracted_path); + assert_eq!( + result.expect("result"), + PreparedArchive { + archive_path: path, + archive_file_name: "unused.zip".to_owned(), + extracted_path, + } + ); + } + + #[test] + fn rejects_complete_archive_replacement_that_drops_old_coverage() { + let existing = ArchiveCoverage { + first: 1_672_531_200, // 2023-01-01 + last: 1_735_689_600, // 2025-01-01 + }; + let downloaded = ArchiveCoverage { + first: 1_704_067_200, // 2024-01-01 + last: 1_767_225_600, // 2026-01-01 + }; + + assert!(!accept_complete_archive_replacement( + Some(existing), + downloaded, + )); + } + + #[test] + fn accepts_complete_archive_for_current_run_when_it_reaches_needed_range() { + let existing = ArchiveCoverage { + first: 1_672_531_200, // 2023-01-01 + last: 1_704_067_200, // 2024-01-01 + }; + let downloaded = ArchiveCoverage { + first: 1_704_067_200, // 2024-01-01 + last: 1_735_689_600, // 2025-01-01 + }; + + assert!(!accept_complete_archive_replacement( + Some(existing), + downloaded, + )); + assert!(downloaded.last >= 1_735_689_600); + } + + #[test] + fn resolves_wrapped_archive_entry_by_suffix() { + let cursor = Cursor::new(Vec::::new()); + let mut writer = zip::ZipWriter::new(cursor); + let options: zip::write::SimpleFileOptions = zip::write::SimpleFileOptions::default(); + writer + .start_file("TimeAndSales_Combined/XBTEUR.csv", options) + .expect("start file"); + std::io::Write::write_all(&mut writer, b"1735689600,90167.3,0.1\n").expect("write file"); + let cursor = writer.finish().expect("finish zip"); + + let mut archive = zip::ZipArchive::new(Cursor::new(cursor.into_inner())).expect("archive"); + let resolved = resolve_archive_entry_name(&mut archive, "XBTEUR.csv").expect("resolve"); + + assert_eq!(resolved, "TimeAndSales_Combined/XBTEUR.csv"); + } +} diff --git a/src/commands/export.rs b/src/commands/export.rs index 3a6f8e6..0ccd7f1 100644 --- a/src/commands/export.rs +++ b/src/commands/export.rs @@ -1,17 +1,22 @@ use std::env; +use std::collections::HashSet; use std::fs::File; use std::io::BufWriter; use std::path::PathBuf; use anyhow::{Context, Result, bail}; -use chrono::NaiveDate; +use chrono::{Duration, NaiveDate}; use crate::accounting::{AccountingConfig, build_statement}; use crate::common::{AppConfig, default_bitcoin_datadir}; +use crate::export::{Entry, Statement}; +use crate::export::booking_date_to_date; +use crate::export::camt053::Camt053ParseResult; use crate::exchange_rate::KrakenProvider; use crate::export::camt053::Camt053Exporter; use crate::export::AccountingExporter; use crate::iban::iban_from_fingerprint; +use crate::import::WalletTransaction; use crate::import::TransactionSource; use crate::import::bitcoin_core_rpc::BitcoinCoreRpc; @@ -32,10 +37,10 @@ options: --fifo Use FIFO lot tracking for realized gains (env: FIFO) --start-date Start date YYYY-MM-DD --bank-name Bank/institution name (default: Bitcoin Core - ) - --candle Kraken candle interval + --candle Kraken candle interval (default: DEFAULT_CANDLE_MINUTES or 1440) --ignore-balance-mismatch Warn instead of error on forward/backward balance mismatch"; -#[derive(Debug)] +#[derive(Debug, PartialEq, Eq)] pub struct ExportArgs { pub wallet: Option, pub country: String, @@ -47,16 +52,34 @@ pub struct ExportArgs { pub fifo: bool, pub start_date: Option, pub output: PathBuf, - pub candle_minutes: u32, + pub candle_override_minutes: Option, pub bank_name: Option, pub ignore_balance_mismatch: bool, } -#[derive(Debug)] +#[derive(Debug, PartialEq, Eq)] pub enum ExportFormat { Camt053, } +#[derive(Debug)] +struct ExistingExportPlan { + merge_mode: ExistingMergeMode, + build_opening_balance_cents: i64, + build_start_date: Option, + build_end_exclusive: Option, + existing_opening_balance_cents: i64, + existing_opening_date: String, + existing_entries: Vec, + existing_entry_refs: HashSet, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ExistingMergeMode { + Append, + Prepend, +} + pub fn run(args: ExportArgs) -> Result<()> { // Resolve wallet name: use provided, or auto-detect the single loaded wallet let wallet = match args.wallet { @@ -80,7 +103,7 @@ pub fn run(args: ExportArgs) -> Result<()> { let iban = iban_from_fingerprint(&fingerprint, &args.country, &args.chain)?; eprintln!("Virtual IBAN: {iban}"); - let transactions = rpc.list_transactions()?; + let mut transactions = rpc.list_transactions()?; let wallet_balance_sats = rpc.get_balance()?; // Collect receive addresses and fetch matching watch-only descriptors @@ -91,6 +114,10 @@ pub fn run(args: ExportArgs) -> Result<()> { let descriptors = rpc.get_receive_descriptors(&receive_addresses)?; let app_config = AppConfig::from_env()?; + let candle_minutes = resolve_candle_minutes( + args.candle_override_minutes, + app_config.default_candle_minutes, + ); let provider = KrakenProvider::new(&app_config)?; let currency = if args.fiat_mode { @@ -104,7 +131,7 @@ pub fn run(args: ExportArgs) -> Result<()> { // Auto-detect append mode: if output file exists, parse it for dedup and continuation let append = args.output.exists(); - let (opening_balance_cents, start_date, existing_entries, existing_entry_refs) = if append { + let existing_plan = if append { let output_path = &args.output; let existing_xml = std::fs::read_to_string(output_path) @@ -121,18 +148,38 @@ pub fn run(args: ExportArgs) -> Result<()> { bail!("currency mismatch: file has {} but current config uses {}", parsed.currency, currency); } - // Use last booking date + 1 day as implicit start date (unless explicitly set) - let implicit_start = parsed.last_booking_date - .and_then(|d| chrono::NaiveDate::parse_from_str(&d, "%Y-%m-%d").ok()) - .map(|d| d + chrono::Duration::days(1)); - - let start = args.start_date.or(implicit_start); - - (parsed.closing_balance_cents, start, parsed.existing_entries, parsed.existing_entry_refs) + let plan = existing_export_plan_from_existing_export(args.start_date, parsed)?; + if plan.merge_mode == ExistingMergeMode::Prepend { + eprintln!( + "Requested start date predates the existing export; prepending entries before {} and replacing the opening balance in {}.", + plan.existing_opening_date, + output_path.display(), + ); + } + Some(plan) } else { - (0, args.start_date, Vec::new(), std::collections::HashSet::new()) + None }; + if let Some(plan) = existing_plan.as_ref() { + if !plan.existing_entry_refs.is_empty() { + transactions.retain(|tx| !plan.existing_entry_refs.contains(&transaction_entry_ref(tx))); + } + if let Some(end_exclusive) = plan.build_end_exclusive { + let end_exclusive_ts = end_exclusive + .and_hms_opt(0, 0, 0) + .expect("midnight should be valid") + .and_utc() + .timestamp(); + transactions.retain(|tx| tx.block_time < end_exclusive_ts); + } + } + + let existing_entry_refs = existing_plan + .as_ref() + .map(|plan| plan.existing_entry_refs.clone()) + .unwrap_or_default(); + let bank_name = Some(args.bank_name.unwrap_or_else(|| format!("Bitcoin Core - {wallet}"))); let config = AccountingConfig { @@ -141,27 +188,37 @@ pub fn run(args: ExportArgs) -> Result<()> { fifo: args.fifo, currency: currency.clone(), account_iban: iban, - candle_interval_minutes: args.candle_minutes, - start_date, - opening_balance_cents, + candle_interval_minutes: candle_minutes, + start_date: existing_plan + .as_ref() + .map(|plan| plan.build_start_date) + .unwrap_or(args.start_date), + opening_balance_cents: existing_plan + .as_ref() + .map(|plan| plan.build_opening_balance_cents) + .unwrap_or(0), bank_name, - wallet_balance_sats: Some(wallet_balance_sats), + wallet_balance_sats: if existing_plan + .as_ref() + .is_some_and(|plan| plan.merge_mode == ExistingMergeMode::Prepend) + { + None + } else { + Some(wallet_balance_sats) + }, ignore_balance_mismatch: args.ignore_balance_mismatch, }; let mut statement = build_statement(&transactions, &provider, &config)?; statement.descriptors = descriptors; - // Dedup: remove entries that already exist in the parsed file - if !existing_entry_refs.is_empty() { - statement.entries.retain(|e| !existing_entry_refs.contains(&e.entry_ref)); - } - - // Prepend existing entries - if !existing_entries.is_empty() { - let mut all_entries = existing_entries; - all_entries.append(&mut statement.entries); - statement.entries = all_entries; + if let Some(plan) = existing_plan { + if !existing_entry_refs.is_empty() { + statement + .entries + .retain(|e| !existing_entry_refs.contains(&e.entry_ref)); + } + merge_with_existing_statement(&mut statement, plan); } // Count new entries (excluding those from the existing file) @@ -227,6 +284,123 @@ fn quote_currency_from_pair(pair: &str) -> String { } } +fn resolve_candle_minutes(candle_override_minutes: Option, default_candle_minutes: Option) -> u32 { + candle_override_minutes + .or(default_candle_minutes) + .unwrap_or(1440) +} + +fn existing_export_plan_from_existing_export( + requested_start_date: Option, + parsed: Camt053ParseResult, +) -> Result { + let existing_opening_date = parse_existing_opening_date(parsed.opening_date.as_deref())?; + let last_booking_date = parse_last_booking_date(parsed.last_booking_date.as_deref())?; + + if let Some(requested_start_date) = requested_start_date { + if requested_start_date > existing_opening_date { + bail!( + "output file already starts at {existing_opening_date}; cannot move --start-date forward to {requested_start_date}" + ); + } + + if requested_start_date < existing_opening_date { + return Ok(ExistingExportPlan { + merge_mode: ExistingMergeMode::Prepend, + build_opening_balance_cents: 0, + build_start_date: Some(requested_start_date), + build_end_exclusive: Some(existing_opening_date), + existing_opening_balance_cents: parsed.opening_balance_cents, + existing_opening_date: existing_opening_date.to_string(), + existing_entries: parsed.existing_entries, + existing_entry_refs: parsed.existing_entry_refs, + }); + } + + return Ok(ExistingExportPlan { + merge_mode: ExistingMergeMode::Append, + build_opening_balance_cents: parsed.closing_balance_cents, + build_start_date: last_booking_date.map(|date| date + Duration::days(1)), + build_end_exclusive: None, + existing_opening_balance_cents: parsed.opening_balance_cents, + existing_opening_date: existing_opening_date.to_string(), + existing_entries: parsed.existing_entries, + existing_entry_refs: parsed.existing_entry_refs, + }); + } + + Ok(ExistingExportPlan { + merge_mode: ExistingMergeMode::Append, + build_opening_balance_cents: parsed.closing_balance_cents, + build_start_date: last_booking_date.map(|date| date + Duration::days(1)), + build_end_exclusive: None, + existing_opening_balance_cents: parsed.opening_balance_cents, + existing_opening_date: existing_opening_date.to_string(), + existing_entries: parsed.existing_entries, + existing_entry_refs: parsed.existing_entry_refs, + }) +} + +fn parse_existing_opening_date(opening_date: Option<&str>) -> Result { + let opening_date = opening_date.context("existing CAMT.053 file is missing its opening balance date")?; + NaiveDate::parse_from_str(opening_date, "%Y-%m-%d") + .with_context(|| format!("invalid opening balance date in existing CAMT.053 file: {opening_date}")) +} + +fn parse_last_booking_date(last_booking_date: Option<&str>) -> Result> { + last_booking_date + .map(|date| { + NaiveDate::parse_from_str(date, "%Y-%m-%d") + .with_context(|| format!("invalid booking date in existing CAMT.053 file: {date}")) + }) + .transpose() +} + +fn transaction_entry_ref(tx: &WalletTransaction) -> String { + format!("{}:{}:{}", tx.block_height, &tx.txid[..20.min(tx.txid.len())], tx.vout) +} + +fn merge_with_existing_statement(statement: &mut Statement, plan: ExistingExportPlan) { + let mut combined_entries = match plan.merge_mode { + ExistingMergeMode::Append => { + let mut combined = plan.existing_entries; + combined.append(&mut statement.entries); + statement.opening_balance_cents = plan.existing_opening_balance_cents; + statement.opening_date = plan.existing_opening_date; + statement.opening_balance_sats = 0; + statement.opening_rate = None; + combined + } + ExistingMergeMode::Prepend => { + let mut combined = statement.entries.clone(); + combined.extend(plan.existing_entries); + combined + } + }; + statement.entries.clear(); + statement.entries.append(&mut combined_entries); + refresh_statement_totals(statement); +} + +fn refresh_statement_totals(statement: &mut Statement) { + let net_entries = statement.entries.iter().map(signed_entry_amount).sum::(); + statement.closing_balance_cents = statement.opening_balance_cents + net_entries; + statement.statement_date = statement + .entries + .last() + .map(|entry| booking_date_to_date(&entry.booking_date).to_owned()) + .unwrap_or_else(|| statement.opening_date.clone()); + statement.statement_id = format!("STMT-{}", statement.statement_date); +} + +fn signed_entry_amount(entry: &Entry) -> i64 { + if entry.is_credit { + entry.amount_cents + } else { + -entry.amount_cents + } +} + pub fn parse_args_from(args: I, usage: &str) -> Result where I: IntoIterator, @@ -346,8 +520,6 @@ where .map(|v| v.eq_ignore_ascii_case("true")) .unwrap_or(false); - let candle_minutes = candle_minutes.unwrap_or(1440); - Ok(ExportArgs { wallet, country, @@ -359,8 +531,220 @@ where fifo: fifo || fifo_env, start_date, output, - candle_minutes, + candle_override_minutes: candle_minutes, bank_name, ignore_balance_mismatch, }) } + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use chrono::NaiveDate; + + use crate::export::Entry; + use crate::import::{TxCategory, WalletTransaction}; + + use super::{ + Camt053ParseResult, ExistingMergeMode, ExportArgs, ExportFormat, USAGE, + existing_export_plan_from_existing_export, parse_args_from, resolve_candle_minutes, + transaction_entry_ref, + }; + + #[test] + fn parses_export_args_without_candle_override() { + let args = parse_args_from( + vec![ + "--country".to_owned(), + "NL".to_owned(), + "--wallet".to_owned(), + "test_wallet".to_owned(), + "--output".to_owned(), + "my-wallet.xml".to_owned(), + ], + USAGE, + ) + .expect("args"); + + assert_eq!( + args, + ExportArgs { + wallet: Some("test_wallet".to_owned()), + country: "NL".to_owned(), + datadir: crate::common::default_bitcoin_datadir(), + chain: "main".to_owned(), + format: ExportFormat::Camt053, + fiat_mode: false, + mark_to_market: None, + fifo: false, + start_date: None, + output: "my-wallet.xml".into(), + candle_override_minutes: None, + bank_name: None, + ignore_balance_mismatch: false, + } + ); + } + + #[test] + fn parses_export_args_with_candle_override() { + let args = parse_args_from( + vec![ + "--country".to_owned(), + "NL".to_owned(), + "--wallet".to_owned(), + "test_wallet".to_owned(), + "--output".to_owned(), + "my-wallet.xml".to_owned(), + "--candle".to_owned(), + "60".to_owned(), + ], + USAGE, + ) + .expect("args"); + + assert_eq!(args.candle_override_minutes, Some(60)); + } + + #[test] + fn export_uses_default_candle_minutes_when_present() { + assert_eq!(resolve_candle_minutes(None, Some(60)), 60); + } + + #[test] + fn export_candle_override_beats_default_candle_minutes() { + assert_eq!(resolve_candle_minutes(Some(240), Some(60)), 240); + } + + #[test] + fn export_defaults_to_daily_when_no_override_or_env_default_exists() { + assert_eq!(resolve_candle_minutes(None, None), 1440); + } + + #[test] + fn append_plan_uses_day_after_last_booking_when_no_start_date_is_provided() { + let parsed = Camt053ParseResult { + opening_balance_cents: 100, + opening_date: Some("2025-01-01".to_owned()), + existing_entry_refs: HashSet::from(["100:abcd:0".to_owned()]), + existing_entries: vec![Entry { + entry_ref: "100:abcd:0".to_owned(), + full_ref: "full".to_owned(), + booking_date: "2025-01-01T00:00:00".to_owned(), + amount_cents: 100, + is_credit: true, + description: "test".to_owned(), + is_fee: false, + }], + closing_balance_cents: 123, + account_iban: "NL00XBTC0000000000".to_owned(), + currency: "EUR".to_owned(), + last_booking_date: Some("2025-01-01".to_owned()), + descriptors: vec![], + }; + + let plan = existing_export_plan_from_existing_export(None, parsed).expect("plan"); + + assert_eq!(plan.build_opening_balance_cents, 123); + assert_eq!( + plan.build_start_date, + Some(NaiveDate::from_ymd_opt(2025, 1, 2).expect("date")) + ); + assert_eq!(plan.existing_entries.len(), 1); + assert!(plan.existing_entry_refs.contains("100:abcd:0")); + assert_eq!(plan.merge_mode, ExistingMergeMode::Append); + } + + #[test] + fn earlier_start_date_triggers_prepend_mode() { + let parsed = Camt053ParseResult { + opening_balance_cents: 100, + opening_date: Some("2025-01-01".to_owned()), + existing_entry_refs: HashSet::from(["100:abcd:0".to_owned()]), + existing_entries: vec![Entry { + entry_ref: "100:abcd:0".to_owned(), + full_ref: "full".to_owned(), + booking_date: "2025-01-01T00:00:00".to_owned(), + amount_cents: 100, + is_credit: true, + description: "test".to_owned(), + is_fee: false, + }], + closing_balance_cents: 123, + account_iban: "NL00XBTC0000000000".to_owned(), + currency: "EUR".to_owned(), + last_booking_date: Some("2025-12-31".to_owned()), + descriptors: vec![], + }; + + let plan = existing_export_plan_from_existing_export( + Some(NaiveDate::from_ymd_opt(2024, 1, 1).expect("date")), + parsed, + ) + .expect("plan"); + + assert_eq!(plan.build_opening_balance_cents, 0); + assert_eq!( + plan.build_start_date, + Some(NaiveDate::from_ymd_opt(2024, 1, 1).expect("date")) + ); + assert_eq!( + plan.build_end_exclusive, + Some(NaiveDate::from_ymd_opt(2025, 1, 1).expect("date")) + ); + assert_eq!(plan.existing_entries.len(), 1); + assert!(plan.existing_entry_refs.contains("100:abcd:0")); + assert_eq!(plan.merge_mode, ExistingMergeMode::Prepend); + } + + #[test] + fn moving_start_date_forward_is_rejected() { + let parsed = Camt053ParseResult { + opening_balance_cents: 100, + opening_date: Some("2025-01-01".to_owned()), + existing_entry_refs: HashSet::from(["100:abcd:0".to_owned()]), + existing_entries: vec![Entry { + entry_ref: "100:abcd:0".to_owned(), + full_ref: "full".to_owned(), + booking_date: "2025-01-01T00:00:00".to_owned(), + amount_cents: 100, + is_credit: true, + description: "test".to_owned(), + is_fee: false, + }], + closing_balance_cents: 123, + account_iban: "NL00XBTC0000000000".to_owned(), + currency: "EUR".to_owned(), + last_booking_date: Some("2025-12-31".to_owned()), + descriptors: vec![], + }; + + let err = existing_export_plan_from_existing_export( + Some(NaiveDate::from_ymd_opt(2025, 6, 1).expect("date")), + parsed, + ) + .expect_err("moving start date forward should fail"); + + assert!(err.to_string().contains("already starts at 2025-01-01")); + assert!(err.to_string().contains("--start-date forward to 2025-06-01")); + } + + #[test] + fn transaction_entry_ref_matches_export_entry_reference_format() { + let tx = WalletTransaction { + txid: "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789".to_owned(), + vout: 7, + amount_sats: 1, + fee_sats: None, + category: TxCategory::Receive, + block_time: 1_735_700_000, + block_height: 123, + block_hash: "bb".repeat(32), + address: "bc1qtest".to_owned(), + label: String::new(), + }; + + assert_eq!(transaction_entry_ref(&tx), "123:abcdef0123456789abcd:7"); + } +} diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 4d2b414..26b3343 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -1,3 +1,4 @@ +pub mod cache_rates; pub mod export; pub mod received_value; pub mod reconstruct; diff --git a/src/common.rs b/src/common.rs index 820b864..fbe0f10 100644 --- a/src/common.rs +++ b/src/common.rs @@ -434,6 +434,56 @@ pub fn fetch_candle_for_timestamp( return Err(anyhow::Error::new(RateLimitedError { retry_after_secs })); } + let candle_rows = parse_kraken_ohlc_response(response, config, &url)?; + + candle_rows + .iter() + .filter_map(parse_candle_row) + .find(|candle| timestamp >= candle.time && timestamp < candle.time + interval_seconds) + .ok_or_else(|| { + anyhow!( + "Kraken did not return the {} minute candle covering {}", + interval_minutes, + format_local_timestamp(timestamp) + ) + }) +} + +pub fn fetch_candles_since( + client: &Client, + config: &AppConfig, + interval_minutes: u32, + since: i64, +) -> Result> { + let url = format!( + "{}/0/public/OHLC?pair={}&interval={interval_minutes}&since={since}", + KRAKEN_BASE_URL, config.kraken_pair + ); + + let response = client + .get(&url) + .send() + .with_context(|| format!("failed to query Kraken at {url}"))?; + + let candle_rows = parse_kraken_ohlc_response(response, config, &url)?; + + Ok(candle_rows.iter().filter_map(parse_candle_row).collect()) +} + +fn parse_kraken_ohlc_response( + response: reqwest::blocking::Response, + config: &AppConfig, + url: &str, +) -> Result> { + if response.status() == StatusCode::TOO_MANY_REQUESTS { + let retry_after_secs = response + .headers() + .get(reqwest::header::RETRY_AFTER) + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.parse::().ok()); + return Err(anyhow::Error::new(RateLimitedError { retry_after_secs })); + } + let response = response .error_for_status() .with_context(|| format!("Kraken returned an error for {url}"))?; @@ -452,22 +502,12 @@ pub fn fetch_candle_for_timestamp( let result = payload .result .context("Kraken response is missing a result field")?; - let candle_rows = result + + result .get(&config.kraken_pair) .and_then(Value::as_array) - .ok_or_else(|| anyhow!("Kraken response does not include pair {}", config.kraken_pair))?; - - candle_rows - .iter() - .filter_map(parse_candle_row) - .find(|candle| timestamp >= candle.time && timestamp < candle.time + interval_seconds) - .ok_or_else(|| { - anyhow!( - "Kraken did not return the {} minute candle covering {}", - interval_minutes, - format_local_timestamp(timestamp) - ) - }) + .cloned() + .ok_or_else(|| anyhow!("Kraken response does not include pair {}", config.kraken_pair)) } fn parse_candle_row(row: &Value) -> Option { diff --git a/src/exchange_rate.rs b/src/exchange_rate.rs index 542d427..53ef4d4 100644 --- a/src/exchange_rate.rs +++ b/src/exchange_rate.rs @@ -10,8 +10,8 @@ use reqwest::blocking::Client; use crate::common::{AppConfig, Candle, RateLimitedError, build_http_client, fetch_candle_for_timestamp}; -const CACHE_DIR: &str = ".cache"; -const CACHE_FILE: &str = "rates.json"; +pub const CACHE_DIR: &str = ".cache"; +pub const CACHE_FILE: &str = "rates.json"; const REQUEST_DELAY: Duration = Duration::from_millis(1500); /// Provides VWAP exchange rates for a given timestamp. @@ -37,7 +37,7 @@ impl KrakenProvider { .transpose()?; let clearnet_client = build_http_client("clearnet Kraken", None)?; - let disk_cache = Self::load_disk_cache(); + let disk_cache = load_disk_cache(); let initial_cache_size = disk_cache.len(); @@ -51,27 +51,9 @@ impl KrakenProvider { }) } - fn cache_key(pair: &str, interval_minutes: u32, candle_start: i64) -> String { - format!("{pair}:{interval_minutes}:{candle_start}") - } - - fn cache_path() -> PathBuf { - PathBuf::from(CACHE_DIR).join(CACHE_FILE) - } - - fn load_disk_cache() -> HashMap { - let path = Self::cache_path(); - fs::read_to_string(&path) - .ok() - .and_then(|s| serde_json::from_str(&s).ok()) - .unwrap_or_default() - } - fn save_disk_cache(&self) { let Ok(cache) = self.cache.lock() else { return }; - let Ok(json) = serde_json::to_string_pretty(&*cache) else { return }; - let _ = fs::create_dir_all(CACHE_DIR); - let _ = fs::write(Self::cache_path(), json); + let _ = save_disk_cache(&cache); } /// Returns true if new entries were written to the disk cache. @@ -120,7 +102,7 @@ impl ExchangeRateProvider for KrakenProvider { fn get_vwap(&self, timestamp: i64, interval_minutes: u32) -> Result { let interval_seconds = i64::from(interval_minutes) * 60; let candle_start = (timestamp / interval_seconds) * interval_seconds; - let key = Self::cache_key(&self.config.kraken_pair, interval_minutes, candle_start); + let key = cache_key(&self.config.kraken_pair, interval_minutes, candle_start); { let cache = self.cache.lock().map_err(|e| anyhow!("cache lock: {e}"))?; @@ -142,6 +124,29 @@ impl ExchangeRateProvider for KrakenProvider { } } +pub fn cache_key(pair: &str, interval_minutes: u32, candle_start: i64) -> String { + format!("{pair}:{interval_minutes}:{candle_start}") +} + +pub fn cache_path() -> PathBuf { + PathBuf::from(CACHE_DIR).join(CACHE_FILE) +} + +pub fn load_disk_cache() -> HashMap { + let path = cache_path(); + fs::read_to_string(&path) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default() +} + +pub fn save_disk_cache(cache: &HashMap) -> Result<()> { + let json = serde_json::to_string_pretty(cache)?; + fs::create_dir_all(CACHE_DIR)?; + fs::write(cache_path(), json)?; + Ok(()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/export/camt053.rs b/src/export/camt053.rs index ff6a65a..e61ab43 100644 --- a/src/export/camt053.rs +++ b/src/export/camt053.rs @@ -213,6 +213,10 @@ fn xml_escape_comment(s: &str) -> String { /// Data extracted from an existing CAMT.053 file for append mode. pub struct Camt053ParseResult { + /// Opening balance of the existing file. + pub opening_balance_cents: i64, + /// Opening balance date of the existing file. + pub opening_date: Option, /// Set of NtryRef values already in the file (for dedup). pub existing_entry_refs: HashSet, /// All existing entries (preserved in output). @@ -238,6 +242,8 @@ pub fn parse_camt053(xml: &str) -> Result { let mut account_iban = String::new(); let mut currency = String::new(); + let mut opening_balance_cents: i64 = 0; + let mut opening_date: Option = None; let mut closing_balance_cents: i64 = 0; let mut existing_entry_refs = HashSet::new(); let mut existing_entries = Vec::new(); @@ -262,6 +268,7 @@ pub fn parse_camt053(xml: &str) -> Result { let mut bal_amount_cents: i64 = 0; let mut bal_is_credit = true; let mut bal_is_btc = false; + let mut bal_date = String::new(); loop { match reader.read_event() { @@ -286,6 +293,7 @@ pub fn parse_camt053(xml: &str) -> Result { bal_amount_cents = 0; bal_is_credit = true; bal_is_btc = false; + bal_date.clear(); } "Amt" if in_bal => { // Skip BTC-denominated balances @@ -316,12 +324,19 @@ pub fn parse_camt053(xml: &str) -> Result { if name == "Bal" && in_bal { in_bal = false; + let signed_balance = if bal_is_credit { + bal_amount_cents + } else { + -bal_amount_cents + }; + if bal_code == "OPBD" { + opening_balance_cents = signed_balance; + if !bal_date.is_empty() { + opening_date = Some(bal_date.clone()); + } + } if bal_code == "CLBD" { - closing_balance_cents = if bal_is_credit { - bal_amount_cents - } else { - -bal_amount_cents - }; + closing_balance_cents = signed_balance; } } @@ -364,6 +379,12 @@ pub fn parse_camt053(xml: &str) -> Result { bal_amount_cents = parse_amount_cents(&text)?; } "CdtDbtInd" => bal_is_credit = text == "CRDT", + "Dt" | "DtTm" if path_contains(&path, "Bal") && path_contains(&path, "Dt") => { + let trimmed = text.trim(); + if bal_date.is_empty() && !trimmed.is_empty() { + bal_date = trimmed.to_owned(); + } + } _ => {} } } else { @@ -405,6 +426,8 @@ pub fn parse_camt053(xml: &str) -> Result { .map(|e| booking_date_to_date(&e.booking_date).to_owned()); Ok(Camt053ParseResult { + opening_balance_cents, + opening_date, existing_entry_refs, existing_entries, closing_balance_cents, @@ -557,6 +580,8 @@ mod tests { assert_eq!(parsed.account_iban, "NL00XBTC0000000000"); assert_eq!(parsed.currency, "EUR"); + assert_eq!(parsed.opening_balance_cents, 0); + assert_eq!(parsed.opening_date, Some("2025-01-02".to_owned())); assert_eq!(parsed.closing_balance_cents, 499_985); assert_eq!(parsed.existing_entry_refs.len(), 2); assert!(parsed.existing_entry_refs.contains("100:abcdef01234567890abc:0")); diff --git a/src/main.rs b/src/main.rs index 58338e8..d9675de 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,11 +3,12 @@ use std::env; use anyhow::{Result, anyhow, bail}; use dotenvy::dotenv; +use btc_fiat_value::commands::cache_rates::{self as cache_rates_cmd, CacheRatesArgs}; use btc_fiat_value::commands::export::{self as export_cmd, ExportArgs}; use btc_fiat_value::commands::received_value::{self, ReceivedValueArgs}; use btc_fiat_value::commands::reconstruct::{self as reconstruct_cmd, ReconstructArgs}; -const ROOT_USAGE: &str = "usage: btc_fiat_value [options]\n\nsubcommands:\n received-value find the quote-currency value when BTC was received\n export export wallet transactions to accounting format\n reconstruct verify an export by reconstructing the wallet"; +const ROOT_USAGE: &str = "usage: btc_fiat_value [options]\n\nsubcommands:\n received-value find the quote-currency value when BTC was received\n cache-rates populate .cache/rates.json for one year of rates\n export export wallet transactions to accounting format\n reconstruct verify an export by reconstructing the wallet"; fn main() { if let Err(err) = run() { @@ -20,6 +21,7 @@ fn run() -> Result<()> { let _ = dotenv(); match parse_command()? { Command::ReceivedValue(args) => received_value::run(args), + Command::CacheRates(args) => cache_rates_cmd::run(args), Command::Export(args) => export_cmd::run(args), Command::Reconstruct(args) => reconstruct_cmd::run(args), } @@ -28,6 +30,7 @@ fn run() -> Result<()> { #[derive(Debug)] enum Command { ReceivedValue(ReceivedValueArgs), + CacheRates(CacheRatesArgs), Export(ExportArgs), Reconstruct(ReconstructArgs), } @@ -47,6 +50,9 @@ where received_value::SUBCOMMAND_NAME => Ok(Command::ReceivedValue( received_value::parse_args_from(args, received_value::USAGE)?, )), + cache_rates_cmd::SUBCOMMAND_NAME => Ok(Command::CacheRates( + cache_rates_cmd::parse_args_from(args, cache_rates_cmd::USAGE)?, + )), export_cmd::SUBCOMMAND_NAME => Ok(Command::Export( export_cmd::parse_args_from(args, export_cmd::USAGE)?, )), @@ -107,6 +113,61 @@ mod tests { } } + #[test] + fn parses_cache_rates_subcommand() { + let command = parse_command_from(vec!["cache-rates".to_owned(), "2024".to_owned()]) + .expect("command"); + + match command { + Command::CacheRates(args) => { + assert_eq!(args.year, 2024); + assert!(!args.use_vwap_archive); + assert_eq!(args.candle_override_minutes, None); + } + _ => panic!("expected CacheRates"), + } + } + + #[test] + fn parses_cache_rates_vwap_subcommand() { + let command = parse_command_from(vec![ + "cache-rates".to_owned(), + "--vwap".to_owned(), + "2024".to_owned(), + ]) + .expect("command"); + + match command { + Command::CacheRates(args) => { + assert_eq!(args.year, 2024); + assert!(args.use_vwap_archive); + assert_eq!(args.candle_override_minutes, None); + } + _ => panic!("expected CacheRates"), + } + } + + #[test] + fn parses_cache_rates_vwap_candle_override_subcommand() { + let command = parse_command_from(vec![ + "cache-rates".to_owned(), + "--vwap".to_owned(), + "--candle".to_owned(), + "60".to_owned(), + "2024".to_owned(), + ]) + .expect("command"); + + match command { + Command::CacheRates(args) => { + assert_eq!(args.year, 2024); + assert!(args.use_vwap_archive); + assert_eq!(args.candle_override_minutes, Some(60)); + } + _ => panic!("expected CacheRates"), + } + } + #[test] fn parses_reconstruct_subcommand() { let command = parse_command_from(vec![ diff --git a/tests/fixtures/kraken_xbteur_2026-03-21_trades.csv b/tests/fixtures/kraken_xbteur_2026-03-21_trades.csv new file mode 100644 index 0000000..55b6726 --- /dev/null +++ b/tests/fixtures/kraken_xbteur_2026-03-21_trades.csv @@ -0,0 +1,767 @@ +1774058400,60912.70000,0.00008127 +1774058400,60912.70000,0.00026138 +1774058400,60912.70000,0.00001634 +1774058400,60912.70000,0.00016174 +1774058400,60912.70000,0.00243816 +1774058400,60912.70000,0.00016335 +1774058402,60912.70000,0.00024600 +1774058402,60912.70000,0.00013081 +1774058402,60912.70000,0.00058864 +1774058403,60912.70000,0.00022656 +1774058405,60912.60000,0.00016400 +1774058405,60912.60000,0.00009850 +1774058409,60914.80000,0.00057171 +1774058410,60914.80000,0.00019776 +1774058410,60914.80000,0.00016480 +1774058411,60914.80000,0.00010877 +1774058411,60914.80000,0.00013184 +1774058411,60914.80000,0.00013184 +1774058411,60914.80000,0.00009888 +1774058411,60914.80000,0.00023553 +1774058411,60914.80000,0.00019776 +1774058411,60914.80000,0.00008240 +1774058412,60914.80000,0.00049921 +1774058412,60914.80000,0.00024294 +1774058412,60914.80000,0.00013184 +1774058412,60914.80000,0.00019776 +1774058412,60914.80000,0.00013184 +1774058413,60914.80000,0.00033441 +1774058413,60914.80000,0.00019776 +1774058413,60914.80000,0.00026368 +1774058413,60914.80000,0.00016480 +1774058413,60914.80000,0.00016196 +1774058413,60914.80000,0.00010921 +1774058413,60914.80000,0.00026368 +1774058413,60914.80000,0.00026368 +1774058413,60914.80000,0.00008000 +1774058414,60914.80000,0.00049921 +1774058414,60914.80000,0.00008000 +1774058417,60914.80000,0.00001617 +1774058419,60914.80000,0.00016000 +1774058419,60914.80000,0.00016000 +1774058419,60914.80000,0.00016000 +1774058419,60914.80000,0.00016000 +1774058419,60914.80000,0.00049249 +1774058422,60914.80000,0.00010836 +1774058427,60921.30000,0.00016093 +1774058438,60929.10000,0.00016169 +1774058450,60922.80000,0.00035998 +1774058459,60929.10000,0.00161700 +1774058495,60939.10000,0.00243686 +1774058496,60938.80000,0.00160012 +1774058517,60943.30000,0.00016162 +1774058553,60905.80000,0.00032273 +1774058555,60905.80000,0.00324000 +1774058585,60896.20000,0.00240313 +1774058607,60883.20000,0.00122225 +1774058610,60883.20000,0.00048787 +1774058625,60884.10000,0.00040437 +1774058632,60886.40000,0.00023634 +1774058639,60877.30000,0.00016860 +1774058641,60900.80000,0.01550000 +1774058644,60922.40000,0.00162830 +1774058653,60903.80000,0.00247122 +1774058699,60899.40000,0.00169502 +1774058700,60901.70000,0.00003220 +1774058705,60909.30000,0.00036748 +1774058709,60914.10000,0.00015994 +1774058826,60907.90000,0.01492875 +1774058892,60941.20000,0.00180780 +1774058905,60926.90000,0.04584000 +1774058933,60918.30000,0.00061970 +1774058966,60918.30000,0.00003219 +1774058984,60918.30000,0.01332554 +1774058984,60931.00000,0.00427676 +1774059001,60927.30000,0.00197034 +1774059005,60927.20000,0.00134915 +1774059012,60927.30000,0.01480000 +1774059021,60927.20000,0.00079193 +1774059021,60927.30000,0.00519508 +1774059023,60927.20000,0.00033734 +1774059038,60925.40000,0.00538293 +1774059039,60925.40000,0.00001616 +1774059076,60925.30000,0.00080696 +1774059076,60922.20000,0.00020844 +1774059103,60913.10000,0.00002566 +1774059133,60913.00000,0.00740000 +1774059152,60913.10000,0.00023966 +1774059181,60913.10000,0.00259212 +1774059194,60913.10000,0.00080871 +1774059220,60905.30000,0.00079117 +1774059244,60913.00000,0.00051618 +1774059293,60913.00000,0.00006902 +1774059302,60913.10000,0.01260595 +1774059305,60941.60000,0.00023027 +1774059305,60941.60000,0.00019777 +1774059312,60941.60000,0.00016087 +1774059328,60943.00000,0.00044969 +1774059331,60942.90000,0.00161665 +1774059331,60942.90000,0.00283096 +1774059347,60942.90000,0.00027348 +1774059363,60943.00000,0.00015555 +1774059377,60954.70000,0.00151900 +1774059394,60958.30000,0.00014964 +1774059410,60941.60000,0.00024614 +1774059422,60951.10000,0.00013176 +1774059481,60972.60000,0.00010000 +1774059483,60977.80000,0.00010000 +1774059536,60970.50000,0.00001581 +1774059548,60983.40000,0.00010000 +1774059549,60984.10000,0.00023535 +1774059601,60974.20000,0.00015100 +1774059605,60974.90000,0.00009878 +1774059608,60977.00000,0.00010000 +1774059613,60977.40000,0.00120000 +1774059642,60979.80000,0.00047142 +1774059662,60980.70000,0.00500000 +1774059667,60976.90000,0.00395570 +1774059671,60978.50000,0.00038926 +1774059681,60981.30000,0.00362781 +1774059695,60977.60000,0.01550000 +1774059695,60976.50000,0.03202137 +1774059698,60988.40000,0.00010000 +1774059737,60990.00000,0.00003231 +1774059737,60990.00000,0.00010000 +1774059747,60989.70000,0.00038912 +1774059756,60990.80000,0.00031383 +1774059796,60999.00000,0.00010000 +1774059802,60999.50000,0.00290277 +1774059812,60999.30000,0.00038906 +1774059822,60995.40000,0.00014950 +1774059827,60995.30000,0.00033773 +1774059838,60998.20000,0.00176500 +1774059845,60995.30000,0.00075051 +1774059853,60998.50000,0.00010000 +1774059905,61000.00000,0.00010000 +1774059925,61000.00000,0.00002403 +1774059925,61008.90000,0.00002597 +1774059946,61008.90000,0.00044382 +1774059965,61021.30000,0.00010000 +1774060020,61055.00000,0.00008149 +1774060024,61054.00000,0.00010000 +1774060040,61054.90000,0.00079703 +1774060073,61064.30000,0.00296660 +1774060085,61070.30000,0.00012156 +1774060113,61073.20000,0.00032105 +1774060127,61073.00000,0.00008106 +1774060135,61073.00000,0.00006630 +1774060163,61098.70000,0.00010000 +1774060200,61055.60000,0.00008029 +1774060201,61045.70000,0.00008181 +1774060203,61048.30000,0.00019730 +1774060203,61048.00000,0.00026307 +1774060210,61042.60000,0.00010000 +1774060215,61061.90000,0.00066845 +1774060219,61044.90000,0.00053665 +1774060228,61045.90000,0.00016060 +1774060254,61047.40000,0.00014833 +1774060261,61045.40000,0.00034900 +1774060262,61046.10000,0.00050870 +1774060270,61046.10000,0.00010000 +1774060278,61045.30000,0.00469873 +1774060312,61041.20000,0.00015945 +1774060326,61040.00000,0.00010000 +1774060342,61025.10000,0.00006686 +1774060353,61023.00000,0.00024099 +1774060386,61003.30000,0.00013165 +1774060386,61003.30000,0.00292532 +1774060389,61003.30000,0.00016150 +1774060391,61003.30000,0.00001615 +1774060391,61003.30000,0.00057133 +1774060396,61003.70000,0.00015999 +1774060397,61003.40000,0.00010000 +1774060402,61003.20000,0.00086232 +1774060403,61003.30000,0.00049178 +1774060406,61003.20000,0.00037001 +1774060408,61004.40000,0.00041074 +1774060430,61014.40000,0.00116744 +1774060442,61015.50000,0.00819421 +1774060444,61027.10000,0.00080720 +1774060469,61030.60000,0.00010000 +1774060511,61039.10000,0.00005000 +1774060520,61036.90000,0.00063854 +1774060523,61038.70000,0.00010000 +1774060574,61039.50000,0.00001626 +1774060576,61034.80000,0.00010000 +1774060623,61045.50000,0.00003558 +1774060635,61043.30000,0.00010000 +1774060661,61028.20000,0.00508900 +1774060687,61047.60000,0.00010000 +1774060727,61056.00000,0.00046564 +1774060778,61065.80000,0.00010000 +1774060796,61068.10000,0.00086176 +1774060814,61109.90000,0.01209700 +1774060815,61132.90000,0.00607358 +1774060815,61139.60000,0.00474177 +1774060816,61142.70000,0.00007369 +1774060818,61138.20000,0.00014777 +1774060819,61129.80000,0.00035786 +1774060830,61134.10000,0.00010000 +1774060848,61132.70000,0.00008485 +1774060878,61114.60000,0.00010000 +1774060884,61117.80000,0.00007729 +1774060886,61116.30000,0.00047826 +1774060888,61116.20000,0.00023941 +1774060913,61120.20000,0.00015467 +1774060921,61115.70000,0.00013141 +1774060928,61117.40000,0.00021874 +1774060952,61115.50000,0.00016477 +1774060981,61112.70000,0.00788493 +1774060986,61117.90000,0.00040787 +1774060991,61122.50000,0.00010000 +1774061007,61114.30000,0.00001701 +1774061015,61117.80000,0.00635440 +1774061016,61116.60000,0.00040792 +1774061049,61117.40000,0.00010000 +1774061081,61109.20000,0.00128847 +1774061083,61114.00000,0.00076431 +1774061086,61115.30000,0.00061350 +1774061092,61112.40000,0.00001684 +1774061103,61116.60000,0.00017181 +1774061104,61117.20000,0.00015999 +1774061115,61091.90000,0.00036902 +1774061148,61101.50000,0.00010000 +1774061152,61096.60000,0.00310066 +1774061171,61099.90000,0.00011576 +1774061177,61098.60000,0.00040070 +1774061201,61117.50000,0.00036895 +1774061220,61116.50000,0.00001612 +1774061245,61126.60000,0.00015693 +1774061253,61128.70000,0.00014142 +1774061253,61128.80000,0.00022674 +1774061289,61135.70000,0.00036813 +1774061302,61133.60000,0.00010000 +1774061316,61137.80000,0.00045451 +1774061330,61138.20000,0.00003239 +1774061346,61114.50000,0.00003224 +1774061358,61111.90000,0.00028900 +1774061398,61135.00000,0.00769274 +1774061401,61138.00000,0.00100833 +1774061455,61142.70000,0.00147426 +1774061472,61146.20000,0.00010000 +1774061506,61141.00000,0.00013275 +1774061510,61141.60000,0.00313404 +1774061517,61145.50000,0.00036807 +1774061538,61145.90000,0.00079920 +1774061540,61148.90000,0.01241250 +1774061541,61151.50000,0.00041344 +1774061553,61145.70000,0.00036807 +1774061573,61148.10000,0.00036805 +1774061574,61145.80000,0.00106679 +1774061609,61162.80000,0.00034388 +1774061624,61169.40000,0.00014872 +1774061628,61168.80000,0.00033353 +1774061647,61170.30000,0.00015091 +1774061690,61179.70000,0.00040749 +1774061696,61168.60000,0.00686297 +1774061736,61138.00000,0.00006804 +1774061736,61133.30000,0.00033945 +1774061782,61134.00000,0.00040778 +1774061787,61132.90000,0.01337524 +1774061808,61140.60000,0.00040778 +1774061835,61112.10000,0.00014145 +1774061835,61107.60000,0.00067873 +1774061837,61112.20000,0.00040794 +1774061839,61113.20000,0.00014143 +1774061839,61113.30000,0.00022688 +1774061865,61124.90000,0.00197090 +1774061879,61123.30000,0.00032723 +1774061896,61113.20000,0.00008071 +1774061924,61133.50000,0.00014504 +1774061930,61126.30000,0.00037046 +1774061940,61132.50000,0.00001612 +1774062000,61131.40000,0.00008058 +1774062000,61131.40000,0.00080187 +1774062000,61136.00000,0.00320716 +1774062000,61136.00000,0.00001612 +1774062000,61136.00000,0.00001612 +1774062000,61136.00000,0.00008177 +1774062000,61136.00000,0.00080975 +1774062000,61136.00000,0.00016036 +1774062000,61136.00000,0.00161152 +1774062000,61136.00000,0.00001604 +1774062000,61136.00000,0.00107972 +1774062001,61136.00000,0.00027154 +1774062002,61152.50000,0.00015026 +1774062002,61151.60000,0.00016359 +1774062008,61152.60000,0.00016271 +1774062010,61153.40000,0.00016415 +1774062011,61153.40000,0.00009521 +1774062011,61153.40000,0.00019699 +1774062011,61153.40000,0.00049663 +1774062011,61153.40000,0.00009849 +1774062011,61153.40000,0.00016416 +1774062011,61153.40000,0.00016416 +1774062011,61153.40000,0.00013132 +1774062011,61153.40000,0.00013132 +1774062011,61153.40000,0.00009849 +1774062012,61153.40000,0.00026265 +1774062012,61153.40000,0.00013132 +1774062012,61153.40000,0.00026265 +1774062012,61153.40000,0.00009849 +1774062013,61153.40000,0.00023442 +1774062014,61151.60000,0.00809545 +1774062016,61151.60000,0.00016002 +1774062025,61153.30000,0.00117851 +1774062026,61153.30000,0.00046940 +1774062027,61152.00000,0.00016352 +1774062030,61151.60000,0.00004857 +1774062052,61151.60000,0.00040477 +1774062058,61151.50000,0.00179881 +1774062059,61129.30000,0.00012088 +1774062130,61115.70000,0.00040792 +1774062181,61103.50000,0.00064172 +1774062185,61107.00000,0.00030633 +1774062234,61089.90000,0.00138861 +1774062305,61091.70000,0.00096762 +1774062337,61104.50000,0.00008102 +1774062339,61104.20000,0.00326310 +1774062345,61104.10000,0.00037750 +1774062381,61116.70000,0.00188167 +1774062387,61104.80000,0.00040792 +1774062401,61114.10000,0.00100340 +1774062441,61140.20000,0.00029686 +1774062441,61140.50000,0.00014144 +1774062441,61140.60000,0.00131861 +1774062470,61136.00000,0.00003207 +1774062487,61137.50000,0.00794011 +1774062495,61137.40000,0.00039912 +1774062509,61154.80000,0.00005000 +1774062536,61137.20000,0.00001550 +1774062553,61144.70000,0.00039908 +1774062581,61141.40000,0.00033897 +1774062582,61147.60000,0.00039617 +1774062590,61147.10000,0.00040461 +1774062644,61150.40000,0.00327562 +1774062674,61173.60000,0.00007058 +1774062686,61177.20000,0.00003099 +1774062709,61175.80000,0.00024398 +1774062711,61177.20000,0.00025610 +1774062729,61162.60000,0.01635084 +1774062732,61141.90000,0.00473035 +1774062761,61113.20000,0.00010000 +1774062763,61113.20000,0.00016201 +1774062765,61108.60000,0.00055490 +1774062799,61084.60000,0.00159898 +1774062801,61085.80000,0.00102668 +1774062861,61102.70000,0.00128829 +1774062889,61101.60000,0.00053500 +1774062900,61104.60000,0.00019714 +1774062902,61105.20000,0.00016365 +1774062906,61106.90000,0.00080219 +1774062919,61104.20000,0.00036467 +1774062930,61103.00000,0.00004885 +1774062969,61078.40000,0.00030888 +1774062971,61078.40000,0.00030888 +1774062971,61078.00000,0.00030888 +1774062974,61078.60000,0.00030888 +1774062975,61078.20000,0.00030888 +1774062977,61077.10000,0.00030887 +1774062978,61076.10000,0.00030889 +1774062980,61075.90000,0.00038302 +1774062982,61071.60000,0.00061783 +1774062986,61080.20000,0.00061774 +1774062999,61094.00000,0.00155235 +1774063007,61089.00000,0.00033232 +1774063020,61095.00000,0.00001613 +1774063026,61084.50000,0.00030885 +1774063027,61084.50000,0.00037837 +1774063028,61084.50000,0.00030885 +1774063031,61084.50000,0.00061770 +1774063037,61086.60000,0.00014151 +1774063037,61080.40000,0.00387312 +1774063055,61099.30000,0.00473562 +1774063073,61096.50000,0.00152161 +1774063078,61094.80000,0.00042899 +1774063080,61092.10000,0.00049596 +1774063086,61098.70000,0.00016003 +1774063105,61092.80000,0.00005000 +1774063141,61096.70000,0.00069889 +1774063144,61097.50000,0.00003209 +1774063161,61106.50000,0.00042899 +1774063167,61105.70000,0.00071497 +1774063185,61085.10000,0.00061985 +1774063200,61087.30000,0.00008025 +1774063206,61082.70000,0.00049737 +1774063206,61082.70000,0.00013147 +1774063207,61083.20000,0.00016003 +1774063218,61078.80000,0.00110753 +1774063248,61074.10000,0.00057434 +1774063295,61099.30000,0.00005000 +1774063312,61099.00000,0.00049002 +1774063321,61094.20000,0.00008074 +1774063327,61093.20000,0.00042885 +1774063401,61102.20000,0.00723045 +1774063532,61092.20000,0.00016369 +1774063539,61073.20000,0.00023472 +1774063590,61101.80000,0.00008102 +1774063596,61101.80000,0.00042863 +1774063642,61101.80000,0.00087618 +1774063647,61101.80000,0.00002161 +1774063741,61101.80000,0.00042847 +1774063792,61101.70000,0.00039783 +1774063800,61101.80000,0.00040712 +1774063800,61096.60000,0.16611649 +1774063802,61094.80000,0.00009201 +1774063802,61094.80000,0.00009859 +1774063803,61094.80000,0.00031688 +1774063820,61083.00000,0.00038917 +1774063855,61086.60000,0.00003210 +1774063883,61076.70000,0.00010000 +1774063905,61069.70000,0.00106199 +1774063910,61071.40000,0.00050216 +1774063931,61075.80000,0.00010000 +1774063949,61067.60000,0.00121261 +1774063950,61073.10000,0.00080787 +1774063951,61067.50000,0.00040312 +1774063952,61067.50000,0.00010361 +1774063954,61065.10000,0.00016028 +1774063954,61062.10000,0.00040312 +1774063955,61062.10000,0.00032218 +1774063957,61060.60000,0.00161736 +1774063958,61060.60000,0.00161736 +1774063959,61060.60000,0.00016028 +1774063960,61060.60000,0.00020075 +1774063964,61060.60000,0.00080787 +1774063965,61060.60000,0.00080793 +1774063967,61060.60000,0.00080793 +1774063968,61060.60000,0.00040316 +1774063969,61060.60000,0.00040316 +1774063971,61060.60000,0.00040316 +1774063972,61060.60000,0.00080793 +1774063974,61058.80000,0.00080793 +1774063975,61058.80000,0.00048411 +1774063977,61058.80000,0.00024125 +1774063978,61058.80000,0.00040317 +1774063979,61058.80000,0.00129370 +1774063980,61058.80000,0.00010746 +1774063980,61058.80000,0.00080795 +1774063981,61058.80000,0.00080795 +1774063982,61056.50000,0.00249450 +1774063982,61058.80000,0.00161753 +1774063983,61058.80000,0.00032221 +1774063984,61058.80000,0.00161753 +1774063985,61058.80000,0.00040317 +1774063986,61058.80000,0.00024125 +1774063987,61058.80000,0.00161753 +1774063988,61058.80000,0.00080795 +1774063988,61058.80000,0.00161753 +1774063989,61058.80000,0.00080795 +1774063992,61058.80000,0.00161753 +1774063992,61058.80000,0.00080795 +1774063993,61058.80000,0.00040317 +1774063994,61076.90000,0.00040317 +1774063995,61072.00000,0.00024125 +1774064011,61079.70000,0.00042867 +1774064014,61074.60000,0.00010000 +1774064050,61060.30000,0.00010000 +1774064074,61065.90000,0.00046661 +1774064124,61050.10000,0.00032257 +1774064132,61050.30000,0.00022612 +1774064155,61050.00000,0.00000607 +1774064155,61050.00000,0.00092332 +1774064155,61049.80000,0.00009859 +1774064183,61045.50000,0.00010000 +1774064193,61053.80000,0.00031056 +1774064200,61050.90000,0.00968064 +1774064200,61050.80000,0.00023816 +1774064208,61051.10000,0.00057452 +1774064234,61060.40000,0.00014156 +1774064234,61060.40000,0.00131838 +1774064252,61055.10000,0.00010000 +1774064296,61048.60000,0.00010387 +1774064321,61047.30000,0.00602784 +1774064326,61043.20000,0.15000000 +1774064347,61044.20000,0.00026357 +1774064410,61051.40000,0.00011627 +1774064420,61050.60000,0.00014369 +1774064463,61057.80000,0.00093037 +1774064467,61063.50000,0.00012574 +1774064494,61063.00000,0.00003180 +1774064524,61067.20000,0.00318977 +1774064525,61067.20000,0.00010000 +1774064598,61057.60000,0.00010000 +1774064612,61060.30000,0.01550000 +1774064612,61061.20000,0.05372800 +1774064643,61064.70000,0.00019728 +1774064644,61063.90000,0.00029321 +1774064647,61059.20000,0.00060602 +1774064703,61055.80000,0.00009865 +1774064708,61059.40000,0.00020862 +1774064739,61045.50000,0.01550000 +1774064739,61045.40000,0.04991085 +1774064748,61043.30000,0.00036823 +1774064760,61046.90000,0.00032437 +1774064764,61051.30000,0.00010000 +1774064783,61052.60000,0.00347290 +1774064795,61065.20000,0.00013230 +1774064804,61062.70000,0.00067060 +1774064825,61059.80000,0.00010000 +1774064834,61059.70000,0.00014157 +1774064834,61059.10000,0.00009255 +1774064856,61059.90000,0.04786331 +1774064863,61060.10000,0.00015900 +1774064898,61063.50000,0.00010000 +1774064912,61063.50000,0.03138530 +1774064921,61062.20000,0.00821015 +1774064946,61061.40000,0.00016568 +1774064946,61061.40000,0.00010000 +1774064972,61063.60000,0.00008188 +1774065005,61047.90000,0.00010000 +1774065033,61045.80000,0.00016060 +1774065066,61061.10000,0.00010000 +1774065077,61080.50000,0.00036274 +1774065077,61092.30000,0.00050654 +1774065107,61082.30000,0.02288700 +1774065108,61084.10000,0.00054024 +1774065124,61076.60000,0.00010000 +1774065128,61076.60000,0.01316750 +1774065135,61067.40000,0.00204973 +1774065191,61071.80000,0.00015116 +1774065218,61065.40000,0.00372131 +1774065221,61063.80000,0.00706130 +1774065243,61070.00000,0.00311209 +1774065245,61074.90000,0.00010000 +1774065262,61077.20000,0.01552236 +1774065300,61073.00000,0.00008147 +1774065314,61081.50000,0.00018352 +1774065358,61076.90000,0.01506150 +1774065360,61078.50000,0.00031791 +1774065385,61082.20000,0.00010000 +1774065417,61087.00000,0.00029999 +1774065423,61087.00000,0.00010000 +1774065458,61087.00000,0.00015187 +1774065500,61087.00000,0.00010000 +1774065534,61087.00000,0.00093640 +1774065544,61087.00000,0.00010000 +1774065594,61087.00000,0.00007985 +1774065596,61087.00000,0.00637279 +1774065600,61087.00000,0.00080246 +1774065600,61087.00000,0.00001613 +1774065600,61087.00000,0.00012903 +1774065600,61087.00000,0.00029031 +1774065600,61087.00000,0.00016049 +1774065600,61087.00000,0.00040722 +1774065600,61087.00000,0.00001613 +1774065600,61087.00000,0.00016208 +1774065600,61087.00000,0.00008144 +1774065600,61087.00000,0.00001613 +1774065600,61087.00000,0.00016128 +1774065600,61087.00000,0.00012746 +1774065604,61087.00000,0.00084547 +1774065609,61087.00000,0.00081443 +1774065610,61087.00000,0.00008217 +1774065610,61087.00000,0.00009860 +1774065610,61087.00000,0.00019720 +1774065610,61087.00000,0.00019720 +1774065610,61087.00000,0.00016301 +1774065610,61087.00000,0.00008545 +1774065611,61087.00000,0.00008217 +1774065611,61087.00000,0.00016150 +1774065611,61087.00000,0.00016868 +1774065611,61087.00000,0.00016868 +1774065611,61087.00000,0.00024376 +1774065611,61087.00000,0.00008874 +1774065611,61087.00000,0.00018182 +1774065611,61087.00000,0.00016868 +1774065616,61087.00000,0.00016000 +1774065616,61087.00000,0.00016000 +1774065616,61087.00000,0.00016000 +1774065616,61087.00000,0.00016000 +1774065617,61087.00000,0.00016000 +1774065617,61087.00000,0.00016000 +1774065618,61087.00000,0.00032251 +1774065619,61087.00000,0.00032251 +1774065620,61087.00000,0.00019171 +1774065629,61100.30000,0.00010372 +1774065637,61101.80000,0.00016045 +1774065639,61101.80000,0.00016301 +1774065643,61101.80000,0.00000299 +1774065643,61111.50000,0.00004586 +1774065646,61114.80000,0.00010000 +1774065652,61107.50000,0.00004071 +1774065683,61117.10000,0.00016211 +1774065688,61116.50000,0.00024243 +1774065688,61102.70000,0.00055757 +1774065694,61103.70000,0.00005000 +1774065700,61112.40000,0.00014780 +1774065709,61106.50000,0.00010000 +1774065757,61135.00000,0.01477774 +1774065757,61136.20000,0.01088047 +1774065780,61114.20000,0.00016042 +1774065784,61112.40000,0.00010000 +1774065808,61114.10000,0.00025476 +1774065868,61128.20000,0.00033877 +1774065875,61135.00000,0.00010000 +1774065902,61143.00000,0.00477521 +1774065905,61143.00000,0.00080567 +1774065905,61143.00000,0.00064454 +1774065911,61143.00000,0.00798876 +1774066028,61167.50000,0.00014133 +1774066028,61167.60000,0.00461405 +1774066110,61146.50000,0.00049070 +1774066110,61146.40000,0.00045436 +1774066120,61146.20000,0.00245314 +1774066158,61120.80000,0.00060895 +1774066161,61120.80000,0.02630890 +1774066227,61140.80000,0.00013444 +1774066270,61141.10000,0.00008138 +1774066335,61156.60000,0.00036799 +1774066440,61153.40000,0.00076744 +1774066455,61141.30000,0.00081393 +1774066494,61142.00000,0.00059960 +1774066495,61142.00000,0.21203009 +1774066495,61144.60000,0.25444800 +1774066495,61144.60000,0.02000000 +1774066502,61150.60000,0.00009850 +1774066503,61150.20000,0.00013133 +1774066503,61150.20000,0.00008000 +1774066521,61142.00000,0.00370652 +1774066561,61146.60000,0.00130026 +1774066585,61149.30000,0.00127595 +1774066586,61149.70000,0.00032223 +1774066620,61151.40000,0.00001611 +1774066631,61153.70000,0.00326797 +1774066645,61152.40000,0.00168023 +1774066664,61165.70000,0.00005000 +1774066706,61153.20000,0.00044807 +1774066766,61179.60000,0.00234990 +1774066772,61179.50000,0.00040863 +1774066789,61175.00000,0.00010000 +1774066789,61175.00000,0.55975148 +1774066791,61175.00000,0.00672925 +1774066791,61175.00000,0.00672925 +1774066791,61175.00000,0.00332841 +1774066791,61175.00000,0.00332841 +1774066795,61175.00000,0.00333184 +1774066795,61175.00000,0.00333184 +1774066797,61175.00000,0.06569000 +1774066797,61175.00000,0.00000673 +1774066804,61135.60000,0.00242927 +1774066804,61139.90000,0.02023794 +1774066804,61141.90000,0.32710792 +1774066819,61144.40000,0.00006803 +1774066819,61122.20000,0.00048793 +1774066819,61130.70000,0.00009625 +1774066819,61130.30000,0.00960527 +1774066828,61130.30000,0.00213550 +1774066838,61130.20000,0.00119019 +1774066841,61130.10000,0.00167588 +1774066843,61130.30000,0.00007849 +1774066845,61123.60000,0.00250308 +1774066886,61138.70000,0.00010000 +1774066906,61137.10000,0.00003312 +1774066920,61137.10000,0.00001604 +1774067031,61172.80000,0.00037194 +1774067035,61180.80000,0.00032049 +1774067049,61177.30000,0.00010000 +1774067136,61184.80000,0.00010000 +1774067160,61190.20000,0.00639873 +1774067184,61193.10000,0.00139495 +1774067184,61194.10000,0.13559505 +1774067186,61181.30000,0.00300996 +1774067189,61180.20000,0.00010000 +1774067190,61180.20000,0.02235360 +1774067221,61166.30000,0.00159101 +1774067237,61180.10000,0.00014129 +1774067237,61174.40000,0.00158466 +1774067277,61196.40000,0.00010000 +1774067280,61193.30000,0.00001610 +1774067309,61197.50000,0.00081708 +1774067319,61187.70000,0.00010000 +1774067349,61186.00000,0.00347646 +1774067351,61184.50000,0.00011059 +1774067353,61181.80000,0.00061100 +1774067395,61172.00000,0.00033348 +1774067400,61147.80000,0.00016272 +1774067401,61143.90000,0.00030229 +1774067403,61124.30000,0.00013134 +1774067403,61134.20000,0.00036538 +1774067404,61130.20000,0.00033255 +1774067404,61123.50000,0.00016008 +1774067404,61123.50000,0.00016007 +1774067412,61127.70000,0.01603843 +1774067427,61125.60000,0.00021276 +1774067449,61119.60000,0.00080997 +1774067467,61110.30000,0.00015106 +1774067500,61110.20000,0.00014148 +1774067500,61110.20000,0.00014148 +1774067504,61110.20000,0.00004431 +1774067520,61081.00000,0.00001613 +1774067556,61090.40000,0.00001613 +1774067601,61087.30000,0.00014930 +1774067656,61081.00000,0.00008025 +1774067700,61081.00000,0.00040324 +1774067701,61081.00000,0.00005000 +1774067760,61080.90000,0.00009823 +1774067760,61060.20000,0.00267206 +1774067760,61060.00000,0.05576580 +1774067763,61066.20000,0.00010000 +1774067773,61061.60000,0.00036866 +1774067802,61061.50000,0.00025556 +1774067808,61061.30000,0.00015118 +1774067842,61053.00000,0.00014157 +1774067842,61053.00000,0.00009332 +1774067844,61053.00000,0.00015675 +1774067847,61053.50000,0.00498917 +1774067861,61058.50000,0.00014157 +1774067861,61058.50000,0.00649463 +1774067881,61056.80000,0.00147188 +1774067914,61072.80000,0.00410310 +1774067914,61072.90000,0.00444615 +1774067937,61081.50000,0.00016051 +1774067949,61081.10000,0.00116718 +1774067985,61073.00000,0.00008494 +1774067996,61073.00000,0.00115947 +1774067996,61073.00000,0.02107735 +1774067996,61073.00000,0.00302318 +1774067999,61073.00000,0.00047932 +1774067999,61072.90000,0.00005563 +1774068000,61073.00000,0.00017337 +1774068002,61072.90000,0.00023162 +1774068027,61072.90000,0.00071155 +1774068027,61055.10000,0.06647903 +1774068109,61063.60000,0.00402424 +1774068113,61063.80000,0.00080485 +1774068129,61065.60000,0.00032109 +1774068138,61060.10000,0.00016132 +1774068181,61060.10000,0.01307698 +1774068199,61060.00000,0.00005733 +1774068248,61055.00000,0.00010000 +1774068283,61070.00000,0.00005000 +1774068288,61066.50000,0.00033254 +1774068306,61060.40000,0.00010000 +1774068352,61067.60000,0.00158982 +1774068364,61059.90000,0.00010000 +1774068369,61060.30000,0.00023161 +1774068424,61064.60000,0.00010000 +1774068506,61068.70000,0.00065786 +1774068508,61068.80000,0.00081064 +1774068511,61065.80000,0.00010000 +1774068524,61068.40000,0.02634990 +1774068545,61068.80000,0.00010000 +1774068546,61071.30000,0.00054952 +1774068616,61076.70000,0.00004839 +1774068618,61070.40000,0.00010000 +1774068678,61053.30000,0.00006455 +1774068720,61053.30000,0.00026901 +1774068731,61053.30000,0.00077248 +1774068736,61053.20000,0.00010000 +1774068755,61054.20000,0.00154154 +1774068791,61053.20000,0.00010000 +1774068801,61053.30000,0.00036927 +1774068857,61053.20000,0.00039888 +1774068858,61053.20000,0.00010000 +1774068906,61053.70000,0.00033012 +1774068971,61059.10000,0.03244264 +1774068978,61053.30000,0.00037078 +1774068987,61053.20000,0.00010000 +1774069037,61062.20000,0.00014157 +1774069037,61064.60000,0.00002724 +1774069042,61062.20000,0.00014157 +1774069042,61064.60000,0.00001843 +1774069101,61060.10000,0.00193802 +1774069113,61054.90000,0.00337396 +1774069140,61060.00000,0.00004817 +1774069141,61054.80000,0.00058209 +1774069160,61053.70000,0.00010000