From 1e56097505af170b2edf3145146e10be8ee97d83 Mon Sep 17 00:00:00 2001 From: rob Date: Thu, 25 Apr 2024 13:12:21 +0200 Subject: [PATCH 001/103] Add event server --- src/event_publisher.rs | 35 +++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/subcommand.rs | 12 ++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 src/event_publisher.rs diff --git a/src/event_publisher.rs b/src/event_publisher.rs new file mode 100644 index 0000000000..2e7bd32a6b --- /dev/null +++ b/src/event_publisher.rs @@ -0,0 +1,35 @@ +use std::sync::Arc; + +use tokio::runtime::Runtime; +use tokio::sync::mpsc; + +use crate::index::event::Event; + +pub struct EventPublisher { + pub event_sender: mpsc::Sender, + #[allow(dead_code)] + runtime: Arc, +} + +impl EventPublisher { + pub fn new() -> Self { + let (event_sender, mut event_receiver) = mpsc::channel::(128); + let runtime = Arc::new(Runtime::new().unwrap()); + + let rt_clone = runtime.clone(); + rt_clone.spawn(async move { + while let Some(event) = event_receiver.recv().await { + EventPublisher::handle_event(event); + } + }); + + EventPublisher { + event_sender, + runtime, + } + } + + fn handle_event(event: Event) { + println!("Received event: {:?}", event); + } +} diff --git a/src/lib.rs b/src/lib.rs index 372f7eef2f..99f53249eb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -120,6 +120,7 @@ pub mod subcommand; mod tally; pub mod templates; pub mod wallet; +mod event_publisher; type Result = std::result::Result; diff --git a/src/subcommand.rs b/src/subcommand.rs index 2da87735db..18aa83f6f2 100644 --- a/src/subcommand.rs +++ b/src/subcommand.rs @@ -1,3 +1,5 @@ +use crate::event_publisher::EventPublisher; + use super::*; pub mod balances; @@ -39,6 +41,8 @@ pub(crate) enum Subcommand { Runes, #[command(about = "Run the explorer server")] Server(server::Server), + #[command(about = "Run the explorer server in event emit mode")] + EventServer(server::Server), #[command(about = "Display settings")] Settings, #[command(about = "Display information about a block's subsidy")] @@ -71,6 +75,14 @@ impl Subcommand { LISTENERS.lock().unwrap().push(handle.clone()); server.run(settings, index, handle) } + Self::EventServer(server) => { + let event_publisher = EventPublisher::new(); + let event_sender = event_publisher.event_sender.clone(); + let index = Arc::new(Index::open_with_event_sender(&settings, Some(event_sender))?); + let handle = axum_server::Handle::new(); + LISTENERS.lock().unwrap().push(handle.clone()); + server.run(settings, index, handle) + } Self::Settings => settings::run(settings), Self::Subsidy(subsidy) => subsidy.run(), Self::Supply => supply::run(), From c2443a9662506ecffa7be5783fe07046ee665959 Mon Sep 17 00:00:00 2001 From: rob Date: Thu, 25 Apr 2024 15:44:54 +0200 Subject: [PATCH 002/103] Publish events to rmq --- Cargo.lock | 4106 ---------------------------------------- Cargo.toml | 2 + src/event_publisher.rs | 60 +- src/index/event.rs | 2 +- src/options.rs | 8 + src/settings.rs | 58 +- src/subcommand.rs | 5 +- 7 files changed, 110 insertions(+), 4131 deletions(-) delete mode 100644 Cargo.lock diff --git a/Cargo.lock b/Cargo.lock deleted file mode 100644 index 4db78ae39c..0000000000 --- a/Cargo.lock +++ /dev/null @@ -1,4106 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "addr2line" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - -[[package]] -name = "aho-corasick" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] - -[[package]] -name = "alloc-no-stdlib" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" - -[[package]] -name = "alloc-stdlib" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" -dependencies = [ - "alloc-no-stdlib", -] - -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "anes" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" - -[[package]] -name = "anstream" -version = "0.6.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" - -[[package]] -name = "anstyle-parse" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" -dependencies = [ - "windows-sys 0.52.0", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" -dependencies = [ - "anstyle", - "windows-sys 0.52.0", -] - -[[package]] -name = "anyhow" -version = "1.0.82" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" -dependencies = [ - "backtrace", -] - -[[package]] -name = "arc-swap" -version = "1.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" - -[[package]] -name = "array-init" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23589ecb866b460d3a0f1278834750268c607e8e28a1b982c907219f3178cd72" -dependencies = [ - "nodrop", -] - -[[package]] -name = "asn1-rs" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ff05a702273012438132f449575dbc804e27b2f3cbe3069aa237d26c98fa33" -dependencies = [ - "asn1-rs-derive", - "asn1-rs-impl", - "displaydoc", - "nom", - "num-traits", - "rusticata-macros", - "thiserror", - "time", -] - -[[package]] -name = "asn1-rs-derive" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db8b7511298d5b7784b40b092d9e9dcd3a627a5707e4b5e507931ab0d44eeebf" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", - "synstructure", -] - -[[package]] -name = "asn1-rs-impl" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "async-channel" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136d4d23bcc79e27423727b36823d86233aad06dfea531837b038394d11e9928" -dependencies = [ - "concurrent-queue", - "event-listener 5.3.0", - "event-listener-strategy 0.5.1", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-compression" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07dbbf24db18d609b1462965249abdf49129ccad073ec257da372adc83259c60" -dependencies = [ - "brotli 4.0.0", - "flate2", - "futures-core", - "memchr", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "async-http-codec" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "096146020b08dbc4587685b0730a7ba905625af13c65f8028035cdfd69573c91" -dependencies = [ - "anyhow", - "futures", - "http 1.1.0", - "httparse", - "log", -] - -[[package]] -name = "async-io" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" -dependencies = [ - "async-lock 2.8.0", - "autocfg", - "cfg-if 1.0.0", - "concurrent-queue", - "futures-lite 1.13.0", - "log", - "parking", - "polling", - "rustix 0.37.27", - "slab", - "socket2 0.4.10", - "waker-fn", -] - -[[package]] -name = "async-lock" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" -dependencies = [ - "event-listener 2.5.3", -] - -[[package]] -name = "async-lock" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" -dependencies = [ - "event-listener 4.0.3", - "event-listener-strategy 0.4.0", - "pin-project-lite", -] - -[[package]] -name = "async-net" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0434b1ed18ce1cf5769b8ac540e33f01fa9471058b5e89da9e06f3c882a8c12f" -dependencies = [ - "async-io", - "blocking", - "futures-lite 1.13.0", -] - -[[package]] -name = "async-task" -version = "4.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799" - -[[package]] -name = "async-trait" -version = "0.1.80" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.60", -] - -[[package]] -name = "async-web-client" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e05ea882322ee3a2c62d3a2d68d8596fd8594fd1d6e93919f6f620eb8b0bf46" -dependencies = [ - "async-http-codec", - "async-net", - "async-ws", - "futures", - "futures-rustls", - "gloo-net", - "http 1.1.0", - "js-sys", - "lazy_static", - "log", - "rustls 0.22.4", - "rustls-pki-types", - "thiserror", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "webpki-roots", -] - -[[package]] -name = "async-ws" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70b425fc1a1d5a3988aa2e1c94c4823740416406f3416322c9dcb542fb997d37" -dependencies = [ - "async-io", - "base64 0.13.1", - "futures", - "futures-lite 1.13.0", - "generic_static", - "http 1.1.0", - "log", - "rand", - "ring 0.16.20", - "strum", - "thiserror", - "utf-8", -] - -[[package]] -name = "atom_syndication" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "571832dcff775e26562e8e6930cd483de5587301d40d3a3b85d532b6383e15a7" -dependencies = [ - "chrono", - "derive_builder", - "diligent-date-parser", - "never", - "quick-xml", -] - -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - -[[package]] -name = "audit-cache" -version = "0.0.0" -dependencies = [ - "colored", - "reqwest", -] - -[[package]] -name = "audit-content-security-policy" -version = "0.0.0" -dependencies = [ - "colored", - "reqwest", -] - -[[package]] -name = "autocfg" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" - -[[package]] -name = "axum" -version = "0.6.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" -dependencies = [ - "async-trait", - "axum-core", - "bitflags 1.3.2", - "bytes", - "futures-util", - "http 0.2.12", - "http-body", - "hyper", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http 0.2.12", - "http-body", - "mime", - "rustversion", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-server" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447f28c85900215cc1bea282f32d4a2f22d55c5a300afdfbc661c8d6a632e063" -dependencies = [ - "arc-swap", - "bytes", - "futures-util", - "http 0.2.12", - "http-body", - "hyper", - "pin-project-lite", - "rustls 0.21.11", - "rustls-pemfile", - "tokio", - "tokio-rustls", - "tower-service", -] - -[[package]] -name = "backtrace" -version = "0.3.71" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" -dependencies = [ - "addr2line", - "cc", - "cfg-if 1.0.0", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", -] - -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - -[[package]] -name = "base64" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" - -[[package]] -name = "bech32" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" - -[[package]] -name = "bech32" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d" - -[[package]] -name = "bip39" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f2635620bf0b9d4576eb7bb9a38a55df78bd1205d26fa994b25911a69f212f" -dependencies = [ - "bitcoin_hashes 0.11.0", - "serde", - "unicode-normalization", -] - -[[package]] -name = "bitcoin" -version = "0.30.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1945a5048598e4189e239d3f809b19bdad4845c4b2ba400d304d2dcf26d2c462" -dependencies = [ - "bech32 0.9.1", - "bitcoin-private", - "bitcoin_hashes 0.12.0", - "hex_lit", - "secp256k1", - "serde", -] - -[[package]] -name = "bitcoin-private" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73290177011694f38ec25e165d0387ab7ea749a4b81cd4c80dae5988229f7a57" - -[[package]] -name = "bitcoin_hashes" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90064b8dee6815a6470d60bad07bbbaee885c0e12d04177138fa3291a01b7bc4" - -[[package]] -name = "bitcoin_hashes" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d7066118b13d4b20b23645932dfb3a81ce7e29f95726c2036fa33cd7b092501" -dependencies = [ - "bitcoin-private", - "serde", -] - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "blocking" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" -dependencies = [ - "async-channel", - "async-lock 3.3.0", - "async-task", - "fastrand 2.0.2", - "futures-io", - "futures-lite 2.3.0", - "piper", - "tracing", -] - -[[package]] -name = "boilerplate" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1906889b1f805a715eac02b2dea416e25c5cfa00f099530fa9d137a3cff93113" -dependencies = [ - "darling 0.20.8", - "mime", - "new_mime_guess", - "proc-macro2", - "quote", - "syn 2.0.60", -] - -[[package]] -name = "brotli" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "125740193d7fee5cc63ab9e16c2fdc4e07c74ba755cc53b327d6ea029e9fc569" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor 3.0.0", -] - -[[package]] -name = "brotli" -version = "5.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19483b140a7ac7174d34b5a581b406c64f84da5409d3e09cf4fff604f9270e67" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor 4.0.0", -] - -[[package]] -name = "brotli-decompressor" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65622a320492e09b5e0ac436b14c54ff68199bac392d0e89a6832c4518eea525" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", -] - -[[package]] -name = "brotli-decompressor" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6221fe77a248b9117d431ad93761222e1cf8ff282d9d1d5d9f53d6299a1cf76" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", -] - -[[package]] -name = "bstr" -version = "1.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" -dependencies = [ - "memchr", - "serde", -] - -[[package]] -name = "bumpalo" -version = "3.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "bytes" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" - -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - -[[package]] -name = "cc" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6e324229dc011159fcc089755d1e2e216a90d43a7dea6853ca740b84f35e7" - -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "cfg_aliases" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" - -[[package]] -name = "chrono" -version = "0.4.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" -dependencies = [ - "android-tzdata", - "iana-time-zone", - "js-sys", - "num-traits", - "serde", - "wasm-bindgen", - "windows-targets 0.52.5", -] - -[[package]] -name = "ciborium" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" -dependencies = [ - "ciborium-io", - "ciborium-ll", - "serde", -] - -[[package]] -name = "ciborium-io" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" - -[[package]] -name = "ciborium-ll" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" -dependencies = [ - "ciborium-io", - "half", -] - -[[package]] -name = "clap" -version = "4.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim 0.11.1", -] - -[[package]] -name = "clap_derive" -version = "4.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.60", -] - -[[package]] -name = "clap_lex" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" - -[[package]] -name = "colorchoice" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" - -[[package]] -name = "colored" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" -dependencies = [ - "lazy_static", - "windows-sys 0.48.0", -] - -[[package]] -name = "concurrent-queue" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "console" -version = "0.15.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" -dependencies = [ - "encode_unicode", - "lazy_static", - "libc", - "unicode-width", - "windows-sys 0.52.0", -] - -[[package]] -name = "convert_case" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" - -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" - -[[package]] -name = "cpufeatures" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" -dependencies = [ - "libc", -] - -[[package]] -name = "crc32fast" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" -dependencies = [ - "cfg-if 1.0.0", -] - -[[package]] -name = "criterion" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" -dependencies = [ - "anes", - "cast", - "ciborium", - "clap", - "criterion-plot", - "is-terminal", - "itertools", - "num-traits", - "once_cell", - "oorandom", - "plotters", - "rayon", - "regex", - "serde", - "serde_derive", - "serde_json", - "tinytemplate", - "walkdir", -] - -[[package]] -name = "criterion-plot" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" -dependencies = [ - "cast", - "itertools", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" - -[[package]] -name = "crunchy" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "ctrlc" -version = "3.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "672465ae37dc1bc6380a6547a8883d5dd397b0f1faaad4f265726cc7042a5345" -dependencies = [ - "nix", - "windows-sys 0.52.0", -] - -[[package]] -name = "darling" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" -dependencies = [ - "darling_core 0.14.4", - "darling_macro 0.14.4", -] - -[[package]] -name = "darling" -version = "0.20.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" -dependencies = [ - "darling_core 0.20.8", - "darling_macro 0.20.8", -] - -[[package]] -name = "darling_core" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.10.0", - "syn 1.0.109", -] - -[[package]] -name = "darling_core" -version = "0.20.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.10.0", - "syn 2.0.60", -] - -[[package]] -name = "darling_macro" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" -dependencies = [ - "darling_core 0.14.4", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "darling_macro" -version = "0.20.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" -dependencies = [ - "darling_core 0.20.8", - "quote", - "syn 2.0.60", -] - -[[package]] -name = "data-encoding" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" - -[[package]] -name = "der-parser" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe398ac75057914d7d07307bf67dc7f3f574a26783b4fc7805a20ffa9f506e82" -dependencies = [ - "asn1-rs", - "displaydoc", - "nom", - "num-bigint", - "num-traits", - "rusticata-macros", -] - -[[package]] -name = "deranged" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" -dependencies = [ - "powerfmt", - "serde", -] - -[[package]] -name = "derive_builder" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" -dependencies = [ - "darling 0.14.4", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "derive_builder_macro" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" -dependencies = [ - "derive_builder_core", - "syn 1.0.109", -] - -[[package]] -name = "derive_more" -version = "0.99.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" -dependencies = [ - "convert_case", - "proc-macro2", - "quote", - "rustc_version", - "syn 1.0.109", -] - -[[package]] -name = "diff" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", -] - -[[package]] -name = "diligent-date-parser" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6cf7fe294274a222363f84bcb63cdea762979a0443b4cf1f4f8fd17c86b1182" -dependencies = [ - "chrono", -] - -[[package]] -name = "dirs" -version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.48.0", -] - -[[package]] -name = "displaydoc" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.60", -] - -[[package]] -name = "either" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" - -[[package]] -name = "encode_unicode" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" - -[[package]] -name = "encoding_rs" -version = "0.8.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" -dependencies = [ - "cfg-if 1.0.0", -] - -[[package]] -name = "env_filter" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" -dependencies = [ - "log", - "regex", -] - -[[package]] -name = "env_logger" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9" -dependencies = [ - "anstream", - "anstyle", - "env_filter", - "humantime", - "log", -] - -[[package]] -name = "equivalent" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" - -[[package]] -name = "errno" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" - -[[package]] -name = "event-listener" -version = "4.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9944b8ca13534cdfb2800775f8dd4902ff3fc75a50101466decadfdf322a24" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" -dependencies = [ - "event-listener 4.0.3", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "332f51cb23d20b0de8458b86580878211da09bcd4503cb579c225b3d124cabb3" -dependencies = [ - "event-listener 5.3.0", - "pin-project-lite", -] - -[[package]] -name = "executable-path" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ebc5a6d89e3c90b84e8f33c8737933dda8f1c106b5415900b38b9d433841478" - -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - -[[package]] -name = "fastrand" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" - -[[package]] -name = "flate2" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" -dependencies = [ - "crc32fast", - "miniz_oxide", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - -[[package]] -name = "form_urlencoded" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "futures" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" - -[[package]] -name = "futures-executor" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" - -[[package]] -name = "futures-lite" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" -dependencies = [ - "fastrand 1.9.0", - "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite", - "waker-fn", -] - -[[package]] -name = "futures-lite" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" -dependencies = [ - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "futures-macro" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.60", -] - -[[package]] -name = "futures-rustls" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d8a2499f0fecc0492eb3e47eab4e92da7875e1028ad2528f214ac3346ca04e" -dependencies = [ - "futures-io", - "rustls 0.22.4", - "rustls-pki-types", -] - -[[package]] -name = "futures-sink" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" - -[[package]] -name = "futures-task" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" - -[[package]] -name = "futures-util" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "generic_static" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28ccff179d8070317671db09aee6d20affc26e88c5394714553b04f509b43a60" -dependencies = [ - "once_cell", -] - -[[package]] -name = "getopts" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" -dependencies = [ - "unicode-width", -] - -[[package]] -name = "getrandom" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "wasi", -] - -[[package]] -name = "gimli" -version = "0.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" - -[[package]] -name = "globset" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57da3b9b5b85bd66f31093f8c408b90a74431672542466497dcbdfdc02034be1" -dependencies = [ - "aho-corasick", - "bstr", - "log", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "gloo-net" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9902a044653b26b99f7e3693a42f171312d9be8b26b5697bd1e43ad1f8a35e10" -dependencies = [ - "gloo-utils", - "js-sys", - "thiserror", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] -name = "gloo-utils" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037fcb07216cb3a30f7292bd0176b050b7b9a052ba830ef7d5d65f6dc64ba58e" -dependencies = [ - "js-sys", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "h2" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap 2.2.6", - "slab", - "tokio", - "tokio-util 0.7.10", - "tracing", -] - -[[package]] -name = "half" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" -dependencies = [ - "cfg-if 1.0.0", - "crunchy", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "hashbrown" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hermit-abi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" - -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "hex_lit" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd" - -[[package]] -name = "html-escaper" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459a0ca33ee92551e0a3bb1774f2d3bdd1c09fb6341845736662dd25e1fcb52a" - -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - -[[package]] -name = "http-range-header" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" - -[[package]] -name = "httparse" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - -[[package]] -name = "hyper" -version = "0.14.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http 0.2.12", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.5.6", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - -[[package]] -name = "iana-time-zone" -version = "0.1.60" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - -[[package]] -name = "idna" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", - "serde", -] - -[[package]] -name = "indexmap" -version = "2.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" -dependencies = [ - "equivalent", - "hashbrown 0.14.3", - "serde", -] - -[[package]] -name = "indicatif" -version = "0.17.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3" -dependencies = [ - "console", - "instant", - "number_prefix", - "portable-atomic", - "unicode-width", -] - -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if 1.0.0", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "ipnet" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" - -[[package]] -name = "is-terminal" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" - -[[package]] -name = "js-sys" -version = "0.3.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" -dependencies = [ - "wasm-bindgen", -] - -[[package]] -name = "jsonrpc" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8128f36b47411cd3f044be8c1f5cc0c9e24d1d1bfdc45f0a57897b32513053f2" -dependencies = [ - "base64 0.13.1", - "serde", - "serde_json", -] - -[[package]] -name = "jsonrpc-core" -version = "18.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb" -dependencies = [ - "futures", - "futures-executor", - "futures-util", - "log", - "serde", - "serde_derive", - "serde_json", -] - -[[package]] -name = "jsonrpc-derive" -version = "18.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b939a78fa820cdfcb7ee7484466746a7377760970f6f9c6fe19f9edcc8a38d2" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "jsonrpc-http-server" -version = "18.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1dea6e07251d9ce6a552abfb5d7ad6bc290a4596c8dcc3d795fae2bbdc1f3ff" -dependencies = [ - "futures", - "hyper", - "jsonrpc-core", - "jsonrpc-server-utils", - "log", - "net2", - "parking_lot", - "unicase", -] - -[[package]] -name = "jsonrpc-server-utils" -version = "18.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4fdea130485b572c39a460d50888beb00afb3e35de23ccd7fad8ff19f0e0d4" -dependencies = [ - "bytes", - "futures", - "globset", - "jsonrpc-core", - "lazy_static", - "log", - "tokio", - "tokio-stream", - "tokio-util 0.6.10", - "unicase", -] - -[[package]] -name = "keccak" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" -dependencies = [ - "cpufeatures", -] - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "libc" -version = "0.2.153" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" - -[[package]] -name = "libredox" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" -dependencies = [ - "bitflags 2.5.0", - "libc", -] - -[[package]] -name = "linux-raw-sys" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" - -[[package]] -name = "linux-raw-sys" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" - -[[package]] -name = "lock_api" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" - -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - -[[package]] -name = "memchr" -version = "2.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "mime_guess" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" -dependencies = [ - "mime", - "unicase", -] - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "miniscript" -version = "10.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1eb102b66b2127a872dbcc73095b7b47aeb9d92f7b03c2b2298253ffc82c7594" -dependencies = [ - "bitcoin", - "bitcoin-private", -] - -[[package]] -name = "miniz_oxide" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" -dependencies = [ - "adler", -] - -[[package]] -name = "mio" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" -dependencies = [ - "libc", - "wasi", - "windows-sys 0.48.0", -] - -[[package]] -name = "mockcore" -version = "0.0.1" -dependencies = [ - "base64 0.21.7", - "bitcoin", - "hex", - "jsonrpc-core", - "jsonrpc-derive", - "jsonrpc-http-server", - "ord-bitcoincore-rpc", - "reqwest", - "serde", - "serde_json", - "tempfile", -] - -[[package]] -name = "mp4" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9ef834d5ed55e494a2ae350220314dc4aacd1c43a9498b00e320e0ea352a5c3" -dependencies = [ - "byteorder", - "bytes", - "num-rational", - "serde", - "serde_json", - "thiserror", -] - -[[package]] -name = "native-tls" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - -[[package]] -name = "net2" -version = "0.2.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b13b648036a2339d06de780866fbdfda0dde886de7b3af2ddeba8b14f4ee34ac" -dependencies = [ - "cfg-if 0.1.10", - "libc", - "winapi", -] - -[[package]] -name = "never" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c96aba5aa877601bb3f6dd6a63a969e1f82e60646e81e71b14496995e9853c91" - -[[package]] -name = "new_mime_guess" -version = "4.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2d684d1b59e0dc07b37e2203ef576987473288f530082512aff850585c61b1f" -dependencies = [ - "mime", - "unicase", -] - -[[package]] -name = "nix" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" -dependencies = [ - "bitflags 2.5.0", - "cfg-if 1.0.0", - "cfg_aliases", - "libc", -] - -[[package]] -name = "nodrop" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] -name = "ntapi" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" -dependencies = [ - "winapi", -] - -[[package]] -name = "num-bigint" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-conv" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" -dependencies = [ - "autocfg", - "num-bigint", - "num-integer", - "num-traits", - "serde", -] - -[[package]] -name = "num-traits" -version = "0.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" -dependencies = [ - "autocfg", -] - -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", -] - -[[package]] -name = "number_prefix" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" - -[[package]] -name = "object" -version = "0.32.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" -dependencies = [ - "memchr", -] - -[[package]] -name = "oid-registry" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38e20717fa0541f39bd146692035c37bedfa532b3e5071b35761082407546b2a" -dependencies = [ - "asn1-rs", -] - -[[package]] -name = "once_cell" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" - -[[package]] -name = "oorandom" -version = "11.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" - -[[package]] -name = "openssl" -version = "0.10.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" -dependencies = [ - "bitflags 2.5.0", - "cfg-if 1.0.0", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.60", -] - -[[package]] -name = "openssl-probe" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" - -[[package]] -name = "openssl-sys" -version = "0.9.102" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - -[[package]] -name = "ord" -version = "0.18.3" -dependencies = [ - "anyhow", - "async-trait", - "axum", - "axum-server", - "base64 0.22.0", - "bech32 0.11.0", - "bip39", - "bitcoin", - "boilerplate", - "brotli 5.0.0", - "chrono", - "ciborium", - "clap", - "colored", - "criterion", - "ctrlc", - "dirs", - "env_logger", - "executable-path", - "futures", - "hex", - "html-escaper", - "http 0.2.12", - "humantime", - "hyper", - "indicatif", - "lazy_static", - "log", - "mime", - "mime_guess", - "miniscript", - "mockcore", - "mp4", - "nix", - "ord-bitcoincore-rpc", - "ordinals", - "pretty_assertions", - "pulldown-cmark", - "redb", - "regex", - "reqwest", - "rss", - "rust-embed", - "rustls 0.22.4", - "rustls-acme", - "serde", - "serde-hex", - "serde_json", - "serde_with", - "serde_yaml", - "sha3", - "sysinfo", - "tempfile", - "tokio", - "tokio-stream", - "tokio-util 0.7.10", - "tower-http", - "unindent", - "urlencoding", -] - -[[package]] -name = "ord-bitcoincore-rpc" -version = "0.17.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b622f69d68d7201d5186615978aca36ceb7e7c57d7771491d3e261c64ff4d8" -dependencies = [ - "bitcoin-private", - "jsonrpc", - "log", - "ord-bitcoincore-rpc-json", - "serde", - "serde_json", -] - -[[package]] -name = "ord-bitcoincore-rpc-json" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb35088f918c775bc27fa79e372d034892b216fb7900aeedd6e06654879ad33" -dependencies = [ - "bitcoin", - "bitcoin-private", - "serde", - "serde_json", -] - -[[package]] -name = "ordinals" -version = "0.0.8" -dependencies = [ - "bitcoin", - "derive_more", - "pretty_assertions", - "serde", - "serde_json", - "serde_with", - "thiserror", -] - -[[package]] -name = "parking" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" - -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if 1.0.0", - "instant", - "libc", - "redox_syscall", - "smallvec 1.13.2", - "winapi", -] - -[[package]] -name = "pem" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" -dependencies = [ - "base64 0.13.1", -] - -[[package]] -name = "percent-encoding" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" - -[[package]] -name = "pin-project" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.60", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "piper" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" -dependencies = [ - "atomic-waker", - "fastrand 2.0.2", - "futures-io", -] - -[[package]] -name = "pkg-config" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" - -[[package]] -name = "plotters" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" - -[[package]] -name = "plotters-svg" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" -dependencies = [ - "plotters-backend", -] - -[[package]] -name = "polling" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" -dependencies = [ - "autocfg", - "bitflags 1.3.2", - "cfg-if 1.0.0", - "concurrent-queue", - "libc", - "log", - "pin-project-lite", - "windows-sys 0.48.0", -] - -[[package]] -name = "portable-atomic" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" - -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - -[[package]] -name = "ppv-lite86" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" - -[[package]] -name = "pretty_assertions" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" -dependencies = [ - "diff", - "yansi", -] - -[[package]] -name = "proc-macro-crate" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" -dependencies = [ - "toml", -] - -[[package]] -name = "proc-macro2" -version = "1.0.81" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "pulldown-cmark" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0530d13d87d1f549b66a3e8d0c688952abe5994e204ed62615baaf25dc029c" -dependencies = [ - "bitflags 2.5.0", - "getopts", - "memchr", - "pulldown-cmark-escape", - "unicase", -] - -[[package]] -name = "pulldown-cmark-escape" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5d8f9aa0e3cbcfaf8bf00300004ee3b72f74770f9cbac93f6928771f613276b" - -[[package]] -name = "quick-xml" -version = "0.30.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" -dependencies = [ - "encoding_rs", - "memchr", -] - -[[package]] -name = "quote" -version = "1.0.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom", -] - -[[package]] -name = "rayon" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "rcgen" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" -dependencies = [ - "pem", - "ring 0.16.20", - "time", - "yasna", -] - -[[package]] -name = "redb" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1100a056c5dcdd4e5513d5333385223b26ef1bf92f31eb38f407e8c20549256" -dependencies = [ - "libc", -] - -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "redox_users" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" -dependencies = [ - "getrandom", - "libredox", - "thiserror", -] - -[[package]] -name = "regex" -version = "1.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" - -[[package]] -name = "reqwest" -version = "0.11.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" -dependencies = [ - "async-compression", - "base64 0.21.7", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http 0.2.12", - "http-body", - "hyper", - "hyper-tls", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls-pemfile", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "system-configuration", - "tokio", - "tokio-native-tls", - "tokio-util 0.7.10", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "winreg", -] - -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", -] - -[[package]] -name = "ring" -version = "0.17.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" -dependencies = [ - "cc", - "cfg-if 1.0.0", - "getrandom", - "libc", - "spin 0.9.8", - "untrusted 0.9.0", - "windows-sys 0.52.0", -] - -[[package]] -name = "rss" -version = "2.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7b2c77eb4450d7d5f98df52c381cd6c4e19b75dad9209a9530b85a44510219a" -dependencies = [ - "atom_syndication", - "derive_builder", - "never", - "quick-xml", -] - -[[package]] -name = "rust-embed" -version = "8.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb78f46d0066053d16d4ca7b898e9343bc3530f71c61d5ad84cd404ada068745" -dependencies = [ - "rust-embed-impl", - "rust-embed-utils", - "walkdir", -] - -[[package]] -name = "rust-embed-impl" -version = "8.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91ac2a3c6c0520a3fb3dd89321177c3c692937c4eb21893378219da10c44fc8" -dependencies = [ - "proc-macro2", - "quote", - "rust-embed-utils", - "syn 2.0.60", - "walkdir", -] - -[[package]] -name = "rust-embed-utils" -version = "8.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f69089032567ffff4eada41c573fc43ff466c7db7c5688b2e7969584345581" -dependencies = [ - "sha2", - "walkdir", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" - -[[package]] -name = "rustc_version" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" -dependencies = [ - "semver", -] - -[[package]] -name = "rusticata-macros" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" -dependencies = [ - "nom", -] - -[[package]] -name = "rustix" -version = "0.37.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" -dependencies = [ - "bitflags 1.3.2", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys 0.3.8", - "windows-sys 0.48.0", -] - -[[package]] -name = "rustix" -version = "0.38.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" -dependencies = [ - "bitflags 2.5.0", - "errno", - "libc", - "linux-raw-sys 0.4.13", - "windows-sys 0.52.0", -] - -[[package]] -name = "rustls" -version = "0.21.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" -dependencies = [ - "log", - "ring 0.17.8", - "rustls-webpki 0.101.7", - "sct", -] - -[[package]] -name = "rustls" -version = "0.22.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" -dependencies = [ - "log", - "ring 0.17.8", - "rustls-pki-types", - "rustls-webpki 0.102.2", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-acme" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c132c57c13d708da4320d515289db08754f0eb539b9481ea8e35476a01d290" -dependencies = [ - "async-io", - "async-trait", - "async-web-client", - "axum-server", - "base64 0.13.1", - "blocking", - "chrono", - "futures", - "futures-rustls", - "http 1.1.0", - "log", - "pem", - "rcgen", - "ring 0.16.20", - "serde", - "serde_json", - "thiserror", - "tokio", - "tokio-util 0.7.10", - "webpki-roots", - "x509-parser", -] - -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", -] - -[[package]] -name = "rustls-pki-types" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd36cc4259e3e4514335c4a138c6b43171a8d61d8f5c9348f9fc7529416f247" - -[[package]] -name = "rustls-webpki" -version = "0.101.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" -dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", -] - -[[package]] -name = "rustls-webpki" -version = "0.102.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" -dependencies = [ - "ring 0.17.8", - "rustls-pki-types", - "untrusted 0.9.0", -] - -[[package]] -name = "rustversion" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" - -[[package]] -name = "ryu" -version = "1.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" - -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "schannel" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" -dependencies = [ - "windows-sys 0.52.0", -] - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", -] - -[[package]] -name = "secp256k1" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25996b82292a7a57ed3508f052cfff8640d38d32018784acd714758b43da9c8f" -dependencies = [ - "bitcoin_hashes 0.12.0", - "rand", - "secp256k1-sys", - "serde", -] - -[[package]] -name = "secp256k1-sys" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a129b9e9efbfb223753b9163c4ab3b13cff7fd9c7f010fbac25ab4099fa07e" -dependencies = [ - "cc", -] - -[[package]] -name = "security-framework" -version = "2.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "semver" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" - -[[package]] -name = "serde" -version = "1.0.198" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9846a40c979031340571da2545a4e5b7c4163bdae79b301d5f86d03979451fcc" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde-hex" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca37e3e4d1b39afd7ff11ee4e947efae85adfddf4841787bfa47c470e96dc26d" -dependencies = [ - "array-init", - "serde", - "smallvec 0.6.14", -] - -[[package]] -name = "serde_derive" -version = "1.0.198" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.60", -] - -[[package]] -name = "serde_json" -version = "1.0.116" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" -dependencies = [ - "indexmap 2.2.6", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_path_to_error" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" -dependencies = [ - "itoa", - "serde", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_with" -version = "3.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee80b0e361bbf88fd2f6e242ccd19cfda072cb0faa6ae694ecee08199938569a" -dependencies = [ - "base64 0.21.7", - "chrono", - "hex", - "indexmap 1.9.3", - "indexmap 2.2.6", - "serde", - "serde_derive", - "serde_json", - "serde_with_macros", - "time", -] - -[[package]] -name = "serde_with_macros" -version = "3.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6561dc161a9224638a31d876ccdfefbc1df91d3f3a8342eddb35f055d48c7655" -dependencies = [ - "darling 0.20.8", - "proc-macro2", - "quote", - "syn 2.0.60", -] - -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap 2.2.6", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - -[[package]] -name = "sha2" -version = "0.10.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" -dependencies = [ - "cfg-if 1.0.0", - "cpufeatures", - "digest", -] - -[[package]] -name = "sha3" -version = "0.10.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" -dependencies = [ - "digest", - "keccak", -] - -[[package]] -name = "slab" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] - -[[package]] -name = "smallvec" -version = "0.6.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" -dependencies = [ - "maybe-uninit", -] - -[[package]] -name = "smallvec" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" - -[[package]] -name = "socket2" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "socket2" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" - -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "strum" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" -dependencies = [ - "strum_macros", -] - -[[package]] -name = "strum_macros" -version = "0.24.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "rustversion", - "syn 1.0.109", -] - -[[package]] -name = "subtle" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.60" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - -[[package]] -name = "synstructure" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", - "unicode-xid", -] - -[[package]] -name = "sysinfo" -version = "0.30.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87341a165d73787554941cd5ef55ad728011566fe714e987d1b976c15dbc3a83" -dependencies = [ - "cfg-if 1.0.0", - "core-foundation-sys", - "libc", - "ntapi", - "once_cell", - "rayon", - "windows", -] - -[[package]] -name = "system-configuration" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "tempfile" -version = "3.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" -dependencies = [ - "cfg-if 1.0.0", - "fastrand 2.0.2", - "rustix 0.38.32", - "windows-sys 0.52.0", -] - -[[package]] -name = "thiserror" -version = "1.0.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.60", -] - -[[package]] -name = "time" -version = "0.3.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" -dependencies = [ - "deranged", - "itoa", - "num-conv", - "powerfmt", - "serde", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" - -[[package]] -name = "time-macros" -version = "0.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" -dependencies = [ - "num-conv", - "time-core", -] - -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] - -[[package]] -name = "tinyvec" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.37.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" -dependencies = [ - "backtrace", - "bytes", - "libc", - "mio", - "num_cpus", - "pin-project-lite", - "socket2 0.5.6", - "tokio-macros", - "windows-sys 0.48.0", -] - -[[package]] -name = "tokio-macros" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.60", -] - -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - -[[package]] -name = "tokio-rustls" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" -dependencies = [ - "rustls 0.21.11", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.6.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "log", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" -dependencies = [ - "bytes", - "futures-core", - "futures-io", - "futures-sink", - "pin-project-lite", - "tokio", - "tracing", -] - -[[package]] -name = "toml" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" -dependencies = [ - "serde", -] - -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tokio", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-http" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" -dependencies = [ - "async-compression", - "base64 0.21.7", - "bitflags 2.5.0", - "bytes", - "futures-core", - "futures-util", - "http 0.2.12", - "http-body", - "http-range-header", - "mime", - "pin-project-lite", - "tokio", - "tokio-util 0.7.10", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" - -[[package]] -name = "tower-service" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" - -[[package]] -name = "tracing" -version = "0.1.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" -dependencies = [ - "log", - "pin-project-lite", - "tracing-core", -] - -[[package]] -name = "tracing-core" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" -dependencies = [ - "once_cell", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] -name = "typenum" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" - -[[package]] -name = "unicase" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] - -[[package]] -name = "unicode-bidi" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" - -[[package]] -name = "unicode-ident" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" - -[[package]] -name = "unicode-normalization" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "unicode-width" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" - -[[package]] -name = "unicode-xid" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" - -[[package]] -name = "unindent" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" - -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "update-contributors" -version = "0.0.0" -dependencies = [ - "regex", -] - -[[package]] -name = "url" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - -[[package]] -name = "urlencoding" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" - -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - -[[package]] -name = "utf8parse" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - -[[package]] -name = "waker-fn" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" - -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "wasm-bindgen" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" -dependencies = [ - "cfg-if 1.0.0", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn 2.0.60", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" -dependencies = [ - "cfg-if 1.0.0", - "js-sys", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.60", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" - -[[package]] -name = "web-sys" -version = "0.3.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "webpki-roots" -version = "0.25.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" -dependencies = [ - "winapi", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" -dependencies = [ - "windows-core", - "windows-targets 0.52.5", -] - -[[package]] -name = "windows-core" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" -dependencies = [ - "windows-targets 0.52.5", -] - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets 0.52.5", -] - -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - -[[package]] -name = "windows-targets" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" -dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", - "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" - -[[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if 1.0.0", - "windows-sys 0.48.0", -] - -[[package]] -name = "x509-parser" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9bace5b5589ffead1afb76e43e34cff39cd0f3ce7e170ae0c29e53b88eb1c" -dependencies = [ - "asn1-rs", - "base64 0.13.1", - "data-encoding", - "der-parser", - "lazy_static", - "nom", - "oid-registry", - "rusticata-macros", - "thiserror", - "time", -] - -[[package]] -name = "yansi" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" - -[[package]] -name = "yasna" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" -dependencies = [ - "time", -] - -[[package]] -name = "zeroize" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" diff --git a/Cargo.toml b/Cargo.toml index fac6cc4d77..5eef92ccb5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,7 @@ http = "0.2.6" humantime = "2.1.0" hyper = { version = "0.14.24", features = ["client", "http2"] } indicatif = "0.17.1" +lapin = "2.3.3" lazy_static = "1.4.0" log = "0.4.14" mime = "0.3.16" @@ -59,6 +60,7 @@ rustls = "0.22.0" rustls-acme = { version = "0.8.1", features = ["axum"] } serde = { version = "1.0.137", features = ["derive"] } serde-hex = "0.1.0" +serde-reflection = "0.3.6" serde_json = { version = "1.0.81", features = ["preserve_order"] } serde_with = "3.7.0" serde_yaml = "0.9.17" diff --git a/src/event_publisher.rs b/src/event_publisher.rs index 2e7bd32a6b..3d71b24415 100644 --- a/src/event_publisher.rs +++ b/src/event_publisher.rs @@ -1,35 +1,55 @@ -use std::sync::Arc; +use std::thread; -use tokio::runtime::Runtime; +use anyhow::{anyhow, Result}; +use lapin::{BasicProperties, Channel, Connection, ConnectionProperties, options::BasicPublishOptions}; +use serde_json::to_vec; use tokio::sync::mpsc; +use tokio::sync::mpsc::Receiver; use crate::index::event::Event; +use crate::settings::Settings; pub struct EventPublisher { - pub event_sender: mpsc::Sender, - #[allow(dead_code)] - runtime: Arc, + pub(crate) sender: mpsc::Sender, } impl EventPublisher { - pub fn new() -> Self { - let (event_sender, mut event_receiver) = mpsc::channel::(128); - let runtime = Arc::new(Runtime::new().unwrap()); - - let rt_clone = runtime.clone(); - rt_clone.spawn(async move { - while let Some(event) = event_receiver.recv().await { - EventPublisher::handle_event(event); - } + pub fn new(settings: &Settings) -> Result { + let url = settings.rabbitmq_url() + .ok_or_else(|| anyhow!("RabbitMQ URL is not set in settings"))?.clone(); + let username = settings.rabbitmq_username() + .ok_or_else(|| anyhow!("RabbitMQ username is not set in settings"))?.clone(); + let password = settings.rabbitmq_password() + .ok_or_else(|| anyhow!("RabbitMQ password is not set in settings"))?.clone(); + let exchange = settings.rabbitmq_exchange() + .ok_or_else(|| anyhow!("RabbitMQ exchange is not set in settings"))?.clone(); + + let addr = format!("amqp://{}:{}@{}", username, password, url); + + let (sender, receiver) = mpsc::channel::(128); + + thread::spawn(move || { + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let conn = Connection::connect(&addr, ConnectionProperties::default()).await.unwrap(); + let channel = conn.create_channel().await.unwrap(); + Self::run_event_loop(receiver, channel, exchange).await; + }); }); - EventPublisher { - event_sender, - runtime, - } + Ok(EventPublisher { sender }) } - fn handle_event(event: Event) { - println!("Received event: {:?}", event); + async fn run_event_loop(mut receiver: Receiver, channel: Channel, exchange: String) { + while let Some(event) = receiver.recv().await { + let message = to_vec(&event).expect("Failed to serialize event"); + let _ = channel.basic_publish( + &exchange, + "", + BasicPublishOptions::default(), + &message, + BasicProperties::default(), + ).await.expect("Failed to publish message"); + } } } diff --git a/src/index/event.rs b/src/index/event.rs index 96e1b41f99..b5a7b3511e 100644 --- a/src/index/event.rs +++ b/src/index/event.rs @@ -1,6 +1,6 @@ use super::*; -#[derive(Debug, Clone, PartialEq)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub enum Event { InscriptionCreated { block_height: u32, diff --git a/src/options.rs b/src/options.rs index b77002d6d6..5db2c33b22 100644 --- a/src/options.rs +++ b/src/options.rs @@ -82,6 +82,14 @@ pub struct Options { help = "Require basic HTTP authentication with . Credentials are sent in cleartext. Consider using authentication in conjunction with HTTPS." )] pub(crate) server_username: Option, + #[arg(long, help = "RMQ url.")] + pub(crate) rabbitmq_url: Option, + #[arg(long, help = "RMQ username.")] + pub(crate) rabbitmq_username: Option, + #[arg(long, help = "RMQ password.")] + pub(crate) rabbitmq_password: Option, + #[arg(long, help = "RMQ exchange to publish index events.")] + pub(crate) rabbitmq_exchange: Option, #[arg(long, short, help = "Use regtest. Equivalent to `--chain regtest`.")] pub(crate) regtest: bool, #[arg(long, short, help = "Use signet. Equivalent to `--chain signet`.")] diff --git a/src/settings.rs b/src/settings.rs index a0a487002a..caf87c9cfa 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -27,6 +27,10 @@ pub struct Settings { server_password: Option, server_url: Option, server_username: Option, + rabbitmq_url: Option, + rabbitmq_username: Option, + rabbitmq_password: Option, + rabbitmq_exchange: Option, } impl Settings { @@ -141,6 +145,10 @@ impl Settings { server_password: self.server_password.or(source.server_password), server_url: self.server_url.or(source.server_url), server_username: self.server_username.or(source.server_username), + rabbitmq_url: self.rabbitmq_url.or(source.rabbitmq_url), + rabbitmq_username: self.rabbitmq_username.or(source.rabbitmq_username), + rabbitmq_password: self.rabbitmq_password.or(source.rabbitmq_password), + rabbitmq_exchange: self.rabbitmq_exchange.or(source.rabbitmq_exchange), } } @@ -175,6 +183,10 @@ impl Settings { server_password: options.server_password, server_url: None, server_username: options.server_username, + rabbitmq_url: options.rabbitmq_url, + rabbitmq_username: options.rabbitmq_username, + rabbitmq_password: options.rabbitmq_password, + rabbitmq_exchange: options.rabbitmq_exchange, } } @@ -253,6 +265,10 @@ impl Settings { server_password: get_string("SERVER_PASSWORD"), server_url: get_string("SERVER_URL"), server_username: get_string("SERVER_USERNAME"), + rabbitmq_url: get_string("RMQ_URL"), + rabbitmq_username: get_string("RMQ_USERNAME"), + rabbitmq_password: get_string("RMQ_PASSWORD"), + rabbitmq_exchange: get_string("RMQ_EXCHANGE"), }) } @@ -282,6 +298,10 @@ impl Settings { server_password: None, server_url: Some(server_url.into()), server_username: None, + rabbitmq_url: None, + rabbitmq_username: None, + rabbitmq_password: None, + rabbitmq_exchange: None, } } @@ -361,6 +381,10 @@ impl Settings { server_password: self.server_password, server_url: self.server_url, server_username: self.server_username, + rabbitmq_url: self.rabbitmq_url, + rabbitmq_username: self.rabbitmq_username, + rabbitmq_password: self.rabbitmq_password, + rabbitmq_exchange: self.rabbitmq_exchange, }) } @@ -553,6 +577,23 @@ impl Settings { pub(crate) fn server_url(&self) -> Option<&str> { self.server_url.as_deref() } + + pub fn rabbitmq_url(&self) -> Option<&String> { + self.rabbitmq_url.as_ref() + } + + pub fn rabbitmq_username(&self) -> Option<&String> { + self.rabbitmq_username.as_ref() + } + + pub fn rabbitmq_password(&self) -> Option<&String> { + self.rabbitmq_password.as_ref() + } + + pub fn rabbitmq_exchange(&self) -> Option<&String> { + self.rabbitmq_exchange.as_ref() + } + } #[cfg(test)] @@ -1000,7 +1041,10 @@ mod tests { ("NO_INDEX_INSCRIPTIONS", "1"), ("SERVER_PASSWORD", "server password"), ("SERVER_URL", "server url"), - ("SERVER_USERNAME", "server username"), + ("RMQ_URL", "http://127.0.0.1"), + ("RMQ_USERNAME", "rmq username"), + ("RMQ_PASSWORD", "rmq password"), + ("RMQ_EXCHANGE", "rmq exchange"), ] .into_iter() .map(|(key, value)| (key.into(), value.into())) @@ -1044,6 +1088,10 @@ mod tests { server_password: Some("server password".into()), server_url: Some("server url".into()), server_username: Some("server username".into()), + rabbitmq_url: Some("http://127.0.0.1".into()), + rabbitmq_username: Some("rmq username".into()), + rabbitmq_password: Some("rmq password".into()), + rabbitmq_exchange: Some("rmq exchange".into()), } ); } @@ -1076,6 +1124,10 @@ mod tests { "--no-index-inscriptions", "--server-password=server password", "--server-username=server username", + "--rabbitmq-url=http://127.0.0.1", + "--rabbitmq-username=rmq username", + "--rabbitmq-password=rmq password", + "--rabbitmq-exchange=rmq exchange", ]) .unwrap() ), @@ -1104,6 +1156,10 @@ mod tests { server_password: Some("server password".into()), server_url: None, server_username: Some("server username".into()), + rabbitmq_url: Some("http://127.0.0.1".into()), + rabbitmq_username: Some("rmq username".into()), + rabbitmq_password: Some("rmq password".into()), + rabbitmq_exchange: Some("rmq exchange".into()), } ); } diff --git a/src/subcommand.rs b/src/subcommand.rs index 18aa83f6f2..17b0aaea39 100644 --- a/src/subcommand.rs +++ b/src/subcommand.rs @@ -76,9 +76,8 @@ impl Subcommand { server.run(settings, index, handle) } Self::EventServer(server) => { - let event_publisher = EventPublisher::new(); - let event_sender = event_publisher.event_sender.clone(); - let index = Arc::new(Index::open_with_event_sender(&settings, Some(event_sender))?); + let event_publisher = EventPublisher::new(&settings)?; + let index = Arc::new(Index::open_with_event_sender(&settings, Some(event_publisher.sender.clone()))?); let handle = axum_server::Handle::new(); LISTENERS.lock().unwrap().push(handle.clone()); server.run(settings, index, handle) From 4487376129413412fcd9468c5249319ead4c03f0 Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Thu, 25 Apr 2024 23:42:40 +0700 Subject: [PATCH 003/103] Minor Refactor RMQ Publish Code --- Cargo.lock | 4966 ++++++++++++++++++++++++++++++++++++++++ src/event_publisher.rs | 81 +- src/lib.rs | 2 +- src/settings.rs | 19 +- src/subcommand.rs | 8 +- 5 files changed, 5021 insertions(+), 55 deletions(-) create mode 100644 Cargo.lock diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000000..109de679d0 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,4966 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if 1.0.0", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "amq-protocol" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f051d4d77904272e9be7e292607378dc9900d15b8d314bfd3ed4b82fdd84f125" +dependencies = [ + "amq-protocol-tcp", + "amq-protocol-types", + "amq-protocol-uri", + "cookie-factory", + "nom", + "serde", +] + +[[package]] +name = "amq-protocol-tcp" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e3d51dd36e67d757c9ba80a7b2a2a2a69254c1dbe4d8c631824ec7f5b69f60e" +dependencies = [ + "amq-protocol-uri", + "tcp-stream", + "tracing", +] + +[[package]] +name = "amq-protocol-types" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0acdd47054ced8b9bc89ee0dbb42ccc8028de48d8658b24de4c255a226c9bfec" +dependencies = [ + "cookie-factory", + "nom", + "serde", + "serde_json", +] + +[[package]] +name = "amq-protocol-uri" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17881b7575dab3e71403f28a3e50b71f0d1bd026829abca3c48664522ce0df0" +dependencies = [ + "amq-protocol-types", + "percent-encoding", + "url", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstream" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" + +[[package]] +name = "anstyle-parse" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + +[[package]] +name = "anyhow" +version = "1.0.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" +dependencies = [ + "backtrace", +] + +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + +[[package]] +name = "array-init" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23589ecb866b460d3a0f1278834750268c607e8e28a1b982c907219f3178cd72" +dependencies = [ + "nodrop", +] + +[[package]] +name = "asn1-rs" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ff05a702273012438132f449575dbc804e27b2f3cbe3069aa237d26c98fa33" +dependencies = [ + "asn1-rs-derive 0.1.0", + "asn1-rs-impl 0.1.0", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror", + "time", +] + +[[package]] +name = "asn1-rs" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ad1373757efa0f70ec53939aabc7152e1591cb485208052993070ac8d2429d" +dependencies = [ + "asn1-rs-derive 0.5.0", + "asn1-rs-impl 0.2.0", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db8b7511298d5b7784b40b092d9e9dcd3a627a5707e4b5e507931ab0d44eeebf" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure 0.12.6", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7378575ff571966e99a744addeff0bff98b8ada0dedf1956d59e634db95eaac1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", + "synstructure 0.13.1", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "async-channel" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d4d23bcc79e27423727b36823d86233aad06dfea531837b038394d11e9928" +dependencies = [ + "concurrent-queue", + "event-listener 5.3.0", + "event-listener-strategy 0.5.1", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-compression" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07dbbf24db18d609b1462965249abdf49129ccad073ec257da372adc83259c60" +dependencies = [ + "brotli 4.0.0", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-executor" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b10202063978b3351199d68f8b22c4e47e4b1b822f8d43fd862d5ea8c006b29a" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand 2.0.2", + "futures-lite 2.3.0", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" +dependencies = [ + "async-channel", + "async-executor", + "async-io 2.3.2", + "async-lock 3.3.0", + "blocking", + "futures-lite 2.3.0", + "once_cell", +] + +[[package]] +name = "async-global-executor-trait" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33dd14c5a15affd2abcff50d84efd4009ada28a860f01c14f9d654f3e81b3f75" +dependencies = [ + "async-global-executor", + "async-trait", + "executor-trait", +] + +[[package]] +name = "async-http-codec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "096146020b08dbc4587685b0730a7ba905625af13c65f8028035cdfd69573c91" +dependencies = [ + "anyhow", + "futures", + "http 1.1.0", + "httparse", + "log", +] + +[[package]] +name = "async-io" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc5b45d93ef0529756f812ca52e44c221b35341892d3dcc34132ac02f3dd2af" +dependencies = [ + "async-lock 2.8.0", + "autocfg", + "cfg-if 1.0.0", + "concurrent-queue", + "futures-lite 1.13.0", + "log", + "parking", + "polling 2.8.0", + "rustix 0.37.27", + "slab", + "socket2 0.4.10", + "waker-fn", +] + +[[package]] +name = "async-io" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcccb0f599cfa2f8ace422d3555572f47424da5648a4382a9dd0310ff8210884" +dependencies = [ + "async-lock 3.3.0", + "cfg-if 1.0.0", + "concurrent-queue", + "futures-io", + "futures-lite 2.3.0", + "parking", + "polling 3.7.0", + "rustix 0.38.34", + "slab", + "tracing", + "windows-sys 0.52.0", +] + +[[package]] +name = "async-lock" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" +dependencies = [ + "event-listener 2.5.3", +] + +[[package]] +name = "async-lock" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" +dependencies = [ + "event-listener 4.0.3", + "event-listener-strategy 0.4.0", + "pin-project-lite", +] + +[[package]] +name = "async-net" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0434b1ed18ce1cf5769b8ac540e33f01fa9471058b5e89da9e06f3c882a8c12f" +dependencies = [ + "async-io 1.13.0", + "blocking", + "futures-lite 1.13.0", +] + +[[package]] +name = "async-reactor-trait" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6012d170ad00de56c9ee354aef2e358359deb1ec504254e0e5a3774771de0e" +dependencies = [ + "async-io 1.13.0", + "async-trait", + "futures-core", + "reactor-trait", +] + +[[package]] +name = "async-task" +version = "4.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799" + +[[package]] +name = "async-trait" +version = "0.1.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "async-web-client" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e05ea882322ee3a2c62d3a2d68d8596fd8594fd1d6e93919f6f620eb8b0bf46" +dependencies = [ + "async-http-codec", + "async-net", + "async-ws", + "futures", + "futures-rustls", + "gloo-net", + "http 1.1.0", + "js-sys", + "lazy_static", + "log", + "rustls 0.22.4", + "rustls-pki-types", + "thiserror", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "async-ws" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70b425fc1a1d5a3988aa2e1c94c4823740416406f3416322c9dcb542fb997d37" +dependencies = [ + "async-io 1.13.0", + "base64 0.13.1", + "futures", + "futures-lite 1.13.0", + "generic_static", + "http 1.1.0", + "log", + "rand", + "ring 0.16.20", + "strum", + "thiserror", + "utf-8", +] + +[[package]] +name = "atom_syndication" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "571832dcff775e26562e8e6930cd483de5587301d40d3a3b85d532b6383e15a7" +dependencies = [ + "chrono", + "derive_builder", + "diligent-date-parser", + "never", + "quick-xml", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "audit-cache" +version = "0.0.0" +dependencies = [ + "colored", + "reqwest", +] + +[[package]] +name = "audit-content-security-policy" +version = "0.0.0" +dependencies = [ + "colored", + "reqwest", +] + +[[package]] +name = "autocfg" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" + +[[package]] +name = "aws-lc-rs" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5509d663b2c00ee421bda8d6a24d6c42e15970957de1701b8df9f6fbe5707df1" +dependencies = [ + "aws-lc-sys", + "mirai-annotations", + "paste", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d5d317212c2a78d86ba6622e969413c38847b62f48111f8b763af3dac2f9840" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", + "libc", + "paste", +] + +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http 0.2.12", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 0.2.12", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-server" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "447f28c85900215cc1bea282f32d4a2f22d55c5a300afdfbc661c8d6a632e063" +dependencies = [ + "arc-swap", + "bytes", + "futures-util", + "http 0.2.12", + "http-body", + "hyper", + "pin-project-lite", + "rustls 0.21.11", + "rustls-pemfile 1.0.4", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "backtrace" +version = "0.3.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + +[[package]] +name = "bech32" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" + +[[package]] +name = "bech32" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d" + +[[package]] +name = "bindgen" +version = "0.69.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +dependencies = [ + "bitflags 2.5.0", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.60", + "which", +] + +[[package]] +name = "bip39" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f2635620bf0b9d4576eb7bb9a38a55df78bd1205d26fa994b25911a69f212f" +dependencies = [ + "bitcoin_hashes 0.11.0", + "serde", + "unicode-normalization", +] + +[[package]] +name = "bitcoin" +version = "0.30.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1945a5048598e4189e239d3f809b19bdad4845c4b2ba400d304d2dcf26d2c462" +dependencies = [ + "bech32 0.9.1", + "bitcoin-private", + "bitcoin_hashes 0.12.0", + "hex_lit", + "secp256k1", + "serde", +] + +[[package]] +name = "bitcoin-private" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73290177011694f38ec25e165d0387ab7ea749a4b81cd4c80dae5988229f7a57" + +[[package]] +name = "bitcoin_hashes" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90064b8dee6815a6470d60bad07bbbaee885c0e12d04177138fa3291a01b7bc4" + +[[package]] +name = "bitcoin_hashes" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d7066118b13d4b20b23645932dfb3a81ce7e29f95726c2036fa33cd7b092501" +dependencies = [ + "bitcoin-private", + "serde", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + +[[package]] +name = "blocking" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" +dependencies = [ + "async-channel", + "async-lock 3.3.0", + "async-task", + "fastrand 2.0.2", + "futures-io", + "futures-lite 2.3.0", + "piper", + "tracing", +] + +[[package]] +name = "boilerplate" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1906889b1f805a715eac02b2dea416e25c5cfa00f099530fa9d137a3cff93113" +dependencies = [ + "darling 0.20.8", + "mime", + "new_mime_guess", + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "brotli" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "125740193d7fee5cc63ab9e16c2fdc4e07c74ba755cc53b327d6ea029e9fc569" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor 3.0.0", +] + +[[package]] +name = "brotli" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19483b140a7ac7174d34b5a581b406c64f84da5409d3e09cf4fff604f9270e67" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor 4.0.0", +] + +[[package]] +name = "brotli-decompressor" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65622a320492e09b5e0ac436b14c54ff68199bac392d0e89a6832c4518eea525" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "brotli-decompressor" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6221fe77a248b9117d431ad93761222e1cf8ff282d9d1d5d9f53d6299a1cf76" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bstr" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + +[[package]] +name = "cc" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32a725bc159af97c3e629873bb9f88fb8cf8a4867175f76dc987815ea07c83b" +dependencies = [ + "jobserver", + "libc", + "once_cell", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets 0.52.5", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "clang-sys" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "4.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim 0.11.1", +] + +[[package]] +name = "clap_derive" +version = "4.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "clap_lex" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" + +[[package]] +name = "cmake" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +dependencies = [ + "cc", +] + +[[package]] +name = "cms" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b77c319abfd5219629c45c34c89ba945ed3c5e49fcde9d16b6c3885f118a730" +dependencies = [ + "const-oid", + "der", + "spki", + "x509-cert", +] + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + +[[package]] +name = "colored" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +dependencies = [ + "lazy_static", + "windows-sys 0.48.0", +] + +[[package]] +name = "concurrent-queue" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "console" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.52.0", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + +[[package]] +name = "cookie-factory" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "ctrlc" +version = "3.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "672465ae37dc1bc6380a6547a8883d5dd397b0f1faaad4f265726cc7042a5345" +dependencies = [ + "nix", + "windows-sys 0.52.0", +] + +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core 0.14.4", + "darling_macro 0.14.4", +] + +[[package]] +name = "darling" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" +dependencies = [ + "darling_core 0.20.8", + "darling_macro 0.20.8", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn 1.0.109", +] + +[[package]] +name = "darling_core" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn 2.0.60", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core 0.14.4", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.20.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" +dependencies = [ + "darling_core 0.20.8", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "data-encoding" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" + +[[package]] +name = "der" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +dependencies = [ + "const-oid", + "der_derive", + "flagset", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "der-parser" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe398ac75057914d7d07307bf67dc7f3f574a26783b4fc7805a20ffa9f506e82" +dependencies = [ + "asn1-rs 0.3.1", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "der-parser" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" +dependencies = [ + "asn1-rs 0.6.1", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "der_derive" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fe87ce4529967e0ba1dcf8450bab64d97dfd5010a6256187ffe2e43e6f0e049" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", + "serde", +] + +[[package]] +name = "derive_builder" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_builder_macro" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" +dependencies = [ + "derive_builder_core", + "syn 1.0.109", +] + +[[package]] +name = "derive_more" +version = "0.99.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 1.0.109", +] + +[[package]] +name = "des" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdd80ce8ce993de27e9f063a444a4d53ce8e8db4c1f00cc03af5ad5a9867a1e" +dependencies = [ + "cipher", +] + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "diligent-date-parser" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6cf7fe294274a222363f84bcb63cdea762979a0443b4cf1f4f8fd17c86b1182" +dependencies = [ + "chrono", +] + +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + +[[package]] +name = "displaydoc" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + +[[package]] +name = "dunce" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" + +[[package]] +name = "either" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" + +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + +[[package]] +name = "encoding_rs" +version = "0.8.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "env_filter" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "event-listener" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9944b8ca13534cdfb2800775f8dd4902ff3fc75a50101466decadfdf322a24" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" +dependencies = [ + "event-listener 4.0.3", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332f51cb23d20b0de8458b86580878211da09bcd4503cb579c225b3d124cabb3" +dependencies = [ + "event-listener 5.3.0", + "pin-project-lite", +] + +[[package]] +name = "executable-path" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ebc5a6d89e3c90b84e8f33c8737933dda8f1c106b5415900b38b9d433841478" + +[[package]] +name = "executor-trait" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a1052dd43212a7777ec6a69b117da52f5e52f07aec47d00c1a2b33b85d06b08" +dependencies = [ + "async-trait", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "fastrand" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" + +[[package]] +name = "flagset" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb3aa5e95cf9aabc17f060cfa0ced7b83f042390760ca53bf09df9968acaa1" + +[[package]] +name = "flate2" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "flume" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" +dependencies = [ + "futures-core", + "futures-sink", + "spin 0.9.8", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand 1.9.0", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + +[[package]] +name = "futures-lite" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" +dependencies = [ + "fastrand 2.0.2", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "futures-rustls" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d8a2499f0fecc0492eb3e47eab4e92da7875e1028ad2528f214ac3346ca04e" +dependencies = [ + "futures-io", + "rustls 0.22.4", + "rustls-pki-types", +] + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "generic_static" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28ccff179d8070317671db09aee6d20affc26e88c5394714553b04f509b43a60" +dependencies = [ + "once_cell", +] + +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + +[[package]] +name = "globset" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57da3b9b5b85bd66f31093f8c408b90a74431672542466497dcbdfdc02034be1" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "gloo-net" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9902a044653b26b99f7e3693a42f171312d9be8b26b5697bd1e43ad1f8a35e10" +dependencies = [ + "gloo-utils", + "js-sys", + "thiserror", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "gloo-utils" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037fcb07216cb3a30f7292bd0176b050b7b9a052ba830ef7d5d65f6dc64ba58e" +dependencies = [ + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util 0.7.10", + "tracing", +] + +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if 1.0.0", + "crunchy", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hex_lit" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "html-escaper" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459a0ca33ee92551e0a3bb1774f2d3bdd1c09fb6341845736662dd25e1fcb52a" + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http 0.2.12", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.6", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown 0.14.3", + "serde", +] + +[[package]] +name = "indicatif" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "unicode-width", +] + +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "is-terminal" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "jobserver" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "jsonrpc" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8128f36b47411cd3f044be8c1f5cc0c9e24d1d1bfdc45f0a57897b32513053f2" +dependencies = [ + "base64 0.13.1", + "serde", + "serde_json", +] + +[[package]] +name = "jsonrpc-core" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb" +dependencies = [ + "futures", + "futures-executor", + "futures-util", + "log", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "jsonrpc-derive" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b939a78fa820cdfcb7ee7484466746a7377760970f6f9c6fe19f9edcc8a38d2" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "jsonrpc-http-server" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1dea6e07251d9ce6a552abfb5d7ad6bc290a4596c8dcc3d795fae2bbdc1f3ff" +dependencies = [ + "futures", + "hyper", + "jsonrpc-core", + "jsonrpc-server-utils", + "log", + "net2", + "parking_lot 0.11.2", + "unicase", +] + +[[package]] +name = "jsonrpc-server-utils" +version = "18.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4fdea130485b572c39a460d50888beb00afb3e35de23ccd7fad8ff19f0e0d4" +dependencies = [ + "bytes", + "futures", + "globset", + "jsonrpc-core", + "lazy_static", + "log", + "tokio", + "tokio-stream", + "tokio-util 0.6.10", + "unicase", +] + +[[package]] +name = "keccak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" +dependencies = [ + "cpufeatures", +] + +[[package]] +name = "lapin" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0454336697a39b3eb8d06339aa1b4da3cad94862fca62b0e32fd4b7f71b563" +dependencies = [ + "amq-protocol", + "async-global-executor-trait", + "async-reactor-trait", + "async-trait", + "executor-trait", + "flume", + "futures-core", + "futures-io", + "parking_lot 0.12.1", + "pinky-swear", + "reactor-trait", + "serde", + "tracing", + "waker-fn", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "libloading" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" +dependencies = [ + "cfg-if 1.0.0", + "windows-targets 0.52.5", +] + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.5.0", + "libc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + +[[package]] +name = "linux-raw-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" + +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "memchr" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniscript" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1eb102b66b2127a872dbcc73095b7b47aeb9d92f7b03c2b2298253ffc82c7594" +dependencies = [ + "bitcoin", + "bitcoin-private", +] + +[[package]] +name = "miniz_oxide" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "mirai-annotations" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" + +[[package]] +name = "mockcore" +version = "0.0.1" +dependencies = [ + "base64 0.21.7", + "bitcoin", + "hex", + "jsonrpc-core", + "jsonrpc-derive", + "jsonrpc-http-server", + "ord-bitcoincore-rpc", + "reqwest", + "serde", + "serde_json", + "tempfile", +] + +[[package]] +name = "mp4" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9ef834d5ed55e494a2ae350220314dc4aacd1c43a9498b00e320e0ea352a5c3" +dependencies = [ + "byteorder", + "bytes", + "num-rational", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "net2" +version = "0.2.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b13b648036a2339d06de780866fbdfda0dde886de7b3af2ddeba8b14f4ee34ac" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi", +] + +[[package]] +name = "never" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96aba5aa877601bb3f6dd6a63a969e1f82e60646e81e71b14496995e9853c91" + +[[package]] +name = "new_mime_guess" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d684d1b59e0dc07b37e2203ef576987473288f530082512aff850585c61b1f" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "nix" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +dependencies = [ + "bitflags 2.5.0", + "cfg-if 1.0.0", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + +[[package]] +name = "num-bigint" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +dependencies = [ + "autocfg", + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "num-traits" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + +[[package]] +name = "oid-registry" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38e20717fa0541f39bd146692035c37bedfa532b3e5071b35761082407546b2a" +dependencies = [ + "asn1-rs 0.3.1", +] + +[[package]] +name = "oid-registry" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c958dd45046245b9c3c2547369bb634eb461670b2e7e0de552905801a648d1d" +dependencies = [ + "asn1-rs 0.6.1", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + +[[package]] +name = "openssl" +version = "0.10.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +dependencies = [ + "bitflags 2.5.0", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "ord" +version = "0.18.3" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "axum-server", + "base64 0.22.0", + "bech32 0.11.0", + "bip39", + "bitcoin", + "boilerplate", + "brotli 5.0.0", + "chrono", + "ciborium", + "clap", + "colored", + "criterion", + "ctrlc", + "dirs", + "env_logger", + "executable-path", + "futures", + "hex", + "html-escaper", + "http 0.2.12", + "humantime", + "hyper", + "indicatif", + "lapin", + "lazy_static", + "log", + "mime", + "mime_guess", + "miniscript", + "mockcore", + "mp4", + "nix", + "ord-bitcoincore-rpc", + "ordinals", + "pretty_assertions", + "pulldown-cmark", + "redb", + "regex", + "reqwest", + "rss", + "rust-embed", + "rustls 0.22.4", + "rustls-acme", + "serde", + "serde-hex", + "serde-reflection", + "serde_json", + "serde_with", + "serde_yaml", + "sha3", + "sysinfo", + "tempfile", + "tokio", + "tokio-stream", + "tokio-util 0.7.10", + "tower-http", + "unindent", + "urlencoding", +] + +[[package]] +name = "ord-bitcoincore-rpc" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b622f69d68d7201d5186615978aca36ceb7e7c57d7771491d3e261c64ff4d8" +dependencies = [ + "bitcoin-private", + "jsonrpc", + "log", + "ord-bitcoincore-rpc-json", + "serde", + "serde_json", +] + +[[package]] +name = "ord-bitcoincore-rpc-json" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bb35088f918c775bc27fa79e372d034892b216fb7900aeedd6e06654879ad33" +dependencies = [ + "bitcoin", + "bitcoin-private", + "serde", + "serde_json", +] + +[[package]] +name = "ordinals" +version = "0.0.8" +dependencies = [ + "bitcoin", + "derive_more", + "pretty_assertions", + "serde", + "serde_json", + "serde_with", + "thiserror", +] + +[[package]] +name = "p12-keystore" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbd7792ed56836118732faffa19b8c2bb20d5f3ff8b403002cd817d6c4ffc96c" +dependencies = [ + "cbc", + "cms", + "der", + "des", + "hex", + "hmac", + "pkcs12", + "pkcs5", + "rand", + "rc2", + "sha1", + "sha2", + "thiserror", + "x509-parser 0.16.0", +] + +[[package]] +name = "parking" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core 0.9.9", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec 1.13.2", + "winapi", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall 0.4.1", + "smallvec 1.13.2", + "windows-targets 0.48.5", +] + +[[package]] +name = "paste" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "pem" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +dependencies = [ + "base64 0.13.1", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pinky-swear" +version = "6.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cfae3ead413ca051a681152bd266438d3bfa301c9bdf836939a14c721bb2a21" +dependencies = [ + "doc-comment", + "flume", + "parking_lot 0.12.1", + "tracing", +] + +[[package]] +name = "piper" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668d31b1c4eba19242f2088b2bf3316b82ca31082a8335764db4e083db7485d4" +dependencies = [ + "atomic-waker", + "fastrand 2.0.2", + "futures-io", +] + +[[package]] +name = "pkcs12" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "695b3df3d3cc1015f12d70235e35b6b79befc5fa7a9b95b951eab1dd07c9efc2" +dependencies = [ + "cms", + "const-oid", + "der", + "digest", + "spki", + "x509-cert", + "zeroize", +] + +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "pbkdf2", + "scrypt", + "sha2", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + +[[package]] +name = "plotters" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2c224ba00d7cadd4d5c660deaf2098e5e80e07846537c51f9cfa4be50c1fd45" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e76628b4d3a7581389a35d5b6e2139607ad7c75b17aed325f210aa91f4a9609" + +[[package]] +name = "plotters-svg" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f6d39893cca0701371e3c27294f09797214b86f1fb951b89ade8ec04e2abab" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "polling" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b2d323e8ca7996b3e23126511a523f7e62924d93ecd5ae73b333815b0eb3dce" +dependencies = [ + "autocfg", + "bitflags 1.3.2", + "cfg-if 1.0.0", + "concurrent-queue", + "libc", + "log", + "pin-project-lite", + "windows-sys 0.48.0", +] + +[[package]] +name = "polling" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645493cf344456ef24219d02a768cf1fb92ddf8c92161679ae3d91b91a637be3" +dependencies = [ + "cfg-if 1.0.0", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix 0.38.34", + "tracing", + "windows-sys 0.52.0", +] + +[[package]] +name = "portable-atomic" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "pretty_assertions" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +dependencies = [ + "diff", + "yansi", +] + +[[package]] +name = "prettyplease" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ac2cf0f2e4f42b49f5ffd07dae8d746508ef7526c13940e5f524012ae6c6550" +dependencies = [ + "proc-macro2", + "syn 2.0.60", +] + +[[package]] +name = "proc-macro-crate" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" +dependencies = [ + "toml", +] + +[[package]] +name = "proc-macro2" +version = "1.0.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pulldown-cmark" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76979bea66e7875e7509c4ec5300112b316af87fa7a252ca91c448b32dfe3993" +dependencies = [ + "bitflags 2.5.0", + "getopts", + "memchr", + "pulldown-cmark-escape", + "unicase", +] + +[[package]] +name = "pulldown-cmark-escape" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd348ff538bc9caeda7ee8cad2d1d48236a1f443c1fa3913c6a02fe0043b1dd3" + +[[package]] +name = "quick-xml" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" +dependencies = [ + "encoding_rs", + "memchr", +] + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rc2" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62c64daa8e9438b84aaae55010a93f396f8e60e3911590fcba770d04643fc1dd" +dependencies = [ + "cipher", +] + +[[package]] +name = "rcgen" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" +dependencies = [ + "pem", + "ring 0.16.20", + "time", + "yasna", +] + +[[package]] +name = "reactor-trait" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "438a4293e4d097556730f4711998189416232f009c137389e0f961d2bc0ddc58" +dependencies = [ + "async-trait", + "futures-core", + "futures-io", +] + +[[package]] +name = "redb" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7508e692a49b6b2290b56540384ccae9b1fb4d77065640b165835b56ffe3bb" +dependencies = [ + "libc", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "redox_users" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" +dependencies = [ + "getrandom", + "libredox", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" + +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "async-compression", + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http 0.2.12", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile 1.0.4", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-util 0.7.10", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin 0.5.2", + "untrusted 0.7.1", + "web-sys", + "winapi", +] + +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if 1.0.0", + "getrandom", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.52.0", +] + +[[package]] +name = "rss" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7b2c77eb4450d7d5f98df52c381cd6c4e19b75dad9209a9530b85a44510219a" +dependencies = [ + "atom_syndication", + "derive_builder", + "never", + "quick-xml", +] + +[[package]] +name = "rust-embed" +version = "8.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb78f46d0066053d16d4ca7b898e9343bc3530f71c61d5ad84cd404ada068745" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91ac2a3c6c0520a3fb3dd89321177c3c692937c4eb21893378219da10c44fc8" +dependencies = [ + "proc-macro2", + "quote", + "rust-embed-utils", + "syn 2.0.60", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86f69089032567ffff4eada41c573fc43ff466c7db7c5688b2e7969584345581" +dependencies = [ + "sha2", + "walkdir", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + +[[package]] +name = "rustix" +version = "0.37.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2" +dependencies = [ + "bitflags 1.3.2", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys 0.3.8", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustix" +version = "0.38.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +dependencies = [ + "bitflags 2.5.0", + "errno", + "libc", + "linux-raw-sys 0.4.13", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.21.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" +dependencies = [ + "log", + "ring 0.17.8", + "rustls-webpki 0.101.7", + "sct", +] + +[[package]] +name = "rustls" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +dependencies = [ + "log", + "ring 0.17.8", + "rustls-pki-types", + "rustls-webpki 0.102.3", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls" +version = "0.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afabcee0551bd1aa3e18e5adbf2c0544722014b899adb31bd186ec638d3da97e" +dependencies = [ + "aws-lc-rs", + "log", + "once_cell", + "rustls-pki-types", + "rustls-webpki 0.102.3", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-acme" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c132c57c13d708da4320d515289db08754f0eb539b9481ea8e35476a01d290" +dependencies = [ + "async-io 1.13.0", + "async-trait", + "async-web-client", + "axum-server", + "base64 0.13.1", + "blocking", + "chrono", + "futures", + "futures-rustls", + "http 1.1.0", + "log", + "pem", + "rcgen", + "ring 0.16.20", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-util 0.7.10", + "webpki-roots", + "x509-parser 0.13.2", +] + +[[package]] +name = "rustls-connector" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727a826801254b6cfcd2508a0508c01b7c1bca21d3673e84d86da084781b83d5" +dependencies = [ + "log", + "rustls 0.23.5", + "rustls-native-certs", + "rustls-pki-types", + "rustls-webpki 0.102.3", +] + +[[package]] +name = "rustls-native-certs" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" +dependencies = [ + "openssl-probe", + "rustls-pemfile 2.1.2", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + +[[package]] +name = "rustls-pemfile" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +dependencies = [ + "base64 0.22.0", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beb461507cee2c2ff151784c52762cf4d9ff6a61f3e80968600ed24fa837fa54" + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring 0.17.8", + "untrusted 0.9.0", +] + +[[package]] +name = "rustls-webpki" +version = "0.102.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3bce581c0dd41bce533ce695a1437fa16a7ab5ac3ccfa99fe1a620a7885eabf" +dependencies = [ + "aws-lc-rs", + "ring 0.17.8", + "rustls-pki-types", + "untrusted 0.9.0", +] + +[[package]] +name = "rustversion" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" + +[[package]] +name = "ryu" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" + +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2", +] + +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring 0.17.8", + "untrusted 0.9.0", +] + +[[package]] +name = "secp256k1" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25996b82292a7a57ed3508f052cfff8640d38d32018784acd714758b43da9c8f" +dependencies = [ + "bitcoin_hashes 0.12.0", + "rand", + "secp256k1-sys", + "serde", +] + +[[package]] +name = "secp256k1-sys" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a129b9e9efbfb223753b9163c4ab3b13cff7fd9c7f010fbac25ab4099fa07e" +dependencies = [ + "cc", +] + +[[package]] +name = "security-framework" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" + +[[package]] +name = "serde" +version = "1.0.198" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9846a40c979031340571da2545a4e5b7c4163bdae79b301d5f86d03979451fcc" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde-hex" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca37e3e4d1b39afd7ff11ee4e947efae85adfddf4841787bfa47c470e96dc26d" +dependencies = [ + "array-init", + "serde", + "smallvec 0.6.14", +] + +[[package]] +name = "serde-reflection" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05a5f801ac62a51a49d378fdb3884480041b99aced450b28990673e8ff99895" +dependencies = [ + "once_cell", + "serde", + "thiserror", +] + +[[package]] +name = "serde_derive" +version = "1.0.198" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "serde_json" +version = "1.0.116" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" +dependencies = [ + "indexmap 2.2.6", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" +dependencies = [ + "itoa", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c85f8e96d1d6857f13768fcbd895fcb06225510022a2774ed8b5150581847b0" +dependencies = [ + "base64 0.22.0", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.2.6", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8b3a576c4eb2924262d5951a3b737ccaf16c931e39a2810c36f9a7e25575557" +dependencies = [ + "darling 0.20.8", + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap 2.2.6", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest", + "keccak", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" +dependencies = [ + "maybe-uninit", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "socket2" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "rustversion", + "syn 1.0.109", +] + +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "sysinfo" +version = "0.30.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87341a165d73787554941cd5ef55ad728011566fe714e987d1b976c15dbc3a83" +dependencies = [ + "cfg-if 1.0.0", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "rayon", + "windows", +] + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tcp-stream" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "495b0abdce3dc1f8fd27240651c9e68890c14e9d9c61527b1ce44d8a5a7bd3d5" +dependencies = [ + "cfg-if 1.0.0", + "p12-keystore", + "rustls-connector", + "rustls-pemfile 2.1.2", +] + +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if 1.0.0", + "fastrand 2.0.2", + "rustix 0.38.34", + "windows-sys 0.52.0", +] + +[[package]] +name = "thiserror" +version = "1.0.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0126ad08bff79f29fc3ae6a55cc72352056dfff61e3ff8bb7129476d44b23aa" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "pin-project-lite", + "socket2 0.5.6", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.11", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-io", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" +dependencies = [ + "async-compression", + "base64 0.21.7", + "bitflags 2.5.0", + "bytes", + "futures-core", + "futures-util", + "http 0.2.12", + "http-body", + "http-range-header", + "mime", + "pin-project-lite", + "tokio", + "tokio-util 0.7.10", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "log", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unicase" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + +[[package]] +name = "unindent" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "update-contributors" +version = "0.0.0" +dependencies = [ + "regex", +] + +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "waker-fn" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c4517f54858c779bbcbf228f4fca63d121bf85fbecb2dc578cdf4a39395690" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.60", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + +[[package]] +name = "web-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.34", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +dependencies = [ + "windows-core", + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if 1.0.0", + "windows-sys 0.48.0", +] + +[[package]] +name = "x509-cert" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" +dependencies = [ + "const-oid", + "der", + "spki", +] + +[[package]] +name = "x509-parser" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9bace5b5589ffead1afb76e43e34cff39cd0f3ce7e170ae0c29e53b88eb1c" +dependencies = [ + "asn1-rs 0.3.1", + "base64 0.13.1", + "data-encoding", + "der-parser 7.0.0", + "lazy_static", + "nom", + "oid-registry 0.4.0", + "rusticata-macros", + "thiserror", + "time", +] + +[[package]] +name = "x509-parser" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" +dependencies = [ + "asn1-rs 0.6.1", + "data-encoding", + "der-parser 9.0.0", + "lazy_static", + "nom", + "oid-registry 0.7.0", + "rusticata-macros", + "thiserror", + "time", +] + +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + +[[package]] +name = "zeroize" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" diff --git a/src/event_publisher.rs b/src/event_publisher.rs index 3d71b24415..dcf727b5b6 100644 --- a/src/event_publisher.rs +++ b/src/event_publisher.rs @@ -1,10 +1,7 @@ -use std::thread; - -use anyhow::{anyhow, Result}; -use lapin::{BasicProperties, Channel, Connection, ConnectionProperties, options::BasicPublishOptions}; -use serde_json::to_vec; +use anyhow::{Context, Result}; +use lapin::{options::BasicPublishOptions, BasicProperties, Connection, ConnectionProperties}; +use tokio::runtime::Runtime; use tokio::sync::mpsc; -use tokio::sync::mpsc::Receiver; use crate::index::event::Event; use crate::settings::Settings; @@ -14,42 +11,46 @@ pub struct EventPublisher { } impl EventPublisher { - pub fn new(settings: &Settings) -> Result { - let url = settings.rabbitmq_url() - .ok_or_else(|| anyhow!("RabbitMQ URL is not set in settings"))?.clone(); - let username = settings.rabbitmq_username() - .ok_or_else(|| anyhow!("RabbitMQ username is not set in settings"))?.clone(); - let password = settings.rabbitmq_password() - .ok_or_else(|| anyhow!("RabbitMQ password is not set in settings"))?.clone(); - let exchange = settings.rabbitmq_exchange() - .ok_or_else(|| anyhow!("RabbitMQ exchange is not set in settings"))?.clone(); - - let addr = format!("amqp://{}:{}@{}", username, password, url); - - let (sender, receiver) = mpsc::channel::(128); - - thread::spawn(move || { - let runtime = tokio::runtime::Runtime::new().unwrap(); - runtime.block_on(async { - let conn = Connection::connect(&addr, ConnectionProperties::default()).await.unwrap(); - let channel = conn.create_channel().await.unwrap(); - Self::run_event_loop(receiver, channel, exchange).await; - }); + pub fn run(settings: &Settings) -> Result { + let addr = settings + .rabbitmq_addr() + .context("rabbitmq amqp credentials and url must be defined")?; + + let exchange = settings + .rabbitmq_exchange() + .context("rabbitmq exchange path must be defined")? + .to_owned(); + + let (sender, mut receiver) = mpsc::channel::(128); + + std::thread::spawn(move || { + Runtime::new().expect("runtime setup ok").block_on(async { + let conn = Connection::connect(&addr, ConnectionProperties::default()) + .await + .expect("connects to rabbitmq ok"); + + let channel = conn + .create_channel() + .await + .expect("creates rmq connection channel"); + + while let Some(event) = receiver.recv().await { + let message = serde_json::to_vec(&event).expect("failed to serialize event"); + + channel + .basic_publish( + &exchange, + "", + BasicPublishOptions::default(), + &message, + BasicProperties::default(), + ) + .await + .expect("Failed to publish message"); + } + }) }); Ok(EventPublisher { sender }) } - - async fn run_event_loop(mut receiver: Receiver, channel: Channel, exchange: String) { - while let Some(event) = receiver.recv().await { - let message = to_vec(&event).expect("Failed to serialize event"); - let _ = channel.basic_publish( - &exchange, - "", - BasicPublishOptions::default(), - &message, - BasicProperties::default(), - ).await.expect("Failed to publish message"); - } - } } diff --git a/src/lib.rs b/src/lib.rs index 99f53249eb..b375427074 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -104,6 +104,7 @@ mod blocktime; pub mod chain; pub mod decimal; mod deserialize_from_str; +mod event_publisher; mod fee_rate; pub mod index; mod inscriptions; @@ -120,7 +121,6 @@ pub mod subcommand; mod tally; pub mod templates; pub mod wallet; -mod event_publisher; type Result = std::result::Result; diff --git a/src/settings.rs b/src/settings.rs index caf87c9cfa..0f93fbdf1e 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -578,22 +578,17 @@ impl Settings { self.server_url.as_deref() } - pub fn rabbitmq_url(&self) -> Option<&String> { - self.rabbitmq_url.as_ref() + pub fn rabbitmq_exchange(&self) -> Option<&str> { + self.rabbitmq_exchange.as_deref() } - pub fn rabbitmq_username(&self) -> Option<&String> { - self.rabbitmq_username.as_ref() - } - - pub fn rabbitmq_password(&self) -> Option<&String> { - self.rabbitmq_password.as_ref() - } + pub fn rabbitmq_addr(&self) -> Option { + let user = self.rabbitmq_username.as_ref()?; + let pass = self.rabbitmq_password.as_ref()?; + let url = self.rabbitmq_url.as_ref()?; - pub fn rabbitmq_exchange(&self) -> Option<&String> { - self.rabbitmq_exchange.as_ref() + Some(format!("amqp://{}:{}@{}", user, pass, url)) } - } #[cfg(test)] diff --git a/src/subcommand.rs b/src/subcommand.rs index 17b0aaea39..3f433a5388 100644 --- a/src/subcommand.rs +++ b/src/subcommand.rs @@ -76,9 +76,13 @@ impl Subcommand { server.run(settings, index, handle) } Self::EventServer(server) => { - let event_publisher = EventPublisher::new(&settings)?; - let index = Arc::new(Index::open_with_event_sender(&settings, Some(event_publisher.sender.clone()))?); + let publisher = EventPublisher::run(&settings)?; let handle = axum_server::Handle::new(); + let index = Arc::new(Index::open_with_event_sender( + &settings, + Some(publisher.sender.clone()), + )?); + LISTENERS.lock().unwrap().push(handle.clone()); server.run(settings, index, handle) } From 3f4db6102695f9b2bdbfbbac0ec5b3b943e75d68 Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Fri, 26 Apr 2024 10:46:55 +0700 Subject: [PATCH 004/103] Refactor --- src/event_publisher.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/event_publisher.rs b/src/event_publisher.rs index dcf727b5b6..b431bf6bad 100644 --- a/src/event_publisher.rs +++ b/src/event_publisher.rs @@ -21,10 +21,10 @@ impl EventPublisher { .context("rabbitmq exchange path must be defined")? .to_owned(); - let (sender, mut receiver) = mpsc::channel::(128); + let (tx, mut rx) = mpsc::channel::(128); - std::thread::spawn(move || { - Runtime::new().expect("runtime setup ok").block_on(async { + let receiver = std::thread::spawn(move || { + Runtime::new().expect("runtime is setup").block_on(async { let conn = Connection::connect(&addr, ConnectionProperties::default()) .await .expect("connects to rabbitmq ok"); @@ -34,7 +34,7 @@ impl EventPublisher { .await .expect("creates rmq connection channel"); - while let Some(event) = receiver.recv().await { + while let Some(event) = rx.recv().await { let message = serde_json::to_vec(&event).expect("failed to serialize event"); channel @@ -46,11 +46,13 @@ impl EventPublisher { BasicProperties::default(), ) .await - .expect("Failed to publish message"); + .expect("published message to rmq"); } }) }); - Ok(EventPublisher { sender }) + receiver.join().expect("spawn blocking event rx thread"); + + Ok(EventPublisher { sender: tx }) } } From fd77c30ce412905d273e498875c0f9961d774d6e Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Fri, 26 Apr 2024 10:47:35 +0700 Subject: [PATCH 005/103] Refactor --- Cargo.lock | 12 ------------ Cargo.toml | 1 - 2 files changed, 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 109de679d0..a5773961a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2830,7 +2830,6 @@ dependencies = [ "rustls-acme", "serde", "serde-hex", - "serde-reflection", "serde_json", "serde_with", "serde_yaml", @@ -3847,17 +3846,6 @@ dependencies = [ "smallvec 0.6.14", ] -[[package]] -name = "serde-reflection" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05a5f801ac62a51a49d378fdb3884480041b99aced450b28990673e8ff99895" -dependencies = [ - "once_cell", - "serde", - "thiserror", -] - [[package]] name = "serde_derive" version = "1.0.198" diff --git a/Cargo.toml b/Cargo.toml index 5eef92ccb5..ecc9259614 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,6 @@ rustls = "0.22.0" rustls-acme = { version = "0.8.1", features = ["axum"] } serde = { version = "1.0.137", features = ["derive"] } serde-hex = "0.1.0" -serde-reflection = "0.3.6" serde_json = { version = "1.0.81", features = ["preserve_order"] } serde_with = "3.7.0" serde_yaml = "0.9.17" From 803a33c0e3e9301bd6ea1a4d4c4829af0077ed03 Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Fri, 26 Apr 2024 11:17:58 +0700 Subject: [PATCH 006/103] Refactor --- src/event_publisher.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/event_publisher.rs b/src/event_publisher.rs index b431bf6bad..fc2ad3e1a8 100644 --- a/src/event_publisher.rs +++ b/src/event_publisher.rs @@ -1,4 +1,5 @@ use anyhow::{Context, Result}; +use lapin::options::ConfirmSelectOptions; use lapin::{options::BasicPublishOptions, BasicProperties, Connection, ConnectionProperties}; use tokio::runtime::Runtime; use tokio::sync::mpsc; @@ -34,10 +35,15 @@ impl EventPublisher { .await .expect("creates rmq connection channel"); + channel + .confirm_select(ConfirmSelectOptions::default()) + .await + .expect("enable msg confirms"); + while let Some(event) = rx.recv().await { let message = serde_json::to_vec(&event).expect("failed to serialize event"); - channel + let publish = channel .basic_publish( &exchange, "", @@ -46,7 +52,11 @@ impl EventPublisher { BasicProperties::default(), ) .await - .expect("published message to rmq"); + .expect("published rmq msg") + .await + .expect("confirms rmq msg received"); + + assert!(publish.is_ack()); } }) }); From 918171c6b910052670b951a5730f98b309110c9f Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Fri, 26 Apr 2024 12:03:46 +0700 Subject: [PATCH 007/103] Refactor --- rmqconfig.json | 32 ++++++++++++++++++++++++++++++++ src/event_publisher.rs | 2 ++ src/subcommand.rs | 1 + 3 files changed, 35 insertions(+) create mode 100644 rmqconfig.json diff --git a/rmqconfig.json b/rmqconfig.json new file mode 100644 index 0000000000..f667dbc4b4 --- /dev/null +++ b/rmqconfig.json @@ -0,0 +1,32 @@ +{ + "queues": [ + { + "name": "ord-q", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": {} + } + ], + "exchanges": [ + { + "name": "ord-tx", + "vhost": "/", + "type": "topic", + "durable": true, + "auto_delete": false, + "internal": false, + "arguments": {} + } + ], + "bindings": [ + { + "source": "ord-tx", + "vhost": "/", + "destination": "ord-q", + "destination_type": "queue", + "routing_key": "", + "arguments": {} + } + ] +} diff --git a/src/event_publisher.rs b/src/event_publisher.rs index fc2ad3e1a8..f21528711f 100644 --- a/src/event_publisher.rs +++ b/src/event_publisher.rs @@ -43,6 +43,8 @@ impl EventPublisher { while let Some(event) = rx.recv().await { let message = serde_json::to_vec(&event).expect("failed to serialize event"); + log::info!("publishing event: {:#?}", event); + let publish = channel .basic_publish( &exchange, diff --git a/src/subcommand.rs b/src/subcommand.rs index 3f433a5388..6f70f13f6e 100644 --- a/src/subcommand.rs +++ b/src/subcommand.rs @@ -76,6 +76,7 @@ impl Subcommand { server.run(settings, index, handle) } Self::EventServer(server) => { + log::info!("Starting event server"); let publisher = EventPublisher::run(&settings)?; let handle = axum_server::Handle::new(); let index = Arc::new(Index::open_with_event_sender( From 1956cf3468902944bc263bd8b78e0c033f1c2fc6 Mon Sep 17 00:00:00 2001 From: Bodhi <106667397+RareBodhi@users.noreply.github.com> Date: Fri, 26 Apr 2024 13:21:24 +0700 Subject: [PATCH 008/103] Update src/event_publisher.rs --- src/event_publisher.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/event_publisher.rs b/src/event_publisher.rs index f21528711f..e637f724e6 100644 --- a/src/event_publisher.rs +++ b/src/event_publisher.rs @@ -63,7 +63,6 @@ impl EventPublisher { }) }); - receiver.join().expect("spawn blocking event rx thread"); Ok(EventPublisher { sender: tx }) } From 9399ac63b550cb8e213668751c1ff0c0a650c35c Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Fri, 26 Apr 2024 13:33:26 +0700 Subject: [PATCH 009/103] Log reduce --- Cargo.lock | 28 ++++++++++++++-------------- src/event_publisher.rs | 5 +---- src/subcommand.rs | 1 - 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a5773961a1..43ec21fc95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2362,7 +2362,7 @@ dependencies = [ "flume", "futures-core", "futures-io", - "parking_lot 0.12.1", + "parking_lot 0.12.2", "pinky-swear", "reactor-trait", "serde", @@ -2422,9 +2422,9 @@ checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -2924,12 +2924,12 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" dependencies = [ "lock_api", - "parking_lot_core 0.9.9", + "parking_lot_core 0.9.10", ] [[package]] @@ -2948,15 +2948,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.4.1", + "redox_syscall 0.5.1", "smallvec 1.13.2", - "windows-targets 0.48.5", + "windows-targets 0.52.5", ] [[package]] @@ -3039,7 +3039,7 @@ checksum = "6cfae3ead413ca051a681152bd266438d3bfa301c9bdf836939a14c721bb2a21" dependencies = [ "doc-comment", "flume", - "parking_lot 0.12.1", + "parking_lot 0.12.2", "tracing", ] @@ -3345,11 +3345,11 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", ] [[package]] diff --git a/src/event_publisher.rs b/src/event_publisher.rs index e637f724e6..da0e3aeb30 100644 --- a/src/event_publisher.rs +++ b/src/event_publisher.rs @@ -24,7 +24,7 @@ impl EventPublisher { let (tx, mut rx) = mpsc::channel::(128); - let receiver = std::thread::spawn(move || { + std::thread::spawn(move || { Runtime::new().expect("runtime is setup").block_on(async { let conn = Connection::connect(&addr, ConnectionProperties::default()) .await @@ -43,8 +43,6 @@ impl EventPublisher { while let Some(event) = rx.recv().await { let message = serde_json::to_vec(&event).expect("failed to serialize event"); - log::info!("publishing event: {:#?}", event); - let publish = channel .basic_publish( &exchange, @@ -63,7 +61,6 @@ impl EventPublisher { }) }); - Ok(EventPublisher { sender: tx }) } } diff --git a/src/subcommand.rs b/src/subcommand.rs index 6f70f13f6e..3f433a5388 100644 --- a/src/subcommand.rs +++ b/src/subcommand.rs @@ -76,7 +76,6 @@ impl Subcommand { server.run(settings, index, handle) } Self::EventServer(server) => { - log::info!("Starting event server"); let publisher = EventPublisher::run(&settings)?; let handle = axum_server::Handle::new(); let index = Arc::new(Index::open_with_event_sender( From 22ef211d39e91cef3bd46f4f7fdf8f8d5a439a6b Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Fri, 26 Apr 2024 13:36:53 +0700 Subject: [PATCH 010/103] Dont push index --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 632b51ef39..ec17c72dd0 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ /target /test-times.txt /tmp +/index-data \ No newline at end of file From d7db01b9d37b21b8b7df3e0dbf08fc79e22c3dab Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 26 Apr 2024 09:15:45 +0200 Subject: [PATCH 011/103] Consume rmq events --- src/event_consumer.rs | 66 +++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/subcommand.rs | 2 ++ 3 files changed, 69 insertions(+) create mode 100644 src/event_consumer.rs diff --git a/src/event_consumer.rs b/src/event_consumer.rs new file mode 100644 index 0000000000..5aee32e4b7 --- /dev/null +++ b/src/event_consumer.rs @@ -0,0 +1,66 @@ +use anyhow::Context; +use futures::StreamExt; +use lapin::{options::*, types::FieldTable, Connection, ConnectionProperties}; +use tokio::runtime::Runtime; + +use crate::settings::Settings; + +pub struct EventConsumer {} + +impl EventConsumer { + pub fn run(settings: &Settings) -> Result { + let addr = settings + .rabbitmq_addr() + .context("rabbitmq amqp credentials and url must be defined")?; + + // let queue = settings + // .rabbitmq_queue() + // .context("rabbitmq queue path must be defined")? + // .to_owned(); + + let queue = "ord-q"; + + std::thread::spawn(move || { + Runtime::new().expect("runtime is setup").block_on(async { + let conn = Connection::connect(&addr, ConnectionProperties::default()) + .await + .expect("connects to rabbitmq ok"); + + let channel = conn + .create_channel() + .await + .expect("creates rmq connection channel"); + + channel + .confirm_select(ConfirmSelectOptions::default()) + .await + .expect("enable msg confirms"); + + let mut consumer = channel + .basic_consume( + queue, + "lr-ord", //TODO pod name + BasicConsumeOptions::default(), + FieldTable::default(), + ) + .await + .expect("creates rmq consumer"); + + log::info!("Starting to consume messages from {}", queue); + while let Some(delivery) = consumer.next().await { + let delivery = delivery.expect("error in consumer"); + log::info!( + "Received message: {:?}", + String::from_utf8_lossy(&delivery.data) + ); + delivery + .ack(BasicAckOptions::default()) + .await + .expect("confirms rmq msg received"); + } + }) + }); + + Ok(EventConsumer {}) + } +} diff --git a/src/lib.rs b/src/lib.rs index b375427074..efcf6b30f4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -104,6 +104,7 @@ mod blocktime; pub mod chain; pub mod decimal; mod deserialize_from_str; +mod event_consumer; mod event_publisher; mod fee_rate; pub mod index; diff --git a/src/subcommand.rs b/src/subcommand.rs index 3f433a5388..b142f66a74 100644 --- a/src/subcommand.rs +++ b/src/subcommand.rs @@ -1,3 +1,4 @@ +use crate::event_consumer::EventConsumer; use crate::event_publisher::EventPublisher; use super::*; @@ -76,6 +77,7 @@ impl Subcommand { server.run(settings, index, handle) } Self::EventServer(server) => { + let consumer = EventConsumer::run(&settings)?; let publisher = EventPublisher::run(&settings)?; let handle = axum_server::Handle::new(); let index = Arc::new(Index::open_with_event_sender( From 48d2b7caa5d0ee4ca2e3e18bcfc5f467959e2ed3 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 26 Apr 2024 09:22:19 +0200 Subject: [PATCH 012/103] Add settings flag --- src/event_consumer.rs | 12 +++++------- src/options.rs | 2 ++ src/settings.rs | 14 ++++++++++++++ src/subcommand.rs | 2 +- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/event_consumer.rs b/src/event_consumer.rs index 5aee32e4b7..7a586cf82a 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -13,12 +13,10 @@ impl EventConsumer { .rabbitmq_addr() .context("rabbitmq amqp credentials and url must be defined")?; - // let queue = settings - // .rabbitmq_queue() - // .context("rabbitmq queue path must be defined")? - // .to_owned(); - - let queue = "ord-q"; + let queue = settings + .rabbitmq_queue() + .context("rabbitmq queue path must be defined")? + .to_owned(); std::thread::spawn(move || { Runtime::new().expect("runtime is setup").block_on(async { @@ -38,7 +36,7 @@ impl EventConsumer { let mut consumer = channel .basic_consume( - queue, + &queue, "lr-ord", //TODO pod name BasicConsumeOptions::default(), FieldTable::default(), diff --git a/src/options.rs b/src/options.rs index 5db2c33b22..7ebc3d8866 100644 --- a/src/options.rs +++ b/src/options.rs @@ -90,6 +90,8 @@ pub struct Options { pub(crate) rabbitmq_password: Option, #[arg(long, help = "RMQ exchange to publish index events.")] pub(crate) rabbitmq_exchange: Option, + #[arg(long, help = "RMQ queue to consume index events.")] + pub(crate) rabbitmq_queue: Option, #[arg(long, short, help = "Use regtest. Equivalent to `--chain regtest`.")] pub(crate) regtest: bool, #[arg(long, short, help = "Use signet. Equivalent to `--chain signet`.")] diff --git a/src/settings.rs b/src/settings.rs index 0f93fbdf1e..ba47f79270 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -31,6 +31,7 @@ pub struct Settings { rabbitmq_username: Option, rabbitmq_password: Option, rabbitmq_exchange: Option, + rabbitmq_queue: Option, } impl Settings { @@ -149,6 +150,7 @@ impl Settings { rabbitmq_username: self.rabbitmq_username.or(source.rabbitmq_username), rabbitmq_password: self.rabbitmq_password.or(source.rabbitmq_password), rabbitmq_exchange: self.rabbitmq_exchange.or(source.rabbitmq_exchange), + rabbitmq_queue: self.rabbitmq_queue.or(source.rabbitmq_queue), } } @@ -187,6 +189,7 @@ impl Settings { rabbitmq_username: options.rabbitmq_username, rabbitmq_password: options.rabbitmq_password, rabbitmq_exchange: options.rabbitmq_exchange, + rabbitmq_queue: options.rabbitmq_queue, } } @@ -269,6 +272,7 @@ impl Settings { rabbitmq_username: get_string("RMQ_USERNAME"), rabbitmq_password: get_string("RMQ_PASSWORD"), rabbitmq_exchange: get_string("RMQ_EXCHANGE"), + rabbitmq_queue: get_string("RMQ_QUEUE"), }) } @@ -302,6 +306,7 @@ impl Settings { rabbitmq_username: None, rabbitmq_password: None, rabbitmq_exchange: None, + rabbitmq_queue: None, } } @@ -385,6 +390,7 @@ impl Settings { rabbitmq_username: self.rabbitmq_username, rabbitmq_password: self.rabbitmq_password, rabbitmq_exchange: self.rabbitmq_exchange, + rabbitmq_queue: self.rabbitmq_queue, }) } @@ -582,6 +588,10 @@ impl Settings { self.rabbitmq_exchange.as_deref() } + pub fn rabbitmq_queue(&self) -> Option<&str> { + self.rabbitmq_queue.as_deref() + } + pub fn rabbitmq_addr(&self) -> Option { let user = self.rabbitmq_username.as_ref()?; let pass = self.rabbitmq_password.as_ref()?; @@ -1040,6 +1050,7 @@ mod tests { ("RMQ_USERNAME", "rmq username"), ("RMQ_PASSWORD", "rmq password"), ("RMQ_EXCHANGE", "rmq exchange"), + ("RMQ_QUEUE", "rmq queue"), ] .into_iter() .map(|(key, value)| (key.into(), value.into())) @@ -1087,6 +1098,7 @@ mod tests { rabbitmq_username: Some("rmq username".into()), rabbitmq_password: Some("rmq password".into()), rabbitmq_exchange: Some("rmq exchange".into()), + rabbitmq_queue: Some("rmq queue".into()), } ); } @@ -1123,6 +1135,7 @@ mod tests { "--rabbitmq-username=rmq username", "--rabbitmq-password=rmq password", "--rabbitmq-exchange=rmq exchange", + "--rabbitmq-queue=rmq queue", ]) .unwrap() ), @@ -1155,6 +1168,7 @@ mod tests { rabbitmq_username: Some("rmq username".into()), rabbitmq_password: Some("rmq password".into()), rabbitmq_exchange: Some("rmq exchange".into()), + rabbitmq_queue: Some("rmq queue".into()), } ); } diff --git a/src/subcommand.rs b/src/subcommand.rs index b142f66a74..ed4934bea1 100644 --- a/src/subcommand.rs +++ b/src/subcommand.rs @@ -77,7 +77,7 @@ impl Subcommand { server.run(settings, index, handle) } Self::EventServer(server) => { - let consumer = EventConsumer::run(&settings)?; + let _consumer = EventConsumer::run(&settings)?; let publisher = EventPublisher::run(&settings)?; let handle = axum_server::Handle::new(); let index = Arc::new(Index::open_with_event_sender( From 17ee8202aba8e4cdd02be5fe3ece24726f29dcd4 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 26 Apr 2024 09:54:52 +0200 Subject: [PATCH 013/103] Add consumer as a subcommand --- src/event_consumer.rs | 96 ++++++++++++++++++++++--------------------- src/options.rs | 2 - src/settings.rs | 12 ------ src/subcommand.rs | 5 ++- 4 files changed, 53 insertions(+), 62 deletions(-) diff --git a/src/event_consumer.rs b/src/event_consumer.rs index 7a586cf82a..59171741d8 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -1,64 +1,68 @@ use anyhow::Context; +use clap::Parser; use futures::StreamExt; use lapin::{options::*, types::FieldTable, Connection, ConnectionProperties}; use tokio::runtime::Runtime; use crate::settings::Settings; +use crate::subcommand::SubcommandResult; -pub struct EventConsumer {} +#[derive(Debug, Parser, Clone)] +pub struct EventConsumer { + #[arg(long, help = "RMQ queue to consume index events.")] + pub(crate) rabbitmq_queue: Option, +} impl EventConsumer { - pub fn run(settings: &Settings) -> Result { - let addr = settings - .rabbitmq_addr() - .context("rabbitmq amqp credentials and url must be defined")?; + pub fn run(self, settings: &Settings) -> SubcommandResult { + Runtime::new()?.block_on(async { + let addr = settings + .rabbitmq_addr() + .context("rabbitmq amqp credentials and url must be defined")?; - let queue = settings - .rabbitmq_queue() - .context("rabbitmq queue path must be defined")? - .to_owned(); + let queue = self + .rabbitmq_queue + .context("rabbitmq queue path must be defined")? + .to_owned(); - std::thread::spawn(move || { - Runtime::new().expect("runtime is setup").block_on(async { - let conn = Connection::connect(&addr, ConnectionProperties::default()) - .await - .expect("connects to rabbitmq ok"); + let conn = Connection::connect(&addr, ConnectionProperties::default()) + .await + .expect("connects to rabbitmq ok"); - let channel = conn - .create_channel() - .await - .expect("creates rmq connection channel"); + let channel = conn + .create_channel() + .await + .expect("creates rmq connection channel"); - channel - .confirm_select(ConfirmSelectOptions::default()) - .await - .expect("enable msg confirms"); + channel + .confirm_select(ConfirmSelectOptions::default()) + .await + .expect("enable msg confirms"); - let mut consumer = channel - .basic_consume( - &queue, - "lr-ord", //TODO pod name - BasicConsumeOptions::default(), - FieldTable::default(), - ) - .await - .expect("creates rmq consumer"); + let mut consumer = channel + .basic_consume( + &queue, + "lr-ord", //TODO pod name + BasicConsumeOptions::default(), + FieldTable::default(), + ) + .await + .expect("creates rmq consumer"); - log::info!("Starting to consume messages from {}", queue); - while let Some(delivery) = consumer.next().await { - let delivery = delivery.expect("error in consumer"); - log::info!( - "Received message: {:?}", - String::from_utf8_lossy(&delivery.data) - ); - delivery - .ack(BasicAckOptions::default()) - .await - .expect("confirms rmq msg received"); - } - }) - }); + log::info!("Starting to consume messages from {}", queue); + while let Some(delivery) = consumer.next().await { + let delivery = delivery.expect("error in consumer"); + log::info!( + "Received message: {:?}", + String::from_utf8_lossy(&delivery.data) + ); + delivery + .ack(BasicAckOptions::default()) + .await + .expect("confirms rmq msg received"); + } - Ok(EventConsumer {}) + Ok(None) + }) } } diff --git a/src/options.rs b/src/options.rs index 7ebc3d8866..5db2c33b22 100644 --- a/src/options.rs +++ b/src/options.rs @@ -90,8 +90,6 @@ pub struct Options { pub(crate) rabbitmq_password: Option, #[arg(long, help = "RMQ exchange to publish index events.")] pub(crate) rabbitmq_exchange: Option, - #[arg(long, help = "RMQ queue to consume index events.")] - pub(crate) rabbitmq_queue: Option, #[arg(long, short, help = "Use regtest. Equivalent to `--chain regtest`.")] pub(crate) regtest: bool, #[arg(long, short, help = "Use signet. Equivalent to `--chain signet`.")] diff --git a/src/settings.rs b/src/settings.rs index ba47f79270..ed26ee5708 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -31,7 +31,6 @@ pub struct Settings { rabbitmq_username: Option, rabbitmq_password: Option, rabbitmq_exchange: Option, - rabbitmq_queue: Option, } impl Settings { @@ -150,7 +149,6 @@ impl Settings { rabbitmq_username: self.rabbitmq_username.or(source.rabbitmq_username), rabbitmq_password: self.rabbitmq_password.or(source.rabbitmq_password), rabbitmq_exchange: self.rabbitmq_exchange.or(source.rabbitmq_exchange), - rabbitmq_queue: self.rabbitmq_queue.or(source.rabbitmq_queue), } } @@ -189,7 +187,6 @@ impl Settings { rabbitmq_username: options.rabbitmq_username, rabbitmq_password: options.rabbitmq_password, rabbitmq_exchange: options.rabbitmq_exchange, - rabbitmq_queue: options.rabbitmq_queue, } } @@ -272,7 +269,6 @@ impl Settings { rabbitmq_username: get_string("RMQ_USERNAME"), rabbitmq_password: get_string("RMQ_PASSWORD"), rabbitmq_exchange: get_string("RMQ_EXCHANGE"), - rabbitmq_queue: get_string("RMQ_QUEUE"), }) } @@ -306,7 +302,6 @@ impl Settings { rabbitmq_username: None, rabbitmq_password: None, rabbitmq_exchange: None, - rabbitmq_queue: None, } } @@ -390,7 +385,6 @@ impl Settings { rabbitmq_username: self.rabbitmq_username, rabbitmq_password: self.rabbitmq_password, rabbitmq_exchange: self.rabbitmq_exchange, - rabbitmq_queue: self.rabbitmq_queue, }) } @@ -588,10 +582,6 @@ impl Settings { self.rabbitmq_exchange.as_deref() } - pub fn rabbitmq_queue(&self) -> Option<&str> { - self.rabbitmq_queue.as_deref() - } - pub fn rabbitmq_addr(&self) -> Option { let user = self.rabbitmq_username.as_ref()?; let pass = self.rabbitmq_password.as_ref()?; @@ -1098,7 +1088,6 @@ mod tests { rabbitmq_username: Some("rmq username".into()), rabbitmq_password: Some("rmq password".into()), rabbitmq_exchange: Some("rmq exchange".into()), - rabbitmq_queue: Some("rmq queue".into()), } ); } @@ -1168,7 +1157,6 @@ mod tests { rabbitmq_username: Some("rmq username".into()), rabbitmq_password: Some("rmq password".into()), rabbitmq_exchange: Some("rmq exchange".into()), - rabbitmq_queue: Some("rmq queue".into()), } ); } diff --git a/src/subcommand.rs b/src/subcommand.rs index ed4934bea1..6f428ad704 100644 --- a/src/subcommand.rs +++ b/src/subcommand.rs @@ -1,4 +1,3 @@ -use crate::event_consumer::EventConsumer; use crate::event_publisher::EventPublisher; use super::*; @@ -44,6 +43,8 @@ pub(crate) enum Subcommand { Server(server::Server), #[command(about = "Run the explorer server in event emit mode")] EventServer(server::Server), + #[command(about = "Run the index event consumer")] + EventConsumer(event_consumer::EventConsumer), #[command(about = "Display settings")] Settings, #[command(about = "Display information about a block's subsidy")] @@ -77,7 +78,6 @@ impl Subcommand { server.run(settings, index, handle) } Self::EventServer(server) => { - let _consumer = EventConsumer::run(&settings)?; let publisher = EventPublisher::run(&settings)?; let handle = axum_server::Handle::new(); let index = Arc::new(Index::open_with_event_sender( @@ -88,6 +88,7 @@ impl Subcommand { LISTENERS.lock().unwrap().push(handle.clone()); server.run(settings, index, handle) } + Self::EventConsumer(event_consumer) => event_consumer.run(&settings), Self::Settings => settings::run(settings), Self::Subsidy(subsidy) => subsidy.run(), Self::Supply => supply::run(), From f1ce1926d86b351e955e0afe173dc34d0155ed42 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 26 Apr 2024 09:58:29 +0200 Subject: [PATCH 014/103] Clean up --- src/settings.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/settings.rs b/src/settings.rs index ed26ee5708..0f93fbdf1e 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -1040,7 +1040,6 @@ mod tests { ("RMQ_USERNAME", "rmq username"), ("RMQ_PASSWORD", "rmq password"), ("RMQ_EXCHANGE", "rmq exchange"), - ("RMQ_QUEUE", "rmq queue"), ] .into_iter() .map(|(key, value)| (key.into(), value.into())) @@ -1124,7 +1123,6 @@ mod tests { "--rabbitmq-username=rmq username", "--rabbitmq-password=rmq password", "--rabbitmq-exchange=rmq exchange", - "--rabbitmq-queue=rmq queue", ]) .unwrap() ), From 6489034aaa04bbf0cfd96f8a3d04a965863bfe2c Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 26 Apr 2024 10:35:10 +0200 Subject: [PATCH 015/103] Serialize back to event --- src/event_consumer.rs | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/src/event_consumer.rs b/src/event_consumer.rs index 59171741d8..88b76523de 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -4,6 +4,7 @@ use futures::StreamExt; use lapin::{options::*, types::FieldTable, Connection, ConnectionProperties}; use tokio::runtime::Runtime; +use crate::index::event::Event; use crate::settings::Settings; use crate::subcommand::SubcommandResult; @@ -51,15 +52,29 @@ impl EventConsumer { log::info!("Starting to consume messages from {}", queue); while let Some(delivery) = consumer.next().await { - let delivery = delivery.expect("error in consumer"); - log::info!( - "Received message: {:?}", - String::from_utf8_lossy(&delivery.data) - ); - delivery - .ack(BasicAckOptions::default()) - .await - .expect("confirms rmq msg received"); + match delivery { + Ok(delivery) => { + let event: Result = serde_json::from_slice(&delivery.data); + match event { + Ok(event) => { + log::info!("Received event: {:?}", event); + delivery + .ack(BasicAckOptions::default()) + .await + .expect("confirms rmq msg received"); + } + Err(e) => { + log::error!("Error deserializing event: {}", e); + delivery + .reject(BasicRejectOptions { requeue: false }) + .await?; + } + } + } + Err(e) => { + log::error!("Error receiving delivery: {}", e); + } + } } Ok(None) From 977d3edde2329431b626dae20e11231b1c04275d Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 26 Apr 2024 13:50:48 +0200 Subject: [PATCH 016/103] Consume inscription events --- Cargo.lock | 552 +++++++++++++++++++++++++++++++++++++++--- Cargo.toml | 1 + src/event_consumer.rs | 50 +++- 3 files changed, 571 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 43ec21fc95..fbc2044805 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,6 +28,19 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if 1.0.0", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -52,6 +65,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + [[package]] name = "amq-protocol" version = "7.2.0" @@ -286,11 +305,11 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07dbbf24db18d609b1462965249abdf49129ccad073ec257da372adc83259c60" +checksum = "4e9eabd7a98fe442131a17c316bd9349c43695e49e730c3c8e12cfb5f4da2693" dependencies = [ - "brotli 4.0.0", + "brotli", "flate2", "futures-core", "memchr", @@ -494,6 +513,15 @@ dependencies = [ "utf-8", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atom_syndication" version = "0.12.2" @@ -763,6 +791,9 @@ name = "bitflags" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +dependencies = [ + "serde", +] [[package]] name = "block-buffer" @@ -812,17 +843,6 @@ dependencies = [ "syn 2.0.60", ] -[[package]] -name = "brotli" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "125740193d7fee5cc63ab9e16c2fdc4e07c74ba755cc53b327d6ea029e9fc569" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor 3.0.0", -] - [[package]] name = "brotli" version = "5.0.0" @@ -831,17 +851,7 @@ checksum = "19483b140a7ac7174d34b5a581b406c64f84da5409d3e09cf4fff604f9270e67" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", - "brotli-decompressor 4.0.0", -] - -[[package]] -name = "brotli-decompressor" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65622a320492e09b5e0ac436b14c54ff68199bac392d0e89a6832c4518eea525" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", + "brotli-decompressor", ] [[package]] @@ -1140,6 +1150,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.4.0" @@ -1204,6 +1229,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.19" @@ -1440,6 +1474,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", "subtle", ] @@ -1491,6 +1526,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dunce" version = "1.0.4" @@ -1502,6 +1543,9 @@ name = "either" version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" +dependencies = [ + "serde", +] [[package]] name = "encode_unicode" @@ -1557,6 +1601,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if 1.0.0", + "home", + "windows-sys 0.48.0", +] + [[package]] name = "event-listener" version = "2.5.3" @@ -1635,6 +1690,12 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" +[[package]] +name = "finl_unicode" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" + [[package]] name = "flagset" version = "0.4.5" @@ -1643,9 +1704,9 @@ checksum = "cdeb3aa5e95cf9aabc17f060cfa0ced7b83f042390760ca53bf09df9968acaa1" [[package]] name = "flate2" -version = "1.0.28" +version = "1.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +checksum = "4556222738635b7a3417ae6130d8f52201e45a0c4d1a907f0826383adb5f85e7" dependencies = [ "crc32fast", "miniz_oxide", @@ -1740,6 +1801,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot 0.12.2", +] + [[package]] name = "futures-io" version = "0.3.30" @@ -1955,12 +2027,28 @@ name = "hashbrown" version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown 0.14.3", +] [[package]] name = "heck" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] [[package]] name = "heck" @@ -1986,6 +2074,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3011d1213f159867b13cfd6ac92d2cd5f1345762c63be3554e84092d85a50bbd" +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + [[package]] name = "hmac" version = "0.12.1" @@ -2375,6 +2472,9 @@ name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin 0.5.2", +] [[package]] name = "lazycell" @@ -2398,6 +2498,12 @@ dependencies = [ "windows-targets 0.52.5", ] +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + [[package]] name = "libredox" version = "0.1.3" @@ -2408,6 +2514,17 @@ dependencies = [ "libc", ] +[[package]] +name = "libsqlite3-sys" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.3.8" @@ -2448,6 +2565,16 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if 1.0.0", + "digest", +] + [[package]] name = "memchr" version = "2.7.2" @@ -2636,6 +2763,23 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec 1.13.2", + "zeroize", +] + [[package]] name = "num-conv" version = "0.1.0" @@ -2651,6 +2795,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-rational" version = "0.4.1" @@ -2671,6 +2826,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -2791,7 +2947,7 @@ dependencies = [ "bip39", "bitcoin", "boilerplate", - "brotli 5.0.0", + "brotli", "chrono", "ciborium", "clap", @@ -2834,6 +2990,7 @@ dependencies = [ "serde_with", "serde_yaml", "sha3", + "sqlx", "sysinfo", "tempfile", "tokio", @@ -3054,6 +3211,17 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + [[package]] name = "pkcs12" version = "0.1.0" @@ -3084,6 +3252,16 @@ dependencies = [ "spki", ] +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.30" @@ -3343,6 +3521,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.1" @@ -3464,6 +3651,26 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rsa" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rss" version = "2.0.7" @@ -3972,6 +4179,16 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core", +] + [[package]] name = "slab" version = "0.4.9" @@ -4041,6 +4258,225 @@ dependencies = [ "der", ] +[[package]] +name = "sqlformat" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce81b7bd7c4493975347ef60d8c7e8b742d4694f4c49f93e0a12ea263938176c" +dependencies = [ + "itertools 0.12.1", + "nom", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9a2ccff1a000a5a59cd33da541d9f2fdcd9e6e8229cc200565942bff36d0aaa" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24ba59a9342a3d9bab6c56c118be528b27c9b60e490080e9711a04dccac83ef6" +dependencies = [ + "ahash", + "atoi", + "byteorder", + "bytes", + "crc", + "crossbeam-queue", + "either", + "event-listener 2.5.3", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashlink", + "hex", + "indexmap 2.2.6", + "log", + "memchr", + "once_cell", + "paste", + "percent-encoding", + "rustls 0.21.11", + "rustls-pemfile 1.0.4", + "serde", + "serde_json", + "sha2", + "smallvec 1.13.2", + "sqlformat", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", + "webpki-roots", +] + +[[package]] +name = "sqlx-macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea40e2345eb2faa9e1e5e326db8c34711317d2b5e08d0d5741619048a803127" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 1.0.109", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5833ef53aaa16d860e92123292f1f6a3d53c34ba8b1969f152ef1a7bb803f3c8" +dependencies = [ + "dotenvy", + "either", + "heck 0.4.1", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 1.0.109", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ed31390216d20e538e447a7a9b959e06ed9fc51c37b514b46eb758016ecd418" +dependencies = [ + "atoi", + "base64 0.21.7", + "bitflags 2.5.0", + "byteorder", + "bytes", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec 1.13.2", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c824eb80b894f926f89a0b9da0c7f435d27cdd35b8c655b114e58223918577e" +dependencies = [ + "atoi", + "base64 0.21.7", + "bitflags 2.5.0", + "byteorder", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand", + "serde", + "serde_json", + "sha2", + "smallvec 1.13.2", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b244ef0a8414da0bed4bb1910426e890b19e5e9bccc27ada6b797d05c55ae0aa" +dependencies = [ + "atoi", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "tracing", + "url", + "urlencoding", +] + +[[package]] +name = "stringprep" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" +dependencies = [ + "finl_unicode", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "strsim" version = "0.10.0" @@ -4424,9 +4860,21 @@ checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ "log", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + [[package]] name = "tracing-core" version = "0.1.32" @@ -4478,6 +4926,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" + [[package]] name = "unicode-width" version = "0.1.11" @@ -4490,6 +4944,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "unindent" version = "0.2.3" @@ -4593,6 +5053,12 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.92" @@ -4687,6 +5153,16 @@ dependencies = [ "rustix 0.38.34", ] +[[package]] +name = "whoami" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" +dependencies = [ + "redox_syscall 0.4.1", + "wasite", +] + [[package]] name = "winapi" version = "0.3.9" @@ -4947,6 +5423,26 @@ dependencies = [ "time", ] +[[package]] +name = "zerocopy" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.60", +] + [[package]] name = "zeroize" version = "1.7.0" diff --git a/Cargo.toml b/Cargo.toml index ecc9259614..0b7eadbad7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,6 +64,7 @@ serde_json = { version = "1.0.81", features = ["preserve_order"] } serde_with = "3.7.0" serde_yaml = "0.9.17" sha3 = "0.10.8" +sqlx = { version = "0.7", features = [ "runtime-tokio", "tls-rustls" , "postgres" ] } sysinfo = "0.30.3" tempfile = "3.2.0" tokio = { version = "1.17.0", features = ["rt-multi-thread"] } diff --git a/src/event_consumer.rs b/src/event_consumer.rs index 88b76523de..50a59d1d1b 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -1,10 +1,13 @@ use anyhow::Context; use clap::Parser; use futures::StreamExt; -use lapin::{options::*, types::FieldTable, Connection, ConnectionProperties}; +use lapin::{Connection, ConnectionProperties, options::*, types::FieldTable}; use tokio::runtime::Runtime; +use ordinals::SatPoint; + use crate::index::event::Event; +use crate::InscriptionId; use crate::settings::Settings; use crate::subcommand::SubcommandResult; @@ -23,8 +26,8 @@ impl EventConsumer { let queue = self .rabbitmq_queue - .context("rabbitmq queue path must be defined")? - .to_owned(); + .as_deref() + .context("rabbitmq queue path must be defined")?; let conn = Connection::connect(&addr, ConnectionProperties::default()) .await @@ -57,7 +60,9 @@ impl EventConsumer { let event: Result = serde_json::from_slice(&delivery.data); match event { Ok(event) => { - log::info!("Received event: {:?}", event); + self.handle_event(&event) + .await + .expect("confirms rmq msg processed"); delivery .ack(BasicAckOptions::default()) .await @@ -80,4 +85,41 @@ impl EventConsumer { Ok(None) }) } + + async fn handle_event(&self, event: &Event) -> Result<(), sqlx::Error> { + match event { + Event::InscriptionCreated { + block_height, + charms, + inscription_id, + location, + parent_inscription_ids, + sequence_number, + } => { + self.handle_inscription_created(block_height, charms, inscription_id, location, parent_inscription_ids, sequence_number).await + } + Event::InscriptionTransferred { + block_height, + inscription_id, + new_location, + old_location, + sequence_number, + } => { + self.handle_inscription_transferred(block_height, inscription_id, new_location, old_location, sequence_number).await + } + _ => { + Ok(()) + } + } + } + + async fn handle_inscription_created(&self, block_height: &u32, charms: &u16, inscription_id: &InscriptionId, location: &Option, parent_inscription_ids: &Vec, sequence_number: &u32) -> Result<(), sqlx::Error> { + log::info!("Received inscription created event: {:?}", inscription_id); + Ok(()) + } + + async fn handle_inscription_transferred(&self, block_height: &u32, inscription_id: &InscriptionId, new_location: &SatPoint, old_location: &SatPoint, sequence_number: &u32) -> Result<(), sqlx::Error> { + log::info!("Received inscription transferred event: {:?}", inscription_id); + Ok(()) + } } From a7d19c0b23c480f24088cd8921891a308d556c4a Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 26 Apr 2024 14:27:33 +0200 Subject: [PATCH 017/103] Fetch data via db --- src/event_consumer.rs | 31 +++++++++++++++++++++++-------- src/subcommand.rs | 5 ++++- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/event_consumer.rs b/src/event_consumer.rs index 50a59d1d1b..033adf148b 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use anyhow::Context; use clap::Parser; use futures::StreamExt; @@ -6,9 +8,10 @@ use tokio::runtime::Runtime; use ordinals::SatPoint; +use crate::{Index, InscriptionId}; use crate::index::event::Event; -use crate::InscriptionId; use crate::settings::Settings; +use crate::subcommand::server::query; use crate::subcommand::SubcommandResult; #[derive(Debug, Parser, Clone)] @@ -18,7 +21,7 @@ pub struct EventConsumer { } impl EventConsumer { - pub fn run(self, settings: &Settings) -> SubcommandResult { + pub fn run(self, settings: &Settings, index: Arc) -> SubcommandResult { Runtime::new()?.block_on(async { let addr = settings .rabbitmq_addr() @@ -60,7 +63,7 @@ impl EventConsumer { let event: Result = serde_json::from_slice(&delivery.data); match event { Ok(event) => { - self.handle_event(&event) + self.handle_event(&index, &event) .await .expect("confirms rmq msg processed"); delivery @@ -86,7 +89,7 @@ impl EventConsumer { }) } - async fn handle_event(&self, event: &Event) -> Result<(), sqlx::Error> { + async fn handle_event(&self, index: &Arc, event: &Event) -> Result<(), anyhow::Error> { match event { Event::InscriptionCreated { block_height, @@ -96,7 +99,7 @@ impl EventConsumer { parent_inscription_ids, sequence_number, } => { - self.handle_inscription_created(block_height, charms, inscription_id, location, parent_inscription_ids, sequence_number).await + self.handle_inscription_created(index, block_height, charms, inscription_id, location, parent_inscription_ids, sequence_number).await } Event::InscriptionTransferred { block_height, @@ -113,12 +116,24 @@ impl EventConsumer { } } - async fn handle_inscription_created(&self, block_height: &u32, charms: &u16, inscription_id: &InscriptionId, location: &Option, parent_inscription_ids: &Vec, sequence_number: &u32) -> Result<(), sqlx::Error> { + async fn handle_inscription_created(&self, index: &Arc, block_height: &u32, charms: &u16, inscription_id: &InscriptionId, location: &Option, parent_inscription_ids: &Vec, sequence_number: &u32) -> Result<(), anyhow::Error> { log::info!("Received inscription created event: {:?}", inscription_id); - Ok(()) + let response = index + .inscription_info(query::Inscription::Id(inscription_id.clone()))?; + match response { + Some((info, txout, inscription)) => { + log::info!("Persisting inscription: {:?}", info.id); + Ok(()) + } + None => { + // TODO handle error properly + // Err(format!("Failed to handle inscription created event: {:?}", inscription_id)); + Ok(()) + } + } } - async fn handle_inscription_transferred(&self, block_height: &u32, inscription_id: &InscriptionId, new_location: &SatPoint, old_location: &SatPoint, sequence_number: &u32) -> Result<(), sqlx::Error> { + async fn handle_inscription_transferred(&self, block_height: &u32, inscription_id: &InscriptionId, new_location: &SatPoint, old_location: &SatPoint, sequence_number: &u32) -> Result<(), anyhow::Error> { log::info!("Received inscription transferred event: {:?}", inscription_id); Ok(()) } diff --git a/src/subcommand.rs b/src/subcommand.rs index 6f428ad704..a75bf4a2e1 100644 --- a/src/subcommand.rs +++ b/src/subcommand.rs @@ -88,7 +88,10 @@ impl Subcommand { LISTENERS.lock().unwrap().push(handle.clone()); server.run(settings, index, handle) } - Self::EventConsumer(event_consumer) => event_consumer.run(&settings), + Self::EventConsumer(event_consumer) => { + let index = Arc::new(Index::open(&settings)?); + event_consumer.run(&settings, index) + }, Self::Settings => settings::run(settings), Self::Subsidy(subsidy) => subsidy.run(), Self::Supply => supply::run(), From 3043639ab5b731646dbf6bc6065f4e25c3087774 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 26 Apr 2024 15:46:54 +0200 Subject: [PATCH 018/103] Consume events and fetch details via api --- src/event_consumer.rs | 93 ++++++++++++++++++++++++++++++------------- src/lib.rs | 1 + src/options.rs | 2 + src/ord_api_client.rs | 51 ++++++++++++++++++++++++ src/settings.rs | 22 ++++++++-- src/subcommand.rs | 7 ++-- 6 files changed, 142 insertions(+), 34 deletions(-) create mode 100644 src/ord_api_client.rs diff --git a/src/event_consumer.rs b/src/event_consumer.rs index 033adf148b..27c69d1523 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -1,18 +1,16 @@ -use std::sync::Arc; - use anyhow::Context; use clap::Parser; use futures::StreamExt; -use lapin::{Connection, ConnectionProperties, options::*, types::FieldTable}; +use lapin::{options::*, types::FieldTable, Connection, ConnectionProperties}; use tokio::runtime::Runtime; use ordinals::SatPoint; -use crate::{Index, InscriptionId}; use crate::index::event::Event; +use crate::ord_api_client::OrdApiClient; use crate::settings::Settings; -use crate::subcommand::server::query; use crate::subcommand::SubcommandResult; +use crate::InscriptionId; #[derive(Debug, Parser, Clone)] pub struct EventConsumer { @@ -21,7 +19,7 @@ pub struct EventConsumer { } impl EventConsumer { - pub fn run(self, settings: &Settings, index: Arc) -> SubcommandResult { + pub fn run(self, settings: &Settings, ord_api_client: OrdApiClient) -> SubcommandResult { Runtime::new()?.block_on(async { let addr = settings .rabbitmq_addr() @@ -63,7 +61,8 @@ impl EventConsumer { let event: Result = serde_json::from_slice(&delivery.data); match event { Ok(event) => { - self.handle_event(&index, &event) + self + .handle_event(&ord_api_client, &event) .await .expect("confirms rmq msg processed"); delivery @@ -89,7 +88,11 @@ impl EventConsumer { }) } - async fn handle_event(&self, index: &Arc, event: &Event) -> Result<(), anyhow::Error> { + async fn handle_event( + &self, + ord_api_client: &OrdApiClient, + event: &Event, + ) -> Result<(), anyhow::Error> { match event { Event::InscriptionCreated { block_height, @@ -99,7 +102,17 @@ impl EventConsumer { parent_inscription_ids, sequence_number, } => { - self.handle_inscription_created(index, block_height, charms, inscription_id, location, parent_inscription_ids, sequence_number).await + self + .handle_inscription_created( + ord_api_client, + block_height, + charms, + inscription_id, + location, + parent_inscription_ids, + sequence_number, + ) + .await } Event::InscriptionTransferred { block_height, @@ -108,33 +121,59 @@ impl EventConsumer { old_location, sequence_number, } => { - self.handle_inscription_transferred(block_height, inscription_id, new_location, old_location, sequence_number).await - } - _ => { - Ok(()) + self + .handle_inscription_transferred( + ord_api_client, + block_height, + inscription_id, + new_location, + old_location, + sequence_number, + ) + .await } + _ => Ok(()), } } - async fn handle_inscription_created(&self, index: &Arc, block_height: &u32, charms: &u16, inscription_id: &InscriptionId, location: &Option, parent_inscription_ids: &Vec, sequence_number: &u32) -> Result<(), anyhow::Error> { + async fn handle_inscription_created( + &self, + ord_api_client: &OrdApiClient, + block_height: &u32, + charms: &u16, + inscription_id: &InscriptionId, + location: &Option, + parent_inscription_ids: &Vec, + sequence_number: &u32, + ) -> Result<(), anyhow::Error> { log::info!("Received inscription created event: {:?}", inscription_id); - let response = index - .inscription_info(query::Inscription::Id(inscription_id.clone()))?; - match response { - Some((info, txout, inscription)) => { - log::info!("Persisting inscription: {:?}", info.id); - Ok(()) + match ord_api_client + .fetch_inscription_details(inscription_id) + .await + { + Ok(inscription) => { + log::info!("Received inscription detailed response: {:?}", inscription); } - None => { - // TODO handle error properly - // Err(format!("Failed to handle inscription created event: {:?}", inscription_id)); - Ok(()) + Err(e) => { + log::error!("Failed to fetch: {:?}", e); } - } + }; + Ok(()) } - async fn handle_inscription_transferred(&self, block_height: &u32, inscription_id: &InscriptionId, new_location: &SatPoint, old_location: &SatPoint, sequence_number: &u32) -> Result<(), anyhow::Error> { - log::info!("Received inscription transferred event: {:?}", inscription_id); + async fn handle_inscription_transferred( + &self, + ord_api_client: &OrdApiClient, + block_height: &u32, + inscription_id: &InscriptionId, + new_location: &SatPoint, + old_location: &SatPoint, + sequence_number: &u32, + ) -> Result<(), anyhow::Error> { + log::info!( + "Received inscription transferred event: {:?}", + inscription_id + ); Ok(()) } } diff --git a/src/lib.rs b/src/lib.rs index efcf6b30f4..418914b4d6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -113,6 +113,7 @@ mod into_usize; mod macros; mod object; pub mod options; +pub mod ord_api_client; pub mod outgoing; mod re; mod representation; diff --git a/src/options.rs b/src/options.rs index 5db2c33b22..69921fcf0e 100644 --- a/src/options.rs +++ b/src/options.rs @@ -90,6 +90,8 @@ pub struct Options { pub(crate) rabbitmq_password: Option, #[arg(long, help = "RMQ exchange to publish index events.")] pub(crate) rabbitmq_exchange: Option, + #[arg(long, help = "Ord api url.")] + pub(crate) ord_api_url: Option, #[arg(long, short, help = "Use regtest. Equivalent to `--chain regtest`.")] pub(crate) regtest: bool, #[arg(long, short, help = "Use signet. Equivalent to `--chain signet`.")] diff --git a/src/ord_api_client.rs b/src/ord_api_client.rs new file mode 100644 index 0000000000..b423243ff7 --- /dev/null +++ b/src/ord_api_client.rs @@ -0,0 +1,51 @@ +use anyhow::Context; +use reqwest::Client; + +use crate::api::Inscription; +use crate::settings::Settings; +use crate::InscriptionId; + +pub struct OrdApiClient { + ord_api_url: String, + client: Client, +} + +impl OrdApiClient { + pub fn run(settings: &Settings) -> anyhow::Result { + let ord_api_url = settings + .ord_api_url() + .context("ord api url must be defined")? + .to_owned(); + + let client = Client::builder() + .timeout(std::time::Duration::from_secs(30)) + // TODO add retries + // .timeout_retry_times(5) + // .timeout_retry_strategy(ExponentialBackoff) + .build()?; + + Ok(OrdApiClient { + ord_api_url, + client, + }) + } + + pub async fn fetch_inscription_details( + &self, + inscription_id: &InscriptionId, + ) -> Result { + let response = self + .client + .get(format!( + "{}/inscription/{}", + self.ord_api_url, inscription_id + )) + .header("Accept", "application/json") + .send() + .await? + .error_for_status()? + .json::() + .await?; + Ok(response) + } +} diff --git a/src/settings.rs b/src/settings.rs index 0f93fbdf1e..3671eb20f0 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -31,6 +31,7 @@ pub struct Settings { rabbitmq_username: Option, rabbitmq_password: Option, rabbitmq_exchange: Option, + ord_api_url: Option, } impl Settings { @@ -149,6 +150,7 @@ impl Settings { rabbitmq_username: self.rabbitmq_username.or(source.rabbitmq_username), rabbitmq_password: self.rabbitmq_password.or(source.rabbitmq_password), rabbitmq_exchange: self.rabbitmq_exchange.or(source.rabbitmq_exchange), + ord_api_url: self.ord_api_url.or(source.ord_api_url), } } @@ -187,6 +189,7 @@ impl Settings { rabbitmq_username: options.rabbitmq_username, rabbitmq_password: options.rabbitmq_password, rabbitmq_exchange: options.rabbitmq_exchange, + ord_api_url: options.ord_api_url, } } @@ -269,6 +272,7 @@ impl Settings { rabbitmq_username: get_string("RMQ_USERNAME"), rabbitmq_password: get_string("RMQ_PASSWORD"), rabbitmq_exchange: get_string("RMQ_EXCHANGE"), + ord_api_url: get_string("ORD_API_URL"), }) } @@ -302,6 +306,7 @@ impl Settings { rabbitmq_username: None, rabbitmq_password: None, rabbitmq_exchange: None, + ord_api_url: None, } } @@ -385,6 +390,7 @@ impl Settings { rabbitmq_username: self.rabbitmq_username, rabbitmq_password: self.rabbitmq_password, rabbitmq_exchange: self.rabbitmq_exchange, + ord_api_url: self.ord_api_url, }) } @@ -444,7 +450,7 @@ impl Settings { "regtest" => Chain::Regtest, "signet" => Chain::Signet, other => bail!("Bitcoin RPC server on unknown chain: {other}"), - } + }; } Err(bitcoincore_rpc::Error::JsonRpc(bitcoincore_rpc::jsonrpc::Error::Rpc(err))) if err.code == -28 => {} @@ -589,6 +595,10 @@ impl Settings { Some(format!("amqp://{}:{}@{}", user, pass, url)) } + + pub fn ord_api_url(&self) -> Option<&str> { + self.ord_api_url.as_deref() + } } #[cfg(test)] @@ -1040,10 +1050,11 @@ mod tests { ("RMQ_USERNAME", "rmq username"), ("RMQ_PASSWORD", "rmq password"), ("RMQ_EXCHANGE", "rmq exchange"), + ("ORD_API_URL", "http://127.0.0.1:8080"), ] - .into_iter() - .map(|(key, value)| (key.into(), value.into())) - .collect::>(); + .into_iter() + .map(|(key, value)| (key.into(), value.into())) + .collect::>(); pretty_assert_eq!( Settings::from_env(env).unwrap(), @@ -1087,6 +1098,7 @@ mod tests { rabbitmq_username: Some("rmq username".into()), rabbitmq_password: Some("rmq password".into()), rabbitmq_exchange: Some("rmq exchange".into()), + ord_api_url: Some("http://127.0.0.1:8080".into()), } ); } @@ -1123,6 +1135,7 @@ mod tests { "--rabbitmq-username=rmq username", "--rabbitmq-password=rmq password", "--rabbitmq-exchange=rmq exchange", + "--ord-api-url=http://127.0.0.1:8080", ]) .unwrap() ), @@ -1155,6 +1168,7 @@ mod tests { rabbitmq_username: Some("rmq username".into()), rabbitmq_password: Some("rmq password".into()), rabbitmq_exchange: Some("rmq exchange".into()), + ord_api_url: Some("http://127.0.0.1:8080".into()), } ); } diff --git a/src/subcommand.rs b/src/subcommand.rs index a75bf4a2e1..34e55e8831 100644 --- a/src/subcommand.rs +++ b/src/subcommand.rs @@ -1,4 +1,5 @@ use crate::event_publisher::EventPublisher; +use crate::ord_api_client::OrdApiClient; use super::*; @@ -89,9 +90,9 @@ impl Subcommand { server.run(settings, index, handle) } Self::EventConsumer(event_consumer) => { - let index = Arc::new(Index::open(&settings)?); - event_consumer.run(&settings, index) - }, + let ord_api_client = OrdApiClient::run(&settings)?; + event_consumer.run(&settings, ord_api_client) + } Self::Settings => settings::run(settings), Self::Subsidy(subsidy) => subsidy.run(), Self::Supply => supply::run(), From 29ff371b41d3d0b53851417e2a5e2aa2005d981a Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 26 Apr 2024 16:30:40 +0200 Subject: [PATCH 019/103] Persist to db --- Cargo.toml | 2 +- db/config.sql | 16 ++++++++ .../20240431000000_ordinals/migration.sql | 23 +++++++++++ docker-compose.yaml | 40 +++++++++++++++++++ src/event_consumer.rs | 31 ++++++++++++-- src/lib.rs | 2 + src/options.rs | 2 - src/ord_api_client.rs | 9 +---- src/ord_db_client.rs | 28 +++++++++++++ src/settings.rs | 13 ------ src/subcommand.rs | 4 +- 11 files changed, 140 insertions(+), 30 deletions(-) create mode 100644 db/config.sql create mode 100644 db/migrations/20240431000000_ordinals/migration.sql create mode 100644 docker-compose.yaml create mode 100644 src/ord_db_client.rs diff --git a/Cargo.toml b/Cargo.toml index 0b7eadbad7..4deaabc550 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,7 @@ serde_json = { version = "1.0.81", features = ["preserve_order"] } serde_with = "3.7.0" serde_yaml = "0.9.17" sha3 = "0.10.8" -sqlx = { version = "0.7", features = [ "runtime-tokio", "tls-rustls" , "postgres" ] } +sqlx = { version = "0.7", features = ["runtime-tokio", "tls-rustls", "postgres"] } sysinfo = "0.30.3" tempfile = "3.2.0" tokio = { version = "1.17.0", features = ["rt-multi-thread"] } diff --git a/db/config.sql b/db/config.sql new file mode 100644 index 0000000000..e7908988e6 --- /dev/null +++ b/db/config.sql @@ -0,0 +1,16 @@ +/* Create database */ +DROP DATABASE IF EXISTS ordinals; + +DO $$ +BEGIN + IF NOT EXISTS(SELECT FROM pg_catalog.pg_roles WHERE rolname = 'backend') THEN + -- Create role with password and grant the appropriate permissions to that role +CREATE ROLE backend WITH ENCRYPTED PASSWORD 'looks-backend' LOGIN; +ALTER ROLE backend CREATEDB; +END IF; +END +$$; + +CREATE DATABASE ordinals WITH OWNER backend; + +GRANT ALL PRIVILEGES ON DATABASE ordinals to backend; \ No newline at end of file diff --git a/db/migrations/20240431000000_ordinals/migration.sql b/db/migrations/20240431000000_ordinals/migration.sql new file mode 100644 index 0000000000..fd04e82985 --- /dev/null +++ b/db/migrations/20240431000000_ordinals/migration.sql @@ -0,0 +1,23 @@ +create table inscriptions +( + id bigserial + primary key, + genesis_id text not null, --TODO tx_id + index, should be unique + number bigint not null --TODO jubilee number, uniqueness depends on indexer implementation + constraint inscriptions_number_unique + unique, + sat_ordinal numeric not null, + sat_rarity text not null, + sat_coinbase_height bigint not null, + mime_type text not null, --TODO not sure we need it + content_type text not null, + content_length bigint not null, --TODO not sure we need it + content bytea not null, --TODO should be moved to S3 + fee numeric not null, --TODO mint fee should belong to event? + curse_type text, + updated_at timestamp with time zone default now() not null, + recursive boolean default false, + classic_number bigint, + metadata text, --TODO needs structuring, currently dump of whatever + parent text --genesis_id of another inscription, used to create "collections" +); diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000000..1f9ace4ab1 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,40 @@ +version: '3.8' +services: + + db: + image: "postgres:15.3" + volumes: + - ./db/volume:/var/lib/postgresql/data + ports: + - "55432:5432" + environment: + POSTGRES_USER: backend + POSTGRES_PASSWORD: looks-backend + POSTGRES_DB: ordinals + POSTGRES_PORT: 5432 + + db-setup: + image: postgres:15.3 + depends_on: + - db + command: > + bash -c ' + until pg_isready -h db -U $$POSTGRES_USER; do + echo "Waiting for PostgreSQL to start..."; + sleep 1; + done; + echo "Applying config.sql to set up the database..."; + psql -h db -U $$POSTGRES_USER -d postgres -q -f /db/config.sql; + echo "Applying migrations..."; + for file in /db/migrations/*/*.sql; do + echo "Applying $$file"; + psql -h db -U $$POSTGRES_USER -d $$POSTGRES_DB -q -f $$file; + done; + ' + environment: + PGOPTIONS: '--client-min-messages=error' + POSTGRES_USER: backend + POSTGRES_DB: ordinals + PGPASSWORD: looks-backend + volumes: + - "./db:/db" diff --git a/src/event_consumer.rs b/src/event_consumer.rs index 27c69d1523..eb7f5c71d4 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -1,25 +1,30 @@ use anyhow::Context; use clap::Parser; use futures::StreamExt; -use lapin::{options::*, types::FieldTable, Connection, ConnectionProperties}; +use lapin::{Connection, ConnectionProperties, options::*, types::FieldTable}; use tokio::runtime::Runtime; use ordinals::SatPoint; use crate::index::event::Event; +use crate::InscriptionId; use crate::ord_api_client::OrdApiClient; +use crate::ord_db_client::OrdDbClient; use crate::settings::Settings; use crate::subcommand::SubcommandResult; -use crate::InscriptionId; #[derive(Debug, Parser, Clone)] pub struct EventConsumer { #[arg(long, help = "RMQ queue to consume index events.")] pub(crate) rabbitmq_queue: Option, + #[arg(long, help = "Ord api url to fetch inscriptions.")] + pub(crate) ord_api_url: Option, + #[arg(long, help = "DB url to persist inscriptions.")] + pub(crate) database_url: Option, } impl EventConsumer { - pub fn run(self, settings: &Settings, ord_api_client: OrdApiClient) -> SubcommandResult { + pub fn run(self, settings: &Settings) -> SubcommandResult { Runtime::new()?.block_on(async { let addr = settings .rabbitmq_addr() @@ -54,6 +59,18 @@ impl EventConsumer { .await .expect("creates rmq consumer"); + let ord_api_url = self + .ord_api_url + .clone() + .context("ord api url must be defined")?; + let ord_api_client = OrdApiClient::run(ord_api_url)?; + + let database_url = self + .database_url + .as_deref() + .context("db url must be defined")?; + let ord_db_client = OrdDbClient::run(database_url).await?; + log::info!("Starting to consume messages from {}", queue); while let Some(delivery) = consumer.next().await { match delivery { @@ -62,7 +79,7 @@ impl EventConsumer { match event { Ok(event) => { self - .handle_event(&ord_api_client, &event) + .handle_event(&ord_api_client, &ord_db_client, &event) .await .expect("confirms rmq msg processed"); delivery @@ -91,6 +108,7 @@ impl EventConsumer { async fn handle_event( &self, ord_api_client: &OrdApiClient, + ord_db_client: &OrdDbClient, event: &Event, ) -> Result<(), anyhow::Error> { match event { @@ -105,6 +123,7 @@ impl EventConsumer { self .handle_inscription_created( ord_api_client, + ord_db_client, block_height, charms, inscription_id, @@ -124,6 +143,7 @@ impl EventConsumer { self .handle_inscription_transferred( ord_api_client, + ord_db_client, block_height, inscription_id, new_location, @@ -139,6 +159,7 @@ impl EventConsumer { async fn handle_inscription_created( &self, ord_api_client: &OrdApiClient, + ord_db_client: &OrdDbClient, block_height: &u32, charms: &u16, inscription_id: &InscriptionId, @@ -153,6 +174,7 @@ impl EventConsumer { { Ok(inscription) => { log::info!("Received inscription detailed response: {:?}", inscription); + ord_db_client.save_inscription(inscription).await?; } Err(e) => { log::error!("Failed to fetch: {:?}", e); @@ -164,6 +186,7 @@ impl EventConsumer { async fn handle_inscription_transferred( &self, ord_api_client: &OrdApiClient, + ord_db_client: &OrdDbClient, block_height: &u32, inscription_id: &InscriptionId, new_location: &SatPoint, diff --git a/src/lib.rs b/src/lib.rs index 418914b4d6..93fdebc8e3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -114,6 +114,8 @@ mod macros; mod object; pub mod options; pub mod ord_api_client; + +pub mod ord_db_client; pub mod outgoing; mod re; mod representation; diff --git a/src/options.rs b/src/options.rs index 69921fcf0e..5db2c33b22 100644 --- a/src/options.rs +++ b/src/options.rs @@ -90,8 +90,6 @@ pub struct Options { pub(crate) rabbitmq_password: Option, #[arg(long, help = "RMQ exchange to publish index events.")] pub(crate) rabbitmq_exchange: Option, - #[arg(long, help = "Ord api url.")] - pub(crate) ord_api_url: Option, #[arg(long, short, help = "Use regtest. Equivalent to `--chain regtest`.")] pub(crate) regtest: bool, #[arg(long, short, help = "Use signet. Equivalent to `--chain signet`.")] diff --git a/src/ord_api_client.rs b/src/ord_api_client.rs index b423243ff7..9664c4fb23 100644 --- a/src/ord_api_client.rs +++ b/src/ord_api_client.rs @@ -1,8 +1,6 @@ -use anyhow::Context; use reqwest::Client; use crate::api::Inscription; -use crate::settings::Settings; use crate::InscriptionId; pub struct OrdApiClient { @@ -11,12 +9,7 @@ pub struct OrdApiClient { } impl OrdApiClient { - pub fn run(settings: &Settings) -> anyhow::Result { - let ord_api_url = settings - .ord_api_url() - .context("ord api url must be defined")? - .to_owned(); - + pub fn run(ord_api_url: String) -> anyhow::Result { let client = Client::builder() .timeout(std::time::Duration::from_secs(30)) // TODO add retries diff --git a/src/ord_db_client.rs b/src/ord_db_client.rs new file mode 100644 index 0000000000..78955e4189 --- /dev/null +++ b/src/ord_db_client.rs @@ -0,0 +1,28 @@ +use sqlx::PgPool; + +use crate::api::Inscription; + +pub struct OrdDbClient { + pool: PgPool, +} + +impl OrdDbClient { + pub async fn run(database_url: &str) -> anyhow::Result { + let pool = PgPool::connect(database_url) + .await + .expect("connects to db ok"); + + Ok(OrdDbClient { pool }) + } + + pub async fn save_inscription(&self, inscription: Inscription) -> Result<(), sqlx::Error> { + log::info!("Saving inscription detailed response: {:?}", inscription); + // let query = sqlx::query!( + // "INSERT INTO inscriptions (genesis_id, number, sat_ordinal, sat_rarity, sat_coinbase_height, mime_type, content_type, content_length, content, fee, curse_type, classic_number, metadata, parent) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + // ?? + // ) + // .execute(&self.pool) + // .await?; + Ok(()) + } +} diff --git a/src/settings.rs b/src/settings.rs index 3671eb20f0..1466662896 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -31,7 +31,6 @@ pub struct Settings { rabbitmq_username: Option, rabbitmq_password: Option, rabbitmq_exchange: Option, - ord_api_url: Option, } impl Settings { @@ -150,7 +149,6 @@ impl Settings { rabbitmq_username: self.rabbitmq_username.or(source.rabbitmq_username), rabbitmq_password: self.rabbitmq_password.or(source.rabbitmq_password), rabbitmq_exchange: self.rabbitmq_exchange.or(source.rabbitmq_exchange), - ord_api_url: self.ord_api_url.or(source.ord_api_url), } } @@ -189,7 +187,6 @@ impl Settings { rabbitmq_username: options.rabbitmq_username, rabbitmq_password: options.rabbitmq_password, rabbitmq_exchange: options.rabbitmq_exchange, - ord_api_url: options.ord_api_url, } } @@ -272,7 +269,6 @@ impl Settings { rabbitmq_username: get_string("RMQ_USERNAME"), rabbitmq_password: get_string("RMQ_PASSWORD"), rabbitmq_exchange: get_string("RMQ_EXCHANGE"), - ord_api_url: get_string("ORD_API_URL"), }) } @@ -306,7 +302,6 @@ impl Settings { rabbitmq_username: None, rabbitmq_password: None, rabbitmq_exchange: None, - ord_api_url: None, } } @@ -390,7 +385,6 @@ impl Settings { rabbitmq_username: self.rabbitmq_username, rabbitmq_password: self.rabbitmq_password, rabbitmq_exchange: self.rabbitmq_exchange, - ord_api_url: self.ord_api_url, }) } @@ -596,9 +590,6 @@ impl Settings { Some(format!("amqp://{}:{}@{}", user, pass, url)) } - pub fn ord_api_url(&self) -> Option<&str> { - self.ord_api_url.as_deref() - } } #[cfg(test)] @@ -1050,7 +1041,6 @@ mod tests { ("RMQ_USERNAME", "rmq username"), ("RMQ_PASSWORD", "rmq password"), ("RMQ_EXCHANGE", "rmq exchange"), - ("ORD_API_URL", "http://127.0.0.1:8080"), ] .into_iter() .map(|(key, value)| (key.into(), value.into())) @@ -1098,7 +1088,6 @@ mod tests { rabbitmq_username: Some("rmq username".into()), rabbitmq_password: Some("rmq password".into()), rabbitmq_exchange: Some("rmq exchange".into()), - ord_api_url: Some("http://127.0.0.1:8080".into()), } ); } @@ -1135,7 +1124,6 @@ mod tests { "--rabbitmq-username=rmq username", "--rabbitmq-password=rmq password", "--rabbitmq-exchange=rmq exchange", - "--ord-api-url=http://127.0.0.1:8080", ]) .unwrap() ), @@ -1168,7 +1156,6 @@ mod tests { rabbitmq_username: Some("rmq username".into()), rabbitmq_password: Some("rmq password".into()), rabbitmq_exchange: Some("rmq exchange".into()), - ord_api_url: Some("http://127.0.0.1:8080".into()), } ); } diff --git a/src/subcommand.rs b/src/subcommand.rs index 34e55e8831..4f181c980b 100644 --- a/src/subcommand.rs +++ b/src/subcommand.rs @@ -1,5 +1,6 @@ use crate::event_publisher::EventPublisher; use crate::ord_api_client::OrdApiClient; +use crate::ord_db_client::OrdDbClient; use super::*; @@ -90,8 +91,7 @@ impl Subcommand { server.run(settings, index, handle) } Self::EventConsumer(event_consumer) => { - let ord_api_client = OrdApiClient::run(&settings)?; - event_consumer.run(&settings, ord_api_client) + event_consumer.run(&settings) } Self::Settings => settings::run(settings), Self::Subsidy(subsidy) => subsidy.run(), From c5fe810aa8ef4c11b164423065c07e627d7eb339 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 26 Apr 2024 16:37:38 +0200 Subject: [PATCH 020/103] Persist to db --- src/event_consumer.rs | 4 +-- src/ord_db_client.rs | 57 ++++++++++++++++++++++++++++++++++++++----- src/settings.rs | 1 - src/subcommand.rs | 4 +-- 4 files changed, 54 insertions(+), 12 deletions(-) diff --git a/src/event_consumer.rs b/src/event_consumer.rs index eb7f5c71d4..0999d31f9f 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -1,17 +1,17 @@ use anyhow::Context; use clap::Parser; use futures::StreamExt; -use lapin::{Connection, ConnectionProperties, options::*, types::FieldTable}; +use lapin::{options::*, types::FieldTable, Connection, ConnectionProperties}; use tokio::runtime::Runtime; use ordinals::SatPoint; use crate::index::event::Event; -use crate::InscriptionId; use crate::ord_api_client::OrdApiClient; use crate::ord_db_client::OrdDbClient; use crate::settings::Settings; use crate::subcommand::SubcommandResult; +use crate::InscriptionId; #[derive(Debug, Parser, Clone)] pub struct EventConsumer { diff --git a/src/ord_db_client.rs b/src/ord_db_client.rs index 78955e4189..543e2bc26d 100644 --- a/src/ord_db_client.rs +++ b/src/ord_db_client.rs @@ -17,12 +17,57 @@ impl OrdDbClient { pub async fn save_inscription(&self, inscription: Inscription) -> Result<(), sqlx::Error> { log::info!("Saving inscription detailed response: {:?}", inscription); - // let query = sqlx::query!( - // "INSERT INTO inscriptions (genesis_id, number, sat_ordinal, sat_rarity, sat_coinbase_height, mime_type, content_type, content_length, content, fee, curse_type, classic_number, metadata, parent) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - // ?? - // ) - // .execute(&self.pool) - // .await?; + + let genesis_id = format!("{}i{}", inscription.id.txid, inscription.id.index); + let parents = inscription + .parents + .iter() + .map(|pid| format!("{}i{}", pid.txid, pid.index)) + .collect::>() + .join(","); + + let sql = " + INSERT INTO inscriptions ( + genesis_id, + number, + sat_ordinal, + sat_rarity, + sat_coinbase_height, + mime_type, + content_type, + content_length, + content, + fee, + curse_type, + classic_number, + metadata, + parent + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) + "; + + sqlx::query(sql) + .bind(genesis_id) + .bind(inscription.number as i64) + .bind(inscription.sat.map_or(0, |s| s.0 as i64)) + .bind("") + .bind(inscription.height as i64) + .bind( + inscription + .effective_content_type + .clone() + .unwrap_or_default(), + ) + .bind(inscription.content_type.clone().unwrap_or_default()) + .bind(inscription.content_length.map_or(0, |len| len as i64)) + .bind(vec![] as Vec) + .bind(inscription.fee as i64) + .bind("") + .bind(0) + .bind("") + .bind(parents) + .execute(&self.pool) + .await?; + Ok(()) } } diff --git a/src/settings.rs b/src/settings.rs index 1466662896..1521dee238 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -589,7 +589,6 @@ impl Settings { Some(format!("amqp://{}:{}@{}", user, pass, url)) } - } #[cfg(test)] diff --git a/src/subcommand.rs b/src/subcommand.rs index 4f181c980b..30d4d9d4f6 100644 --- a/src/subcommand.rs +++ b/src/subcommand.rs @@ -90,9 +90,7 @@ impl Subcommand { LISTENERS.lock().unwrap().push(handle.clone()); server.run(settings, index, handle) } - Self::EventConsumer(event_consumer) => { - event_consumer.run(&settings) - } + Self::EventConsumer(event_consumer) => event_consumer.run(&settings), Self::Settings => settings::run(settings), Self::Subsidy(subsidy) => subsidy.run(), Self::Supply => supply::run(), From a8408a6c22fb88307a6ee54fb0c85fb49d4fae7b Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 26 Apr 2024 16:38:42 +0200 Subject: [PATCH 021/103] Add todo --- src/subcommand/server.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/subcommand/server.rs b/src/subcommand/server.rs index d67e678b33..93e1ca471e 100644 --- a/src/subcommand/server.rs +++ b/src/subcommand/server.rs @@ -1520,6 +1520,7 @@ impl Server { }) } + // TODO need to enhance this api to return all necessary data async fn inscription( Extension(server_config): Extension>, Extension(index): Extension>, From 8492288528385e346dc7cef0d1d06f34181148f5 Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 1 May 2024 11:13:45 +0200 Subject: [PATCH 022/103] Persist raw ord events --- .../20240431000000_ordinals/migration.sql | 32 +++---- src/event_consumer.rs | 87 ++----------------- src/ord_db_client.rs | 81 +++++++---------- 3 files changed, 47 insertions(+), 153 deletions(-) diff --git a/db/migrations/20240431000000_ordinals/migration.sql b/db/migrations/20240431000000_ordinals/migration.sql index fd04e82985..1431587e3b 100644 --- a/db/migrations/20240431000000_ordinals/migration.sql +++ b/db/migrations/20240431000000_ordinals/migration.sql @@ -1,23 +1,13 @@ -create table inscriptions +CREATE TABLE events ( - id bigserial - primary key, - genesis_id text not null, --TODO tx_id + index, should be unique - number bigint not null --TODO jubilee number, uniqueness depends on indexer implementation - constraint inscriptions_number_unique - unique, - sat_ordinal numeric not null, - sat_rarity text not null, - sat_coinbase_height bigint not null, - mime_type text not null, --TODO not sure we need it - content_type text not null, - content_length bigint not null, --TODO not sure we need it - content bytea not null, --TODO should be moved to S3 - fee numeric not null, --TODO mint fee should belong to event? - curse_type text, - updated_at timestamp with time zone default now() not null, - recursive boolean default false, - classic_number bigint, - metadata text, --TODO needs structuring, currently dump of whatever - parent text --genesis_id of another inscription, used to create "collections" + id SERIAL PRIMARY KEY, + type_id SMALLINT NOT NULL, --1,InscriptionCreated;2,InscriptionTransferred + block_height BIGINT NOT NULL, + inscription_id TEXT NOT NULL, + location TEXT, -- Will hold either 'location' or 'new_location' based on type + old_location TEXT, -- Only used for InscriptionTransferred + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP ); + +CREATE INDEX idx_events_block_height ON events (block_height); +CREATE INDEX idx_events_inscription_id ON events (inscription_id); diff --git a/src/event_consumer.rs b/src/event_consumer.rs index 0999d31f9f..c923293ddd 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -1,24 +1,18 @@ use anyhow::Context; use clap::Parser; use futures::StreamExt; -use lapin::{options::*, types::FieldTable, Connection, ConnectionProperties}; +use lapin::{Connection, ConnectionProperties, options::*, types::FieldTable}; use tokio::runtime::Runtime; -use ordinals::SatPoint; - use crate::index::event::Event; -use crate::ord_api_client::OrdApiClient; use crate::ord_db_client::OrdDbClient; use crate::settings::Settings; use crate::subcommand::SubcommandResult; -use crate::InscriptionId; #[derive(Debug, Parser, Clone)] pub struct EventConsumer { #[arg(long, help = "RMQ queue to consume index events.")] pub(crate) rabbitmq_queue: Option, - #[arg(long, help = "Ord api url to fetch inscriptions.")] - pub(crate) ord_api_url: Option, #[arg(long, help = "DB url to persist inscriptions.")] pub(crate) database_url: Option, } @@ -59,12 +53,6 @@ impl EventConsumer { .await .expect("creates rmq consumer"); - let ord_api_url = self - .ord_api_url - .clone() - .context("ord api url must be defined")?; - let ord_api_client = OrdApiClient::run(ord_api_url)?; - let database_url = self .database_url .as_deref() @@ -79,7 +67,7 @@ impl EventConsumer { match event { Ok(event) => { self - .handle_event(&ord_api_client, &ord_db_client, &event) + .handle_event(&ord_db_client, &event) .await .expect("confirms rmq msg processed"); delivery @@ -107,7 +95,6 @@ impl EventConsumer { async fn handle_event( &self, - ord_api_client: &OrdApiClient, ord_db_client: &OrdDbClient, event: &Event, ) -> Result<(), anyhow::Error> { @@ -120,18 +107,8 @@ impl EventConsumer { parent_inscription_ids, sequence_number, } => { - self - .handle_inscription_created( - ord_api_client, - ord_db_client, - block_height, - charms, - inscription_id, - location, - parent_inscription_ids, - sequence_number, - ) - .await + ord_db_client.save_inscription_created(block_height, inscription_id, location).await?; + Ok(()) } Event::InscriptionTransferred { block_height, @@ -140,63 +117,11 @@ impl EventConsumer { old_location, sequence_number, } => { - self - .handle_inscription_transferred( - ord_api_client, - ord_db_client, - block_height, - inscription_id, - new_location, - old_location, - sequence_number, - ) - .await + ord_db_client.save_inscription_transferred(block_height, inscription_id, new_location, old_location).await?; + Ok(()) } _ => Ok(()), } } - async fn handle_inscription_created( - &self, - ord_api_client: &OrdApiClient, - ord_db_client: &OrdDbClient, - block_height: &u32, - charms: &u16, - inscription_id: &InscriptionId, - location: &Option, - parent_inscription_ids: &Vec, - sequence_number: &u32, - ) -> Result<(), anyhow::Error> { - log::info!("Received inscription created event: {:?}", inscription_id); - match ord_api_client - .fetch_inscription_details(inscription_id) - .await - { - Ok(inscription) => { - log::info!("Received inscription detailed response: {:?}", inscription); - ord_db_client.save_inscription(inscription).await?; - } - Err(e) => { - log::error!("Failed to fetch: {:?}", e); - } - }; - Ok(()) - } - - async fn handle_inscription_transferred( - &self, - ord_api_client: &OrdApiClient, - ord_db_client: &OrdDbClient, - block_height: &u32, - inscription_id: &InscriptionId, - new_location: &SatPoint, - old_location: &SatPoint, - sequence_number: &u32, - ) -> Result<(), anyhow::Error> { - log::info!( - "Received inscription transferred event: {:?}", - inscription_id - ); - Ok(()) - } } diff --git a/src/ord_db_client.rs b/src/ord_db_client.rs index 543e2bc26d..da52941bf7 100644 --- a/src/ord_db_client.rs +++ b/src/ord_db_client.rs @@ -1,6 +1,8 @@ use sqlx::PgPool; -use crate::api::Inscription; +use ordinals::SatPoint; + +use crate::InscriptionId; pub struct OrdDbClient { pool: PgPool, @@ -15,59 +17,36 @@ impl OrdDbClient { Ok(OrdDbClient { pool }) } - pub async fn save_inscription(&self, inscription: Inscription) -> Result<(), sqlx::Error> { - log::info!("Saving inscription detailed response: {:?}", inscription); - - let genesis_id = format!("{}i{}", inscription.id.txid, inscription.id.index); - let parents = inscription - .parents - .iter() - .map(|pid| format!("{}i{}", pid.txid, pid.index)) - .collect::>() - .join(","); - - let sql = " - INSERT INTO inscriptions ( - genesis_id, - number, - sat_ordinal, - sat_rarity, - sat_coinbase_height, - mime_type, - content_type, - content_length, - content, - fee, - curse_type, - classic_number, - metadata, - parent - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) - "; - - sqlx::query(sql) - .bind(genesis_id) - .bind(inscription.number as i64) - .bind(inscription.sat.map_or(0, |s| s.0 as i64)) - .bind("") - .bind(inscription.height as i64) - .bind( - inscription - .effective_content_type - .clone() - .unwrap_or_default(), - ) - .bind(inscription.content_type.clone().unwrap_or_default()) - .bind(inscription.content_length.map_or(0, |len| len as i64)) - .bind(vec![] as Vec) - .bind(inscription.fee as i64) - .bind("") - .bind(0) - .bind("") - .bind(parents) + pub async fn save_inscription_created(&self, + block_height: &u32, + inscription_id: &InscriptionId, + location: &Option) -> Result<(), sqlx::Error> { + let query = "INSERT INTO events (type_id, block_height, inscription_id, location) VALUES ($1, $2, $3, $4)"; + sqlx::query(query) + .bind(1_i32) // Type ID for InscriptionCreated + .bind(*block_height as i64) + .bind(inscription_id.to_string()) + .bind(location.map(|loc| loc.to_string())) .execute(&self.pool) .await?; + Ok(()) + } + pub async fn save_inscription_transferred(&self, + block_height: &u32, + inscription_id: &InscriptionId, + new_location: &SatPoint, + old_location: &SatPoint) -> Result<(), sqlx::Error> { + let query = "INSERT INTO events (type_id, block_height, inscription_id, location, old_location) VALUES ($1, $2, $3, $4, $5)"; + sqlx::query(query) + .bind(2_i32) // Type ID for InscriptionTransferred + .bind(*block_height as i64) + .bind(inscription_id.to_string()) + .bind(new_location.to_string()) + .bind(old_location.to_string()) + .execute(&self.pool) + .await?; Ok(()) } + } From 87f94b5b01d14945eb2384c95506540449cd319f Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 1 May 2024 13:21:24 +0200 Subject: [PATCH 023/103] Publish when blocks are committed --- rmqconfig.json | 25 ++++++++++++++++++++++++- src/event_publisher.rs | 16 ++++++++++++++-- src/index/event.rs | 3 +++ src/index/updater.rs | 8 +++++++- 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/rmqconfig.json b/rmqconfig.json index f667dbc4b4..1e30f1866a 100644 --- a/rmqconfig.json +++ b/rmqconfig.json @@ -6,6 +6,13 @@ "durable": true, "auto_delete": false, "arguments": {} + }, + { + "name": "blocks-q", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": {} } ], "exchanges": [ @@ -20,12 +27,28 @@ } ], "bindings": [ + { + "source": "ord-tx", + "vhost": "/", + "destination": "blocks-q", + "destination_type": "queue", + "routing_key": "BlockCommitted", + "arguments": {} + }, + { + "source": "ord-tx", + "vhost": "/", + "destination": "ord-q", + "destination_type": "queue", + "routing_key": "InscriptionCreated", + "arguments": {} + }, { "source": "ord-tx", "vhost": "/", "destination": "ord-q", "destination_type": "queue", - "routing_key": "", + "routing_key": "InscriptionTransferred", "arguments": {} } ] diff --git a/src/event_publisher.rs b/src/event_publisher.rs index da0e3aeb30..0fbd307c0d 100644 --- a/src/event_publisher.rs +++ b/src/event_publisher.rs @@ -1,6 +1,6 @@ use anyhow::{Context, Result}; +use lapin::{BasicProperties, Connection, ConnectionProperties, options::BasicPublishOptions}; use lapin::options::ConfirmSelectOptions; -use lapin::{options::BasicPublishOptions, BasicProperties, Connection, ConnectionProperties}; use tokio::runtime::Runtime; use tokio::sync::mpsc; @@ -46,7 +46,7 @@ impl EventPublisher { let publish = channel .basic_publish( &exchange, - "", + EventPublisher::type_name(&event), BasicPublishOptions::default(), &message, BasicProperties::default(), @@ -63,4 +63,16 @@ impl EventPublisher { Ok(EventPublisher { sender: tx }) } + + fn type_name(event: &Event) -> &'static str { + match event { + Event::InscriptionCreated { .. } => "InscriptionCreated", + Event::InscriptionTransferred { .. } => "InscriptionTransferred", + Event::RuneBurned { .. } => "RuneBurned", + Event::RuneEtched { .. } => "RuneEtched", + Event::RuneMinted { .. } => "RuneMinted", + Event::RuneTransferred { .. } => "RuneTransferred", + Event::BlockCommitted { .. } => "BlockCommitted", + } + } } diff --git a/src/index/event.rs b/src/index/event.rs index b5a7b3511e..4309a48e40 100644 --- a/src/index/event.rs +++ b/src/index/event.rs @@ -41,4 +41,7 @@ pub enum Event { rune_id: RuneId, txid: Txid, }, + BlockCommitted { + block_height: u32, + }, } diff --git a/src/index/updater.rs b/src/index/updater.rs index 3d446f2eb0..0fc837d766 100644 --- a/src/index/updater.rs +++ b/src/index/updater.rs @@ -1,8 +1,8 @@ use { self::{inscription_updater::InscriptionUpdater, rune_updater::RuneUpdater}, - super::{fetcher::Fetcher, *}, futures::future::try_join_all, std::sync::mpsc, + super::{*, fetcher::Fetcher}, tokio::sync::mpsc::{error::TryRecvError, Receiver, Sender}, }; @@ -739,6 +739,12 @@ impl<'index> Updater<'index> { Reorg::update_savepoints(self.index, self.height)?; + if let Some(sender) = self.index.event_sender.as_ref() { + sender.blocking_send(Event::BlockCommitted { + block_height: self.height, + })?; + } + Ok(()) } } From 739aee39274aa107ca6830a3d167f982b477c6bf Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 1 May 2024 14:40:20 +0200 Subject: [PATCH 024/103] Add consumer for blocks-q --- Cargo.lock | 11 +++ Cargo.toml | 3 +- src/event_consumer.rs | 177 ++++++++++++++++++++++++------------------ src/ord_db_client.rs | 16 ++-- 4 files changed, 126 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fbc2044805..526d38e3f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2977,6 +2977,7 @@ dependencies = [ "ordinals", "pretty_assertions", "pulldown-cmark", + "rand", "redb", "regex", "reqwest", @@ -4179,6 +4180,15 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + [[package]] name = "signature" version = "2.2.0" @@ -4716,6 +4726,7 @@ dependencies = [ "mio", "num_cpus", "pin-project-lite", + "signal-hook-registry", "socket2 0.5.6", "tokio-macros", "windows-sys 0.48.0", diff --git a/Cargo.toml b/Cargo.toml index 4deaabc550..ff030f5cad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,11 +67,12 @@ sha3 = "0.10.8" sqlx = { version = "0.7", features = ["runtime-tokio", "tls-rustls", "postgres"] } sysinfo = "0.30.3" tempfile = "3.2.0" -tokio = { version = "1.17.0", features = ["rt-multi-thread"] } +tokio = { version = "1.17.0", features = ["rt-multi-thread", "signal"] } tokio-stream = "0.1.9" tokio-util = {version = "0.7.3", features = ["compat"] } tower-http = { version = "0.4.0", features = ["auth", "compression-br", "compression-gzip", "cors", "set-header"] } urlencoding = "2.1.3" +rand = "0.8.5" [dev-dependencies] criterion = "0.5.1" diff --git a/src/event_consumer.rs b/src/event_consumer.rs index c923293ddd..325287353a 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -1,7 +1,12 @@ +use std::sync::Arc; + use anyhow::Context; +use bitcoin::secp256k1::rand::distributions::Alphanumeric; +use chrono::Utc; use clap::Parser; use futures::StreamExt; use lapin::{Connection, ConnectionProperties, options::*, types::FieldTable}; +use rand::distributions::DistString; use tokio::runtime::Runtime; use crate::index::event::Event; @@ -11,8 +16,10 @@ use crate::subcommand::SubcommandResult; #[derive(Debug, Parser, Clone)] pub struct EventConsumer { - #[arg(long, help = "RMQ queue to consume index events.")] - pub(crate) rabbitmq_queue: Option, + #[arg(long, help = "RMQ queue to consume blocks.")] + pub(crate) blocks_queue: Option, + #[arg(long, help = "RMQ queue to consume inscription events.")] + pub(crate) inscriptions_queue: Option, #[arg(long, help = "DB url to persist inscriptions.")] pub(crate) database_url: Option, } @@ -24,11 +31,6 @@ impl EventConsumer { .rabbitmq_addr() .context("rabbitmq amqp credentials and url must be defined")?; - let queue = self - .rabbitmq_queue - .as_deref() - .context("rabbitmq queue path must be defined")?; - let conn = Connection::connect(&addr, ConnectionProperties::default()) .await .expect("connects to rabbitmq ok"); @@ -43,85 +45,110 @@ impl EventConsumer { .await .expect("enable msg confirms"); - let mut consumer = channel - .basic_consume( - &queue, - "lr-ord", //TODO pod name - BasicConsumeOptions::default(), - FieldTable::default(), - ) - .await - .expect("creates rmq consumer"); - let database_url = self .database_url .as_deref() .context("db url must be defined")?; - let ord_db_client = OrdDbClient::run(database_url).await?; - - log::info!("Starting to consume messages from {}", queue); - while let Some(delivery) = consumer.next().await { - match delivery { - Ok(delivery) => { - let event: Result = serde_json::from_slice(&delivery.data); - match event { - Ok(event) => { - self - .handle_event(&ord_db_client, &event) - .await - .expect("confirms rmq msg processed"); - delivery - .ack(BasicAckOptions::default()) - .await - .expect("confirms rmq msg received"); - } - Err(e) => { - log::error!("Error deserializing event: {}", e); - delivery - .reject(BasicRejectOptions { requeue: false }) - .await?; - } - } - } - Err(e) => { - log::error!("Error receiving delivery: {}", e); - } - } - } + let ord_db_client = Arc::new(OrdDbClient::run(database_url).await?); + + let blocks_queue = self.blocks_queue.as_deref().context("rabbitmq blocks queue path must be defined")?; + let blocks_queue_str = blocks_queue.to_string(); + let blocks_channel = channel.clone(); + let blocks_ord_db_client = Arc::clone(&ord_db_client); + let blocks_consumer_tag = Self::generate_consumer_tag(); + + let inscriptions_queue = self.inscriptions_queue.as_deref().context("rabbitmq inscriptions queue path must be defined")?; + let inscriptions_queue_str = inscriptions_queue.to_string(); + let inscriptions_channel = channel.clone(); + let inscriptions_ord_db_client = Arc::clone(&ord_db_client); + let inscriptions_consumer_tag = Self::generate_consumer_tag(); + + let blocks_consumer_handle = tokio::spawn(async move { + EventConsumer::consume_queue(blocks_channel, blocks_queue_str, blocks_consumer_tag, blocks_ord_db_client).await + }); + + let inscriptions_consumer_handle = tokio::spawn(async move { + EventConsumer::consume_queue(inscriptions_channel, inscriptions_queue_str, inscriptions_consumer_tag, inscriptions_ord_db_client).await + }); + + let _ = tokio::try_join!(blocks_consumer_handle, inscriptions_consumer_handle); + + // TODO handle shutdown? + // let mut sigterm = signal(SignalKind::terminate()).expect("Failed to bind signal handler"); + // sigterm.recv().await; + // log::info!("Termination signal received, shutting down."); Ok(None) }) } - async fn handle_event( - &self, - ord_db_client: &OrdDbClient, - event: &Event, - ) -> Result<(), anyhow::Error> { - match event { - Event::InscriptionCreated { - block_height, - charms, - inscription_id, - location, - parent_inscription_ids, - sequence_number, - } => { - ord_db_client.save_inscription_created(block_height, inscription_id, location).await?; - Ok(()) - } - Event::InscriptionTransferred { - block_height, - inscription_id, - new_location, - old_location, - sequence_number, - } => { - ord_db_client.save_inscription_transferred(block_height, inscription_id, new_location, old_location).await?; - Ok(()) + async fn consume_queue(channel: lapin::Channel, queue_name: String, consumer_tag: String, ord_db_client: Arc) -> Result<(), anyhow::Error> { + let mut consumer = channel + .basic_consume( + &queue_name, + consumer_tag.as_str(), + BasicConsumeOptions::default(), + FieldTable::default(), + ) + .await + .expect("creates rmq consumer"); + + log::info!("Starting to consume messages from {}", queue_name); + while let Some(delivery) = consumer.next().await { + match delivery { + Ok(delivery) => { + let event: Result = serde_json::from_slice(&delivery.data); + match event { + Ok(event) => { + match &event { + Event::BlockCommitted { + block_height, + } => { + ord_db_client.sync_blocks(block_height).await? + } + Event::InscriptionCreated { + block_height, + charms, + inscription_id, + location, + parent_inscription_ids, + sequence_number, + } => { + ord_db_client.save_inscription_created(block_height, inscription_id, location).await? + } + Event::InscriptionTransferred { + block_height, + inscription_id, + new_location, + old_location, + sequence_number, + } => { + ord_db_client.save_inscription_transferred(block_height, inscription_id, new_location, old_location).await?; + } + _ => { + log::warn!("Received an unhandled event type"); + } + } + delivery.ack(BasicAckOptions::default()).await?; + } + Err(e) => { + log::error!("Error deserializing event: {}", e); + delivery.reject(BasicRejectOptions { requeue: false }).await?; + } + } + } + Err(e) => { + log::error!("Error receiving delivery: {}", e); + } } - _ => Ok(()), } + + Ok(()) } + fn generate_consumer_tag() -> String { + // TODO get pod name from k8s? + let timestamp = Utc::now().format("%Y%m%d%H%M%S"); + format!("{}-{}-{}", "lr-ord", timestamp, Alphanumeric.sample_string(&mut rand::thread_rng(), 16)) + } } diff --git a/src/ord_db_client.rs b/src/ord_db_client.rs index da52941bf7..20c71eb609 100644 --- a/src/ord_db_client.rs +++ b/src/ord_db_client.rs @@ -9,14 +9,21 @@ pub struct OrdDbClient { } impl OrdDbClient { - pub async fn run(database_url: &str) -> anyhow::Result { - let pool = PgPool::connect(database_url) - .await - .expect("connects to db ok"); + pub async fn run(database_url: &str) -> anyhow::Result { + let pool = PgPool::connect(database_url).await?; + + // TODO handle shutdown? Ok(OrdDbClient { pool }) } + pub async fn sync_blocks(&self, + block_height: &u32) -> Result<(), sqlx::Error> { + log::info!("Block committed event {}", block_height); + // TODO consume all blocks from this to last consumed + Ok(()) + } + pub async fn save_inscription_created(&self, block_height: &u32, inscription_id: &InscriptionId, @@ -48,5 +55,4 @@ impl OrdDbClient { .await?; Ok(()) } - } From 0879aacb1435dea05cc3f4b6cfa65bcab46f4e9d Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 1 May 2024 15:18:09 +0200 Subject: [PATCH 025/103] Publish block range not single block --- src/event_consumer.rs | 5 +++-- src/index/event.rs | 3 ++- src/index/updater.rs | 9 +++++---- src/ord_db_client.rs | 5 +++-- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/event_consumer.rs b/src/event_consumer.rs index 325287353a..bef20ecb5a 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -102,9 +102,10 @@ impl EventConsumer { Ok(event) => { match &event { Event::BlockCommitted { - block_height, + from_height, + to_height, } => { - ord_db_client.sync_blocks(block_height).await? + ord_db_client.sync_blocks(from_height, to_height).await? } Event::InscriptionCreated { block_height, diff --git a/src/index/event.rs b/src/index/event.rs index 4309a48e40..6fd9f83be8 100644 --- a/src/index/event.rs +++ b/src/index/event.rs @@ -42,6 +42,7 @@ pub enum Event { txid: Txid, }, BlockCommitted { - block_height: u32, + from_height: u32, + to_height: u32, }, } diff --git a/src/index/updater.rs b/src/index/updater.rs index 0fc837d766..3c475cc16e 100644 --- a/src/index/updater.rs +++ b/src/index/updater.rs @@ -101,7 +101,7 @@ impl<'index> Updater<'index> { uncommitted += 1; if uncommitted == self.index.settings.commit_interval() { - self.commit(wtx, value_cache)?; + self.commit(wtx, value_cache, uncommitted)?; value_cache = HashMap::new(); uncommitted = 0; wtx = self.index.begin_write()?; @@ -140,7 +140,7 @@ impl<'index> Updater<'index> { } if uncommitted > 0 { - self.commit(wtx, value_cache)?; + self.commit(wtx, value_cache, uncommitted)?; } if let Some(progress_bar) = &mut progress_bar { @@ -696,7 +696,7 @@ impl<'index> Updater<'index> { Ok(()) } - fn commit(&mut self, wtx: WriteTransaction, value_cache: HashMap) -> Result { + fn commit(&mut self, wtx: WriteTransaction, value_cache: HashMap, uncommitted: usize) -> Result { log::info!( "Committing at block height {}, {} outputs traversed, {} in map, {} cached", self.height, @@ -741,7 +741,8 @@ impl<'index> Updater<'index> { if let Some(sender) = self.index.event_sender.as_ref() { sender.blocking_send(Event::BlockCommitted { - block_height: self.height, + from_height: self.height - uncommitted as u32, + to_height: self.height, })?; } diff --git a/src/ord_db_client.rs b/src/ord_db_client.rs index 20c71eb609..cb739afb41 100644 --- a/src/ord_db_client.rs +++ b/src/ord_db_client.rs @@ -18,8 +18,9 @@ impl OrdDbClient { } pub async fn sync_blocks(&self, - block_height: &u32) -> Result<(), sqlx::Error> { - log::info!("Block committed event {}", block_height); + from_height: &u32, + to_height: &u32) -> Result<(), sqlx::Error> { + log::info!("Blocks committed event from={from_height} (excluded), to={to_height} (included)"); // TODO consume all blocks from this to last consumed Ok(()) } From b5bddc3f409b671c8e6a5210106f8802eef3dc2f Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 1 May 2024 16:02:38 +0200 Subject: [PATCH 026/103] Move db out of ordclient for shutdown --- Cargo.lock | 245 +++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 1 + src/event_consumer.rs | 21 +++- src/ord_db_client.rs | 16 ++- 4 files changed, 267 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 526d38e3f9..712092e412 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1094,6 +1094,26 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "config" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7328b20597b53c2454f0b1919720c25c7339051c02b72b7e05409e00b14132be" +dependencies = [ + "async-trait", + "convert_case 0.6.0", + "json5", + "lazy_static", + "nom", + "pathdiff", + "ron", + "rust-ini", + "serde", + "serde_json", + "toml 0.8.12", + "yaml-rust", +] + [[package]] name = "console" version = "0.15.8" @@ -1113,12 +1133,41 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom", + "once_cell", + "tiny-keccak", +] + [[package]] name = "convert_case" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cookie-factory" version = "0.3.3" @@ -1445,7 +1494,7 @@ version = "0.99.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ - "convert_case", + "convert_case 0.4.0", "proc-macro2", "quote", "rustc_version", @@ -1520,6 +1569,15 @@ dependencies = [ "syn 2.0.60", ] +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + [[package]] name = "doc-comment" version = "0.3.3" @@ -2022,6 +2080,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" + [[package]] name = "hashbrown" version = "0.14.3" @@ -2364,6 +2428,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json5" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" +dependencies = [ + "pest", + "pest_derive", + "serde", +] + [[package]] name = "jsonrpc" version = "0.14.1" @@ -2525,6 +2600,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "linux-raw-sys" version = "0.3.8" @@ -2952,6 +3033,7 @@ dependencies = [ "ciborium", "clap", "colored", + "config", "criterion", "ctrlc", "dirs", @@ -3028,6 +3110,16 @@ dependencies = [ "serde_json", ] +[[package]] +name = "ordered-multimap" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ed8acf08e98e744e5384c8bc63ceb0364e68a6854187221c18df61c4797690e" +dependencies = [ + "dlv-list", + "hashbrown 0.13.2", +] + [[package]] name = "ordinals" version = "0.0.8" @@ -3123,6 +3215,12 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +[[package]] +name = "pathdiff" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" + [[package]] name = "pbkdf2" version = "0.12.2" @@ -3157,6 +3255,51 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pest" +version = "2.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "560131c633294438da9f7c4b08189194b20946c8274c6b9e38881a7874dc8ee8" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26293c9193fbca7b1a3bf9b79dc1e388e927e6cacaa78b4a3ab705a1d3d41459" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ec22af7d3fb470a85dd2ca96b7c577a1eb4ef6f1683a9fe9a8c16e136c04687" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.60", +] + +[[package]] +name = "pest_meta" +version = "2.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a240022f37c361ec1878d646fc5b7d7c4d28d5946e1a80ad5a7a4f4ca0bdcd" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + [[package]] name = "pin-project" version = "1.1.5" @@ -3372,7 +3515,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" dependencies = [ - "toml", + "toml 0.5.11", ] [[package]] @@ -3652,6 +3795,18 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "ron" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" +dependencies = [ + "base64 0.21.7", + "bitflags 2.5.0", + "serde", + "serde_derive", +] + [[package]] name = "rsa" version = "0.9.6" @@ -3718,6 +3873,16 @@ dependencies = [ "walkdir", ] +[[package]] +name = "rust-ini" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e2a3bcec1f113553ef1c88aae6c020a369d03d55b58de9869a0908930385091" +dependencies = [ + "cfg-if 1.0.0", + "ordered-multimap", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -4087,6 +4252,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -4689,6 +4863,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -4812,6 +4995,40 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3328d4f68a705b2a4498da1d580585d39a6510f98318a2cec3018a7ec61ddef" +dependencies = [ + "indexmap 2.2.6", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" @@ -4907,6 +5124,12 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "ucd-trie" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" + [[package]] name = "unicase" version = "2.7.0" @@ -5363,6 +5586,15 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +[[package]] +name = "winnow" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14b9415ee827af173ebb3f15f9083df5a122eb93572ec28741fb153356ea2578" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" @@ -5419,6 +5651,15 @@ dependencies = [ "time", ] +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "yansi" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index ff030f5cad..62a8b90a62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,6 +73,7 @@ tokio-util = {version = "0.7.3", features = ["compat"] } tower-http = { version = "0.4.0", features = ["auth", "compression-br", "compression-gzip", "cors", "set-header"] } urlencoding = "2.1.3" rand = "0.8.5" +config = "0.14.0" [dev-dependencies] criterion = "0.5.1" diff --git a/src/event_consumer.rs b/src/event_consumer.rs index bef20ecb5a..8ac0c57f62 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -7,6 +7,7 @@ use clap::Parser; use futures::StreamExt; use lapin::{Connection, ConnectionProperties, options::*, types::FieldTable}; use rand::distributions::DistString; +use sqlx::postgres::PgPoolOptions; use tokio::runtime::Runtime; use crate::index::event::Event; @@ -49,7 +50,12 @@ impl EventConsumer { .database_url .as_deref() .context("db url must be defined")?; - let ord_db_client = Arc::new(OrdDbClient::run(database_url).await?); + let pool = PgPoolOptions::new() + .max_connections(5) + .connect(database_url) + .await?; + let shared_pool = Arc::new(pool); + let ord_db_client = Arc::new(OrdDbClient::new(shared_pool.clone())); let blocks_queue = self.blocks_queue.as_deref().context("rabbitmq blocks queue path must be defined")?; let blocks_queue_str = blocks_queue.to_string(); @@ -73,10 +79,15 @@ impl EventConsumer { let _ = tokio::try_join!(blocks_consumer_handle, inscriptions_consumer_handle); - // TODO handle shutdown? - // let mut sigterm = signal(SignalKind::terminate()).expect("Failed to bind signal handler"); - // sigterm.recv().await; - // log::info!("Termination signal received, shutting down."); + // let mut sigterm = signal(SignalKind::terminate())?; + // tokio::select! { + // _ = sigterm.recv() => { + // log::info!("Signal received, shutting down."); + // shared_pool.close().await; + // blocks_channel.close(200, "Closing channel").await?; + // inscriptions_channel.close(200, "Closing channel").await?; + // }, + // } Ok(None) }) diff --git a/src/ord_db_client.rs b/src/ord_db_client.rs index cb739afb41..1727b0cdd8 100644 --- a/src/ord_db_client.rs +++ b/src/ord_db_client.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use sqlx::PgPool; use ordinals::SatPoint; @@ -5,16 +7,12 @@ use ordinals::SatPoint; use crate::InscriptionId; pub struct OrdDbClient { - pool: PgPool, + pool: Arc, } impl OrdDbClient { - pub async fn run(database_url: &str) -> anyhow::Result { - let pool = PgPool::connect(database_url).await?; - - // TODO handle shutdown? - - Ok(OrdDbClient { pool }) + pub fn new(pool: Arc) -> Self { + Self { pool } } pub async fn sync_blocks(&self, @@ -35,7 +33,7 @@ impl OrdDbClient { .bind(*block_height as i64) .bind(inscription_id.to_string()) .bind(location.map(|loc| loc.to_string())) - .execute(&self.pool) + .execute(&*self.pool) .await?; Ok(()) } @@ -52,7 +50,7 @@ impl OrdDbClient { .bind(inscription_id.to_string()) .bind(new_location.to_string()) .bind(old_location.to_string()) - .execute(&self.pool) + .execute(&*self.pool) .await?; Ok(()) } From 5e7eb214e24a02b9564929d2f08e73808589fad6 Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 1 May 2024 16:34:06 +0200 Subject: [PATCH 027/103] Graceful shutdown --- src/event_consumer.rs | 131 +++++++++++++++++++++++++----------------- 1 file changed, 78 insertions(+), 53 deletions(-) diff --git a/src/event_consumer.rs b/src/event_consumer.rs index 8ac0c57f62..e97ec56557 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -9,6 +9,7 @@ use lapin::{Connection, ConnectionProperties, options::*, types::FieldTable}; use rand::distributions::DistString; use sqlx::postgres::PgPoolOptions; use tokio::runtime::Runtime; +use tokio::sync::oneshot; use crate::index::event::Event; use crate::ord_db_client::OrdDbClient; @@ -62,38 +63,41 @@ impl EventConsumer { let blocks_channel = channel.clone(); let blocks_ord_db_client = Arc::clone(&ord_db_client); let blocks_consumer_tag = Self::generate_consumer_tag(); + let (blocks_shutdown_tx, blocks_shutdown_rx) = oneshot::channel::<()>(); let inscriptions_queue = self.inscriptions_queue.as_deref().context("rabbitmq inscriptions queue path must be defined")?; let inscriptions_queue_str = inscriptions_queue.to_string(); let inscriptions_channel = channel.clone(); let inscriptions_ord_db_client = Arc::clone(&ord_db_client); let inscriptions_consumer_tag = Self::generate_consumer_tag(); + let (inscriptions_shutdown_tx, inscriptions_shutdown_rx) = oneshot::channel::<()>(); let blocks_consumer_handle = tokio::spawn(async move { - EventConsumer::consume_queue(blocks_channel, blocks_queue_str, blocks_consumer_tag, blocks_ord_db_client).await + EventConsumer::consume_queue(blocks_channel, blocks_queue_str, blocks_consumer_tag, blocks_ord_db_client, blocks_shutdown_rx).await }); let inscriptions_consumer_handle = tokio::spawn(async move { - EventConsumer::consume_queue(inscriptions_channel, inscriptions_queue_str, inscriptions_consumer_tag, inscriptions_ord_db_client).await + EventConsumer::consume_queue(inscriptions_channel, inscriptions_queue_str, inscriptions_consumer_tag, inscriptions_ord_db_client, inscriptions_shutdown_rx).await }); + let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + sigterm.recv().await; + let _ = blocks_shutdown_tx.send(()); + let _ = inscriptions_shutdown_tx.send(()); let _ = tokio::try_join!(blocks_consumer_handle, inscriptions_consumer_handle); - // let mut sigterm = signal(SignalKind::terminate())?; - // tokio::select! { - // _ = sigterm.recv() => { - // log::info!("Signal received, shutting down."); - // shared_pool.close().await; - // blocks_channel.close(200, "Closing channel").await?; - // inscriptions_channel.close(200, "Closing channel").await?; - // }, - // } + shared_pool.close().await; Ok(None) }) } - async fn consume_queue(channel: lapin::Channel, queue_name: String, consumer_tag: String, ord_db_client: Arc) -> Result<(), anyhow::Error> { + async fn consume_queue(channel: lapin::Channel, + queue_name: String, + consumer_tag: String, + ord_db_client: Arc, + mut shutdown_signal: oneshot::Receiver<()>, + ) -> Result<(), anyhow::Error> { let mut consumer = channel .basic_consume( &queue_name, @@ -105,49 +109,16 @@ impl EventConsumer { .expect("creates rmq consumer"); log::info!("Starting to consume messages from {}", queue_name); - while let Some(delivery) = consumer.next().await { - match delivery { + while let Some(result) = consumer.next().await { + match result { Ok(delivery) => { - let event: Result = serde_json::from_slice(&delivery.data); - match event { - Ok(event) => { - match &event { - Event::BlockCommitted { - from_height, - to_height, - } => { - ord_db_client.sync_blocks(from_height, to_height).await? - } - Event::InscriptionCreated { - block_height, - charms, - inscription_id, - location, - parent_inscription_ids, - sequence_number, - } => { - ord_db_client.save_inscription_created(block_height, inscription_id, location).await? - } - Event::InscriptionTransferred { - block_height, - inscription_id, - new_location, - old_location, - sequence_number, - } => { - ord_db_client.save_inscription_transferred(block_height, inscription_id, new_location, old_location).await?; - } - _ => { - log::warn!("Received an unhandled event type"); - } - } - delivery.ack(BasicAckOptions::default()).await?; + tokio::select! { + _ = EventConsumer::handle_delivery(delivery, &ord_db_client) => {}, + _ = &mut shutdown_signal => { + log::info!("Shutdown signal received, stopping consumer."); + break; + }, } - Err(e) => { - log::error!("Error deserializing event: {}", e); - delivery.reject(BasicRejectOptions { requeue: false }).await?; - } - } } Err(e) => { log::error!("Error receiving delivery: {}", e); @@ -155,6 +126,60 @@ impl EventConsumer { } } + log::info!("Closing consumer channel {}", queue_name); + channel.close(200, "Closing channel due to shutdown").await?; + + Ok(()) + } + + async fn handle_delivery(delivery: lapin::message::Delivery, + ord_db_client: &Arc) -> Result<(), anyhow::Error> { + let event: Result = serde_json::from_slice(&delivery.data); + match event { + Ok(event) => { + EventConsumer::process_event(event, ord_db_client).await?; + delivery.ack(BasicAckOptions::default()).await?; + } + Err(e) => { + log::error!("Failed to deserialize event, rejecting: {}", e); + delivery.reject(BasicRejectOptions { requeue: false }).await?; + } + } + + Ok(()) + } + + async fn process_event(event: Event, ord_db_client: &Arc) -> Result<(), anyhow::Error> { + match &event { + Event::BlockCommitted { + from_height, + to_height, + } => { + ord_db_client.sync_blocks(from_height, to_height).await? + } + Event::InscriptionCreated { + block_height, + charms, + inscription_id, + location, + parent_inscription_ids, + sequence_number, + } => { + ord_db_client.save_inscription_created(block_height, inscription_id, location).await? + } + Event::InscriptionTransferred { + block_height, + inscription_id, + new_location, + old_location, + sequence_number, + } => { + ord_db_client.save_inscription_transferred(block_height, inscription_id, new_location, old_location).await?; + } + _ => { + log::warn!("Received an unhandled event type"); + } + } Ok(()) } From 8d62582d84b61fc3776745771c918e4ed6d79b15 Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 1 May 2024 16:38:07 +0200 Subject: [PATCH 028/103] Reject message if processing fails --- src/event_consumer.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/event_consumer.rs b/src/event_consumer.rs index e97ec56557..4ad08467cf 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -137,8 +137,12 @@ impl EventConsumer { let event: Result = serde_json::from_slice(&delivery.data); match event { Ok(event) => { - EventConsumer::process_event(event, ord_db_client).await?; - delivery.ack(BasicAckOptions::default()).await?; + if let Err(err) = EventConsumer::process_event(event, ord_db_client).await { + log::error!("Failed to process event: {}", err); + delivery.reject(BasicRejectOptions { requeue: false }).await?; + } else { + delivery.ack(BasicAckOptions::default()).await?; + } } Err(e) => { log::error!("Failed to deserialize event, rejecting: {}", e); From e458be983602156552f6d8b26d70a0dece694ca6 Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 1 May 2024 16:46:25 +0200 Subject: [PATCH 029/103] Insert events only if they don't exist. fmt --- src/event_consumer.rs | 95 +++++++++++++++++++++++++++++------------- src/event_publisher.rs | 2 +- src/index/updater.rs | 9 +++- src/ord_db_client.rs | 42 ++++++++++++------- 4 files changed, 103 insertions(+), 45 deletions(-) diff --git a/src/event_consumer.rs b/src/event_consumer.rs index 4ad08467cf..7b41e625cc 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -5,7 +5,7 @@ use bitcoin::secp256k1::rand::distributions::Alphanumeric; use chrono::Utc; use clap::Parser; use futures::StreamExt; -use lapin::{Connection, ConnectionProperties, options::*, types::FieldTable}; +use lapin::{options::*, types::FieldTable, Connection, ConnectionProperties}; use rand::distributions::DistString; use sqlx::postgres::PgPoolOptions; use tokio::runtime::Runtime; @@ -58,14 +58,20 @@ impl EventConsumer { let shared_pool = Arc::new(pool); let ord_db_client = Arc::new(OrdDbClient::new(shared_pool.clone())); - let blocks_queue = self.blocks_queue.as_deref().context("rabbitmq blocks queue path must be defined")?; + let blocks_queue = self + .blocks_queue + .as_deref() + .context("rabbitmq blocks queue path must be defined")?; let blocks_queue_str = blocks_queue.to_string(); let blocks_channel = channel.clone(); let blocks_ord_db_client = Arc::clone(&ord_db_client); let blocks_consumer_tag = Self::generate_consumer_tag(); let (blocks_shutdown_tx, blocks_shutdown_rx) = oneshot::channel::<()>(); - let inscriptions_queue = self.inscriptions_queue.as_deref().context("rabbitmq inscriptions queue path must be defined")?; + let inscriptions_queue = self + .inscriptions_queue + .as_deref() + .context("rabbitmq inscriptions queue path must be defined")?; let inscriptions_queue_str = inscriptions_queue.to_string(); let inscriptions_channel = channel.clone(); let inscriptions_ord_db_client = Arc::clone(&ord_db_client); @@ -73,11 +79,25 @@ impl EventConsumer { let (inscriptions_shutdown_tx, inscriptions_shutdown_rx) = oneshot::channel::<()>(); let blocks_consumer_handle = tokio::spawn(async move { - EventConsumer::consume_queue(blocks_channel, blocks_queue_str, blocks_consumer_tag, blocks_ord_db_client, blocks_shutdown_rx).await + EventConsumer::consume_queue( + blocks_channel, + blocks_queue_str, + blocks_consumer_tag, + blocks_ord_db_client, + blocks_shutdown_rx, + ) + .await }); let inscriptions_consumer_handle = tokio::spawn(async move { - EventConsumer::consume_queue(inscriptions_channel, inscriptions_queue_str, inscriptions_consumer_tag, inscriptions_ord_db_client, inscriptions_shutdown_rx).await + EventConsumer::consume_queue( + inscriptions_channel, + inscriptions_queue_str, + inscriptions_consumer_tag, + inscriptions_ord_db_client, + inscriptions_shutdown_rx, + ) + .await }); let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; @@ -92,11 +112,12 @@ impl EventConsumer { }) } - async fn consume_queue(channel: lapin::Channel, - queue_name: String, - consumer_tag: String, - ord_db_client: Arc, - mut shutdown_signal: oneshot::Receiver<()>, + async fn consume_queue( + channel: lapin::Channel, + queue_name: String, + consumer_tag: String, + ord_db_client: Arc, + mut shutdown_signal: oneshot::Receiver<()>, ) -> Result<(), anyhow::Error> { let mut consumer = channel .basic_consume( @@ -113,12 +134,12 @@ impl EventConsumer { match result { Ok(delivery) => { tokio::select! { - _ = EventConsumer::handle_delivery(delivery, &ord_db_client) => {}, - _ = &mut shutdown_signal => { - log::info!("Shutdown signal received, stopping consumer."); - break; - }, - } + _ = EventConsumer::handle_delivery(delivery, &ord_db_client) => {}, + _ = &mut shutdown_signal => { + log::info!("Shutdown signal received, stopping consumer."); + break; + }, + } } Err(e) => { log::error!("Error receiving delivery: {}", e); @@ -127,40 +148,49 @@ impl EventConsumer { } log::info!("Closing consumer channel {}", queue_name); - channel.close(200, "Closing channel due to shutdown").await?; + channel + .close(200, "Closing channel due to shutdown") + .await?; Ok(()) } - async fn handle_delivery(delivery: lapin::message::Delivery, - ord_db_client: &Arc) -> Result<(), anyhow::Error> { + async fn handle_delivery( + delivery: lapin::message::Delivery, + ord_db_client: &Arc, + ) -> Result<(), anyhow::Error> { let event: Result = serde_json::from_slice(&delivery.data); match event { Ok(event) => { if let Err(err) = EventConsumer::process_event(event, ord_db_client).await { log::error!("Failed to process event: {}", err); - delivery.reject(BasicRejectOptions { requeue: false }).await?; + delivery + .reject(BasicRejectOptions { requeue: false }) + .await?; } else { delivery.ack(BasicAckOptions::default()).await?; } } Err(e) => { log::error!("Failed to deserialize event, rejecting: {}", e); - delivery.reject(BasicRejectOptions { requeue: false }).await?; + delivery + .reject(BasicRejectOptions { requeue: false }) + .await?; } } Ok(()) } - async fn process_event(event: Event, ord_db_client: &Arc) -> Result<(), anyhow::Error> { + async fn process_event( + event: Event, + ord_db_client: &Arc, + ) -> Result<(), anyhow::Error> { match &event { Event::BlockCommitted { from_height, to_height, - } => { - ord_db_client.sync_blocks(from_height, to_height).await? - } + } => ord_db_client.sync_blocks(from_height, to_height).await?, Event::InscriptionCreated { block_height, charms, @@ -169,7 +199,9 @@ impl EventConsumer { parent_inscription_ids, sequence_number, } => { - ord_db_client.save_inscription_created(block_height, inscription_id, location).await? + ord_db_client + .save_inscription_created(block_height, inscription_id, location) + .await? } Event::InscriptionTransferred { block_height, @@ -178,7 +210,9 @@ impl EventConsumer { old_location, sequence_number, } => { - ord_db_client.save_inscription_transferred(block_height, inscription_id, new_location, old_location).await?; + ord_db_client + .save_inscription_transferred(block_height, inscription_id, new_location, old_location) + .await?; } _ => { log::warn!("Received an unhandled event type"); @@ -190,6 +224,11 @@ impl EventConsumer { fn generate_consumer_tag() -> String { // TODO get pod name from k8s? let timestamp = Utc::now().format("%Y%m%d%H%M%S"); - format!("{}-{}-{}", "lr-ord", timestamp, Alphanumeric.sample_string(&mut rand::thread_rng(), 16)) + format!( + "{}-{}-{}", + "lr-ord", + timestamp, + Alphanumeric.sample_string(&mut rand::thread_rng(), 16) + ) } } diff --git a/src/event_publisher.rs b/src/event_publisher.rs index 0fbd307c0d..9bf15c5bf3 100644 --- a/src/event_publisher.rs +++ b/src/event_publisher.rs @@ -1,6 +1,6 @@ use anyhow::{Context, Result}; -use lapin::{BasicProperties, Connection, ConnectionProperties, options::BasicPublishOptions}; use lapin::options::ConfirmSelectOptions; +use lapin::{options::BasicPublishOptions, BasicProperties, Connection, ConnectionProperties}; use tokio::runtime::Runtime; use tokio::sync::mpsc; diff --git a/src/index/updater.rs b/src/index/updater.rs index 3c475cc16e..72790d245b 100644 --- a/src/index/updater.rs +++ b/src/index/updater.rs @@ -1,8 +1,8 @@ use { self::{inscription_updater::InscriptionUpdater, rune_updater::RuneUpdater}, + super::{fetcher::Fetcher, *}, futures::future::try_join_all, std::sync::mpsc, - super::{*, fetcher::Fetcher}, tokio::sync::mpsc::{error::TryRecvError, Receiver, Sender}, }; @@ -696,7 +696,12 @@ impl<'index> Updater<'index> { Ok(()) } - fn commit(&mut self, wtx: WriteTransaction, value_cache: HashMap, uncommitted: usize) -> Result { + fn commit( + &mut self, + wtx: WriteTransaction, + value_cache: HashMap, + uncommitted: usize, + ) -> Result { log::info!( "Committing at block height {}, {} outputs traversed, {} in map, {} cached", self.height, diff --git a/src/ord_db_client.rs b/src/ord_db_client.rs index 1727b0cdd8..b3f9c705a2 100644 --- a/src/ord_db_client.rs +++ b/src/ord_db_client.rs @@ -15,19 +15,25 @@ impl OrdDbClient { Self { pool } } - pub async fn sync_blocks(&self, - from_height: &u32, - to_height: &u32) -> Result<(), sqlx::Error> { + pub async fn sync_blocks(&self, from_height: &u32, to_height: &u32) -> Result<(), sqlx::Error> { log::info!("Blocks committed event from={from_height} (excluded), to={to_height} (included)"); // TODO consume all blocks from this to last consumed Ok(()) } - pub async fn save_inscription_created(&self, - block_height: &u32, - inscription_id: &InscriptionId, - location: &Option) -> Result<(), sqlx::Error> { - let query = "INSERT INTO events (type_id, block_height, inscription_id, location) VALUES ($1, $2, $3, $4)"; + pub async fn save_inscription_created( + &self, + block_height: &u32, + inscription_id: &InscriptionId, + location: &Option, + ) -> Result<(), sqlx::Error> { + let query = " + INSERT INTO events (type_id, block_height, inscription_id, location) + SELECT $1, $2, $3, $4 + WHERE NOT EXISTS ( + SELECT 1 FROM events + WHERE type_id = $1 AND block_height = $2 AND inscription_id = $3 AND location = $4 + )"; sqlx::query(query) .bind(1_i32) // Type ID for InscriptionCreated .bind(*block_height as i64) @@ -38,12 +44,20 @@ impl OrdDbClient { Ok(()) } - pub async fn save_inscription_transferred(&self, - block_height: &u32, - inscription_id: &InscriptionId, - new_location: &SatPoint, - old_location: &SatPoint) -> Result<(), sqlx::Error> { - let query = "INSERT INTO events (type_id, block_height, inscription_id, location, old_location) VALUES ($1, $2, $3, $4, $5)"; + pub async fn save_inscription_transferred( + &self, + block_height: &u32, + inscription_id: &InscriptionId, + new_location: &SatPoint, + old_location: &SatPoint, + ) -> Result<(), sqlx::Error> { + let query = " + INSERT INTO events (type_id, block_height, inscription_id, location, old_location) + SELECT $1, $2, $3, $4, $5 + WHERE NOT EXISTS ( + SELECT 1 FROM events + WHERE type_id = $1 AND block_height = $2 AND inscription_id = $3 AND location = $4 AND old_location = $5 + )"; sqlx::query(query) .bind(2_i32) // Type ID for InscriptionTransferred .bind(*block_height as i64) From e9540e4e7d50b42543ca4c8bec80c06dceed3f68 Mon Sep 17 00:00:00 2001 From: rob Date: Thu, 2 May 2024 12:17:39 +0200 Subject: [PATCH 030/103] Fetch events per block --- src/event_consumer.rs | 37 ++++++++++++++++-------- src/event_publisher.rs | 3 ++ src/ord_db_client.rs | 50 ++++++++++++++++++++++++++++---- src/ord_indexation.rs | 65 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 138 insertions(+), 17 deletions(-) create mode 100644 src/ord_indexation.rs diff --git a/src/event_consumer.rs b/src/event_consumer.rs index 7b41e625cc..cf29a90ff1 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -12,7 +12,9 @@ use tokio::runtime::Runtime; use tokio::sync::oneshot; use crate::index::event::Event; +use crate::ord_api_client::OrdApiClient; use crate::ord_db_client::OrdDbClient; +use crate::ord_indexation::OrdIndexation; use crate::settings::Settings; use crate::subcommand::SubcommandResult; @@ -24,6 +26,8 @@ pub struct EventConsumer { pub(crate) inscriptions_queue: Option, #[arg(long, help = "DB url to persist inscriptions.")] pub(crate) database_url: Option, + #[arg(long, help = "Ord api url to fetch inscriptions.")] + pub(crate) ord_api_url: Option, } impl EventConsumer { @@ -58,13 +62,22 @@ impl EventConsumer { let shared_pool = Arc::new(pool); let ord_db_client = Arc::new(OrdDbClient::new(shared_pool.clone())); + let api_url = self.ord_api_url.context("api url must be defined")?; + let ord_api_c = OrdApiClient::new(api_url.clone()).expect("api client must exist"); + let ord_api_client = Arc::new(ord_api_c); + + let ord_indexation = Arc::new(OrdIndexation::new( + Arc::clone(&ord_db_client), + Arc::clone(&ord_api_client), + )); + let blocks_queue = self .blocks_queue .as_deref() .context("rabbitmq blocks queue path must be defined")?; let blocks_queue_str = blocks_queue.to_string(); let blocks_channel = channel.clone(); - let blocks_ord_db_client = Arc::clone(&ord_db_client); + let blocks_ord_indexation = Arc::clone(&ord_indexation); let blocks_consumer_tag = Self::generate_consumer_tag(); let (blocks_shutdown_tx, blocks_shutdown_rx) = oneshot::channel::<()>(); @@ -74,7 +87,7 @@ impl EventConsumer { .context("rabbitmq inscriptions queue path must be defined")?; let inscriptions_queue_str = inscriptions_queue.to_string(); let inscriptions_channel = channel.clone(); - let inscriptions_ord_db_client = Arc::clone(&ord_db_client); + let inscriptions_ord_indexation = Arc::clone(&ord_indexation); let inscriptions_consumer_tag = Self::generate_consumer_tag(); let (inscriptions_shutdown_tx, inscriptions_shutdown_rx) = oneshot::channel::<()>(); @@ -83,7 +96,7 @@ impl EventConsumer { blocks_channel, blocks_queue_str, blocks_consumer_tag, - blocks_ord_db_client, + blocks_ord_indexation, blocks_shutdown_rx, ) .await @@ -94,7 +107,7 @@ impl EventConsumer { inscriptions_channel, inscriptions_queue_str, inscriptions_consumer_tag, - inscriptions_ord_db_client, + inscriptions_ord_indexation, inscriptions_shutdown_rx, ) .await @@ -116,7 +129,7 @@ impl EventConsumer { channel: lapin::Channel, queue_name: String, consumer_tag: String, - ord_db_client: Arc, + ord_indexation: Arc, mut shutdown_signal: oneshot::Receiver<()>, ) -> Result<(), anyhow::Error> { let mut consumer = channel @@ -134,7 +147,7 @@ impl EventConsumer { match result { Ok(delivery) => { tokio::select! { - _ = EventConsumer::handle_delivery(delivery, &ord_db_client) => {}, + _ = EventConsumer::handle_delivery(delivery, &ord_indexation) => {}, _ = &mut shutdown_signal => { log::info!("Shutdown signal received, stopping consumer."); break; @@ -157,12 +170,12 @@ impl EventConsumer { async fn handle_delivery( delivery: lapin::message::Delivery, - ord_db_client: &Arc, + ord_indexation: &Arc, ) -> Result<(), anyhow::Error> { let event: Result = serde_json::from_slice(&delivery.data); match event { Ok(event) => { - if let Err(err) = EventConsumer::process_event(event, ord_db_client).await { + if let Err(err) = EventConsumer::process_event(event, ord_indexation).await { log::error!("Failed to process event: {}", err); delivery .reject(BasicRejectOptions { requeue: false }) @@ -184,13 +197,13 @@ impl EventConsumer { async fn process_event( event: Event, - ord_db_client: &Arc, + ord_indexation: &Arc, ) -> Result<(), anyhow::Error> { match &event { Event::BlockCommitted { from_height, to_height, - } => ord_db_client.sync_blocks(from_height, to_height).await?, + } => ord_indexation.sync_blocks(from_height, to_height).await?, Event::InscriptionCreated { block_height, charms, @@ -199,7 +212,7 @@ impl EventConsumer { parent_inscription_ids, sequence_number, } => { - ord_db_client + ord_indexation .save_inscription_created(block_height, inscription_id, location) .await? } @@ -210,7 +223,7 @@ impl EventConsumer { old_location, sequence_number, } => { - ord_db_client + ord_indexation .save_inscription_transferred(block_height, inscription_id, new_location, old_location) .await?; } diff --git a/src/event_publisher.rs b/src/event_publisher.rs index 9bf15c5bf3..53d1f69eb2 100644 --- a/src/event_publisher.rs +++ b/src/event_publisher.rs @@ -41,6 +41,9 @@ impl EventPublisher { .expect("enable msg confirms"); while let Some(event) = rx.recv().await { + // TODO we might want to panic if rmq is down so we don't miss any messages + // if we miss messages only way to replay them is run `ord` instance from scratch + // maybe we can trigger fake reorg to force it to reindex from savepoint? let message = serde_json::to_vec(&event).expect("failed to serialize event"); let publish = channel diff --git a/src/ord_db_client.rs b/src/ord_db_client.rs index b3f9c705a2..dfd7b1c1ce 100644 --- a/src/ord_db_client.rs +++ b/src/ord_db_client.rs @@ -1,11 +1,33 @@ use std::sync::Arc; -use sqlx::PgPool; +use futures::TryStreamExt; +use sqlx::{PgPool, Row}; use ordinals::SatPoint; use crate::InscriptionId; +#[derive(Debug, Clone)] +pub struct Event { + pub type_id: i16, + pub block_height: i64, + pub inscription_id: String, + pub location: Option, + pub old_location: Option, +} + +impl Event { + fn from_row(row: sqlx::postgres::PgRow) -> Self { + Event { + type_id: row.get("type_id"), + block_height: row.get("block_height"), + inscription_id: row.get("inscription_id"), + location: row.get("location"), + old_location: row.get("old_location"), + } + } +} + pub struct OrdDbClient { pool: Arc, } @@ -15,10 +37,28 @@ impl OrdDbClient { Self { pool } } - pub async fn sync_blocks(&self, from_height: &u32, to_height: &u32) -> Result<(), sqlx::Error> { - log::info!("Blocks committed event from={from_height} (excluded), to={to_height} (included)"); - // TODO consume all blocks from this to last consumed - Ok(()) + pub async fn fetch_events_by_block_height( + &self, + block_height: u32, + ) -> Result, sqlx::Error> { + let query = " + SELECT type_id, block_height, inscription_id, location, old_location + FROM events + WHERE block_height = $1 + ORDER BY type_id ASC, id ASC"; + + let mut rows = sqlx::query(query) + .bind(block_height as i64) + .fetch(&*self.pool); + + let mut events = Vec::new(); + + while let Some(row) = rows.try_next().await? { + let event = Event::from_row(row); + events.push(event); + } + + Ok(events) } pub async fn save_inscription_created( diff --git a/src/ord_indexation.rs b/src/ord_indexation.rs new file mode 100644 index 0000000000..2e1d53f664 --- /dev/null +++ b/src/ord_indexation.rs @@ -0,0 +1,65 @@ +use std::sync::Arc; + +use ordinals::SatPoint; + +use crate::ord_api_client::OrdApiClient; +use crate::ord_db_client::OrdDbClient; +use crate::InscriptionId; + +pub struct OrdIndexation { + ord_db_client: Arc, + ord_api_client: Arc, +} + +impl OrdIndexation { + pub fn new(ord_db_client: Arc, ord_api_client: Arc) -> Self { + Self { + ord_api_client, + ord_db_client, + } + } + + pub async fn sync_blocks(&self, from_height: &u32, to_height: &u32) -> Result<(), anyhow::Error> { + log::info!("Blocks committed event from={from_height} (excluded), to={to_height} (included)"); + + for block_height in *from_height + 1..=*to_height { + let events = self + .ord_db_client + .fetch_events_by_block_height(block_height) + .await?; + for event in events { + log::info!("Event: {:?}", event); + // let inscription_details = self.ord_api_client.fetch_inscription_details(event.inscription_id).await?; + } + } + + Ok(()) + } + + pub async fn save_inscription_created( + &self, + block_height: &u32, + inscription_id: &InscriptionId, + location: &Option, + ) -> Result<(), anyhow::Error> { + self + .ord_db_client + .save_inscription_created(block_height, inscription_id, location) + .await?; + Ok(()) + } + + pub async fn save_inscription_transferred( + &self, + block_height: &u32, + inscription_id: &InscriptionId, + new_location: &SatPoint, + old_location: &SatPoint, + ) -> Result<(), anyhow::Error> { + self + .ord_db_client + .save_inscription_transferred(block_height, inscription_id, new_location, old_location) + .await?; + Ok(()) + } +} From cf60ac7b30600df89e475cc74997aaeea6418c0e Mon Sep 17 00:00:00 2001 From: rob Date: Thu, 2 May 2024 14:27:27 +0200 Subject: [PATCH 031/103] Fetch inscription for event --- src/api.rs | 27 ++++++++++++++ src/ord_api_client.rs | 80 +++++++++++++++++++++++++++++++--------- src/ord_indexation.rs | 7 +++- src/subcommand/server.rs | 35 ++++++++++++++++-- 4 files changed, 127 insertions(+), 22 deletions(-) diff --git a/src/api.rs b/src/api.rs index 33f566e868..b785914d6a 100644 --- a/src/api.rs +++ b/src/api.rs @@ -76,6 +76,33 @@ pub struct Children { pub page: usize, } +#[derive(Debug, Serialize, Deserialize)] +pub struct InscriptionDetails { + pub id: InscriptionId, + pub number: i32, + pub owner: Option, + pub content_type: Option, + pub content_length: Option, + pub metadata: Option>, + + pub genesis_block_height: u32, + pub genesis_block_time: i64, + + pub sat_number: Option, + pub sat_name: Option, + pub sat_rarity: Option, + pub sat_block_height: Option, + pub sat_block_time: Option, + + pub satpoint: SatPoint, + pub value: Option, + pub fee: u64, + pub charms: u16, + + pub children: Vec, + pub parents: Vec, +} + #[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] pub struct Inscription { pub address: Option, diff --git a/src/ord_api_client.rs b/src/ord_api_client.rs index 9664c4fb23..67ec2af510 100644 --- a/src/ord_api_client.rs +++ b/src/ord_api_client.rs @@ -1,7 +1,11 @@ -use reqwest::Client; +use std::time::Duration; -use crate::api::Inscription; -use crate::InscriptionId; +use anyhow::anyhow; +use http::StatusCode; +use reqwest::{Client, RequestBuilder}; +use tokio::time::sleep; + +use crate::api::InscriptionDetails; pub struct OrdApiClient { ord_api_url: String, @@ -9,12 +13,9 @@ pub struct OrdApiClient { } impl OrdApiClient { - pub fn run(ord_api_url: String) -> anyhow::Result { + pub fn new(ord_api_url: String) -> anyhow::Result { let client = Client::builder() .timeout(std::time::Duration::from_secs(30)) - // TODO add retries - // .timeout_retry_times(5) - // .timeout_retry_strategy(ExponentialBackoff) .build()?; Ok(OrdApiClient { @@ -23,22 +24,67 @@ impl OrdApiClient { }) } + async fn execute_with_retries( + &self, + request_builder: RequestBuilder, + max_attempts: u32, + ) -> Result + where + T: for<'de> serde::Deserialize<'de> + 'static, + { + let mut attempts = 0; + let mut delay = Duration::from_secs(1); + + while attempts < max_attempts { + let mut request = request_builder + .try_clone() + .ok_or_else(|| anyhow!("Failed to clone request"))?; + + let response = request.send().await; + + match response { + Ok(resp) => match resp.error_for_status() { + Ok(valid_response) => { + return valid_response + .json::() + .await + .map_err(anyhow::Error::from); + } + Err(e) + if e.status() == Some(StatusCode::TOO_MANY_REQUESTS) + || e + .status() + .map_or_else(|| false, |status_code| status_code.is_server_error()) => + { + attempts += 1; + sleep(delay).await; + delay *= 2; + } + Err(e) => return Err(anyhow!(e)), + }, + Err(e) => { + attempts += 1; + sleep(delay).await; + delay *= 2; + } + } + } + + Err(anyhow!("Exceeded maximum retry attempts")) + } + pub async fn fetch_inscription_details( &self, - inscription_id: &InscriptionId, - ) -> Result { - let response = self + inscription_id: String, + ) -> Result { + let request_builder = self .client .get(format!( "{}/inscription/{}", self.ord_api_url, inscription_id )) - .header("Accept", "application/json") - .send() - .await? - .error_for_status()? - .json::() - .await?; - Ok(response) + .header("Accept", "application/json"); + + self.execute_with_retries(request_builder, 3).await } } diff --git a/src/ord_indexation.rs b/src/ord_indexation.rs index 2e1d53f664..49e597bdcf 100644 --- a/src/ord_indexation.rs +++ b/src/ord_indexation.rs @@ -28,8 +28,11 @@ impl OrdIndexation { .fetch_events_by_block_height(block_height) .await?; for event in events { - log::info!("Event: {:?}", event); - // let inscription_details = self.ord_api_client.fetch_inscription_details(event.inscription_id).await?; + let inscription_details = self + .ord_api_client + .fetch_inscription_details(event.inscription_id) + .await?; + log::info!("inscription_details: {:?}", inscription_details); } } diff --git a/src/subcommand/server.rs b/src/subcommand/server.rs index 93e1ca471e..aa2ad255d5 100644 --- a/src/subcommand/server.rs +++ b/src/subcommand/server.rs @@ -1,3 +1,4 @@ +pub(crate) use server_config::ServerConfig; use { self::{ accept_encoding::AcceptEncoding, @@ -39,7 +40,7 @@ use { }, }; -pub(crate) use server_config::ServerConfig; +use crate::api::InscriptionDetails; mod accept_encoding; mod accept_json; @@ -1520,7 +1521,6 @@ impl Server { }) } - // TODO need to enhance this api to return all necessary data async fn inscription( Extension(server_config): Extension>, Extension(index): Extension>, @@ -1539,7 +1539,36 @@ impl Server { .ok_or_not_found(|| format!("inscription {query}"))?; Ok(if accept_json { - Json(info).into_response() + let sat_block_time = info + .sat + .and_then(|sat| index.block_time(sat.height()).ok()) + .map(|block_time| block_time.timestamp().timestamp()); + + let response = InscriptionDetails { + id: info.id, + number: info.number, + owner: info.address, + content_type: info.content_type, + content_length: info.content_length, + metadata: inscription.metadata, + genesis_block_height: info.height, + genesis_block_time: info.timestamp, + sat_number: info.sat.map(|sat| sat.0), + sat_name: info.sat.map(|sat| sat.name()), + sat_rarity: info.sat.map(|sat| sat.rarity()), + sat_block_height: info.sat.map(|sat| sat.height().0), + sat_block_time, + satpoint: info.satpoint, + value: info.value, + fee: info.fee, + charms: Charm::Vindicated.unset(info.charms.iter().fold(0, |mut acc, charm| { + charm.set(&mut acc); + acc + })), + children: info.children, + parents: info.parents, + }; + Json(response).into_response() } else { InscriptionHtml { chain: server_config.chain, From 75480fafeaa1f5e523aa9072589fcdab7a19b9b8 Mon Sep 17 00:00:00 2001 From: rob Date: Thu, 2 May 2024 19:30:34 +0200 Subject: [PATCH 032/103] Handle inscription created event --- crates/ordinals/src/rarity.rs | 13 ++ .../20240431000000_ordinals/migration.sql | 42 ++++++ src/event_consumer.rs | 1 + src/ord_api_client.rs | 12 +- src/ord_db_client.rs | 141 +++++++++++++++++- src/ord_indexation.rs | 111 +++++++++++++- 6 files changed, 308 insertions(+), 12 deletions(-) diff --git a/crates/ordinals/src/rarity.rs b/crates/ordinals/src/rarity.rs index a92fce72a9..845795aeb9 100644 --- a/crates/ordinals/src/rarity.rs +++ b/crates/ordinals/src/rarity.rs @@ -10,6 +10,19 @@ pub enum Rarity { Mythic, } +impl Rarity { + pub fn to_i16(&self) -> i16 { + match self { + Rarity::Common => 0, + Rarity::Uncommon => 1, + Rarity::Rare => 2, + Rarity::Epic => 3, + Rarity::Legendary => 4, + Rarity::Mythic => 5, + } + } +} + impl From for u8 { fn from(rarity: Rarity) -> Self { rarity as u8 diff --git a/db/migrations/20240431000000_ordinals/migration.sql b/db/migrations/20240431000000_ordinals/migration.sql index 1431587e3b..d48c9de5fd 100644 --- a/db/migrations/20240431000000_ordinals/migration.sql +++ b/db/migrations/20240431000000_ordinals/migration.sql @@ -11,3 +11,45 @@ CREATE TABLE events CREATE INDEX idx_events_block_height ON events (block_height); CREATE INDEX idx_events_inscription_id ON events (inscription_id); + +CREATE TABLE inscription +( + id SERIAL PRIMARY KEY, + genesis_id TEXT NOT NULL, + number INTEGER NOT NULL, + content_type VARCHAR, + content_length INTEGER, + metadata TEXT, + + genesis_block_height INTEGER NOT NULL, + genesis_block_time BIGINT NOT NULL, + + sat_number BIGINT, + sat_rarity INTEGER, + sat_block_height INTEGER, + sat_block_time BIGINT, + + fee BIGINT NOT NULL, + charms SMALLINT NOT NULL, + + children TEXT, + parents TEXT +); + +CREATE UNIQUE INDEX idx_unique_inscription_genesis_id ON inscription (genesis_id); + +CREATE TABLE location +( + id SERIAL PRIMARY KEY, + inscription_id INTEGER REFERENCES inscription (id), + block_height INTEGER NOT NULL, + block_time BIGINT NOT NULL, + tx_id TEXT NOT NULL, + to_address TEXT NOT NULL, + cur_output TEXT NOT NULL, + cur_offset BIGINT NOT NULL, + from_address TEXT, + prev_output TEXT, + prev_offset BIGINT, + value BIGINT +); diff --git a/src/event_consumer.rs b/src/event_consumer.rs index cf29a90ff1..ae30527f3a 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -67,6 +67,7 @@ impl EventConsumer { let ord_api_client = Arc::new(ord_api_c); let ord_indexation = Arc::new(OrdIndexation::new( + settings, Arc::clone(&ord_db_client), Arc::clone(&ord_api_client), )); diff --git a/src/ord_api_client.rs b/src/ord_api_client.rs index 67ec2af510..ccfe47d917 100644 --- a/src/ord_api_client.rs +++ b/src/ord_api_client.rs @@ -1,11 +1,12 @@ use std::time::Duration; use anyhow::anyhow; +use bitcoin::Txid; use http::StatusCode; use reqwest::{Client, RequestBuilder}; use tokio::time::sleep; -use crate::api::InscriptionDetails; +use crate::api::{InscriptionDetails, Transaction}; pub struct OrdApiClient { ord_api_url: String, @@ -87,4 +88,13 @@ impl OrdApiClient { self.execute_with_retries(request_builder, 3).await } + + pub async fn fetch_tx(&self, tx_id: Txid) -> Result { + let request_builder = self + .client + .get(format!("{}/tx/{}", self.ord_api_url, tx_id)) + .header("Accept", "application/json"); + + self.execute_with_retries(request_builder, 3).await + } } diff --git a/src/ord_db_client.rs b/src/ord_db_client.rs index dfd7b1c1ce..2794941bdd 100644 --- a/src/ord_db_client.rs +++ b/src/ord_db_client.rs @@ -1,10 +1,14 @@ +use std::str::FromStr; use std::sync::Arc; +use bitcoin::{OutPoint, Txid}; use futures::TryStreamExt; +use sqlx::types::Json; use sqlx::{PgPool, Row}; use ordinals::SatPoint; +use crate::api::InscriptionDetails; use crate::InscriptionId; #[derive(Debug, Clone)] @@ -12,8 +16,8 @@ pub struct Event { pub type_id: i16, pub block_height: i64, pub inscription_id: String, - pub location: Option, - pub old_location: Option, + pub location: Option, + pub old_location: Option, } impl Event { @@ -22,8 +26,14 @@ impl Event { type_id: row.get("type_id"), block_height: row.get("block_height"), inscription_id: row.get("inscription_id"), - location: row.get("location"), - old_location: row.get("old_location"), + location: { + let location_str: Option = row.get("location"); + location_str.and_then(|s| SatPoint::from_str(&s).ok()) + }, + old_location: { + let old_location_str: Option = row.get("old_location"); + old_location_str.and_then(|s| SatPoint::from_str(&s).ok()) + }, } } } @@ -108,4 +118,127 @@ impl OrdDbClient { .await?; Ok(()) } + + pub async fn save_inscription( + &self, + inscription_details: &InscriptionDetails, + ) -> Result<(), sqlx::Error> { + let query = " + INSERT INTO inscription ( + genesis_id, + number, + content_type, + content_length, + metadata, + genesis_block_height, + genesis_block_time, + sat_number, + sat_rarity, + sat_block_height, + sat_block_time, + fee, + charms, + children, + parents + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) + ON CONFLICT (genesis_id) DO UPDATE SET + number = EXCLUDED.number, + content_type = EXCLUDED.content_type, + content_length = COALESCE(EXCLUDED.content_length, inscription.content_length), + metadata = COALESCE(EXCLUDED.metadata, inscription.metadata), + genesis_block_height = EXCLUDED.genesis_block_height, + genesis_block_time = EXCLUDED.genesis_block_time, + sat_number = COALESCE(EXCLUDED.sat_number, inscription.sat_number), + sat_rarity = COALESCE(EXCLUDED.sat_rarity, inscription.sat_rarity), + sat_block_height = COALESCE(EXCLUDED.sat_block_height, inscription.sat_block_height), + sat_block_time = COALESCE(EXCLUDED.sat_block_time, inscription.sat_block_time), + fee = EXCLUDED.fee, + charms = EXCLUDED.charms, + children = COALESCE(EXCLUDED.children, inscription.children), + parents = COALESCE(EXCLUDED.parents, inscription.parents) + "; + sqlx::query(query) + .bind(inscription_details.id.to_string()) + .bind(inscription_details.number) + .bind(inscription_details.content_type.as_deref()) + .bind(inscription_details.content_length.map(|n| n as i32)) + .bind(Json(&inscription_details.metadata)) //TODO need to ascii decode + .bind(inscription_details.genesis_block_height as i32) + .bind(inscription_details.genesis_block_time) + .bind(inscription_details.sat_number.map(|n| n as i64)) + .bind(inscription_details.sat_rarity.map(|r| r.to_i16())) + .bind(inscription_details.sat_block_height.map(|n| n as i32)) + .bind(inscription_details.sat_block_time) + .bind(inscription_details.fee as i64) + .bind(inscription_details.charms as i16) + .bind(Json(&inscription_details.children)) + .bind(Json(&inscription_details.parents)) + .execute(&*self.pool) + .await?; + Ok(()) + } + + pub async fn save_location( + &self, + id: InscriptionId, + block_height: i64, + block_time: i64, + tx_id: Option, + to_address: Option, + to_outpoint: Option, + to_offset: Option, + from_address: Option, + from_outpoint: Option, + from_offset: Option, + value: Option, + ) -> Result<(), sqlx::Error> { + let query = " + INSERT INTO location ( + inscription_id, + block_height, + block_time, + tx_id, + to_address, + cur_output, + cur_offset, + from_address, + prev_output, + prev_offset, + value + ) + SELECT + (SELECT id FROM inscription WHERE genesis_id = $1), + $2, $3, $4, $5, $6, $7, $8, $9, $10, $11 + WHERE NOT EXISTS ( + SELECT 1 FROM location + WHERE + inscription_id = (SELECT id FROM inscription WHERE genesis_id = $1) AND + block_height = $2 AND + block_time = $3 AND + tx_id = $4 AND + to_address = $5 AND + cur_output = $6 AND + cur_offset = $7 AND + from_address = $8 AND + prev_output = $9 AND + prev_offset = $10 AND + value = $11 + ); + "; + sqlx::query(query) + .bind(id.to_string()) + .bind(block_height) + .bind(block_time) + .bind(tx_id.map(|n| n.to_string())) + .bind(to_address) + .bind(to_outpoint.map(|n| n.to_string())) + .bind(to_offset.map(|n| n as i64)) + .bind(from_address) + .bind(from_outpoint.map(|n| n.to_string())) + .bind(from_offset.map(|n| n as i64)) + .bind(value.map(|n| n as i64)) + .execute(&*self.pool) + .await?; + Ok(()) + } } diff --git a/src/ord_indexation.rs b/src/ord_indexation.rs index 49e597bdcf..aed656cf2b 100644 --- a/src/ord_indexation.rs +++ b/src/ord_indexation.rs @@ -3,17 +3,24 @@ use std::sync::Arc; use ordinals::SatPoint; use crate::ord_api_client::OrdApiClient; -use crate::ord_db_client::OrdDbClient; +use crate::ord_db_client::{Event, OrdDbClient}; +use crate::settings::Settings; use crate::InscriptionId; pub struct OrdIndexation { + settings: Settings, ord_db_client: Arc, ord_api_client: Arc, } impl OrdIndexation { - pub fn new(ord_db_client: Arc, ord_api_client: Arc) -> Self { + pub fn new( + settings: &Settings, + ord_db_client: Arc, + ord_api_client: Arc, + ) -> Self { Self { + settings: settings.clone(), ord_api_client, ord_db_client, } @@ -28,17 +35,107 @@ impl OrdIndexation { .fetch_events_by_block_height(block_height) .await?; for event in events { - let inscription_details = self - .ord_api_client - .fetch_inscription_details(event.inscription_id) - .await?; - log::info!("inscription_details: {:?}", inscription_details); + match event.type_id { + 1 => { + if let Err(e) = self.process_inscription_created(&event).await { + log::error!( + "Error processing inscription creation for event {:?}: {}", + event, + e + ); + } + } + 2 => { + // TODO: Handle type 2 events + } + _ => { + log::warn!("Unhandled event type: {}", event.type_id); + } + } } } Ok(()) } + async fn process_inscription_created(&self, event: &Event) -> Result<(), anyhow::Error> { + let inscription_details = self + .ord_api_client + .fetch_inscription_details(event.inscription_id.clone()) + .await?; + self + .ord_db_client + .save_inscription(&inscription_details) + .await?; + + let block_time = 0; //TODO need to fetch + + let mut to_location_details: Option<(String, u64)> = None; + if let Some(location) = &event.location { + to_location_details = self.process_location(location).await?; + } + + let mut from_location_details: Option<(String, u64)> = None; + if let Some(location) = &event.old_location { + from_location_details = self.process_location(location).await?; + } + + self + .ord_db_client + .save_location( + inscription_details.id.clone(), + event.block_height, + block_time, + event.location.as_ref().map(|loc| loc.outpoint.txid.clone()), + to_location_details + .as_ref() + .map(|details| details.0.clone()), + event.location.as_ref().map(|loc| loc.outpoint.clone()), + event.location.as_ref().map(|loc| loc.offset), + from_location_details + .as_ref() + .map(|details| details.0.clone()), + event.old_location.as_ref().map(|loc| loc.outpoint.clone()), + event.old_location.as_ref().map(|loc| loc.offset), + to_location_details.as_ref().map(|details| details.1), + ) + .await?; + + Ok(()) + } + + async fn process_location( + &self, + location: &SatPoint, + ) -> Result, anyhow::Error> { + let tx_details = self + .ord_api_client + .fetch_tx(location.outpoint.txid.clone()) + .await?; + + let output = tx_details + .transaction + .output + .into_iter() + .nth(location.outpoint.vout.try_into().unwrap()); + + let address = output + .as_ref() + .and_then(|o| { + self + .settings + .chain() + .address_from_script(&o.script_pubkey) + .ok() + }) + .map(|address| address.to_string()); + + match address { + Some(addr) => Ok(Some((addr, output.unwrap().value))), + None => Ok(None), + } + } + pub async fn save_inscription_created( &self, block_height: &u32, From 4b9f219e30157a3234f756d61fbe6c5757ee24ee Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 3 May 2024 11:28:30 +0200 Subject: [PATCH 033/103] Handle inscription transfers, fix metadata --- src/lib.rs | 1 + src/ord_api_client.rs | 11 ++++- src/ord_db_client.rs | 43 ++++++++++++------ src/ord_indexation.rs | 101 ++++++++++++++++++++++++++++++++++++++---- 4 files changed, 134 insertions(+), 22 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 93fdebc8e3..43fad300a7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -116,6 +116,7 @@ pub mod options; pub mod ord_api_client; pub mod ord_db_client; +mod ord_indexation; pub mod outgoing; mod re; mod representation; diff --git a/src/ord_api_client.rs b/src/ord_api_client.rs index ccfe47d917..93cefdb1db 100644 --- a/src/ord_api_client.rs +++ b/src/ord_api_client.rs @@ -6,7 +6,7 @@ use http::StatusCode; use reqwest::{Client, RequestBuilder}; use tokio::time::sleep; -use crate::api::{InscriptionDetails, Transaction}; +use crate::api::{BlockInfo, InscriptionDetails, Transaction}; pub struct OrdApiClient { ord_api_url: String, @@ -97,4 +97,13 @@ impl OrdApiClient { self.execute_with_retries(request_builder, 3).await } + + pub async fn fetch_block_info(&self, block_height: u32) -> Result { + let request_builder = self + .client + .get(format!("{}/r/blockinfo/{}", self.ord_api_url, block_height)) + .header("Accept", "application/json"); + + self.execute_with_retries(request_builder, 3).await + } } diff --git a/src/ord_db_client.rs b/src/ord_db_client.rs index 2794941bdd..41593e8dfe 100644 --- a/src/ord_db_client.rs +++ b/src/ord_db_client.rs @@ -3,8 +3,8 @@ use std::sync::Arc; use bitcoin::{OutPoint, Txid}; use futures::TryStreamExt; -use sqlx::types::Json; use sqlx::{PgPool, Row}; +use sqlx::types::Json; use ordinals::SatPoint; @@ -119,10 +119,27 @@ impl OrdDbClient { Ok(()) } + pub async fn fetch_inscription_id_by_genesis_id( + &self, + genesis_id: String, + ) -> Result, sqlx::Error> { + let query = " + SELECT id FROM inscription + WHERE genesis_id = $1; + "; + let result = sqlx::query(query) + .bind(genesis_id) + .fetch_optional(&*self.pool) + .await? + .map(|row| row.get(0)); + Ok(result) + } + pub async fn save_inscription( &self, inscription_details: &InscriptionDetails, - ) -> Result<(), sqlx::Error> { + metadata: Option, + ) -> Result { let query = " INSERT INTO inscription ( genesis_id, @@ -156,13 +173,14 @@ impl OrdDbClient { charms = EXCLUDED.charms, children = COALESCE(EXCLUDED.children, inscription.children), parents = COALESCE(EXCLUDED.parents, inscription.parents) + RETURNING id; "; - sqlx::query(query) + let id = sqlx::query_as::<_, (i32, )>(query) .bind(inscription_details.id.to_string()) .bind(inscription_details.number) .bind(inscription_details.content_type.as_deref()) .bind(inscription_details.content_length.map(|n| n as i32)) - .bind(Json(&inscription_details.metadata)) //TODO need to ascii decode + .bind(metadata) .bind(inscription_details.genesis_block_height as i32) .bind(inscription_details.genesis_block_time) .bind(inscription_details.sat_number.map(|n| n as i64)) @@ -173,16 +191,16 @@ impl OrdDbClient { .bind(inscription_details.charms as i16) .bind(Json(&inscription_details.children)) .bind(Json(&inscription_details.parents)) - .execute(&*self.pool) + .fetch_one(&*self.pool) .await?; - Ok(()) + Ok(id.0) } pub async fn save_location( &self, - id: InscriptionId, + id: i32, block_height: i64, - block_time: i64, + block_time: u64, tx_id: Option, to_address: Option, to_outpoint: Option, @@ -207,12 +225,11 @@ impl OrdDbClient { value ) SELECT - (SELECT id FROM inscription WHERE genesis_id = $1), - $2, $3, $4, $5, $6, $7, $8, $9, $10, $11 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11 WHERE NOT EXISTS ( SELECT 1 FROM location WHERE - inscription_id = (SELECT id FROM inscription WHERE genesis_id = $1) AND + inscription_id = $1 AND block_height = $2 AND block_time = $3 AND tx_id = $4 AND @@ -226,9 +243,9 @@ impl OrdDbClient { ); "; sqlx::query(query) - .bind(id.to_string()) + .bind(id) .bind(block_height) - .bind(block_time) + .bind(block_time as i64) .bind(tx_id.map(|n| n.to_string())) .bind(to_address) .bind(to_outpoint.map(|n| n.to_string())) diff --git a/src/ord_indexation.rs b/src/ord_indexation.rs index aed656cf2b..b2eca1c262 100644 --- a/src/ord_indexation.rs +++ b/src/ord_indexation.rs @@ -1,11 +1,16 @@ +use std::io::Cursor; use std::sync::Arc; +use ciborium::from_reader; +use serde_json::Value; + use ordinals::SatPoint; +use crate::api::BlockInfo; +use crate::InscriptionId; use crate::ord_api_client::OrdApiClient; use crate::ord_db_client::{Event, OrdDbClient}; use crate::settings::Settings; -use crate::InscriptionId; pub struct OrdIndexation { settings: Settings, @@ -34,10 +39,11 @@ impl OrdIndexation { .ord_db_client .fetch_events_by_block_height(block_height) .await?; + let block_info = self.ord_api_client.fetch_block_info(block_height).await?; for event in events { match event.type_id { 1 => { - if let Err(e) = self.process_inscription_created(&event).await { + if let Err(e) = self.process_inscription_created(&event, &block_info).await { log::error!( "Error processing inscription creation for event {:?}: {}", event, @@ -46,7 +52,16 @@ impl OrdIndexation { } } 2 => { - // TODO: Handle type 2 events + if let Err(e) = self + .process_inscription_transferred(&event, &block_info) + .await + { + log::error!( + "Error processing inscription transferred for event {:?}: {}", + event, + e + ); + } } _ => { log::warn!("Unhandled event type: {}", event.type_id); @@ -58,17 +73,87 @@ impl OrdIndexation { Ok(()) } - async fn process_inscription_created(&self, event: &Event) -> Result<(), anyhow::Error> { + async fn process_inscription_created( + &self, + event: &Event, + block_info: &BlockInfo, + ) -> Result<(), anyhow::Error> { let inscription_details = self .ord_api_client .fetch_inscription_details(event.inscription_id.clone()) .await?; + + let metadata = self.try_and_extract_metadata(&inscription_details.metadata); + + let inscription_id = self + .ord_db_client + .save_inscription(&inscription_details, metadata) + .await?; + + let mut to_location_details: Option<(String, u64)> = None; + if let Some(location) = &event.location { + to_location_details = self.process_location(location).await?; + } + self .ord_db_client - .save_inscription(&inscription_details) + .save_location( + inscription_id, + event.block_height, + block_info.timestamp, + event.location.as_ref().map(|loc| loc.outpoint.txid.clone()), + to_location_details + .as_ref() + .map(|details| details.0.clone()), + event.location.as_ref().map(|loc| loc.outpoint.clone()), + event.location.as_ref().map(|loc| loc.offset), + None, + None, + None, + to_location_details.as_ref().map(|details| details.1), + ) .await?; - let block_time = 0; //TODO need to fetch + Ok(()) + } + + fn try_and_extract_metadata(&self, metadata: &Option>) -> Option { + metadata.as_ref().and_then(|bytes| { + let cursor = Cursor::new(bytes); + let result = from_reader(cursor); + match result { + Ok(value) => { + match &value { + Value::Object(_) | Value::Array(_) => { + serde_json::to_string(&value).ok() + }, + _ => { + Some(hex::encode(bytes)) + } + } + }, + Err(_) => { + Some(hex::encode(bytes)) + } + } + }) + } + + async fn process_inscription_transferred( + &self, + event: &Event, + block_info: &BlockInfo, + ) -> Result<(), anyhow::Error> { + let inscription_id = self + .ord_db_client + .fetch_inscription_id_by_genesis_id(event.inscription_id.clone()) + .await? + .ok_or_else(|| { + anyhow::anyhow!( + "No inscription found for genesis_id: {}", + event.inscription_id + ) + })?; let mut to_location_details: Option<(String, u64)> = None; if let Some(location) = &event.location { @@ -83,9 +168,9 @@ impl OrdIndexation { self .ord_db_client .save_location( - inscription_details.id.clone(), + inscription_id, event.block_height, - block_time, + block_info.timestamp, event.location.as_ref().map(|loc| loc.outpoint.txid.clone()), to_location_details .as_ref() From a8cce0b7215c48372a998040cd83c79899fd8db9 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 3 May 2024 12:10:25 +0200 Subject: [PATCH 034/103] Update rmq config --- rmqconfig.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rmqconfig.json b/rmqconfig.json index 1e30f1866a..de454be89a 100644 --- a/rmqconfig.json +++ b/rmqconfig.json @@ -1,14 +1,14 @@ { "queues": [ { - "name": "ord-q", + "name": "btc-inscription-q", "vhost": "/", "durable": true, "auto_delete": false, "arguments": {} }, { - "name": "blocks-q", + "name": "btc-blocks-q", "vhost": "/", "durable": true, "auto_delete": false, @@ -30,7 +30,7 @@ { "source": "ord-tx", "vhost": "/", - "destination": "blocks-q", + "destination": "btc-blocks-q", "destination_type": "queue", "routing_key": "BlockCommitted", "arguments": {} @@ -38,7 +38,7 @@ { "source": "ord-tx", "vhost": "/", - "destination": "ord-q", + "destination": "btc-inscription-q", "destination_type": "queue", "routing_key": "InscriptionCreated", "arguments": {} @@ -46,7 +46,7 @@ { "source": "ord-tx", "vhost": "/", - "destination": "ord-q", + "destination": "btc-inscription-q", "destination_type": "queue", "routing_key": "InscriptionTransferred", "arguments": {} From 070a98b62edcabffe13b0da3c1e27be7d72b39b8 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 3 May 2024 12:42:17 +0200 Subject: [PATCH 035/103] Rename events to event --- db/migrations/20240431000000_ordinals/migration.sql | 6 +++--- src/ord_db_client.rs | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/db/migrations/20240431000000_ordinals/migration.sql b/db/migrations/20240431000000_ordinals/migration.sql index d48c9de5fd..0392318e07 100644 --- a/db/migrations/20240431000000_ordinals/migration.sql +++ b/db/migrations/20240431000000_ordinals/migration.sql @@ -1,4 +1,4 @@ -CREATE TABLE events +CREATE TABLE event ( id SERIAL PRIMARY KEY, type_id SMALLINT NOT NULL, --1,InscriptionCreated;2,InscriptionTransferred @@ -9,8 +9,8 @@ CREATE TABLE events created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP ); -CREATE INDEX idx_events_block_height ON events (block_height); -CREATE INDEX idx_events_inscription_id ON events (inscription_id); +CREATE INDEX idx_events_block_height ON event (block_height); +CREATE INDEX idx_events_inscription_id ON event (inscription_id); CREATE TABLE inscription ( diff --git a/src/ord_db_client.rs b/src/ord_db_client.rs index 41593e8dfe..077ad62939 100644 --- a/src/ord_db_client.rs +++ b/src/ord_db_client.rs @@ -53,7 +53,7 @@ impl OrdDbClient { ) -> Result, sqlx::Error> { let query = " SELECT type_id, block_height, inscription_id, location, old_location - FROM events + FROM event WHERE block_height = $1 ORDER BY type_id ASC, id ASC"; @@ -78,10 +78,10 @@ impl OrdDbClient { location: &Option, ) -> Result<(), sqlx::Error> { let query = " - INSERT INTO events (type_id, block_height, inscription_id, location) + INSERT INTO event (type_id, block_height, inscription_id, location) SELECT $1, $2, $3, $4 WHERE NOT EXISTS ( - SELECT 1 FROM events + SELECT 1 FROM event WHERE type_id = $1 AND block_height = $2 AND inscription_id = $3 AND location = $4 )"; sqlx::query(query) @@ -102,10 +102,10 @@ impl OrdDbClient { old_location: &SatPoint, ) -> Result<(), sqlx::Error> { let query = " - INSERT INTO events (type_id, block_height, inscription_id, location, old_location) + INSERT INTO event (type_id, block_height, inscription_id, location, old_location) SELECT $1, $2, $3, $4, $5 WHERE NOT EXISTS ( - SELECT 1 FROM events + SELECT 1 FROM event WHERE type_id = $1 AND block_height = $2 AND inscription_id = $3 AND location = $4 AND old_location = $5 )"; sqlx::query(query) From f5950d1ef4dc46774ae4c8341b536ef6ae4fe798 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 3 May 2024 12:51:21 +0200 Subject: [PATCH 036/103] Update README.md --- README.md | 143 +++++------------------------------------------------- 1 file changed, 12 insertions(+), 131 deletions(-) diff --git a/README.md b/README.md index 3f4465ef7a..78785e9e35 100644 --- a/README.md +++ b/README.md @@ -42,48 +42,6 @@ as well as hosting costs for [ordinals.com](https://ordinals.com). Thank you for donating! -Wallet ------- - -`ord` relies on Bitcoin Core for private key management and transaction signing. -This has a number of implications that you must understand in order to use -`ord` wallet commands safely: - -- Bitcoin Core is not aware of inscriptions and does not perform sat - control. Using `bitcoin-cli` commands and RPC calls with `ord` wallets may - lead to loss of inscriptions. - -- `ord wallet` commands automatically load the `ord` wallet given by the - `--name` option, which defaults to 'ord'. Keep in mind that after running - an `ord wallet` command, an `ord` wallet may be loaded. - -- Because `ord` has access to your Bitcoin Core wallets, `ord` should not be - used with wallets that contain a material amount of funds. Keep ordinal and - cardinal wallets segregated. - -### Pre-alpha wallet migration - -Alpha `ord` wallets are not compatible with wallets created by previous -versions of `ord`. To migrate, use `ord wallet send` from the old wallet to -send sats and inscriptions to addresses generated by the new wallet with `ord -wallet receive`. - -Installation ------------- - -`ord` is written in Rust and can be built from -[source](https://github.com/ordinals/ord). Pre-built binaries are available on the -[releases page](https://github.com/ordinals/ord/releases). - -You can install the latest pre-built binary from the command line with: - -```sh -curl --proto '=https' --tlsv1.2 -fsLS https://ordinals.com/install.sh | bash -s -``` - -Once `ord` is installed, you should be able to run `ord --version` on the -command line. - Building -------- @@ -140,23 +98,6 @@ A Docker image can be built with: docker build -t ordinals/ord . ``` -### Homebrew - -`ord` is available in [Homebrew](https://brew.sh/): - -``` -brew install ord -``` - -### Debian Package - -To build a `.deb` package: - -``` -cargo install cargo-deb -cargo deb -``` - Contributing ------------ @@ -255,6 +196,18 @@ bitcoin_rpc_username: foo bitcoin_rpc_password: bar ``` +Examples running via command line: + +To run ord event server +``` +ord --bitcoin-rpc-url "http://localhost:18332" --bitcoin-rpc-username user --bitcoin-rpc-password password --commit-interval 10 --rabbitmq-url localhost --rabbitmq-password s3cr3t --rabbitmq-username indexer --rabbitmq-exchange ord-tx --chain testnet --data-dir ./index-data event-server --http-port 8080 +``` + +To run ord event indexer +``` +ord --rabbitmq-url localhost --rabbitmq-password s3cr3t --rabbitmq-username indexer event-consumer --blocks-queue btc-blocks-q --inscriptions-queue btc-inscription-q --database-url postgresql://backend:looks-backend@localhost:55432/ordinals --ord-api-url http://localhost:8080 +``` + Logging -------- @@ -265,75 +218,3 @@ the server and show `info`-level log messages and above: ``` $ RUST_LOG=info cargo run server ``` - -New Releases ------------- - -Release commit messages use the following template: - -``` -Release x.y.z - -- Bump version: x.y.z → x.y.z -- Update changelog -- Update changelog contributor credits -- Update dependencies -``` - -Translations ------------- - -To translate [the docs](https://docs.ordinals.com) we use -[mdBook i18n helper](https://github.com/google/mdbook-i18n-helpers). - -See -[mdbook-i18n-helpers usage guide](https://github.com/google/mdbook-i18n-helpers/blob/main/i18n-helpers/USAGE.md) -for help. - -Adding a new translations is somewhat involved, so feel free to start -translation and open a pull request, even if your translation is incomplete. - -Take a look at -[this commit](https://github.com/ordinals/ord/commit/329f31bf6dac207dad001507dd6f18c87fdef355) -for an example of adding a new translation. A maintainer will help you integrate it -into our build system. - -To start a new translation: - -1. Install `mdbook`, `mdbook-i18n-helpers`, and `mdbook-linkcheck`: - - ``` - cargo install mdbook mdbook-i18n-helpers mdbook-linkcheck - ``` - -2. Generate a new `pot` file named `messages.pot`: - - ``` - MDBOOK_OUTPUT='{"xgettext": {"pot-file": "messages.pot"}}' - mdbook build -d po - ``` - -3. Run `msgmerge` on `XX.po` where `XX` is the two-letter - [ISO-639](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) code for - the language you are translating into. This will update the `po` file with - the text of the most recent English version: - - ``` - msgmerge --update po/XX.po po/messages.pot - ``` - -4. Untranslated sections are marked with `#, fuzzy` in `XX.po`. Edit the - `msgstr` string with the translated text. - -5. Execute the `mdbook` command to rebuild the docs. For Chinese, whose - two-letter ISO-639 code is `zh`: - - ``` - mdbook build docs -d build - MDBOOK_BOOK__LANGUAGE=zh mdbook build docs -d build/zh - mv docs/build/zh/html docs/build/html/zh - python3 -m http.server --directory docs/build/html --bind 127.0.0.1 8080 - ``` - -6. If everything looks good, commit `XX.po` and open a pull request on GitHub. - Other changed files should be omitted from the pull request. From 915dbcf747e7693248e807c1588b602c574612d4 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 3 May 2024 12:57:06 +0200 Subject: [PATCH 037/103] Fix warnings --- src/event_consumer.rs | 8 ++++---- src/ord_api_client.rs | 4 ++-- src/subcommand.rs | 2 -- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/event_consumer.rs b/src/event_consumer.rs index ae30527f3a..ffed1107a6 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -207,11 +207,11 @@ impl EventConsumer { } => ord_indexation.sync_blocks(from_height, to_height).await?, Event::InscriptionCreated { block_height, - charms, + charms: _charms, inscription_id, location, - parent_inscription_ids, - sequence_number, + parent_inscription_ids: _parent_inscription_ids, + sequence_number: _sequence_number, } => { ord_indexation .save_inscription_created(block_height, inscription_id, location) @@ -222,7 +222,7 @@ impl EventConsumer { inscription_id, new_location, old_location, - sequence_number, + sequence_number: _sequence_number, } => { ord_indexation .save_inscription_transferred(block_height, inscription_id, new_location, old_location) diff --git a/src/ord_api_client.rs b/src/ord_api_client.rs index 93cefdb1db..0cda4eac38 100644 --- a/src/ord_api_client.rs +++ b/src/ord_api_client.rs @@ -37,7 +37,7 @@ impl OrdApiClient { let mut delay = Duration::from_secs(1); while attempts < max_attempts { - let mut request = request_builder + let request = request_builder .try_clone() .ok_or_else(|| anyhow!("Failed to clone request"))?; @@ -63,7 +63,7 @@ impl OrdApiClient { } Err(e) => return Err(anyhow!(e)), }, - Err(e) => { + Err(_e) => { attempts += 1; sleep(delay).await; delay *= 2; diff --git a/src/subcommand.rs b/src/subcommand.rs index 30d4d9d4f6..6f428ad704 100644 --- a/src/subcommand.rs +++ b/src/subcommand.rs @@ -1,6 +1,4 @@ use crate::event_publisher::EventPublisher; -use crate::ord_api_client::OrdApiClient; -use crate::ord_db_client::OrdDbClient; use super::*; From a02a4964f6849ee16cd9a972cdf82b8b3c050e14 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 3 May 2024 13:07:50 +0200 Subject: [PATCH 038/103] Add GH actions --- .github/workflows/ci.yaml | 135 ++++++++++----------------------- .github/workflows/docker.yaml | 83 ++++++++++++++++++++ .github/workflows/release.yaml | 81 -------------------- 3 files changed, 125 insertions(+), 174 deletions(-) create mode 100644 .github/workflows/docker.yaml delete mode 100644 .github/workflows/release.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2f753fd1fc..8afecfb0d3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -3,10 +3,10 @@ name: CI on: push: branches: - - master + - main + tags: + - '*' pull_request: - branches: - - master defaults: run: @@ -14,105 +14,54 @@ defaults: env: RUSTFLAGS: --deny warnings - LANGUAGES: de fr es pt ru zh ja ko fil ar hi it jobs: - docs: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Install Rust Toolchain Components - uses: actions-rs/toolchain@v1 - with: - override: true - toolchain: stable - - - uses: Swatinem/rust-cache@v2 - - - uses: peaceiris/actions-mdbook@v1 - with: - mdbook-version: latest - - - name: Install mdbook-i18n-helpers - run: cargo install mdbook-i18n-helpers - - - name: Install mdbook-linkcheck - run: | - mkdir -p mdbook-linkcheck - cd mdbook-linkcheck - wget https://github.com/Michael-F-Bryan/mdbook-linkcheck/releases/latest/download/mdbook-linkcheck.x86_64-unknown-linux-gnu.zip - unzip mdbook-linkcheck.x86_64-unknown-linux-gnu.zip - chmod +x mdbook-linkcheck - pwd >> $GITHUB_PATH - - - name: Build docs - run: mdbook build docs -d build - - - name: Build all translations for docs - run: | - for lang in ${{ env.LANGUAGES }}; do - echo "::group::Building $lang translation" - MDBOOK_BOOK__LANGUAGE=$lang \ - mdbook build docs -d build/$lang - mv docs/build/$lang/html docs/build/html/$lang - echo "::endgroup::" - done - - - name: Deploy Pages - uses: peaceiris/actions-gh-pages@v3 - if: github.ref == 'refs/heads/master' - with: - github_token: ${{secrets.GITHUB_TOKEN}} - publish_branch: gh-pages - publish_dir: docs/build/html - lint: - runs-on: ubuntu-latest - + timeout-minutes: 5 + runs-on: self-hosted steps: - - uses: actions/checkout@v2 + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 - - name: Install Rust Toolchain Components - uses: actions-rs/toolchain@v1 - with: - components: clippy, rustfmt - override: true - toolchain: stable + - name: Install Rust Toolchain Components + uses: actions-rs/toolchain@v1 + with: + components: clippy, rustfmt + override: true + toolchain: stable - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@v2 - - name: Clippy - run: cargo clippy --all --all-targets + - name: Clippy + run: cargo clippy --all --all-targets - - name: Format - run: cargo fmt --all -- --check + - name: Format + run: cargo fmt --all -- --check - - name: Check for Forbidden Words - run: | - sudo apt-get install ripgrep - ./bin/forbid + - name: Check for Forbidden Words + run: | + sudo apt-get install ripgrep + ./bin/forbid test: - strategy: - matrix: - os: - - macos-latest - - ubuntu-latest - - windows-latest - - runs-on: ${{matrix.os}} - + timeout-minutes: 5 + runs-on: self-hosted steps: - - uses: actions/checkout@v2 - - - name: Install Rust Toolchain Components - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - - - uses: Swatinem/rust-cache@v2 - - - name: Test - run: cargo test --all + - + name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install Rust Toolchain Components + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + + - uses: Swatinem/rust-cache@v2 + + - name: Test + run: cargo test --all diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml new file mode 100644 index 0000000000..59cdc4085e --- /dev/null +++ b/.github/workflows/docker.yaml @@ -0,0 +1,83 @@ +name: docker + +on: + push: + branches: + - main + tags: + - '*' + pull_request: + +jobs: + ordinals-indexer: + timeout-minutes: 10 + runs-on: self-hosted + steps: + - + name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - + name: Login to ECR + uses: docker/login-action@v3 + with: + registry: 726418723176.dkr.ecr.us-east-1.amazonaws.com + username: ${{ secrets.LR_PRD_AWS_ACCESS_KEY_ID }} + password: ${{ secrets.LR_PRD_AWS_SECRET_ACCESS_KEY }} + - + name: Build SHA tag + id: vars + run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + - + name: Create Docker builder context + run: docker context create builders + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker-container + endpoint: builders + platforms: linux/arm64 + version: latest + - + name: Bump version and push tag + uses: LooksRare/github-tag-action@1.0.0 + if: github.event_name != 'pull_request' + id: version + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERBOSE: false + WITH_V: true + - + name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + flavor: | + latest=auto + prefix= + suffix= + # list of Docker images to use as base name for tags + images: | + 726418723176.dkr.ecr.us-east-1.amazonaws.com/ord + # generate Docker tags based on the following events/attributes + tags: | + latest + type=schedule + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{raw}},value=${{ steps.version.outputs.tag }} + type=semver,pattern=v{{major}}.{{minor}},value=${{ steps.version.outputs.tag }},enable={{is_default_branch}} + type=semver,pattern=v{{major}},value=${{ steps.version.outputs.tag }},enable={{is_default_branch}} + type=sha,prefix=,format=short + - + name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + file: Dockerfile + platforms: linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml deleted file mode 100644 index 374d8f0b1b..0000000000 --- a/.github/workflows/release.yaml +++ /dev/null @@ -1,81 +0,0 @@ -name: Release - -on: - push: - tags: - - '*' - -defaults: - run: - shell: bash - -jobs: - release: - strategy: - fail-fast: false - matrix: - target: - - aarch64-apple-darwin - - x86_64-apple-darwin - - x86_64-pc-windows-msvc - - x86_64-unknown-linux-gnu - include: - - target: aarch64-apple-darwin - os: macos-latest - target_rustflags: '' - - target: x86_64-apple-darwin - os: macos-latest - target_rustflags: '' - - target: x86_64-pc-windows-msvc - os: windows-latest - target_rustflags: '' - - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - target_rustflags: '' - - runs-on: ${{matrix.os}} - - steps: - - uses: actions/checkout@v2 - - - name: Install Rust Toolchain Components - uses: actions-rs/toolchain@v1 - with: - override: true - target: ${{ matrix.target }} - toolchain: stable - - - name: Install Linux Dependencies - if: ${{ matrix.os == 'ubuntu-latest' }} - run: | - sudo apt-get update - sudo apt-get install musl-tools libssl-dev pkg-config - - - name: Release Type - id: release-type - run: | - if [[ ${{ github.ref }} =~ ^refs/tags/[0-9]+[.][0-9]+[.][0-9]+$ ]]; then - echo ::set-output name=value::release - else - echo ::set-output name=value::prerelease - fi - - - name: Package - id: package - env: - TARGET: ${{ matrix.target }} - REF: ${{ github.ref }} - OS: ${{ matrix.os }} - TARGET_RUSTFLAGS: ${{ matrix.target_rustflags }} - run: ./bin/package - shell: bash - - - name: Publish Archive - uses: softprops/action-gh-release@v0.1.15 - if: ${{ startsWith(github.ref, 'refs/tags/') }} - with: - draft: false - files: ${{ steps.package.outputs.archive }} - prerelease: ${{ steps.release-type.outputs.value == 'prerelease' }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From edbdb319803edd5187e8ab4423c9959ed46b1640 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 3 May 2024 13:53:10 +0200 Subject: [PATCH 039/103] Filter out empty json --- src/ord_indexation.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ord_indexation.rs b/src/ord_indexation.rs index b2eca1c262..026f7afa65 100644 --- a/src/ord_indexation.rs +++ b/src/ord_indexation.rs @@ -124,9 +124,8 @@ impl OrdIndexation { match result { Ok(value) => { match &value { - Value::Object(_) | Value::Array(_) => { - serde_json::to_string(&value).ok() - }, + Value::Object(obj) if obj.is_empty() => None, + Value::Object(_) | Value::Array(_) => serde_json::to_string(&value).ok(), _ => { Some(hex::encode(bytes)) } From db0ff208fbc47a61701a63b2faf70a346ba34e23 Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Fri, 3 May 2024 18:58:30 +0700 Subject: [PATCH 040/103] Runner --- .github/workflows/docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 59cdc4085e..a2418108da 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -11,7 +11,7 @@ on: jobs: ordinals-indexer: timeout-minutes: 10 - runs-on: self-hosted + runs-on: [self-hosted] steps: - name: Checkout From bcb574b7b6d1d608c7367883a900c5e9717aee55 Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Fri, 3 May 2024 19:00:23 +0700 Subject: [PATCH 041/103] Runner only build --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8afecfb0d3..e9c79b7ddd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,7 +18,7 @@ env: jobs: lint: timeout-minutes: 5 - runs-on: self-hosted + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 From 8fc832487625304ac448169534b82ba8b55a51d5 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 3 May 2024 14:22:27 +0200 Subject: [PATCH 042/103] Increase timeout --- .github/workflows/docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index a2418108da..0c295e6e90 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -10,7 +10,7 @@ on: jobs: ordinals-indexer: - timeout-minutes: 10 + timeout-minutes: 60 runs-on: [self-hosted] steps: - From 2543b1faf824add188e7d03f8f71da81b3dd3b8d Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 3 May 2024 14:44:12 +0200 Subject: [PATCH 043/103] Fix lint --- src/ord_db_client.rs | 4 ++-- src/ord_indexation.rs | 18 ++++++------------ 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/ord_db_client.rs b/src/ord_db_client.rs index 077ad62939..35c80c27d9 100644 --- a/src/ord_db_client.rs +++ b/src/ord_db_client.rs @@ -3,8 +3,8 @@ use std::sync::Arc; use bitcoin::{OutPoint, Txid}; use futures::TryStreamExt; -use sqlx::{PgPool, Row}; use sqlx::types::Json; +use sqlx::{PgPool, Row}; use ordinals::SatPoint; @@ -175,7 +175,7 @@ impl OrdDbClient { parents = COALESCE(EXCLUDED.parents, inscription.parents) RETURNING id; "; - let id = sqlx::query_as::<_, (i32, )>(query) + let id = sqlx::query_as::<_, (i32,)>(query) .bind(inscription_details.id.to_string()) .bind(inscription_details.number) .bind(inscription_details.content_type.as_deref()) diff --git a/src/ord_indexation.rs b/src/ord_indexation.rs index 026f7afa65..ff6de8a7fe 100644 --- a/src/ord_indexation.rs +++ b/src/ord_indexation.rs @@ -7,10 +7,10 @@ use serde_json::Value; use ordinals::SatPoint; use crate::api::BlockInfo; -use crate::InscriptionId; use crate::ord_api_client::OrdApiClient; use crate::ord_db_client::{Event, OrdDbClient}; use crate::settings::Settings; +use crate::InscriptionId; pub struct OrdIndexation { settings: Settings, @@ -122,18 +122,12 @@ impl OrdIndexation { let cursor = Cursor::new(bytes); let result = from_reader(cursor); match result { - Ok(value) => { - match &value { - Value::Object(obj) if obj.is_empty() => None, - Value::Object(_) | Value::Array(_) => serde_json::to_string(&value).ok(), - _ => { - Some(hex::encode(bytes)) - } - } + Ok(value) => match &value { + Value::Object(obj) if obj.is_empty() => None, + Value::Object(_) | Value::Array(_) => serde_json::to_string(&value).ok(), + _ => Some(hex::encode(bytes)), }, - Err(_) => { - Some(hex::encode(bytes)) - } + Err(_) => Some(hex::encode(bytes)), } }) } From 869ef89cf07726ba1e6494041cd4f034a8569c30 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 3 May 2024 15:29:03 +0200 Subject: [PATCH 044/103] Fix lint --- src/index/updater.rs | 16 +++++++++++---- src/ord_db_client.rs | 48 +++++++++++++++++++++---------------------- src/ord_indexation.rs | 15 ++++++-------- 3 files changed, 42 insertions(+), 37 deletions(-) diff --git a/src/index/updater.rs b/src/index/updater.rs index 72790d245b..14855e4b47 100644 --- a/src/index/updater.rs +++ b/src/index/updater.rs @@ -745,10 +745,18 @@ impl<'index> Updater<'index> { Reorg::update_savepoints(self.index, self.height)?; if let Some(sender) = self.index.event_sender.as_ref() { - sender.blocking_send(Event::BlockCommitted { - from_height: self.height - uncommitted as u32, - to_height: self.height, - })?; + if let Ok(uncommitted) = u32::try_from(uncommitted) { + sender.blocking_send(Event::BlockCommitted { + from_height: self.height - uncommitted, + to_height: self.height, + })?; + } else { + log::error!( + "Failed to publish block range from_height: {}, uncommitted: {}", + self.height, + uncommitted + ); + } } Ok(()) diff --git a/src/ord_db_client.rs b/src/ord_db_client.rs index 35c80c27d9..d4be4e1890 100644 --- a/src/ord_db_client.rs +++ b/src/ord_db_client.rs @@ -54,11 +54,11 @@ impl OrdDbClient { let query = " SELECT type_id, block_height, inscription_id, location, old_location FROM event - WHERE block_height = $1 + WHERE block_height = $1::BIGINT ORDER BY type_id ASC, id ASC"; let mut rows = sqlx::query(query) - .bind(block_height as i64) + .bind(block_height.to_string()) .fetch(&*self.pool); let mut events = Vec::new(); @@ -79,14 +79,14 @@ impl OrdDbClient { ) -> Result<(), sqlx::Error> { let query = " INSERT INTO event (type_id, block_height, inscription_id, location) - SELECT $1, $2, $3, $4 + SELECT $1, $2::BIGINT, $3, $4 WHERE NOT EXISTS ( SELECT 1 FROM event - WHERE type_id = $1 AND block_height = $2 AND inscription_id = $3 AND location = $4 + WHERE type_id = $1 AND block_height = $2::BIGINT AND inscription_id = $3 AND location = $4 )"; sqlx::query(query) .bind(1_i32) // Type ID for InscriptionCreated - .bind(*block_height as i64) + .bind(block_height.to_string()) .bind(inscription_id.to_string()) .bind(location.map(|loc| loc.to_string())) .execute(&*self.pool) @@ -103,14 +103,14 @@ impl OrdDbClient { ) -> Result<(), sqlx::Error> { let query = " INSERT INTO event (type_id, block_height, inscription_id, location, old_location) - SELECT $1, $2, $3, $4, $5 + SELECT $1, $2::BIGINT, $3, $4, $5 WHERE NOT EXISTS ( SELECT 1 FROM event - WHERE type_id = $1 AND block_height = $2 AND inscription_id = $3 AND location = $4 AND old_location = $5 + WHERE type_id = $1 AND block_height = $2::BIGINT AND inscription_id = $3 AND location = $4 AND old_location = $5 )"; sqlx::query(query) .bind(2_i32) // Type ID for InscriptionTransferred - .bind(*block_height as i64) + .bind(block_height.to_string()) .bind(inscription_id.to_string()) .bind(new_location.to_string()) .bind(old_location.to_string()) @@ -157,7 +157,7 @@ impl OrdDbClient { charms, children, parents - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) + ) VALUES ($1, $2, $3, $4::INTEGER, $5, $6::INTEGER, $7::BIGINT, $8::BIGINT, $9, $10::INTEGER, $11::BIGINT, $12::BIGINT, $13::SMALLINT, $14, $15) ON CONFLICT (genesis_id) DO UPDATE SET number = EXCLUDED.number, content_type = EXCLUDED.content_type, @@ -179,16 +179,16 @@ impl OrdDbClient { .bind(inscription_details.id.to_string()) .bind(inscription_details.number) .bind(inscription_details.content_type.as_deref()) - .bind(inscription_details.content_length.map(|n| n as i32)) + .bind(inscription_details.content_length.map(|n| n.to_string())) .bind(metadata) - .bind(inscription_details.genesis_block_height as i32) + .bind(inscription_details.genesis_block_height.to_string()) .bind(inscription_details.genesis_block_time) - .bind(inscription_details.sat_number.map(|n| n as i64)) + .bind(inscription_details.sat_number.map(|n| n.to_string())) .bind(inscription_details.sat_rarity.map(|r| r.to_i16())) - .bind(inscription_details.sat_block_height.map(|n| n as i32)) + .bind(inscription_details.sat_block_height.map(|n| n.to_string())) .bind(inscription_details.sat_block_time) - .bind(inscription_details.fee as i64) - .bind(inscription_details.charms as i16) + .bind(inscription_details.fee.to_string()) + .bind(inscription_details.charms.to_string()) .bind(Json(&inscription_details.children)) .bind(Json(&inscription_details.parents)) .fetch_one(&*self.pool) @@ -225,35 +225,35 @@ impl OrdDbClient { value ) SELECT - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11 + $1, $2, $3::BIGINT, $4, $5, $6, $7::BIGINT, $8, $9, $10::BIGINT, $11::BIGINT WHERE NOT EXISTS ( SELECT 1 FROM location WHERE inscription_id = $1 AND block_height = $2 AND - block_time = $3 AND + block_time = $3::BIGINT AND tx_id = $4 AND to_address = $5 AND cur_output = $6 AND - cur_offset = $7 AND + cur_offset = $7::BIGINT AND from_address = $8 AND prev_output = $9 AND - prev_offset = $10 AND - value = $11 + prev_offset = $10::BIGINT AND + value = $10::BIGINT ); "; sqlx::query(query) .bind(id) .bind(block_height) - .bind(block_time as i64) + .bind(block_time.to_string()) .bind(tx_id.map(|n| n.to_string())) .bind(to_address) .bind(to_outpoint.map(|n| n.to_string())) - .bind(to_offset.map(|n| n as i64)) + .bind(to_offset.map(|n| n.to_string())) .bind(from_address) .bind(from_outpoint.map(|n| n.to_string())) - .bind(from_offset.map(|n| n as i64)) - .bind(value.map(|n| n as i64)) + .bind(from_offset.map(|n| n.to_string())) + .bind(value.map(|n| n.to_string())) .execute(&*self.pool) .await?; Ok(()) diff --git a/src/ord_indexation.rs b/src/ord_indexation.rs index ff6de8a7fe..ca5fa317fa 100644 --- a/src/ord_indexation.rs +++ b/src/ord_indexation.rs @@ -101,11 +101,11 @@ impl OrdIndexation { inscription_id, event.block_height, block_info.timestamp, - event.location.as_ref().map(|loc| loc.outpoint.txid.clone()), + event.location.as_ref().map(|loc| loc.outpoint.txid), to_location_details .as_ref() .map(|details| details.0.clone()), - event.location.as_ref().map(|loc| loc.outpoint.clone()), + event.location.as_ref().map(|loc| loc.outpoint), event.location.as_ref().map(|loc| loc.offset), None, None, @@ -164,16 +164,16 @@ impl OrdIndexation { inscription_id, event.block_height, block_info.timestamp, - event.location.as_ref().map(|loc| loc.outpoint.txid.clone()), + event.location.as_ref().map(|loc| loc.outpoint.txid), to_location_details .as_ref() .map(|details| details.0.clone()), - event.location.as_ref().map(|loc| loc.outpoint.clone()), + event.location.as_ref().map(|loc| loc.outpoint), event.location.as_ref().map(|loc| loc.offset), from_location_details .as_ref() .map(|details| details.0.clone()), - event.old_location.as_ref().map(|loc| loc.outpoint.clone()), + event.old_location.as_ref().map(|loc| loc.outpoint), event.old_location.as_ref().map(|loc| loc.offset), to_location_details.as_ref().map(|details| details.1), ) @@ -186,10 +186,7 @@ impl OrdIndexation { &self, location: &SatPoint, ) -> Result, anyhow::Error> { - let tx_details = self - .ord_api_client - .fetch_tx(location.outpoint.txid.clone()) - .await?; + let tx_details = self.ord_api_client.fetch_tx(location.outpoint.txid).await?; let output = tx_details .transaction From f33d02efdccff21eb939e7dabd14897d36ff517e Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 3 May 2024 16:32:01 +0200 Subject: [PATCH 045/103] Remove unnecessary check --- .github/workflows/ci.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e9c79b7ddd..33e50a3187 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -40,11 +40,6 @@ jobs: - name: Format run: cargo fmt --all -- --check - - name: Check for Forbidden Words - run: | - sudo apt-get install ripgrep - ./bin/forbid - test: timeout-minutes: 5 runs-on: self-hosted From 935c78d27e2fe95412d5529833bdbcb5cee4999c Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 3 May 2024 16:32:24 +0200 Subject: [PATCH 046/103] Fix tests --- src/index.rs | 163 ++++++++++++++++++++++----------------- src/settings.rs | 1 + src/subcommand/server.rs | 56 +++++++++++++- 3 files changed, 147 insertions(+), 73 deletions(-) diff --git a/src/index.rs b/src/index.rs index bb2569d6e7..f90be1b314 100644 --- a/src/index.rs +++ b/src/index.rs @@ -6318,23 +6318,28 @@ mod tests { }; let create_event = event_receiver.blocking_recv().unwrap(); let expected_charms = if context.index.index_sats { 513 } else { 0 }; - assert_eq!( - create_event, - Event::InscriptionCreated { - inscription_id, - location: Some(SatPoint { - outpoint: OutPoint { - txid: create_txid, - vout: 0 - }, - offset: 0 - }), - sequence_number: 0, - block_height: 2, - charms: expected_charms, - parent_inscription_ids: Vec::new(), + + let expected_event = Event::InscriptionCreated { + inscription_id, + location: Some(SatPoint { + outpoint: OutPoint { + txid: create_txid, + vout: 0 + }, + offset: 0 + }), + sequence_number: 0, + block_height: 2, + charms: expected_charms, + parent_inscription_ids: Vec::new(), + }; + loop { + let received_event = event_receiver.blocking_recv().unwrap(); + if received_event == expected_event { + assert_eq!(received_event, expected_event); + break; } - ); + } // Transfer inscription let transfer_txid = context.core.broadcast_tx(TransactionTemplate { @@ -6347,28 +6352,32 @@ mod tests { context.mine_blocks(1); let transfer_event = event_receiver.blocking_recv().unwrap(); - assert_eq!( - transfer_event, - Event::InscriptionTransferred { - block_height: 3, - inscription_id, - new_location: SatPoint { - outpoint: OutPoint { - txid: transfer_txid, - vout: 0 - }, - offset: 0 + let expected_event = Event::InscriptionTransferred { + block_height: 3, + inscription_id, + new_location: SatPoint { + outpoint: OutPoint { + txid: transfer_txid, + vout: 0 }, - old_location: SatPoint { - outpoint: OutPoint { - txid: create_txid, - vout: 0 - }, - offset: 0 + offset: 0 + }, + old_location: SatPoint { + outpoint: OutPoint { + txid: create_txid, + vout: 0 }, - sequence_number: 0, + offset: 0 + }, + sequence_number: 0, + }; + loop { + let received_event = event_receiver.blocking_recv().unwrap(); + if received_event == expected_event { + assert_eq!(received_event, expected_event); + break; } - ); + } } #[test] @@ -6420,14 +6429,18 @@ mod tests { [], ); - assert_eq!( - event_receiver.blocking_recv().unwrap(), - Event::RuneEtched { - block_height: 8, - txid: txid0, - rune_id: id, + let expected_event = Event::RuneEtched { + block_height: 8, + txid: txid0, + rune_id: id, + }; + loop { + let received_event = event_receiver.blocking_recv().unwrap(); + if received_event == expected_event { + pretty_assert_eq!(received_event, expected_event); + break; } - ); + } let txid1 = context.core.broadcast_tx(TransactionTemplate { inputs: &[(2, 0, 0, Witness::new())], @@ -6473,15 +6486,19 @@ mod tests { )], ); - assert_eq!( - event_receiver.blocking_recv().unwrap(), - Event::RuneMinted { - block_height: 9, - txid: txid1, - rune_id: id, - amount: 1000, + let expected_event = Event::RuneMinted { + block_height: 9, + txid: txid1, + rune_id: id, + amount: 1000, + }; + loop { + let received_event = event_receiver.blocking_recv().unwrap(); + if received_event == expected_event { + assert_eq!(received_event, expected_event); + break; } - ); + } let txid2 = context.core.broadcast_tx(TransactionTemplate { inputs: &[(9, 1, 0, Witness::new())], @@ -6532,19 +6549,23 @@ mod tests { event_receiver.blocking_recv().unwrap(); - pretty_assert_eq!( - event_receiver.blocking_recv().unwrap(), - Event::RuneTransferred { - block_height: 10, + let expected_event = Event::RuneTransferred { + block_height: 10, + txid: txid2, + rune_id: id, + amount: 1000, + outpoint: OutPoint { txid: txid2, - rune_id: id, - amount: 1000, - outpoint: OutPoint { - txid: txid2, - vout: 0, - }, + vout: 0, + }, + }; + loop { + let received_event = event_receiver.blocking_recv().unwrap(); + if received_event == expected_event { + pretty_assert_eq!(received_event, expected_event); + break; } - ); + } let txid3 = context.core.broadcast_tx(TransactionTemplate { inputs: &[(10, 1, 0, Witness::new())], @@ -6597,14 +6618,18 @@ mod tests { event_receiver.blocking_recv().unwrap(); - pretty_assert_eq!( - event_receiver.blocking_recv().unwrap(), - Event::RuneBurned { - block_height: 11, - txid: txid3, - amount: 111, - rune_id: id, + let expected_event = Event::RuneBurned { + block_height: 11, + txid: txid3, + amount: 111, + rune_id: id, + }; + loop { + let received_event = event_receiver.blocking_recv().unwrap(); + if received_event == expected_event { + pretty_assert_eq!(received_event, expected_event); + break; } - ); + } } } diff --git a/src/settings.rs b/src/settings.rs index 1521dee238..25f1babbba 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -1036,6 +1036,7 @@ mod tests { ("NO_INDEX_INSCRIPTIONS", "1"), ("SERVER_PASSWORD", "server password"), ("SERVER_URL", "server url"), + ("SERVER_USERNAME", "server username"), ("RMQ_URL", "http://127.0.0.1"), ("RMQ_USERNAME", "rmq username"), ("RMQ_PASSWORD", "rmq password"), diff --git a/src/subcommand/server.rs b/src/subcommand/server.rs index aa2ad255d5..5e382ca93e 100644 --- a/src/subcommand/server.rs +++ b/src/subcommand/server.rs @@ -203,6 +203,7 @@ impl Server { .route("/feed.xml", get(Self::feed)) .route("/input/:block/:transaction/:input", get(Self::input)) .route("/inscription/:inscription_query", get(Self::inscription)) + .route("/inscription/:inscription_query/details", get(Self::inscription_details)) .route("/inscriptions", get(Self::inscriptions)) .route("/inscriptions", post(Self::inscriptions_json)) .route("/inscriptions/:page", get(Self::inscriptions_paginated)) @@ -1534,6 +1535,53 @@ impl Server { } } + let (info, txout, inscription) = index + .inscription_info(query)? + .ok_or_not_found(|| format!("inscription {query}"))?; + + Ok(if accept_json { + Json(info).into_response() + } else { + InscriptionHtml { + chain: server_config.chain, + charms: Charm::Vindicated.unset(info.charms.iter().fold(0, |mut acc, charm| { + charm.set(&mut acc); + acc + })), + children: info.children, + fee: info.fee, + height: info.height, + inscription, + id: info.id, + number: info.number, + next: info.next, + output: txout, + parents: info.parents, + previous: info.previous, + rune: info.rune, + sat: info.sat, + satpoint: info.satpoint, + timestamp: Utc.timestamp_opt(info.timestamp, 0).unwrap(), + } + .page(server_config) + .into_response() + }) + }) + } + + async fn inscription_details( + Extension(server_config): Extension>, + Extension(index): Extension>, + Path(DeserializeFromStr(query)): Path>, + AcceptJson(accept_json): AcceptJson, + ) -> ServerResult { + task::block_in_place(|| { + if let query::Inscription::Sat(_) = query { + if !index.has_sat_index() { + return Err(ServerError::NotFound("sat index required".into())); + } + } + let (info, txout, inscription) = index .inscription_info(query)? .ok_or_not_found(|| format!("inscription {query}"))?; @@ -1547,19 +1595,19 @@ impl Server { let response = InscriptionDetails { id: info.id, number: info.number, - owner: info.address, + owner: info.address, //not used content_type: info.content_type, content_length: info.content_length, metadata: inscription.metadata, genesis_block_height: info.height, genesis_block_time: info.timestamp, sat_number: info.sat.map(|sat| sat.0), - sat_name: info.sat.map(|sat| sat.name()), + sat_name: info.sat.map(|sat| sat.name()), //not used sat_rarity: info.sat.map(|sat| sat.rarity()), sat_block_height: info.sat.map(|sat| sat.height().0), sat_block_time, - satpoint: info.satpoint, - value: info.value, + satpoint: info.satpoint, //not used + value: info.value, //not used fee: info.fee, charms: Charm::Vindicated.unset(info.charms.iter().fold(0, |mut acc, charm| { charm.set(&mut acc); From beb21b5fbf2871299b1f15b7090eb6cd7cb51c7f Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 3 May 2024 16:34:32 +0200 Subject: [PATCH 047/103] Fix tests --- src/index.rs | 14 ++++++-------- src/subcommand/server.rs | 5 ++++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/index.rs b/src/index.rs index f90be1b314..62db87282e 100644 --- a/src/index.rs +++ b/src/index.rs @@ -6316,7 +6316,6 @@ mod tests { txid: create_txid, index: 0, }; - let create_event = event_receiver.blocking_recv().unwrap(); let expected_charms = if context.index.index_sats { 513 } else { 0 }; let expected_event = Event::InscriptionCreated { @@ -6324,9 +6323,9 @@ mod tests { location: Some(SatPoint { outpoint: OutPoint { txid: create_txid, - vout: 0 + vout: 0, }, - offset: 0 + offset: 0, }), sequence_number: 0, block_height: 2, @@ -6351,23 +6350,22 @@ mod tests { context.mine_blocks(1); - let transfer_event = event_receiver.blocking_recv().unwrap(); let expected_event = Event::InscriptionTransferred { block_height: 3, inscription_id, new_location: SatPoint { outpoint: OutPoint { txid: transfer_txid, - vout: 0 + vout: 0, }, - offset: 0 + offset: 0, }, old_location: SatPoint { outpoint: OutPoint { txid: create_txid, - vout: 0 + vout: 0, }, - offset: 0 + offset: 0, }, sequence_number: 0, }; diff --git a/src/subcommand/server.rs b/src/subcommand/server.rs index 5e382ca93e..baf7c5c410 100644 --- a/src/subcommand/server.rs +++ b/src/subcommand/server.rs @@ -203,7 +203,10 @@ impl Server { .route("/feed.xml", get(Self::feed)) .route("/input/:block/:transaction/:input", get(Self::input)) .route("/inscription/:inscription_query", get(Self::inscription)) - .route("/inscription/:inscription_query/details", get(Self::inscription_details)) + .route( + "/inscription/:inscription_query/details", + get(Self::inscription_details), + ) .route("/inscriptions", get(Self::inscriptions)) .route("/inscriptions", post(Self::inscriptions_json)) .route("/inscriptions/:page", get(Self::inscriptions_paginated)) From 4759dc9cc99a5df36a786ca688d49086efb905c1 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 3 May 2024 16:35:32 +0200 Subject: [PATCH 048/103] Move to new api endpoint to not clash with tests --- src/ord_api_client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ord_api_client.rs b/src/ord_api_client.rs index 0cda4eac38..1c6fec64a0 100644 --- a/src/ord_api_client.rs +++ b/src/ord_api_client.rs @@ -81,7 +81,7 @@ impl OrdApiClient { let request_builder = self .client .get(format!( - "{}/inscription/{}", + "{}/inscription/{}/details", self.ord_api_url, inscription_id )) .header("Accept", "application/json"); From b3dd4c10b3a276225b80e496e5c2a430dea56e26 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 3 May 2024 17:11:04 +0200 Subject: [PATCH 049/103] Retry any error, it sometimes gives 404 even though data exists --- src/ord_api_client.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/ord_api_client.rs b/src/ord_api_client.rs index 1c6fec64a0..84a90daa10 100644 --- a/src/ord_api_client.rs +++ b/src/ord_api_client.rs @@ -2,7 +2,6 @@ use std::time::Duration; use anyhow::anyhow; use bitcoin::Txid; -use http::StatusCode; use reqwest::{Client, RequestBuilder}; use tokio::time::sleep; @@ -52,10 +51,10 @@ impl OrdApiClient { .map_err(anyhow::Error::from); } Err(e) - if e.status() == Some(StatusCode::TOO_MANY_REQUESTS) - || e - .status() - .map_or_else(|| false, |status_code| status_code.is_server_error()) => + if e.status().map_or_else( + || false, + |status_code| status_code.is_server_error() || status_code.is_client_error(), + ) => { attempts += 1; sleep(delay).await; From c7ce8da72f1902813a71b8966b6e99d07f1a21d5 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 3 May 2024 17:50:12 +0200 Subject: [PATCH 050/103] Use amqps --- src/settings.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/settings.rs b/src/settings.rs index 1521dee238..3db33a4ac8 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -587,7 +587,7 @@ impl Settings { let pass = self.rabbitmq_password.as_ref()?; let url = self.rabbitmq_url.as_ref()?; - Some(format!("amqp://{}:{}@{}", user, pass, url)) + Some(format!("amqps://{}:{}@{}", user, pass, url)) } } From 0dd8f4cfb7fced699ef79311c96ec556aaf60ada Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 3 May 2024 18:15:46 +0200 Subject: [PATCH 051/103] Pass amq conn param --- src/event_publisher.rs | 3 ++- src/options.rs | 4 ---- src/settings.rs | 26 ++------------------------ 3 files changed, 4 insertions(+), 29 deletions(-) diff --git a/src/event_publisher.rs b/src/event_publisher.rs index 53d1f69eb2..c3469f9234 100644 --- a/src/event_publisher.rs +++ b/src/event_publisher.rs @@ -15,7 +15,8 @@ impl EventPublisher { pub fn run(settings: &Settings) -> Result { let addr = settings .rabbitmq_addr() - .context("rabbitmq amqp credentials and url must be defined")?; + .context("rabbitmq amqp credentials and url must be defined")? + .to_owned(); let exchange = settings .rabbitmq_exchange() diff --git a/src/options.rs b/src/options.rs index 5db2c33b22..bde569f9a7 100644 --- a/src/options.rs +++ b/src/options.rs @@ -84,10 +84,6 @@ pub struct Options { pub(crate) server_username: Option, #[arg(long, help = "RMQ url.")] pub(crate) rabbitmq_url: Option, - #[arg(long, help = "RMQ username.")] - pub(crate) rabbitmq_username: Option, - #[arg(long, help = "RMQ password.")] - pub(crate) rabbitmq_password: Option, #[arg(long, help = "RMQ exchange to publish index events.")] pub(crate) rabbitmq_exchange: Option, #[arg(long, short, help = "Use regtest. Equivalent to `--chain regtest`.")] diff --git a/src/settings.rs b/src/settings.rs index 3db33a4ac8..17ba40c35e 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -28,8 +28,6 @@ pub struct Settings { server_url: Option, server_username: Option, rabbitmq_url: Option, - rabbitmq_username: Option, - rabbitmq_password: Option, rabbitmq_exchange: Option, } @@ -146,8 +144,6 @@ impl Settings { server_url: self.server_url.or(source.server_url), server_username: self.server_username.or(source.server_username), rabbitmq_url: self.rabbitmq_url.or(source.rabbitmq_url), - rabbitmq_username: self.rabbitmq_username.or(source.rabbitmq_username), - rabbitmq_password: self.rabbitmq_password.or(source.rabbitmq_password), rabbitmq_exchange: self.rabbitmq_exchange.or(source.rabbitmq_exchange), } } @@ -184,8 +180,6 @@ impl Settings { server_url: None, server_username: options.server_username, rabbitmq_url: options.rabbitmq_url, - rabbitmq_username: options.rabbitmq_username, - rabbitmq_password: options.rabbitmq_password, rabbitmq_exchange: options.rabbitmq_exchange, } } @@ -266,8 +260,6 @@ impl Settings { server_url: get_string("SERVER_URL"), server_username: get_string("SERVER_USERNAME"), rabbitmq_url: get_string("RMQ_URL"), - rabbitmq_username: get_string("RMQ_USERNAME"), - rabbitmq_password: get_string("RMQ_PASSWORD"), rabbitmq_exchange: get_string("RMQ_EXCHANGE"), }) } @@ -299,8 +291,6 @@ impl Settings { server_url: Some(server_url.into()), server_username: None, rabbitmq_url: None, - rabbitmq_username: None, - rabbitmq_password: None, rabbitmq_exchange: None, } } @@ -382,8 +372,6 @@ impl Settings { server_url: self.server_url, server_username: self.server_username, rabbitmq_url: self.rabbitmq_url, - rabbitmq_username: self.rabbitmq_username, - rabbitmq_password: self.rabbitmq_password, rabbitmq_exchange: self.rabbitmq_exchange, }) } @@ -582,12 +570,8 @@ impl Settings { self.rabbitmq_exchange.as_deref() } - pub fn rabbitmq_addr(&self) -> Option { - let user = self.rabbitmq_username.as_ref()?; - let pass = self.rabbitmq_password.as_ref()?; - let url = self.rabbitmq_url.as_ref()?; - - Some(format!("amqps://{}:{}@{}", user, pass, url)) + pub fn rabbitmq_addr(&self) -> Option<&String> { + self.rabbitmq_url.as_ref() } } @@ -1084,8 +1068,6 @@ mod tests { server_url: Some("server url".into()), server_username: Some("server username".into()), rabbitmq_url: Some("http://127.0.0.1".into()), - rabbitmq_username: Some("rmq username".into()), - rabbitmq_password: Some("rmq password".into()), rabbitmq_exchange: Some("rmq exchange".into()), } ); @@ -1120,8 +1102,6 @@ mod tests { "--server-password=server password", "--server-username=server username", "--rabbitmq-url=http://127.0.0.1", - "--rabbitmq-username=rmq username", - "--rabbitmq-password=rmq password", "--rabbitmq-exchange=rmq exchange", ]) .unwrap() @@ -1152,8 +1132,6 @@ mod tests { server_url: None, server_username: Some("server username".into()), rabbitmq_url: Some("http://127.0.0.1".into()), - rabbitmq_username: Some("rmq username".into()), - rabbitmq_password: Some("rmq password".into()), rabbitmq_exchange: Some("rmq exchange".into()), } ); From 9841d0ca0cca6765791b9b2e1b5493fe089722e6 Mon Sep 17 00:00:00 2001 From: rob Date: Fri, 3 May 2024 19:40:06 +0200 Subject: [PATCH 052/103] ignore amq certs --- Cargo.lock | 1 + Cargo.toml | 2 +- src/event_consumer.rs | 25 ++++++++++++++++++++++--- src/event_publisher.rs | 17 ++++++++++++++++- src/ord_db_client.rs | 4 ++-- src/ord_indexation.rs | 18 ++++++------------ 6 files changed, 48 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 712092e412..2d78cece84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4795,6 +4795,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "495b0abdce3dc1f8fd27240651c9e68890c14e9d9c61527b1ce44d8a5a7bd3d5" dependencies = [ "cfg-if 1.0.0", + "native-tls", "p12-keystore", "rustls-connector", "rustls-pemfile 2.1.2", diff --git a/Cargo.toml b/Cargo.toml index 62a8b90a62..971bea537a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,7 @@ http = "0.2.6" humantime = "2.1.0" hyper = { version = "0.14.24", features = ["client", "http2"] } indicatif = "0.17.1" -lapin = "2.3.3" +lapin = { version = "2.3.3", features = ["native-tls"] } lazy_static = "1.4.0" log = "0.4.14" mime = "0.3.16" diff --git a/src/event_consumer.rs b/src/event_consumer.rs index ffed1107a6..719f32de0d 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -4,12 +4,18 @@ use anyhow::Context; use bitcoin::secp256k1::rand::distributions::Alphanumeric; use chrono::Utc; use clap::Parser; -use futures::StreamExt; -use lapin::{options::*, types::FieldTable, Connection, ConnectionProperties}; +use lapin::options::{ + BasicAckOptions, BasicConsumeOptions, BasicRejectOptions, ConfirmSelectOptions, +}; +use lapin::tcp::{AMQPUriTcpExt, NativeTlsConnector}; +use lapin::types::FieldTable; +use lapin::uri::AMQPUri; +use lapin::{Connection, ConnectionProperties}; use rand::distributions::DistString; use sqlx::postgres::PgPoolOptions; use tokio::runtime::Runtime; use tokio::sync::oneshot; +use tokio_stream::StreamExt; use crate::index::event::Event; use crate::ord_api_client::OrdApiClient; @@ -37,7 +43,20 @@ impl EventConsumer { .rabbitmq_addr() .context("rabbitmq amqp credentials and url must be defined")?; - let conn = Connection::connect(&addr, ConnectionProperties::default()) + let uri = addr.parse::().unwrap(); + + let connect = move |uri: &AMQPUri| { + uri.connect().and_then(|stream| { + let mut tls_builder = NativeTlsConnector::builder(); + tls_builder.danger_accept_invalid_certs(true); + stream.into_native_tls( + &tls_builder.build().expect("TLS configuration failed"), + &uri.authority.host, + ) + }) + }; + + let conn = Connection::connector(uri, Box::new(connect), ConnectionProperties::default()) .await .expect("connects to rabbitmq ok"); diff --git a/src/event_publisher.rs b/src/event_publisher.rs index c3469f9234..f25b26b432 100644 --- a/src/event_publisher.rs +++ b/src/event_publisher.rs @@ -1,5 +1,7 @@ use anyhow::{Context, Result}; use lapin::options::ConfirmSelectOptions; +use lapin::tcp::{AMQPUriTcpExt, NativeTlsConnector}; +use lapin::uri::AMQPUri; use lapin::{options::BasicPublishOptions, BasicProperties, Connection, ConnectionProperties}; use tokio::runtime::Runtime; use tokio::sync::mpsc; @@ -27,7 +29,20 @@ impl EventPublisher { std::thread::spawn(move || { Runtime::new().expect("runtime is setup").block_on(async { - let conn = Connection::connect(&addr, ConnectionProperties::default()) + let uri = addr.parse::().unwrap(); + + let connect = move |uri: &AMQPUri| { + uri.connect().and_then(|stream| { + let mut tls_builder = NativeTlsConnector::builder(); + tls_builder.danger_accept_invalid_certs(true); + stream.into_native_tls( + &tls_builder.build().expect("TLS configuration failed"), + &uri.authority.host, + ) + }) + }; + + let conn = Connection::connector(uri, Box::new(connect), ConnectionProperties::default()) .await .expect("connects to rabbitmq ok"); diff --git a/src/ord_db_client.rs b/src/ord_db_client.rs index 077ad62939..35c80c27d9 100644 --- a/src/ord_db_client.rs +++ b/src/ord_db_client.rs @@ -3,8 +3,8 @@ use std::sync::Arc; use bitcoin::{OutPoint, Txid}; use futures::TryStreamExt; -use sqlx::{PgPool, Row}; use sqlx::types::Json; +use sqlx::{PgPool, Row}; use ordinals::SatPoint; @@ -175,7 +175,7 @@ impl OrdDbClient { parents = COALESCE(EXCLUDED.parents, inscription.parents) RETURNING id; "; - let id = sqlx::query_as::<_, (i32, )>(query) + let id = sqlx::query_as::<_, (i32,)>(query) .bind(inscription_details.id.to_string()) .bind(inscription_details.number) .bind(inscription_details.content_type.as_deref()) diff --git a/src/ord_indexation.rs b/src/ord_indexation.rs index 026f7afa65..ff6de8a7fe 100644 --- a/src/ord_indexation.rs +++ b/src/ord_indexation.rs @@ -7,10 +7,10 @@ use serde_json::Value; use ordinals::SatPoint; use crate::api::BlockInfo; -use crate::InscriptionId; use crate::ord_api_client::OrdApiClient; use crate::ord_db_client::{Event, OrdDbClient}; use crate::settings::Settings; +use crate::InscriptionId; pub struct OrdIndexation { settings: Settings, @@ -122,18 +122,12 @@ impl OrdIndexation { let cursor = Cursor::new(bytes); let result = from_reader(cursor); match result { - Ok(value) => { - match &value { - Value::Object(obj) if obj.is_empty() => None, - Value::Object(_) | Value::Array(_) => serde_json::to_string(&value).ok(), - _ => { - Some(hex::encode(bytes)) - } - } + Ok(value) => match &value { + Value::Object(obj) if obj.is_empty() => None, + Value::Object(_) | Value::Array(_) => serde_json::to_string(&value).ok(), + _ => Some(hex::encode(bytes)), }, - Err(_) => { - Some(hex::encode(bytes)) - } + Err(_) => Some(hex::encode(bytes)), } }) } From cadfe5b58c2053e1e75b834cd4b664398117aa73 Mon Sep 17 00:00:00 2001 From: rob Date: Sat, 4 May 2024 09:22:14 +0200 Subject: [PATCH 053/103] Revert "ignore amq certs" This reverts commit 9841d0ca0cca6765791b9b2e1b5493fe089722e6. --- Cargo.lock | 1 - Cargo.toml | 2 +- src/event_consumer.rs | 25 +++---------------------- src/event_publisher.rs | 17 +---------------- src/ord_db_client.rs | 4 ++-- src/ord_indexation.rs | 18 ++++++++++++------ 6 files changed, 19 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2d78cece84..712092e412 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4795,7 +4795,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "495b0abdce3dc1f8fd27240651c9e68890c14e9d9c61527b1ce44d8a5a7bd3d5" dependencies = [ "cfg-if 1.0.0", - "native-tls", "p12-keystore", "rustls-connector", "rustls-pemfile 2.1.2", diff --git a/Cargo.toml b/Cargo.toml index 971bea537a..62a8b90a62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,7 @@ http = "0.2.6" humantime = "2.1.0" hyper = { version = "0.14.24", features = ["client", "http2"] } indicatif = "0.17.1" -lapin = { version = "2.3.3", features = ["native-tls"] } +lapin = "2.3.3" lazy_static = "1.4.0" log = "0.4.14" mime = "0.3.16" diff --git a/src/event_consumer.rs b/src/event_consumer.rs index 719f32de0d..ffed1107a6 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -4,18 +4,12 @@ use anyhow::Context; use bitcoin::secp256k1::rand::distributions::Alphanumeric; use chrono::Utc; use clap::Parser; -use lapin::options::{ - BasicAckOptions, BasicConsumeOptions, BasicRejectOptions, ConfirmSelectOptions, -}; -use lapin::tcp::{AMQPUriTcpExt, NativeTlsConnector}; -use lapin::types::FieldTable; -use lapin::uri::AMQPUri; -use lapin::{Connection, ConnectionProperties}; +use futures::StreamExt; +use lapin::{options::*, types::FieldTable, Connection, ConnectionProperties}; use rand::distributions::DistString; use sqlx::postgres::PgPoolOptions; use tokio::runtime::Runtime; use tokio::sync::oneshot; -use tokio_stream::StreamExt; use crate::index::event::Event; use crate::ord_api_client::OrdApiClient; @@ -43,20 +37,7 @@ impl EventConsumer { .rabbitmq_addr() .context("rabbitmq amqp credentials and url must be defined")?; - let uri = addr.parse::().unwrap(); - - let connect = move |uri: &AMQPUri| { - uri.connect().and_then(|stream| { - let mut tls_builder = NativeTlsConnector::builder(); - tls_builder.danger_accept_invalid_certs(true); - stream.into_native_tls( - &tls_builder.build().expect("TLS configuration failed"), - &uri.authority.host, - ) - }) - }; - - let conn = Connection::connector(uri, Box::new(connect), ConnectionProperties::default()) + let conn = Connection::connect(&addr, ConnectionProperties::default()) .await .expect("connects to rabbitmq ok"); diff --git a/src/event_publisher.rs b/src/event_publisher.rs index f25b26b432..c3469f9234 100644 --- a/src/event_publisher.rs +++ b/src/event_publisher.rs @@ -1,7 +1,5 @@ use anyhow::{Context, Result}; use lapin::options::ConfirmSelectOptions; -use lapin::tcp::{AMQPUriTcpExt, NativeTlsConnector}; -use lapin::uri::AMQPUri; use lapin::{options::BasicPublishOptions, BasicProperties, Connection, ConnectionProperties}; use tokio::runtime::Runtime; use tokio::sync::mpsc; @@ -29,20 +27,7 @@ impl EventPublisher { std::thread::spawn(move || { Runtime::new().expect("runtime is setup").block_on(async { - let uri = addr.parse::().unwrap(); - - let connect = move |uri: &AMQPUri| { - uri.connect().and_then(|stream| { - let mut tls_builder = NativeTlsConnector::builder(); - tls_builder.danger_accept_invalid_certs(true); - stream.into_native_tls( - &tls_builder.build().expect("TLS configuration failed"), - &uri.authority.host, - ) - }) - }; - - let conn = Connection::connector(uri, Box::new(connect), ConnectionProperties::default()) + let conn = Connection::connect(&addr, ConnectionProperties::default()) .await .expect("connects to rabbitmq ok"); diff --git a/src/ord_db_client.rs b/src/ord_db_client.rs index 35c80c27d9..077ad62939 100644 --- a/src/ord_db_client.rs +++ b/src/ord_db_client.rs @@ -3,8 +3,8 @@ use std::sync::Arc; use bitcoin::{OutPoint, Txid}; use futures::TryStreamExt; -use sqlx::types::Json; use sqlx::{PgPool, Row}; +use sqlx::types::Json; use ordinals::SatPoint; @@ -175,7 +175,7 @@ impl OrdDbClient { parents = COALESCE(EXCLUDED.parents, inscription.parents) RETURNING id; "; - let id = sqlx::query_as::<_, (i32,)>(query) + let id = sqlx::query_as::<_, (i32, )>(query) .bind(inscription_details.id.to_string()) .bind(inscription_details.number) .bind(inscription_details.content_type.as_deref()) diff --git a/src/ord_indexation.rs b/src/ord_indexation.rs index ff6de8a7fe..026f7afa65 100644 --- a/src/ord_indexation.rs +++ b/src/ord_indexation.rs @@ -7,10 +7,10 @@ use serde_json::Value; use ordinals::SatPoint; use crate::api::BlockInfo; +use crate::InscriptionId; use crate::ord_api_client::OrdApiClient; use crate::ord_db_client::{Event, OrdDbClient}; use crate::settings::Settings; -use crate::InscriptionId; pub struct OrdIndexation { settings: Settings, @@ -122,12 +122,18 @@ impl OrdIndexation { let cursor = Cursor::new(bytes); let result = from_reader(cursor); match result { - Ok(value) => match &value { - Value::Object(obj) if obj.is_empty() => None, - Value::Object(_) | Value::Array(_) => serde_json::to_string(&value).ok(), - _ => Some(hex::encode(bytes)), + Ok(value) => { + match &value { + Value::Object(obj) if obj.is_empty() => None, + Value::Object(_) | Value::Array(_) => serde_json::to_string(&value).ok(), + _ => { + Some(hex::encode(bytes)) + } + } }, - Err(_) => Some(hex::encode(bytes)), + Err(_) => { + Some(hex::encode(bytes)) + } } }) } From 81f9688e34375816ebc45d2ff9538095ab2a7169 Mon Sep 17 00:00:00 2001 From: rob Date: Sat, 4 May 2024 10:19:40 +0200 Subject: [PATCH 054/103] Test connection --- src/event_consumer.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/event_consumer.rs b/src/event_consumer.rs index ffed1107a6..0cae5d87fa 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -5,7 +5,9 @@ use bitcoin::secp256k1::rand::distributions::Alphanumeric; use chrono::Utc; use clap::Parser; use futures::StreamExt; -use lapin::{options::*, types::FieldTable, Connection, ConnectionProperties}; +use lapin::{Connection, ConnectionProperties, options::*, types::FieldTable}; +use lapin::tcp::{AMQPUriTcpExt, RustlsConnector}; +use lapin::uri::AMQPUri; use rand::distributions::DistString; use sqlx::postgres::PgPoolOptions; use tokio::runtime::Runtime; @@ -37,7 +39,17 @@ impl EventConsumer { .rabbitmq_addr() .context("rabbitmq amqp credentials and url must be defined")?; - let conn = Connection::connect(&addr, ConnectionProperties::default()) + let uri = addr.parse::().unwrap(); + + let connect = move |uri: &AMQPUri| { + let conn = uri.connect().and_then(|stream| { + let connector = RustlsConnector::new_with_native_certs().unwrap(); + stream.into_rustls(&connector, &uri.authority.host) + }); + conn + }; + + let conn = Connection::connector(uri, Box::new(connect), ConnectionProperties::default()) .await .expect("connects to rabbitmq ok"); From 8a945ab958a1a372629615a0f2135e1043559e4c Mon Sep 17 00:00:00 2001 From: rob Date: Sat, 4 May 2024 10:36:27 +0200 Subject: [PATCH 055/103] Move event publisher to new connection way --- src/event_consumer.rs | 2 +- src/event_publisher.rs | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/event_consumer.rs b/src/event_consumer.rs index 0cae5d87fa..afaecbdc8f 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -5,9 +5,9 @@ use bitcoin::secp256k1::rand::distributions::Alphanumeric; use chrono::Utc; use clap::Parser; use futures::StreamExt; -use lapin::{Connection, ConnectionProperties, options::*, types::FieldTable}; use lapin::tcp::{AMQPUriTcpExt, RustlsConnector}; use lapin::uri::AMQPUri; +use lapin::{options::*, types::FieldTable, Connection, ConnectionProperties}; use rand::distributions::DistString; use sqlx::postgres::PgPoolOptions; use tokio::runtime::Runtime; diff --git a/src/event_publisher.rs b/src/event_publisher.rs index c3469f9234..071e38906d 100644 --- a/src/event_publisher.rs +++ b/src/event_publisher.rs @@ -1,5 +1,7 @@ use anyhow::{Context, Result}; use lapin::options::ConfirmSelectOptions; +use lapin::tcp::{AMQPUriTcpExt, RustlsConnector}; +use lapin::uri::AMQPUri; use lapin::{options::BasicPublishOptions, BasicProperties, Connection, ConnectionProperties}; use tokio::runtime::Runtime; use tokio::sync::mpsc; @@ -27,7 +29,17 @@ impl EventPublisher { std::thread::spawn(move || { Runtime::new().expect("runtime is setup").block_on(async { - let conn = Connection::connect(&addr, ConnectionProperties::default()) + let uri = addr.parse::().unwrap(); + + let connect = move |uri: &AMQPUri| { + let conn = uri.connect().and_then(|stream| { + let connector = RustlsConnector::new_with_native_certs().unwrap(); + stream.into_rustls(&connector, &uri.authority.host) + }); + conn + }; + + let conn = Connection::connector(uri, Box::new(connect), ConnectionProperties::default()) .await .expect("connects to rabbitmq ok"); From bb022a845eed12da7355946b0b41fa4b7930730c Mon Sep 17 00:00:00 2001 From: rob Date: Sun, 5 May 2024 09:13:46 +0200 Subject: [PATCH 056/103] Change how we build uri --- src/event_consumer.rs | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/event_consumer.rs b/src/event_consumer.rs index afaecbdc8f..9022e781e3 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -6,14 +6,16 @@ use chrono::Utc; use clap::Parser; use futures::StreamExt; use lapin::tcp::{AMQPUriTcpExt, RustlsConnector}; -use lapin::uri::AMQPUri; +use lapin::uri::{AMQPAuthority, AMQPQueryString, AMQPScheme, AMQPUri, AMQPUserInfo}; use lapin::{options::*, types::FieldTable, Connection, ConnectionProperties}; use rand::distributions::DistString; +use rustls::ClientConfig; use sqlx::postgres::PgPoolOptions; use tokio::runtime::Runtime; use tokio::sync::oneshot; use crate::index::event::Event; +use crate::no_op_verifier::NoOpVerifier; use crate::ord_api_client::OrdApiClient; use crate::ord_db_client::OrdDbClient; use crate::ord_indexation::OrdIndexation; @@ -39,10 +41,31 @@ impl EventConsumer { .rabbitmq_addr() .context("rabbitmq amqp credentials and url must be defined")?; - let uri = addr.parse::().unwrap(); + // let uri = addr.parse::().unwrap(); + + let uri = AMQPUri { + scheme: AMQPScheme::AMQP, + authority: AMQPAuthority { + userinfo: AMQPUserInfo { + username: "indexer".into(), + password: addr.into(), + }, + host: "b-3a9cd5ac-5f8c-457b-a8d7-e1ba88a07c29.mq.us-east-1.amazonaws.com".into(), + port: 5671, + }, + vhost: "/".into(), + query: AMQPQueryString::default(), + }; let connect = move |uri: &AMQPUri| { let conn = uri.connect().and_then(|stream| { + // let config = ClientConfig::builder() + // .dangerous() + // .with_custom_certificate_verifier(Arc::new(NoOpVerifier)) + // .with_no_client_auth(); + // + // let connector = RustlsConnector::from(config); + let connector = RustlsConnector::new_with_native_certs().unwrap(); stream.into_rustls(&connector, &uri.authority.host) }); From aea38394436afb6ec5c17180b736e62c5f5aeab9 Mon Sep 17 00:00:00 2001 From: rob Date: Sun, 5 May 2024 09:25:03 +0200 Subject: [PATCH 057/103] Test native tls --- Cargo.lock | 5 +++-- Cargo.toml | 2 +- src/event_consumer.rs | 37 ++++++++----------------------------- 3 files changed, 12 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 712092e412..05aa639a75 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2522,9 +2522,9 @@ dependencies = [ [[package]] name = "lapin" -version = "2.3.3" +version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be0454336697a39b3eb8d06339aa1b4da3cad94862fca62b0e32fd4b7f71b563" +checksum = "fae02c316a8a5922ce7518afa6b6c00e9a099f8e59587567e3331efdd11b8ceb" dependencies = [ "amq-protocol", "async-global-executor-trait", @@ -4795,6 +4795,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "495b0abdce3dc1f8fd27240651c9e68890c14e9d9c61527b1ce44d8a5a7bd3d5" dependencies = [ "cfg-if 1.0.0", + "native-tls", "p12-keystore", "rustls-connector", "rustls-pemfile 2.1.2", diff --git a/Cargo.toml b/Cargo.toml index 62a8b90a62..971bea537a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,7 @@ http = "0.2.6" humantime = "2.1.0" hyper = { version = "0.14.24", features = ["client", "http2"] } indicatif = "0.17.1" -lapin = "2.3.3" +lapin = { version = "2.3.3", features = ["native-tls"] } lazy_static = "1.4.0" log = "0.4.14" mime = "0.3.16" diff --git a/src/event_consumer.rs b/src/event_consumer.rs index 9022e781e3..72ad254e92 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -5,17 +5,15 @@ use bitcoin::secp256k1::rand::distributions::Alphanumeric; use chrono::Utc; use clap::Parser; use futures::StreamExt; -use lapin::tcp::{AMQPUriTcpExt, RustlsConnector}; -use lapin::uri::{AMQPAuthority, AMQPQueryString, AMQPScheme, AMQPUri, AMQPUserInfo}; -use lapin::{options::*, types::FieldTable, Connection, ConnectionProperties}; +use lapin::{Connection, ConnectionProperties, options::*, types::FieldTable}; +use lapin::tcp::{AMQPUriTcpExt, NativeTlsConnector}; +use lapin::uri::AMQPUri; use rand::distributions::DistString; -use rustls::ClientConfig; use sqlx::postgres::PgPoolOptions; use tokio::runtime::Runtime; use tokio::sync::oneshot; use crate::index::event::Event; -use crate::no_op_verifier::NoOpVerifier; use crate::ord_api_client::OrdApiClient; use crate::ord_db_client::OrdDbClient; use crate::ord_indexation::OrdIndexation; @@ -41,33 +39,14 @@ impl EventConsumer { .rabbitmq_addr() .context("rabbitmq amqp credentials and url must be defined")?; - // let uri = addr.parse::().unwrap(); - - let uri = AMQPUri { - scheme: AMQPScheme::AMQP, - authority: AMQPAuthority { - userinfo: AMQPUserInfo { - username: "indexer".into(), - password: addr.into(), - }, - host: "b-3a9cd5ac-5f8c-457b-a8d7-e1ba88a07c29.mq.us-east-1.amazonaws.com".into(), - port: 5671, - }, - vhost: "/".into(), - query: AMQPQueryString::default(), - }; + let uri = addr.parse::().unwrap(); let connect = move |uri: &AMQPUri| { let conn = uri.connect().and_then(|stream| { - // let config = ClientConfig::builder() - // .dangerous() - // .with_custom_certificate_verifier(Arc::new(NoOpVerifier)) - // .with_no_client_auth(); - // - // let connector = RustlsConnector::from(config); - - let connector = RustlsConnector::new_with_native_certs().unwrap(); - stream.into_rustls(&connector, &uri.authority.host) + let mut tls_builder = NativeTlsConnector::builder(); + tls_builder.danger_accept_invalid_certs(true); + let connector = &tls_builder.build().expect("TLS configuration failed"); + stream.into_native_tls(connector, &uri.authority.host) }); conn }; From 597061d52eb968e3d5a95f0afacfaefb09366ed6 Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 6 May 2024 09:06:13 +0200 Subject: [PATCH 058/103] Add rmq connection that supports secured and unsecured conn --- src/connect_rmq.rs | 37 +++++++++++++++++++++++++++++++++++++ src/event_consumer.rs | 21 +++------------------ src/event_publisher.rs | 17 +++-------------- src/lib.rs | 1 + 4 files changed, 44 insertions(+), 32 deletions(-) create mode 100644 src/connect_rmq.rs diff --git a/src/connect_rmq.rs b/src/connect_rmq.rs new file mode 100644 index 0000000000..c62f9ee25d --- /dev/null +++ b/src/connect_rmq.rs @@ -0,0 +1,37 @@ +use std::collections::HashSet; + +use anyhow::Context; +use lapin::tcp::{AMQPUriTcpExt, NativeTlsConnector}; +use lapin::uri::AMQPUri; +use lapin::{Connection, ConnectionProperties}; + +pub async fn connect_to_rabbitmq(addr: &String) -> Result { + let uri = addr + .parse::() + .map_err(anyhow::Error::msg) + .context("Failed to parse AMQP URI")?; + let host = uri.authority.host.clone(); + + let non_tls_hosts: HashSet = vec!["localhost".to_string()].into_iter().collect(); + + let is_secure = !non_tls_hosts.contains(&host); + + if is_secure { + let connect = move |uri: &AMQPUri| { + let conn = uri.connect().and_then(|stream| { + let mut tls_builder = NativeTlsConnector::builder(); + tls_builder.danger_accept_invalid_certs(true); + let connector = &tls_builder.build().expect("TLS configuration failed"); + stream.into_native_tls(&connector, &uri.authority.host) + }); + conn + }; + Connection::connector(uri, Box::new(connect), ConnectionProperties::default()) + .await + .context("Failed to establish a secure AMQP connection") + } else { + Connection::connect(addr, ConnectionProperties::default()) + .await + .context("Failed to establish an unsecure AMQP connection") + } +} diff --git a/src/event_consumer.rs b/src/event_consumer.rs index 72ad254e92..1df1072dc2 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -5,14 +5,13 @@ use bitcoin::secp256k1::rand::distributions::Alphanumeric; use chrono::Utc; use clap::Parser; use futures::StreamExt; -use lapin::{Connection, ConnectionProperties, options::*, types::FieldTable}; -use lapin::tcp::{AMQPUriTcpExt, NativeTlsConnector}; -use lapin::uri::AMQPUri; +use lapin::{options::*, types::FieldTable}; use rand::distributions::DistString; use sqlx::postgres::PgPoolOptions; use tokio::runtime::Runtime; use tokio::sync::oneshot; +use crate::connect_rmq::connect_to_rabbitmq; use crate::index::event::Event; use crate::ord_api_client::OrdApiClient; use crate::ord_db_client::OrdDbClient; @@ -39,21 +38,7 @@ impl EventConsumer { .rabbitmq_addr() .context("rabbitmq amqp credentials and url must be defined")?; - let uri = addr.parse::().unwrap(); - - let connect = move |uri: &AMQPUri| { - let conn = uri.connect().and_then(|stream| { - let mut tls_builder = NativeTlsConnector::builder(); - tls_builder.danger_accept_invalid_certs(true); - let connector = &tls_builder.build().expect("TLS configuration failed"); - stream.into_native_tls(connector, &uri.authority.host) - }); - conn - }; - - let conn = Connection::connector(uri, Box::new(connect), ConnectionProperties::default()) - .await - .expect("connects to rabbitmq ok"); + let conn = connect_to_rabbitmq(addr).await?; let channel = conn .create_channel() diff --git a/src/event_publisher.rs b/src/event_publisher.rs index 071e38906d..8e0e754e25 100644 --- a/src/event_publisher.rs +++ b/src/event_publisher.rs @@ -1,11 +1,10 @@ use anyhow::{Context, Result}; use lapin::options::ConfirmSelectOptions; -use lapin::tcp::{AMQPUriTcpExt, RustlsConnector}; -use lapin::uri::AMQPUri; -use lapin::{options::BasicPublishOptions, BasicProperties, Connection, ConnectionProperties}; +use lapin::{options::BasicPublishOptions, BasicProperties}; use tokio::runtime::Runtime; use tokio::sync::mpsc; +use crate::connect_rmq::connect_to_rabbitmq; use crate::index::event::Event; use crate::settings::Settings; @@ -29,17 +28,7 @@ impl EventPublisher { std::thread::spawn(move || { Runtime::new().expect("runtime is setup").block_on(async { - let uri = addr.parse::().unwrap(); - - let connect = move |uri: &AMQPUri| { - let conn = uri.connect().and_then(|stream| { - let connector = RustlsConnector::new_with_native_certs().unwrap(); - stream.into_rustls(&connector, &uri.authority.host) - }); - conn - }; - - let conn = Connection::connector(uri, Box::new(connect), ConnectionProperties::default()) + let conn = connect_to_rabbitmq(&addr) .await .expect("connects to rabbitmq ok"); diff --git a/src/lib.rs b/src/lib.rs index 43fad300a7..7a9ea33527 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -115,6 +115,7 @@ mod object; pub mod options; pub mod ord_api_client; +pub mod connect_rmq; pub mod ord_db_client; mod ord_indexation; pub mod outgoing; From a5411accf752cdbc2c6e3de59abb61f120cf9dc7 Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 6 May 2024 09:31:58 +0200 Subject: [PATCH 059/103] Fix lint --- src/connect_rmq.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/connect_rmq.rs b/src/connect_rmq.rs index c62f9ee25d..57e570363f 100644 --- a/src/connect_rmq.rs +++ b/src/connect_rmq.rs @@ -5,7 +5,7 @@ use lapin::tcp::{AMQPUriTcpExt, NativeTlsConnector}; use lapin::uri::AMQPUri; use lapin::{Connection, ConnectionProperties}; -pub async fn connect_to_rabbitmq(addr: &String) -> Result { +pub async fn connect_to_rabbitmq(addr: &str) -> Result { let uri = addr .parse::() .map_err(anyhow::Error::msg) @@ -22,7 +22,7 @@ pub async fn connect_to_rabbitmq(addr: &String) -> Result Date: Mon, 6 May 2024 14:13:20 +0200 Subject: [PATCH 060/103] Stop service if issues with rmq publishing --- src/event_publisher.rs | 81 ++++++++++++++++++++++++------------------ src/lib.rs | 32 +++++++++-------- 2 files changed, 63 insertions(+), 50 deletions(-) diff --git a/src/event_publisher.rs b/src/event_publisher.rs index 8e0e754e25..38e61e01d5 100644 --- a/src/event_publisher.rs +++ b/src/event_publisher.rs @@ -1,9 +1,12 @@ +use std::process; + use anyhow::{Context, Result}; +use lapin::{BasicProperties, options::BasicPublishOptions}; use lapin::options::ConfirmSelectOptions; -use lapin::{options::BasicPublishOptions, BasicProperties}; use tokio::runtime::Runtime; use tokio::sync::mpsc; +use crate::{gracefully_shutdown_indexer, shutdown_process}; use crate::connect_rmq::connect_to_rabbitmq; use crate::index::event::Event; use crate::settings::Settings; @@ -24,49 +27,57 @@ impl EventPublisher { .context("rabbitmq exchange path must be defined")? .to_owned(); - let (tx, mut rx) = mpsc::channel::(128); + let (tx, rx) = mpsc::channel::(128); std::thread::spawn(move || { Runtime::new().expect("runtime is setup").block_on(async { - let conn = connect_to_rabbitmq(&addr) - .await - .expect("connects to rabbitmq ok"); + match EventPublisher::consume_channel(addr, exchange, rx).await { + Ok(_) => log::info!("Channel closed."), + Err(e) => { + log::error!("Fatal error publishing to RMQ, exiting {}", e); + shutdown_process(); + }, + } + }) + }); + + Ok(EventPublisher { sender: tx }) + } - let channel = conn - .create_channel() - .await - .expect("creates rmq connection channel"); + async fn consume_channel(addr: String, exchange: String, mut rx: mpsc::Receiver) -> Result<()> { + let conn = connect_to_rabbitmq(&addr) + .await?; - channel - .confirm_select(ConfirmSelectOptions::default()) - .await - .expect("enable msg confirms"); + let channel = conn + .create_channel() + .await?; - while let Some(event) = rx.recv().await { - // TODO we might want to panic if rmq is down so we don't miss any messages - // if we miss messages only way to replay them is run `ord` instance from scratch - // maybe we can trigger fake reorg to force it to reindex from savepoint? - let message = serde_json::to_vec(&event).expect("failed to serialize event"); + channel + .confirm_select(ConfirmSelectOptions::default()) + .await?; - let publish = channel - .basic_publish( - &exchange, - EventPublisher::type_name(&event), - BasicPublishOptions::default(), - &message, - BasicProperties::default(), - ) - .await - .expect("published rmq msg") - .await - .expect("confirms rmq msg received"); + while let Some(event) = rx.recv().await { + let message = serde_json::to_vec(&event)?; - assert!(publish.is_ack()); - } - }) - }); + let publish = channel + .basic_publish( + &exchange, + EventPublisher::type_name(&event), + BasicPublishOptions::default(), + &message, + BasicProperties::default(), + ) + .await? + .await?; - Ok(EventPublisher { sender: tx }) + if !publish.is_ack() { + return Err(anyhow::Error::new(std::io::Error::new( + std::io::ErrorKind::Other, + "Message was not acknowledged", + ))); + } + } + Ok(()) } fn type_name(event: &Event) -> &'static str { diff --git a/src/lib.rs b/src/lib.rs index 7a9ea33527..10452cdb05 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -238,24 +238,26 @@ fn gracefully_shutdown_indexer() { } } -pub fn main() { - env_logger::init(); - ctrlc::set_handler(move || { - if SHUTTING_DOWN.fetch_or(true, atomic::Ordering::Relaxed) { - process::exit(1); - } +fn shutdown_process() { + if SHUTTING_DOWN.fetch_or(true, atomic::Ordering::Relaxed) { + process::exit(1); + } - eprintln!("Shutting down gracefully. Press again to shutdown immediately."); + eprintln!("Shutting down gracefully. Press again to shutdown immediately."); - LISTENERS - .lock() - .unwrap() - .iter() - .for_each(|handle| handle.graceful_shutdown(Some(Duration::from_millis(100)))); + LISTENERS + .lock() + .unwrap() + .iter() + .for_each(|handle| handle.graceful_shutdown(Some(Duration::from_millis(100)))); + + gracefully_shutdown_indexer(); +} - gracefully_shutdown_indexer(); - }) - .expect("Error setting handler"); +pub fn main() { + env_logger::init(); + ctrlc::set_handler(move || shutdown_process()) + .expect("Error setting handler"); let args = Arguments::parse(); From c0dba6668cc1f55c6c9e4e0a3e76c96bb6580f8c Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 6 May 2024 14:46:07 +0200 Subject: [PATCH 061/103] fix lint --- src/event_publisher.rs | 21 ++++++++++----------- src/lib.rs | 3 +-- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/event_publisher.rs b/src/event_publisher.rs index 38e61e01d5..88fa9eba70 100644 --- a/src/event_publisher.rs +++ b/src/event_publisher.rs @@ -1,15 +1,13 @@ -use std::process; - use anyhow::{Context, Result}; -use lapin::{BasicProperties, options::BasicPublishOptions}; use lapin::options::ConfirmSelectOptions; +use lapin::{options::BasicPublishOptions, BasicProperties}; use tokio::runtime::Runtime; use tokio::sync::mpsc; -use crate::{gracefully_shutdown_indexer, shutdown_process}; use crate::connect_rmq::connect_to_rabbitmq; use crate::index::event::Event; use crate::settings::Settings; +use crate::shutdown_process; pub struct EventPublisher { pub(crate) sender: mpsc::Sender, @@ -36,7 +34,7 @@ impl EventPublisher { Err(e) => { log::error!("Fatal error publishing to RMQ, exiting {}", e); shutdown_process(); - }, + } } }) }); @@ -44,13 +42,14 @@ impl EventPublisher { Ok(EventPublisher { sender: tx }) } - async fn consume_channel(addr: String, exchange: String, mut rx: mpsc::Receiver) -> Result<()> { - let conn = connect_to_rabbitmq(&addr) - .await?; + async fn consume_channel( + addr: String, + exchange: String, + mut rx: mpsc::Receiver, + ) -> Result<()> { + let conn = connect_to_rabbitmq(&addr).await?; - let channel = conn - .create_channel() - .await?; + let channel = conn.create_channel().await?; channel .confirm_select(ConfirmSelectOptions::default()) diff --git a/src/lib.rs b/src/lib.rs index 10452cdb05..97e8cfca1e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -256,8 +256,7 @@ fn shutdown_process() { pub fn main() { env_logger::init(); - ctrlc::set_handler(move || shutdown_process()) - .expect("Error setting handler"); + ctrlc::set_handler(shutdown_process).expect("Error setting handler"); let args = Arguments::parse(); From cff889c0e29e8f65d9629ada305c36c01f294ba1 Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 6 May 2024 15:17:27 +0200 Subject: [PATCH 062/103] Make consumer fail if rmq error --- src/event_consumer.rs | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/event_consumer.rs b/src/event_consumer.rs index 1df1072dc2..eb1ee12f6f 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -1,3 +1,4 @@ +use std::process; use std::sync::Arc; use anyhow::Context; @@ -100,6 +101,10 @@ impl EventConsumer { blocks_shutdown_rx, ) .await + .map_err(|e| { + log::error!("Error consuming blocks queue: {}", e); + process::exit(1); + }) }); let inscriptions_consumer_handle = tokio::spawn(async move { @@ -111,6 +116,10 @@ impl EventConsumer { inscriptions_shutdown_rx, ) .await + .map_err(|e| { + log::error!("Error consuming inscriptions queue: {}", e); + process::exit(1); + }) }); let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; @@ -139,24 +148,19 @@ impl EventConsumer { BasicConsumeOptions::default(), FieldTable::default(), ) - .await - .expect("creates rmq consumer"); + .await?; log::info!("Starting to consume messages from {}", queue_name); while let Some(result) = consumer.next().await { - match result { - Ok(delivery) => { - tokio::select! { - _ = EventConsumer::handle_delivery(delivery, &ord_indexation) => {}, - _ = &mut shutdown_signal => { - log::info!("Shutdown signal received, stopping consumer."); - break; - }, - } - } - Err(e) => { - log::error!("Error receiving delivery: {}", e); - } + let delivery = result?; + tokio::select! { + process_result = EventConsumer::handle_delivery(delivery, &ord_indexation) => { + process_result?; + }, + _ = &mut shutdown_signal => { + log::info!("Shutdown signal received, stopping consumer."); + break; + }, } } From 86ff1854687a1b6176e728ab1db2418f68107374 Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 6 May 2024 16:41:33 +0200 Subject: [PATCH 063/103] Log db url --- src/event_consumer.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/event_consumer.rs b/src/event_consumer.rs index eb1ee12f6f..472e3cb76a 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -55,6 +55,8 @@ impl EventConsumer { .database_url .as_deref() .context("db url must be defined")?; + log::info!("Connecting to database at {}", EventConsumer::mask_password_in_url(database_url)); + let pool = PgPoolOptions::new() .max_connections(5) .connect(database_url) @@ -134,6 +136,11 @@ impl EventConsumer { }) } + fn mask_password_in_url(url: &str) -> String { + let re = regex::Regex::new(r"(\w+://)([^:]+):([^@]+)@").unwrap(); + re.replace(url, "$1$2:***@").to_string() + } + async fn consume_queue( channel: lapin::Channel, queue_name: String, From 51122cacba9fc7e71962ddda5a9294be0f356cac Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 6 May 2024 17:22:56 +0200 Subject: [PATCH 064/103] Encode password --- src/event_consumer.rs | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/event_consumer.rs b/src/event_consumer.rs index 472e3cb76a..29f99d3ce1 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -8,12 +8,16 @@ use clap::Parser; use futures::StreamExt; use lapin::{options::*, types::FieldTable}; use rand::distributions::DistString; -use sqlx::postgres::PgPoolOptions; +use serde::__private::de::IdentifierDeserializer; +use sqlx::{Connection, database}; +use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; use tokio::runtime::Runtime; use tokio::sync::oneshot; +use urlencoding::encode; use crate::connect_rmq::connect_to_rabbitmq; use crate::index::event::Event; +use crate::Options; use crate::ord_api_client::OrdApiClient; use crate::ord_db_client::OrdDbClient; use crate::ord_indexation::OrdIndexation; @@ -59,7 +63,7 @@ impl EventConsumer { let pool = PgPoolOptions::new() .max_connections(5) - .connect(database_url) + .connect(encoded_database_url.as_ref()) .await?; let shared_pool = Arc::new(pool); let ord_db_client = Arc::new(OrdDbClient::new(shared_pool.clone())); @@ -136,6 +140,19 @@ impl EventConsumer { }) } + fn encode_password_in_url(url: &str) -> String { + let re = regex::Regex::new(r"(\w+://)([^:]+):([^@]+)@(.*)").unwrap(); + if let Some(caps) = re.captures(url) { + let protocol_and_user = caps.get(1).map_or("", |m| m.as_str()); + let username = caps.get(2).map_or("", |m| m.as_str()); + let password = caps.get(3).map_or("", |m| m.as_str()); + let rest_of_url = caps.get(4).map_or("", |m| m.as_str()); + format!("{}{}:{}@{}", protocol_and_user, username, encode(password), rest_of_url) + } else { + url.to_string() + } + } + fn mask_password_in_url(url: &str) -> String { let re = regex::Regex::new(r"(\w+://)([^:]+):([^@]+)@").unwrap(); re.replace(url, "$1$2:***@").to_string() From e6a0d150e365514a23d8851ae46531af3a4437b1 Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 6 May 2024 17:27:15 +0200 Subject: [PATCH 065/103] Encode password --- src/event_consumer.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/event_consumer.rs b/src/event_consumer.rs index 29f99d3ce1..82d74d891a 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -60,6 +60,7 @@ impl EventConsumer { .as_deref() .context("db url must be defined")?; log::info!("Connecting to database at {}", EventConsumer::mask_password_in_url(database_url)); + let encoded_database_url = EventConsumer::encode_password_in_url(database_url); let pool = PgPoolOptions::new() .max_connections(5) From cdf4c305427345ebd33ff55e46963274a521bf82 Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 6 May 2024 18:16:58 +0200 Subject: [PATCH 066/103] Log http errors --- src/ord_api_client.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/ord_api_client.rs b/src/ord_api_client.rs index 84a90daa10..a20a4bd8d3 100644 --- a/src/ord_api_client.rs +++ b/src/ord_api_client.rs @@ -2,6 +2,7 @@ use std::time::Duration; use anyhow::anyhow; use bitcoin::Txid; +use http::StatusCode; use reqwest::{Client, RequestBuilder}; use tokio::time::sleep; @@ -35,6 +36,8 @@ impl OrdApiClient { let mut attempts = 0; let mut delay = Duration::from_secs(1); + let mut last_error: Option = None; + while attempts < max_attempts { let request = request_builder .try_clone() @@ -56,13 +59,19 @@ impl OrdApiClient { |status_code| status_code.is_server_error() || status_code.is_client_error(), ) => { + last_error = Some(format!( + "{}: {}", + e.status().unwrap_or(StatusCode::from_u16(0).unwrap()), + e + )); attempts += 1; sleep(delay).await; delay *= 2; } Err(e) => return Err(anyhow!(e)), }, - Err(_e) => { + Err(e) => { + last_error = Some(e.to_string()); attempts += 1; sleep(delay).await; delay *= 2; @@ -70,7 +79,12 @@ impl OrdApiClient { } } - Err(anyhow!("Exceeded maximum retry attempts")) + Err(anyhow!( + "Exceeded maximum retry attempts after {} tries. Last error: {}. Attempted endpoint: {}", + max_attempts, + last_error.unwrap_or_else(|| "No error captured".to_string()), + request_builder.build().unwrap().url().to_string() + )) } pub async fn fetch_inscription_details( From 79e459455586052fb33adc621556c59f1cfc2a3a Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 6 May 2024 18:17:02 +0200 Subject: [PATCH 067/103] Format --- src/event_consumer.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/event_consumer.rs b/src/event_consumer.rs index 82d74d891a..ddc7effaef 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -8,16 +8,13 @@ use clap::Parser; use futures::StreamExt; use lapin::{options::*, types::FieldTable}; use rand::distributions::DistString; -use serde::__private::de::IdentifierDeserializer; -use sqlx::{Connection, database}; -use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; +use sqlx::postgres::PgPoolOptions; use tokio::runtime::Runtime; use tokio::sync::oneshot; use urlencoding::encode; use crate::connect_rmq::connect_to_rabbitmq; use crate::index::event::Event; -use crate::Options; use crate::ord_api_client::OrdApiClient; use crate::ord_db_client::OrdDbClient; use crate::ord_indexation::OrdIndexation; @@ -59,7 +56,10 @@ impl EventConsumer { .database_url .as_deref() .context("db url must be defined")?; - log::info!("Connecting to database at {}", EventConsumer::mask_password_in_url(database_url)); + log::info!( + "Connecting to database at {}", + EventConsumer::mask_password_in_url(database_url) + ); let encoded_database_url = EventConsumer::encode_password_in_url(database_url); let pool = PgPoolOptions::new() @@ -148,7 +148,13 @@ impl EventConsumer { let username = caps.get(2).map_or("", |m| m.as_str()); let password = caps.get(3).map_or("", |m| m.as_str()); let rest_of_url = caps.get(4).map_or("", |m| m.as_str()); - format!("{}{}:{}@{}", protocol_and_user, username, encode(password), rest_of_url) + format!( + "{}{}:{}@{}", + protocol_and_user, + username, + encode(password), + rest_of_url + ) } else { url.to_string() } From 5af7c124eeb5f01330747856fbd2367f1f491eb5 Mon Sep 17 00:00:00 2001 From: rob Date: Mon, 6 May 2024 18:24:51 +0200 Subject: [PATCH 068/103] Fix error status propagation --- src/ord_api_client.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ord_api_client.rs b/src/ord_api_client.rs index a20a4bd8d3..207a17c422 100644 --- a/src/ord_api_client.rs +++ b/src/ord_api_client.rs @@ -2,7 +2,6 @@ use std::time::Duration; use anyhow::anyhow; use bitcoin::Txid; -use http::StatusCode; use reqwest::{Client, RequestBuilder}; use tokio::time::sleep; @@ -61,7 +60,7 @@ impl OrdApiClient { { last_error = Some(format!( "{}: {}", - e.status().unwrap_or(StatusCode::from_u16(0).unwrap()), + e.status().map_or("No Status Code".to_string(), |s| s.as_str().to_string()), e )); attempts += 1; From d5f53c126f766b96cc125da3ef8df98839d426a6 Mon Sep 17 00:00:00 2001 From: Bodhi <106667397+RareBodhi@users.noreply.github.com> Date: Tue, 7 May 2024 09:31:55 +0700 Subject: [PATCH 069/103] Update .github/workflows/ci.yaml --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 33e50a3187..78de9b0d64 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -41,7 +41,7 @@ jobs: run: cargo fmt --all -- --check test: - timeout-minutes: 5 + timeout-minutes: 10 runs-on: self-hosted steps: - From 30f8891f3287be641e3b7ca47061cc58e8b32f2b Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Tue, 7 May 2024 09:51:03 +0700 Subject: [PATCH 070/103] CI Adjustments --- .github/workflows/ci.yaml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 78de9b0d64..ff06a1783d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,7 +17,7 @@ env: jobs: lint: - timeout-minutes: 5 + timeout-minutes: 10 runs-on: ubuntu-latest steps: - name: Checkout @@ -26,11 +26,7 @@ jobs: fetch-depth: 0 - name: Install Rust Toolchain Components - uses: actions-rs/toolchain@v1 - with: - components: clippy, rustfmt - override: true - toolchain: stable + uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 @@ -51,10 +47,7 @@ jobs: fetch-depth: 0 - name: Install Rust Toolchain Components - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable + uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 From 1fe8b1af85e7a6bef786fb3fb4043ee390847ffe Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Tue, 7 May 2024 09:56:58 +0700 Subject: [PATCH 071/103] Test CI Change --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ff06a1783d..3b727e06cd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -38,7 +38,7 @@ jobs: test: timeout-minutes: 10 - runs-on: self-hosted + runs-on: ubuntu-latest steps: - name: Checkout From b6c809ec81e2541193e6e8ca8e7d412c84002778 Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Tue, 7 May 2024 10:02:29 +0700 Subject: [PATCH 072/103] Lint fix --- src/ord_api_client.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ord_api_client.rs b/src/ord_api_client.rs index 207a17c422..777d0c23fc 100644 --- a/src/ord_api_client.rs +++ b/src/ord_api_client.rs @@ -60,7 +60,8 @@ impl OrdApiClient { { last_error = Some(format!( "{}: {}", - e.status().map_or("No Status Code".to_string(), |s| s.as_str().to_string()), + e.status() + .map_or("No Status Code".to_string(), |s| s.as_str().to_string()), e )); attempts += 1; From b7252683b701709bc0dd7edae33e073db6f1ab82 Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Tue, 7 May 2024 10:34:47 +0700 Subject: [PATCH 073/103] Test fix --- README.md | 37 +++++++++++++++++++++++++------------ tests/settings.rs | 4 +++- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 78785e9e35..3e36d568ed 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,31 @@ currently prioritized issues. Join [the Discord server](https://discord.gg/87cjuz4FYg) to chat with fellow ordinal degenerates. +LooksRare Ordinals +------------------ + +**Setup** + +1. Setup the local `ordinals` database +`$ psql -h localhost -d postgres -q -f db/config.sql` + +2. Run the migrations. +`cargo sqlx database setup --database-url $DATABASE_URL --source db/migrations/` + +Note database url is likely to be _postgres://backend:looks-backend@localhost:5432/ordinals"_ + +**Examples running via command line:** + +To run ord event server +``` +ord --bitcoin-rpc-url "http://localhost:18332" --bitcoin-rpc-username user --bitcoin-rpc-password password --commit-interval 10 --rabbitmq-url localhost --rabbitmq-password s3cr3t --rabbitmq-username indexer --rabbitmq-exchange ord-tx --chain testnet --data-dir ./index-data event-server --http-port 8080 +``` + +To run ord event indexer +``` +ord --rabbitmq-url localhost --rabbitmq-password s3cr3t --rabbitmq-username indexer event-consumer --blocks-queue btc-blocks-q --inscriptions-queue btc-inscription-q --database-url postgresql://backend:looks-backend@localhost:55432/ordinals --ord-api-url http://localhost:8080 +``` + Donate ------ @@ -196,18 +221,6 @@ bitcoin_rpc_username: foo bitcoin_rpc_password: bar ``` -Examples running via command line: - -To run ord event server -``` -ord --bitcoin-rpc-url "http://localhost:18332" --bitcoin-rpc-username user --bitcoin-rpc-password password --commit-interval 10 --rabbitmq-url localhost --rabbitmq-password s3cr3t --rabbitmq-username indexer --rabbitmq-exchange ord-tx --chain testnet --data-dir ./index-data event-server --http-port 8080 -``` - -To run ord event indexer -``` -ord --rabbitmq-url localhost --rabbitmq-password s3cr3t --rabbitmq-username indexer event-consumer --blocks-queue btc-blocks-q --inscriptions-queue btc-inscription-q --database-url postgresql://backend:looks-backend@localhost:55432/ordinals --ord-api-url http://localhost:8080 -``` - Logging -------- diff --git a/tests/settings.rs b/tests/settings.rs index 0244259be4..bb0b7128cf 100644 --- a/tests/settings.rs +++ b/tests/settings.rs @@ -30,7 +30,9 @@ fn default() { "no_index_inscriptions": false, "server_password": null, "server_url": null, - "server_username": null + "server_username": null, + "rabbitmq_url": null, + "rabbitmq_exchange": null \} "#, ) From 43734fe2c2509b208522faa415cc7542312e4583 Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Tue, 7 May 2024 12:11:19 +0700 Subject: [PATCH 074/103] Compile time checked queries --- .env.example | 1 + .gitignore | 3 +- ...39bc7809996d1e479898c906cfe39f3350acf.json | 17 + ...9306d21d5940ae37871166afce9f664bf53d4.json | 24 ++ ...7caacffb6de8323126e4529e5bcf875e6cc41.json | 46 +++ ...4d8b24981488c6fe23aa9d6b06b28e7d21ddb.json | 36 ++ ...87408dd4c93132eeb1f1af111079bbe0d8d49.json | 22 ++ ...314d0d235da28ead0b89fdaa090cf2f16a151.json | 18 + README.md | 15 +- crates/ordinals/src/rarity.rs | 2 +- ...ration.sql => 20240431000000_ordinals.sql} | 0 src/ord_db_client.rs | 350 +++++++++--------- 12 files changed, 344 insertions(+), 190 deletions(-) create mode 100644 .env.example create mode 100644 .sqlx/query-03b5007410af0c37827309e7e2439bc7809996d1e479898c906cfe39f3350acf.json create mode 100644 .sqlx/query-1289edfa88c5539ccee82bb3d389306d21d5940ae37871166afce9f664bf53d4.json create mode 100644 .sqlx/query-37a57ddabacea1d4357e4c906c37caacffb6de8323126e4529e5bcf875e6cc41.json create mode 100644 .sqlx/query-3d3201c09d01f1fe7776db4abb54d8b24981488c6fe23aa9d6b06b28e7d21ddb.json create mode 100644 .sqlx/query-4fbd66a8177f12a3291abc8506287408dd4c93132eeb1f1af111079bbe0d8d49.json create mode 100644 .sqlx/query-7d3e8f006c4f66cff514bb7bcf5314d0d235da28ead0b89fdaa090cf2f16a151.json rename db/migrations/{20240431000000_ordinals/migration.sql => 20240431000000_ordinals.sql} (100%) diff --git a/.env.example b/.env.example new file mode 100644 index 0000000000..5f64bd27f3 --- /dev/null +++ b/.env.example @@ -0,0 +1 @@ +DATABASE_URL=postgres://backend:looks-backend@localhost:5432/ordinals \ No newline at end of file diff --git a/.gitignore b/.gitignore index ec17c72dd0..cffa3eb598 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.env /*.redb /.idea/ /.vagrant @@ -12,4 +13,4 @@ /target /test-times.txt /tmp -/index-data \ No newline at end of file +/index-data diff --git a/.sqlx/query-03b5007410af0c37827309e7e2439bc7809996d1e479898c906cfe39f3350acf.json b/.sqlx/query-03b5007410af0c37827309e7e2439bc7809996d1e479898c906cfe39f3350acf.json new file mode 100644 index 0000000000..763de586cd --- /dev/null +++ b/.sqlx/query-03b5007410af0c37827309e7e2439bc7809996d1e479898c906cfe39f3350acf.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO event (type_id, block_height, inscription_id, location)\n SELECT $1, $2, $3, $4\n WHERE NOT EXISTS (\n SELECT 1 FROM event\n WHERE type_id = $1 AND block_height = $2 AND inscription_id = $3 AND location = $4\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int2", + "Int8", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "03b5007410af0c37827309e7e2439bc7809996d1e479898c906cfe39f3350acf" +} diff --git a/.sqlx/query-1289edfa88c5539ccee82bb3d389306d21d5940ae37871166afce9f664bf53d4.json b/.sqlx/query-1289edfa88c5539ccee82bb3d389306d21d5940ae37871166afce9f664bf53d4.json new file mode 100644 index 0000000000..21c1ae17b6 --- /dev/null +++ b/.sqlx/query-1289edfa88c5539ccee82bb3d389306d21d5940ae37871166afce9f664bf53d4.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO location (\n inscription_id\n , block_height\n , block_time\n , tx_id\n , to_address\n , cur_output\n , cur_offset\n , from_address\n , prev_output\n , prev_offset\n , value\n )\n SELECT\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11\n WHERE NOT EXISTS (\n SELECT 1 FROM location\n WHERE\n inscription_id = $1\n AND block_height = $2\n AND block_time = $3\n AND tx_id = $4\n AND to_address = $5\n AND cur_output = $6\n AND cur_offset = $7\n AND from_address = $8\n AND prev_output = $9\n AND prev_offset = $10\n AND value = $11\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4", + "Int8", + "Text", + "Text", + "Text", + "Int8", + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1289edfa88c5539ccee82bb3d389306d21d5940ae37871166afce9f664bf53d4" +} diff --git a/.sqlx/query-37a57ddabacea1d4357e4c906c37caacffb6de8323126e4529e5bcf875e6cc41.json b/.sqlx/query-37a57ddabacea1d4357e4c906c37caacffb6de8323126e4529e5bcf875e6cc41.json new file mode 100644 index 0000000000..2f3a58392b --- /dev/null +++ b/.sqlx/query-37a57ddabacea1d4357e4c906c37caacffb6de8323126e4529e5bcf875e6cc41.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT type_id, block_height, inscription_id, location, old_location\n FROM event WHERE block_height = $1\n ORDER BY type_id ASC, id ASC\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "type_id", + "type_info": "Int2" + }, + { + "ordinal": 1, + "name": "block_height", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "inscription_id", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "location", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "old_location", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + true, + true + ] + }, + "hash": "37a57ddabacea1d4357e4c906c37caacffb6de8323126e4529e5bcf875e6cc41" +} diff --git a/.sqlx/query-3d3201c09d01f1fe7776db4abb54d8b24981488c6fe23aa9d6b06b28e7d21ddb.json b/.sqlx/query-3d3201c09d01f1fe7776db4abb54d8b24981488c6fe23aa9d6b06b28e7d21ddb.json new file mode 100644 index 0000000000..46e5f38b5d --- /dev/null +++ b/.sqlx/query-3d3201c09d01f1fe7776db4abb54d8b24981488c6fe23aa9d6b06b28e7d21ddb.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO inscription (\n genesis_id\n , number\n , content_type\n , content_length\n , metadata\n , genesis_block_height\n , genesis_block_time\n , sat_number\n , sat_rarity\n , sat_block_height\n , sat_block_time\n , fee\n , charms\n , children\n , parents\n ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)\n ON CONFLICT (genesis_id) DO UPDATE SET\n number = EXCLUDED.number\n , content_type = EXCLUDED.content_type\n , content_length = COALESCE(EXCLUDED.content_length, inscription.content_length)\n , metadata = COALESCE(EXCLUDED.metadata, inscription.metadata)\n , genesis_block_height = EXCLUDED.genesis_block_height\n , genesis_block_time = EXCLUDED.genesis_block_time\n , sat_number = COALESCE(EXCLUDED.sat_number, inscription.sat_number)\n , sat_rarity = COALESCE(EXCLUDED.sat_rarity, inscription.sat_rarity)\n , sat_block_height = COALESCE(EXCLUDED.sat_block_height, inscription.sat_block_height)\n , sat_block_time = COALESCE(EXCLUDED.sat_block_time, inscription.sat_block_time)\n , fee = EXCLUDED.fee\n , charms = EXCLUDED.charms\n , children = COALESCE(EXCLUDED.children, inscription.children)\n , parents = COALESCE(EXCLUDED.parents, inscription.parents)\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Int4", + "Varchar", + "Int4", + "Text", + "Int4", + "Int8", + "Int8", + "Int4", + "Int4", + "Int8", + "Int8", + "Int2", + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "3d3201c09d01f1fe7776db4abb54d8b24981488c6fe23aa9d6b06b28e7d21ddb" +} diff --git a/.sqlx/query-4fbd66a8177f12a3291abc8506287408dd4c93132eeb1f1af111079bbe0d8d49.json b/.sqlx/query-4fbd66a8177f12a3291abc8506287408dd4c93132eeb1f1af111079bbe0d8d49.json new file mode 100644 index 0000000000..81b601f334 --- /dev/null +++ b/.sqlx/query-4fbd66a8177f12a3291abc8506287408dd4c93132eeb1f1af111079bbe0d8d49.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM inscription WHERE genesis_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "4fbd66a8177f12a3291abc8506287408dd4c93132eeb1f1af111079bbe0d8d49" +} diff --git a/.sqlx/query-7d3e8f006c4f66cff514bb7bcf5314d0d235da28ead0b89fdaa090cf2f16a151.json b/.sqlx/query-7d3e8f006c4f66cff514bb7bcf5314d0d235da28ead0b89fdaa090cf2f16a151.json new file mode 100644 index 0000000000..be09dd1923 --- /dev/null +++ b/.sqlx/query-7d3e8f006c4f66cff514bb7bcf5314d0d235da28ead0b89fdaa090cf2f16a151.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO event (type_id, block_height, inscription_id, location, old_location)\n SELECT $1, $2, $3, $4, $5\n WHERE NOT EXISTS (\n SELECT 1 FROM event\n WHERE type_id = $1 AND block_height = $2 AND inscription_id = $3 AND location = $4 AND old_location = $5\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int2", + "Int8", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "7d3e8f006c4f66cff514bb7bcf5314d0d235da28ead0b89fdaa090cf2f16a151" +} diff --git a/README.md b/README.md index 3e36d568ed..89428b24fe 100644 --- a/README.md +++ b/README.md @@ -24,17 +24,22 @@ ordinal degenerates. LooksRare Ordinals ------------------ -**Setup** +**Database & Sqlx Setup** 1. Setup the local `ordinals` database `$ psql -h localhost -d postgres -q -f db/config.sql` -2. Run the migrations. -`cargo sqlx database setup --database-url $DATABASE_URL --source db/migrations/` +2. Run the database migrations. +`$ cargo sqlx migrate run --database-url $DATABASE_URL --source db/migrations/` -Note database url is likely to be _postgres://backend:looks-backend@localhost:5432/ordinals"_ +3. Clone `.env.example` into `.env` for compile time checked sql queries -**Examples running via command line:** +4. After changing queries you can run +`$ cargo sqlx prepare --database-url $DATABASE_URL` + +Note `$DATABASE_URL` is likely to be _"postgres://backend:looks-backend@localhost:5432/ordinals"_. + +**Running via command line:** To run ord event server ``` diff --git a/crates/ordinals/src/rarity.rs b/crates/ordinals/src/rarity.rs index 845795aeb9..ae5a850998 100644 --- a/crates/ordinals/src/rarity.rs +++ b/crates/ordinals/src/rarity.rs @@ -11,7 +11,7 @@ pub enum Rarity { } impl Rarity { - pub fn to_i16(&self) -> i16 { + pub fn to_i32(&self) -> i32 { match self { Rarity::Common => 0, Rarity::Uncommon => 1, diff --git a/db/migrations/20240431000000_ordinals/migration.sql b/db/migrations/20240431000000_ordinals.sql similarity index 100% rename from db/migrations/20240431000000_ordinals/migration.sql rename to db/migrations/20240431000000_ordinals.sql diff --git a/src/ord_db_client.rs b/src/ord_db_client.rs index d4be4e1890..df42fda43c 100644 --- a/src/ord_db_client.rs +++ b/src/ord_db_client.rs @@ -1,17 +1,14 @@ -use std::str::FromStr; -use std::sync::Arc; - use bitcoin::{OutPoint, Txid}; -use futures::TryStreamExt; -use sqlx::types::Json; -use sqlx::{PgPool, Row}; - use ordinals::SatPoint; +use sqlx::types::Json; +use sqlx::PgPool; +use std::str::FromStr; +use std::sync::Arc; use crate::api::InscriptionDetails; use crate::InscriptionId; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, sqlx::FromRow)] pub struct Event { pub type_id: i16, pub block_height: i64, @@ -20,24 +17,6 @@ pub struct Event { pub old_location: Option, } -impl Event { - fn from_row(row: sqlx::postgres::PgRow) -> Self { - Event { - type_id: row.get("type_id"), - block_height: row.get("block_height"), - inscription_id: row.get("inscription_id"), - location: { - let location_str: Option = row.get("location"); - location_str.and_then(|s| SatPoint::from_str(&s).ok()) - }, - old_location: { - let old_location_str: Option = row.get("old_location"); - old_location_str.and_then(|s| SatPoint::from_str(&s).ok()) - }, - } - } -} - pub struct OrdDbClient { pool: Arc, } @@ -51,24 +30,23 @@ impl OrdDbClient { &self, block_height: u32, ) -> Result, sqlx::Error> { - let query = " - SELECT type_id, block_height, inscription_id, location, old_location - FROM event - WHERE block_height = $1::BIGINT - ORDER BY type_id ASC, id ASC"; - - let mut rows = sqlx::query(query) - .bind(block_height.to_string()) - .fetch(&*self.pool); - - let mut events = Vec::new(); - - while let Some(row) = rows.try_next().await? { - let event = Event::from_row(row); - events.push(event); - } - - Ok(events) + sqlx::query!( + r#" + SELECT type_id, block_height, inscription_id, location, old_location + FROM event WHERE block_height = $1 + ORDER BY type_id ASC, id ASC + "#, + block_height as i64, + ) + .map(|r| Event { + type_id: r.type_id, + block_height: r.block_height, + inscription_id: r.inscription_id, + location: r.location.and_then(|s| SatPoint::from_str(&s).ok()), + old_location: r.old_location.and_then(|s| SatPoint::from_str(&s).ok()), + }) + .fetch_all(&*self.pool) + .await } pub async fn save_inscription_created( @@ -77,20 +55,23 @@ impl OrdDbClient { inscription_id: &InscriptionId, location: &Option, ) -> Result<(), sqlx::Error> { - let query = " - INSERT INTO event (type_id, block_height, inscription_id, location) - SELECT $1, $2::BIGINT, $3, $4 - WHERE NOT EXISTS ( - SELECT 1 FROM event - WHERE type_id = $1 AND block_height = $2::BIGINT AND inscription_id = $3 AND location = $4 - )"; - sqlx::query(query) - .bind(1_i32) // Type ID for InscriptionCreated - .bind(block_height.to_string()) - .bind(inscription_id.to_string()) - .bind(location.map(|loc| loc.to_string())) - .execute(&*self.pool) - .await?; + sqlx::query!( + r#" + INSERT INTO event (type_id, block_height, inscription_id, location) + SELECT $1, $2, $3, $4 + WHERE NOT EXISTS ( + SELECT 1 FROM event + WHERE type_id = $1 AND block_height = $2 AND inscription_id = $3 AND location = $4 + ) + "#, + 1, // Type ID for `InscriptionCreated` + block_height.to_owned() as i64, + inscription_id.to_string(), + location.map(|loc| loc.to_string()) + ) + .execute(&*self.pool) + .await?; + Ok(()) } @@ -101,21 +82,24 @@ impl OrdDbClient { new_location: &SatPoint, old_location: &SatPoint, ) -> Result<(), sqlx::Error> { - let query = " - INSERT INTO event (type_id, block_height, inscription_id, location, old_location) - SELECT $1, $2::BIGINT, $3, $4, $5 - WHERE NOT EXISTS ( - SELECT 1 FROM event - WHERE type_id = $1 AND block_height = $2::BIGINT AND inscription_id = $3 AND location = $4 AND old_location = $5 - )"; - sqlx::query(query) - .bind(2_i32) // Type ID for InscriptionTransferred - .bind(block_height.to_string()) - .bind(inscription_id.to_string()) - .bind(new_location.to_string()) - .bind(old_location.to_string()) - .execute(&*self.pool) - .await?; + sqlx::query!( + r#" + INSERT INTO event (type_id, block_height, inscription_id, location, old_location) + SELECT $1, $2, $3, $4, $5 + WHERE NOT EXISTS ( + SELECT 1 FROM event + WHERE type_id = $1 AND block_height = $2 AND inscription_id = $3 AND location = $4 AND old_location = $5 + ) + "#, + 2, // Type ID for `InscriptionCreated` + block_height.to_owned() as i64, + inscription_id.to_string(), + new_location.to_string(), + old_location.to_string() + ) + .execute(&*self.pool) + .await?; + Ok(()) } @@ -123,16 +107,13 @@ impl OrdDbClient { &self, genesis_id: String, ) -> Result, sqlx::Error> { - let query = " - SELECT id FROM inscription - WHERE genesis_id = $1; - "; - let result = sqlx::query(query) - .bind(genesis_id) - .fetch_optional(&*self.pool) - .await? - .map(|row| row.get(0)); - Ok(result) + sqlx::query!( + r#"SELECT id FROM inscription WHERE genesis_id = $1"#, + genesis_id + ) + .map(|r| r.id) + .fetch_optional(&*self.pool) + .await } pub async fn save_inscription( @@ -140,60 +121,61 @@ impl OrdDbClient { inscription_details: &InscriptionDetails, metadata: Option, ) -> Result { - let query = " - INSERT INTO inscription ( - genesis_id, - number, - content_type, - content_length, - metadata, - genesis_block_height, - genesis_block_time, - sat_number, - sat_rarity, - sat_block_height, - sat_block_time, - fee, - charms, - children, - parents - ) VALUES ($1, $2, $3, $4::INTEGER, $5, $6::INTEGER, $7::BIGINT, $8::BIGINT, $9, $10::INTEGER, $11::BIGINT, $12::BIGINT, $13::SMALLINT, $14, $15) - ON CONFLICT (genesis_id) DO UPDATE SET - number = EXCLUDED.number, - content_type = EXCLUDED.content_type, - content_length = COALESCE(EXCLUDED.content_length, inscription.content_length), - metadata = COALESCE(EXCLUDED.metadata, inscription.metadata), - genesis_block_height = EXCLUDED.genesis_block_height, - genesis_block_time = EXCLUDED.genesis_block_time, - sat_number = COALESCE(EXCLUDED.sat_number, inscription.sat_number), - sat_rarity = COALESCE(EXCLUDED.sat_rarity, inscription.sat_rarity), - sat_block_height = COALESCE(EXCLUDED.sat_block_height, inscription.sat_block_height), - sat_block_time = COALESCE(EXCLUDED.sat_block_time, inscription.sat_block_time), - fee = EXCLUDED.fee, - charms = EXCLUDED.charms, - children = COALESCE(EXCLUDED.children, inscription.children), - parents = COALESCE(EXCLUDED.parents, inscription.parents) - RETURNING id; - "; - let id = sqlx::query_as::<_, (i32,)>(query) - .bind(inscription_details.id.to_string()) - .bind(inscription_details.number) - .bind(inscription_details.content_type.as_deref()) - .bind(inscription_details.content_length.map(|n| n.to_string())) - .bind(metadata) - .bind(inscription_details.genesis_block_height.to_string()) - .bind(inscription_details.genesis_block_time) - .bind(inscription_details.sat_number.map(|n| n.to_string())) - .bind(inscription_details.sat_rarity.map(|r| r.to_i16())) - .bind(inscription_details.sat_block_height.map(|n| n.to_string())) - .bind(inscription_details.sat_block_time) - .bind(inscription_details.fee.to_string()) - .bind(inscription_details.charms.to_string()) - .bind(Json(&inscription_details.children)) - .bind(Json(&inscription_details.parents)) - .fetch_one(&*self.pool) - .await?; - Ok(id.0) + sqlx::query!( + r#" + INSERT INTO inscription ( + genesis_id + , number + , content_type + , content_length + , metadata + , genesis_block_height + , genesis_block_time + , sat_number + , sat_rarity + , sat_block_height + , sat_block_time + , fee + , charms + , children + , parents + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) + ON CONFLICT (genesis_id) DO UPDATE SET + number = EXCLUDED.number + , content_type = EXCLUDED.content_type + , content_length = COALESCE(EXCLUDED.content_length, inscription.content_length) + , metadata = COALESCE(EXCLUDED.metadata, inscription.metadata) + , genesis_block_height = EXCLUDED.genesis_block_height + , genesis_block_time = EXCLUDED.genesis_block_time + , sat_number = COALESCE(EXCLUDED.sat_number, inscription.sat_number) + , sat_rarity = COALESCE(EXCLUDED.sat_rarity, inscription.sat_rarity) + , sat_block_height = COALESCE(EXCLUDED.sat_block_height, inscription.sat_block_height) + , sat_block_time = COALESCE(EXCLUDED.sat_block_time, inscription.sat_block_time) + , fee = EXCLUDED.fee + , charms = EXCLUDED.charms + , children = COALESCE(EXCLUDED.children, inscription.children) + , parents = COALESCE(EXCLUDED.parents, inscription.parents) + RETURNING id + "#, + inscription_details.id.to_string(), + inscription_details.number, + inscription_details.content_type.as_deref(), + inscription_details.content_length.map(|n| n as i32), + metadata, + inscription_details.genesis_block_height as i32, + inscription_details.genesis_block_time, + inscription_details.sat_number.map(|n| n as i64), + inscription_details.sat_rarity.map(|r| r.to_i32()), + inscription_details.sat_block_height.map(|n| n as i32), + inscription_details.sat_block_time, + inscription_details.fee as i64, + inscription_details.charms as i16, + Json(&inscription_details.children).encode_to_string(), + Json(&inscription_details.parents).encode_to_string() + ) + .map(|r| r.id) + .fetch_one(&*self.pool) + .await } pub async fn save_location( @@ -210,52 +192,54 @@ impl OrdDbClient { from_offset: Option, value: Option, ) -> Result<(), sqlx::Error> { - let query = " - INSERT INTO location ( - inscription_id, - block_height, - block_time, - tx_id, - to_address, - cur_output, - cur_offset, - from_address, - prev_output, - prev_offset, - value - ) - SELECT - $1, $2, $3::BIGINT, $4, $5, $6, $7::BIGINT, $8, $9, $10::BIGINT, $11::BIGINT - WHERE NOT EXISTS ( - SELECT 1 FROM location - WHERE - inscription_id = $1 AND - block_height = $2 AND - block_time = $3::BIGINT AND - tx_id = $4 AND - to_address = $5 AND - cur_output = $6 AND - cur_offset = $7::BIGINT AND - from_address = $8 AND - prev_output = $9 AND - prev_offset = $10::BIGINT AND - value = $10::BIGINT - ); - "; - sqlx::query(query) - .bind(id) - .bind(block_height) - .bind(block_time.to_string()) - .bind(tx_id.map(|n| n.to_string())) - .bind(to_address) - .bind(to_outpoint.map(|n| n.to_string())) - .bind(to_offset.map(|n| n.to_string())) - .bind(from_address) - .bind(from_outpoint.map(|n| n.to_string())) - .bind(from_offset.map(|n| n.to_string())) - .bind(value.map(|n| n.to_string())) - .execute(&*self.pool) - .await?; + sqlx::query!( + r#" + INSERT INTO location ( + inscription_id + , block_height + , block_time + , tx_id + , to_address + , cur_output + , cur_offset + , from_address + , prev_output + , prev_offset + , value + ) + SELECT + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11 + WHERE NOT EXISTS ( + SELECT 1 FROM location + WHERE + inscription_id = $1 + AND block_height = $2 + AND block_time = $3 + AND tx_id = $4 + AND to_address = $5 + AND cur_output = $6 + AND cur_offset = $7 + AND from_address = $8 + AND prev_output = $9 + AND prev_offset = $10 + AND value = $11 + ) + "#, + id, + block_height as i64, + block_time as i64, + tx_id.map(|n| n.to_string()), + to_address, + to_outpoint.map(|n| n.to_string()), + to_offset.map(|n| n as i64), + from_address, + from_outpoint.map(|n| n.to_string()), + from_offset.map(|n| n as i64), + value.map(|n| n as i64), + ) + .execute(&*self.pool) + .await?; + Ok(()) } } From 50691f3ca5dd6a030265d02ed409cee709ce0094 Mon Sep 17 00:00:00 2001 From: rob Date: Tue, 7 May 2024 07:36:05 +0200 Subject: [PATCH 075/103] fix sql --- src/ord_db_client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ord_db_client.rs b/src/ord_db_client.rs index d4be4e1890..d5147a5811 100644 --- a/src/ord_db_client.rs +++ b/src/ord_db_client.rs @@ -239,7 +239,7 @@ impl OrdDbClient { from_address = $8 AND prev_output = $9 AND prev_offset = $10::BIGINT AND - value = $10::BIGINT + value = $11::BIGINT ); "; sqlx::query(query) From 09953d47d32057b04aef34698dc5465e1e882d4b Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Tue, 7 May 2024 12:45:26 +0700 Subject: [PATCH 076/103] Redundant from --- crates/ordinals/src/rarity.rs | 13 ---------- src/connect_rmq.rs | 46 ++++++++++++++++------------------- src/ord_db_client.rs | 2 +- 3 files changed, 22 insertions(+), 39 deletions(-) diff --git a/crates/ordinals/src/rarity.rs b/crates/ordinals/src/rarity.rs index ae5a850998..a92fce72a9 100644 --- a/crates/ordinals/src/rarity.rs +++ b/crates/ordinals/src/rarity.rs @@ -10,19 +10,6 @@ pub enum Rarity { Mythic, } -impl Rarity { - pub fn to_i32(&self) -> i32 { - match self { - Rarity::Common => 0, - Rarity::Uncommon => 1, - Rarity::Rare => 2, - Rarity::Epic => 3, - Rarity::Legendary => 4, - Rarity::Mythic => 5, - } - } -} - impl From for u8 { fn from(rarity: Rarity) -> Self { rarity as u8 diff --git a/src/connect_rmq.rs b/src/connect_rmq.rs index 57e570363f..62c6cc936c 100644 --- a/src/connect_rmq.rs +++ b/src/connect_rmq.rs @@ -1,37 +1,33 @@ -use std::collections::HashSet; - -use anyhow::Context; +use anyhow::{Context, Error}; use lapin::tcp::{AMQPUriTcpExt, NativeTlsConnector}; use lapin::uri::AMQPUri; use lapin::{Connection, ConnectionProperties}; pub async fn connect_to_rabbitmq(addr: &str) -> Result { + let opt = ConnectionProperties::default(); let uri = addr .parse::() - .map_err(anyhow::Error::msg) - .context("Failed to parse AMQP URI")?; - let host = uri.authority.host.clone(); + .map_err(Error::msg) + .context("failed to parse AMQP URI")?; - let non_tls_hosts: HashSet = vec!["localhost".to_string()].into_iter().collect(); + match uri.authority.host.as_str() { + "localhost" => Connection::connect(addr, opt) + .await + .context("failed to establish an unsecure AMQP connection"), - let is_secure = !non_tls_hosts.contains(&host); + _ => { + let connect = move |uri: &AMQPUri| { + uri.connect().and_then(|stream| { + let mut tls_builder = NativeTlsConnector::builder(); + tls_builder.danger_accept_invalid_certs(true); + let connector = &tls_builder.build().expect("TLS configuration failed"); + stream.into_native_tls(connector, &uri.authority.host) + }) + }; - if is_secure { - let connect = move |uri: &AMQPUri| { - let conn = uri.connect().and_then(|stream| { - let mut tls_builder = NativeTlsConnector::builder(); - tls_builder.danger_accept_invalid_certs(true); - let connector = &tls_builder.build().expect("TLS configuration failed"); - stream.into_native_tls(connector, &uri.authority.host) - }); - conn - }; - Connection::connector(uri, Box::new(connect), ConnectionProperties::default()) - .await - .context("Failed to establish a secure AMQP connection") - } else { - Connection::connect(addr, ConnectionProperties::default()) - .await - .context("Failed to establish an unsecure AMQP connection") + Connection::connector(uri, Box::new(connect), opt) + .await + .context("failed to establish a secure AMQP connection") + } } } diff --git a/src/ord_db_client.rs b/src/ord_db_client.rs index df42fda43c..6225750ab4 100644 --- a/src/ord_db_client.rs +++ b/src/ord_db_client.rs @@ -165,7 +165,7 @@ impl OrdDbClient { inscription_details.genesis_block_height as i32, inscription_details.genesis_block_time, inscription_details.sat_number.map(|n| n as i64), - inscription_details.sat_rarity.map(|r| r.to_i32()), + inscription_details.sat_rarity.map(|r| r as i32), inscription_details.sat_block_height.map(|n| n as i32), inscription_details.sat_block_time, inscription_details.fee as i64, From ffe8297ae9a39e73a41f342e9fac0034365b6c98 Mon Sep 17 00:00:00 2001 From: rob Date: Tue, 7 May 2024 12:14:33 +0200 Subject: [PATCH 077/103] Publish one block at a time --- src/index/updater.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/index/updater.rs b/src/index/updater.rs index 8e79843812..bbc2c0f25a 100644 --- a/src/index/updater.rs +++ b/src/index/updater.rs @@ -746,10 +746,12 @@ impl<'index> Updater<'index> { if let Some(sender) = self.index.event_sender.as_ref() { if let Ok(uncommitted) = u32::try_from(uncommitted) { - sender.blocking_send(Event::BlockCommitted { - from_height: self.height - uncommitted, - to_height: self.height, - })?; + for current_height in (self.height - uncommitted)..=self.height { + sender.blocking_send(Event::BlockCommitted { + from_height: current_height - 1, + to_height: current_height, + })?; + } } else { log::error!( "Failed to publish block range from_height: {}, uncommitted: {}", From 817d1c91d986678bc1fd85848b55d1723b64a3c2 Mon Sep 17 00:00:00 2001 From: rob Date: Tue, 7 May 2024 14:40:40 +0200 Subject: [PATCH 078/103] Refactor --- .../api_client.rs} | 10 +- src/indexer/block_consumer.rs | 173 ++++++++++++++++++ .../db_client.rs} | 30 +-- src/indexer/db_con.rs | 31 ++++ src/{ => indexer}/event_consumer.rs | 142 ++------------ src/{ => indexer}/event_publisher.rs | 11 +- .../inscription_indexation.rs} | 44 +---- src/{connect_rmq.rs => indexer/rmq_con.rs} | 29 ++- src/lib.rs | 17 +- src/subcommand.rs | 9 +- 10 files changed, 294 insertions(+), 202 deletions(-) rename src/{ord_api_client.rs => indexer/api_client.rs} (94%) create mode 100644 src/indexer/block_consumer.rs rename src/{ord_db_client.rs => indexer/db_client.rs} (92%) create mode 100644 src/indexer/db_con.rs rename src/{ => indexer}/event_consumer.rs (50%) rename src/{ => indexer}/event_publisher.rs (89%) rename src/{ord_indexation.rs => indexer/inscription_indexation.rs} (84%) rename src/{connect_rmq.rs => indexer/rmq_con.rs} (59%) diff --git a/src/ord_api_client.rs b/src/indexer/api_client.rs similarity index 94% rename from src/ord_api_client.rs rename to src/indexer/api_client.rs index 777d0c23fc..c5b178450a 100644 --- a/src/ord_api_client.rs +++ b/src/indexer/api_client.rs @@ -7,18 +7,16 @@ use tokio::time::sleep; use crate::api::{BlockInfo, InscriptionDetails, Transaction}; -pub struct OrdApiClient { +pub struct ApiClient { ord_api_url: String, client: Client, } -impl OrdApiClient { +impl ApiClient { pub fn new(ord_api_url: String) -> anyhow::Result { - let client = Client::builder() - .timeout(std::time::Duration::from_secs(30)) - .build()?; + let client = Client::builder().timeout(Duration::from_secs(30)).build()?; - Ok(OrdApiClient { + Ok(ApiClient { ord_api_url, client, }) diff --git a/src/indexer/block_consumer.rs b/src/indexer/block_consumer.rs new file mode 100644 index 0000000000..0dfdd92ef5 --- /dev/null +++ b/src/indexer/block_consumer.rs @@ -0,0 +1,173 @@ +use std::process; +use std::sync::Arc; + +use anyhow::Context; +use clap::Parser; +use futures::StreamExt; +use lapin::{options::*, types::FieldTable}; +use tokio::runtime::Runtime; +use tokio::sync::oneshot; + +use crate::index::event::Event; +use crate::indexer::api_client::ApiClient; +use crate::indexer::db_client::DbClient; +use crate::indexer::db_con::setup_db_connection; +use crate::indexer::inscription_indexation::InscriptionIndexation; +use crate::indexer::rmq_con::{generate_consumer_tag, setup_rabbitmq_connection}; +use crate::settings::Settings; +use crate::subcommand::SubcommandResult; + +#[derive(Debug, Parser, Clone)] +pub struct BlockConsumer { + #[arg(long, help = "RMQ queue to consume blocks.")] + pub(crate) blocks_queue: Option, + #[arg(long, help = "DB url to persist inscriptions.")] + pub(crate) database_url: Option, + #[arg(long, help = "Ord api url to fetch inscriptions.")] + pub(crate) ord_api_url: Option, +} + +impl BlockConsumer { + pub fn run(self, settings: &Settings) -> SubcommandResult { + Runtime::new()?.block_on(async { + let addr = settings + .rabbitmq_addr() + .context("rabbitmq amqp credentials and url must be defined")?; + let channel = setup_rabbitmq_connection(addr).await?; + + let database_url = self + .database_url + .as_deref() + .context("db url must be defined")?; + let pool = setup_db_connection(database_url).await?; + + let shared_pool = Arc::new(pool); + let db_client = Arc::new(DbClient::new(shared_pool.clone())); + + let api_url = self.ord_api_url.context("api url must be defined")?; + let api_c = ApiClient::new(api_url.clone()).expect("api client must exist"); + let api_client = Arc::new(api_c); + + let inscription_indexation = + Arc::new(InscriptionIndexation::new(settings, db_client, api_client)); + + let blocks_queue = self + .blocks_queue + .as_deref() + .context("rabbitmq blocks queue path must be defined")?; + let blocks_queue_str = blocks_queue.to_string(); + let blocks_channel = channel.clone(); + let blocks_consumer_tag = generate_consumer_tag("lr-ord"); + let (blocks_shutdown_tx, blocks_shutdown_rx) = oneshot::channel::<()>(); + + let blocks_consumer_handle = tokio::spawn(async move { + BlockConsumer::consume_queue( + blocks_channel, + blocks_queue_str, + blocks_consumer_tag, + inscription_indexation, + blocks_shutdown_rx, + ) + .await + .map_err(|e| { + log::error!("Error consuming blocks queue: {}", e); + process::exit(1); + }) + }); + + let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + sigterm.recv().await; + let _ = blocks_shutdown_tx.send(()); + let _ = tokio::try_join!(blocks_consumer_handle); + + shared_pool.close().await; + + Ok(None) + }) + } + + async fn consume_queue( + channel: lapin::Channel, + queue_name: String, + consumer_tag: String, + inscription_indexation: Arc, + mut shutdown_signal: oneshot::Receiver<()>, + ) -> Result<(), anyhow::Error> { + let mut consumer = channel + .basic_consume( + &queue_name, + consumer_tag.as_str(), + BasicConsumeOptions::default(), + FieldTable::default(), + ) + .await?; + + log::info!("Starting to consume messages from {}", queue_name); + while let Some(result) = consumer.next().await { + let delivery = result?; + tokio::select! { + process_result = BlockConsumer::handle_delivery(delivery, &inscription_indexation) => { + process_result?; + }, + _ = &mut shutdown_signal => { + log::info!("Shutdown signal received, stopping consumer."); + break; + }, + } + } + + log::info!("Closing consumer channel {}", queue_name); + channel + .close(200, "Closing channel due to shutdown") + .await?; + + Ok(()) + } + + async fn handle_delivery( + delivery: lapin::message::Delivery, + inscription_indexation: &Arc, + ) -> Result<(), anyhow::Error> { + let event: Result = serde_json::from_slice(&delivery.data); + match event { + Ok(event) => { + if let Err(err) = BlockConsumer::process_event(event, inscription_indexation).await { + log::error!("Failed to process event: {}", err); + delivery + .reject(BasicRejectOptions { requeue: false }) + .await?; + } else { + delivery.ack(BasicAckOptions::default()).await?; + } + } + Err(e) => { + log::error!("Failed to deserialize event, rejecting: {}", e); + delivery + .reject(BasicRejectOptions { requeue: false }) + .await?; + } + } + + Ok(()) + } + + async fn process_event( + event: Event, + inscription_indexation: &Arc, + ) -> Result<(), anyhow::Error> { + match &event { + Event::BlockCommitted { + from_height, + to_height, + } => { + inscription_indexation + .sync_blocks(from_height, to_height) + .await? + } + _ => { + log::warn!("Received an unhandled event type {:?}", event); + } + } + Ok(()) + } +} diff --git a/src/ord_db_client.rs b/src/indexer/db_client.rs similarity index 92% rename from src/ord_db_client.rs rename to src/indexer/db_client.rs index d5147a5811..499966a07f 100644 --- a/src/ord_db_client.rs +++ b/src/indexer/db_client.rs @@ -38,11 +38,11 @@ impl Event { } } -pub struct OrdDbClient { +pub struct DbClient { pool: Arc, } -impl OrdDbClient { +impl DbClient { pub fn new(pool: Arc) -> Self { Self { pool } } @@ -85,12 +85,12 @@ impl OrdDbClient { WHERE type_id = $1 AND block_height = $2::BIGINT AND inscription_id = $3 AND location = $4 )"; sqlx::query(query) - .bind(1_i32) // Type ID for InscriptionCreated - .bind(block_height.to_string()) - .bind(inscription_id.to_string()) - .bind(location.map(|loc| loc.to_string())) - .execute(&*self.pool) - .await?; + .bind(1_i32) // Type ID for InscriptionCreated + .bind(block_height.to_string()) + .bind(inscription_id.to_string()) + .bind(location.map(|loc| loc.to_string())) + .execute(&*self.pool) + .await?; Ok(()) } @@ -109,13 +109,13 @@ impl OrdDbClient { WHERE type_id = $1 AND block_height = $2::BIGINT AND inscription_id = $3 AND location = $4 AND old_location = $5 )"; sqlx::query(query) - .bind(2_i32) // Type ID for InscriptionTransferred - .bind(block_height.to_string()) - .bind(inscription_id.to_string()) - .bind(new_location.to_string()) - .bind(old_location.to_string()) - .execute(&*self.pool) - .await?; + .bind(2_i32) // Type ID for InscriptionTransferred + .bind(block_height.to_string()) + .bind(inscription_id.to_string()) + .bind(new_location.to_string()) + .bind(old_location.to_string()) + .execute(&*self.pool) + .await?; Ok(()) } diff --git a/src/indexer/db_con.rs b/src/indexer/db_con.rs new file mode 100644 index 0000000000..20c2622ed1 --- /dev/null +++ b/src/indexer/db_con.rs @@ -0,0 +1,31 @@ +use sqlx::postgres::PgPoolOptions; +use sqlx::{Pool, Postgres}; +use urlencoding::encode; + +pub async fn setup_db_connection(database_url: &str) -> Result, anyhow::Error> { + let encoded_database_url = encode_password_in_url(database_url); + let pool = PgPoolOptions::new() + .max_connections(5) + .connect(encoded_database_url.as_ref()) + .await?; + Ok(pool) +} + +fn encode_password_in_url(url: &str) -> String { + let re = regex::Regex::new(r"(\w+://)([^:]+):([^@]+)@(.*)").unwrap(); + if let Some(caps) = re.captures(url) { + let protocol_and_user = caps.get(1).map_or("", |m| m.as_str()); + let username = caps.get(2).map_or("", |m| m.as_str()); + let password = caps.get(3).map_or("", |m| m.as_str()); + let rest_of_url = caps.get(4).map_or("", |m| m.as_str()); + format!( + "{}{}:{}@{}", + protocol_and_user, + username, + encode(password), + rest_of_url + ) + } else { + url.to_string() + } +} diff --git a/src/event_consumer.rs b/src/indexer/event_consumer.rs similarity index 50% rename from src/event_consumer.rs rename to src/indexer/event_consumer.rs index ddc7effaef..9dc1621631 100644 --- a/src/event_consumer.rs +++ b/src/indexer/event_consumer.rs @@ -2,35 +2,25 @@ use std::process; use std::sync::Arc; use anyhow::Context; -use bitcoin::secp256k1::rand::distributions::Alphanumeric; -use chrono::Utc; use clap::Parser; use futures::StreamExt; use lapin::{options::*, types::FieldTable}; -use rand::distributions::DistString; -use sqlx::postgres::PgPoolOptions; use tokio::runtime::Runtime; use tokio::sync::oneshot; -use urlencoding::encode; -use crate::connect_rmq::connect_to_rabbitmq; use crate::index::event::Event; -use crate::ord_api_client::OrdApiClient; -use crate::ord_db_client::OrdDbClient; -use crate::ord_indexation::OrdIndexation; +use crate::indexer::db_client::DbClient; +use crate::indexer::db_con::setup_db_connection; +use crate::indexer::rmq_con::{generate_consumer_tag, setup_rabbitmq_connection}; use crate::settings::Settings; use crate::subcommand::SubcommandResult; #[derive(Debug, Parser, Clone)] pub struct EventConsumer { - #[arg(long, help = "RMQ queue to consume blocks.")] - pub(crate) blocks_queue: Option, #[arg(long, help = "RMQ queue to consume inscription events.")] pub(crate) inscriptions_queue: Option, #[arg(long, help = "DB url to persist inscriptions.")] pub(crate) database_url: Option, - #[arg(long, help = "Ord api url to fetch inscriptions.")] - pub(crate) ord_api_url: Option, } impl EventConsumer { @@ -39,55 +29,16 @@ impl EventConsumer { let addr = settings .rabbitmq_addr() .context("rabbitmq amqp credentials and url must be defined")?; - - let conn = connect_to_rabbitmq(addr).await?; - - let channel = conn - .create_channel() - .await - .expect("creates rmq connection channel"); - - channel - .confirm_select(ConfirmSelectOptions::default()) - .await - .expect("enable msg confirms"); + let channel = setup_rabbitmq_connection(addr).await?; let database_url = self .database_url .as_deref() .context("db url must be defined")?; - log::info!( - "Connecting to database at {}", - EventConsumer::mask_password_in_url(database_url) - ); - let encoded_database_url = EventConsumer::encode_password_in_url(database_url); + let pool = setup_db_connection(database_url).await?; - let pool = PgPoolOptions::new() - .max_connections(5) - .connect(encoded_database_url.as_ref()) - .await?; let shared_pool = Arc::new(pool); - let ord_db_client = Arc::new(OrdDbClient::new(shared_pool.clone())); - - let api_url = self.ord_api_url.context("api url must be defined")?; - let ord_api_c = OrdApiClient::new(api_url.clone()).expect("api client must exist"); - let ord_api_client = Arc::new(ord_api_c); - - let ord_indexation = Arc::new(OrdIndexation::new( - settings, - Arc::clone(&ord_db_client), - Arc::clone(&ord_api_client), - )); - - let blocks_queue = self - .blocks_queue - .as_deref() - .context("rabbitmq blocks queue path must be defined")?; - let blocks_queue_str = blocks_queue.to_string(); - let blocks_channel = channel.clone(); - let blocks_ord_indexation = Arc::clone(&ord_indexation); - let blocks_consumer_tag = Self::generate_consumer_tag(); - let (blocks_shutdown_tx, blocks_shutdown_rx) = oneshot::channel::<()>(); + let db_client = Arc::new(DbClient::new(shared_pool.clone())); let inscriptions_queue = self .inscriptions_queue @@ -95,31 +46,15 @@ impl EventConsumer { .context("rabbitmq inscriptions queue path must be defined")?; let inscriptions_queue_str = inscriptions_queue.to_string(); let inscriptions_channel = channel.clone(); - let inscriptions_ord_indexation = Arc::clone(&ord_indexation); - let inscriptions_consumer_tag = Self::generate_consumer_tag(); + let inscriptions_consumer_tag = generate_consumer_tag("lr-ord"); let (inscriptions_shutdown_tx, inscriptions_shutdown_rx) = oneshot::channel::<()>(); - let blocks_consumer_handle = tokio::spawn(async move { - EventConsumer::consume_queue( - blocks_channel, - blocks_queue_str, - blocks_consumer_tag, - blocks_ord_indexation, - blocks_shutdown_rx, - ) - .await - .map_err(|e| { - log::error!("Error consuming blocks queue: {}", e); - process::exit(1); - }) - }); - let inscriptions_consumer_handle = tokio::spawn(async move { EventConsumer::consume_queue( inscriptions_channel, inscriptions_queue_str, inscriptions_consumer_tag, - inscriptions_ord_indexation, + db_client, inscriptions_shutdown_rx, ) .await @@ -131,9 +66,8 @@ impl EventConsumer { let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; sigterm.recv().await; - let _ = blocks_shutdown_tx.send(()); let _ = inscriptions_shutdown_tx.send(()); - let _ = tokio::try_join!(blocks_consumer_handle, inscriptions_consumer_handle); + let _ = tokio::try_join!(inscriptions_consumer_handle); shared_pool.close().await; @@ -141,35 +75,11 @@ impl EventConsumer { }) } - fn encode_password_in_url(url: &str) -> String { - let re = regex::Regex::new(r"(\w+://)([^:]+):([^@]+)@(.*)").unwrap(); - if let Some(caps) = re.captures(url) { - let protocol_and_user = caps.get(1).map_or("", |m| m.as_str()); - let username = caps.get(2).map_or("", |m| m.as_str()); - let password = caps.get(3).map_or("", |m| m.as_str()); - let rest_of_url = caps.get(4).map_or("", |m| m.as_str()); - format!( - "{}{}:{}@{}", - protocol_and_user, - username, - encode(password), - rest_of_url - ) - } else { - url.to_string() - } - } - - fn mask_password_in_url(url: &str) -> String { - let re = regex::Regex::new(r"(\w+://)([^:]+):([^@]+)@").unwrap(); - re.replace(url, "$1$2:***@").to_string() - } - async fn consume_queue( channel: lapin::Channel, queue_name: String, consumer_tag: String, - ord_indexation: Arc, + db_client: Arc, mut shutdown_signal: oneshot::Receiver<()>, ) -> Result<(), anyhow::Error> { let mut consumer = channel @@ -185,7 +95,7 @@ impl EventConsumer { while let Some(result) = consumer.next().await { let delivery = result?; tokio::select! { - process_result = EventConsumer::handle_delivery(delivery, &ord_indexation) => { + process_result = EventConsumer::handle_delivery(delivery, &db_client) => { process_result?; }, _ = &mut shutdown_signal => { @@ -205,12 +115,12 @@ impl EventConsumer { async fn handle_delivery( delivery: lapin::message::Delivery, - ord_indexation: &Arc, + db_client: &Arc, ) -> Result<(), anyhow::Error> { let event: Result = serde_json::from_slice(&delivery.data); match event { Ok(event) => { - if let Err(err) = EventConsumer::process_event(event, ord_indexation).await { + if let Err(err) = EventConsumer::process_event(event, db_client).await { log::error!("Failed to process event: {}", err); delivery .reject(BasicRejectOptions { requeue: false }) @@ -230,15 +140,8 @@ impl EventConsumer { Ok(()) } - async fn process_event( - event: Event, - ord_indexation: &Arc, - ) -> Result<(), anyhow::Error> { + async fn process_event(event: Event, db_client: &Arc) -> Result<(), anyhow::Error> { match &event { - Event::BlockCommitted { - from_height, - to_height, - } => ord_indexation.sync_blocks(from_height, to_height).await?, Event::InscriptionCreated { block_height, charms: _charms, @@ -247,7 +150,7 @@ impl EventConsumer { parent_inscription_ids: _parent_inscription_ids, sequence_number: _sequence_number, } => { - ord_indexation + db_client .save_inscription_created(block_height, inscription_id, location) .await? } @@ -258,25 +161,14 @@ impl EventConsumer { old_location, sequence_number: _sequence_number, } => { - ord_indexation + db_client .save_inscription_transferred(block_height, inscription_id, new_location, old_location) .await?; } _ => { - log::warn!("Received an unhandled event type"); + log::warn!("Received an unhandled event type {:?}", event); } } Ok(()) } - - fn generate_consumer_tag() -> String { - // TODO get pod name from k8s? - let timestamp = Utc::now().format("%Y%m%d%H%M%S"); - format!( - "{}-{}-{}", - "lr-ord", - timestamp, - Alphanumeric.sample_string(&mut rand::thread_rng(), 16) - ) - } } diff --git a/src/event_publisher.rs b/src/indexer/event_publisher.rs similarity index 89% rename from src/event_publisher.rs rename to src/indexer/event_publisher.rs index 88fa9eba70..6a28946736 100644 --- a/src/event_publisher.rs +++ b/src/indexer/event_publisher.rs @@ -1,11 +1,10 @@ use anyhow::{Context, Result}; -use lapin::options::ConfirmSelectOptions; use lapin::{options::BasicPublishOptions, BasicProperties}; use tokio::runtime::Runtime; use tokio::sync::mpsc; -use crate::connect_rmq::connect_to_rabbitmq; use crate::index::event::Event; +use crate::indexer::rmq_con::setup_rabbitmq_connection; use crate::settings::Settings; use crate::shutdown_process; @@ -47,13 +46,7 @@ impl EventPublisher { exchange: String, mut rx: mpsc::Receiver, ) -> Result<()> { - let conn = connect_to_rabbitmq(&addr).await?; - - let channel = conn.create_channel().await?; - - channel - .confirm_select(ConfirmSelectOptions::default()) - .await?; + let channel = setup_rabbitmq_connection(&addr).await?; while let Some(event) = rx.recv().await { let message = serde_json::to_vec(&event)?; diff --git a/src/ord_indexation.rs b/src/indexer/inscription_indexation.rs similarity index 84% rename from src/ord_indexation.rs rename to src/indexer/inscription_indexation.rs index ca5fa317fa..5a4105e8c1 100644 --- a/src/ord_indexation.rs +++ b/src/indexer/inscription_indexation.rs @@ -7,22 +7,21 @@ use serde_json::Value; use ordinals::SatPoint; use crate::api::BlockInfo; -use crate::ord_api_client::OrdApiClient; -use crate::ord_db_client::{Event, OrdDbClient}; +use crate::indexer::api_client::ApiClient; +use crate::indexer::db_client::{DbClient, Event}; use crate::settings::Settings; -use crate::InscriptionId; -pub struct OrdIndexation { +pub struct InscriptionIndexation { settings: Settings, - ord_db_client: Arc, - ord_api_client: Arc, + ord_db_client: Arc, + ord_api_client: Arc, } -impl OrdIndexation { +impl InscriptionIndexation { pub fn new( settings: &Settings, - ord_db_client: Arc, - ord_api_client: Arc, + ord_db_client: Arc, + ord_api_client: Arc, ) -> Self { Self { settings: settings.clone(), @@ -210,31 +209,4 @@ impl OrdIndexation { None => Ok(None), } } - - pub async fn save_inscription_created( - &self, - block_height: &u32, - inscription_id: &InscriptionId, - location: &Option, - ) -> Result<(), anyhow::Error> { - self - .ord_db_client - .save_inscription_created(block_height, inscription_id, location) - .await?; - Ok(()) - } - - pub async fn save_inscription_transferred( - &self, - block_height: &u32, - inscription_id: &InscriptionId, - new_location: &SatPoint, - old_location: &SatPoint, - ) -> Result<(), anyhow::Error> { - self - .ord_db_client - .save_inscription_transferred(block_height, inscription_id, new_location, old_location) - .await?; - Ok(()) - } } diff --git a/src/connect_rmq.rs b/src/indexer/rmq_con.rs similarity index 59% rename from src/connect_rmq.rs rename to src/indexer/rmq_con.rs index 57e570363f..96cadd81ee 100644 --- a/src/connect_rmq.rs +++ b/src/indexer/rmq_con.rs @@ -1,11 +1,27 @@ use std::collections::HashSet; use anyhow::Context; +use chrono::Utc; +use lapin::options::ConfirmSelectOptions; use lapin::tcp::{AMQPUriTcpExt, NativeTlsConnector}; use lapin::uri::AMQPUri; use lapin::{Connection, ConnectionProperties}; +use rand::distributions::{Alphanumeric, DistString}; -pub async fn connect_to_rabbitmq(addr: &str) -> Result { +pub async fn setup_rabbitmq_connection(addr: &str) -> Result { + let conn = connect_to_rabbitmq(addr).await?; + let channel = conn + .create_channel() + .await + .context("creates rmq connection channel")?; + channel + .confirm_select(ConfirmSelectOptions::default()) + .await + .context("enable msg confirms")?; + Ok(channel) +} + +async fn connect_to_rabbitmq(addr: &str) -> Result { let uri = addr .parse::() .map_err(anyhow::Error::msg) @@ -35,3 +51,14 @@ pub async fn connect_to_rabbitmq(addr: &str) -> Result String { + let timestamp = Utc::now().format("%Y%m%d%H%M%S"); + format!( + "{}-{}-{}", + prefix, + timestamp, + Alphanumeric.sample_string(&mut rand::thread_rng(), 16) + ) +} diff --git a/src/lib.rs b/src/lib.rs index 241bffa6bc..42e543107e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -104,8 +104,6 @@ mod blocktime; pub mod chain; pub mod decimal; mod deserialize_from_str; -mod event_consumer; -mod event_publisher; mod fee_rate; pub mod index; mod inscriptions; @@ -113,11 +111,6 @@ mod into_usize; mod macros; mod object; pub mod options; -pub mod ord_api_client; - -pub mod connect_rmq; -pub mod ord_db_client; -mod ord_indexation; pub mod outgoing; mod re; mod representation; @@ -127,6 +120,16 @@ pub mod subcommand; mod tally; pub mod templates; pub mod wallet; +pub mod indexer { + pub mod api_client; + pub mod block_consumer; + pub mod db_client; + pub mod db_con; + pub mod event_consumer; + pub mod event_publisher; + pub mod inscription_indexation; + pub mod rmq_con; +} type Result = std::result::Result; diff --git a/src/subcommand.rs b/src/subcommand.rs index b792f16350..e6ef5b87c0 100644 --- a/src/subcommand.rs +++ b/src/subcommand.rs @@ -1,6 +1,6 @@ -use crate::event_publisher::EventPublisher; - use super::*; +use crate::indexer::event_publisher::EventPublisher; +use crate::indexer::{block_consumer, event_consumer}; pub mod balances; pub mod decode; @@ -43,8 +43,10 @@ pub(crate) enum Subcommand { Server(server::Server), #[command(about = "Run the explorer server in event emit mode")] EventServer(server::Server), - #[command(about = "Run the index event consumer")] + #[command(about = "Run inscription event consumer")] EventConsumer(event_consumer::EventConsumer), + #[command(about = "Run block event consumer")] + BlockConsumer(block_consumer::BlockConsumer), #[command(about = "Display settings")] Settings, #[command(about = "Display information about a block's subsidy")] @@ -89,6 +91,7 @@ impl Subcommand { server.run(settings, index, handle) } Self::EventConsumer(event_consumer) => event_consumer.run(&settings), + Self::BlockConsumer(block_consumer) => block_consumer.run(&settings), Self::Settings => settings::run(settings), Self::Subsidy(subsidy) => subsidy.run(), Self::Supply => supply::run(), From 2d663d571b6c40b2a2b62db19b5f5b288dc06bce Mon Sep 17 00:00:00 2001 From: rob Date: Tue, 7 May 2024 15:00:34 +0200 Subject: [PATCH 079/103] Process one block at a time --- src/index/event.rs | 3 +- src/index/updater.rs | 3 +- src/indexer/api_client.rs | 2 +- src/indexer/block_consumer.rs | 9 +--- src/indexer/db_client.rs | 2 +- src/indexer/inscription_indexation.rs | 64 +++++++++++++-------------- 6 files changed, 36 insertions(+), 47 deletions(-) diff --git a/src/index/event.rs b/src/index/event.rs index 6fd9f83be8..6e9d91cbce 100644 --- a/src/index/event.rs +++ b/src/index/event.rs @@ -42,7 +42,6 @@ pub enum Event { txid: Txid, }, BlockCommitted { - from_height: u32, - to_height: u32, + height: u32, }, } diff --git a/src/index/updater.rs b/src/index/updater.rs index bbc2c0f25a..0600804353 100644 --- a/src/index/updater.rs +++ b/src/index/updater.rs @@ -748,8 +748,7 @@ impl<'index> Updater<'index> { if let Ok(uncommitted) = u32::try_from(uncommitted) { for current_height in (self.height - uncommitted)..=self.height { sender.blocking_send(Event::BlockCommitted { - from_height: current_height - 1, - to_height: current_height, + height: current_height, })?; } } else { diff --git a/src/indexer/api_client.rs b/src/indexer/api_client.rs index c5b178450a..9c34dcb065 100644 --- a/src/indexer/api_client.rs +++ b/src/indexer/api_client.rs @@ -109,7 +109,7 @@ impl ApiClient { self.execute_with_retries(request_builder, 3).await } - pub async fn fetch_block_info(&self, block_height: u32) -> Result { + pub async fn fetch_block_info(&self, block_height: &u32) -> Result { let request_builder = self .client .get(format!("{}/r/blockinfo/{}", self.ord_api_url, block_height)) diff --git a/src/indexer/block_consumer.rs b/src/indexer/block_consumer.rs index 0dfdd92ef5..e6a4a06c51 100644 --- a/src/indexer/block_consumer.rs +++ b/src/indexer/block_consumer.rs @@ -156,14 +156,7 @@ impl BlockConsumer { inscription_indexation: &Arc, ) -> Result<(), anyhow::Error> { match &event { - Event::BlockCommitted { - from_height, - to_height, - } => { - inscription_indexation - .sync_blocks(from_height, to_height) - .await? - } + Event::BlockCommitted { height } => inscription_indexation.sync_blocks(height).await?, _ => { log::warn!("Received an unhandled event type {:?}", event); } diff --git a/src/indexer/db_client.rs b/src/indexer/db_client.rs index 499966a07f..892195ed38 100644 --- a/src/indexer/db_client.rs +++ b/src/indexer/db_client.rs @@ -49,7 +49,7 @@ impl DbClient { pub async fn fetch_events_by_block_height( &self, - block_height: u32, + block_height: &u32, ) -> Result, sqlx::Error> { let query = " SELECT type_id, block_height, inscription_id, location, old_location diff --git a/src/indexer/inscription_indexation.rs b/src/indexer/inscription_indexation.rs index 5a4105e8c1..81eb9809b7 100644 --- a/src/indexer/inscription_indexation.rs +++ b/src/indexer/inscription_indexation.rs @@ -30,42 +30,40 @@ impl InscriptionIndexation { } } - pub async fn sync_blocks(&self, from_height: &u32, to_height: &u32) -> Result<(), anyhow::Error> { - log::info!("Blocks committed event from={from_height} (excluded), to={to_height} (included)"); - - for block_height in *from_height + 1..=*to_height { - let events = self - .ord_db_client - .fetch_events_by_block_height(block_height) - .await?; - let block_info = self.ord_api_client.fetch_block_info(block_height).await?; - for event in events { - match event.type_id { - 1 => { - if let Err(e) = self.process_inscription_created(&event, &block_info).await { - log::error!( - "Error processing inscription creation for event {:?}: {}", - event, - e - ); - } - } - 2 => { - if let Err(e) = self - .process_inscription_transferred(&event, &block_info) - .await - { - log::error!( - "Error processing inscription transferred for event {:?}: {}", - event, - e - ); - } + pub async fn sync_blocks(&self, block_height: &u32) -> Result<(), anyhow::Error> { + log::info!("Blocks committed event for={block_height}"); + + let events = self + .ord_db_client + .fetch_events_by_block_height(block_height) + .await?; + let block_info = self.ord_api_client.fetch_block_info(block_height).await?; + for event in events { + match event.type_id { + 1 => { + if let Err(e) = self.process_inscription_created(&event, &block_info).await { + log::error!( + "Error processing inscription creation for event {:?}: {}", + event, + e + ); } - _ => { - log::warn!("Unhandled event type: {}", event.type_id); + } + 2 => { + if let Err(e) = self + .process_inscription_transferred(&event, &block_info) + .await + { + log::error!( + "Error processing inscription transferred for event {:?}: {}", + event, + e + ); } } + _ => { + log::warn!("Unhandled event type: {}", event.type_id); + } } } From a59f81f3558fc9fbbfc246ab79e77021eb49eb47 Mon Sep 17 00:00:00 2001 From: rob Date: Tue, 7 May 2024 15:16:49 +0200 Subject: [PATCH 080/103] Process one block at a time --- src/index/updater.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index/updater.rs b/src/index/updater.rs index 0600804353..47144c518c 100644 --- a/src/index/updater.rs +++ b/src/index/updater.rs @@ -746,7 +746,7 @@ impl<'index> Updater<'index> { if let Some(sender) = self.index.event_sender.as_ref() { if let Ok(uncommitted) = u32::try_from(uncommitted) { - for current_height in (self.height - uncommitted)..=self.height { + for current_height in (self.height - uncommitted)..self.height { sender.blocking_send(Event::BlockCommitted { height: current_height, })?; From 799c6dc0d567ba776866c580df7c019808eb7dee Mon Sep 17 00:00:00 2001 From: rob Date: Tue, 7 May 2024 15:30:03 +0200 Subject: [PATCH 081/103] Refactor --- src/indexer/block_consumer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/indexer/block_consumer.rs b/src/indexer/block_consumer.rs index e6a4a06c51..288cd92d8d 100644 --- a/src/indexer/block_consumer.rs +++ b/src/indexer/block_consumer.rs @@ -45,7 +45,7 @@ impl BlockConsumer { let db_client = Arc::new(DbClient::new(shared_pool.clone())); let api_url = self.ord_api_url.context("api url must be defined")?; - let api_c = ApiClient::new(api_url.clone()).expect("api client must exist"); + let api_c = ApiClient::new(api_url.clone()).context("Failed to create API client")?; let api_client = Arc::new(api_c); let inscription_indexation = From 6676e3c84c504e8935e93403f5c032950716dab2 Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Wed, 8 May 2024 09:07:11 +0700 Subject: [PATCH 082/103] Explicit conversions --- db/migrations/20240431000000_ordinals.sql | 83 +++++++++++------------ src/event_consumer.rs | 21 ++---- src/lib.rs | 2 +- src/ord_db_client.rs | 38 ++++++----- 4 files changed, 68 insertions(+), 76 deletions(-) diff --git a/db/migrations/20240431000000_ordinals.sql b/db/migrations/20240431000000_ordinals.sql index 0392318e07..57779b10ff 100644 --- a/db/migrations/20240431000000_ordinals.sql +++ b/db/migrations/20240431000000_ordinals.sql @@ -1,55 +1,48 @@ -CREATE TABLE event -( - id SERIAL PRIMARY KEY, - type_id SMALLINT NOT NULL, --1,InscriptionCreated;2,InscriptionTransferred - block_height BIGINT NOT NULL, - inscription_id TEXT NOT NULL, - location TEXT, -- Will hold either 'location' or 'new_location' based on type - old_location TEXT, -- Only used for InscriptionTransferred - created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +CREATE TABLE event ( + id SERIAL PRIMARY KEY + , type_id SMALLINT NOT NULL --1=InscriptionCreated; 2=InscriptionTransferred + , block_height INTEGER NOT NULL + , inscription_id TEXT NOT NULL + , location TEXT -- Will hold either 'location' or 'new_location' based on type + , old_location TEXT -- Only used for InscriptionTransferred + , created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP ); CREATE INDEX idx_events_block_height ON event (block_height); CREATE INDEX idx_events_inscription_id ON event (inscription_id); -CREATE TABLE inscription -( - id SERIAL PRIMARY KEY, - genesis_id TEXT NOT NULL, - number INTEGER NOT NULL, - content_type VARCHAR, - content_length INTEGER, - metadata TEXT, - - genesis_block_height INTEGER NOT NULL, - genesis_block_time BIGINT NOT NULL, - - sat_number BIGINT, - sat_rarity INTEGER, - sat_block_height INTEGER, - sat_block_time BIGINT, - - fee BIGINT NOT NULL, - charms SMALLINT NOT NULL, - - children TEXT, - parents TEXT +CREATE TABLE inscription ( + id SERIAL PRIMARY KEY + , genesis_id TEXT NOT NULL + , number INTEGER NOT NULL + , content_type VARCHAR + , content_length INTEGER + , metadata TEXT + , genesis_block_height INTEGER NOT NULL + , genesis_block_time BIGINT NOT NULL + , sat_number BIGINT + , sat_rarity INTEGER + , sat_block_height INTEGER + , sat_block_time BIGINT + , fee BIGINT NOT NULL + , charms SMALLINT NOT NULL + , children TEXT + , parents TEXT ); CREATE UNIQUE INDEX idx_unique_inscription_genesis_id ON inscription (genesis_id); -CREATE TABLE location -( - id SERIAL PRIMARY KEY, - inscription_id INTEGER REFERENCES inscription (id), - block_height INTEGER NOT NULL, - block_time BIGINT NOT NULL, - tx_id TEXT NOT NULL, - to_address TEXT NOT NULL, - cur_output TEXT NOT NULL, - cur_offset BIGINT NOT NULL, - from_address TEXT, - prev_output TEXT, - prev_offset BIGINT, - value BIGINT +CREATE TABLE location ( + id SERIAL PRIMARY KEY + , inscription_id INTEGER REFERENCES inscription (id) + , block_height INTEGER NOT NULL + , block_time BIGINT NOT NULL + , tx_id TEXT NOT NULL + , to_address TEXT NOT NULL + , cur_output TEXT NOT NULL + , cur_offset BIGINT NOT NULL + , from_address TEXT + , prev_output TEXT + , prev_offset BIGINT + , value BIGINT ); diff --git a/src/event_consumer.rs b/src/event_consumer.rs index ddc7effaef..70ceaa744f 100644 --- a/src/event_consumer.rs +++ b/src/event_consumer.rs @@ -1,6 +1,3 @@ -use std::process; -use std::sync::Arc; - use anyhow::Context; use bitcoin::secp256k1::rand::distributions::Alphanumeric; use chrono::Utc; @@ -9,6 +6,7 @@ use futures::StreamExt; use lapin::{options::*, types::FieldTable}; use rand::distributions::DistString; use sqlx::postgres::PgPoolOptions; +use std::sync::Arc; use tokio::runtime::Runtime; use tokio::sync::oneshot; use urlencoding::encode; @@ -56,12 +54,13 @@ impl EventConsumer { .database_url .as_deref() .context("db url must be defined")?; + log::info!( - "Connecting to database at {}", + "connecting to database at {}", EventConsumer::mask_password_in_url(database_url) ); - let encoded_database_url = EventConsumer::encode_password_in_url(database_url); + let encoded_database_url = EventConsumer::encode_password_in_url(database_url); let pool = PgPoolOptions::new() .max_connections(5) .connect(encoded_database_url.as_ref()) @@ -70,7 +69,7 @@ impl EventConsumer { let ord_db_client = Arc::new(OrdDbClient::new(shared_pool.clone())); let api_url = self.ord_api_url.context("api url must be defined")?; - let ord_api_c = OrdApiClient::new(api_url.clone()).expect("api client must exist"); + let ord_api_c = OrdApiClient::new(api_url).expect("api client must exist"); let ord_api_client = Arc::new(ord_api_c); let ord_indexation = Arc::new(OrdIndexation::new( @@ -108,10 +107,7 @@ impl EventConsumer { blocks_shutdown_rx, ) .await - .map_err(|e| { - log::error!("Error consuming blocks queue: {}", e); - process::exit(1); - }) + .expect("error consuming blocks queue") }); let inscriptions_consumer_handle = tokio::spawn(async move { @@ -123,10 +119,7 @@ impl EventConsumer { inscriptions_shutdown_rx, ) .await - .map_err(|e| { - log::error!("Error consuming inscriptions queue: {}", e); - process::exit(1); - }) + .expect("error consuming inscriptions queue") }); let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; diff --git a/src/lib.rs b/src/lib.rs index 241bffa6bc..681df4655c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,10 +2,10 @@ clippy::large_enum_variant, clippy::result_large_err, clippy::too_many_arguments, + clippy::cast_lossless, clippy::type_complexity )] #![deny( - clippy::cast_lossless, clippy::cast_possible_truncation, clippy::cast_possible_wrap, clippy::cast_sign_loss diff --git a/src/ord_db_client.rs b/src/ord_db_client.rs index 6225750ab4..e593c9ec5d 100644 --- a/src/ord_db_client.rs +++ b/src/ord_db_client.rs @@ -11,7 +11,7 @@ use crate::InscriptionId; #[derive(Debug, Clone, sqlx::FromRow)] pub struct Event { pub type_id: i16, - pub block_height: i64, + pub block_height: i32, pub inscription_id: String, pub location: Option, pub old_location: Option, @@ -36,7 +36,7 @@ impl OrdDbClient { FROM event WHERE block_height = $1 ORDER BY type_id ASC, id ASC "#, - block_height as i64, + i32::try_from(block_height).expect("block_height should fit in pg integer"), ) .map(|r| Event { type_id: r.type_id, @@ -160,16 +160,23 @@ impl OrdDbClient { inscription_details.id.to_string(), inscription_details.number, inscription_details.content_type.as_deref(), - inscription_details.content_length.map(|n| n as i32), + inscription_details + .content_length + .map(|n| i32::try_from(n).expect("content_length should fit in pg integer")), metadata, - inscription_details.genesis_block_height as i32, + i32::try_from(inscription_details.genesis_block_height) + .expect("genesis_block_height should fit in pg integer"), inscription_details.genesis_block_time, - inscription_details.sat_number.map(|n| n as i64), + inscription_details + .sat_number + .map(|n| i64::try_from(n).expect("sat_number should fit in pg bigint")), inscription_details.sat_rarity.map(|r| r as i32), - inscription_details.sat_block_height.map(|n| n as i32), + inscription_details + .sat_block_height + .map(|n| i32::try_from(n).expect("sat_block_height should fit in pg integer")), inscription_details.sat_block_time, - inscription_details.fee as i64, - inscription_details.charms as i16, + i64::try_from(inscription_details.fee).expect("fee should fit in pg bigint"), + i16::try_from(inscription_details.charms).expect("charts should fit in pg smallint"), Json(&inscription_details.children).encode_to_string(), Json(&inscription_details.parents).encode_to_string() ) @@ -181,7 +188,7 @@ impl OrdDbClient { pub async fn save_location( &self, id: i32, - block_height: i64, + block_height: i32, block_time: u64, tx_id: Option, to_address: Option, @@ -211,8 +218,7 @@ impl OrdDbClient { $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11 WHERE NOT EXISTS ( SELECT 1 FROM location - WHERE - inscription_id = $1 + WHERE inscription_id = $1 AND block_height = $2 AND block_time = $3 AND tx_id = $4 @@ -226,16 +232,16 @@ impl OrdDbClient { ) "#, id, - block_height as i64, - block_time as i64, + block_height, + i64::try_from(block_time).expect("block_time should fit in pg bigint"), tx_id.map(|n| n.to_string()), to_address, to_outpoint.map(|n| n.to_string()), - to_offset.map(|n| n as i64), + to_offset.map(|n| i64::try_from(n).expect("to_offset should fit in pg bigint")), from_address, from_outpoint.map(|n| n.to_string()), - from_offset.map(|n| n as i64), - value.map(|n| n as i64), + from_offset.map(|n| i64::try_from(n).expect("from_offset should fit in pg bigint")), + value.map(|n| i64::try_from(n).expect("value should fit in pg bigint")), ) .execute(&*self.pool) .await?; From 13b1b644d9957550fb95d022d0493732a768192c Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Wed, 8 May 2024 09:09:33 +0700 Subject: [PATCH 083/103] Test fix --- ...f0c37827309e7e2439bc7809996d1e479898c906cfe39f3350acf.json | 2 +- ...ea1d4357e4c906c37caacffb6de8323126e4529e5bcf875e6cc41.json | 4 ++-- ...b072aa3e8bf3e37bb7f90cd11f74847a8cadb41326032b580b23.json} | 4 ++-- ...f66cff514bb7bcf5314d0d235da28ead0b89fdaa090cf2f16a151.json | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) rename .sqlx/{query-1289edfa88c5539ccee82bb3d389306d21d5940ae37871166afce9f664bf53d4.json => query-46ed37001552b072aa3e8bf3e37bb7f90cd11f74847a8cadb41326032b580b23.json} (58%) diff --git a/.sqlx/query-03b5007410af0c37827309e7e2439bc7809996d1e479898c906cfe39f3350acf.json b/.sqlx/query-03b5007410af0c37827309e7e2439bc7809996d1e479898c906cfe39f3350acf.json index 763de586cd..01db865bce 100644 --- a/.sqlx/query-03b5007410af0c37827309e7e2439bc7809996d1e479898c906cfe39f3350acf.json +++ b/.sqlx/query-03b5007410af0c37827309e7e2439bc7809996d1e479898c906cfe39f3350acf.json @@ -6,7 +6,7 @@ "parameters": { "Left": [ "Int2", - "Int8", + "Int4", "Text", "Text" ] diff --git a/.sqlx/query-37a57ddabacea1d4357e4c906c37caacffb6de8323126e4529e5bcf875e6cc41.json b/.sqlx/query-37a57ddabacea1d4357e4c906c37caacffb6de8323126e4529e5bcf875e6cc41.json index 2f3a58392b..c6bce2eed5 100644 --- a/.sqlx/query-37a57ddabacea1d4357e4c906c37caacffb6de8323126e4529e5bcf875e6cc41.json +++ b/.sqlx/query-37a57ddabacea1d4357e4c906c37caacffb6de8323126e4529e5bcf875e6cc41.json @@ -11,7 +11,7 @@ { "ordinal": 1, "name": "block_height", - "type_info": "Int8" + "type_info": "Int4" }, { "ordinal": 2, @@ -31,7 +31,7 @@ ], "parameters": { "Left": [ - "Int8" + "Int4" ] }, "nullable": [ diff --git a/.sqlx/query-1289edfa88c5539ccee82bb3d389306d21d5940ae37871166afce9f664bf53d4.json b/.sqlx/query-46ed37001552b072aa3e8bf3e37bb7f90cd11f74847a8cadb41326032b580b23.json similarity index 58% rename from .sqlx/query-1289edfa88c5539ccee82bb3d389306d21d5940ae37871166afce9f664bf53d4.json rename to .sqlx/query-46ed37001552b072aa3e8bf3e37bb7f90cd11f74847a8cadb41326032b580b23.json index 21c1ae17b6..4a27327a6a 100644 --- a/.sqlx/query-1289edfa88c5539ccee82bb3d389306d21d5940ae37871166afce9f664bf53d4.json +++ b/.sqlx/query-46ed37001552b072aa3e8bf3e37bb7f90cd11f74847a8cadb41326032b580b23.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO location (\n inscription_id\n , block_height\n , block_time\n , tx_id\n , to_address\n , cur_output\n , cur_offset\n , from_address\n , prev_output\n , prev_offset\n , value\n )\n SELECT\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11\n WHERE NOT EXISTS (\n SELECT 1 FROM location\n WHERE\n inscription_id = $1\n AND block_height = $2\n AND block_time = $3\n AND tx_id = $4\n AND to_address = $5\n AND cur_output = $6\n AND cur_offset = $7\n AND from_address = $8\n AND prev_output = $9\n AND prev_offset = $10\n AND value = $11\n )\n ", + "query": "\n INSERT INTO location (\n inscription_id\n , block_height\n , block_time\n , tx_id\n , to_address\n , cur_output\n , cur_offset\n , from_address\n , prev_output\n , prev_offset\n , value\n )\n SELECT\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11\n WHERE NOT EXISTS (\n SELECT 1 FROM location\n WHERE inscription_id = $1\n AND block_height = $2\n AND block_time = $3\n AND tx_id = $4\n AND to_address = $5\n AND cur_output = $6\n AND cur_offset = $7\n AND from_address = $8\n AND prev_output = $9\n AND prev_offset = $10\n AND value = $11\n )\n ", "describe": { "columns": [], "parameters": { @@ -20,5 +20,5 @@ }, "nullable": [] }, - "hash": "1289edfa88c5539ccee82bb3d389306d21d5940ae37871166afce9f664bf53d4" + "hash": "46ed37001552b072aa3e8bf3e37bb7f90cd11f74847a8cadb41326032b580b23" } diff --git a/.sqlx/query-7d3e8f006c4f66cff514bb7bcf5314d0d235da28ead0b89fdaa090cf2f16a151.json b/.sqlx/query-7d3e8f006c4f66cff514bb7bcf5314d0d235da28ead0b89fdaa090cf2f16a151.json index be09dd1923..2e8656483f 100644 --- a/.sqlx/query-7d3e8f006c4f66cff514bb7bcf5314d0d235da28ead0b89fdaa090cf2f16a151.json +++ b/.sqlx/query-7d3e8f006c4f66cff514bb7bcf5314d0d235da28ead0b89fdaa090cf2f16a151.json @@ -6,7 +6,7 @@ "parameters": { "Left": [ "Int2", - "Int8", + "Int4", "Text", "Text", "Text" From 4292baf28b6e4d66a5e99804b5d96f130b9c9981 Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Wed, 8 May 2024 09:20:22 +0700 Subject: [PATCH 084/103] Fix --- src/indexer/db_client.rs | 2 +- src/indexer/event_consumer.rs | 1 + src/indexer/rmq_con.rs | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/indexer/db_client.rs b/src/indexer/db_client.rs index de8d40fbe8..dbe574f28b 100644 --- a/src/indexer/db_client.rs +++ b/src/indexer/db_client.rs @@ -36,7 +36,7 @@ impl DbClient { FROM event WHERE block_height = $1 ORDER BY type_id ASC, id ASC "#, - i32::try_from(block_height).expect("block_height should fit in pg integer"), + i32::try_from(block_height.to_owned()).expect("block_height should fit in pg integer"), ) .map(|r| Event { type_id: r.type_id, diff --git a/src/indexer/event_consumer.rs b/src/indexer/event_consumer.rs index b5313799f2..55a33358c1 100644 --- a/src/indexer/event_consumer.rs +++ b/src/indexer/event_consumer.rs @@ -2,6 +2,7 @@ use anyhow::Context; use clap::Parser; use futures::StreamExt; use lapin::{options::*, types::FieldTable}; +use std::sync::Arc; use tokio::runtime::Runtime; use tokio::sync::oneshot; diff --git a/src/indexer/rmq_con.rs b/src/indexer/rmq_con.rs index 9c02bc7255..043e8e3a1d 100644 --- a/src/indexer/rmq_con.rs +++ b/src/indexer/rmq_con.rs @@ -1,4 +1,6 @@ use anyhow::{Context, Error}; +use chrono::Utc; +use lapin::options::ConfirmSelectOptions; use lapin::tcp::{AMQPUriTcpExt, NativeTlsConnector}; use lapin::uri::AMQPUri; use lapin::{Connection, ConnectionProperties}; From bcd57382725e2b15d021d6616a5e39678b2a5fc7 Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Wed, 8 May 2024 11:59:32 +0700 Subject: [PATCH 085/103] Refactors --- README.md | 12 ++- src/indexer/block_consumer.rs | 20 ++--- src/indexer/db_client.rs | 30 ++++--- src/indexer/db_con.rs | 31 -------- src/indexer/event_consumer.rs | 143 ++++++++++------------------------ 5 files changed, 76 insertions(+), 160 deletions(-) diff --git a/README.md b/README.md index 89428b24fe..49483ab853 100644 --- a/README.md +++ b/README.md @@ -41,14 +41,18 @@ Note `$DATABASE_URL` is likely to be _"postgres://backend:looks-backend@localhos **Running via command line:** -To run ord event server +To run ord event server (signet): ``` -ord --bitcoin-rpc-url "http://localhost:18332" --bitcoin-rpc-username user --bitcoin-rpc-password password --commit-interval 10 --rabbitmq-url localhost --rabbitmq-password s3cr3t --rabbitmq-username indexer --rabbitmq-exchange ord-tx --chain testnet --data-dir ./index-data event-server --http-port 8080 +RUST_LOG=info cargo run --package ord --bin ord -- --bitcoin-rpc-url "http://localhost:38332" --bitcoin-rpc-username user --bitcoin-rpc-password password --commit-interval 10 --rabbitmq-url amqp://indexer:s3cr3t@localhost --rabbitmq-exchange ord-tx --chain signet --data-dir ./index-data event-server --http-port 8080 ``` -To run ord event indexer +To run ord event-consumer "inscription" indexer: ``` -ord --rabbitmq-url localhost --rabbitmq-password s3cr3t --rabbitmq-username indexer event-consumer --blocks-queue btc-blocks-q --inscriptions-queue btc-inscription-q --database-url postgresql://backend:looks-backend@localhost:55432/ordinals --ord-api-url http://localhost:8080 +RUST_LOG=info cargo run --package ord --bin ord -- --rabbitmq-url amqp://indexer:s3cr3t@localhost event-consumer --inscriptions-queue btc-inscription-q --database-url postgresql://backend:looks-backend@localhost:5432/ordinals +``` + +``` +RUST_LOG=info cargo run --package ord --bin ord -- --rabbitmq-url amqp://indexer:s3cr3t@localhost block-consumer --blocks-queue btc-blocks-q --database-url postgresql://backend:looks-backend@localhost:5432/ordinals --ord-api-url http://localhost:8080 ``` Donate diff --git a/src/indexer/block_consumer.rs b/src/indexer/block_consumer.rs index 288cd92d8d..fcce01d678 100644 --- a/src/indexer/block_consumer.rs +++ b/src/indexer/block_consumer.rs @@ -11,7 +11,6 @@ use tokio::sync::oneshot; use crate::index::event::Event; use crate::indexer::api_client::ApiClient; use crate::indexer::db_client::DbClient; -use crate::indexer::db_con::setup_db_connection; use crate::indexer::inscription_indexation::InscriptionIndexation; use crate::indexer::rmq_con::{generate_consumer_tag, setup_rabbitmq_connection}; use crate::settings::Settings; @@ -35,21 +34,18 @@ impl BlockConsumer { .context("rabbitmq amqp credentials and url must be defined")?; let channel = setup_rabbitmq_connection(addr).await?; - let database_url = self - .database_url - .as_deref() - .context("db url must be defined")?; - let pool = setup_db_connection(database_url).await?; - - let shared_pool = Arc::new(pool); - let db_client = Arc::new(DbClient::new(shared_pool.clone())); + let database_url = self.database_url.context("db url is required")?; + let db_client = Arc::new(DbClient::new(database_url, 2).await?); let api_url = self.ord_api_url.context("api url must be defined")?; let api_c = ApiClient::new(api_url.clone()).context("Failed to create API client")?; let api_client = Arc::new(api_c); - let inscription_indexation = - Arc::new(InscriptionIndexation::new(settings, db_client, api_client)); + let inscription_indexation = Arc::new(InscriptionIndexation::new( + settings, + db_client.clone(), + api_client, + )); let blocks_queue = self .blocks_queue @@ -80,7 +76,7 @@ impl BlockConsumer { let _ = blocks_shutdown_tx.send(()); let _ = tokio::try_join!(blocks_consumer_handle); - shared_pool.close().await; + db_client.close().await; Ok(None) }) diff --git a/src/indexer/db_client.rs b/src/indexer/db_client.rs index dbe574f28b..abda7f8fca 100644 --- a/src/indexer/db_client.rs +++ b/src/indexer/db_client.rs @@ -1,9 +1,10 @@ use bitcoin::{OutPoint, Txid}; use ordinals::SatPoint; +use sqlx::postgres::PgPoolOptions; use sqlx::types::Json; use sqlx::PgPool; +use sqlx::Result; use std::str::FromStr; -use std::sync::Arc; use crate::api::InscriptionDetails; use crate::InscriptionId; @@ -18,12 +19,21 @@ pub struct Event { } pub struct DbClient { - pool: Arc, + pool: PgPool, } impl DbClient { - pub fn new(pool: Arc) -> Self { - Self { pool } + pub async fn new(database_url: String, max_conn: u32) -> Result { + let pool = PgPoolOptions::new() + .max_connections(max_conn) + .connect(database_url.as_str()) + .await?; + + Ok(Self { pool }) + } + + pub async fn close(&self) { + self.pool.close().await; } pub async fn fetch_events_by_block_height( @@ -45,7 +55,7 @@ impl DbClient { location: r.location.and_then(|s| SatPoint::from_str(&s).ok()), old_location: r.old_location.and_then(|s| SatPoint::from_str(&s).ok()), }) - .fetch_all(&*self.pool) + .fetch_all(&self.pool) .await } @@ -69,7 +79,7 @@ impl DbClient { inscription_id.to_string(), location.map(|loc| loc.to_string()) ) - .execute(&*self.pool) + .execute(&self.pool) .await?; Ok(()) @@ -97,7 +107,7 @@ impl DbClient { new_location.to_string(), old_location.to_string() ) - .execute(&*self.pool) + .execute(&self.pool) .await?; Ok(()) @@ -112,7 +122,7 @@ impl DbClient { genesis_id ) .map(|r| r.id) - .fetch_optional(&*self.pool) + .fetch_optional(&self.pool) .await } @@ -181,7 +191,7 @@ impl DbClient { Json(&inscription_details.parents).encode_to_string() ) .map(|r| r.id) - .fetch_one(&*self.pool) + .fetch_one(&self.pool) .await } @@ -243,7 +253,7 @@ impl DbClient { from_offset.map(|n| i64::try_from(n).expect("from_offset should fit in pg bigint")), value.map(|n| i64::try_from(n).expect("value should fit in pg bigint")), ) - .execute(&*self.pool) + .execute(&self.pool) .await?; Ok(()) diff --git a/src/indexer/db_con.rs b/src/indexer/db_con.rs index 20c2622ed1..e69de29bb2 100644 --- a/src/indexer/db_con.rs +++ b/src/indexer/db_con.rs @@ -1,31 +0,0 @@ -use sqlx::postgres::PgPoolOptions; -use sqlx::{Pool, Postgres}; -use urlencoding::encode; - -pub async fn setup_db_connection(database_url: &str) -> Result, anyhow::Error> { - let encoded_database_url = encode_password_in_url(database_url); - let pool = PgPoolOptions::new() - .max_connections(5) - .connect(encoded_database_url.as_ref()) - .await?; - Ok(pool) -} - -fn encode_password_in_url(url: &str) -> String { - let re = regex::Regex::new(r"(\w+://)([^:]+):([^@]+)@(.*)").unwrap(); - if let Some(caps) = re.captures(url) { - let protocol_and_user = caps.get(1).map_or("", |m| m.as_str()); - let username = caps.get(2).map_or("", |m| m.as_str()); - let password = caps.get(3).map_or("", |m| m.as_str()); - let rest_of_url = caps.get(4).map_or("", |m| m.as_str()); - format!( - "{}{}:{}@{}", - protocol_and_user, - username, - encode(password), - rest_of_url - ) - } else { - url.to_string() - } -} diff --git a/src/indexer/event_consumer.rs b/src/indexer/event_consumer.rs index 55a33358c1..0d756810c2 100644 --- a/src/indexer/event_consumer.rs +++ b/src/indexer/event_consumer.rs @@ -2,13 +2,10 @@ use anyhow::Context; use clap::Parser; use futures::StreamExt; use lapin::{options::*, types::FieldTable}; -use std::sync::Arc; use tokio::runtime::Runtime; -use tokio::sync::oneshot; use crate::index::event::Event; use crate::indexer::db_client::DbClient; -use crate::indexer::db_con::setup_db_connection; use crate::indexer::rmq_con::{generate_consumer_tag, setup_rabbitmq_connection}; use crate::settings::Settings; use crate::subcommand::SubcommandResult; @@ -24,146 +21,86 @@ pub struct EventConsumer { impl EventConsumer { pub fn run(self, settings: &Settings) -> SubcommandResult { Runtime::new()?.block_on(async { - let addr = settings - .rabbitmq_addr() - .context("rabbitmq amqp credentials and url must be defined")?; - let channel = setup_rabbitmq_connection(addr).await?; - - let database_url = self - .database_url - .as_deref() - .context("db url must be defined")?; - let pool = setup_db_connection(database_url).await?; - - let shared_pool = Arc::new(pool); - let db_client = Arc::new(DbClient::new(shared_pool.clone())); + let database_url = self.database_url.context("db url is required")?; + let db_client = DbClient::new(database_url, 2).await?; - let inscriptions_queue = self - .inscriptions_queue - .as_deref() - .context("rabbitmq inscriptions queue path must be defined")?; - let inscriptions_queue_str = inscriptions_queue.to_string(); - let inscriptions_channel = channel.clone(); - let inscriptions_consumer_tag = generate_consumer_tag("lr-ord"); - let (inscriptions_shutdown_tx, inscriptions_shutdown_rx) = oneshot::channel::<()>(); + let tag = generate_consumer_tag("lr-ord"); + let addr = settings.rabbitmq_addr().context("rmq url is required")?; + let queue_name = self.inscriptions_queue.context("rmq queue is required")?; + let channel = setup_rabbitmq_connection(addr).await?; - let inscriptions_consumer_handle = tokio::spawn(async move { - EventConsumer::consume_queue( - inscriptions_channel, - inscriptions_queue_str, - inscriptions_consumer_tag, - db_client, - inscriptions_shutdown_rx, + let mut consumer = channel + .basic_consume( + &queue_name, + tag.as_str(), + BasicConsumeOptions::default(), + FieldTable::default(), ) - .await - .expect("error consuming inscriptions queue") - }); - - let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; - sigterm.recv().await; - let _ = inscriptions_shutdown_tx.send(()); - let _ = tokio::try_join!(inscriptions_consumer_handle); - - shared_pool.close().await; - - Ok(None) - }) - } - - async fn consume_queue( - channel: lapin::Channel, - queue_name: String, - consumer_tag: String, - db_client: Arc, - mut shutdown_signal: oneshot::Receiver<()>, - ) -> Result<(), anyhow::Error> { - let mut consumer = channel - .basic_consume( - &queue_name, - consumer_tag.as_str(), - BasicConsumeOptions::default(), - FieldTable::default(), - ) - .await?; - - log::info!("Starting to consume messages from {}", queue_name); - while let Some(result) = consumer.next().await { - let delivery = result?; - tokio::select! { - process_result = EventConsumer::handle_delivery(delivery, &db_client) => { - process_result?; - }, - _ = &mut shutdown_signal => { - log::info!("Shutdown signal received, stopping consumer."); - break; - }, + .await?; + + loop { + if let Some(message) = consumer.next().await { + match message { + Ok(delivery) => EventConsumer::handle_delivery(delivery, &db_client).await?, + Err(e) => log::error!("error consuming message: {}", e), + } + } } - } - - log::info!("Closing consumer channel {}", queue_name); - channel - .close(200, "Closing channel due to shutdown") - .await?; - - Ok(()) + }) } async fn handle_delivery( delivery: lapin::message::Delivery, - db_client: &Arc, - ) -> Result<(), anyhow::Error> { + db_client: &DbClient, + ) -> Result<(), lapin::Error> { let event: Result = serde_json::from_slice(&delivery.data); + match event { Ok(event) => { if let Err(err) = EventConsumer::process_event(event, db_client).await { - log::error!("Failed to process event: {}", err); - delivery - .reject(BasicRejectOptions { requeue: false }) - .await?; + log::error!("failed to process event: {}", err); + delivery.reject(BasicRejectOptions { requeue: false }).await } else { - delivery.ack(BasicAckOptions::default()).await?; + delivery.ack(BasicAckOptions::default()).await } } + Err(e) => { - log::error!("Failed to deserialize event, rejecting: {}", e); - delivery - .reject(BasicRejectOptions { requeue: false }) - .await?; + log::error!("failed to deserialize event, rejecting: {}", e); + delivery.reject(BasicRejectOptions { requeue: false }).await } } - - Ok(()) } - async fn process_event(event: Event, db_client: &Arc) -> Result<(), anyhow::Error> { + async fn process_event(event: Event, db_client: &DbClient) -> Result<(), sqlx::Error> { match &event { Event::InscriptionCreated { block_height, - charms: _charms, inscription_id, location, - parent_inscription_ids: _parent_inscription_ids, - sequence_number: _sequence_number, + .. } => { db_client .save_inscription_created(block_height, inscription_id, location) - .await? + .await } + Event::InscriptionTransferred { block_height, inscription_id, new_location, old_location, - sequence_number: _sequence_number, + .. } => { db_client .save_inscription_transferred(block_height, inscription_id, new_location, old_location) - .await?; + .await } + _ => { - log::warn!("Received an unhandled event type {:?}", event); + log::warn!("skipped unhandled event type {:?}", event); + Ok(()) } } - Ok(()) } } From 9bd812a7b163b391750bcffae099b60f0f98daa2 Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Wed, 8 May 2024 16:03:19 +0700 Subject: [PATCH 086/103] Refactors Continued --- src/indexer/block_consumer.rs | 152 ++++++++------------------ src/indexer/db_client.rs | 2 +- src/indexer/db_con.rs | 0 src/indexer/event_consumer.rs | 58 +++++----- src/indexer/inscription_indexation.rs | 17 +-- src/lib.rs | 1 - 6 files changed, 82 insertions(+), 148 deletions(-) delete mode 100644 src/indexer/db_con.rs diff --git a/src/indexer/block_consumer.rs b/src/indexer/block_consumer.rs index fcce01d678..54e0479466 100644 --- a/src/indexer/block_consumer.rs +++ b/src/indexer/block_consumer.rs @@ -1,12 +1,8 @@ -use std::process; -use std::sync::Arc; - use anyhow::Context; use clap::Parser; use futures::StreamExt; -use lapin::{options::*, types::FieldTable}; +use lapin::{message::Delivery, options::*, types::FieldTable}; use tokio::runtime::Runtime; -use tokio::sync::oneshot; use crate::index::event::Event; use crate::indexer::api_client::ApiClient; @@ -16,12 +12,14 @@ use crate::indexer::rmq_con::{generate_consumer_tag, setup_rabbitmq_connection}; use crate::settings::Settings; use crate::subcommand::SubcommandResult; -#[derive(Debug, Parser, Clone)] +#[derive(Debug, Parser)] pub struct BlockConsumer { #[arg(long, help = "RMQ queue to consume blocks.")] pub(crate) blocks_queue: Option, + #[arg(long, help = "DB url to persist inscriptions.")] pub(crate) database_url: Option, + #[arg(long, help = "Ord api url to fetch inscriptions.")] pub(crate) ord_api_url: Option, } @@ -29,134 +27,72 @@ pub struct BlockConsumer { impl BlockConsumer { pub fn run(self, settings: &Settings) -> SubcommandResult { Runtime::new()?.block_on(async { - let addr = settings - .rabbitmq_addr() - .context("rabbitmq amqp credentials and url must be defined")?; - let channel = setup_rabbitmq_connection(addr).await?; - let database_url = self.database_url.context("db url is required")?; - let db_client = Arc::new(DbClient::new(database_url, 2).await?); + let db = DbClient::new(database_url, 2).await?; let api_url = self.ord_api_url.context("api url must be defined")?; - let api_c = ApiClient::new(api_url.clone()).context("Failed to create API client")?; - let api_client = Arc::new(api_c); + let api_client = ApiClient::new(api_url.clone()).context("Failed to create API client")?; - let inscription_indexation = Arc::new(InscriptionIndexation::new( - settings, - db_client.clone(), - api_client, - )); + let tag = generate_consumer_tag("lr-ord-evts"); + let addr = settings.rabbitmq_addr().context("rmq url is required")?; + let queue_name = self.blocks_queue.context("rmq queue is required")?; + let channel = setup_rabbitmq_connection(addr).await?; - let blocks_queue = self - .blocks_queue - .as_deref() - .context("rabbitmq blocks queue path must be defined")?; - let blocks_queue_str = blocks_queue.to_string(); - let blocks_channel = channel.clone(); - let blocks_consumer_tag = generate_consumer_tag("lr-ord"); - let (blocks_shutdown_tx, blocks_shutdown_rx) = oneshot::channel::<()>(); + let inscription_indexer = InscriptionIndexation::new(settings, db, api_client); - let blocks_consumer_handle = tokio::spawn(async move { - BlockConsumer::consume_queue( - blocks_channel, - blocks_queue_str, - blocks_consumer_tag, - inscription_indexation, - blocks_shutdown_rx, + let mut consumer = channel + .basic_consume( + &queue_name, + tag.as_str(), + BasicConsumeOptions::default(), + FieldTable::default(), ) - .await - .map_err(|e| { - log::error!("Error consuming blocks queue: {}", e); - process::exit(1); - }) - }); + .await?; - let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; - sigterm.recv().await; - let _ = blocks_shutdown_tx.send(()); - let _ = tokio::try_join!(blocks_consumer_handle); - - db_client.close().await; - - Ok(None) - }) - } - - async fn consume_queue( - channel: lapin::Channel, - queue_name: String, - consumer_tag: String, - inscription_indexation: Arc, - mut shutdown_signal: oneshot::Receiver<()>, - ) -> Result<(), anyhow::Error> { - let mut consumer = channel - .basic_consume( - &queue_name, - consumer_tag.as_str(), - BasicConsumeOptions::default(), - FieldTable::default(), - ) - .await?; + log::info!("started block consumer {tag} for {queue_name}"); - log::info!("Starting to consume messages from {}", queue_name); - while let Some(result) = consumer.next().await { - let delivery = result?; - tokio::select! { - process_result = BlockConsumer::handle_delivery(delivery, &inscription_indexation) => { - process_result?; - }, - _ = &mut shutdown_signal => { - log::info!("Shutdown signal received, stopping consumer."); - break; - }, + loop { + if let Some(msg) = consumer.next().await { + match msg { + Ok(d) => BlockConsumer::handle_delivery(d, &inscription_indexer).await?, + Err(e) => log::error!("error consuming message: {}", e), + } + } } - } - - log::info!("Closing consumer channel {}", queue_name); - channel - .close(200, "Closing channel due to shutdown") - .await?; - - Ok(()) + }) } async fn handle_delivery( - delivery: lapin::message::Delivery, - inscription_indexation: &Arc, - ) -> Result<(), anyhow::Error> { + delivery: Delivery, + indexer: &InscriptionIndexation, + ) -> Result<(), lapin::Error> { + let reject = BasicRejectOptions { requeue: false }; let event: Result = serde_json::from_slice(&delivery.data); + match event { Ok(event) => { - if let Err(err) = BlockConsumer::process_event(event, inscription_indexation).await { - log::error!("Failed to process event: {}", err); - delivery - .reject(BasicRejectOptions { requeue: false }) - .await?; + if let Err(err) = BlockConsumer::process_event(&event, indexer).await { + log::error!("failed to process event: {}", err); + delivery.reject(reject).await } else { - delivery.ack(BasicAckOptions::default()).await?; + delivery.ack(BasicAckOptions::default()).await } } + Err(e) => { - log::error!("Failed to deserialize event, rejecting: {}", e); - delivery - .reject(BasicRejectOptions { requeue: false }) - .await?; + log::error!("failed to deserialize event, rejecting: {}", e); + delivery.reject(reject).await } } - - Ok(()) } async fn process_event( - event: Event, - inscription_indexation: &Arc, + event: &Event, + indexer: &InscriptionIndexation, ) -> Result<(), anyhow::Error> { - match &event { - Event::BlockCommitted { height } => inscription_indexation.sync_blocks(height).await?, - _ => { - log::warn!("Received an unhandled event type {:?}", event); - } + match event { + Event::BlockCommitted { height } => indexer.sync_blocks(height).await, + _ => Ok(log::warn!("skipped unhandled event type {:?}", event)), } - Ok(()) } } diff --git a/src/indexer/db_client.rs b/src/indexer/db_client.rs index abda7f8fca..314ce5dd0b 100644 --- a/src/indexer/db_client.rs +++ b/src/indexer/db_client.rs @@ -9,7 +9,7 @@ use std::str::FromStr; use crate::api::InscriptionDetails; use crate::InscriptionId; -#[derive(Debug, Clone, sqlx::FromRow)] +#[derive(Debug, Clone)] pub struct Event { pub type_id: i16, pub block_height: i32, diff --git a/src/indexer/db_con.rs b/src/indexer/db_con.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/indexer/event_consumer.rs b/src/indexer/event_consumer.rs index 0d756810c2..d715bec500 100644 --- a/src/indexer/event_consumer.rs +++ b/src/indexer/event_consumer.rs @@ -1,7 +1,7 @@ use anyhow::Context; use clap::Parser; use futures::StreamExt; -use lapin::{options::*, types::FieldTable}; +use lapin::{message::Delivery, options::*, types::FieldTable}; use tokio::runtime::Runtime; use crate::index::event::Event; @@ -10,7 +10,7 @@ use crate::indexer::rmq_con::{generate_consumer_tag, setup_rabbitmq_connection}; use crate::settings::Settings; use crate::subcommand::SubcommandResult; -#[derive(Debug, Parser, Clone)] +#[derive(Debug, Parser)] pub struct EventConsumer { #[arg(long, help = "RMQ queue to consume inscription events.")] pub(crate) inscriptions_queue: Option, @@ -22,9 +22,9 @@ impl EventConsumer { pub fn run(self, settings: &Settings) -> SubcommandResult { Runtime::new()?.block_on(async { let database_url = self.database_url.context("db url is required")?; - let db_client = DbClient::new(database_url, 2).await?; + let db = DbClient::new(database_url, 2).await?; - let tag = generate_consumer_tag("lr-ord"); + let tag = generate_consumer_tag("lr-ord-evts"); let addr = settings.rabbitmq_addr().context("rmq url is required")?; let queue_name = self.inscriptions_queue.context("rmq queue is required")?; let channel = setup_rabbitmq_connection(addr).await?; @@ -38,10 +38,12 @@ impl EventConsumer { ) .await?; + log::info!("started event consumer {tag} for {queue_name}"); + loop { - if let Some(message) = consumer.next().await { - match message { - Ok(delivery) => EventConsumer::handle_delivery(delivery, &db_client).await?, + if let Some(msg) = consumer.next().await { + match msg { + Ok(d) => EventConsumer::handle_delivery(d, &db).await?, Err(e) => log::error!("error consuming message: {}", e), } } @@ -49,30 +51,28 @@ impl EventConsumer { }) } - async fn handle_delivery( - delivery: lapin::message::Delivery, - db_client: &DbClient, - ) -> Result<(), lapin::Error> { + async fn handle_delivery(delivery: Delivery, db: &DbClient) -> Result<(), lapin::Error> { + let reject = BasicRejectOptions { requeue: false }; let event: Result = serde_json::from_slice(&delivery.data); match event { Ok(event) => { - if let Err(err) = EventConsumer::process_event(event, db_client).await { + if let Err(err) = EventConsumer::process_event(event, db).await { log::error!("failed to process event: {}", err); - delivery.reject(BasicRejectOptions { requeue: false }).await + delivery.reject(reject).await } else { delivery.ack(BasicAckOptions::default()).await } } Err(e) => { - log::error!("failed to deserialize event, rejecting: {}", e); - delivery.reject(BasicRejectOptions { requeue: false }).await + log::error!("failed to deserialize event, rejecting: {e}"); + delivery.reject(reject).await } } } - async fn process_event(event: Event, db_client: &DbClient) -> Result<(), sqlx::Error> { + async fn process_event(event: Event, db: &DbClient) -> Result<(), sqlx::Error> { match &event { Event::InscriptionCreated { block_height, @@ -80,9 +80,13 @@ impl EventConsumer { location, .. } => { - db_client - .save_inscription_created(block_height, inscription_id, location) - .await + db.save_inscription_created( + // + block_height, + inscription_id, + location, + ) + .await } Event::InscriptionTransferred { @@ -92,15 +96,17 @@ impl EventConsumer { old_location, .. } => { - db_client - .save_inscription_transferred(block_height, inscription_id, new_location, old_location) - .await + db.save_inscription_transferred( + // + block_height, + inscription_id, + new_location, + old_location, + ) + .await } - _ => { - log::warn!("skipped unhandled event type {:?}", event); - Ok(()) - } + _ => Ok(log::warn!("skipped unhandled event type {:?}", event)), } } } diff --git a/src/indexer/inscription_indexation.rs b/src/indexer/inscription_indexation.rs index 81eb9809b7..32a3de60ac 100644 --- a/src/indexer/inscription_indexation.rs +++ b/src/indexer/inscription_indexation.rs @@ -1,10 +1,7 @@ -use std::io::Cursor; -use std::sync::Arc; - use ciborium::from_reader; -use serde_json::Value; - use ordinals::SatPoint; +use serde_json::Value; +use std::io::Cursor; use crate::api::BlockInfo; use crate::indexer::api_client::ApiClient; @@ -13,16 +10,12 @@ use crate::settings::Settings; pub struct InscriptionIndexation { settings: Settings, - ord_db_client: Arc, - ord_api_client: Arc, + ord_db_client: DbClient, + ord_api_client: ApiClient, } impl InscriptionIndexation { - pub fn new( - settings: &Settings, - ord_db_client: Arc, - ord_api_client: Arc, - ) -> Self { + pub fn new(settings: &Settings, ord_db_client: DbClient, ord_api_client: ApiClient) -> Self { Self { settings: settings.clone(), ord_api_client, diff --git a/src/lib.rs b/src/lib.rs index 428176a83a..e10a25f1cd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -124,7 +124,6 @@ pub mod indexer { pub mod api_client; pub mod block_consumer; pub mod db_client; - pub mod db_con; pub mod event_consumer; pub mod event_publisher; pub mod inscription_indexation; From c100d2ce0865fb1ff95703ab85d5c0ed49431750 Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Wed, 8 May 2024 17:12:34 +0700 Subject: [PATCH 087/103] Refactors Continued --- src/indexer/block_consumer.rs | 6 +- src/indexer/db_client.rs | 15 +++++ src/indexer/inscription_indexation.rs | 82 ++++++++++----------------- 3 files changed, 48 insertions(+), 55 deletions(-) diff --git a/src/indexer/block_consumer.rs b/src/indexer/block_consumer.rs index 54e0479466..e7b9416b5f 100644 --- a/src/indexer/block_consumer.rs +++ b/src/indexer/block_consumer.rs @@ -55,7 +55,7 @@ impl BlockConsumer { if let Some(msg) = consumer.next().await { match msg { Ok(d) => BlockConsumer::handle_delivery(d, &inscription_indexer).await?, - Err(e) => log::error!("error consuming message: {}", e), + Err(err) => log::error!("error consuming message: {err}"), } } } @@ -72,7 +72,7 @@ impl BlockConsumer { match event { Ok(event) => { if let Err(err) = BlockConsumer::process_event(&event, indexer).await { - log::error!("failed to process event: {}", err); + log::error!("failed to process event: {err}"); delivery.reject(reject).await } else { delivery.ack(BasicAckOptions::default()).await @@ -80,7 +80,7 @@ impl BlockConsumer { } Err(e) => { - log::error!("failed to deserialize event, rejecting: {}", e); + log::error!("failed to deserialize event: {:?}", e); delivery.reject(reject).await } } diff --git a/src/indexer/db_client.rs b/src/indexer/db_client.rs index 314ce5dd0b..1ad5a7853d 100644 --- a/src/indexer/db_client.rs +++ b/src/indexer/db_client.rs @@ -9,6 +9,11 @@ use std::str::FromStr; use crate::api::InscriptionDetails; use crate::InscriptionId; +pub enum EventType { + InscriptionCreated, + InscriptionTransferred, +} + #[derive(Debug, Clone)] pub struct Event { pub type_id: i16, @@ -18,6 +23,16 @@ pub struct Event { pub old_location: Option, } +impl Event { + pub fn get_type(&self) -> EventType { + match self.type_id { + 1 => EventType::InscriptionCreated, + 2 => EventType::InscriptionTransferred, + _ => unreachable!(), + } + } +} + pub struct DbClient { pool: PgPool, } diff --git a/src/indexer/inscription_indexation.rs b/src/indexer/inscription_indexation.rs index 32a3de60ac..f66a06c87d 100644 --- a/src/indexer/inscription_indexation.rs +++ b/src/indexer/inscription_indexation.rs @@ -5,59 +5,42 @@ use std::io::Cursor; use crate::api::BlockInfo; use crate::indexer::api_client::ApiClient; -use crate::indexer::db_client::{DbClient, Event}; +use crate::indexer::db_client::{DbClient, Event, EventType}; use crate::settings::Settings; pub struct InscriptionIndexation { settings: Settings, - ord_db_client: DbClient, - ord_api_client: ApiClient, + db: DbClient, + api: ApiClient, } impl InscriptionIndexation { - pub fn new(settings: &Settings, ord_db_client: DbClient, ord_api_client: ApiClient) -> Self { + pub fn new(settings: &Settings, db: DbClient, api: ApiClient) -> Self { Self { settings: settings.clone(), - ord_api_client, - ord_db_client, + db, + api, } } pub async fn sync_blocks(&self, block_height: &u32) -> Result<(), anyhow::Error> { log::info!("Blocks committed event for={block_height}"); - let events = self - .ord_db_client - .fetch_events_by_block_height(block_height) - .await?; - let block_info = self.ord_api_client.fetch_block_info(block_height).await?; + let events = self.db.fetch_events_by_block_height(block_height).await?; + let block_info = self.api.fetch_block_info(block_height).await?; + for event in events { - match event.type_id { - 1 => { - if let Err(e) = self.process_inscription_created(&event, &block_info).await { - log::error!( - "Error processing inscription creation for event {:?}: {}", - event, - e - ); - } - } - 2 => { - if let Err(e) = self - .process_inscription_transferred(&event, &block_info) - .await - { - log::error!( - "Error processing inscription transferred for event {:?}: {}", - event, - e - ); - } - } - _ => { - log::warn!("Unhandled event type: {}", event.type_id); - } - } + match event.get_type() { + EventType::InscriptionCreated => self + .process_inscription_created(&event, &block_info) + .await + .inspect_err(|e| log::error!("error with inscription_created {:?}: {e}", event)), + + EventType::InscriptionTransferred => self + .process_inscription_transferred(&event, &block_info) + .await + .inspect_err(|e| log::error!("error with inscription_transferred {:?}: {e}", event)), + }; } Ok(()) @@ -68,27 +51,22 @@ impl InscriptionIndexation { event: &Event, block_info: &BlockInfo, ) -> Result<(), anyhow::Error> { - let inscription_details = self - .ord_api_client - .fetch_inscription_details(event.inscription_id.clone()) - .await?; + let inscription_id = event.inscription_id.clone(); + let inscription = self.api.fetch_inscription_details(inscription_id).await?; - let metadata = self.try_and_extract_metadata(&inscription_details.metadata); - - let inscription_id = self - .ord_db_client - .save_inscription(&inscription_details, metadata) - .await?; + let metadata = self.try_and_extract_metadata(&inscription.metadata); + let id = self.db.save_inscription(&inscription, metadata).await?; let mut to_location_details: Option<(String, u64)> = None; + if let Some(location) = &event.location { to_location_details = self.process_location(location).await?; } self - .ord_db_client + .db .save_location( - inscription_id, + id, event.block_height, block_info.timestamp, event.location.as_ref().map(|loc| loc.outpoint.txid), @@ -128,7 +106,7 @@ impl InscriptionIndexation { block_info: &BlockInfo, ) -> Result<(), anyhow::Error> { let inscription_id = self - .ord_db_client + .db .fetch_inscription_id_by_genesis_id(event.inscription_id.clone()) .await? .ok_or_else(|| { @@ -149,7 +127,7 @@ impl InscriptionIndexation { } self - .ord_db_client + .db .save_location( inscription_id, event.block_height, @@ -176,7 +154,7 @@ impl InscriptionIndexation { &self, location: &SatPoint, ) -> Result, anyhow::Error> { - let tx_details = self.ord_api_client.fetch_tx(location.outpoint.txid).await?; + let tx_details = self.api.fetch_tx(location.outpoint.txid).await?; let output = tx_details .transaction From 19be6f7fd972d5b0522536a7f733d36189496751 Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 8 May 2024 12:53:29 +0200 Subject: [PATCH 088/103] Return if no events --- src/indexer/inscription_indexation.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/indexer/inscription_indexation.rs b/src/indexer/inscription_indexation.rs index 81eb9809b7..f6ea8b7ce5 100644 --- a/src/indexer/inscription_indexation.rs +++ b/src/indexer/inscription_indexation.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use ciborium::from_reader; use serde_json::Value; +use sha3::digest::const_oid::db::rfc4519::L; use ordinals::SatPoint; @@ -37,6 +38,11 @@ impl InscriptionIndexation { .ord_db_client .fetch_events_by_block_height(block_height) .await?; + + if (events.is_empty()) { + return Ok(()); + } + let block_info = self.ord_api_client.fetch_block_info(block_height).await?; for event in events { match event.type_id { From 19b168553fb89893f4b9dbbf771b4d50460679eb Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 8 May 2024 12:58:12 +0200 Subject: [PATCH 089/103] Publish block only if it's passed first inscription height --- src/index/updater.rs | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/index/updater.rs b/src/index/updater.rs index 47144c518c..b0b984a597 100644 --- a/src/index/updater.rs +++ b/src/index/updater.rs @@ -744,19 +744,21 @@ impl<'index> Updater<'index> { Reorg::update_savepoints(self.index, self.height)?; - if let Some(sender) = self.index.event_sender.as_ref() { - if let Ok(uncommitted) = u32::try_from(uncommitted) { - for current_height in (self.height - uncommitted)..self.height { - sender.blocking_send(Event::BlockCommitted { - height: current_height, - })?; + if self.height >= self.index.first_inscription_height { + if let Some(sender) = self.index.event_sender.as_ref() { + if let Ok(uncommitted) = u32::try_from(uncommitted) { + for current_height in (self.height - uncommitted)..self.height { + sender.blocking_send(Event::BlockCommitted { + height: current_height, + })?; + } + } else { + log::error!( + "Failed to publish block range from_height: {}, uncommitted: {}", + self.height, + uncommitted + ); } - } else { - log::error!( - "Failed to publish block range from_height: {}, uncommitted: {}", - self.height, - uncommitted - ); } } From a89d1f0b0b3f2a8b9a21d3c456d4f53190fbf373 Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 8 May 2024 12:58:21 +0200 Subject: [PATCH 090/103] remove db_con.rs --- src/indexer/db_con.rs | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/indexer/db_con.rs diff --git a/src/indexer/db_con.rs b/src/indexer/db_con.rs deleted file mode 100644 index e69de29bb2..0000000000 From 731cbad838658b36b5ab055dffcff7e6ab411ff9 Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 8 May 2024 12:59:52 +0200 Subject: [PATCH 091/103] Fix build --- src/indexer/inscription_indexation.rs | 3 +-- src/lib.rs | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/indexer/inscription_indexation.rs b/src/indexer/inscription_indexation.rs index f6ea8b7ce5..4bbe9875d6 100644 --- a/src/indexer/inscription_indexation.rs +++ b/src/indexer/inscription_indexation.rs @@ -3,7 +3,6 @@ use std::sync::Arc; use ciborium::from_reader; use serde_json::Value; -use sha3::digest::const_oid::db::rfc4519::L; use ordinals::SatPoint; @@ -39,7 +38,7 @@ impl InscriptionIndexation { .fetch_events_by_block_height(block_height) .await?; - if (events.is_empty()) { + if events.is_empty() { return Ok(()); } diff --git a/src/lib.rs b/src/lib.rs index 428176a83a..e10a25f1cd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -124,7 +124,6 @@ pub mod indexer { pub mod api_client; pub mod block_consumer; pub mod db_client; - pub mod db_con; pub mod event_consumer; pub mod event_publisher; pub mod inscription_indexation; From f21018cf12a8ab44532d64e852095c0928586d62 Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 8 May 2024 14:51:53 +0200 Subject: [PATCH 092/103] Add retries for rmq publish --- src/indexer/event_publisher.rs | 65 ++++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/src/indexer/event_publisher.rs b/src/indexer/event_publisher.rs index 6a28946736..898688a1a4 100644 --- a/src/indexer/event_publisher.rs +++ b/src/indexer/event_publisher.rs @@ -1,7 +1,9 @@ use anyhow::{Context, Result}; -use lapin::{options::BasicPublishOptions, BasicProperties}; +use lapin::{options::BasicPublishOptions, BasicProperties, Channel}; +use std::time::Duration; use tokio::runtime::Runtime; use tokio::sync::mpsc; +use tokio::time::sleep; use crate::index::event::Event; use crate::indexer::rmq_con::setup_rabbitmq_connection; @@ -24,7 +26,7 @@ impl EventPublisher { .context("rabbitmq exchange path must be defined")? .to_owned(); - let (tx, rx) = mpsc::channel::(128); + let (tx, rx) = mpsc::channel::(1); std::thread::spawn(move || { Runtime::new().expect("runtime is setup").block_on(async { @@ -46,32 +48,65 @@ impl EventPublisher { exchange: String, mut rx: mpsc::Receiver, ) -> Result<()> { - let channel = setup_rabbitmq_connection(&addr).await?; + let mut channel = setup_rabbitmq_connection(&addr).await?; while let Some(event) = rx.recv().await { let message = serde_json::to_vec(&event)?; + let routing_key = EventPublisher::type_name(&event); - let publish = channel - .basic_publish( - &exchange, - EventPublisher::type_name(&event), - BasicPublishOptions::default(), - &message, - BasicProperties::default(), - ) - .await? - .await?; + let mut attempts = 3; + let mut backoff_delay = Duration::from_secs(1); + while attempts > 0 { + let result = + EventPublisher::publish_message(&channel, &exchange, routing_key, &message).await; + if let Err(e) = result { + attempts -= 1; + log::error!("Error publishing message: {}. Attempting to recreate the RMQ channel. Retries left: {}. Retrying in {} seconds.", e, attempts, backoff_delay.as_secs()); + sleep(backoff_delay).await; + backoff_delay *= 2; + channel = setup_rabbitmq_connection(&addr).await?; + } else { + break; + } + } - if !publish.is_ack() { + if attempts == 0 { return Err(anyhow::Error::new(std::io::Error::new( std::io::ErrorKind::Other, - "Message was not acknowledged", + "Failed to publish message after retries", ))); } } Ok(()) } + async fn publish_message( + channel: &Channel, + exchange: &str, + routing_key: &str, + message: &[u8], + ) -> Result<()> { + let publish = channel + .basic_publish( + exchange, + routing_key, + BasicPublishOptions::default(), + message, + BasicProperties::default(), + ) + .await? + .await?; + + if !publish.is_ack() { + return Err(anyhow::Error::new(std::io::Error::new( + std::io::ErrorKind::Other, + "Message was not acknowledged", + ))); + } + + Ok(()) + } + fn type_name(event: &Event) -> &'static str { match event { Event::InscriptionCreated { .. } => "InscriptionCreated", From 6250178062aa70b126df7fe233841c9cf384c36f Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 8 May 2024 15:10:24 +0200 Subject: [PATCH 093/103] Add retries to rmq connection, increase retry amount --- src/indexer/event_publisher.rs | 2 +- src/indexer/rmq_con.rs | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/indexer/event_publisher.rs b/src/indexer/event_publisher.rs index 898688a1a4..0881e807c1 100644 --- a/src/indexer/event_publisher.rs +++ b/src/indexer/event_publisher.rs @@ -54,7 +54,7 @@ impl EventPublisher { let message = serde_json::to_vec(&event)?; let routing_key = EventPublisher::type_name(&event); - let mut attempts = 3; + let mut attempts = 10; let mut backoff_delay = Duration::from_secs(1); while attempts > 0 { let result = diff --git a/src/indexer/rmq_con.rs b/src/indexer/rmq_con.rs index 043e8e3a1d..0bd7b491e3 100644 --- a/src/indexer/rmq_con.rs +++ b/src/indexer/rmq_con.rs @@ -1,3 +1,5 @@ +use std::time::Duration; + use anyhow::{Context, Error}; use chrono::Utc; use lapin::options::ConfirmSelectOptions; @@ -5,8 +7,38 @@ use lapin::tcp::{AMQPUriTcpExt, NativeTlsConnector}; use lapin::uri::AMQPUri; use lapin::{Connection, ConnectionProperties}; use rand::distributions::{Alphanumeric, DistString}; +use tokio::time::sleep; pub async fn setup_rabbitmq_connection(addr: &str) -> Result { + let mut attempts = 10; + let mut backoff_delay = Duration::from_secs(1); + while attempts > 0 { + match create_channel(addr).await { + Ok(channel) => return Ok(channel), + Err(e) => { + attempts -= 1; + if attempts < 0 { + return Err(e); + } else { + log::error!( + "Error creating RabbitMQ channel: {}. Retries left: {}. Retrying in {} seconds.", + e, + attempts, + backoff_delay.as_secs() + ); + sleep(backoff_delay).await; + backoff_delay *= 2; + } + } + } + } + Err(anyhow::Error::new(std::io::Error::new( + std::io::ErrorKind::Other, + "Failed to establish RabbitMQ channel after retries", + ))) +} + +async fn create_channel(addr: &str) -> Result { let conn = connect_to_rabbitmq(addr).await?; let channel = conn .create_channel() From 8ecbb3e30866c75ab5418631cfb88cf91407c8f7 Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 8 May 2024 16:33:55 +0200 Subject: [PATCH 094/103] Requeue failed consumption --- rmqconfig.json | 8 ++++++-- src/indexer/event_consumer.rs | 22 ++++++++++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/rmqconfig.json b/rmqconfig.json index de454be89a..773d7b60a3 100644 --- a/rmqconfig.json +++ b/rmqconfig.json @@ -5,14 +5,18 @@ "vhost": "/", "durable": true, "auto_delete": false, - "arguments": {} + "arguments": { + "x-queue-type": "quorum" + } }, { "name": "btc-blocks-q", "vhost": "/", "durable": true, "auto_delete": false, - "arguments": {} + "arguments": { + "x-queue-type": "quorum" + } } ], "exchanges": [ diff --git a/src/indexer/event_consumer.rs b/src/indexer/event_consumer.rs index 0d756810c2..03691596a7 100644 --- a/src/indexer/event_consumer.rs +++ b/src/indexer/event_consumer.rs @@ -1,6 +1,7 @@ use anyhow::Context; use clap::Parser; use futures::StreamExt; +use lapin::types::AMQPValue; use lapin::{options::*, types::FieldTable}; use tokio::runtime::Runtime; @@ -55,11 +56,28 @@ impl EventConsumer { ) -> Result<(), lapin::Error> { let event: Result = serde_json::from_slice(&delivery.data); + let mut delivery_count = 0; + if let Some(headers) = delivery.properties.headers() { + if let Some(AMQPValue::LongLongInt(count)) = headers.inner().get("x-delivery-count") { + delivery_count = *count; + } + } + match event { Ok(event) => { if let Err(err) = EventConsumer::process_event(event, db_client).await { - log::error!("failed to process event: {}", err); - delivery.reject(BasicRejectOptions { requeue: false }).await + if delivery_count >= 3 { + log::error!("failed to process event: {}, rejecting", err); + delivery.reject(BasicRejectOptions { requeue: false }).await + } else { + log::warn!("failed to process event: {}, requeuing", err); + delivery + .nack(BasicNackOptions { + multiple: false, + requeue: true, + }) + .await + } } else { delivery.ack(BasicAckOptions::default()).await } From 82cf880ad205015634f45df5b8da86fbd29d36d5 Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 8 May 2024 16:39:08 +0200 Subject: [PATCH 095/103] Requeue failed consumption --- src/indexer/block_consumer.rs | 36 ++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/src/indexer/block_consumer.rs b/src/indexer/block_consumer.rs index fcce01d678..f4dfb0e9aa 100644 --- a/src/indexer/block_consumer.rs +++ b/src/indexer/block_consumer.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use anyhow::Context; use clap::Parser; use futures::StreamExt; +use lapin::types::AMQPValue; use lapin::{options::*, types::FieldTable}; use tokio::runtime::Runtime; use tokio::sync::oneshot; @@ -123,28 +124,41 @@ impl BlockConsumer { async fn handle_delivery( delivery: lapin::message::Delivery, inscription_indexation: &Arc, - ) -> Result<(), anyhow::Error> { + ) -> Result<(), lapin::Error> { let event: Result = serde_json::from_slice(&delivery.data); + + let mut delivery_count = 0; + if let Some(headers) = delivery.properties.headers() { + if let Some(AMQPValue::LongLongInt(count)) = headers.inner().get("x-delivery-count") { + delivery_count = *count; + } + } + match event { Ok(event) => { if let Err(err) = BlockConsumer::process_event(event, inscription_indexation).await { - log::error!("Failed to process event: {}", err); - delivery - .reject(BasicRejectOptions { requeue: false }) - .await?; + if delivery_count >= 3 { + log::error!("Failed to process event: {}, rejecting", err); + delivery.reject(BasicRejectOptions { requeue: false }).await + } else { + log::warn!("Failed to process event: {}, requeuing", err); + delivery + .nack(BasicNackOptions { + multiple: false, + requeue: true, + }) + .await + } } else { - delivery.ack(BasicAckOptions::default()).await?; + delivery.ack(BasicAckOptions::default()).await } } + Err(e) => { log::error!("Failed to deserialize event, rejecting: {}", e); - delivery - .reject(BasicRejectOptions { requeue: false }) - .await?; + delivery.reject(BasicRejectOptions { requeue: false }).await } } - - Ok(()) } async fn process_event( From efe34c4b4378d2e466838df589d5149cc8c50d8f Mon Sep 17 00:00:00 2001 From: rob Date: Wed, 8 May 2024 17:45:37 +0200 Subject: [PATCH 096/103] Simplified event_publisher retry --- src/indexer/event_publisher.rs | 7 ++++++- src/indexer/rmq_con.rs | 32 -------------------------------- 2 files changed, 6 insertions(+), 33 deletions(-) diff --git a/src/indexer/event_publisher.rs b/src/indexer/event_publisher.rs index 0881e807c1..506f4c3aa0 100644 --- a/src/indexer/event_publisher.rs +++ b/src/indexer/event_publisher.rs @@ -64,7 +64,12 @@ impl EventPublisher { log::error!("Error publishing message: {}. Attempting to recreate the RMQ channel. Retries left: {}. Retrying in {} seconds.", e, attempts, backoff_delay.as_secs()); sleep(backoff_delay).await; backoff_delay *= 2; - channel = setup_rabbitmq_connection(&addr).await?; + channel = setup_rabbitmq_connection(&addr) + .await + .unwrap_or_else(|conn_err| { + log::error!("Failed to recreate RabbitMQ channel: {}", conn_err); + channel + }); } else { break; } diff --git a/src/indexer/rmq_con.rs b/src/indexer/rmq_con.rs index 0bd7b491e3..043e8e3a1d 100644 --- a/src/indexer/rmq_con.rs +++ b/src/indexer/rmq_con.rs @@ -1,5 +1,3 @@ -use std::time::Duration; - use anyhow::{Context, Error}; use chrono::Utc; use lapin::options::ConfirmSelectOptions; @@ -7,38 +5,8 @@ use lapin::tcp::{AMQPUriTcpExt, NativeTlsConnector}; use lapin::uri::AMQPUri; use lapin::{Connection, ConnectionProperties}; use rand::distributions::{Alphanumeric, DistString}; -use tokio::time::sleep; pub async fn setup_rabbitmq_connection(addr: &str) -> Result { - let mut attempts = 10; - let mut backoff_delay = Duration::from_secs(1); - while attempts > 0 { - match create_channel(addr).await { - Ok(channel) => return Ok(channel), - Err(e) => { - attempts -= 1; - if attempts < 0 { - return Err(e); - } else { - log::error!( - "Error creating RabbitMQ channel: {}. Retries left: {}. Retrying in {} seconds.", - e, - attempts, - backoff_delay.as_secs() - ); - sleep(backoff_delay).await; - backoff_delay *= 2; - } - } - } - } - Err(anyhow::Error::new(std::io::Error::new( - std::io::ErrorKind::Other, - "Failed to establish RabbitMQ channel after retries", - ))) -} - -async fn create_channel(addr: &str) -> Result { let conn = connect_to_rabbitmq(addr).await?; let channel = conn .create_channel() From 655b085db4c8eb615b846098e16cf6aeba1f2cb4 Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Thu, 9 May 2024 09:54:01 +0700 Subject: [PATCH 097/103] Refactors Continued --- src/indexer/db_client.rs | 2 +- src/indexer/inscription_indexation.rs | 123 +++++++++++--------------- 2 files changed, 52 insertions(+), 73 deletions(-) diff --git a/src/indexer/db_client.rs b/src/indexer/db_client.rs index 1ad5a7853d..2ac1bb2475 100644 --- a/src/indexer/db_client.rs +++ b/src/indexer/db_client.rs @@ -130,7 +130,7 @@ impl DbClient { pub async fn fetch_inscription_id_by_genesis_id( &self, - genesis_id: String, + genesis_id: &String, ) -> Result, sqlx::Error> { sqlx::query!( r#"SELECT id FROM inscription WHERE genesis_id = $1"#, diff --git a/src/indexer/inscription_indexation.rs b/src/indexer/inscription_indexation.rs index f66a06c87d..7eda3f8ee3 100644 --- a/src/indexer/inscription_indexation.rs +++ b/src/indexer/inscription_indexation.rs @@ -1,3 +1,4 @@ +use anyhow::anyhow; use ciborium::from_reader; use ordinals::SatPoint; use serde_json::Value; @@ -24,8 +25,6 @@ impl InscriptionIndexation { } pub async fn sync_blocks(&self, block_height: &u32) -> Result<(), anyhow::Error> { - log::info!("Blocks committed event for={block_height}"); - let events = self.db.fetch_events_by_block_height(block_height).await?; let block_info = self.api.fetch_block_info(block_height).await?; @@ -40,9 +39,11 @@ impl InscriptionIndexation { .process_inscription_transferred(&event, &block_info) .await .inspect_err(|e| log::error!("error with inscription_transferred {:?}: {e}", event)), - }; + }?; } + log::info!("Blocks committed event for={block_height}"); + Ok(()) } @@ -53,15 +54,14 @@ impl InscriptionIndexation { ) -> Result<(), anyhow::Error> { let inscription_id = event.inscription_id.clone(); let inscription = self.api.fetch_inscription_details(inscription_id).await?; - let metadata = self.try_and_extract_metadata(&inscription.metadata); - let id = self.db.save_inscription(&inscription, metadata).await?; - - let mut to_location_details: Option<(String, u64)> = None; + let location = event.location.as_ref(); + let to_location = match location { + Some(loc) => Some(self.process_location(loc).await?), + None => None, + }; - if let Some(location) = &event.location { - to_location_details = self.process_location(location).await?; - } + let id = self.db.save_inscription(&inscription, metadata).await?; self .db @@ -69,20 +69,17 @@ impl InscriptionIndexation { id, event.block_height, block_info.timestamp, - event.location.as_ref().map(|loc| loc.outpoint.txid), - to_location_details - .as_ref() - .map(|details| details.0.clone()), - event.location.as_ref().map(|loc| loc.outpoint), - event.location.as_ref().map(|loc| loc.offset), + location.map(|loc| loc.outpoint.txid), + to_location.as_ref().map(|d| d.0.clone()), + location.map(|loc| loc.outpoint), + location.map(|loc| loc.offset), None, None, None, - to_location_details.as_ref().map(|details| details.1), + to_location.as_ref().map(|d| d.1), ) - .await?; - - Ok(()) + .await + .map_err(|err| anyhow!("error saving inscription_created location: {err}")) } fn try_and_extract_metadata(&self, metadata: &Option>) -> Option { @@ -107,24 +104,21 @@ impl InscriptionIndexation { ) -> Result<(), anyhow::Error> { let inscription_id = self .db - .fetch_inscription_id_by_genesis_id(event.inscription_id.clone()) + .fetch_inscription_id_by_genesis_id(&event.inscription_id) .await? - .ok_or_else(|| { - anyhow::anyhow!( - "No inscription found for genesis_id: {}", - event.inscription_id - ) - })?; - - let mut to_location_details: Option<(String, u64)> = None; - if let Some(location) = &event.location { - to_location_details = self.process_location(location).await?; - } + .ok_or_else(|| anyhow!("No inscription for genesis_id: {}", event.inscription_id))?; - let mut from_location_details: Option<(String, u64)> = None; - if let Some(location) = &event.old_location { - from_location_details = self.process_location(location).await?; - } + let location = event.location.as_ref(); + let old_location = event.location.as_ref(); + + let to_location = match &event.location { + Some(location) => Some(self.process_location(location).await?), + None => None, + }; + let from_location = match &event.old_location { + Some(location) => Some(self.process_location(location).await?), + None => None, + }; self .db @@ -132,50 +126,35 @@ impl InscriptionIndexation { inscription_id, event.block_height, block_info.timestamp, - event.location.as_ref().map(|loc| loc.outpoint.txid), - to_location_details - .as_ref() - .map(|details| details.0.clone()), - event.location.as_ref().map(|loc| loc.outpoint), - event.location.as_ref().map(|loc| loc.offset), - from_location_details - .as_ref() - .map(|details| details.0.clone()), - event.old_location.as_ref().map(|loc| loc.outpoint), - event.old_location.as_ref().map(|loc| loc.offset), - to_location_details.as_ref().map(|details| details.1), + location.map(|loc| loc.outpoint.txid), + to_location.as_ref().map(|d| d.0.clone()), + location.map(|loc| loc.outpoint), + location.map(|loc| loc.offset), + from_location.as_ref().map(|d| d.0.clone()), + old_location.map(|loc| loc.outpoint), + old_location.map(|loc| loc.offset), + to_location.as_ref().map(|d| d.1), ) - .await?; - - Ok(()) + .await + .map_err(|err| anyhow!("error saving inscription_transferred location: {err}")) } - async fn process_location( - &self, - location: &SatPoint, - ) -> Result, anyhow::Error> { + async fn process_location(&self, location: &SatPoint) -> Result<(String, u64), anyhow::Error> { let tx_details = self.api.fetch_tx(location.outpoint.txid).await?; let output = tx_details .transaction .output .into_iter() - .nth(location.outpoint.vout.try_into().unwrap()); - - let address = output - .as_ref() - .and_then(|o| { - self - .settings - .chain() - .address_from_script(&o.script_pubkey) - .ok() - }) - .map(|address| address.to_string()); - - match address { - Some(addr) => Ok(Some((addr, output.unwrap().value))), - None => Ok(None), - } + .nth(location.outpoint.vout.try_into()?) + .ok_or_else(|| anyhow!("output not found in tx: {}", location.outpoint.txid))?; + + let address = self + .settings + .chain() + .address_from_script(&output.script_pubkey)? + .to_string(); + + Ok((address, output.value)) } } From 8567670dc7826b6bb6b2b8cdca0af003b48c8a35 Mon Sep 17 00:00:00 2001 From: rob Date: Thu, 9 May 2024 09:49:12 +0200 Subject: [PATCH 098/103] Persist inscription on transfer if not exists --- src/indexer/api_client.rs | 2 +- src/indexer/inscription_indexation.rs | 20 ++++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/indexer/api_client.rs b/src/indexer/api_client.rs index 9c34dcb065..9ffa895d18 100644 --- a/src/indexer/api_client.rs +++ b/src/indexer/api_client.rs @@ -87,7 +87,7 @@ impl ApiClient { pub async fn fetch_inscription_details( &self, - inscription_id: String, + inscription_id: &String, ) -> Result { let request_builder = self .client diff --git a/src/indexer/inscription_indexation.rs b/src/indexer/inscription_indexation.rs index 8b498870d3..9a3920c586 100644 --- a/src/indexer/inscription_indexation.rs +++ b/src/indexer/inscription_indexation.rs @@ -55,8 +55,10 @@ impl InscriptionIndexation { event: &Event, block_info: &BlockInfo, ) -> Result<(), anyhow::Error> { - let inscription_id = event.inscription_id.clone(); - let inscription = self.api.fetch_inscription_details(inscription_id).await?; + let inscription = self + .api + .fetch_inscription_details(&event.inscription_id) + .await?; let metadata = self.try_and_extract_metadata(&inscription.metadata); let location = event.location.as_ref(); let to_location = match location { @@ -105,11 +107,21 @@ impl InscriptionIndexation { event: &Event, block_info: &BlockInfo, ) -> Result<(), anyhow::Error> { - let inscription_id = self + let inscription_id = match self .db .fetch_inscription_id_by_genesis_id(&event.inscription_id) .await? - .ok_or_else(|| anyhow!("No inscription for genesis_id: {}", event.inscription_id))?; + { + Some(id) => id, + None => { + let inscription = self + .api + .fetch_inscription_details(&event.inscription_id) + .await?; + let metadata = self.try_and_extract_metadata(&inscription.metadata); + self.db.save_inscription(&inscription, metadata).await? + } + }; let location = event.location.as_ref(); let old_location = event.location.as_ref(); From d5fb398d66fdb2a37a22a1ff4af278014758dec4 Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Thu, 9 May 2024 15:36:11 +0700 Subject: [PATCH 099/103] Merge with refactors --- src/indexer/block_consumer.rs | 2 +- src/indexer/event_publisher.rs | 69 +++++++++++++++++++--------------- 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/src/indexer/block_consumer.rs b/src/indexer/block_consumer.rs index 7cc9db8808..c90c0da10c 100644 --- a/src/indexer/block_consumer.rs +++ b/src/indexer/block_consumer.rs @@ -62,7 +62,7 @@ impl BlockConsumer { }) } - /// Handle the persistence of incoming queue "event" messages. + /// Handle the persistence of incoming queue "block" messages. /// /// Re-enqueues the message up to `max_delivery` times for processing failures. /// Bubbles up the `lapin::Error` only if the ack/nack/reject itself fails. diff --git a/src/indexer/event_publisher.rs b/src/indexer/event_publisher.rs index 506f4c3aa0..ef826fd496 100644 --- a/src/indexer/event_publisher.rs +++ b/src/indexer/event_publisher.rs @@ -1,4 +1,4 @@ -use anyhow::{Context, Result}; +use anyhow::{anyhow, Context, Result}; use lapin::{options::BasicPublishOptions, BasicProperties, Channel}; use std::time::Duration; use tokio::runtime::Runtime; @@ -15,7 +15,7 @@ pub struct EventPublisher { } impl EventPublisher { - pub fn run(settings: &Settings) -> Result { + pub fn run(settings: &Settings) -> Result { let addr = settings .rabbitmq_addr() .context("rabbitmq amqp credentials and url must be defined")? @@ -43,6 +43,11 @@ impl EventPublisher { Ok(EventPublisher { sender: tx }) } + /// Publishes an event to the message exchange. + /// On failures it will retry up to `n` times with exponential backoff. + /// After `n` attempts, the retry loop will break and the main loop continues. + /// + /// TODO: Consider how to handle failure without skipping. async fn consume_channel( addr: String, exchange: String, @@ -54,34 +59,39 @@ impl EventPublisher { let message = serde_json::to_vec(&event)?; let routing_key = EventPublisher::type_name(&event); - let mut attempts = 10; let mut backoff_delay = Duration::from_secs(1); - while attempts > 0 { - let result = - EventPublisher::publish_message(&channel, &exchange, routing_key, &message).await; - if let Err(e) = result { - attempts -= 1; - log::error!("Error publishing message: {}. Attempting to recreate the RMQ channel. Retries left: {}. Retrying in {} seconds.", e, attempts, backoff_delay.as_secs()); - sleep(backoff_delay).await; - backoff_delay *= 2; - channel = setup_rabbitmq_connection(&addr) - .await - .unwrap_or_else(|conn_err| { - log::error!("Failed to recreate RabbitMQ channel: {}", conn_err); - channel - }); - } else { - break; + let mut max_attempts = 8; + + let publish_retried = loop { + match EventPublisher::publish_message(&channel, &exchange, routing_key, &message).await { + Ok(value) => break Ok(value), + Err(e) => { + if max_attempts == 0 { + break Err(e); + } + + max_attempts -= 1; + + log::error!( + "Error: {e}. Retries left: {max_attempts}. Retrying in {}s.", + backoff_delay.as_secs() + ); + + sleep(backoff_delay).await; + + channel = setup_rabbitmq_connection(&addr) + .await + .inspect_err(|e| log::error!("error reconnecting rmq: {e}")) + .unwrap_or_else(|_| channel); + + backoff_delay *= 2; + } } - } - - if attempts == 0 { - return Err(anyhow::Error::new(std::io::Error::new( - std::io::ErrorKind::Other, - "Failed to publish message after retries", - ))); - } + }; + + publish_retried? } + Ok(()) } @@ -103,10 +113,7 @@ impl EventPublisher { .await?; if !publish.is_ack() { - return Err(anyhow::Error::new(std::io::Error::new( - std::io::ErrorKind::Other, - "Message was not acknowledged", - ))); + return Err(anyhow!("message was not acknowledged")); } Ok(()) From ff2661c2d45defd1d3ce9df0400ebc2a266ff010 Mon Sep 17 00:00:00 2001 From: rob Date: Thu, 9 May 2024 10:41:10 +0200 Subject: [PATCH 100/103] Set prefetch --- src/indexer/block_consumer.rs | 4 ++++ src/indexer/event_consumer.rs | 4 ++++ src/indexer/event_publisher.rs | 5 +++++ 3 files changed, 13 insertions(+) diff --git a/src/indexer/block_consumer.rs b/src/indexer/block_consumer.rs index e7b9416b5f..02c80439a9 100644 --- a/src/indexer/block_consumer.rs +++ b/src/indexer/block_consumer.rs @@ -37,6 +37,10 @@ impl BlockConsumer { let addr = settings.rabbitmq_addr().context("rmq url is required")?; let queue_name = self.blocks_queue.context("rmq queue is required")?; let channel = setup_rabbitmq_connection(addr).await?; + channel + .basic_qos(2, BasicQosOptions::default()) + .await + .context("Failed to set basic_qos")?; let inscription_indexer = InscriptionIndexation::new(settings, db, api_client); diff --git a/src/indexer/event_consumer.rs b/src/indexer/event_consumer.rs index d715bec500..de7f255744 100644 --- a/src/indexer/event_consumer.rs +++ b/src/indexer/event_consumer.rs @@ -28,6 +28,10 @@ impl EventConsumer { let addr = settings.rabbitmq_addr().context("rmq url is required")?; let queue_name = self.inscriptions_queue.context("rmq queue is required")?; let channel = setup_rabbitmq_connection(addr).await?; + channel + .basic_qos(2, BasicQosOptions::default()) + .await + .context("Failed to set basic_qos")?; let mut consumer = channel .basic_consume( diff --git a/src/indexer/event_publisher.rs b/src/indexer/event_publisher.rs index 6a28946736..ec357fdf43 100644 --- a/src/indexer/event_publisher.rs +++ b/src/indexer/event_publisher.rs @@ -1,4 +1,5 @@ use anyhow::{Context, Result}; +use lapin::options::BasicQosOptions; use lapin::{options::BasicPublishOptions, BasicProperties}; use tokio::runtime::Runtime; use tokio::sync::mpsc; @@ -47,6 +48,10 @@ impl EventPublisher { mut rx: mpsc::Receiver, ) -> Result<()> { let channel = setup_rabbitmq_connection(&addr).await?; + channel + .basic_qos(2, BasicQosOptions::default()) + .await + .context("Failed to set basic_qos")?; while let Some(event) = rx.recv().await { let message = serde_json::to_vec(&event)?; From b468fcab53defab68ef5efcc340b0fb44df29498 Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Thu, 9 May 2024 15:44:25 +0700 Subject: [PATCH 101/103] Lint --- src/indexer/block_consumer.rs | 2 +- src/indexer/event_consumer.rs | 2 +- src/indexer/event_publisher.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/indexer/block_consumer.rs b/src/indexer/block_consumer.rs index c90c0da10c..53e499dd40 100644 --- a/src/indexer/block_consumer.rs +++ b/src/indexer/block_consumer.rs @@ -91,7 +91,7 @@ impl BlockConsumer { let event = serde_json::from_slice::(&delivery.data).context("should deserialize evt"); if let Ok(ref e) = event { - if let Ok(_) = BlockConsumer::process_event(e, indexer).await { + if BlockConsumer::process_event(e, indexer).await.is_ok() { delivery.ack(BasicAckOptions::default()).await? }; }; diff --git a/src/indexer/event_consumer.rs b/src/indexer/event_consumer.rs index 07b30035a5..c2daded4c7 100644 --- a/src/indexer/event_consumer.rs +++ b/src/indexer/event_consumer.rs @@ -77,7 +77,7 @@ impl EventConsumer { let event = serde_json::from_slice::(&delivery.data).context("should deserialize evt"); if let Ok(ref e) = event { - if let Ok(_) = EventConsumer::process_event(e, db).await { + if EventConsumer::process_event(e, db).await.is_ok() { delivery.ack(BasicAckOptions::default()).await? }; }; diff --git a/src/indexer/event_publisher.rs b/src/indexer/event_publisher.rs index ef826fd496..b1f5d29c9e 100644 --- a/src/indexer/event_publisher.rs +++ b/src/indexer/event_publisher.rs @@ -82,7 +82,7 @@ impl EventPublisher { channel = setup_rabbitmq_connection(&addr) .await .inspect_err(|e| log::error!("error reconnecting rmq: {e}")) - .unwrap_or_else(|_| channel); + .unwrap_or(channel); backoff_delay *= 2; } From 600bbf9e251389f0154668e9aae08c6fb799e7ce Mon Sep 17 00:00:00 2001 From: RareBodhi Date: Thu, 9 May 2024 15:58:02 +0700 Subject: [PATCH 102/103] Merge --- src/indexer/event_publisher.rs | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/indexer/event_publisher.rs b/src/indexer/event_publisher.rs index 0918a213b1..b41cc3a01e 100644 --- a/src/indexer/event_publisher.rs +++ b/src/indexer/event_publisher.rs @@ -1,12 +1,9 @@ -<<<<<<< HEAD use anyhow::{anyhow, Context, Result}; -use lapin::{options::BasicPublishOptions, BasicProperties, Channel}; +use lapin::{ + options::{BasicPublishOptions, BasicQosOptions}, + BasicProperties, Channel, +}; use std::time::Duration; -======= -use anyhow::{Context, Result}; -use lapin::options::BasicQosOptions; -use lapin::{options::BasicPublishOptions, BasicProperties}; ->>>>>>> ff2661c2d45defd1d3ce9df0400ebc2a266ff010 use tokio::runtime::Runtime; use tokio::sync::mpsc; use tokio::time::sleep; @@ -16,6 +13,15 @@ use crate::indexer::rmq_con::setup_rabbitmq_connection; use crate::settings::Settings; use crate::shutdown_process; +async fn rabbit_qos_setup(channel: Channel) -> Result { + channel + .basic_qos(2, BasicQosOptions::default()) + .await + .context("failed to set basic qos")?; + + Ok(channel) +} + pub struct EventPublisher { pub(crate) sender: mpsc::Sender, } @@ -60,11 +66,7 @@ impl EventPublisher { mut rx: mpsc::Receiver, ) -> Result<()> { let channel = setup_rabbitmq_connection(&addr).await?; - - channel - .basic_qos(2, BasicQosOptions::default()) - .await - .context("failed to set basic qos")?; + let mut channel = rabbit_qos_setup(channel).await?; while let Some(event) = rx.recv().await { let message = serde_json::to_vec(&event)?; @@ -90,7 +92,7 @@ impl EventPublisher { sleep(backoff_delay).await; - channel = setup_rabbitmq_connection(&addr) + channel = rabbit_qos_setup(setup_rabbitmq_connection(&addr).await?) .await .inspect_err(|e| log::error!("error reconnecting rmq: {e}")) .unwrap_or(channel); From 7a353a0a0b22faf88a4ac5dbbeec74a53356d323 Mon Sep 17 00:00:00 2001 From: rob Date: Thu, 9 May 2024 12:36:17 +0200 Subject: [PATCH 103/103] Revert quorom queue and roll own retry --- rmqconfig.json | 8 ++------ src/indexer/block_consumer.rs | 29 +++++++++++++++-------------- src/indexer/event_consumer.rs | 34 +++++++++++++++++++--------------- src/indexer/rmq_con.rs | 35 +++++++++++++++++++++++++++++++++-- 4 files changed, 69 insertions(+), 37 deletions(-) diff --git a/rmqconfig.json b/rmqconfig.json index 773d7b60a3..de454be89a 100644 --- a/rmqconfig.json +++ b/rmqconfig.json @@ -5,18 +5,14 @@ "vhost": "/", "durable": true, "auto_delete": false, - "arguments": { - "x-queue-type": "quorum" - } + "arguments": {} }, { "name": "btc-blocks-q", "vhost": "/", "durable": true, "auto_delete": false, - "arguments": { - "x-queue-type": "quorum" - } + "arguments": {} } ], "exchanges": [ diff --git a/src/indexer/block_consumer.rs b/src/indexer/block_consumer.rs index 9df8b089f8..31a4567440 100644 --- a/src/indexer/block_consumer.rs +++ b/src/indexer/block_consumer.rs @@ -1,14 +1,16 @@ use anyhow::Context; use clap::Parser; use futures::StreamExt; -use lapin::{message::Delivery, options::*, types::FieldTable}; +use lapin::{message::Delivery, options::*, types::FieldTable, Channel}; use tokio::runtime::Runtime; use crate::index::event::Event; use crate::indexer::api_client::ApiClient; use crate::indexer::db_client::DbClient; use crate::indexer::inscription_indexation::InscriptionIndexation; -use crate::indexer::rmq_con::{generate_consumer_tag, setup_rabbitmq_connection}; +use crate::indexer::rmq_con::{ + generate_consumer_tag, republish_to_queue, setup_rabbitmq_connection, +}; use crate::settings::Settings; use crate::subcommand::SubcommandResult; @@ -58,7 +60,7 @@ impl BlockConsumer { loop { if let Some(msg) = consumer.next().await { match msg { - Ok(d) => BlockConsumer::handle_delivery(d, &inscription_indexer).await?, + Ok(d) => BlockConsumer::handle_delivery(&channel, d, &inscription_indexer).await?, Err(err) => log::error!("error consuming message: {err}"), } } @@ -69,17 +71,14 @@ impl BlockConsumer { /// Handle the persistence of incoming queue "block" messages. /// /// Re-enqueues the message up to `max_delivery` times for processing failures. - /// Bubbles up the `lapin::Error` only if the ack/nack/reject itself fails. + /// Bubbles up the `lapin::Error` only if the ack/reject itself fails. async fn handle_delivery( + channel: &Channel, delivery: Delivery, indexer: &InscriptionIndexation, ) -> Result<(), lapin::Error> { let max_delivery = 3; let reject = BasicRejectOptions { requeue: false }; - let requeue = BasicNackOptions { - multiple: false, - requeue: true, - }; let delivery_count = delivery .properties @@ -88,20 +87,22 @@ impl BlockConsumer { .and_then(|h| h.inner().get("x-delivery-count")?.as_short_uint()) .unwrap_or(0); + let event = serde_json::from_slice::(&delivery.data).context("should deserialize evt"); + if delivery_count > max_delivery { - delivery.reject(reject).await? + log::error!("failed event dropped {:?}", event); + return delivery.reject(reject).await; } - let event = serde_json::from_slice::(&delivery.data).context("should deserialize evt"); - if let Ok(ref e) = event { if BlockConsumer::process_event(e, indexer).await.is_ok() { - delivery.ack(BasicAckOptions::default()).await? + return delivery.ack(BasicAckOptions::default()).await; }; }; - log::error!("failed event requeued {:?}", event); - delivery.nack(requeue).await + log::warn!("failed event requeued {:?}", event); + republish_to_queue(channel, &delivery, &delivery_count).await?; + delivery.reject(reject).await } async fn process_event( diff --git a/src/indexer/event_consumer.rs b/src/indexer/event_consumer.rs index 737bf957f3..b3380f7f41 100644 --- a/src/indexer/event_consumer.rs +++ b/src/indexer/event_consumer.rs @@ -1,12 +1,14 @@ use anyhow::Context; use clap::Parser; use futures::StreamExt; -use lapin::{message::Delivery, options::*, types::FieldTable}; +use lapin::{message::Delivery, options::*, types::FieldTable, Channel}; use tokio::runtime::Runtime; use crate::index::event::Event; use crate::indexer::db_client::DbClient; -use crate::indexer::rmq_con::{generate_consumer_tag, setup_rabbitmq_connection}; +use crate::indexer::rmq_con::{ + generate_consumer_tag, republish_to_queue, setup_rabbitmq_connection, +}; use crate::settings::Settings; use crate::subcommand::SubcommandResult; @@ -47,7 +49,7 @@ impl EventConsumer { loop { if let Some(msg) = consumer.next().await { match msg { - Ok(d) => EventConsumer::handle_delivery(d, &db).await?, + Ok(d) => EventConsumer::handle_delivery(&channel, d, &db).await?, Err(e) => log::error!("error consuming message: {}", e), } } @@ -58,14 +60,14 @@ impl EventConsumer { /// Handle the persistence of incoming queue "event" messages. /// /// Re-enqueues the message up to `max_delivery` times for processing failures. - /// Bubbles up the `lapin::Error` only if the ack/nack/reject itself fails. - async fn handle_delivery(delivery: Delivery, db: &DbClient) -> Result<(), lapin::Error> { + /// Bubbles up the `lapin::Error` only if the ack/reject itself fails. + async fn handle_delivery( + channel: &Channel, + delivery: Delivery, + db: &DbClient, + ) -> Result<(), lapin::Error> { let max_delivery = 3; let reject = BasicRejectOptions { requeue: false }; - let requeue = BasicNackOptions { - multiple: false, - requeue: true, - }; let delivery_count = delivery .properties @@ -74,20 +76,22 @@ impl EventConsumer { .and_then(|h| h.inner().get("x-delivery-count")?.as_short_uint()) .unwrap_or(0); + let event = serde_json::from_slice::(&delivery.data).context("should deserialize evt"); + if delivery_count > max_delivery { - delivery.reject(reject).await? + log::error!("failed event dropped {:?}", event); + return delivery.reject(reject).await; } - let event = serde_json::from_slice::(&delivery.data).context("should deserialize evt"); - if let Ok(ref e) = event { if EventConsumer::process_event(e, db).await.is_ok() { - delivery.ack(BasicAckOptions::default()).await? + return delivery.ack(BasicAckOptions::default()).await; }; }; - log::error!("failed event requeued {:?}", event); - delivery.nack(requeue).await + log::warn!("failed event requeued {:?}", event); + republish_to_queue(channel, &delivery, &delivery_count).await?; + delivery.reject(reject).await } async fn process_event(event: &Event, db: &DbClient) -> Result<(), sqlx::Error> { diff --git a/src/indexer/rmq_con.rs b/src/indexer/rmq_con.rs index 043e8e3a1d..6dde40403c 100644 --- a/src/indexer/rmq_con.rs +++ b/src/indexer/rmq_con.rs @@ -1,9 +1,12 @@ use anyhow::{Context, Error}; use chrono::Utc; -use lapin::options::ConfirmSelectOptions; +use lapin::message::Delivery; +use lapin::options::{BasicPublishOptions, ConfirmSelectOptions}; +use lapin::publisher_confirm::Confirmation; use lapin::tcp::{AMQPUriTcpExt, NativeTlsConnector}; +use lapin::types::{FieldTable, ShortUInt}; use lapin::uri::AMQPUri; -use lapin::{Connection, ConnectionProperties}; +use lapin::{BasicProperties, Channel, Connection, ConnectionProperties}; use rand::distributions::{Alphanumeric, DistString}; pub async fn setup_rabbitmq_connection(addr: &str) -> Result { @@ -58,3 +61,31 @@ pub fn generate_consumer_tag(prefix: &str) -> String { Alphanumeric.sample_string(&mut rand::thread_rng(), 16) ) } + +pub async fn republish_to_queue( + channel: &Channel, + delivery: &Delivery, + delivery_count: &ShortUInt, +) -> lapin::Result { + let mut new_headers = delivery + .properties + .headers() + .as_ref() + .cloned() + .unwrap_or_else(FieldTable::default); + new_headers.insert( + "x-delivery-count".into(), + ShortUInt::from(delivery_count + 1).into(), + ); + + channel + .basic_publish( + "", + delivery.routing_key.as_str(), + BasicPublishOptions::default(), + &delivery.data, + BasicProperties::default().with_headers(new_headers), + ) + .await? + .await +}