diff --git a/Cargo.lock b/Cargo.lock index a4d2005b6..29012132f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -830,7 +830,7 @@ dependencies = [ "alloy-sol-types", "anyhow", "bincode", - "hyperware_process_lib 2.3.0", + "hyperware_process_lib 3.0.1", "process_macros", "rand 0.8.5", "serde", @@ -1195,7 +1195,7 @@ dependencies = [ "anyhow", "chrono", "hex", - "hyperware_process_lib 2.3.0", + "hyperware_process_lib 3.0.1", "process_macros", "rand 0.8.5", "rmp-serde", @@ -1546,7 +1546,7 @@ dependencies = [ "anyhow", "bincode", "hex", - "hyperware_process_lib 2.2.1", + "hyperware_process_lib 3.0.1", "process_macros", "rand 0.8.5", "serde", @@ -2477,7 +2477,7 @@ name = "download" version = "0.1.0" dependencies = [ "anyhow", - "hyperware_process_lib 2.3.0", + "hyperware_process_lib 3.0.1", "process_macros", "serde", "serde_json", @@ -2489,7 +2489,7 @@ name = "downloads" version = "0.5.0" dependencies = [ "anyhow", - "hyperware_process_lib 2.3.0", + "hyperware_process_lib 3.0.1", "process_macros", "rand 0.8.5", "serde", @@ -2673,6 +2673,7 @@ name = "explorer" version = "0.1.0" dependencies = [ "anyhow", + "file_explorer_caller_utils", "hyperapp_macro 0.1.1", "hyperware_process_lib 2.2.0 (git+https://github.com/hyperware-ai/process_lib?rev=1a6ad9d)", "md5", @@ -2770,6 +2771,22 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "file_explorer_caller_utils" +version = "0.1.0" +dependencies = [ + "anyhow", + "futures", + "futures-util", + "hyperware_process_lib 2.2.0 (git+https://github.com/hyperware-ai/process_lib?rev=1a6ad9d)", + "once_cell", + "process_macros", + "serde", + "serde_json", + "uuid 1.17.0", + "wit-bindgen 0.41.0", +] + [[package]] name = "fixed-hash" version = "0.8.0" @@ -2855,7 +2872,7 @@ version = "0.2.0" dependencies = [ "anyhow", "bincode", - "hyperware_process_lib 2.3.0", + "hyperware_process_lib 3.0.1", "process_macros", "rand 0.8.5", "serde", @@ -3031,7 +3048,7 @@ dependencies = [ name = "get-block" version = "0.1.0" dependencies = [ - "hyperware_process_lib 2.3.0", + "hyperware_process_lib 3.0.1", "serde", "serde_json", "wit-bindgen 0.42.1", @@ -3343,7 +3360,7 @@ dependencies = [ "alloy-sol-types", "anyhow", "hex", - "hyperware_process_lib 2.2.1", + "hyperware_process_lib 3.0.1", "process_macros", "rmp-serde", "serde", @@ -3648,7 +3665,7 @@ dependencies = [ "anyhow", "chrono", "hex", - "hyperware_process_lib 2.3.0", + "hyperware_process_lib 3.0.1", "process_macros", "rand 0.8.5", "rmp-serde", @@ -3744,8 +3761,9 @@ dependencies = [ [[package]] name = "hyperware_process_lib" -version = "2.2.1" -source = "git+https://github.com/hyperware-ai/process_lib?rev=4e91521#4e915218b382e4b3f7f93cddeec30415f02a504b" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe2b6795488c3f7d3410808cf7e29404725f862ad42a3822886464b92451a21" dependencies = [ "alloy", "alloy-primitives", @@ -3755,6 +3773,8 @@ dependencies = [ "base64 0.22.1", "bincode", "color-eyre", + "futures-channel", + "futures-util", "http 1.3.1", "mime_guess", "rand 0.8.5", @@ -3767,14 +3787,14 @@ dependencies = [ "tracing-error", "tracing-subscriber", "url", + "uuid 1.17.0", "wit-bindgen 0.42.1", ] [[package]] name = "hyperware_process_lib" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe2b6795488c3f7d3410808cf7e29404725f862ad42a3822886464b92451a21" +version = "3.0.1" +source = "git+https://github.com/hyperware-ai/process_lib?rev=34b1f1d857f76fb474f7af96f089961445758edb#34b1f1d857f76fb474f7af96f089961445758edb" dependencies = [ "alloy", "alloy-primitives", @@ -3784,8 +3804,6 @@ dependencies = [ "base64 0.22.1", "bincode", "color-eyre", - "futures-channel", - "futures-util", "http 1.3.1", "mime_guess", "rand 0.8.5", @@ -3798,7 +3816,6 @@ dependencies = [ "tracing-error", "tracing-subscriber", "url", - "uuid 1.17.0", "wit-bindgen 0.42.1", ] @@ -4030,7 +4047,7 @@ name = "install" version = "0.1.0" dependencies = [ "anyhow", - "hyperware_process_lib 2.3.0", + "hyperware_process_lib 3.0.1", "process_macros", "serde", "serde_json", @@ -4803,7 +4820,7 @@ dependencies = [ name = "node-info" version = "0.1.0" dependencies = [ - "hyperware_process_lib 2.3.0", + "hyperware_process_lib 3.0.1", "process_macros", "serde", "serde_json", @@ -5841,7 +5858,7 @@ dependencies = [ name = "reset" version = "0.1.0" dependencies = [ - "hyperware_process_lib 2.3.0", + "hyperware_process_lib 3.0.1", "process_macros", "serde", "serde_json", @@ -5852,7 +5869,7 @@ dependencies = [ name = "reset-cache" version = "0.1.0" dependencies = [ - "hyperware_process_lib 2.3.0", + "hyperware_process_lib 3.0.1", "process_macros", "serde", "serde_json", @@ -5864,7 +5881,7 @@ name = "reset-store" version = "0.1.0" dependencies = [ "anyhow", - "hyperware_process_lib 2.3.0", + "hyperware_process_lib 3.0.1", "process_macros", "serde", "serde_json", @@ -6378,7 +6395,7 @@ dependencies = [ name = "set-nodes" version = "0.1.0" dependencies = [ - "hyperware_process_lib 2.3.0", + "hyperware_process_lib 3.0.1", "process_macros", "serde", "serde_json", @@ -6623,12 +6640,29 @@ dependencies = [ "serde", "serde_json", "sha2", + "spider_caller_utils", "url", "uuid 1.17.0", "wit-bindgen 0.42.1", "wit-parser 0.220.1", ] +[[package]] +name = "spider_caller_utils" +version = "0.1.0" +dependencies = [ + "anyhow", + "futures", + "futures-util", + "hyperware_process_lib 2.2.0 (git+https://github.com/hyperware-ai/process_lib?rev=41f25ce)", + "once_cell", + "process_macros", + "serde", + "serde_json", + "uuid 1.17.0", + "wit-bindgen 0.41.0", +] + [[package]] name = "spin" version = "0.9.8" @@ -6671,7 +6705,7 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" name = "start-providing" version = "0.1.0" dependencies = [ - "hyperware_process_lib 2.3.0", + "hyperware_process_lib 3.0.1", "process_macros", "serde", "serde_json", @@ -6682,7 +6716,7 @@ dependencies = [ name = "state" version = "0.1.0" dependencies = [ - "hyperware_process_lib 2.3.0", + "hyperware_process_lib 3.0.1", "process_macros", "serde", "serde_json", @@ -6716,7 +6750,7 @@ dependencies = [ name = "stop-providing" version = "0.1.0" dependencies = [ - "hyperware_process_lib 2.3.0", + "hyperware_process_lib 3.0.1", "process_macros", "serde", "serde_json", @@ -7569,7 +7603,7 @@ name = "uninstall" version = "0.1.0" dependencies = [ "anyhow", - "hyperware_process_lib 2.3.0", + "hyperware_process_lib 3.0.1", "process_macros", "serde", "serde_json", diff --git a/hyperdrive/Cargo.toml b/hyperdrive/Cargo.toml index 3e8a9362f..a798db974 100644 --- a/hyperdrive/Cargo.toml +++ b/hyperdrive/Cargo.toml @@ -67,7 +67,7 @@ p256 = { version = "0.13", features = ["ecdsa"] } public-ip = "0.2.2" rand = "0.8.4" regex = "1.11.0" -reqwest = "0.12.4" +reqwest = { version = "0.12.4", features = ["json"] } ring = "0.17.8" rmp-serde = "1.1.2" rocksdb = { version = "0.22.0", features = ["multi-threaded-cf"] } diff --git a/hyperdrive/packages/app-store/Cargo.lock b/hyperdrive/packages/app-store/Cargo.lock index a5db5f05d..730674f6c 100644 --- a/hyperdrive/packages/app-store/Cargo.lock +++ b/hyperdrive/packages/app-store/Cargo.lock @@ -585,7 +585,7 @@ dependencies = [ "alloy-sol-types", "anyhow", "bincode", - "hyperware_process_lib 2.1.0", + "hyperware_process_lib", "process_macros", "rand", "serde", @@ -952,7 +952,7 @@ dependencies = [ "anyhow", "bincode", "hex", - "hyperware_process_lib 2.2.1", + "hyperware_process_lib", "process_macros", "rand", "serde", @@ -1156,7 +1156,7 @@ name = "download" version = "0.1.0" dependencies = [ "anyhow", - "hyperware_process_lib 2.1.0", + "hyperware_process_lib", "process_macros", "serde", "serde_json", @@ -1168,7 +1168,7 @@ name = "downloads" version = "0.5.0" dependencies = [ "anyhow", - "hyperware_process_lib 2.1.0", + "hyperware_process_lib", "process_macros", "rand", "serde", @@ -1344,7 +1344,7 @@ version = "0.2.0" dependencies = [ "anyhow", "bincode", - "hyperware_process_lib 2.1.0", + "hyperware_process_lib", "process_macros", "rand", "serde", @@ -1656,35 +1656,8 @@ dependencies = [ [[package]] name = "hyperware_process_lib" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3abd008d22c3b96ee43300c4c8dffbf1d072a680a13635b5f9da11a0ce9395" -dependencies = [ - "alloy", - "alloy-primitives", - "alloy-sol-macro", - "alloy-sol-types", - "anyhow", - "base64", - "bincode", - "hex", - "http", - "mime_guess", - "rand", - "regex", - "rmp-serde", - "serde", - "serde_json", - "sha3", - "thiserror 1.0.69", - "url", - "wit-bindgen", -] - -[[package]] -name = "hyperware_process_lib" -version = "2.2.1" -source = "git+https://github.com/hyperware-ai/process_lib?rev=4e91521#4e915218b382e4b3f7f93cddeec30415f02a504b" +version = "3.0.1" +source = "git+https://github.com/hyperware-ai/process_lib?rev=34b1f1d857f76fb474f7af96f089961445758edb#34b1f1d857f76fb474f7af96f089961445758edb" dependencies = [ "alloy", "alloy-primitives", @@ -1886,7 +1859,7 @@ name = "install" version = "0.1.0" dependencies = [ "anyhow", - "hyperware_process_lib 2.1.0", + "hyperware_process_lib", "process_macros", "serde", "serde_json", @@ -2580,7 +2553,7 @@ name = "reset-store" version = "0.1.0" dependencies = [ "anyhow", - "hyperware_process_lib 2.1.0", + "hyperware_process_lib", "process_macros", "serde", "serde_json", @@ -3341,7 +3314,7 @@ name = "uninstall" version = "0.1.0" dependencies = [ "anyhow", - "hyperware_process_lib 2.1.0", + "hyperware_process_lib", "process_macros", "serde", "serde_json", diff --git a/hyperdrive/packages/app-store/app-store/Cargo.toml b/hyperdrive/packages/app-store/app-store/Cargo.toml index a2a2be749..f830c7c91 100644 --- a/hyperdrive/packages/app-store/app-store/Cargo.toml +++ b/hyperdrive/packages/app-store/app-store/Cargo.toml @@ -11,7 +11,7 @@ alloy-primitives = "0.8.15" alloy-sol-types = "0.8.15" anyhow = "1.0" bincode = "1.3.3" -hyperware_process_lib = "2.1.0" +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "34b1f1d857f76fb474f7af96f089961445758edb" } process_macros = "0.1" rand = "0.8" serde = { version = "1.0", features = ["derive"] } diff --git a/hyperdrive/packages/app-store/app-store/src/lib.rs b/hyperdrive/packages/app-store/app-store/src/lib.rs index b465f2e05..fc2db616e 100644 --- a/hyperdrive/packages/app-store/app-store/src/lib.rs +++ b/hyperdrive/packages/app-store/app-store/src/lib.rs @@ -147,8 +147,13 @@ fn handle_message( response.send().unwrap(); } } - http::server::HttpServerRequest::WebSocketOpen { path, channel_id } => { - http_server.handle_websocket_open(&path, channel_id); + http::server::HttpServerRequest::WebSocketOpen { + path, + channel_id, + source_socket_addr, + forwarded_for, + } => { + http_server.handle_websocket_open(&path, channel_id, source_socket_addr, forwarded_for); } http::server::HttpServerRequest::WebSocketClose(channel_id) => { http_server.handle_websocket_close(channel_id); diff --git a/hyperdrive/packages/app-store/chain/Cargo.toml b/hyperdrive/packages/app-store/chain/Cargo.toml index fcd1b2904..562dba6d9 100644 --- a/hyperdrive/packages/app-store/chain/Cargo.toml +++ b/hyperdrive/packages/app-store/chain/Cargo.toml @@ -12,7 +12,7 @@ alloy-sol-types = "0.8.15" anyhow = "1.0" hex = "0.4" bincode = "1.3.3" -hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "4e91521" } +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "34b1f1d857f76fb474f7af96f089961445758edb" } process_macros = "0.1" rand = "0.8" serde = { version = "1.0", features = ["derive"] } diff --git a/hyperdrive/packages/app-store/download/Cargo.toml b/hyperdrive/packages/app-store/download/Cargo.toml index 1744ce7bf..3d624d433 100644 --- a/hyperdrive/packages/app-store/download/Cargo.toml +++ b/hyperdrive/packages/app-store/download/Cargo.toml @@ -8,7 +8,7 @@ simulation-mode = [] [dependencies] anyhow = "1.0" -hyperware_process_lib = "2.1.0" +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "34b1f1d857f76fb474f7af96f089961445758edb" } process_macros = "0.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/hyperdrive/packages/app-store/downloads/Cargo.toml b/hyperdrive/packages/app-store/downloads/Cargo.toml index 6f90c7c05..f75f17555 100644 --- a/hyperdrive/packages/app-store/downloads/Cargo.toml +++ b/hyperdrive/packages/app-store/downloads/Cargo.toml @@ -8,7 +8,7 @@ simulation-mode = [] [dependencies] anyhow = "1.0" -hyperware_process_lib = "2.1.0" +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "34b1f1d857f76fb474f7af96f089961445758edb" } process_macros = "0.1" rand = "0.8" serde = { version = "1.0", features = ["derive"] } diff --git a/hyperdrive/packages/app-store/ft-worker/Cargo.toml b/hyperdrive/packages/app-store/ft-worker/Cargo.toml index 9f6ea88d3..1b1c8326f 100644 --- a/hyperdrive/packages/app-store/ft-worker/Cargo.toml +++ b/hyperdrive/packages/app-store/ft-worker/Cargo.toml @@ -9,7 +9,7 @@ simulation-mode = [] [dependencies] anyhow = "1.0" bincode = "1.3.3" -hyperware_process_lib = "2.1.0" +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "34b1f1d857f76fb474f7af96f089961445758edb" } process_macros = "0.1" rand = "0.8" serde = { version = "1.0", features = ["derive"] } diff --git a/hyperdrive/packages/app-store/install/Cargo.toml b/hyperdrive/packages/app-store/install/Cargo.toml index 1320b5161..4af498c94 100644 --- a/hyperdrive/packages/app-store/install/Cargo.toml +++ b/hyperdrive/packages/app-store/install/Cargo.toml @@ -8,7 +8,7 @@ simulation-mode = [] [dependencies] anyhow = "1.0" -hyperware_process_lib = "2.1.0" +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "34b1f1d857f76fb474f7af96f089961445758edb" } process_macros = "0.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/hyperdrive/packages/app-store/reset-store/Cargo.toml b/hyperdrive/packages/app-store/reset-store/Cargo.toml index 754ae83cb..6ca79c9eb 100644 --- a/hyperdrive/packages/app-store/reset-store/Cargo.toml +++ b/hyperdrive/packages/app-store/reset-store/Cargo.toml @@ -8,7 +8,7 @@ simulation-mode = [] [dependencies] anyhow = "1.0" -hyperware_process_lib = "2.1.0" +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "34b1f1d857f76fb474f7af96f089961445758edb" } process_macros = "0.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/hyperdrive/packages/app-store/uninstall/Cargo.toml b/hyperdrive/packages/app-store/uninstall/Cargo.toml index 8bbd3559b..64c3fccd2 100644 --- a/hyperdrive/packages/app-store/uninstall/Cargo.toml +++ b/hyperdrive/packages/app-store/uninstall/Cargo.toml @@ -8,7 +8,7 @@ simulation-mode = [] [dependencies] anyhow = "1.0" -hyperware_process_lib = "2.1.0" +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "34b1f1d857f76fb474f7af96f089961445758edb" } process_macros = "0.1" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/hyperdrive/packages/file-explorer/Cargo.lock b/hyperdrive/packages/file-explorer/Cargo.lock index 16213b04f..b83ada160 100644 --- a/hyperdrive/packages/file-explorer/Cargo.lock +++ b/hyperdrive/packages/file-explorer/Cargo.lock @@ -1240,9 +1240,8 @@ name = "explorer" version = "0.1.0" dependencies = [ "anyhow", - "file_explorer_caller_utils", - "hyperprocess_macro", - "hyperware_process_lib 2.2.0 (git+https://github.com/hyperware-ai/process_lib?rev=4beff93)", + "hyperapp_macro", + "hyperware_process_lib 2.2.0", "md5", "process_macros", "serde", @@ -1300,22 +1299,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "file_explorer_caller_utils" -version = "0.1.0" -dependencies = [ - "anyhow", - "futures", - "futures-util", - "hyperware_process_lib 2.2.0 (git+https://github.com/hyperware-ai/process_lib?rev=4beff93)", - "once_cell", - "process_macros", - "serde", - "serde_json", - "uuid", - "wit-bindgen 0.41.0", -] - [[package]] name = "fixed-hash" version = "0.8.0" @@ -1330,9 +1313,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc5a4e564e38c699f2880d3fda590bedc2e69f3f84cd48b457bd892ce61d0aa9" +checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" dependencies = [ "crc32fast", "miniz_oxide 0.8.9", @@ -1682,11 +1665,12 @@ dependencies = [ ] [[package]] -name = "hyperprocess_macro" -version = "0.1.0" -source = "git+https://github.com/hyperware-ai/hyperprocess-macro?rev=66884c0#66884c0a22b845d1db632f0fb8985a7e5bdad3fb" +name = "hyperapp_macro" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e694a2a3dd6c4d1ba3ec2110b054dca52834ae10d98630b39efd7a59ed4887c8" dependencies = [ - "hyperware_process_lib 2.2.0 (git+https://github.com/hyperware-ai/process_lib?rev=b9f1ead)", + "hyperware_process_lib 2.3.0", "proc-macro2", "quote", "syn 2.0.100", @@ -1695,7 +1679,7 @@ dependencies = [ [[package]] name = "hyperware_process_lib" version = "2.2.0" -source = "git+https://github.com/hyperware-ai/process_lib?rev=4beff93#4beff9389598978d2f97e4cd6d1d0f74a7acac0f" +source = "git+https://github.com/hyperware-ai/process_lib?rev=1a6ad9d#1a6ad9d00e98bd6b268a7d434a7e78a0bb3cce7b" dependencies = [ "alloy", "alloy-primitives", @@ -1705,6 +1689,7 @@ dependencies = [ "base64", "bincode", "color-eyre", + "futures-channel", "futures-util", "http", "mime_guess", @@ -1724,8 +1709,9 @@ dependencies = [ [[package]] name = "hyperware_process_lib" -version = "2.2.0" -source = "git+https://github.com/hyperware-ai/process_lib?rev=b9f1ead#b9f1ead63356bfd4b60b337a380fef1be81d81c6" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe2b6795488c3f7d3410808cf7e29404725f862ad42a3822886464b92451a21" dependencies = [ "alloy", "alloy-primitives", @@ -1735,6 +1721,7 @@ dependencies = [ "base64", "bincode", "color-eyre", + "futures-channel", "futures-util", "http", "mime_guess", @@ -3082,9 +3069,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" [[package]] name = "slab" diff --git a/hyperdrive/packages/file-explorer/explorer/Cargo.toml b/hyperdrive/packages/file-explorer/explorer/Cargo.toml index 33fefce8f..bb289e760 100644 --- a/hyperdrive/packages/file-explorer/explorer/Cargo.toml +++ b/hyperdrive/packages/file-explorer/explorer/Cargo.toml @@ -1,13 +1,17 @@ [dependencies] anyhow = "1.0" -md5 = "0.7" hyperapp_macro = "0.1.1" +md5 = "0.7" process_macros = "0.1" serde_json = "1.0" serde_urlencoded = "0.7" tracing = "0.1.37" wit-bindgen = "0.42.1" +[dependencies.file_explorer_caller_utils] +optional = true +path = "../target/file-explorer-caller-utils" + [dependencies.hyperware_process_lib] features = ["hyperapp"] git = "https://github.com/hyperware-ai/process_lib" @@ -17,6 +21,9 @@ rev = "1a6ad9d" features = ["derive"] version = "1.0" +[features] +caller-utils = ["file_explorer_caller_utils"] + [lib] crate-type = ["cdylib"] name = "explorer" diff --git a/hyperdrive/packages/hns-indexer/Cargo.lock b/hyperdrive/packages/hns-indexer/Cargo.lock index 111a29fbf..2bf5a6385 100644 --- a/hyperdrive/packages/hns-indexer/Cargo.lock +++ b/hyperdrive/packages/hns-indexer/Cargo.lock @@ -1379,7 +1379,7 @@ dependencies = [ name = "get-block" version = "0.1.0" dependencies = [ - "hyperware_process_lib 2.1.0", + "hyperware_process_lib", "serde", "serde_json", "wit-bindgen", @@ -1487,7 +1487,7 @@ dependencies = [ "alloy-sol-types", "anyhow", "hex", - "hyperware_process_lib 2.2.1", + "hyperware_process_lib", "process_macros", "rmp-serde", "serde", @@ -1592,35 +1592,8 @@ dependencies = [ [[package]] name = "hyperware_process_lib" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3abd008d22c3b96ee43300c4c8dffbf1d072a680a13635b5f9da11a0ce9395" -dependencies = [ - "alloy", - "alloy-primitives", - "alloy-sol-macro", - "alloy-sol-types", - "anyhow", - "base64", - "bincode", - "hex", - "http", - "mime_guess", - "rand", - "regex", - "rmp-serde", - "serde", - "serde_json", - "sha3", - "thiserror 1.0.69", - "url", - "wit-bindgen", -] - -[[package]] -name = "hyperware_process_lib" -version = "2.2.1" -source = "git+https://github.com/hyperware-ai/process_lib?rev=4e91521#4e915218b382e4b3f7f93cddeec30415f02a504b" +version = "3.0.1" +source = "git+https://github.com/hyperware-ai/process_lib?rev=34b1f1d857f76fb474f7af96f089961445758edb#34b1f1d857f76fb474f7af96f089961445758edb" dependencies = [ "alloy", "alloy-primitives", @@ -2032,7 +2005,7 @@ dependencies = [ name = "node-info" version = "0.1.0" dependencies = [ - "hyperware_process_lib 2.1.0", + "hyperware_process_lib", "process_macros", "serde", "serde_json", @@ -2569,7 +2542,7 @@ dependencies = [ name = "reset" version = "0.1.0" dependencies = [ - "hyperware_process_lib 2.1.0", + "hyperware_process_lib", "process_macros", "serde", "serde_json", @@ -2973,7 +2946,7 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" name = "state" version = "0.1.0" dependencies = [ - "hyperware_process_lib 2.1.0", + "hyperware_process_lib", "process_macros", "serde", "serde_json", diff --git a/hyperdrive/packages/hns-indexer/get-block/Cargo.toml b/hyperdrive/packages/hns-indexer/get-block/Cargo.toml index fd9a55e5f..d185896a0 100644 --- a/hyperdrive/packages/hns-indexer/get-block/Cargo.toml +++ b/hyperdrive/packages/hns-indexer/get-block/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" simulation-mode = [] [dependencies] -hyperware_process_lib = "2.1.0" +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "34b1f1d857f76fb474f7af96f089961445758edb" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" wit-bindgen = "0.42.1" diff --git a/hyperdrive/packages/hns-indexer/hns-indexer/Cargo.toml b/hyperdrive/packages/hns-indexer/hns-indexer/Cargo.toml index 1d4c67035..db195099d 100644 --- a/hyperdrive/packages/hns-indexer/hns-indexer/Cargo.toml +++ b/hyperdrive/packages/hns-indexer/hns-indexer/Cargo.toml @@ -11,7 +11,7 @@ anyhow = "1.0" alloy-primitives = "0.8.15" alloy-sol-types = "0.8.15" hex = "0.4.3" -hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "4e91521", features = ["logging"] } +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "34b1f1d857f76fb474f7af96f089961445758edb", features = ["logging"] } process_macros = "0.1" rmp-serde = "1.1.2" serde = { version = "1.0", features = ["derive"] } diff --git a/hyperdrive/packages/hns-indexer/node-info/Cargo.toml b/hyperdrive/packages/hns-indexer/node-info/Cargo.toml index 09baa1417..552e465d8 100644 --- a/hyperdrive/packages/hns-indexer/node-info/Cargo.toml +++ b/hyperdrive/packages/hns-indexer/node-info/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" simulation-mode = [] [dependencies] -hyperware_process_lib = "2.1.0" +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "34b1f1d857f76fb474f7af96f089961445758edb" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" process_macros = "0.1" diff --git a/hyperdrive/packages/hns-indexer/reset/Cargo.toml b/hyperdrive/packages/hns-indexer/reset/Cargo.toml index 55e12c42d..58e2b35a6 100644 --- a/hyperdrive/packages/hns-indexer/reset/Cargo.toml +++ b/hyperdrive/packages/hns-indexer/reset/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" simulation-mode = [] [dependencies] -hyperware_process_lib = "2.1.0" +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "34b1f1d857f76fb474f7af96f089961445758edb" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" process_macros = "0.1" diff --git a/hyperdrive/packages/hns-indexer/state/Cargo.toml b/hyperdrive/packages/hns-indexer/state/Cargo.toml index 70565c3cd..73587fe9b 100644 --- a/hyperdrive/packages/hns-indexer/state/Cargo.toml +++ b/hyperdrive/packages/hns-indexer/state/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" simulation-mode = [] [dependencies] -hyperware_process_lib = "2.1.0" +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "34b1f1d857f76fb474f7af96f089961445758edb" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" process_macros = "0.1" diff --git a/hyperdrive/packages/hypermap-cacher/Cargo.lock b/hyperdrive/packages/hypermap-cacher/Cargo.lock index 04499b149..93a9d08f9 100644 --- a/hyperdrive/packages/hypermap-cacher/Cargo.lock +++ b/hyperdrive/packages/hypermap-cacher/Cargo.lock @@ -1086,6 +1086,25 @@ dependencies = [ "typenum", ] +[[package]] +name = "dao-cacher" +version = "0.1.0" +dependencies = [ + "alloy", + "alloy-primitives", + "alloy-sol-types", + "anyhow", + "chrono", + "hex", + "hyperware_process_lib", + "process_macros", + "rand 0.8.5", + "rmp-serde", + "serde", + "serde_json", + "wit-bindgen", +] + [[package]] name = "dashmap" version = "6.1.0" @@ -1680,9 +1699,8 @@ dependencies = [ [[package]] name = "hyperware_process_lib" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe2b6795488c3f7d3410808cf7e29404725f862ad42a3822886464b92451a21" +version = "3.0.1" +source = "git+https://github.com/hyperware-ai/process_lib?rev=34b1f1d857f76fb474f7af96f089961445758edb#34b1f1d857f76fb474f7af96f089961445758edb" dependencies = [ "alloy", "alloy-primitives", diff --git a/hyperdrive/packages/hypermap-cacher/Cargo.toml b/hyperdrive/packages/hypermap-cacher/Cargo.toml index ff4a19f1f..ff8510946 100644 --- a/hyperdrive/packages/hypermap-cacher/Cargo.toml +++ b/hyperdrive/packages/hypermap-cacher/Cargo.toml @@ -2,6 +2,7 @@ resolver = "2" members = [ "binding-cacher", + "dao-cacher", "hypermap-cacher", "reset-cache", "set-nodes", diff --git a/hyperdrive/packages/hypermap-cacher/api/hypermap-cacher:sys-v1.wit b/hyperdrive/packages/hypermap-cacher/api/hypermap-cacher:sys-v2.wit similarity index 66% rename from hyperdrive/packages/hypermap-cacher/api/hypermap-cacher:sys-v1.wit rename to hyperdrive/packages/hypermap-cacher/api/hypermap-cacher:sys-v2.wit index 472e9b6e2..e39a427da 100644 --- a/hyperdrive/packages/hypermap-cacher/api/hypermap-cacher:sys-v1.wit +++ b/hyperdrive/packages/hypermap-cacher/api/hypermap-cacher:sys-v2.wit @@ -76,6 +76,84 @@ interface binding-cacher { } } +interface dao-cacher { + // Metadata associated with a batch of Ethereum logs. + record dao-logs-metadata { + chain-id: string, + from-block: string, + to-block: string, + time-created: string, + created-by: string, + signature: string, + } + + // Represents an item in the manifest, detailing a single log cache file. + record dao-manifest-item { + metadata: dao-logs-metadata, + is-empty: bool, + file-hash: string, + file-name: string, + } + + // The main manifest structure, listing all available log cache files. + // WIT does not support direct map types, so a list of key-value tuples is used. + record dao-manifest { + // The key is the filename of the log cache. + items: list>, + manifest-filename: string, + chain-id: string, + protocol-version: string, + } + + record dao-get-logs-by-range-request { + from-block: u64, + to-block: option, // If None, signifies to the latest available/relevant cached block. + } + + variant dao-get-logs-by-range-ok-response { + logs(tuple), + latest(u64), + } + + // Defines the types of requests that can be sent to the Hypermap Cacher process. + variant dao-cacher-request { + get-manifest, + get-log-cache-content(string), + get-status, + get-logs-by-range(dao-get-logs-by-range-request), + start-providing, + stop-providing, + set-nodes(list), + reset(option>), + } + + // Represents the operational status of the cacher. + record dao-cacher-status { + last-cached-block: u64, + chain-id: string, + protocol-version: string, + next-cache-attempt-in-seconds: option, + manifest-filename: string, + log-files-count: u32, + our-address: string, + is-providing: bool, + } + + // Defines the types of responses the Hypermap Cacher process can send. + variant dao-cacher-response { + get-manifest(option), + get-log-cache-content(result, string>), + get-status(dao-cacher-status), + get-logs-by-range(result), + start-providing(result), + stop-providing(result), + set-nodes(result), + reset(result), + rejected, + is-starting, + } +} + interface hypermap-cacher { // Metadata associated with a batch of Ethereum logs. record logs-metadata { @@ -154,9 +232,10 @@ interface hypermap-cacher { } } -world hypermap-cacher-sys-v1 { +world hypermap-cacher-sys-v2 { import sign; import binding-cacher; + import dao-cacher; import hypermap-cacher; include process-v1; } diff --git a/hyperdrive/packages/hypermap-cacher/binding-cacher/Cargo.toml b/hyperdrive/packages/hypermap-cacher/binding-cacher/Cargo.toml index e0504d478..17b56f30d 100644 --- a/hyperdrive/packages/hypermap-cacher/binding-cacher/Cargo.toml +++ b/hyperdrive/packages/hypermap-cacher/binding-cacher/Cargo.toml @@ -18,7 +18,7 @@ alloy = { version = "0.8.1", features = [ ] } chrono = "0.4.41" hex = "0.4.3" -hyperware_process_lib = { version = "2.3.0", features = ["logging"] } +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "34b1f1d857f76fb474f7af96f089961445758edb", features = ["logging"] } process_macros = "0.1.0" rand = "0.8" rmp-serde = "1.1.2" diff --git a/hyperdrive/packages/hypermap-cacher/binding-cacher/src/lib.rs b/hyperdrive/packages/hypermap-cacher/binding-cacher/src/lib.rs index d026da31e..6bb9914c9 100644 --- a/hyperdrive/packages/hypermap-cacher/binding-cacher/src/lib.rs +++ b/hyperdrive/packages/hypermap-cacher/binding-cacher/src/lib.rs @@ -27,7 +27,7 @@ use hyperware_process_lib::{wait_for_process_ready, WaitClassification}; wit_bindgen::generate!({ path: "../target/wit", - world: "hypermap-cacher-sys-v1", + world: "hypermap-cacher-sys-v2", generate_unused_types: true, additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); diff --git a/hyperdrive/packages/hypermap-cacher/dao-cacher/Cargo.toml b/hyperdrive/packages/hypermap-cacher/dao-cacher/Cargo.toml new file mode 100644 index 000000000..7acc80ebb --- /dev/null +++ b/hyperdrive/packages/hypermap-cacher/dao-cacher/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "dao-cacher" +version = "0.1.0" +edition = "2021" +publish = false + +[features] +simulation-mode = ["hyperware_process_lib/simulation-mode"] + +[dependencies] +anyhow = "1.0" +alloy-primitives = "0.8.15" +alloy-sol-types = "0.8.15" +alloy = { version = "0.8.1", features = [ + "json-rpc", + "rpc-client", + "rpc-types", +] } +chrono = "0.4.41" +hex = "0.4.3" +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "34b1f1d857f76fb474f7af96f089961445758edb", features = ["logging"] } +process_macros = "0.1.0" +rand = "0.8" +rmp-serde = "1.1.2" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +wit-bindgen = "0.42.1" + +[lib] +crate-type = ["cdylib"] + +[package.metadata.component] +package = "hyperware:process" diff --git a/hyperdrive/packages/hypermap-cacher/dao-cacher/src/lib.rs b/hyperdrive/packages/hypermap-cacher/dao-cacher/src/lib.rs new file mode 100644 index 000000000..7def5a6af --- /dev/null +++ b/hyperdrive/packages/hypermap-cacher/dao-cacher/src/lib.rs @@ -0,0 +1,1516 @@ +use std::{ + cmp::{max, min}, + collections::HashMap, + str::FromStr, +}; + +use alloy::hex; +use alloy_primitives::keccak256; +use rand::seq::SliceRandom; +use rand::thread_rng; +use serde::{Deserialize, Serialize}; + +use crate::hyperware::process::dao_cacher::{ + DaoCacherRequest as CacherRequest, DaoCacherResponse as CacherResponse, DaoCacherStatus as CacherStatus, + DaoGetLogsByRangeOkResponse as GetLogsByRangeOkResponse, DaoGetLogsByRangeRequest as GetLogsByRangeRequest, + DaoLogsMetadata as WitLogsMetadata, DaoManifest as WitManifest, DaoManifestItem as WitManifestItem, +}; +use hyperware_process_lib::{ + await_message, call_init, dao, eth, get_state, http, + logging::{debug, error, info, init_logging, warn, Level}, + net::{NetAction, NetResponse}, + our, set_state, sign, timer, vfs, Address, ProcessId, Request, Response, +}; +use hyperware_process_lib::{wait_for_process_ready, WaitClassification}; + +wit_bindgen::generate!({ + path: "../target/wit", + world: "hypermap-cacher-sys-v2", + generate_unused_types: true, + additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], +}); + +const PROTOCOL_VERSION: &str = "0"; +const DEFAULT_BLOCK_BATCH_SIZE: u64 = 10; +// Cache cadence: faster in simulation, slower on real chains. +#[cfg(feature = "simulation-mode")] +const DEFAULT_CACHE_INTERVAL_S: u64 = 3_600; +#[cfg(not(feature = "simulation-mode"))] +const DEFAULT_CACHE_INTERVAL_S: u64 = 3_600; // 2s / block -> ~1hr (1800 blocks) +const MAX_LOG_RETRIES: u8 = 3; +const RETRY_DELAY_S: u64 = 10; +const LOG_ITERATION_DELAY_MS: u64 = 200; + +#[cfg(not(feature = "simulation-mode"))] +const DEFAULT_NODES: &[&str] = &[ + "us-cacher-1.hypr", + "eu-cacher-1.hypr", + "nick.hypr", + "nick1udwig.os", +]; +#[cfg(feature = "simulation-mode")] +const DEFAULT_NODES: &[&str] = &["fake.os"]; + +// Internal representation of LogsMetadata, similar to WIT but for Rust logic. +#[derive(Serialize, Deserialize, Debug, Clone)] +struct LogsMetadataInternal { + #[serde(rename = "chainId")] + chain_id: String, + #[serde(rename = "fromBlock")] + from_block: String, + #[serde(rename = "toBlock")] + to_block: String, + #[serde(rename = "timeCreated")] + time_created: String, + #[serde(rename = "createdBy")] + created_by: String, + signature: String, // Keccak256 hash of the log file content. +} + +// Internal representation of a LogCache, containing metadata and actual logs. +#[derive(Serialize, Deserialize, Debug, Clone)] +struct LogCacheInternal { + metadata: LogsMetadataInternal, + logs: Vec, // The actual Ethereum logs. +} + +// Internal representation of a ManifestItem. +#[derive(Serialize, Deserialize, Debug, Clone)] +struct ManifestItemInternal { + metadata: LogsMetadataInternal, + #[serde(rename = "isEmpty")] + is_empty: bool, + #[serde(rename = "fileHash")] + file_hash: String, + #[serde(rename = "fileName")] + file_name: String, +} + +// Internal representation of the Manifest. +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +struct ManifestInternal { + items: HashMap, + manifest_filename: String, + chain_id: String, + protocol_version: String, +} + +// The main state structure for the Hypermap Binding Cacher process. +#[derive(Serialize, Deserialize, Debug)] +struct State { + hypermap_binding_address: eth::Address, + manifest: ManifestInternal, + last_cached_block: u64, + chain_id: String, + protocol_version: String, + cache_interval_s: u64, + block_batch_size: u64, + is_cache_timer_live: bool, + drive_path: String, + is_providing: bool, + nodes: Vec, + #[serde(skip)] + is_starting: bool, +} + +// Generates a timestamp string. +fn get_current_timestamp_str() -> String { + let datetime = chrono::Utc::now(); + datetime.format("%Y%m%dT%H%M%SZ").to_string() +} + +fn is_local_request(our: &Address, source: &Address) -> bool { + our.node == source.node +} + +impl State { + fn new(drive_path: &str) -> Self { + let chain_id = dao::DAO_CHAIN_ID.to_string(); + let hypermap_binding_address = eth::Address::from_str(dao::DAO_GOVERNOR_ADDRESS) + .expect("Failed to parse DAO_GOVERNOR_ADDRESS"); + + let manifest_filename = format!( + "manifest-chain{}-protocol{}.json", + chain_id, PROTOCOL_VERSION + ); + let initial_manifest = ManifestInternal { + items: HashMap::new(), + manifest_filename: manifest_filename.clone(), + chain_id: chain_id.clone(), + protocol_version: PROTOCOL_VERSION.to_string(), + }; + + State { + hypermap_binding_address, + manifest: initial_manifest, + last_cached_block: dao::DAO_FIRST_BLOCK, + chain_id, + protocol_version: PROTOCOL_VERSION.to_string(), + cache_interval_s: DEFAULT_CACHE_INTERVAL_S, + block_batch_size: 0, // Will be determined dynamically + is_cache_timer_live: false, + drive_path: drive_path.to_string(), + is_providing: false, + nodes: DEFAULT_NODES.iter().map(|s| s.to_string()).collect(), + is_starting: true, + } + } + + fn load(drive_path: &str) -> Self { + match get_state() { + Some(state_bytes) => match serde_json::from_slice::(&state_bytes) { + Ok(mut loaded_state) => { + info!("Successfully loaded state from checkpoint."); + // Always start in starting mode to bootstrap from other nodes + // is_starting is not serialized, so it defaults to false and we set it to true + loaded_state.is_starting = true; + loaded_state.drive_path = drive_path.to_string(); + + // Validate state against manifest file on disk + if let Err(e) = loaded_state.validate_state_against_manifest() { + warn!("State validation failed: {:?}. Clearing drive and creating fresh state.", e); + if let Err(clear_err) = loaded_state.clear_drive() { + error!("Failed to clear drive: {:?}", clear_err); + } + return Self::new(drive_path); + } + + loaded_state + } + Err(e) => { + warn!( + "Failed to deserialize saved state: {:?}. Creating new state.", + e + ); + Self::new(drive_path) + } + }, + None => { + info!("No saved state found. Creating new state."); + Self::new(drive_path) + } + } + } + + fn save(&self) { + match serde_json::to_vec(self) { + Ok(state_bytes) => set_state(&state_bytes), + Err(e) => error!("Fatal: Failed to serialize state for saving: {:?}", e), + } + info!( + "State checkpoint saved. Last cached block: {}", + self.last_cached_block + ); + } + + // Core logic for fetching logs, creating cache files, and updating the manifest. + fn cache_logs_and_update_manifest(&mut self, provider: ð::Provider) -> anyhow::Result<()> { + // Ensure batch size is determined + if self.block_batch_size == 0 { + self.determine_batch_size(provider)?; + } + + let current_chain_head = match provider.get_block_number() { + Ok(block_num) => block_num, + Err(e) => { + error!( + "Failed to get current block number: {:?}. Skipping cycle.", + e + ); + return Err(anyhow::anyhow!("Failed to get block number: {:?}", e)); + } + }; + + if self.last_cached_block >= current_chain_head { + info!( + "Already caught up to chain head ({}). Nothing to cache.", + current_chain_head + ); + return Ok(()); + } + + while self.last_cached_block != current_chain_head { + self.cache_logs_and_update_manifest_step(provider, Some(current_chain_head))?; + + std::thread::sleep(std::time::Duration::from_millis(LOG_ITERATION_DELAY_MS)); + } + + Ok(()) + } + + fn cache_logs_and_update_manifest_step( + &mut self, + provider: ð::Provider, + to_block: Option, + ) -> anyhow::Result<()> { + info!( + "Starting caching cycle. From block: {}", + self.last_cached_block + 1 + ); + + let current_chain_head = match to_block { + Some(b) => b, + None => match provider.get_block_number() { + Ok(block_num) => block_num, + Err(e) => { + error!( + "Failed to get current block number: {:?}. Skipping cycle.", + e + ); + return Err(anyhow::anyhow!("Failed to get block number: {:?}", e)); + } + }, + }; + + if self.last_cached_block >= current_chain_head { + info!( + "Already caught up to chain head ({}). Nothing to cache.", + current_chain_head + ); + return Ok(()); + } + + let from_block = self.last_cached_block + 1; + let mut to_block = from_block + self.block_batch_size - 1; + if to_block > current_chain_head { + to_block = current_chain_head; + } + + if from_block > to_block { + info!("From_block {} is greater than to_block {}. Chain might not have advanced enough. Skipping.", from_block, to_block); + return Ok(()); + } + + let filter = eth::Filter::new() + .address(self.hypermap_binding_address) + .from_block(from_block) + .to_block(eth::BlockNumberOrTag::Number(to_block)); + + let logs = { + let mut attempt = 0; + loop { + match provider.get_logs(&filter) { + Ok(logs) => break logs, + Err(e) => { + attempt += 1; + if attempt >= MAX_LOG_RETRIES { + error!( + "Failed to get logs after {} retries: {:?}", + MAX_LOG_RETRIES, e + ); + return Err(anyhow::anyhow!("Failed to get logs: {:?}", e)); + } + warn!( + "Error getting logs (attempt {}/{}): {:?}. Retrying in {}s...", + attempt, MAX_LOG_RETRIES, e, RETRY_DELAY_S + ); + std::thread::sleep(std::time::Duration::from_secs(RETRY_DELAY_S)); + } + } + } + }; + + info!( + "Fetched {} logs from block {} to {}.", + logs.len(), + from_block, + to_block + ); + + let our = our(); + + let metadata = LogsMetadataInternal { + chain_id: self.chain_id.clone(), + from_block: from_block.to_string(), + to_block: to_block.to_string(), + time_created: get_current_timestamp_str(), + created_by: our.to_string(), + signature: "".to_string(), + }; + + let mut log_cache = LogCacheInternal { + metadata, + logs: logs.clone(), + }; + + let mut logs_bytes_for_sig = serde_json::to_vec(&log_cache.logs).unwrap_or_default(); + logs_bytes_for_sig.extend_from_slice(&from_block.to_be_bytes()); + logs_bytes_for_sig.extend_from_slice(&to_block.to_be_bytes()); + let logs_hash_for_sig = keccak256(&logs_bytes_for_sig); + + let signature = sign::net_key_sign(logs_hash_for_sig.to_vec())?; + + log_cache.metadata.signature = format!("0x{}", hex::encode(signature)); + + // Final serialization of LogCacheInternal with the signature. + let final_log_cache_bytes = match serde_json::to_vec(&log_cache) { + Ok(bytes) => bytes, + Err(e) => { + error!( + "Failed to re-serialize LogCacheInternal with signature: {:?}", + e + ); + return Err(e.into()); + } + }; + + let file_hash_for_manifest = + format!("0x{}", hex::encode(keccak256(&final_log_cache_bytes))); + + let log_cache_filename = format!( + "{}-chain{}-from{}-to{}-protocol{}.json", + log_cache + .metadata + .time_created + .replace(":", "") + .replace("-", ""), // Make timestamp filename-safe + self.chain_id, + from_block, + to_block, + self.protocol_version + ); + + if !logs.is_empty() { + let log_cache_path = format!("{}/{}", self.drive_path, log_cache_filename); + let mut log_cache_file = vfs::open_file(&log_cache_path, true, None)?; + + if let Err(e) = log_cache_file.write_all(&final_log_cache_bytes) { + error!("Failed to write log cache file {}: {:?}", log_cache_path, e); + return Err(e.into()); + } + info!("Successfully wrote log cache file: {}", log_cache_path); + } + + let manifest_item = ManifestItemInternal { + metadata: log_cache.metadata.clone(), + is_empty: logs.is_empty(), + file_hash: file_hash_for_manifest, + file_name: if logs.is_empty() { + "".to_string() + } else { + log_cache_filename.clone() + }, + }; + self.manifest + .items + .insert(log_cache_filename.clone(), manifest_item); + self.manifest.chain_id = self.chain_id.clone(); + self.manifest.protocol_version = self.protocol_version.clone(); + + let manifest_bytes = match serde_json::to_vec(&self.manifest) { + Ok(bytes) => bytes, + Err(e) => { + error!("Failed to serialize manifest: {:?}", e); + return Err(e.into()); + } + }; + + let manifest_path = format!("{}/{}", self.drive_path, self.manifest.manifest_filename); + let manifest_file = vfs::open_file(&manifest_path, true, None)?; + + if let Err(e) = manifest_file.write(&manifest_bytes) { + error!("Failed to write manifest file {}: {:?}", manifest_path, e); + return Err(e.into()); + } + info!( + "Successfully updated and wrote manifest file: {}", + manifest_path + ); + + self.last_cached_block = to_block; + self.save(); + + Ok(()) + } + + // Validate that the in-memory state matches the manifest file on disk + fn validate_state_against_manifest(&self) -> anyhow::Result<()> { + let manifest_path = format!("{}/{}", self.drive_path, self.manifest.manifest_filename); + + // Check if manifest file exists + match vfs::open_file(&manifest_path, false, None) { + Ok(manifest_file) => { + match manifest_file.read() { + Ok(disk_manifest_bytes) => { + match serde_json::from_slice::(&disk_manifest_bytes) { + Ok(disk_manifest) => { + // Compare key aspects of the manifests + if self.manifest.chain_id != disk_manifest.chain_id { + return Err(anyhow::anyhow!( + "Chain ID mismatch: state has {}, disk has {}", + self.manifest.chain_id, + disk_manifest.chain_id + )); + } + + if self.manifest.protocol_version != disk_manifest.protocol_version + { + return Err(anyhow::anyhow!( + "Protocol version mismatch: state has {}, disk has {}", + self.manifest.protocol_version, + disk_manifest.protocol_version + )); + } + + // Check if all files mentioned in state manifest exist on disk + for (_filename, item) in &self.manifest.items { + if !item.file_name.is_empty() { + let file_path = + format!("{}/{}", self.drive_path, item.file_name); + if vfs::metadata(&file_path, None).is_err() { + return Err(anyhow::anyhow!( + "File {} mentioned in state manifest does not exist on disk", + item.file_name + )); + } + } + } + + // Check if disk manifest has more recent data than our state + let disk_max_block = disk_manifest + .items + .values() + .filter_map(|item| item.metadata.to_block.parse::().ok()) + .max() + .unwrap_or(0); + + let state_max_block = self + .manifest + .items + .values() + .filter_map(|item| item.metadata.to_block.parse::().ok()) + .max() + .unwrap_or(0); + + if disk_max_block > state_max_block { + return Err(anyhow::anyhow!( + "Disk manifest has more recent data (block {}) than state (block {})", + disk_max_block, state_max_block + )); + } + + info!("State validation passed - state matches manifest file"); + Ok(()) + } + Err(e) => { + Err(anyhow::anyhow!("Failed to parse manifest file: {:?}", e)) + } + } + } + Err(e) => Err(anyhow::anyhow!("Failed to read manifest file: {:?}", e)), + } + } + Err(_) => { + // Manifest file doesn't exist - this is okay for new installs + if self.manifest.items.is_empty() { + info!("No manifest file found, but state is also empty - validation passed"); + Ok(()) + } else { + Err(anyhow::anyhow!( + "State has manifest items but no manifest file exists on disk" + )) + } + } + } + } + + // Clear all files from the drive + fn clear_drive(&self) -> anyhow::Result<()> { + info!("Clearing all files from drive: {}", self.drive_path); + + // Remove the manifest file + let manifest_path = format!("{}/{}", self.drive_path, self.manifest.manifest_filename); + match vfs::remove_file(&manifest_path, None) { + Ok(_) => info!("Removed manifest file: {}", manifest_path), + Err(e) => warn!("Failed to remove manifest file {}: {:?}", manifest_path, e), + } + + // Remove all files mentioned in the manifest + for (_, item) in &self.manifest.items { + if !item.file_name.is_empty() { + let file_path = format!("{}/{}", self.drive_path, item.file_name); + match vfs::remove_file(&file_path, None) { + Ok(_) => info!("Removed cache file: {}", file_path), + Err(e) => warn!("Failed to remove cache file {}: {:?}", file_path, e), + } + } + } + + info!("Drive clearing completed"); + Ok(()) + } + + // Bootstrap state from other nodes, then fallback to RPC + fn bootstrap_state(&mut self, provider: ð::Provider) -> anyhow::Result<()> { + info!("Starting state bootstrap process..."); + + // Try to bootstrap from other nodes first + if let Ok(()) = self.try_bootstrap_from_nodes() { + info!("Successfully bootstrapped from other nodes"); + } + + self.try_bootstrap_from_rpc(provider)?; + + // Mark as no longer starting + self.is_starting = false; + self.save(); + info!("Bootstrap process completed, cacher is now ready"); + Ok(()) + } + + // Try to bootstrap from other dao-cacher nodes + fn try_bootstrap_from_nodes(&mut self) -> anyhow::Result<()> { + // Create alternate drive for initfiles and read the test data + let alt_drive_path = vfs::create_drive(our().package_id(), "initfiles", None).unwrap(); + + // Try to read the cache_sources file from the initfiles drive + match vfs::open_file(&format!("{}/cache_sources", alt_drive_path), false, None) { + Ok(file) => { + match file.read() { + Ok(contents) => { + let content_str = String::from_utf8_lossy(&contents); + info!("Contents of cache_sources: {}", content_str); + + // Parse the JSON to get the vector of node names + match serde_json::from_str::>(&content_str) { + Ok(custom_cache_nodes) => { + if !custom_cache_nodes.is_empty() { + info!( + "Loading custom cache source nodes: {:?}", + custom_cache_nodes + ); + // Clear existing nodes and add custom ones + self.nodes.clear(); + for node_name in custom_cache_nodes { + self.nodes.push(node_name.clone()); + } + } else { + info!("Custom cache nodes list is empty, keeping existing node configuration"); + } + } + Err(e) => { + info!("Failed to parse cache_sources as JSON: {}, keeping existing node configuration", e); + } + } + } + Err(e) => { + info!( + "Failed to read cache_sources: {}, keeping existing node configuration", + e + ); + } + } + } + Err(e) => { + info!( + "Failed to open cache_sources: {}, keeping existing node configuration", + e + ); + } + } + + if self.nodes.is_empty() { + info!("No nodes configured for bootstrap, will fallback to RPC"); + return Err(anyhow::anyhow!("No nodes configured for bootstrap")); + } + + info!("Attempting to bootstrap from {} nodes", self.nodes.len()); + + let mut nodes = self.nodes.clone(); + + // If using default nodes, shuffle them for random order + let default_nodes: Vec = DEFAULT_NODES.iter().map(|s| s.to_string()).collect(); + if nodes == default_nodes { + nodes.shuffle(&mut thread_rng()); + } + + let mut nodes_not_yet_in_net = nodes.clone(); + let num_retries = 10; + for _ in 0..num_retries { + nodes_not_yet_in_net.retain(|node| { + let Ok(Ok(response)) = Request::new() + .target(("our", "net", "distro", "sys")) + .body(rmp_serde::to_vec(&NetAction::GetPeer(node.clone())).unwrap()) + .send_and_await_response(1) + else { + return true; // keep the node + }; + + !matches!( + rmp_serde::from_slice::(response.body()), + Ok(NetResponse::Peer(Some(_))), + ) + }); + if nodes_not_yet_in_net.is_empty() { + break; + } + std::thread::sleep(std::time::Duration::from_secs(1)); + } + if !nodes_not_yet_in_net.is_empty() { + error!("failed to get peering info for {nodes_not_yet_in_net:?}"); + } + + for node in nodes { + info!("Requesting logs from node: {}", node); + + let cacher_process_address = + Address::new(&node, ("dao-cacher", "hypermap-cacher", "sys")); + + if cacher_process_address == our() { + continue; + } + + // ping node for quicker failure if not online/providing/... + let Ok(Ok(response)) = Request::to(cacher_process_address.clone()) + .body(CacherRequest::GetStatus) + .send_and_await_response(3) + else { + warn!("Node {node} failed to respond to ping; trying next one..."); + continue; + }; + let Ok(CacherResponse::GetStatus(_)) = response.body().try_into() else { + warn!("Node {node} failed to respond to ping with expected GetStatus; trying next one..."); + continue; + }; + + // get the logs + let get_logs_request = GetLogsByRangeRequest { + from_block: self.last_cached_block + 1, + to_block: None, // Get all available logs + }; + + match Request::to(cacher_process_address.clone()) + .body(CacherRequest::GetLogsByRange(get_logs_request)) + .send_and_await_response(15) + { + Ok(Ok(response_msg)) => match response_msg.body().try_into() { + Ok(CacherResponse::GetLogsByRange(Ok(get_logs))) => { + match get_logs { + GetLogsByRangeOkResponse::Logs((block, json_string)) => { + if let Ok(log_caches) = + serde_json::from_str::>(&json_string) + { + self.process_received_log_caches(log_caches)?; + } + if block > self.last_cached_block { + self.last_cached_block = block; + } + } + GetLogsByRangeOkResponse::Latest(block) => { + if block > self.last_cached_block { + self.last_cached_block = block; + } + } + } + return Ok(()); + } + Ok(CacherResponse::GetLogsByRange(Err(e))) => { + warn!("Node {} returned error: {}", cacher_process_address, e); + } + Ok(CacherResponse::IsStarting) => { + info!( + "Node {} is still starting, trying next node", + cacher_process_address + ); + } + Ok(CacherResponse::Rejected) => { + warn!("Node {} rejected our request", cacher_process_address); + } + Ok(_) => { + warn!( + "Node {} returned unexpected response type", + cacher_process_address + ); + } + Err(e) => { + warn!( + "Failed to parse response from {}: {:?}", + cacher_process_address, e + ); + } + }, + Ok(Err(e)) => { + warn!("Error response from {}: {:?}", cacher_process_address, e); + } + Err(e) => { + warn!( + "Failed to send request to {}: {:?}", + cacher_process_address, e + ); + } + } + } + + Err(anyhow::anyhow!("Failed to bootstrap from any node")) + } + + // Helper function to write nodes to cache_sources file + fn write_nodes_to_file(&self) -> anyhow::Result<()> { + info!("Beginning of subroutine"); + let alt_drive_path = vfs::create_drive(our().package_id(), "initfiles", None)?; + info!("drive path defined"); + let nodes_json = serde_json::to_string(&self.nodes)?; + info!("nodes_json defined"); + let file_path = format!("{}/cache_sources", alt_drive_path); + info!("file_path defined"); + + // Open file in write mode which should truncate, but to be safe we'll write exact bytes + let mut file = vfs::open_file(&file_path, true, None)?; + + // Get the bytes to write + let bytes = nodes_json.as_bytes(); + + // Write all bytes + file.write_all(bytes)?; + + // Explicitly set the file length to the exact size of what we wrote + // This ensures any old content beyond this point is truncated + file.set_len(bytes.len() as u64)?; + + info!("Updated cache_sources with {} nodes", self.nodes.len()); + Ok(()) + } + + // Process received log caches and write them to VFS + fn process_received_log_caches( + &mut self, + log_caches: Vec, + ) -> anyhow::Result<()> { + info!("Processing {} received log caches", log_caches.len()); + + for log_cache in log_caches { + // Validate the log cache signature + // TODO Remove or find other method + // Temporarily skip + /* + if !self.validate_log_cache(&log_cache)? { + warn!("Invalid log cache signature, skipping"); + continue; + } + */ + + // Generate filename from metadata + let filename = format!( + "{}-chain{}-from{}-to{}-protocol{}.json", + log_cache + .metadata + .time_created + .replace(":", "") + .replace("-", ""), + log_cache.metadata.chain_id, + log_cache.metadata.from_block, + log_cache.metadata.to_block, + PROTOCOL_VERSION + ); + + // Write log cache to VFS + let file_path = format!("{}/{}", self.drive_path, filename); + let log_cache_bytes = serde_json::to_vec(&log_cache)?; + + let mut file = vfs::open_file(&file_path, true, None)?; + file.write_all(&log_cache_bytes)?; + + info!("Wrote log cache file: {}", file_path); + + // Update manifest + let file_hash = format!("0x{}", hex::encode(keccak256(&log_cache_bytes))); + let manifest_item = ManifestItemInternal { + metadata: log_cache.metadata.clone(), + is_empty: log_cache.logs.is_empty(), + file_hash, + file_name: filename.clone(), + }; + + self.manifest.items.insert(filename, manifest_item); + + // Update last cached block if this cache goes beyond it + if let Ok(to_block) = log_cache.metadata.to_block.parse::() { + if to_block > self.last_cached_block { + self.last_cached_block = to_block; + } + } + } + + // Write updated manifest + self.write_manifest()?; + + Ok(()) + } + + // Validate a log cache signature + fn validate_log_cache(&self, log_cache: &LogCacheInternal) -> anyhow::Result { + let from_block = log_cache.metadata.from_block.parse::()?; + let to_block = log_cache.metadata.to_block.parse::()?; + + let mut bytes_to_verify = serde_json::to_vec(&log_cache.logs)?; + bytes_to_verify.extend_from_slice(&from_block.to_be_bytes()); + bytes_to_verify.extend_from_slice(&to_block.to_be_bytes()); + let hashed_data = keccak256(&bytes_to_verify); + + let signature_hex = log_cache.metadata.signature.trim_start_matches("0x"); + let signature_bytes = hex::decode(signature_hex)?; + + let created_by_address = log_cache.metadata.created_by.parse::
()?; + + Ok(sign::net_key_verify( + hashed_data.to_vec(), + &created_by_address, + signature_bytes, + )?) + } + + // Write manifest to VFS + fn write_manifest(&self) -> anyhow::Result<()> { + let manifest_bytes = serde_json::to_vec(&self.manifest)?; + let manifest_path = format!("{}/{}", self.drive_path, self.manifest.manifest_filename); + let manifest_file = vfs::open_file(&manifest_path, true, None)?; + manifest_file.write(&manifest_bytes)?; + info!("Updated manifest file: {}", manifest_path); + Ok(()) + } + + // Determine optimal batch size dynamically + fn determine_batch_size(&mut self, provider: ð::Provider) -> anyhow::Result<()> { + if self.block_batch_size > 0 { + // Already determined + return Ok(()); + } + + let current_block = match provider.get_block_number() { + Ok(block_num) => block_num, + Err(e) => { + error!("Failed to get current block number: {:?}", e); + // Fall back to default if we can't get the current block + self.block_batch_size = DEFAULT_BLOCK_BATCH_SIZE; + return Ok(()); + } + }; + + // Start with the difference between current block and HYPERMAP_FIRST_BLOCK + let mut batch_size = current_block.saturating_sub(dao::DAO_FIRST_BLOCK); + + // Ensure we have at least a minimum batch size + if batch_size < 1 { + batch_size = DEFAULT_BLOCK_BATCH_SIZE; + self.block_batch_size = batch_size; + info!("Using default batch size: {batch_size}"); + return Ok(()); + } + + info!("Determining optimal batch size starting from {batch_size}"); + + // Try progressively smaller batch sizes until we find one that works + loop { + let from_block = dao::DAO_FIRST_BLOCK; + let to_block = from_block + batch_size; + + let filter = eth::Filter::new() + .address(self.hypermap_binding_address) + .from_block(from_block) + .to_block(eth::BlockNumberOrTag::Number(to_block)); + + match provider.get_logs(&filter) { + Ok(_) => { + // Success! This batch size works + self.block_batch_size = batch_size; + info!("Successfully determined batch size: {}", batch_size); + return Ok(()); + } + Err(e) => { + // Request failed or timed out, try smaller batch + warn!("Batch size {} failed: {:?}, halving...", batch_size, e); + batch_size = batch_size / 2; + + // Don't go below a minimum threshold + if batch_size < 10 { + warn!("Could not determine optimal batch size, using minimum: {DEFAULT_BLOCK_BATCH_SIZE}"); + self.block_batch_size = DEFAULT_BLOCK_BATCH_SIZE; + return Ok(()); + } + } + } + } + } + + // Fallback to RPC bootstrap - catch up from where we left off + fn try_bootstrap_from_rpc(&mut self, provider: ð::Provider) -> anyhow::Result<()> { + info!( + "Bootstrapping from RPC, starting from block {}", + self.last_cached_block + 1 + ); + + // Catch up remainder (or as fallback) using RPC + self.cache_logs_and_update_manifest(provider)?; + + // run it twice for fresh boot case: + // - initial bootstrap takes much time + // - in that time, the block you are updating to is no longer the head of the chain + // - so run again to get to the head of the chain + self.cache_logs_and_update_manifest(provider)?; + + Ok(()) + } + + fn to_wit_manifest(&self) -> WitManifest { + let items = self + .manifest + .items + .iter() + .map(|(k, v)| { + let wit_meta = WitLogsMetadata { + chain_id: v.metadata.chain_id.clone(), + from_block: v.metadata.from_block.clone(), + to_block: v.metadata.to_block.clone(), + time_created: v.metadata.time_created.clone(), + created_by: v.metadata.created_by.clone(), + signature: v.metadata.signature.clone(), + }; + let wit_item = WitManifestItem { + metadata: wit_meta, + is_empty: v.is_empty, + file_hash: v.file_hash.clone(), + file_name: v.file_name.clone(), + }; + (k.clone(), wit_item) + }) + .collect::>(); + + WitManifest { + items, + manifest_filename: self.manifest.manifest_filename.clone(), + chain_id: self.manifest.chain_id.clone(), + protocol_version: self.manifest.protocol_version.clone(), + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +enum HttpApi { + GetManifest, + GetLogCacheFile(String), + GetStatus, +} + +fn http_handler( + state: &mut State, + path: &str, +) -> anyhow::Result<(http::server::HttpResponse, Vec)> { + let response = http::server::HttpResponse::new(http::StatusCode::OK) + .header("Content-Type", "application/json"); + + // Basic routing based on path + Ok(if path == "/manifest" || path == "/manifest.json" { + let manifest_path = format!("{}/{}", state.drive_path, state.manifest.manifest_filename); + let manifest_file = vfs::open_file(&manifest_path, true, None)?; + match manifest_file.read() { + Ok(content) => (response, content), + Err(e) => { + error!( + "HTTP: Failed to read manifest file {}: {:?}", + manifest_path, e + ); + ( + http::server::HttpResponse::new(http::StatusCode::NOT_FOUND), + b"Manifest not found".to_vec(), + ) + } + } + } else if path.starts_with("/log-cache/") { + let filename = path.trim_start_matches("/log-cache/"); + if filename.is_empty() || filename.contains("..") { + // Basic security check + return Ok(( + http::server::HttpResponse::new(http::StatusCode::BAD_REQUEST), + b"Invalid filename".to_vec(), + )); + } + let log_cache_path = format!("{}/{}", state.drive_path, filename); + let log_cache_file = vfs::open_file(&log_cache_path, true, None)?; + match log_cache_file.read() { + Ok(content) => (response, content), + Err(e) => { + error!( + "HTTP: Failed to read log cache file {}: {:?}", + log_cache_path, e + ); + ( + http::server::HttpResponse::new(http::StatusCode::NOT_FOUND), + b"Log cache file not found".to_vec(), + ) + } + } + } else if path == "/status" { + let status_info = CacherStatus { + last_cached_block: state.last_cached_block, + chain_id: state.chain_id.clone(), + protocol_version: state.protocol_version.clone(), + next_cache_attempt_in_seconds: if state.is_cache_timer_live { + Some(state.cache_interval_s) + } else { + None + }, + manifest_filename: state.manifest.manifest_filename.clone(), + log_files_count: state.manifest.items.len() as u32, + our_address: our().to_string(), + is_providing: state.is_providing, + }; + match serde_json::to_vec(&status_info) { + Ok(body) => (response, body), + Err(e) => { + error!("HTTP: Failed to serialize status: {:?}", e); + ( + http::server::HttpResponse::new(http::StatusCode::INTERNAL_SERVER_ERROR), + b"Error serializing status".to_vec(), + ) + } + } + } else { + ( + http::server::HttpResponse::new(http::StatusCode::NOT_FOUND), + b"Not Found".to_vec(), + ) + }) +} + +fn handle_request( + our: &Address, + source: &Address, + state: &mut State, + request: CacherRequest, +) -> anyhow::Result<()> { + let is_local = is_local_request(our, source); + + // If we're still starting, respond with IsStarting to all requests + if state.is_starting { + Response::new().body(CacherResponse::IsStarting).send()?; + return Ok(()); + } + + if !is_local && source.process.to_string() != "dao-cacher:hypermap-cacher:sys" { + warn!("Rejecting remote request from non-dao-cacher: {source}"); + Response::new().body(CacherResponse::Rejected).send()?; + return Ok(()); + } + + if !is_local + && !state.is_providing + && source.process.to_string() == "dao-cacher:hypermap-cacher:sys" + { + warn!("Rejecting remote request from {source} - not in provider mode"); + Response::new().body(CacherResponse::Rejected).send()?; + return Ok(()); + } + let response_body = match request { + CacherRequest::GetManifest => { + let manifest_path = + format!("{}/{}", state.drive_path, state.manifest.manifest_filename); + if state.manifest.items.is_empty() && vfs::metadata(&manifest_path, None).is_err() { + CacherResponse::GetManifest(None) + } else { + // Ensure manifest is loaded from VFS if state is fresh and manifest file exists + // This is usually handled by State::load, but as a fallback: + if state.manifest.items.is_empty() { + // If manifest in memory is empty, try to load it + let manifest_file = vfs::open_file(&manifest_path, true, None)?; + if let Ok(bytes) = manifest_file.read() { + if let Ok(disk_manifest) = + serde_json::from_slice::(&bytes) + { + state.manifest = disk_manifest; + } + } + } + CacherResponse::GetManifest(Some(state.to_wit_manifest())) + } + } + CacherRequest::GetLogCacheContent(filename) => { + let log_cache_path = format!("{}/{}", state.drive_path, filename); + let log_cache_file = vfs::open_file(&log_cache_path, true, None)?; + match log_cache_file.read() { + Ok(content_bytes) => { + // Content is raw JSON bytes of LogCacheInternal. + // The WIT expects a string. + match String::from_utf8(content_bytes) { + Ok(content_str) => { + CacherResponse::GetLogCacheContent(Ok(Some(content_str))) + } + Err(e) => { + error!("Failed to convert log cache content to UTF-8 string: {}", e); + CacherResponse::GetLogCacheContent(Err(format!( + "File content not valid UTF-8: {}", + e + ))) + } + } + } + Err(_) => CacherResponse::GetLogCacheContent(Ok(None)), + } + } + CacherRequest::GetStatus => { + let status = CacherStatus { + last_cached_block: state.last_cached_block, + chain_id: state.chain_id.clone(), + protocol_version: state.protocol_version.clone(), + next_cache_attempt_in_seconds: if state.is_cache_timer_live { + Some(state.cache_interval_s) + } else { + None + }, + manifest_filename: state.manifest.manifest_filename.clone(), + log_files_count: state.manifest.items.len() as u32, + our_address: our.to_string(), + is_providing: state.is_providing, + }; + CacherResponse::GetStatus(status) + } + CacherRequest::GetLogsByRange(req_params) => { + let mut relevant_caches: Vec = Vec::new(); + let req_from_block = req_params.from_block; + // If req_params.to_block is None, we effectively want to go up to the highest block available in caches. + // For simplicity in overlap calculation, we can treat None as u64::MAX here. + let effective_req_to_block = req_params.to_block.unwrap_or(u64::MAX); + + for item in state.manifest.items.values() { + // Skip items that don't have an actual file (e.g., empty log ranges not written to disk). + if item.file_name.is_empty() { + continue; + } + + let cache_from = match item.metadata.from_block.parse::() { + Ok(b) => b, + Err(_) => { + warn!( + "Could not parse from_block for cache item {}: {}", + item.file_name, item.metadata.from_block + ); + continue; + } + }; + let cache_to = match item.metadata.to_block.parse::() { + Ok(b) => b, + Err(_) => { + warn!( + "Could not parse to_block for cache item {}: {}", + item.file_name, item.metadata.to_block + ); + continue; + } + }; + + // Check for overlap: max(start1, start2) <= min(end1, end2) + if max(req_from_block, cache_from) <= min(effective_req_to_block, cache_to) { + // This cache file overlaps with the requested range. + let file_vfs_path = format!("{}/{}", state.drive_path, item.file_name); + match vfs::open_file(&file_vfs_path, false, None) { + Ok(file) => match file.read() { + Ok(content_bytes) => { + match serde_json::from_slice::(&content_bytes) { + Ok(log_cache) => relevant_caches.push(log_cache), + Err(e) => { + error!( + "Failed to deserialize LogCacheInternal from {}: {:?}", + item.file_name, e + ); + // Decide: return error or skip this cache? For now, skip. + } + } + } + Err(e) => error!("Failed to read VFS file {}: {:?}", item.file_name, e), + }, + Err(e) => error!("Failed to open VFS file {}: {e:?}", item.file_name), + } + } + } + + // Sort caches by their from_block. + relevant_caches + .sort_by_key(|cache| cache.metadata.from_block.parse::().unwrap_or(0)); + + if relevant_caches.is_empty() { + CacherResponse::GetLogsByRange(Ok(GetLogsByRangeOkResponse::Latest( + state.last_cached_block, + ))) + } else { + match serde_json::to_string(&relevant_caches) { + Ok(json_string) => CacherResponse::GetLogsByRange(Ok( + GetLogsByRangeOkResponse::Logs((state.last_cached_block, json_string)), + )), + Err(e) => CacherResponse::GetLogsByRange(Err(format!( + "Failed to serialize relevant caches: {e}" + ))), + } + } + } + CacherRequest::StartProviding => { + if !is_local { + // should never happen: should be caught in check above + Response::new().body(CacherResponse::Rejected).send()?; + return Ok(()); + } + state.is_providing = true; + state.save(); + info!("Provider mode enabled"); + CacherResponse::StartProviding(Ok("Provider mode enabled".to_string())) + } + CacherRequest::StopProviding => { + if !is_local { + Response::new().body(CacherResponse::Rejected).send()?; + warn!("Rejecting remote request from {source} to alter provider mode"); + return Ok(()); + } + state.is_providing = false; + state.save(); + info!("Provider mode disabled"); + CacherResponse::StopProviding(Ok("Provider mode disabled".to_string())) + } + CacherRequest::SetNodes(new_nodes) => { + if !is_local { + Response::new().body(CacherResponse::Rejected).send()?; + warn!("Rejecting remote request from {source} to set nodes"); + return Ok(()); + } + state.nodes = new_nodes; + state.save(); + if let Err(e) = state.write_nodes_to_file() { + error!("Failed to write nodes to cache_sources: {:?}", e); + } + info!("Nodes updated to: {:?}", state.nodes); + CacherResponse::SetNodes(Ok("Nodes updated successfully".to_string())) + } + CacherRequest::Reset(custom_nodes) => { + if !is_local { + Response::new().body(CacherResponse::Rejected).send()?; + warn!("Rejecting remote request from {source} to reset"); + return Ok(()); + } + + info!("Resetting dao-cacher state and clearing VFS..."); + + // Clear all files from the drive + if let Err(e) = state.clear_drive() { + error!("Failed to clear drive during reset: {:?}", e); + CacherResponse::Reset(Err(format!("Failed to clear drive: {:?}", e))) + } else { + // Create new state with custom nodes if provided, otherwise use defaults + let nodes = match custom_nodes { + Some(nodes) => nodes, + None => DEFAULT_NODES.iter().map(|s| s.to_string()).collect(), + }; + + *state = State::new(&state.drive_path); + state.nodes = nodes; + state.save(); + if let Err(e) = state.write_nodes_to_file() { + error!("Failed to write nodes to cache_sources: {:?}", e); + } + info!( + "dao-cacher reset complete. New nodes: {:?}", + state.nodes + ); + CacherResponse::Reset(Ok( + "Reset completed successfully. DAO Cacher will restart with new settings." + .to_string(), + )) + } + } + }; + + Response::new().body(response_body).send()?; + Ok(()) +} + +fn main_loop( + our: &Address, + state: &mut State, + provider: ð::Provider, + server: &http::server::HttpServer, +) -> anyhow::Result<()> { + info!( + "Hypermap Binding Cacher main_loop started. Our address: {}", + our + ); + info!( + "Monitoring Binding contract: {}", + state.hypermap_binding_address.to_string() + ); + info!( + "Chain ID: {}, Protocol Version: {}", + state.chain_id, state.protocol_version + ); + info!("Last cached block: {}", state.last_cached_block); + + // Always bootstrap on start to get latest state from other nodes or RPC + while state.is_starting { + match state.bootstrap_state(provider) { + Ok(_) => info!("Bootstrap process completed successfully."), + Err(e) => { + error!("Error during bootstrap process: {:?}", e); + std::thread::sleep(std::time::Duration::from_secs(RETRY_DELAY_S)); + } + } + } + + // Set up the main caching timer. + info!( + "Setting cache timer for {} seconds.", + state.cache_interval_s + ); + timer::set_timer(state.cache_interval_s * 1000, Some(b"cache_cycle".to_vec())); + state.is_cache_timer_live = true; + state.save(); + + loop { + let Ok(message) = await_message() else { + warn!("Failed to get message, continuing loop."); + continue; + }; + let source = message.source(); + + if message.is_request() { + if source.process == ProcessId::from_str("http-server:distro:sys").unwrap() { + // HTTP request from the system's HTTP server process + let Ok(http::server::HttpServerRequest::Http(http_request)) = + server.parse_request(message.body()) + else { + error!("Failed to parse HTTP request from http-server:distro:sys"); + // Potentially send an error response back if possible/expected + continue; + }; + let (http_response, body) = http_handler(state, &http_request.path()?)?; + Response::new() + .body(serde_json::to_vec(&http_response).unwrap()) + .blob_bytes(body) + .send()?; + } else { + // Standard process-to-process request + match serde_json::from_slice::(message.body()) { + Ok(request) => { + if let Err(e) = handle_request(our, &source, state, request) { + error!("Error handling request from {:?}: {:?}", source, e); + } + } + Err(e) => { + error!( + "Failed to deserialize CacherRequest from {:?}: {:?}", + source, e + ); + } + } + } + } else { + // It's a Response or other kind of message + if source.process == ProcessId::from_str("timer:distro:sys").unwrap() { + if message.context() == Some(b"cache_cycle") { + info!("Cache timer triggered."); + state.is_cache_timer_live = false; + match state.cache_logs_and_update_manifest(provider) { + Ok(_) => info!("Periodic cache cycle complete."), + Err(e) => error!("Error during periodic cache cycle: {:?}", e), + } + // Reset the timer for the next cycle + if !state.is_cache_timer_live { + timer::set_timer( + state.cache_interval_s * 1000, + Some(b"cache_cycle".to_vec()), + ); + state.is_cache_timer_live = true; + state.save(); + } + } + } else { + debug!( + "Received unhandled response or other message from {:?}.", + source + ); + } + } + } +} + +call_init!(init); +fn init(our: Address) { + init_logging(Level::INFO, Level::DEBUG, None, None, None).unwrap(); + info!("Hypermap DAO Cacher process starting..."); + + let drive_path = vfs::create_drive(our.package_id(), "dao-cache", None).unwrap(); + // Create alternate drive for initfiles and read the test data + let alt_drive_path = vfs::create_drive(our.package_id(), "initfiles", None).unwrap(); + + // Try to read the cache_sources file from the initfiles drive + match vfs::open_file(&format!("{}/cache_sources", alt_drive_path), false, None) { + Ok(file) => match file.read() { + Ok(contents) => { + let content_str = String::from_utf8_lossy(&contents); + info!("Contents of cache_sources: {}", content_str); + } + Err(e) => { + info!("Failed to read cache_sources: {}", e); + } + }, + Err(e) => { + info!("Failed to open cache_sources: {}", e); + } + } + + let bind_config = http::server::HttpBindingConfig::default().authenticated(false); + let mut server = http::server::HttpServer::new(5); + + let provider = eth::Provider::new(dao::DAO_CHAIN_ID, 60); + + server + .bind_http_path("/manifest", bind_config.clone()) + .expect("Failed to bind /manifest"); + server + .bind_http_path("/manifest.json", bind_config.clone()) + .expect("Failed to bind /manifest.json"); + server + .bind_http_path("/log-cache/*", bind_config.clone()) + .expect("Failed to bind /log-cache/*"); + server + .bind_http_path("/status", bind_config.clone()) + .expect("Failed to bind /status"); + info!("Bound HTTP paths: /manifest, /log-cache/*, /status"); + + let mut state = State::load(&drive_path); + + // Wait for binding-cacher to be ready before entering main loop + let cacher_addr = Address::new("our", ("binding-cacher", "hypermap-cacher", "sys")); + info!( + "Waiting for binding-cacher at {} to report ready before starting dao-cacher...", + cacher_addr + ); + wait_for_process_ready( + cacher_addr, + b"\"GetStatus\"".to_vec(), + 15, + 2, + |body| { + let body_str = String::from_utf8_lossy(body); + if body_str.contains("IsStarting") || body_str.contains(r#""IsStarting""#) { + WaitClassification::Starting + } else if body_str.contains("GetStatus") || body_str.contains("last_cached_block") { + WaitClassification::Ready + } else { + WaitClassification::Unknown + } + }, + true, + None, + ); + info!("binding-cacher is ready; continuing dao-cacher startup."); + + loop { + match main_loop(&our, &mut state, &provider, &server) { + Ok(()) => { + // main_loop should not exit with Ok in normal operation as it's an infinite loop. + error!("main_loop exited unexpectedly with Ok. Restarting."); + } + Err(e) => { + error!("main_loop exited with error: {:?}. Restarting.", e); + std::thread::sleep(std::time::Duration::from_secs(5)); + } + } + // Reload state in case of restart, or re-initialize if necessary. + state = State::load(&drive_path); + } +} diff --git a/hyperdrive/packages/hypermap-cacher/hypermap-cacher/Cargo.toml b/hyperdrive/packages/hypermap-cacher/hypermap-cacher/Cargo.toml index 1b316cdfb..765548ffc 100644 --- a/hyperdrive/packages/hypermap-cacher/hypermap-cacher/Cargo.toml +++ b/hyperdrive/packages/hypermap-cacher/hypermap-cacher/Cargo.toml @@ -18,7 +18,7 @@ alloy = { version = "0.8.1", features = [ ] } chrono = "0.4.41" hex = "0.4.3" -hyperware_process_lib = { version = "2.3.0", features = ["logging"] } +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "34b1f1d857f76fb474f7af96f089961445758edb", features = ["logging"] } process_macros = "0.1.0" rand = "0.8" rmp-serde = "1.1.2" diff --git a/hyperdrive/packages/hypermap-cacher/hypermap-cacher/src/lib.rs b/hyperdrive/packages/hypermap-cacher/hypermap-cacher/src/lib.rs index cc98c1549..3ccf9bf88 100644 --- a/hyperdrive/packages/hypermap-cacher/hypermap-cacher/src/lib.rs +++ b/hyperdrive/packages/hypermap-cacher/hypermap-cacher/src/lib.rs @@ -24,7 +24,7 @@ use hyperware_process_lib::{ wit_bindgen::generate!({ path: "../target/wit", - world: "hypermap-cacher-sys-v1", + world: "hypermap-cacher-sys-v2", generate_unused_types: true, additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); diff --git a/hyperdrive/packages/hypermap-cacher/pkg/manifest.json b/hyperdrive/packages/hypermap-cacher/pkg/manifest.json index e4afacf23..f8034b401 100644 --- a/hyperdrive/packages/hypermap-cacher/pkg/manifest.json +++ b/hyperdrive/packages/hypermap-cacher/pkg/manifest.json @@ -7,6 +7,7 @@ "request_capabilities": [ "eth:distro:sys", "http-server:distro:sys", + "dao-cacher:hypermap-cacher:sys", "hypermap-cacher:hypermap-cacher:sys", "net:distro:sys", "sign:sign:sys", @@ -17,6 +18,7 @@ "grant_capabilities": [ "eth:distro:sys", "http-server:distro:sys", + "dao-cacher:hypermap-cacher:sys", "hypermap-cacher:hypermap-cacher:sys", "net:distro:sys", "sign:sign:sys", @@ -27,6 +29,36 @@ "public": false, "wit_version": 1 }, + { + "process_name": "dao-cacher", + "process_wasm_path": "/dao-cacher.wasm", + "on_exit": "Restart", + "request_networking": true, + "request_capabilities": [ + "eth:distro:sys", + "http-server:distro:sys", + "binding-cacher:hypermap-cacher:sys", + "hypr-dao:hypr-dao:ware.hypr", + "net:distro:sys", + "sign:sign:sys", + "terminal:terminal:sys", + "timer:distro:sys", + "vfs:distro:sys" + ], + "grant_capabilities": [ + "eth:distro:sys", + "http-server:distro:sys", + "binding-cacher:hypermap-cacher:sys", + "hypr-dao:hypr-dao:ware.hypr", + "net:distro:sys", + "sign:sign:sys", + "terminal:terminal:sys", + "timer:distro:sys", + "vfs:distro:sys" + ], + "public": false, + "wit_version": 1 + }, { "process_name": "hypermap-cacher", "process_wasm_path": "/hypermap-cacher.wasm", diff --git a/hyperdrive/packages/hypermap-cacher/pkg/scripts.json b/hyperdrive/packages/hypermap-cacher/pkg/scripts.json index 5b1bfaefe..fc774921d 100644 --- a/hyperdrive/packages/hypermap-cacher/pkg/scripts.json +++ b/hyperdrive/packages/hypermap-cacher/pkg/scripts.json @@ -5,10 +5,12 @@ "request_networking": false, "request_capabilities": [ "binding-cacher:hypermap-cacher:sys", + "dao-cacher:hypermap-cacher:sys", "hypermap-cacher:hypermap-cacher:sys" ], "grant_capabilities": [ "binding-cacher:hypermap-cacher:sys", + "dao-cacher:hypermap-cacher:sys", "hypermap-cacher:hypermap-cacher:sys" ], "wit_version": 1 @@ -19,10 +21,12 @@ "request_networking": false, "request_capabilities": [ "binding-cacher:hypermap-cacher:sys", + "dao-cacher:hypermap-cacher:sys", "hypermap-cacher:hypermap-cacher:sys" ], "grant_capabilities": [ "binding-cacher:hypermap-cacher:sys", + "dao-cacher:hypermap-cacher:sys", "hypermap-cacher:hypermap-cacher:sys" ], "wit_version": 1 @@ -33,10 +37,12 @@ "request_networking": false, "request_capabilities": [ "binding-cacher:hypermap-cacher:sys", + "dao-cacher:hypermap-cacher:sys", "hypermap-cacher:hypermap-cacher:sys" ], "grant_capabilities": [ "binding-cacher:hypermap-cacher:sys", + "dao-cacher:hypermap-cacher:sys", "hypermap-cacher:hypermap-cacher:sys" ], "wit_version": 1 @@ -47,10 +53,12 @@ "request_networking": false, "request_capabilities": [ "binding-cacher:hypermap-cacher:sys", + "dao-cacher:hypermap-cacher:sys", "hypermap-cacher:hypermap-cacher:sys" ], "grant_capabilities": [ "binding-cacher:hypermap-cacher:sys", + "dao-cacher:hypermap-cacher:sys", "hypermap-cacher:hypermap-cacher:sys" ], "wit_version": 1 diff --git a/hyperdrive/packages/hypermap-cacher/reset-cache/Cargo.toml b/hyperdrive/packages/hypermap-cacher/reset-cache/Cargo.toml index a796f9b13..3e341f9a2 100644 --- a/hyperdrive/packages/hypermap-cacher/reset-cache/Cargo.toml +++ b/hyperdrive/packages/hypermap-cacher/reset-cache/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" publish = false [dependencies] -hyperware_process_lib = { version = "2.3.0" } +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "34b1f1d857f76fb474f7af96f089961445758edb" } process_macros = "0.1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/hyperdrive/packages/hypermap-cacher/reset-cache/src/lib.rs b/hyperdrive/packages/hypermap-cacher/reset-cache/src/lib.rs index 495847667..65bf5c865 100644 --- a/hyperdrive/packages/hypermap-cacher/reset-cache/src/lib.rs +++ b/hyperdrive/packages/hypermap-cacher/reset-cache/src/lib.rs @@ -13,12 +13,13 @@ //! reset:hypermap-cacher:sys alice.os bob.os # Reset with custom nodes use crate::hyperware::process::binding_cacher::{BindingCacherRequest, BindingCacherResponse}; +use crate::hyperware::process::dao_cacher::{DaoCacherRequest, DaoCacherResponse}; use crate::hyperware::process::hypermap_cacher::{CacherRequest, CacherResponse}; use hyperware_process_lib::{await_next_message_body, call_init, println, Address, Request}; wit_bindgen::generate!({ path: "../target/wit", - world: "hypermap-cacher-sys-v1", + world: "hypermap-cacher-sys-v2", generate_unused_types: true, additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); @@ -42,6 +43,7 @@ fn init(_our: Address) { Some(nodes) }; let binding_custom_nodes = custom_nodes.clone(); + let dao_custom_nodes = custom_nodes.clone(); let response = Request::to(("our", "hypermap-cacher", "hypermap-cacher", "sys")) .body(CacherRequest::Reset(custom_nodes)) @@ -90,4 +92,28 @@ fn init(_our: Address) { println!("✗ Communication error: {:?}", err); } } + + let response = Request::to(("our", "dao-cacher", "hypermap-cacher", "sys")) + .body(DaoCacherRequest::Reset(dao_custom_nodes)) + .send_and_await_response(10); // Give it more time for reset operations + + match response { + Ok(Ok(message)) => match message.body().try_into() { + Ok(DaoCacherResponse::Reset(Ok(msg))) => { + println!("✓ {}", msg); + } + Ok(DaoCacherResponse::Reset(Err(err))) => { + println!("✗ Failed to reset dao-cacher: {}", err); + } + _ => { + println!("✗ Unexpected response from dao-cacher"); + } + }, + Ok(Err(err)) => { + println!("✗ Request failed: {:?}", err); + } + Err(err) => { + println!("✗ Communication error: {:?}", err); + } + } } diff --git a/hyperdrive/packages/hypermap-cacher/set-nodes/Cargo.toml b/hyperdrive/packages/hypermap-cacher/set-nodes/Cargo.toml index 1b3538908..97d80aa99 100644 --- a/hyperdrive/packages/hypermap-cacher/set-nodes/Cargo.toml +++ b/hyperdrive/packages/hypermap-cacher/set-nodes/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" publish = false [dependencies] -hyperware_process_lib = { version = "2.3.0" } +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "34b1f1d857f76fb474f7af96f089961445758edb" } process_macros = "0.1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/hyperdrive/packages/hypermap-cacher/set-nodes/src/lib.rs b/hyperdrive/packages/hypermap-cacher/set-nodes/src/lib.rs index c8479ef9e..d15806b66 100644 --- a/hyperdrive/packages/hypermap-cacher/set-nodes/src/lib.rs +++ b/hyperdrive/packages/hypermap-cacher/set-nodes/src/lib.rs @@ -11,12 +11,13 @@ //! set-nodes:hypermap-cacher:sys alice.os bob.os charlie.os use crate::hyperware::process::binding_cacher::{BindingCacherRequest, BindingCacherResponse}; +use crate::hyperware::process::dao_cacher::{DaoCacherRequest, DaoCacherResponse}; use crate::hyperware::process::hypermap_cacher::{CacherRequest, CacherResponse}; use hyperware_process_lib::{await_next_message_body, call_init, println, Address, Request}; wit_bindgen::generate!({ path: "../target/wit", - world: "hypermap-cacher-sys-v1", + world: "hypermap-cacher-sys-v2", generate_unused_types: true, additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); @@ -39,6 +40,7 @@ fn init(_our: Address) { let nodes: Vec = parts.iter().map(|s| s.to_string()).collect(); let binding_nodes = nodes.clone(); + let dao_nodes = nodes.clone(); println!("Setting hypermap-cacher nodes to: {:?}", nodes); @@ -91,4 +93,30 @@ fn init(_our: Address) { println!("✗ Communication error: {:?}", err); } } + + println!("Setting dao-cacher nodes to: {:?}", dao_nodes); + + let response = Request::to(("our", "dao-cacher", "hypermap-cacher", "sys")) + .body(DaoCacherRequest::SetNodes(dao_nodes)) + .send_and_await_response(5); + + match response { + Ok(Ok(message)) => match message.body().try_into() { + Ok(DaoCacherResponse::SetNodes(Ok(msg))) => { + println!("✓ {}", msg); + } + Ok(DaoCacherResponse::SetNodes(Err(err))) => { + println!("✗ Failed to set nodes for dao-cacher: {}", err); + } + _ => { + println!("✗ Unexpected response from dao-cacher"); + } + }, + Ok(Err(err)) => { + println!("✗ Request failed: {:?}", err); + } + Err(err) => { + println!("✗ Communication error: {:?}", err); + } + } } diff --git a/hyperdrive/packages/hypermap-cacher/start-providing/Cargo.toml b/hyperdrive/packages/hypermap-cacher/start-providing/Cargo.toml index 3797fb50a..aa2d7b15b 100644 --- a/hyperdrive/packages/hypermap-cacher/start-providing/Cargo.toml +++ b/hyperdrive/packages/hypermap-cacher/start-providing/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" publish = false [dependencies] -hyperware_process_lib = { version = "2.3.0" } +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "34b1f1d857f76fb474f7af96f089961445758edb" } process_macros = "0.1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/hyperdrive/packages/hypermap-cacher/start-providing/src/lib.rs b/hyperdrive/packages/hypermap-cacher/start-providing/src/lib.rs index f74f7abe7..bc8fc6263 100644 --- a/hyperdrive/packages/hypermap-cacher/start-providing/src/lib.rs +++ b/hyperdrive/packages/hypermap-cacher/start-providing/src/lib.rs @@ -1,10 +1,11 @@ use crate::hyperware::process::binding_cacher::{BindingCacherRequest, BindingCacherResponse}; +use crate::hyperware::process::dao_cacher::{DaoCacherRequest, DaoCacherResponse}; use crate::hyperware::process::hypermap_cacher::{CacherRequest, CacherResponse}; use hyperware_process_lib::{call_init, println, Address, Request}; wit_bindgen::generate!({ path: "../target/wit", - world: "hypermap-cacher-sys-v1", + world: "hypermap-cacher-sys-v2", generate_unused_types: true, additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); @@ -62,4 +63,30 @@ fn init(_our: Address) { println!("✗ Communication error: {:?}", err); } } + + println!("Enabling dao-cacher provider mode..."); + + let response = Request::to(("our", "dao-cacher", "hypermap-cacher", "sys")) + .body(DaoCacherRequest::StartProviding) + .send_and_await_response(5); + + match response { + Ok(Ok(message)) => match message.body().try_into() { + Ok(DaoCacherResponse::StartProviding(Ok(msg))) => { + println!("✓ {}", msg); + } + Ok(DaoCacherResponse::StartProviding(Err(err))) => { + println!("✗ Failed to enable dao-cacher provider mode: {}", err); + } + _ => { + println!("✗ Unexpected response from dao-cacher"); + } + }, + Ok(Err(err)) => { + println!("✗ Request failed: {:?}", err); + } + Err(err) => { + println!("✗ Communication error: {:?}", err); + } + } } diff --git a/hyperdrive/packages/hypermap-cacher/stop-providing/Cargo.toml b/hyperdrive/packages/hypermap-cacher/stop-providing/Cargo.toml index 4f75e6add..e16795a4d 100644 --- a/hyperdrive/packages/hypermap-cacher/stop-providing/Cargo.toml +++ b/hyperdrive/packages/hypermap-cacher/stop-providing/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" publish = false [dependencies] -hyperware_process_lib = { version = "2.3.0" } +hyperware_process_lib = { git = "https://github.com/hyperware-ai/process_lib", rev = "34b1f1d857f76fb474f7af96f089961445758edb" } process_macros = "0.1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/hyperdrive/packages/hypermap-cacher/stop-providing/src/lib.rs b/hyperdrive/packages/hypermap-cacher/stop-providing/src/lib.rs index e6d55b831..ae39d2972 100644 --- a/hyperdrive/packages/hypermap-cacher/stop-providing/src/lib.rs +++ b/hyperdrive/packages/hypermap-cacher/stop-providing/src/lib.rs @@ -1,10 +1,11 @@ use crate::hyperware::process::binding_cacher::{BindingCacherRequest, BindingCacherResponse}; +use crate::hyperware::process::dao_cacher::{DaoCacherRequest, DaoCacherResponse}; use crate::hyperware::process::hypermap_cacher::{CacherRequest, CacherResponse}; use hyperware_process_lib::{call_init, println, Address, Request}; wit_bindgen::generate!({ path: "../target/wit", - world: "hypermap-cacher-sys-v1", + world: "hypermap-cacher-sys-v2", generate_unused_types: true, additional_derives: [serde::Deserialize, serde::Serialize, process_macros::SerdeJsonInto], }); @@ -62,4 +63,30 @@ fn init(_our: Address) { println!("✗ Communication error: {:?}", err); } } + + println!("Disabling dao-cacher provider mode..."); + + let response = Request::to(("our", "dao-cacher", "hypermap-cacher", "sys")) + .body(DaoCacherRequest::StopProviding) + .send_and_await_response(5); + + match response { + Ok(Ok(message)) => match message.body().try_into() { + Ok(DaoCacherResponse::StopProviding(Ok(msg))) => { + println!("✓ {}", msg); + } + Ok(DaoCacherResponse::StopProviding(Err(err))) => { + println!("✗ Failed to disable dao-cacher provider mode: {}", err); + } + _ => { + println!("✗ Unexpected response from dao-cacher"); + } + }, + Ok(Err(err)) => { + println!("✗ Request failed: {:?}", err); + } + Err(err) => { + println!("✗ Communication error: {:?}", err); + } + } } diff --git a/hyperdrive/packages/spider/Cargo.lock b/hyperdrive/packages/spider/Cargo.lock index a2f073349..3f8fb3310 100644 --- a/hyperdrive/packages/spider/Cargo.lock +++ b/hyperdrive/packages/spider/Cargo.lock @@ -1703,9 +1703,9 @@ dependencies = [ ] [[package]] -name = "hyperprocess_macro" -version = "0.1.0" -source = "git+https://github.com/hyperware-ai/hyperprocess-macro?rev=98fac21#98fac2177df21a39a0950d48b248e1b12013f7f0" +name = "hyperapp_macro" +version = "0.1.2" +source = "git+https://github.com/hyperware-ai/hyperapp-macro?rev=5c7cc7a#5c7cc7a8dfb366bdc767d9dc3e2b954dc2cc786c" dependencies = [ "hyperware_process_lib", "proc-macro2", @@ -1716,7 +1716,7 @@ dependencies = [ [[package]] name = "hyperware-anthropic-sdk" version = "0.1.0" -source = "git+https://github.com/hyperware-ai/hyperware-anthropic-sdk?rev=607bbc6#607bbc6da2601b06f13729a846ebe2419e1dcf6c" +source = "git+https://github.com/hyperware-ai/hyperware-anthropic-sdk?rev=1fc7b0c#1fc7b0c1a0a8f6e8f706cf837bea8173c3198b31" dependencies = [ "hyperware_process_lib", "rand 0.8.5", @@ -1729,7 +1729,7 @@ dependencies = [ [[package]] name = "hyperware-elevenlabs-tts" version = "0.1.0" -source = "git+https://github.com/hyperware-ai/hyperware-elevenlabs-tts?rev=65899de#65899dea2b081ff982404c687040dd0bc92ed917" +source = "git+https://github.com/hyperware-ai/hyperware-elevenlabs-tts?rev=1e15f92#1e15f9299b4c5397e770904f97d23f7db4d68cc5" dependencies = [ "http", "hyperware_process_lib", @@ -1742,7 +1742,7 @@ dependencies = [ [[package]] name = "hyperware-openai-stt" version = "0.1.0" -source = "git+https://github.com/hyperware-ai/hyperware-openai-ttstt?rev=0e6bf83#0e6bf834c39ebe3ab6624829848b50c83f18201f" +source = "git+https://github.com/hyperware-ai/hyperware-openai-ttstt?rev=0887b2b#0887b2baa619503daa45c9cf07c9bd281c5352c0" dependencies = [ "base64 0.22.1", "http", @@ -1757,7 +1757,7 @@ dependencies = [ [[package]] name = "hyperware-openai-tts" version = "0.1.0" -source = "git+https://github.com/hyperware-ai/hyperware-openai-ttstt?rev=0e6bf83#0e6bf834c39ebe3ab6624829848b50c83f18201f" +source = "git+https://github.com/hyperware-ai/hyperware-openai-ttstt?rev=0887b2b#0887b2baa619503daa45c9cf07c9bd281c5352c0" dependencies = [ "http", "hyperware_process_lib", @@ -1780,7 +1780,7 @@ dependencies = [ [[package]] name = "hyperware_process_lib" version = "2.2.0" -source = "git+https://github.com/hyperware-ai/process_lib?rev=232fe25#232fe2526f383c88efc2ef3a289deba2e193d68f" +source = "git+https://github.com/hyperware-ai/process_lib?rev=41f25ce#41f25ceb30935929416137162eb90f101dc720c5" dependencies = [ "alloy", "alloy-primitives", @@ -1790,6 +1790,7 @@ dependencies = [ "base64 0.22.1", "bincode", "color-eyre", + "futures-channel", "futures-util", "http", "mime_guess", @@ -3179,7 +3180,7 @@ dependencies = [ "base64 0.21.7", "chrono", "http", - "hyperprocess_macro", + "hyperapp_macro", "hyperware-anthropic-sdk", "hyperware-parse-wit", "hyperware_process_lib", @@ -3188,29 +3189,12 @@ dependencies = [ "serde", "serde_json", "sha2", - "spider_caller_utils", "url", "uuid", "wit-bindgen 0.42.1", "wit-parser 0.220.1", ] -[[package]] -name = "spider_caller_utils" -version = "0.1.0" -dependencies = [ - "anyhow", - "futures", - "futures-util", - "hyperware_process_lib", - "once_cell", - "process_macros", - "serde", - "serde_json", - "uuid", - "wit-bindgen 0.41.0", -] - [[package]] name = "spki" version = "0.7.3" @@ -3640,7 +3624,7 @@ dependencies = [ "async-trait", "base64 0.21.7", "chrono", - "hyperprocess_macro", + "hyperapp_macro", "hyperware-elevenlabs-tts", "hyperware-openai-stt", "hyperware-openai-tts", @@ -3648,7 +3632,6 @@ dependencies = [ "process_macros", "serde", "serde_json", - "spider_caller_utils", "uuid", "wit-bindgen 0.42.1", ] diff --git a/hyperdrive/packages/spider/spider/Cargo.toml b/hyperdrive/packages/spider/spider/Cargo.toml index 6f5b2539a..88294fb40 100644 --- a/hyperdrive/packages/spider/spider/Cargo.toml +++ b/hyperdrive/packages/spider/spider/Cargo.toml @@ -33,6 +33,10 @@ rev = "41f25ce" features = ["derive"] version = "1.0" +[dependencies.spider_caller_utils] +optional = true +path = "../target/spider-caller-utils" + [dependencies.uuid] features = [ "v4", @@ -45,6 +49,7 @@ features = ["serde"] version = "0.220.0" [features] +caller-utils = ["spider_caller_utils"] public-mode = [] simulation-mode = [] diff --git a/hyperdrive/packages/spider/ttstt/Cargo.toml b/hyperdrive/packages/spider/ttstt/Cargo.toml index 76633dee1..78b337fd9 100644 --- a/hyperdrive/packages/spider/ttstt/Cargo.toml +++ b/hyperdrive/packages/spider/ttstt/Cargo.toml @@ -35,6 +35,10 @@ rev = "41f25ce" features = ["derive"] version = "1.0" +[dependencies.spider_caller_utils] +optional = true +path = "../target/spider-caller-utils" + [dependencies.uuid] features = [ "v4", @@ -43,6 +47,7 @@ features = [ version = "1.0" [features] +caller-utils = ["spider_caller_utils"] simulation-mode = [] [lib] diff --git a/hyperdrive/src/net/mod.rs b/hyperdrive/src/net/mod.rs index c55e0c94e..6a240bd35 100644 --- a/hyperdrive/src/net/mod.rs +++ b/hyperdrive/src/net/mod.rs @@ -88,11 +88,11 @@ pub async fn networking( match &ext.our.routing { NodeRouting::Direct { ip, ports } => { if *ext.our_ip != *ip { - return Err(anyhow::anyhow!( - "net: fatal error: IP address mismatch: {} != {}, update your HNS identity", - ext.our_ip, - ip - )); + let message = format!( + "IP address mismatch detected. Detected from environment: {}, node's data: {}. Please confirm your node is reachable at {} or restart your node to reset your networking data.", + ext.our_ip, ip, ip + ); + utils::print_loud(&ext.print_tx, &message).await; } utils::print_debug(&ext.print_tx, "going online as a direct node").await; if !ports.contains_key(WS_PROTOCOL) && !ports.contains_key(TCP_PROTOCOL) { diff --git a/hyperdrive/src/register-ui/src/App.tsx b/hyperdrive/src/register-ui/src/App.tsx index fbce93750..1c52c1e9d 100644 --- a/hyperdrive/src/register-ui/src/App.tsx +++ b/hyperdrive/src/register-ui/src/App.tsx @@ -9,7 +9,7 @@ import Login from './pages/Login' import ResetName from './pages/ResetName' import HyperdriveHome from "./pages/HyperdriveHome" import ImportKeyfile from "./pages/ImportKeyfile"; -import { UnencryptedIdentity } from "./lib/types"; +import { InfoResponse } from "./lib/types"; import Header from "./components/Header"; import ProgressBar from "./components/ProgressBar"; import { LargeBackgroundVector } from "./components/LargeBackgroundVector"; @@ -23,6 +23,7 @@ function App() { const [keyFileName, setKeyFileName] = useState(''); const [reset, setReset] = useState(false); const [direct, setDirect] = useState(false); + const [directNodeIp, setDirectNodeIp] = useState(''); const [upgradable, setUpgradable] = useState(false); const [hnsName, setHnsName] = useState(''); const [networkingKey, setNetworkingKey] = useState(''); @@ -44,17 +45,28 @@ function App() { try { const infoResponse = await fetch('/info', { method: 'GET', credentials: 'include' }) + const info: InfoResponse | null = await infoResponse.json().catch(() => null) + if (infoResponse.status > 399) { console.log('no info, unbooted') - } else { - const info: UnencryptedIdentity = await infoResponse.json() + } - if (initialVisit) { - setHnsName(info.name) - setRouters(info.allowed_routers) + if (info) { + if (infoResponse.status < 400 && initialVisit) { + if (info.name) { + setHnsName(info.name) + } + if (info.allowed_routers) { + setRouters(info.allowed_routers) + } setNavigateToLogin(true) setInitialVisit(false) } + + // Set detected IP address if available + if (info.detected_ip_address) { + setDirectNodeIp(info.detected_ip_address) + } } } catch { console.log('no info, unbooted') @@ -79,11 +91,13 @@ function App() { useEffect(() => setNavigateToLogin(false), [initialVisit]) + // just pass all the props each time since components won't mind extras // todo, most of these can be removed... const props = { upgradable, setUpgradable, direct, setDirect, + directNodeIp, setDirectNodeIp, key, keyFileName, setKeyFileName, reset, setReset, @@ -99,52 +113,52 @@ function App() { } return ( - <> -
-
- - -
- - - - : - } /> - - - - - } /> - - - - - } /> - - - - - } /> - } /> - } /> - } /> - - - - - } /> - -
- -
- + <> +
+
+ + +
+ + + + : + } /> + + + + + } /> + + + + + } /> + + + + + } /> + } /> + } /> + } /> + + + + + } /> + +
+ +
+ ) } diff --git a/hyperdrive/src/register-ui/src/abis/helpers.ts b/hyperdrive/src/register-ui/src/abis/helpers.ts index d57479383..09414555b 100644 --- a/hyperdrive/src/register-ui/src/abis/helpers.ts +++ b/hyperdrive/src/register-ui/src/abis/helpers.ts @@ -1,4 +1,3 @@ - import { NetworkingInfo } from "../lib/types"; import { hyperhash } from "../utils/hyperhash"; import { ipToBytes, portToBytes } from "../utils/hns_encoding"; @@ -15,6 +14,7 @@ const encodeRouters = (routers: string[]): `0x${string}` => { export const generateNetworkingKeys = async ({ upgradable, direct, + directNodeIp, setNetworkingKey, setWsPort, setTcpPort, @@ -25,6 +25,7 @@ export const generateNetworkingKeys = async ({ }: { upgradable: boolean, direct: boolean, + directNodeIp?: string, label: string, our_address: `0x${string}`, setNetworkingKey: (networkingKey: string) => void; @@ -49,7 +50,9 @@ export const generateNetworkingKeys = async ({ (res) => res.json() )) as NetworkingInfo; - const ipAddress = ipToBytes(ip_address); + // Use directNodeIp if provided and direct is true, otherwise use the generated IP + const ipToUse = direct && directNodeIp ? directNodeIp : ip_address; + const ipAddress = ipToBytes(ipToUse); const routersToUse = customRouters && customRouters.length > 0 ? customRouters : allowed_routers; @@ -60,6 +63,7 @@ export const generateNetworkingKeys = async ({ setRouters(routersToUse); console.log("networking_key: ", networking_key); + console.log("IP address being used: ", ipToUse); console.log("routers being used: ", routersToUse); const netkeycall = encodeFunctionData({ diff --git a/hyperdrive/src/register-ui/src/lib/types.ts b/hyperdrive/src/register-ui/src/lib/types.ts index 5d80f7ede..284149f76 100644 --- a/hyperdrive/src/register-ui/src/lib/types.ts +++ b/hyperdrive/src/register-ui/src/lib/types.ts @@ -12,6 +12,8 @@ export interface PageProps { setRouters: React.Dispatch>, direct: boolean, setDirect: React.Dispatch>, + directNodeIp: string, + setDirectNodeIp: React.Dispatch>, upgradable: boolean, setUpgradable: React.Dispatch>, hnsName: string, @@ -51,6 +53,13 @@ export type InfoResponse = { allowed_routers?: string[]; initial_cache_sources: string[]; initial_base_l2_providers: string[]; + uses_direct_networking?: boolean; + hns_ip_address?: string; + detected_ip_address?: string; +} + +export type IPv4Response = { + ip: string; } export interface RpcProviderConfig { diff --git a/hyperdrive/src/register-ui/src/pages/CommitDotOsName.tsx b/hyperdrive/src/register-ui/src/pages/CommitDotOsName.tsx index bcbd4ee15..341d18469 100644 --- a/hyperdrive/src/register-ui/src/pages/CommitDotOsName.tsx +++ b/hyperdrive/src/register-ui/src/pages/CommitDotOsName.tsx @@ -1,3 +1,4 @@ + import { useState, useEffect, FormEvent, useCallback } from "react"; import { useNavigate } from "react-router-dom"; import { toAscii } from "idna-uts46-hx"; @@ -18,16 +19,21 @@ interface RegisterOsNameProps extends PageProps { } // Regex for valid router names (domain format) const ROUTER_NAME_REGEX = /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)*$/; +// IPv4 validation regex +const IPV4_REGEX = /^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/; + function CommitDotOsName({ - direct, - setDirect, - setHnsName, - setNetworkingKey, - setIpAddress, - setWsPort, - setTcpPort, - setRouters, -}: RegisterOsNameProps) { + direct, + setDirect, + directNodeIp, + setDirectNodeIp, + setHnsName, + setNetworkingKey, + setIpAddress, + setWsPort, + setTcpPort, + setRouters, + }: RegisterOsNameProps) { let { address } = useAccount(); let navigate = useNavigate(); let { openConnectModal } = useConnectModal(); @@ -53,6 +59,40 @@ function CommitDotOsName({ const [customRouters, setCustomRouters] = useState('') const [routerValidationErrors, setRouterValidationErrors] = useState([]) + // Track the initial IPv4 value to determine if it was auto-detected + const [initialDirectNodeIp, setInitialDirectNodeIp] = useState('') + + // Capture the initial directNodeIp value when component mounts + useEffect(() => { + setInitialDirectNodeIp(directNodeIp); + }, []); // Empty dependency array means this runs once on mount + + // Validation function for IPv4 + const isValidIPv4 = (ip: string): boolean => { + return IPV4_REGEX.test(ip); + }; + + // Check if direct node configuration is valid + const isDirectNodeValid = (): boolean => { + if (!direct) return true; // Not required if checkbox is unchecked + return directNodeIp.trim() !== '' && isValidIPv4(directNodeIp.trim()); + }; + + // Determine the appropriate label for the Direct Node IP field + const getDirectNodeIpLabel = (): string => { + const hasValidInitialIp = initialDirectNodeIp && isValidIPv4(initialDirectNodeIp); + + if (!hasValidInitialIp) { + return "Direct Node IP Address (IPv4)"; + } + + if (directNodeIp === initialDirectNodeIp) { + return "Direct Node IP Address (as detected)"; + } + + return "Direct Node IP Address (overridden by user)"; + }; + // Modified setDirect function - no longer clears custom routers const handleSetDirect = (value: boolean) => { setDirect(value); @@ -226,6 +266,35 @@ function CommitDotOsName({ Advanced Network Options
+ {direct && ( +
+ + setDirectNodeIp(e.target.value)} + placeholder="e.g., 192.168.1.100" + className={`input ${ + direct && !isDirectNodeValid() + ? 'border-red-500 focus:border-red-500' + : '' + }`} + /> + {direct && directNodeIp.trim() && !isValidIPv4(directNodeIp.trim()) && ( + + Please enter a valid IPv4 address + + )} + {direct && !directNodeIp.trim() && ( + + IP address is required for direct nodes + + )} +
+ )} {specifyRouters && (
@@ -272,6 +341,7 @@ function CommitDotOsName({ isPending || isConfirming || nameValidities.length !== 0 || + (direct && !isDirectNodeValid()) || (specifyRouters && !isCustomRoutersValid()) } > diff --git a/hyperdrive/src/register-ui/src/pages/Login.tsx b/hyperdrive/src/register-ui/src/pages/Login.tsx index 84709693b..c4b78d556 100644 --- a/hyperdrive/src/register-ui/src/pages/Login.tsx +++ b/hyperdrive/src/register-ui/src/pages/Login.tsx @@ -1,4 +1,3 @@ - import { FormEvent, useCallback, useEffect, useState } from "react"; import { PageProps, InfoResponse } from "../lib/types"; import Loader from "../components/Loader"; @@ -40,6 +39,10 @@ function Login({ const [cacheSourceValidationErrors, setCacheSourceValidationErrors] = useState([]); const [specifyBaseL2AccessProviders, setSpecifyBaseL2AccessProviders] = useState(false); const [rpcProviders, setRpcProviders] = useState([]); + const [usesDirectNetworking, setUsesDirectNetworking] = useState(false); + const [hnsIpAddress, setHnsIpAddress] = useState(""); + const [detectedIpAddress, setDetectedIpAddress] = useState(""); + const [acknowledgeIpMismatch, setAcknowledgeIpMismatch] = useState(false); // Track initial states after data is loaded const [initialCacheSourcesChecked, setInitialCacheSourcesChecked] = useState(false); @@ -64,6 +67,17 @@ function Login({ setHnsName(infoData.name); } + // Set networking information + if (infoData.uses_direct_networking !== undefined) { + setUsesDirectNetworking(infoData.uses_direct_networking); + } + if (infoData.hns_ip_address) { + setHnsIpAddress(infoData.hns_ip_address); + } + if (infoData.detected_ip_address) { + setDetectedIpAddress(infoData.detected_ip_address); + } + // Prepopulate cache sources if (infoData.initial_cache_sources && infoData.initial_cache_sources.length > 0) { setCustomCacheSources(infoData.initial_cache_sources.join('\n')); @@ -133,6 +147,8 @@ function Login({ return errors; }; + const hasIpMismatch = usesDirectNetworking && hnsIpAddress && detectedIpAddress && hnsIpAddress !== detectedIpAddress; + // Handle custom cache sources change with validation const handleCustomCacheSourcesChange = (value: string) => { setCustomCacheSources(value); @@ -253,6 +269,12 @@ function Login({ rpcProviders.some(p => !p.url.trim() || !validateWebSocketUrl(p.url) || (p.auth && !p.auth.value.trim())) ); + // Check if login button should be disabled + const isLoginDisabled = + (specifyCacheSources && !isCustomCacheSourcesValid()) || + hasInvalidRpcProviders || + (hasIpMismatch && !acknowledgeIpMismatch); + return
{loading &&
@@ -281,6 +303,32 @@ function Login({ onChange={(e) => setPw(e.target.value)} autoFocus /> + {/* IP Mismatch Warning */} + {hasIpMismatch && ( +
+
+ ⚠️ IP Address Mismatch Detected +
+
+
HNS IP: {hnsIpAddress}
(your node's published Hypermap directory address)
+
Detected IP: {detectedIpAddress}
(your node's likely current address)
+
+
+ Please use Reset Password & Networking Info to reset your networking information, or acknowledge the following to proceed with login. +
+ +
+ )}
{/* Advanced Options Section */} @@ -356,7 +404,7 @@ function Login({