From 726594167ad24f51d4e7e9400ead1e1b99ce3912 Mon Sep 17 00:00:00 2001 From: Giwook-Han Date: Thu, 6 Nov 2025 14:59:16 +0900 Subject: [PATCH 01/29] init: add k8s yaml --- Cargo.lock | 138 ++++++++++++++++++------------------- k8s/deploy.sequencer.yaml | 46 +++++++++++++ k8s/service.sequencer.yaml | 14 ++++ 3 files changed, 129 insertions(+), 69 deletions(-) create mode 100644 k8s/deploy.sequencer.yaml create mode 100644 k8s/service.sequencer.yaml diff --git a/Cargo.lock b/Cargo.lock index 798d4b5..15d483a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -222,7 +222,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62945a2f7e6de02a31fe400aa489f0e0f5b2502e69f95f853adb82a96c7a6b60" dependencies = [ "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -235,7 +235,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -274,7 +274,7 @@ checksum = "213888f660fddcca0d257e88e54ac05bca01885f258ccdf695bafd77031bb69d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -316,7 +316,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -357,7 +357,7 @@ checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -532,7 +532,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -756,7 +756,7 @@ checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -988,7 +988,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -1431,7 +1431,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -1474,7 +1474,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -1488,7 +1488,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -1499,7 +1499,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -1510,7 +1510,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -1583,7 +1583,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -1593,7 +1593,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -1623,7 +1623,7 @@ dependencies = [ "convert_case 0.6.0", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", "unicode-xid", ] @@ -1636,7 +1636,7 @@ dependencies = [ "convert_case 0.7.1", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -1723,7 +1723,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -1796,7 +1796,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -1884,7 +1884,7 @@ checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -2091,7 +2091,7 @@ dependencies = [ "reqwest 0.11.27", "serde", "serde_json", - "syn 2.0.108", + "syn 2.0.109", "toml", "walkdir", ] @@ -2109,7 +2109,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -2135,7 +2135,7 @@ dependencies = [ "serde", "serde_json", "strum 0.26.3", - "syn 2.0.108", + "syn 2.0.109", "tempfile", "thiserror 1.0.69", "tiny-keccak", @@ -2973,7 +2973,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -3436,7 +3436,7 @@ dependencies = [ "http 1.3.1", "hyper 1.7.0", "hyper-util", - "rustls 0.23.34", + "rustls 0.23.35", "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", @@ -3679,7 +3679,7 @@ checksum = "a0eb5a3343abf848c0984fe4604b2b105da9539376e24fc0a3b0007411ae4fd9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -3745,7 +3745,7 @@ dependencies = [ "indoc", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -3765,9 +3765,9 @@ checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "iri-string" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" dependencies = [ "memchr", "serde", @@ -4518,7 +4518,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -4623,7 +4623,7 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -4776,7 +4776,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -4876,7 +4876,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -5067,7 +5067,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -5222,7 +5222,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -5251,7 +5251,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -5319,7 +5319,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -5443,7 +5443,7 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -5622,7 +5622,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -5819,7 +5819,7 @@ checksum = "bd83f5f173ff41e00337d97f6572e416d022ef8a19f371817259ae960324c482" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -5921,9 +5921,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.34" +version = "0.23.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9586e9ee2b4f8fab52a0048ca7334d7024eef48e2cb9407e3497bb7cab7fa7" +checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" dependencies = [ "once_cell", "rustls-pki-types", @@ -6031,7 +6031,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -6057,9 +6057,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.0.5" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1317c3bf3e7df961da95b0a56a172a02abead31276215a0497241a7624b487ce" +checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289" dependencies = [ "dyn-clone", "ref-cast", @@ -6103,7 +6103,7 @@ checksum = "22f968c5ea23d555e670b449c1c5e7b2fc399fdaec1d304a17cd48e288abc107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -6234,7 +6234,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -6269,7 +6269,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -6305,7 +6305,7 @@ dependencies = [ "indexmap 1.9.3", "indexmap 2.12.0", "schemars 0.9.0", - "schemars 1.0.5", + "schemars 1.1.0", "serde_core", "serde_json", "serde_with_macros", @@ -6321,7 +6321,7 @@ dependencies = [ "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -6652,7 +6652,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -6664,7 +6664,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -6706,9 +6706,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.108" +version = "2.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917" +checksum = "2f17c7e013e88258aa9543dcbe81aca68a667a9ac37cd69c9fbc07858bfe0e2f" dependencies = [ "proc-macro2", "quote", @@ -6738,7 +6738,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -6862,7 +6862,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -6873,7 +6873,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -6986,7 +6986,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -7015,7 +7015,7 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls 0.23.34", + "rustls 0.23.35", "tokio", ] @@ -7212,7 +7212,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -7282,7 +7282,7 @@ checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -7644,7 +7644,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", "wasm-bindgen-shared", ] @@ -7781,7 +7781,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -7792,7 +7792,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -8188,7 +8188,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", "synstructure", ] @@ -8209,7 +8209,7 @@ checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -8229,7 +8229,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", "synstructure", ] @@ -8250,7 +8250,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] @@ -8310,7 +8310,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.108", + "syn 2.0.109", ] [[package]] diff --git a/k8s/deploy.sequencer.yaml b/k8s/deploy.sequencer.yaml new file mode 100644 index 0000000..95dbcda --- /dev/null +++ b/k8s/deploy.sequencer.yaml @@ -0,0 +1,46 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mojave-sequencer-deployment + labels: + app: mojave-sequencer +spec: + replicas: 3 + selector: + matchLabels: + app: mojave-sequencer + template: + metadata: + labels: + app: mojave-sequencer + spec: + containers: + - name: mojave-sequencer-v01 + image: starkgiwook/mojave-sequencer:v0.1.1 + args: + - "init" + - "--private_key" + - "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + - "--network" + - "/data/testnet-genesis.json" # Dockerfile에서 복사한 경로 + - "--http.port" + - "18545" + - "--authrpc.port" + - "18551" + - "--p2p.port" + - "30305" + - "--discovery.port" + - "30305" + - "--no-daemon" # & 와 wait 없이 포어그라운드로 실행 + ports: + - containerPort: 18545 + name: http + - containerPort: 18551 + name: auth-rpc + - containerPort: 30305 # TCP/UDP 둘 다일 수 있음 + name: p2p + protocol: TCP + imagePullPolicy: Always + ports: + - containerPort: 1739 + name: http \ No newline at end of file diff --git a/k8s/service.sequencer.yaml b/k8s/service.sequencer.yaml new file mode 100644 index 0000000..bf7e0d7 --- /dev/null +++ b/k8s/service.sequencer.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: mojave-sequencer-service +spec: + type: ClusterIP + selector: + app: mojave-sequencer + ports: + - name: http + protocol: TCP + + port: 1739 + targetPort: 1739 From 8d899bdf96c12f4cdb31a6b0ff0d2c7549c448dc Mon Sep 17 00:00:00 2001 From: Giwook-Han Date: Tue, 11 Nov 2025 13:23:28 +0900 Subject: [PATCH 02/29] feat: add k8s leader election logic --- Cargo.lock | 601 ++++++++++++++++++++++++++++---- Cargo.toml | 2 + cmd/sequencer/Cargo.toml | 4 +- cmd/sequencer/src/k8s_leader.rs | 144 ++++++++ cmd/sequencer/src/main.rs | 97 +++--- k8s/deploy.sequencer.yaml | 9 + 6 files changed, 731 insertions(+), 126 deletions(-) create mode 100644 cmd/sequencer/src/k8s_leader.rs diff --git a/Cargo.lock b/Cargo.lock index 15d483a..683a465 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -56,6 +56,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if 1.0.4", + "getrandom 0.3.4", "once_cell", "version_check", "zerocopy", @@ -222,7 +223,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62945a2f7e6de02a31fe400aa489f0e0f5b2502e69f95f853adb82a96c7a6b60" dependencies = [ "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -235,7 +236,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -274,7 +275,7 @@ checksum = "213888f660fddcca0d257e88e54ac05bca01885f258ccdf695bafd77031bb69d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -308,6 +309,40 @@ dependencies = [ "term", ] +[[package]] +name = "async-broadcast" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435a87a52755b8f27fcf321ac4f04b2802e337c8c4872923137471ec39c37532" +dependencies = [ + "event-listener", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.110", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -316,7 +351,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -357,7 +392,7 @@ checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -441,6 +476,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers 0.3.0", + "tokio", +] + [[package]] name = "backtrace" version = "0.3.76" @@ -532,7 +578,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -756,7 +802,7 @@ checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -864,9 +910,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.44" +version = "1.2.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37521ac7aabe3d13122dc382493e20c9416f299d2ccd5b3a5340a2570cdeb0f3" +checksum = "35900b6c8d709fb1d854671ae27aeaa9eec2f8b01b364e1619a40da3e6fe2afe" dependencies = [ "find-msvc-tools", "jobserver", @@ -988,7 +1034,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -1105,6 +1151,15 @@ dependencies = [ "digest", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils 0.8.21", +] + [[package]] name = "console" version = "0.15.11" @@ -1196,6 +1251,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -1431,7 +1496,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -1474,7 +1539,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -1488,7 +1553,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -1499,7 +1564,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -1510,7 +1575,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -1583,7 +1648,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -1593,7 +1658,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -1623,7 +1688,7 @@ dependencies = [ "convert_case 0.6.0", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", "unicode-xid", ] @@ -1636,7 +1701,7 @@ dependencies = [ "convert_case 0.7.1", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -1723,7 +1788,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -1796,7 +1861,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -1884,7 +1949,7 @@ checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -2091,7 +2156,7 @@ dependencies = [ "reqwest 0.11.27", "serde", "serde_json", - "syn 2.0.109", + "syn 2.0.110", "toml", "walkdir", ] @@ -2109,7 +2174,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -2135,7 +2200,7 @@ dependencies = [ "serde", "serde_json", "strum 0.26.3", - "syn 2.0.109", + "syn 2.0.110", "tempfile", "thiserror 1.0.69", "tiny-keccak", @@ -2754,6 +2819,27 @@ dependencies = [ "tracing", ] +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "eyre" version = "0.6.12" @@ -2973,7 +3059,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -2994,7 +3080,7 @@ version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" dependencies = [ - "gloo-timers", + "gloo-timers 0.2.6", "send_wrapper 0.4.0", ] @@ -3104,6 +3190,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "group" version = "0.13.0" @@ -3298,6 +3396,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "hostname" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56f203cd1c76362b69e3863fd987520ac36cf70a8c92627449b2f64a8cf7d65" +dependencies = [ + "cfg-if 1.0.4", + "libc", + "windows-link 0.1.3", +] + [[package]] name = "http" version = "0.2.12" @@ -3436,13 +3545,28 @@ dependencies = [ "http 1.3.1", "hyper 1.7.0", "hyper-util", + "log", "rustls 0.23.35", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls 0.26.4", "tower-service", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper 1.7.0", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-tls" version = "0.6.0" @@ -3679,7 +3803,7 @@ checksum = "a0eb5a3343abf848c0984fe4604b2b105da9539376e24fc0a3b0007411ae4fd9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -3745,7 +3869,7 @@ dependencies = [ "indoc", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -3841,6 +3965,41 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json-patch" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f300e415e2134745ef75f04562dd0145405c2f7fd92065db029ac4b16b57fe90" +dependencies = [ + "jsonptr", + "serde", + "serde_json", + "thiserror 1.0.69", +] + +[[package]] +name = "jsonpath-rust" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c00ae348f9f8fd2d09f82a98ca381c60df9e0820d8d79fce43e649b4dc3128b" +dependencies = [ + "pest", + "pest_derive", + "regex", + "serde_json", + "thiserror 2.0.17", +] + +[[package]] +name = "jsonptr" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5a3cc660ba5d72bce0b3bb295bf20847ccbb40fd423f3f05b61273672e561fe" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "jsonrpc" version = "0.18.0" @@ -3896,6 +4055,18 @@ dependencies = [ "signature", ] +[[package]] +name = "k8s-openapi" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d13f06d5326a915becaffabdfab75051b8cdc260c2a5c06c0e90226ede89a692" +dependencies = [ + "base64 0.22.1", + "chrono", + "serde", + "serde_json", +] + [[package]] name = "keccak" version = "0.1.5" @@ -3905,6 +4076,130 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "kube" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e7bb0b6a46502cc20e4575b6ff401af45cfea150b34ba272a3410b78aa014e" +dependencies = [ + "k8s-openapi", + "kube-client", + "kube-core", + "kube-derive", + "kube-runtime", +] + +[[package]] +name = "kube-client" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4987d57a184d2b5294fdad3d7fc7f278899469d21a4da39a8f6ca16426567a36" +dependencies = [ + "base64 0.22.1", + "bytes", + "chrono", + "either", + "futures", + "home", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "hyper 1.7.0", + "hyper-rustls 0.27.7", + "hyper-timeout", + "hyper-util", + "jsonpath-rust", + "k8s-openapi", + "kube-core", + "pem 3.0.6", + "rustls 0.23.35", + "secrecy", + "serde", + "serde_json", + "serde_yaml", + "thiserror 2.0.17", + "tokio", + "tokio-util", + "tower", + "tower-http", + "tracing", +] + +[[package]] +name = "kube-core" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914bbb770e7bb721a06e3538c0edd2babed46447d128f7c21caa68747060ee73" +dependencies = [ + "chrono", + "derive_more 2.0.1", + "form_urlencoded", + "http 1.3.1", + "json-patch", + "k8s-openapi", + "schemars 1.1.0", + "serde", + "serde-value", + "serde_json", + "thiserror 2.0.17", +] + +[[package]] +name = "kube-derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03dee8252be137772a6ab3508b81cd797dee62ee771112a2453bc85cbbe150d2" +dependencies = [ + "darling 0.21.3", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 2.0.110", +] + +[[package]] +name = "kube-leader-election" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0b7fbf9c715967ca639f6ba0c8108481aeb53e69226027ab425469ba877e028" +dependencies = [ + "chrono", + "k8s-openapi", + "kube", + "log", + "serde", + "serde_json", + "thiserror 1.0.69", +] + +[[package]] +name = "kube-runtime" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aea4de4b562c5cc89ab10300bb63474ae1fa57ff5a19275f2e26401a323e3fd" +dependencies = [ + "ahash", + "async-broadcast", + "async-stream", + "backon", + "educe", + "futures", + "hashbrown 0.15.5", + "hostname", + "json-patch", + "k8s-openapi", + "kube-client", + "parking_lot", + "pin-project", + "serde", + "serde_json", + "thiserror 2.0.17", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "kzg-rs" version = "0.2.7" @@ -4518,7 +4813,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -4542,6 +4837,8 @@ dependencies = [ "anyhow", "clap", "hex", + "kube", + "kube-leader-election", "mojave-batch-producer", "mojave-batch-submitter", "mojave-block-producer", @@ -4623,7 +4920,7 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -4638,7 +4935,7 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "security-framework", + "security-framework 2.11.1", "security-framework-sys", "tempfile", ] @@ -4776,7 +5073,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -4855,9 +5152,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.74" +version = "0.10.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24ad14dd45412269e1a30f52ad8f0664f0f4f4a89ee8fe28c3b3527021ebb654" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" dependencies = [ "bitflags 2.10.0", "cfg-if 1.0.4", @@ -4876,7 +5173,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -4887,9 +5184,9 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-sys" -version = "0.9.110" +version = "0.9.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a9f0075ba3c21b09f8e8b2026584b1d18d49388648f2fbbf3c97ea8deced8e2" +checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" dependencies = [ "cc", "libc", @@ -4903,6 +5200,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + [[package]] name = "owo-colors" version = "4.2.3" @@ -5067,9 +5373,15 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.5" @@ -5172,6 +5484,49 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pest" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "989e7521a040efde50c3ab6bbadafbe15ab6dc042686926be59ac35d74607df4" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "187da9a3030dbafabbbfb20cb323b976dc7b7ce91fcd84f2f74d6e31d378e2de" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49b401d98f5757ebe97a26085998d6c0eecec4995cad6ab7fc30ffdf4b052843" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.110", +] + +[[package]] +name = "pest_meta" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f27a2cfee9f9039c4d86faa5af122a0ac3851441a34865b8a043b46be0065a" +dependencies = [ + "pest", + "sha2", +] + [[package]] name = "petgraph" version = "0.6.5" @@ -5222,7 +5577,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -5251,7 +5606,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -5319,7 +5674,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -5443,14 +5798,14 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] name = "quote" -version = "1.0.41" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" dependencies = [ "proc-macro2", ] @@ -5622,7 +5977,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -5819,7 +6174,7 @@ checksum = "bd83f5f173ff41e00337d97f6572e416d022ef8a19f371817259ae960324c482" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -5925,13 +6280,27 @@ version = "0.23.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" dependencies = [ + "log", "once_cell", + "ring 0.17.14", "rustls-pki-types", "rustls-webpki 0.103.8", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework 3.5.1", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -6031,7 +6400,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -6063,10 +6432,23 @@ checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289" dependencies = [ "dyn-clone", "ref-cast", + "schemars_derive", "serde", "serde_json", ] +[[package]] +name = "schemars_derive" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301858a4023d78debd2353c7426dc486001bddc91ae31a76fb1f55132f7e2633" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 2.0.110", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -6103,7 +6485,7 @@ checksum = "22f968c5ea23d555e670b449c1c5e7b2fc399fdaec1d304a17cd48e288abc107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -6153,6 +6535,15 @@ dependencies = [ "cc", ] +[[package]] +name = "secrecy" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a" +dependencies = [ + "zeroize", +] + [[package]] name = "security-framework" version = "2.11.1" @@ -6160,7 +6551,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags 2.10.0", - "core-foundation", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -6208,6 +6612,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + [[package]] name = "serde_arrays" version = "0.2.0" @@ -6234,7 +6648,18 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", +] + +[[package]] +name = "serde_derive_internals" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.110", ] [[package]] @@ -6269,7 +6694,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -6321,7 +6746,20 @@ dependencies = [ "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap 2.12.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", ] [[package]] @@ -6652,7 +7090,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -6664,7 +7102,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -6706,9 +7144,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.109" +version = "2.0.110" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f17c7e013e88258aa9543dcbe81aca68a667a9ac37cd69c9fbc07858bfe0e2f" +checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea" dependencies = [ "proc-macro2", "quote", @@ -6738,7 +7176,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -6762,7 +7200,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", - "core-foundation", + "core-foundation 0.9.4", "system-configuration-sys 0.5.0", ] @@ -6773,7 +7211,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ "bitflags 2.10.0", - "core-foundation", + "core-foundation 0.9.4", "system-configuration-sys 0.6.0", ] @@ -6862,7 +7300,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -6873,7 +7311,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -6986,7 +7424,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -7072,6 +7510,7 @@ dependencies = [ "futures-sink", "futures-util", "pin-project-lite", + "slab", "tokio", ] @@ -7157,6 +7596,7 @@ dependencies = [ "pin-project-lite", "sync_wrapper 1.0.2", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -7168,16 +7608,19 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ + "base64 0.22.1", "bitflags 2.10.0", "bytes", "futures-util", "http 1.3.1", "http-body 1.0.1", "iri-string", + "mime", "pin-project-lite", "tower", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -7212,7 +7655,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -7282,7 +7725,7 @@ checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -7375,6 +7818,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + [[package]] name = "uint" version = "0.9.5" @@ -7446,6 +7895,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.7.1" @@ -7644,7 +8099,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", "wasm-bindgen-shared", ] @@ -7781,7 +8236,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -7792,7 +8247,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -8188,7 +8643,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", "synstructure", ] @@ -8209,7 +8664,7 @@ checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -8229,7 +8684,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", "synstructure", ] @@ -8250,7 +8705,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -8310,7 +8765,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 8a32b4d..173b6a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,6 +83,8 @@ daemonize = "0.5" ed25519-dalek = { version = "2.1.1", features = ["rand_core", "serde"] } futures = "0.3" hex = "0.4.3" +kube = { version = "2.0.1", features = ["runtime", "derive", "client", "rustls-tls"] } +kube-leader-election = "0.42" lazy_static = "1.5.0" local-ip-address = { version = "0.6" } proc-macro2 = "1" diff --git a/cmd/sequencer/Cargo.toml b/cmd/sequencer/Cargo.toml index c185fb3..e01eccd 100644 --- a/cmd/sequencer/Cargo.toml +++ b/cmd/sequencer/Cargo.toml @@ -21,5 +21,7 @@ mojave-utils = { workspace = true } anyhow = { workspace = true } clap = { workspace = true } hex = { workspace = true } -tokio = { workspace = true, features = ["macros", "rt", "sync"] } +kube = { workspace = true } +kube-leader-election = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt", "sync", "time", "signal"] } tracing = { workspace = true } diff --git a/cmd/sequencer/src/k8s_leader.rs b/cmd/sequencer/src/k8s_leader.rs new file mode 100644 index 0000000..b982814 --- /dev/null +++ b/cmd/sequencer/src/k8s_leader.rs @@ -0,0 +1,144 @@ +use kube::Client; +use kube_leader_election::{LeaseLock, LeaseLockParams}; +use std::{env, time::Duration}; +use tokio::time::sleep; + +use mojave_batch_producer::{BatchProducer, types::Request as BatchProducerRequest}; +use mojave_batch_submitter::committer::Committer; +use mojave_block_producer::{ + BlockProducer, + types::{BlockProducerOptions, Request as BlockProducerRequest}, +}; +use mojave_node_lib::types::MojaveNode; +use mojave_proof_coordinator::{ProofCoordinator, types::ProofCoordinatorOptions}; +use mojave_task::{Runner, Task, TaskHandle}; +use tracing::{error, info}; + +pub struct LeaderTasks { + batch: TaskHandle, + block: TaskHandle, + proof: TaskHandle, + committer: tokio::task::JoinHandle>, +} + +const BLOCK_PRODUCER_CAPACITY: usize = 100; + +pub async fn run_with_k8s_coordination( + node: MojaveNode, + node_options: mojave_node_lib::types::NodeOptions, + block_producer_options: BlockProducerOptions, + proof_coordinator_options: ProofCoordinatorOptions, +) -> Result<(), Box> { + let client = Client::try_default().await?; + let identity = env::var("POD_NAME").unwrap_or_else(|_| "sequencer-pod".to_string()); + let namespace = env::var("POD_NAMESPACE").unwrap_or_else(|_| "default".to_string()); + + let lease_name = env::var("LEASE_NAME").unwrap_or_else(|_| "sequencer-leader".to_string()); + let lease_ttl_sec = 15_u64; + let renew_every_secs = lease_ttl_sec / 3; // 1/3 of TTL + + let lease_lock = LeaseLock::new( + client, + &namespace, + LeaseLockParams { + lease_name, + holder_id: identity, + lease_ttl: Duration::from_secs(lease_ttl_sec), + }, + ); + + let mut am_i_leader = false; + let mut leader_tasks: Option = None; + + loop { + match lease_lock.try_acquire_or_renew().await { + Ok(res) => { + let became_leader = res.acquired_lease; + + if !am_i_leader && became_leader { + // GW - Do I need to add Sleep here to wait all the leader task stop? + + info!("Became a leader. Start leader tasks"); + leader_tasks = Some( + start_leader_tasks( + node.clone(), + &node_options, + &block_producer_options, + &proof_coordinator_options, + ) + .await?, + ); + am_i_leader = true; + } else if !became_leader && am_i_leader { + info!("Became a follower. Stop leader tasks"); + if let Some(lt) = leader_tasks.take() { + stop_leader_tasks(lt).await?; + } + am_i_leader = false; + } + + sleep(Duration::from_secs(renew_every_secs)).await; + } + Err(err) => { + error!("Error while k8s leader election: {err:}") + } + } + } +} + +pub async fn start_leader_tasks( + node: MojaveNode, + node_options: &mojave_node_lib::types::NodeOptions, + block_producer_options: &BlockProducerOptions, + proof_coordinator_options: &ProofCoordinatorOptions, +) -> Result> { + let cancel_token = node.cancel_token.clone(); + + // TODO: replace by implementation backed by a real queue + let q = mojave_msgio::dummy::Dummy; + + let batch_producer = BatchProducer::new(node.clone(), 0); + let block_producer = BlockProducer::new(node.clone()); + let proof_coordinator = + ProofCoordinator::new(node.clone(), node_options, proof_coordinator_options)?; + + let batch = batch_producer + .clone() + .spawn_periodic(Duration::from_millis(10_000), || { + BatchProducerRequest::BuildBatch + }); + + let block = block_producer.spawn_with_capacity_periodic( + BLOCK_PRODUCER_CAPACITY, + Duration::from_millis(block_producer_options.block_time), + || BlockProducerRequest::BuildBlock, + ); + + let committer = Runner::new( + Committer::new(batch_producer.subscribe(), q, node.p2p_context.clone()), + cancel_token.clone(), + ) + .spawn(); + + let proof = proof_coordinator.spawn(); + + Ok(LeaderTasks { + batch, + block, + proof, + committer, + }) +} + +pub async fn stop_leader_tasks(lt: LeaderTasks) -> Result<(), Box> { + lt.batch.shutdown().await?; + lt.block.shutdown().await?; + lt.proof.shutdown().await?; + let _ = lt.committer.await?; + Ok(()) +} + +// 간단한 K8s 환경 감지 +pub fn is_k8s_env() -> bool { + std::env::var("KUBERNETES_SERVICE_HOST").is_ok() +} diff --git a/cmd/sequencer/src/main.rs b/cmd/sequencer/src/main.rs index a04f14f..078cb80 100644 --- a/cmd/sequencer/src/main.rs +++ b/cmd/sequencer/src/main.rs @@ -1,32 +1,29 @@ pub mod cli; +mod k8s_leader; -use crate::cli::Command; +use crate::{ + cli::Command, + k8s_leader::{is_k8s_env, run_with_k8s_coordination, start_leader_tasks, stop_leader_tasks}, +}; use anyhow::Result; -use mojave_batch_producer::{BatchProducer, types::Request as BatchProducerRequest}; -use mojave_batch_submitter::committer::Committer; -use mojave_block_producer::{ - BlockProducer, - types::{BlockProducerOptions, Request as BlockProducerRequest}, -}; +use mojave_block_producer::types::BlockProducerOptions; use mojave_node_lib::{ initializers::get_signer, types::{MojaveNode, NodeConfigFile}, utils::store_node_config_file, }; -use mojave_proof_coordinator::{ProofCoordinator, types::ProofCoordinatorOptions}; -use mojave_task::{Runner, Task}; +use mojave_proof_coordinator::types::ProofCoordinatorOptions; use mojave_utils::{ block_on::block_on_current_thread, daemon::{DaemonOptions, run_daemonized, stop_daemonized}, p2p::public_key_from_signing_key, }; -use std::{path::PathBuf, time::Duration}; +use std::path::PathBuf; use tracing::{error, info}; const PID_FILE_NAME: &str = "sequencer.pid"; const LOG_FILE_NAME: &str = "sequencer.log"; -const BLOCK_PRODUCER_CAPACITY: usize = 100; fn main() -> Result<()> { mojave_utils::logging::init(); @@ -65,50 +62,46 @@ fn main() -> Result<()> { let node = MojaveNode::init(&node_options) .await .map_err(|e| Box::new(e) as Box)?; - let cancel_token = node.cancel_token.clone(); - - // TODO: replace by implementation backed by a real queue - let q = mojave_msgio::dummy::Dummy; - - let batch_producer = BatchProducer::new(node.clone(), 0); - let block_producer = BlockProducer::new(node.clone()); - let proof_coordinator = ProofCoordinator::new(node.clone(), &node_options, &proof_coordinator_options)?; - - let batch_producer_task = batch_producer.clone() - .spawn_periodic(Duration::from_millis(10_000), || BatchProducerRequest::BuildBatch); - - let block_producer_task = block_producer - .spawn_with_capacity_periodic(BLOCK_PRODUCER_CAPACITY, Duration::from_millis(block_producer_options.block_time), || BlockProducerRequest::BuildBlock); - - let committer_handle = Runner::new(Committer::new(batch_producer.subscribe(), q, node.p2p_context.clone()), cancel_token.clone()).spawn(); - - let proof_coordinator_task = proof_coordinator.spawn(); - - tokio::select! { - // TODO: replace with api task - _ = mojave_utils::signal::wait_for_shutdown_signal() => { - info!("Termination signal received, shutting down sequencer.."); - cancel_token.cancel(); - batch_producer_task.shutdown().await.map_err(|e| Box::new(e) as Box)?; - block_producer_task.shutdown().await.map_err(|e| Box::new(e) as Box)?; - proof_coordinator_task.shutdown().await.map_err(|e| Box::new(e) as Box)?; - let _ = committer_handle.await.map_err(|e| Box::new(e) as Box)?; - - let node_config_path = PathBuf::from(node.data_dir).join("node_config.json"); - info!("Storing config at {:?}...", node_config_path); - - let node_config = NodeConfigFile::new(node.peer_table.clone(), node.local_node_record.lock().await.clone()).await; - store_node_config_file(node_config, node_config_path).await; - - // TODO: wait for api to stop here - // if let Err(_elapsed) = tokio::time::timeout(std::time::Duration::from_secs(10), api_task).await { - // warn!("Timed out waiting for API to stop"); - // } - info!("Successfully shut down the sequencer."); - Ok(()) + let use_k8s = is_k8s_env(); + + if use_k8s { + run_with_k8s_coordination( + node.clone(), + node_options.clone(), + block_producer_options, + proof_coordinator_options, + ) + .await?; + } else { + let lt = start_leader_tasks( + node.clone(), + &node_options, + &block_producer_options, + &proof_coordinator_options, + ) + .await?; + + tokio::select! { + _ = mojave_utils::signal::wait_for_shutdown_signal() => { + info!("Termination signal received, shutting down sequencer.."); + stop_leader_tasks(lt).await?; + + let node_config_path = PathBuf::from(node.data_dir).join("node_config.json"); + info!("Storing config at {:?}...", node_config_path); + let node_config = NodeConfigFile::new(node.peer_table.clone(), node.local_node_record.lock().await.clone()).await; + store_node_config_file(node_config, node_config_path).await; + + // TODO: wait for api to stop here + // if let Err(_elapsed) = tokio::time::timeout(std::time::Duration::from_secs(10), api_task).await { + // warn!("Timed out waiting for API to stop"); + // } + + info!("Successfully shut down the sequencer."); + } } } + Ok(()) }) .unwrap_or_else(|err| error!("Failed to start daemonized sequencer: {}", err)); } diff --git a/k8s/deploy.sequencer.yaml b/k8s/deploy.sequencer.yaml index 95dbcda..557cb3a 100644 --- a/k8s/deploy.sequencer.yaml +++ b/k8s/deploy.sequencer.yaml @@ -17,6 +17,15 @@ spec: containers: - name: mojave-sequencer-v01 image: starkgiwook/mojave-sequencer:v0.1.1 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace args: - "init" - "--private_key" From ab72315911ee3c0e1148de06b872785cb6f36a2f Mon Sep 17 00:00:00 2001 From: Giwook-Han Date: Tue, 11 Nov 2025 13:34:02 +0900 Subject: [PATCH 03/29] add: k8s yamls --- k8s/deploy.sequencer.yaml | 39 ++++++++++++++++++++++++++------------- k8s/rbac.sequencer.yaml | 25 +++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 13 deletions(-) create mode 100644 k8s/rbac.sequencer.yaml diff --git a/k8s/deploy.sequencer.yaml b/k8s/deploy.sequencer.yaml index 557cb3a..3fe0e1c 100644 --- a/k8s/deploy.sequencer.yaml +++ b/k8s/deploy.sequencer.yaml @@ -14,9 +14,12 @@ spec: labels: app: mojave-sequencer spec: + serviceAccountName: mojave-sequencer-sa + terminationGracePeriodSeconds: 30 containers: - - name: mojave-sequencer-v01 + - name: mojave-sequencer image: starkgiwook/mojave-sequencer:v0.1.1 + imagePullPolicy: Always env: - name: POD_NAME valueFrom: @@ -26,12 +29,16 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: LEASE_NAME + value: sequencer-leader + - name: LEASE_TTL_SECONDS + value: "15" args: - "init" - "--private_key" - "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - "--network" - - "/data/testnet-genesis.json" # Dockerfile에서 복사한 경로 + - "/data/testnet-genesis.json" - "--http.port" - "18545" - "--authrpc.port" @@ -40,16 +47,22 @@ spec: - "30305" - "--discovery.port" - "30305" - - "--no-daemon" # & 와 wait 없이 포어그라운드로 실행 + - "--no-daemon" ports: - - containerPort: 18545 - name: http - - containerPort: 18551 - name: auth-rpc - - containerPort: 30305 # TCP/UDP 둘 다일 수 있음 - name: p2p + - name: http + containerPort: 18545 + - name: auth-rpc + containerPort: 18551 + - name: p2p + containerPort: 30305 protocol: TCP - imagePullPolicy: Always - ports: - - containerPort: 1739 - name: http \ No newline at end of file + livenessProbe: + tcpSocket: + port: 18545 + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + tcpSocket: + port: 18545 + initialDelaySeconds: 10 + periodSeconds: 5 \ No newline at end of file diff --git a/k8s/rbac.sequencer.yaml b/k8s/rbac.sequencer.yaml new file mode 100644 index 0000000..1cf06a8 --- /dev/null +++ b/k8s/rbac.sequencer.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mojave-sequencer-sa + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: sequencer-lease-role +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["lease"] + verbs: ["get", "create", "update", "patch"] + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: sequencer-lease-rb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: sequencer-lease-role +subjects: +- kind: ServiceAccount + name: sequencer-sa \ No newline at end of file From 4c21205528f1e5bb1711507d9de4a1c87f630bfc Mon Sep 17 00:00:00 2001 From: Giwook-Han Date: Tue, 11 Nov 2025 15:45:43 +0900 Subject: [PATCH 04/29] mod: add k8s leader shutdown --- Cargo.lock | 200 +------------------------------- Cargo.toml | 3 +- cmd/sequencer/Cargo.toml | 1 + cmd/sequencer/src/k8s_leader.rs | 79 ++++++++----- 4 files changed, 58 insertions(+), 225 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 683a465..9d88d6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -56,7 +56,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if 1.0.4", - "getrandom 0.3.4", "once_cell", "version_check", "zerocopy", @@ -309,40 +308,6 @@ dependencies = [ "term", ] -[[package]] -name = "async-broadcast" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435a87a52755b8f27fcf321ac4f04b2802e337c8c4872923137471ec39c37532" -dependencies = [ - "event-listener", - "event-listener-strategy", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.110", -] - [[package]] name = "async-trait" version = "0.1.89" @@ -476,17 +441,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "backon" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" -dependencies = [ - "fastrand", - "gloo-timers 0.3.0", - "tokio", -] - [[package]] name = "backtrace" version = "0.3.76" @@ -1151,15 +1105,6 @@ dependencies = [ "digest", ] -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils 0.8.21", -] - [[package]] name = "console" version = "0.15.11" @@ -2819,27 +2764,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "event-listener" -version = "5.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" -dependencies = [ - "event-listener", - "pin-project-lite", -] - [[package]] name = "eyre" version = "0.6.12" @@ -3080,7 +3004,7 @@ version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" dependencies = [ - "gloo-timers 0.2.6", + "gloo-timers", "send_wrapper 0.4.0", ] @@ -3190,18 +3114,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "gloo-timers" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "group" version = "0.13.0" @@ -3396,17 +3308,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "hostname" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56f203cd1c76362b69e3863fd987520ac36cf70a8c92627449b2f64a8cf7d65" -dependencies = [ - "cfg-if 1.0.4", - "libc", - "windows-link 0.1.3", -] - [[package]] name = "http" version = "0.2.12" @@ -3965,18 +3866,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "json-patch" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f300e415e2134745ef75f04562dd0145405c2f7fd92065db029ac4b16b57fe90" -dependencies = [ - "jsonptr", - "serde", - "serde_json", - "thiserror 1.0.69", -] - [[package]] name = "jsonpath-rust" version = "0.7.5" @@ -3990,16 +3879,6 @@ dependencies = [ "thiserror 2.0.17", ] -[[package]] -name = "jsonptr" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5a3cc660ba5d72bce0b3bb295bf20847ccbb40fd423f3f05b61273672e561fe" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "jsonrpc" version = "0.18.0" @@ -4085,8 +3964,6 @@ dependencies = [ "k8s-openapi", "kube-client", "kube-core", - "kube-derive", - "kube-runtime", ] [[package]] @@ -4135,29 +4012,13 @@ dependencies = [ "derive_more 2.0.1", "form_urlencoded", "http 1.3.1", - "json-patch", "k8s-openapi", - "schemars 1.1.0", "serde", "serde-value", "serde_json", "thiserror 2.0.17", ] -[[package]] -name = "kube-derive" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03dee8252be137772a6ab3508b81cd797dee62ee771112a2453bc85cbbe150d2" -dependencies = [ - "darling 0.21.3", - "proc-macro2", - "quote", - "serde", - "serde_json", - "syn 2.0.110", -] - [[package]] name = "kube-leader-election" version = "0.42.0" @@ -4173,33 +4034,6 @@ dependencies = [ "thiserror 1.0.69", ] -[[package]] -name = "kube-runtime" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aea4de4b562c5cc89ab10300bb63474ae1fa57ff5a19275f2e26401a323e3fd" -dependencies = [ - "ahash", - "async-broadcast", - "async-stream", - "backon", - "educe", - "futures", - "hashbrown 0.15.5", - "hostname", - "json-patch", - "k8s-openapi", - "kube-client", - "parking_lot", - "pin-project", - "serde", - "serde_json", - "thiserror 2.0.17", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "kzg-rs" version = "0.2.7" @@ -4837,6 +4671,7 @@ dependencies = [ "anyhow", "clap", "hex", + "k8s-openapi", "kube", "kube-leader-election", "mojave-batch-producer", @@ -5376,12 +5211,6 @@ dependencies = [ "syn 2.0.110", ] -[[package]] -name = "parking" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" - [[package]] name = "parking_lot" version = "0.12.5" @@ -6432,23 +6261,10 @@ checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289" dependencies = [ "dyn-clone", "ref-cast", - "schemars_derive", "serde", "serde_json", ] -[[package]] -name = "schemars_derive" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301858a4023d78debd2353c7426dc486001bddc91ae31a76fb1f55132f7e2633" -dependencies = [ - "proc-macro2", - "quote", - "serde_derive_internals", - "syn 2.0.110", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -6651,17 +6467,6 @@ dependencies = [ "syn 2.0.110", ] -[[package]] -name = "serde_derive_internals" -version = "0.29.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.110", -] - [[package]] name = "serde_json" version = "1.0.145" @@ -7510,7 +7315,6 @@ dependencies = [ "futures-sink", "futures-util", "pin-project-lite", - "slab", "tokio", ] diff --git a/Cargo.toml b/Cargo.toml index 173b6a2..6041de8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,7 +83,8 @@ daemonize = "0.5" ed25519-dalek = { version = "2.1.1", features = ["rand_core", "serde"] } futures = "0.3" hex = "0.4.3" -kube = { version = "2.0.1", features = ["runtime", "derive", "client", "rustls-tls"] } +k8s-openapi = { version = "0.26.0", features = ["v1_34"] } +kube = { version = "2.0.1", features = ["client"] } kube-leader-election = "0.42" lazy_static = "1.5.0" local-ip-address = { version = "0.6" } diff --git a/cmd/sequencer/Cargo.toml b/cmd/sequencer/Cargo.toml index e01eccd..7058044 100644 --- a/cmd/sequencer/Cargo.toml +++ b/cmd/sequencer/Cargo.toml @@ -23,5 +23,6 @@ clap = { workspace = true } hex = { workspace = true } kube = { workspace = true } kube-leader-election = { workspace = true } +k8s-openapi = { workspace = true } tokio = { workspace = true, features = ["macros", "rt", "sync", "time", "signal"] } tracing = { workspace = true } diff --git a/cmd/sequencer/src/k8s_leader.rs b/cmd/sequencer/src/k8s_leader.rs index b982814..bab2766 100644 --- a/cmd/sequencer/src/k8s_leader.rs +++ b/cmd/sequencer/src/k8s_leader.rs @@ -1,6 +1,6 @@ use kube::Client; use kube_leader_election::{LeaseLock, LeaseLockParams}; -use std::{env, time::Duration}; +use std::{env, path::PathBuf, result::Result::*, time::Duration}; use tokio::time::sleep; use mojave_batch_producer::{BatchProducer, types::Request as BatchProducerRequest}; @@ -9,7 +9,10 @@ use mojave_block_producer::{ BlockProducer, types::{BlockProducerOptions, Request as BlockProducerRequest}, }; -use mojave_node_lib::types::MojaveNode; +use mojave_node_lib::{ + types::{MojaveNode, NodeConfigFile}, + utils::store_node_config_file, +}; use mojave_proof_coordinator::{ProofCoordinator, types::ProofCoordinatorOptions}; use mojave_task::{Runner, Task, TaskHandle}; use tracing::{error, info}; @@ -51,36 +54,60 @@ pub async fn run_with_k8s_coordination( let mut leader_tasks: Option = None; loop { - match lease_lock.try_acquire_or_renew().await { - Ok(res) => { - let became_leader = res.acquired_lease; - - if !am_i_leader && became_leader { - // GW - Do I need to add Sleep here to wait all the leader task stop? - - info!("Became a leader. Start leader tasks"); - leader_tasks = Some( - start_leader_tasks( - node.clone(), - &node_options, - &block_producer_options, - &proof_coordinator_options, - ) - .await?, - ); - am_i_leader = true; - } else if !became_leader && am_i_leader { - info!("Became a follower. Stop leader tasks"); + tokio::select! { + _ = mojave_utils::signal::wait_for_shutdown_signal() => { + info!("Termination signal received (K8s). Stopping leader tasks and exiting..."); + if am_i_leader { if let Some(lt) = leader_tasks.take() { stop_leader_tasks(lt).await?; } - am_i_leader = false; + if let Err(err) = lease_lock.step_down().await { + error!("Error while stepping down from leader: {err:?}"); + } } - sleep(Duration::from_secs(renew_every_secs)).await; + let node_config_path = PathBuf::from(node.data_dir.clone()).join("node_config.json"); + info!("Storing config at {:?}...", node_config_path); + let node_config = NodeConfigFile::new(node.peer_table.clone(), node.local_node_record.lock().await.clone()).await; + store_node_config_file(node_config, node_config_path).await; + + info!("Shutdown complete."); + break Ok(()); } - Err(err) => { - error!("Error while k8s leader election: {err:}") + res = lease_lock.try_acquire_or_renew() => { + match res { + Ok(res) => { + let became_leader = res.acquired_lease; + + if !am_i_leader && became_leader { + // GW - Do I need to add Sleep here to wait all the leader task stop? + + info!("Became a leader. Start leader tasks"); + leader_tasks = Some( + start_leader_tasks( + node.clone(), + &node_options, + &block_producer_options, + &proof_coordinator_options, + ) + .await?, + ); + am_i_leader = true; + } else if !became_leader && am_i_leader { + info!("Became a follower. Stop leader tasks"); + if let Some(lt) = leader_tasks.take() { + stop_leader_tasks(lt).await?; + } + am_i_leader = false; + } + + sleep(Duration::from_secs(renew_every_secs)).await; + } + Err(err) => { + error!("Error while k8s leader election: {err:?}"); + break Err(Box::new(err)); + } + } } } } From 156e9cb26b372e4c3b10f213bed3d21b0ff99d4f Mon Sep 17 00:00:00 2001 From: Giwook-Han Date: Thu, 13 Nov 2025 12:14:04 +0900 Subject: [PATCH 05/29] add: k8s node yaml --- cmd/sequencer/src/k8s_leader.rs | 19 +++++++++--- k8s/deploy.node.yaml | 54 +++++++++++++++++++++++++++++++++ k8s/deploy.sequencer.yaml | 9 ++++-- k8s/rbac.sequencer.yaml | 10 +++--- k8s/service.sequencer.yaml | 9 +++++- 5 files changed, 88 insertions(+), 13 deletions(-) create mode 100644 k8s/deploy.node.yaml diff --git a/cmd/sequencer/src/k8s_leader.rs b/cmd/sequencer/src/k8s_leader.rs index bab2766..0d194bf 100644 --- a/cmd/sequencer/src/k8s_leader.rs +++ b/cmd/sequencer/src/k8s_leader.rs @@ -37,8 +37,11 @@ pub async fn run_with_k8s_coordination( let namespace = env::var("POD_NAMESPACE").unwrap_or_else(|_| "default".to_string()); let lease_name = env::var("LEASE_NAME").unwrap_or_else(|_| "sequencer-leader".to_string()); - let lease_ttl_sec = 15_u64; - let renew_every_secs = lease_ttl_sec / 3; // 1/3 of TTL + let lease_ttl_sec = env::var("LEASE_TTL_SECONDS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(15_u64); + let renew_every_secs = lease_ttl_sec / 5; // 1/5 of TTL let lease_lock = LeaseLock::new( client, @@ -81,6 +84,7 @@ pub async fn run_with_k8s_coordination( if !am_i_leader && became_leader { // GW - Do I need to add Sleep here to wait all the leader task stop? + sleep(Duration::from_secs(2)).await; info!("Became a leader. Start leader tasks"); leader_tasks = Some( @@ -131,7 +135,7 @@ pub async fn start_leader_tasks( let batch = batch_producer .clone() - .spawn_periodic(Duration::from_millis(10_000), || { + .spawn_periodic(Duration::from_millis(100_000), || { BatchProducerRequest::BuildBatch }); @@ -165,7 +169,12 @@ pub async fn stop_leader_tasks(lt: LeaderTasks) -> Result<(), Box bool { - std::env::var("KUBERNETES_SERVICE_HOST").is_ok() + match std::env::var("KUBERNETES_SERVICE_HOST") { + Ok(_) => { + info!("Starting service as K8s version"); + true + } + _ => false, + } } diff --git a/k8s/deploy.node.yaml b/k8s/deploy.node.yaml new file mode 100644 index 0000000..922eb86 --- /dev/null +++ b/k8s/deploy.node.yaml @@ -0,0 +1,54 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mojave-node-deployment + labels: + app: mojave-node +spec: + replicas: 1 + selector: + matchLabels: + app: mojave-node + template: + metadata: + labels: + app: mojave-node + spec: + serviceAccountName: node-sa + terminationGracePeriodSeconds: 30 + containers: + - name: mojave-node + image: starkgiwook/mojave-node:v0.1 + imagePullPolicy: Always + args: + - "init" + - "--network" + - "/data/testnet-genesis.json" + - "--p2p.addr" + - "0.0.0.0" + - "--p2p.port" + - "30305" + - "--discovery.addr" + - "0.0.0.0" + - "--discovery.port" + - "30305" + - "--no-daemon" + ports: + - { name: p2p, containerPort: 30305, protocol: TCP } + - { name: p2p-udp, containerPort: 30305, protocol: UDP } +--- +apiVersion: v1 +kind: Service +metadata: + name: mojave-node-p2p +spec: + selector: + app: mojave-node + ports: + - { name: p2p, port: 30305, targetPort: 30305, protocol: TCP } + - { name: p2p-udp, port: 30305, targetPort: 30305, protocol: UDP } +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-sa \ No newline at end of file diff --git a/k8s/deploy.sequencer.yaml b/k8s/deploy.sequencer.yaml index 3fe0e1c..daa3a9f 100644 --- a/k8s/deploy.sequencer.yaml +++ b/k8s/deploy.sequencer.yaml @@ -14,11 +14,11 @@ spec: labels: app: mojave-sequencer spec: - serviceAccountName: mojave-sequencer-sa + serviceAccountName: sequencer-sa terminationGracePeriodSeconds: 30 containers: - name: mojave-sequencer - image: starkgiwook/mojave-sequencer:v0.1.1 + image: starkgiwook/mojave-sequencer:v0.1.2 imagePullPolicy: Always env: - name: POD_NAME @@ -48,6 +48,8 @@ spec: - "--discovery.port" - "30305" - "--no-daemon" + - "--bootnodes" + - "enode://2e4b1f1631ce86cf29c2656c8108cf51f6551dcaafcae7241dcef9d075d5c5c3542b56051eb9ae2841b24e139cad3b1b4e140f334b4c54770c1be2c8a92ef155@10.244.0.46:30305" ports: - name: http containerPort: 18545 @@ -56,6 +58,9 @@ spec: - name: p2p containerPort: 30305 protocol: TCP + - name: p2p-udp + containerPort: 30305 + protocol: UDP livenessProbe: tcpSocket: port: 18545 diff --git a/k8s/rbac.sequencer.yaml b/k8s/rbac.sequencer.yaml index 1cf06a8..624cea4 100644 --- a/k8s/rbac.sequencer.yaml +++ b/k8s/rbac.sequencer.yaml @@ -1,17 +1,17 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: mojave-sequencer-sa - + name: sequencer-sa +--- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: sequencer-lease-role rules: - apiGroups: ["coordination.k8s.io"] - resources: ["lease"] - verbs: ["get", "create", "update", "patch"] - + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/k8s/service.sequencer.yaml b/k8s/service.sequencer.yaml index bf7e0d7..d936dcc 100644 --- a/k8s/service.sequencer.yaml +++ b/k8s/service.sequencer.yaml @@ -9,6 +9,13 @@ spec: ports: - name: http protocol: TCP - port: 1739 targetPort: 1739 + - name: p2p + protocol: TCP + port: 30305 + targetPort: 30305 + - name: p2p-udp + protocol: UDP + port: 30305 + targetPort: 30305 \ No newline at end of file From 47dd8c28f7910bc4dfc609f1a057733029a80b48 Mon Sep 17 00:00:00 2001 From: Giwook-Han Date: Mon, 17 Nov 2025 12:35:09 +0900 Subject: [PATCH 06/29] fix: use pv&pvc to maintain status on k8s --- cmd/sequencer/Cargo.toml | 2 +- cmd/sequencer/src/k8s_leader.rs | 5 ++-- crates/block-producer/src/block_producer.rs | 5 +++- k8s/deploy.sequencer.yaml | 23 ++++++++---------- k8s/pvc.yaml | 27 +++++++++++++++++++++ k8s/service.sequencer.yaml | 4 +-- k8s/setup.sh | 6 +++++ 7 files changed, 53 insertions(+), 19 deletions(-) create mode 100644 k8s/pvc.yaml create mode 100755 k8s/setup.sh diff --git a/cmd/sequencer/Cargo.toml b/cmd/sequencer/Cargo.toml index 7058044..87f6cef 100644 --- a/cmd/sequencer/Cargo.toml +++ b/cmd/sequencer/Cargo.toml @@ -21,8 +21,8 @@ mojave-utils = { workspace = true } anyhow = { workspace = true } clap = { workspace = true } hex = { workspace = true } +k8s-openapi = { workspace = true } kube = { workspace = true } kube-leader-election = { workspace = true } -k8s-openapi = { workspace = true } tokio = { workspace = true, features = ["macros", "rt", "sync", "time", "signal"] } tracing = { workspace = true } diff --git a/cmd/sequencer/src/k8s_leader.rs b/cmd/sequencer/src/k8s_leader.rs index 0d194bf..8786571 100644 --- a/cmd/sequencer/src/k8s_leader.rs +++ b/cmd/sequencer/src/k8s_leader.rs @@ -1,6 +1,6 @@ use kube::Client; use kube_leader_election::{LeaseLock, LeaseLockParams}; -use std::{env, path::PathBuf, result::Result::*, time::Duration}; +use std::{env, result::Result::*, time::Duration}; use tokio::time::sleep; use mojave_batch_producer::{BatchProducer, types::Request as BatchProducerRequest}; @@ -69,7 +69,8 @@ pub async fn run_with_k8s_coordination( } } - let node_config_path = PathBuf::from(node.data_dir.clone()).join("node_config.json"); + let (data_dir, _) = mojave_node_lib::utils::resolve_data_dir(&node_options.datadir).await?; + let node_config_path = data_dir.join("node_config.json"); info!("Storing config at {:?}...", node_config_path); let node_config = NodeConfigFile::new(node.peer_table.clone(), node.local_node_record.lock().await.clone()).await; store_node_config_file(node_config, node_config_path).await; diff --git a/crates/block-producer/src/block_producer.rs b/crates/block-producer/src/block_producer.rs index 3f67bd4..586332d 100644 --- a/crates/block-producer/src/block_producer.rs +++ b/crates/block-producer/src/block_producer.rs @@ -83,7 +83,10 @@ impl Task for BlockProducer { Ok(block) } - Err(e) => Err(e), + Err(e) => { + error!(error = %e, "Error while producing block:"); + Err(e) + } } } } diff --git a/k8s/deploy.sequencer.yaml b/k8s/deploy.sequencer.yaml index daa3a9f..983406d 100644 --- a/k8s/deploy.sequencer.yaml +++ b/k8s/deploy.sequencer.yaml @@ -18,7 +18,7 @@ spec: terminationGracePeriodSeconds: 30 containers: - name: mojave-sequencer - image: starkgiwook/mojave-sequencer:v0.1.2 + image: starkgiwook/mojave-sequencer:v0.1.5 imagePullPolicy: Always env: - name: POD_NAME @@ -39,6 +39,8 @@ spec: - "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - "--network" - "/data/testnet-genesis.json" + - "--datadir" + - "/Users/Shared/mojave/sequencer" - "--http.port" - "18545" - "--authrpc.port" @@ -48,8 +50,9 @@ spec: - "--discovery.port" - "30305" - "--no-daemon" - - "--bootnodes" - - "enode://2e4b1f1631ce86cf29c2656c8108cf51f6551dcaafcae7241dcef9d075d5c5c3542b56051eb9ae2841b24e139cad3b1b4e140f334b4c54770c1be2c8a92ef155@10.244.0.46:30305" + volumeMounts: + - name: sequencer-datadir + mountPath: /Users/Shared/mojave ports: - name: http containerPort: 18545 @@ -61,13 +64,7 @@ spec: - name: p2p-udp containerPort: 30305 protocol: UDP - livenessProbe: - tcpSocket: - port: 18545 - initialDelaySeconds: 30 - periodSeconds: 10 - readinessProbe: - tcpSocket: - port: 18545 - initialDelaySeconds: 10 - periodSeconds: 5 \ No newline at end of file + volumes: + - name: sequencer-datadir + persistentVolumeClaim: + claimName: sequencer-pvc \ No newline at end of file diff --git a/k8s/pvc.yaml b/k8s/pvc.yaml new file mode 100644 index 0000000..f813e53 --- /dev/null +++ b/k8s/pvc.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: sequencer-pv +spec: + capacity: + storage: 20Gi + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: mojave-rwx + hostPath: + path: /data/k8s-data + type: DirectoryOrCreate +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: sequencer-pvc +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 20Gi + storageClassName: mojave-rwx + volumeName: sequencer-pv \ No newline at end of file diff --git a/k8s/service.sequencer.yaml b/k8s/service.sequencer.yaml index d936dcc..ca56f7c 100644 --- a/k8s/service.sequencer.yaml +++ b/k8s/service.sequencer.yaml @@ -9,8 +9,8 @@ spec: ports: - name: http protocol: TCP - port: 1739 - targetPort: 1739 + port: 18545 + targetPort: 18545 - name: p2p protocol: TCP port: 30305 diff --git a/k8s/setup.sh b/k8s/setup.sh new file mode 100755 index 0000000..47dac4e --- /dev/null +++ b/k8s/setup.sh @@ -0,0 +1,6 @@ +kubectl delete -f k8s/ + +kubectl apply -f k8s/pvc.yaml +kubectl apply -f k8s/rbac.sequencer.yaml +kubectl apply -f k8s/service.sequencer.yaml +kubectl apply -f k8s/deploy.sequencer.yaml \ No newline at end of file From 1be0daf98d31c0bc7997ed55811ce5b63beffbe9 Mon Sep 17 00:00:00 2001 From: Giwook-Han Date: Tue, 18 Nov 2025 12:52:22 +0900 Subject: [PATCH 07/29] chore: yaml && add docs --- docs/k8s.md | 246 ++++++++++++++++++++++++++++++++++++++ k8s/deploy.node.yaml | 54 --------- k8s/deploy.sequencer.yaml | 2 +- k8s/pvc.yaml | 2 +- 4 files changed, 248 insertions(+), 56 deletions(-) create mode 100644 docs/k8s.md delete mode 100644 k8s/deploy.node.yaml diff --git a/docs/k8s.md b/docs/k8s.md new file mode 100644 index 0000000..fbf008c --- /dev/null +++ b/docs/k8s.md @@ -0,0 +1,246 @@ +## Kubernetes guide (with Minikube) for Mojave + +This document explains: + +- Where Mojave currently uses Kubernetes (today this is focused on the Sequencer for high availability). +- How to spin up a local Minikube cluster, deploy the Sequencer with leader election, and test failover using the manifests in `k8s/`. + +### 1. How Mojave uses Kubernetes + +#### 1.1 Scope + +- Kubernetes is currently used for **highly-available Sequencer deployment**. +- The provided manifests run **3 Sequencer pods** behind a `ClusterIP` service. +- At any moment, **only one pod is the leader** and runs the "leader tasks"; the other pods stay ready to take over if the leader fails. +- Other components (Node, Prover, etc.) can be run as regular processes or containers; this repository does not currently ship production-ready Kubernetes manifests for them. + +#### 1.2 Packaged Kubernetes resources + +All manifests live under `k8s/`: + +- `k8s/deploy.sequencer.yaml`: `Deployment` with 3 replicas of `mojave-sequencer`, configured for Kubernetes leader election and persistent storage. +- `k8s/service.sequencer.yaml`: `ClusterIP` `Service` exposing: + - HTTP JSON-RPC on port `18545` (Currently not using). + - P2P networking on port `30305` (TCP and UDP). +- `k8s/rbac.sequencer.yaml`: + - `ServiceAccount` (`sequencer-sa`). + - `Role` and `RoleBinding` that grant access to `coordination.k8s.io/v1` `Lease` objects. +- `k8s/pvc.yaml`: + - `PersistentVolume` (`sequencer-pv`) that uses a `hostPath` directory **inside the Minikube node** at `/Users/Shared/mojave`. + - `PersistentVolumeClaim` (`sequencer-pvc`) bound to that PV. +- `k8s/setup.sh`: Helper script that deletes everything under `k8s/` and then re-applies the core resources (PVC, RBAC, Service, Deployment). + +#### 1.3 Sequencer behavior on Kubernetes + +The Sequencer binary detects a Kubernetes environment by checking the `KUBERNETES_SERVICE_HOST` environment variable. When running in Kubernetes: + +- The pod receives the following environment variables from `deploy.sequencer.yaml`: + - `POD_NAME`: the pod name (from the downward API). + - `POD_NAMESPACE`: the namespace. + - `LEASE_NAME`: the name of the `Lease` object used for leader election (default: `sequencer-leader`). + - `LEASE_TTL_SECONDS`: lease time to live (default: `15` seconds). +- The code in `cmd/sequencer/src/k8s_leader.rs` uses these to participate in leader election via `kube-leader-election`: + - If the pod **acquires** the lease, it becomes the leader and starts the leader tasks: + - Batch producer. + - Block producer. + - Proof coordinator. + - Committer. + - If the pod **loses** the lease, it steps down and stops the leader tasks. +- On shutdown, the Sequencer writes a `node_config.json` under its data directory so that state can be reused on restart. + +The data directory is backed by the PVC: + +- The PV on the Minikube node uses the host path `/Users/Shared/mojave`. +- Inside the container, this PV is mounted at `/Users/Shared/mojave`. +- The Sequencer is started with `--datadir /Users/Shared/mojave/sequencer`, so the effective data directory is `/Users/Shared/mojave/sequencer` on the Minikube node. + +--- + +### 2. Local Minikube setup for Sequencer HA test (step-by-step) + +The following steps show how to run a **local, highly-available Sequencer** on Minikube using only commands you can copy and paste. + +#### 2.1 Prerequisites + +Make sure you have: + +- Docker (or another container runtime supported by Minikube). +- `kubectl`. +- Minikube (Docker driver is recommended on macOS). +- Rust toolchain (if you plan to build Mojave locally). +- `just` (optional but recommended for building Docker images; see the repository `justfile`). + +#### 2.2 Start Minikube + +Start a Minikube cluster: + +```bash +minikube start --driver=docker + +# Optional: enable metrics-server for better visibility +minikube addons enable metrics-server +``` + +You only need to do this once per Minikube profile. + +#### 2.3 Prepare persistent storage inside Minikube + +The `k8s/pvc.yaml` manifest expects a hostPath directory at `/Users/Shared/mojave` on the Minikube node. Create it and make it writable: + +```bash +minikube ssh "sudo mkdir -p /Users/Shared/mojave && sudo chmod 777 /Users/Shared/mojave" +``` + +Notes: + +- This directory lives **inside the Minikube VM/container**, not on your macOS host filesystem. +- The PV (`sequencer-pv`) references `/Users/Shared/mojave`, and the Sequencer pod mounts that PV at `/Users/Shared/mojave` inside the container. + +#### 2.4 Choose a Sequencer Docker image + +For this HA test you need a Docker image that contains the `mojave-sequencer` binary. There are two main options: + +**Option A (recommended for a quick test): Use the prebuilt Mojave image** + +- `k8s/deploy.sequencer.yaml` already references a published image: + - `starkgiwook/mojave-sequencer:latest` +- If your Minikube cluster can pull from Docker Hub, you **do not need to build anything**. You can skip to **2.6 Apply the core Kubernetes resources**. + +**Option B: Build and push your own image under your Docker namespace** + +1. Build the Sequencer image locally using the `justfile` helper: + + ```bash + just docker-build mojave-sequencer + ``` + + This creates a local image tagged as `1sixtech/mojave-sequencer`. + +2. Retag it into your own Docker namespace and push it: + + ```bash + # Replace with your Docker Hub (or registry) username + docker tag 1sixtech/mojave-sequencer /mojave-sequencer:latest + docker push /mojave-sequencer:latest + ``` + +3. Update the Deployment to use your image instead of the default: + + - Option 1: Edit `k8s/deploy.sequencer.yaml`: + + ```yaml + # inside k8s/deploy.sequencer.yaml + containers: + - name: mojave-sequencer + image: /mojave-sequencer:latest + imagePullPolicy: IfNotPresent + ``` + + - Option 2: Patch the live Deployment after applying the manifests: + + ```bash + kubectl set image deployment/mojave-sequencer-deployment \ + mojave-sequencer=/mojave-sequencer:latest + ``` + +#### 2.6 Apply the core Kubernetes resources + +You can apply the manifests one by one: + +```bash +kubectl apply -f k8s/pvc.yaml +kubectl apply -f k8s/rbac.sequencer.yaml +kubectl apply -f k8s/service.sequencer.yaml +kubectl apply -f k8s/deploy.sequencer.yaml +``` + +Or use the helper script (note: this **first deletes** all manifests under `k8s/`): + +```bash +bash k8s/setup.sh +``` + +#### 2.7 Verify the deployment + +Check that all resources are created: + +```bash +kubectl get pods +kubectl get svc mojave-sequencer-service +kubectl get pvc sequencer-pvc +kubectl get lease sequencer-leader -o yaml +``` + +You should see: + +- 3 pods with `app=mojave-sequencer`. +- A single `Lease` named `sequencer-leader` with one of the pods listed as the current holder. + +#### 2.8 Check that the current leader is producing blocks (via logs) + +Before testing failover, confirm that the current leader pod is actively producing blocks. + +1. **Identify the current leader pod** (via the `Lease` holder identity): + + ```bash + LEADER_POD=$(kubectl get lease sequencer-leader -o jsonpath='{.spec.holderIdentity}') + echo "Current leader pod: $LEADER_POD" + ``` + +2. **Print the last 100 log lines from the leader pod**: + + ```bash + kubectl logs "$LEADER_POD" -c mojave-sequencer | tail -n 100 + ``` + +Review the logs and confirm that the leader is producing blocks (or whatever periodic work you expect the Sequencer leader to perform). + +#### 2.9 Leader failover test + +To see Kubernetes-based HA in action: + +1. **Delete the current leader pod** (identified in the previous step): + + ```bash + kubectl delete pod "$LEADER_POD" + ``` + +2. **Wait for a new leader to be elected and identify it**: + + ```bash + # After some time (up to LEASE_TTL_SECONDS), check which pod is the new leader + NEW_LEADER_POD=$(kubectl get lease sequencer-leader -o jsonpath='{.spec.holderIdentity}') + echo "New leader pod: $NEW_LEADER_POD" + + # (Optional) list all sequencer pods + kubectl get pods -l app=mojave-sequencer + ``` + +3. **Print the last 100 log lines from the new leader pod**: + + ```bash + kubectl logs "$NEW_LEADER_POD" -c mojave-sequencer | tail -n 100 + ``` + +Within approximately the lease TTL window (`LEASE_TTL_SECONDS`, default 15s), another pod should acquire the lease and start the leader tasks. By comparing the logs before and after the failover, you can verify that block production (or other leader activity) continues on the new leader. + +#### 2.10 Cleanup + +To remove all Mojave-related Kubernetes resources created from `k8s/`: + +```bash +kubectl delete -f k8s/ +``` + +--- + +### 3. Notes and recommendations (towards production) + +The manifests in this repository are designed for **local development and testing**, not as a final production setup. For a production deployment, you should at least: + +- Avoid passing private keys on the command line; prefer Kubernetes `Secret` objects and environment variables or mounted files. +- Configure proper resource requests and limits for each container. +- Add liveness and readiness probes for the Sequencer container to improve reliability and rollout behavior. +- Consider a `Headless Service` or separate Services if you need stable pod identities or direct pod-to-pod communication. +- Configure `imagePullSecrets` if you use private container registries. +- Use a production-grade storage class instead of `hostPath` volumes. diff --git a/k8s/deploy.node.yaml b/k8s/deploy.node.yaml deleted file mode 100644 index 922eb86..0000000 --- a/k8s/deploy.node.yaml +++ /dev/null @@ -1,54 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: mojave-node-deployment - labels: - app: mojave-node -spec: - replicas: 1 - selector: - matchLabels: - app: mojave-node - template: - metadata: - labels: - app: mojave-node - spec: - serviceAccountName: node-sa - terminationGracePeriodSeconds: 30 - containers: - - name: mojave-node - image: starkgiwook/mojave-node:v0.1 - imagePullPolicy: Always - args: - - "init" - - "--network" - - "/data/testnet-genesis.json" - - "--p2p.addr" - - "0.0.0.0" - - "--p2p.port" - - "30305" - - "--discovery.addr" - - "0.0.0.0" - - "--discovery.port" - - "30305" - - "--no-daemon" - ports: - - { name: p2p, containerPort: 30305, protocol: TCP } - - { name: p2p-udp, containerPort: 30305, protocol: UDP } ---- -apiVersion: v1 -kind: Service -metadata: - name: mojave-node-p2p -spec: - selector: - app: mojave-node - ports: - - { name: p2p, port: 30305, targetPort: 30305, protocol: TCP } - - { name: p2p-udp, port: 30305, targetPort: 30305, protocol: UDP } ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: node-sa \ No newline at end of file diff --git a/k8s/deploy.sequencer.yaml b/k8s/deploy.sequencer.yaml index 983406d..f78e75f 100644 --- a/k8s/deploy.sequencer.yaml +++ b/k8s/deploy.sequencer.yaml @@ -18,7 +18,7 @@ spec: terminationGracePeriodSeconds: 30 containers: - name: mojave-sequencer - image: starkgiwook/mojave-sequencer:v0.1.5 + image: starkgiwook/mojave-sequencer:latest imagePullPolicy: Always env: - name: POD_NAME diff --git a/k8s/pvc.yaml b/k8s/pvc.yaml index f813e53..4dffac7 100644 --- a/k8s/pvc.yaml +++ b/k8s/pvc.yaml @@ -10,7 +10,7 @@ spec: persistentVolumeReclaimPolicy: Retain storageClassName: mojave-rwx hostPath: - path: /data/k8s-data + path: /Users/Shared/mojave type: DirectoryOrCreate --- apiVersion: v1 From bed857cbf1e5e2645b608ac99794173d673568f5 Mon Sep 17 00:00:00 2001 From: Giwook Han <73291175+Giwook-Han@users.noreply.github.com> Date: Tue, 18 Nov 2025 16:08:18 +0900 Subject: [PATCH 08/29] feat: mod batch producer load batch number when it starts (#272) --- Cargo.lock | 1 + cmd/sequencer/src/k8s_leader.rs | 2 +- crates/batch-producer/Cargo.toml | 1 + crates/batch-producer/src/batch_producer.rs | 10 +++++++++- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d88d6a..8045985 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4439,6 +4439,7 @@ dependencies = [ "ethrex-vm", "mojave-node-lib", "mojave-task", + "mojave-utils", "thiserror 2.0.17", "tokio", "tracing", diff --git a/cmd/sequencer/src/k8s_leader.rs b/cmd/sequencer/src/k8s_leader.rs index 8786571..82d5567 100644 --- a/cmd/sequencer/src/k8s_leader.rs +++ b/cmd/sequencer/src/k8s_leader.rs @@ -129,7 +129,7 @@ pub async fn start_leader_tasks( // TODO: replace by implementation backed by a real queue let q = mojave_msgio::dummy::Dummy; - let batch_producer = BatchProducer::new(node.clone(), 0); + let batch_producer = BatchProducer::new(node.clone()); let block_producer = BlockProducer::new(node.clone()); let proof_coordinator = ProofCoordinator::new(node.clone(), node_options, proof_coordinator_options)?; diff --git a/crates/batch-producer/Cargo.toml b/crates/batch-producer/Cargo.toml index 247aeab..f95066e 100644 --- a/crates/batch-producer/Cargo.toml +++ b/crates/batch-producer/Cargo.toml @@ -11,6 +11,7 @@ documentation = { workspace = true } [dependencies] mojave-node-lib = { workspace = true } mojave-task = { workspace = true } +mojave-utils = { workspace = true } ethrex-blockchain = { workspace = true, default-features = false } ethrex-common = { workspace = true } diff --git a/crates/batch-producer/src/batch_producer.rs b/crates/batch-producer/src/batch_producer.rs index a84a37c..506f129 100644 --- a/crates/batch-producer/src/batch_producer.rs +++ b/crates/batch-producer/src/batch_producer.rs @@ -73,9 +73,17 @@ impl Task for BatchProducer { } impl BatchProducer { - pub fn new(node: MojaveNode, batch_counter: u64) -> Self { + pub fn new(node: MojaveNode) -> Self { let (broadcast, _) = tokio::sync::broadcast::channel(MAX_BATCH_TO_BROADCAST); + let batch_counter = mojave_utils::block_on::block_on_current_thread(async || { + if let Some(batch_number) = node.rollup_store.get_batch_number().await? { + return Ok(batch_number); + } + Ok(0) + }) + .unwrap_or(0); + BatchProducer { batch_counter, store: node.store.clone(), From 1c96a8ef8e98067dbeac7397bbe44ce07b153fcc Mon Sep 17 00:00:00 2001 From: Giwook-Han Date: Tue, 18 Nov 2025 16:10:35 +0900 Subject: [PATCH 09/29] fix: cargo.toml --- cmd/sequencer/Cargo.toml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/sequencer/Cargo.toml b/cmd/sequencer/Cargo.toml index 87f6cef..da4765e 100644 --- a/cmd/sequencer/Cargo.toml +++ b/cmd/sequencer/Cargo.toml @@ -24,5 +24,11 @@ hex = { workspace = true } k8s-openapi = { workspace = true } kube = { workspace = true } kube-leader-election = { workspace = true } -tokio = { workspace = true, features = ["macros", "rt", "sync", "time", "signal"] } +tokio = { workspace = true, features = [ + "macros", + "rt", + "sync", + "time", + "signal", +] } tracing = { workspace = true } From 9a6b3ae68c5d88d955ba295f71abcbd360868415 Mon Sep 17 00:00:00 2001 From: Giwook-Han Date: Wed, 19 Nov 2025 12:35:11 +0900 Subject: [PATCH 10/29] chore: toml --- cmd/sequencer/Cargo.toml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/sequencer/Cargo.toml b/cmd/sequencer/Cargo.toml index da4765e..55dab0b 100644 --- a/cmd/sequencer/Cargo.toml +++ b/cmd/sequencer/Cargo.toml @@ -25,10 +25,10 @@ k8s-openapi = { workspace = true } kube = { workspace = true } kube-leader-election = { workspace = true } tokio = { workspace = true, features = [ - "macros", - "rt", - "sync", - "time", - "signal", + "macros", + "rt", + "sync", + "time", + "signal", ] } tracing = { workspace = true } From 78df5f25f4a5f321cebde3d485b1c1e45a4dc93c Mon Sep 17 00:00:00 2001 From: Giwook-Han Date: Thu, 20 Nov 2025 19:04:38 +0900 Subject: [PATCH 11/29] mod: k8s + ovh --- cmd/sequencer/src/k8s_leader.rs | 7 ++++++- crates/batch-producer/src/batch_producer.rs | 10 +--------- justfile | 2 +- k8s/deploy.sequencer.yaml | 10 +--------- k8s/ovh.pvc.yaml | 11 +++++++++++ k8s/ovh.setup.sh | 6 ++++++ 6 files changed, 26 insertions(+), 20 deletions(-) create mode 100644 k8s/ovh.pvc.yaml create mode 100755 k8s/ovh.setup.sh diff --git a/cmd/sequencer/src/k8s_leader.rs b/cmd/sequencer/src/k8s_leader.rs index 82d5567..cb51621 100644 --- a/cmd/sequencer/src/k8s_leader.rs +++ b/cmd/sequencer/src/k8s_leader.rs @@ -129,7 +129,12 @@ pub async fn start_leader_tasks( // TODO: replace by implementation backed by a real queue let q = mojave_msgio::dummy::Dummy; - let batch_producer = BatchProducer::new(node.clone()); + let batch_counter = match node.rollup_store.get_batch_number().await? { + Some(batch_number) => batch_number, + _ => 0, + }; + + let batch_producer = BatchProducer::new(node.clone(), batch_counter); let block_producer = BlockProducer::new(node.clone()); let proof_coordinator = ProofCoordinator::new(node.clone(), node_options, proof_coordinator_options)?; diff --git a/crates/batch-producer/src/batch_producer.rs b/crates/batch-producer/src/batch_producer.rs index 506f129..a84a37c 100644 --- a/crates/batch-producer/src/batch_producer.rs +++ b/crates/batch-producer/src/batch_producer.rs @@ -73,17 +73,9 @@ impl Task for BatchProducer { } impl BatchProducer { - pub fn new(node: MojaveNode) -> Self { + pub fn new(node: MojaveNode, batch_counter: u64) -> Self { let (broadcast, _) = tokio::sync::broadcast::channel(MAX_BATCH_TO_BROADCAST); - let batch_counter = mojave_utils::block_on::block_on_current_thread(async || { - if let Some(batch_number) = node.rollup_store.get_batch_number().await? { - return Ok(batch_number); - } - Ok(0) - }) - .unwrap_or(0); - BatchProducer { batch_counter, store: node.store.clone(), diff --git a/justfile b/justfile index 1842383..29a66f3 100644 --- a/justfile +++ b/justfile @@ -74,7 +74,7 @@ sequencer: export $(cat .env | xargs) && \ mkdir -p {{home-dir}}/.mojave/sequencer && \ echo "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" > {{home-dir}}/.mojave/sequencer/node.key && \ - if [ -z "$SKIP_BUILD" ]; then cargo build --bin mojave-sequencer; fi && \ + cargo build --bin mojave-sequencer && \ ( \ "${BIN_DIR:-target/debug}"/mojave-sequencer init \ --private_key 0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ diff --git a/k8s/deploy.sequencer.yaml b/k8s/deploy.sequencer.yaml index f78e75f..e4d0d03 100644 --- a/k8s/deploy.sequencer.yaml +++ b/k8s/deploy.sequencer.yaml @@ -18,7 +18,7 @@ spec: terminationGracePeriodSeconds: 30 containers: - name: mojave-sequencer - image: starkgiwook/mojave-sequencer:latest + image: starkgiwook/mojave-sequencer:v0.1.9 imagePullPolicy: Always env: - name: POD_NAME @@ -41,10 +41,6 @@ spec: - "/data/testnet-genesis.json" - "--datadir" - "/Users/Shared/mojave/sequencer" - - "--http.port" - - "18545" - - "--authrpc.port" - - "18551" - "--p2p.port" - "30305" - "--discovery.port" @@ -54,10 +50,6 @@ spec: - name: sequencer-datadir mountPath: /Users/Shared/mojave ports: - - name: http - containerPort: 18545 - - name: auth-rpc - containerPort: 18551 - name: p2p containerPort: 30305 protocol: TCP diff --git a/k8s/ovh.pvc.yaml b/k8s/ovh.pvc.yaml new file mode 100644 index 0000000..d41510d --- /dev/null +++ b/k8s/ovh.pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: sequencer-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + storageClassName: csi-cinder-high-speed-gen2 \ No newline at end of file diff --git a/k8s/ovh.setup.sh b/k8s/ovh.setup.sh new file mode 100755 index 0000000..f5e56dd --- /dev/null +++ b/k8s/ovh.setup.sh @@ -0,0 +1,6 @@ +kubectl --kubeconfig=/Users/Giwook/.kube/mojave-sequencer-test.yml delete -f k8s/ + +kubectl --kubeconfig=/Users/Giwook/.kube/mojave-sequencer-test.yml apply -f k8s/ovh.pvc.yaml +kubectl --kubeconfig=/Users/Giwook/.kube/mojave-sequencer-test.yml apply -f k8s/rbac.sequencer.yaml +kubectl --kubeconfig=/Users/Giwook/.kube/mojave-sequencer-test.yml apply -f k8s/service.sequencer.yaml +kubectl --kubeconfig=/Users/Giwook/.kube/mojave-sequencer-test.yml apply -f k8s/deploy.sequencer.yaml \ No newline at end of file From 20df9c4021817015c517173f8182cd53582e4e9f Mon Sep 17 00:00:00 2001 From: Sacha Froment Date: Mon, 24 Nov 2025 13:05:04 +0100 Subject: [PATCH 12/29] chore: lint Signed-off-by: Sacha Froment --- cmd/sequencer/src/k8s_leader.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cmd/sequencer/src/k8s_leader.rs b/cmd/sequencer/src/k8s_leader.rs index cb51621..b59e115 100644 --- a/cmd/sequencer/src/k8s_leader.rs +++ b/cmd/sequencer/src/k8s_leader.rs @@ -129,11 +129,7 @@ pub async fn start_leader_tasks( // TODO: replace by implementation backed by a real queue let q = mojave_msgio::dummy::Dummy; - let batch_counter = match node.rollup_store.get_batch_number().await? { - Some(batch_number) => batch_number, - _ => 0, - }; - + let batch_counter = node.rollup_store.get_batch_number().await?.unwrap_or(0); let batch_producer = BatchProducer::new(node.clone(), batch_counter); let block_producer = BlockProducer::new(node.clone()); let proof_coordinator = From 36eb56680f93c7a1eaed852c879196f4b753bf77 Mon Sep 17 00:00:00 2001 From: Sacha Froment Date: Mon, 24 Nov 2025 13:30:55 +0100 Subject: [PATCH 13/29] chore: fix here we have 2 result imbricated Signed-off-by: Sacha Froment --- cmd/sequencer/src/k8s_leader.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/sequencer/src/k8s_leader.rs b/cmd/sequencer/src/k8s_leader.rs index b59e115..e8d0667 100644 --- a/cmd/sequencer/src/k8s_leader.rs +++ b/cmd/sequencer/src/k8s_leader.rs @@ -1,6 +1,6 @@ use kube::Client; use kube_leader_election::{LeaseLock, LeaseLockParams}; -use std::{env, result::Result::*, time::Duration}; +use std::{env, time::Duration}; use tokio::time::sleep; use mojave_batch_producer::{BatchProducer, types::Request as BatchProducerRequest}; @@ -167,7 +167,7 @@ pub async fn stop_leader_tasks(lt: LeaderTasks) -> Result<(), Box Date: Mon, 24 Nov 2025 14:40:59 +0100 Subject: [PATCH 14/29] chore: change a bit Signed-off-by: Sacha Froment --- docs/k8s.md | 16 ++++---- k8s/deploy.sequencer.yaml | 83 ++++++++++++++++++++------------------- k8s/ovh.pvc.yaml | 11 ------ k8s/ovh.setup.sh | 6 --- k8s/pvc.yaml | 4 +- k8s/setup.sh | 6 ++- 6 files changed, 57 insertions(+), 69 deletions(-) delete mode 100644 k8s/ovh.pvc.yaml delete mode 100755 k8s/ovh.setup.sh diff --git a/docs/k8s.md b/docs/k8s.md index fbf008c..d731b73 100644 --- a/docs/k8s.md +++ b/docs/k8s.md @@ -26,7 +26,7 @@ All manifests live under `k8s/`: - `ServiceAccount` (`sequencer-sa`). - `Role` and `RoleBinding` that grant access to `coordination.k8s.io/v1` `Lease` objects. - `k8s/pvc.yaml`: - - `PersistentVolume` (`sequencer-pv`) that uses a `hostPath` directory **inside the Minikube node** at `/Users/Shared/mojave`. + - `PersistentVolume` (`sequencer-pv`) that uses a `hostPath` directory **inside the Minikube node** at `/data/mojave`. - `PersistentVolumeClaim` (`sequencer-pvc`) bound to that PV. - `k8s/setup.sh`: Helper script that deletes everything under `k8s/` and then re-applies the core resources (PVC, RBAC, Service, Deployment). @@ -50,9 +50,9 @@ The Sequencer binary detects a Kubernetes environment by checking the `KUBERNETE The data directory is backed by the PVC: -- The PV on the Minikube node uses the host path `/Users/Shared/mojave`. -- Inside the container, this PV is mounted at `/Users/Shared/mojave`. -- The Sequencer is started with `--datadir /Users/Shared/mojave/sequencer`, so the effective data directory is `/Users/Shared/mojave/sequencer` on the Minikube node. +- The PV on the Minikube node uses the host path `/data/mojave`. +- Inside the container, this PV is mounted at `/data/mojave`. +- The Sequencer is started with `--datadir /data/mojave/sequencer`, so the effective data directory is `/data/mojave/sequencer` on the Minikube node. --- @@ -85,16 +85,16 @@ You only need to do this once per Minikube profile. #### 2.3 Prepare persistent storage inside Minikube -The `k8s/pvc.yaml` manifest expects a hostPath directory at `/Users/Shared/mojave` on the Minikube node. Create it and make it writable: +The `k8s/pvc.yaml` manifest expects a hostPath directory at `/data/mojave` on the Minikube node. Create it and make it writable: ```bash -minikube ssh "sudo mkdir -p /Users/Shared/mojave && sudo chmod 777 /Users/Shared/mojave" +minikube ssh "sudo mkdir -p /data/mojave && sudo chmod 777 /data/mojave" ``` Notes: -- This directory lives **inside the Minikube VM/container**, not on your macOS host filesystem. -- The PV (`sequencer-pv`) references `/Users/Shared/mojave`, and the Sequencer pod mounts that PV at `/Users/Shared/mojave` inside the container. +- This directory lives **inside the Minikube VM/container**, not on your host filesystem. +- The PV (`sequencer-pv`) references `/data/mojave`, and the Sequencer pod mounts that PV at `/data/mojave` inside the container. #### 2.4 Choose a Sequencer Docker image diff --git a/k8s/deploy.sequencer.yaml b/k8s/deploy.sequencer.yaml index e4d0d03..8a70896 100644 --- a/k8s/deploy.sequencer.yaml +++ b/k8s/deploy.sequencer.yaml @@ -17,46 +17,49 @@ spec: serviceAccountName: sequencer-sa terminationGracePeriodSeconds: 30 containers: - - name: mojave-sequencer - image: starkgiwook/mojave-sequencer:v0.1.9 - imagePullPolicy: Always - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: LEASE_NAME - value: sequencer-leader - - name: LEASE_TTL_SECONDS - value: "15" - args: - - "init" - - "--private_key" - - "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - - "--network" - - "/data/testnet-genesis.json" - - "--datadir" - - "/Users/Shared/mojave/sequencer" - - "--p2p.port" - - "30305" - - "--discovery.port" - - "30305" - - "--no-daemon" - volumeMounts: - - name: sequencer-datadir - mountPath: /Users/Shared/mojave - ports: - - name: p2p - containerPort: 30305 - protocol: TCP - - name: p2p-udp - containerPort: 30305 - protocol: UDP + - name: mojave-sequencer + image: starkgiwook/mojave-sequencer:v0.1.9 + imagePullPolicy: Always + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LEASE_NAME + value: sequencer-leader + - name: LEASE_TTL_SECONDS + value: "15" + - name: PRIVATE_KEY + valueFrom: + secretKeyRef: + name: sequencer-secrets + key: private-key + args: + - "init" + - "--network" + - "/data/testnet-genesis.json" + - "--datadir" + - "/data/mojave/sequencer" + - "--p2p.port" + - "30305" + - "--discovery.port" + - "30305" + - "--no-daemon" + volumeMounts: + - name: sequencer-datadir + mountPath: /data/mojave + ports: + - name: p2p + containerPort: 30305 + protocol: TCP + - name: p2p-udp + containerPort: 30305 + protocol: UDP volumes: - name: sequencer-datadir persistentVolumeClaim: - claimName: sequencer-pvc \ No newline at end of file + claimName: sequencer-pvc diff --git a/k8s/ovh.pvc.yaml b/k8s/ovh.pvc.yaml deleted file mode 100644 index d41510d..0000000 --- a/k8s/ovh.pvc.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: sequencer-pvc -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 20Gi - storageClassName: csi-cinder-high-speed-gen2 \ No newline at end of file diff --git a/k8s/ovh.setup.sh b/k8s/ovh.setup.sh deleted file mode 100755 index f5e56dd..0000000 --- a/k8s/ovh.setup.sh +++ /dev/null @@ -1,6 +0,0 @@ -kubectl --kubeconfig=/Users/Giwook/.kube/mojave-sequencer-test.yml delete -f k8s/ - -kubectl --kubeconfig=/Users/Giwook/.kube/mojave-sequencer-test.yml apply -f k8s/ovh.pvc.yaml -kubectl --kubeconfig=/Users/Giwook/.kube/mojave-sequencer-test.yml apply -f k8s/rbac.sequencer.yaml -kubectl --kubeconfig=/Users/Giwook/.kube/mojave-sequencer-test.yml apply -f k8s/service.sequencer.yaml -kubectl --kubeconfig=/Users/Giwook/.kube/mojave-sequencer-test.yml apply -f k8s/deploy.sequencer.yaml \ No newline at end of file diff --git a/k8s/pvc.yaml b/k8s/pvc.yaml index 4dffac7..79256b7 100644 --- a/k8s/pvc.yaml +++ b/k8s/pvc.yaml @@ -10,7 +10,7 @@ spec: persistentVolumeReclaimPolicy: Retain storageClassName: mojave-rwx hostPath: - path: /Users/Shared/mojave + path: /data/mojave type: DirectoryOrCreate --- apiVersion: v1 @@ -24,4 +24,4 @@ spec: requests: storage: 20Gi storageClassName: mojave-rwx - volumeName: sequencer-pv \ No newline at end of file + volumeName: sequencer-pv diff --git a/k8s/setup.sh b/k8s/setup.sh index 47dac4e..e3c0524 100755 --- a/k8s/setup.sh +++ b/k8s/setup.sh @@ -1,6 +1,8 @@ -kubectl delete -f k8s/ +#!/bin/bash + +kubectl delete -f k8s/ --ignore-not-found=true kubectl apply -f k8s/pvc.yaml kubectl apply -f k8s/rbac.sequencer.yaml kubectl apply -f k8s/service.sequencer.yaml -kubectl apply -f k8s/deploy.sequencer.yaml \ No newline at end of file +kubectl apply -f k8s/deploy.sequencer.yaml From 843521884d3049e500f48d8412860335db547c8f Mon Sep 17 00:00:00 2001 From: Sacha Froment Date: Tue, 25 Nov 2025 16:12:40 +0100 Subject: [PATCH 15/29] feat: add node deploy Signed-off-by: Sacha Froment --- cmd/node/src/cli.rs | 64 +++++++++++++++++++++++++++++++++++--- cmd/sequencer/src/cli.rs | 8 +++-- justfile | 9 ++++-- k8s/deploy.node.yaml | 63 +++++++++++++++++++++++++++++++++++++ k8s/deploy.sequencer.yaml | 54 +++++++++++++++++++++++--------- k8s/namespace.yaml | 4 +++ k8s/pvc.yaml | 27 ---------------- k8s/rbac.sequencer.yaml | 6 +++- k8s/secret.sequencer.yaml | 8 +++++ k8s/service.sequencer.yaml | 13 ++++---- k8s/setup.sh | 8 ++++- 11 files changed, 206 insertions(+), 58 deletions(-) create mode 100644 k8s/deploy.node.yaml create mode 100644 k8s/namespace.yaml delete mode 100644 k8s/pvc.yaml create mode 100644 k8s/secret.sequencer.yaml diff --git a/cmd/node/src/cli.rs b/cmd/node/src/cli.rs index 6adb9f9..b1a3d95 100644 --- a/cmd/node/src/cli.rs +++ b/cmd/node/src/cli.rs @@ -1,8 +1,59 @@ +use std::{fmt::format, str::FromStr}; + use clap::{ArgAction, Parser, Subcommand}; use mojave_node_lib::types::{Node, SyncMode}; use mojave_utils::network::Network; +use std::net::ToSocketAddrs; use tracing::Level; +fn resolve_dns_host_port(addr: &str) -> Result { + let mut iter = addr.to_socket_addrs().map_err(|e| { + mojave_node_lib::error::Error::Custom(format!("to_socket failed {}", e).to_string()) + })?; + + let socket_addr = iter.next().ok_or_else(|| { + mojave_node_lib::error::Error::Custom(format!( + "DNS resolution for `{addr}` returned no addresses" + )) + })?; + + // This will be something like "10.42.0.12:3030" + Ok(socket_addr.to_string()) +} + +#[derive(Debug, Clone)] +pub struct DNSNode { + inner: Node, +} + +impl FromStr for DNSNode { + type Err = mojave_node_lib::error::Error; + + fn from_str(enode: &str) -> Result { + let at_pos = enode.find('@').ok_or_else(|| { + mojave_node_lib::error::Error::Custom("Invalid enode: missing `@`".into()) + })?; + + let after_at = &enode[at_pos + 1..]; // "host:port?discport=..." + let (host_port, rest) = match after_at.find('?') { + Some(pos) => (&after_at[..pos], &after_at[pos..]), // ("host:port", "?discport=...") + None => (after_at, ""), + }; + + let resolved = resolve_dns_host_port(host_port)?; // "IP:port" + + let enode = format!("{}{}{}", &enode[..=at_pos], resolved, rest); + let node = Node::from_str(&enode)?; + Ok(DNSNode { inner: node }) + } +} + +impl From for Node { + fn from(dns_node: DNSNode) -> Self { + dns_node.inner + } +} + #[derive(Parser)] pub struct Options { #[arg( @@ -19,14 +70,14 @@ pub struct Options { #[arg( long = "bootnodes", - value_parser = clap::value_parser!(Node), + value_parser = clap::value_parser!(DNSNode), value_name = "BOOTNODE_LIST", value_delimiter = ',', num_args = 1.., help = "Comma separated enode URLs for P2P discovery bootstrap.", help_heading = "P2P options" )] - pub bootnodes: Vec, + pub bootnodes: Vec, #[arg( long = "syncmode", @@ -186,7 +237,12 @@ impl From<&Options> for mojave_node_lib::types::NodeOptions { discovery_addr: options.discovery_addr.clone(), discovery_port: options.discovery_port.clone(), network: options.network.clone(), - bootnodes: options.bootnodes.clone(), + bootnodes: options + .bootnodes + .iter() + .cloned() + .map(|dn| dn.into()) + .collect(), datadir: Default::default(), syncmode: options.syncmode.unwrap_or(SyncMode::Full), sponsorable_addresses_file_path: options.sponsorable_addresses_file_path.clone(), @@ -328,7 +384,7 @@ mod tests { assert_eq!(node_opts.discovery_addr, options.discovery_addr); assert_eq!(node_opts.discovery_port, options.discovery_port); assert!(matches!(node_opts.network, Network::DefaultNet)); - assert_eq!(node_opts.bootnodes, options.bootnodes); + //assert_eq!(node_opts.bootnodes.iter().cloned(), options.bootnodes); assert!(matches!(node_opts.syncmode, SyncMode::Full)); // syncmode is not set from Options. Override to default assert_eq!( node_opts.sponsorable_addresses_file_path, diff --git a/cmd/sequencer/src/cli.rs b/cmd/sequencer/src/cli.rs index cb6bcad..c299aff 100644 --- a/cmd/sequencer/src/cli.rs +++ b/cmd/sequencer/src/cli.rs @@ -10,7 +10,7 @@ pub struct Options { #[arg( long = "network", default_value_t = Network::default(), - value_name = "GENESIS_FILE_PATH", + value_name = "GENESIS_FILE_PATH", help = "Receives a `Genesis` struct in json format. This is the only argument which is required. You can look at some example genesis files at `test_data/genesis*`.", long_help = "Alternatively, the name of a known network can be provided instead to use its preset genesis file and include its preset bootnodes. The networks currently supported include holesky, sepolia, hoodi and mainnet.", help_heading = "Node options", @@ -233,7 +233,11 @@ pub struct SequencerOptions { default_value = "1000" )] pub block_time: u64, - #[arg(long = "private_key", help = "Private key used for signing blocks")] + #[arg( + long = "private_key", + help = "Private key used for signing blocks", + env = "PRIVATE_KEY" + )] pub private_key: String, } diff --git a/justfile b/justfile index 29a66f3..9ce7d99 100644 --- a/justfile +++ b/justfile @@ -182,16 +182,16 @@ doc: doc-watch: cargo watch -x "doc --no-deps" -registry := "ghcr.io/1sixtech" +registry := "1sixtech" # Build the docker image for a specific binary # Binary name should be one of: mojave-node, mojave-sequencer, mojave-prover docker-build bin registry=registry: role="{{bin}}"; \ role="${role#mojave-}"; \ - docker build --platform=linux/amd64,linux/arm64 \ + docker build --platform=linux/arm64 \ -f "docker/Dockerfile.target" \ - -t "{{registry}}/{{bin}}" \ + -t {{ if registry == '' { bin } else { registry + '/' + bin } }} \ --build-arg "TARGET_BIN={{bin}}" \ . @@ -200,3 +200,6 @@ docker-run bin *ARGS: test: clean bash tests/tests-e2e.sh + +run-k8s: + bash k8s/setup.sh diff --git a/k8s/deploy.node.yaml b/k8s/deploy.node.yaml new file mode 100644 index 0000000..72dccaf --- /dev/null +++ b/k8s/deploy.node.yaml @@ -0,0 +1,63 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mojave-node-deployment + namespace: 1sixtech + labels: + app: mojave-node +spec: + replicas: 1 + selector: + matchLabels: + app: mojave-node + template: + metadata: + labels: + app: mojave-node + spec: + terminationGracePeriodSeconds: 30 + containers: + - name: mojave-node + image: mojave-node:latest + imagePullPolicy: IfNotPresent + args: + - "init" + - "--datadir" + - "/data/mojave/node" + - "--http.addr" + - "0.0.0.0" + - "--http.port" + - "8545" + - "--authrpc.addr" + - "0.0.0.0" + - "--network" + - "/data/testnet-genesis.json" + - "--authrpc.port" + - "8551" + - "--authrpc.jwtsecret" + - "/data/mojave/node/jwt.hex" + - "--p2p.addr" + - "0.0.0.0" + - "--discovery.addr" + - "0.0.0.0" + - "--bootnodes=enode://3e9c8a6bc193671ef87ea714ba2bcc979ae820672d5c93ff0ed265129b22180264eecebeae70ba947a6ffad76ab47eef41031838039f8f0ba84ea98b4d8734e5@mojave-sequencer-0.mojave-sequencer.1sixtech.svc.cluster.local:30303" + - "--no-daemon" + volumeMounts: + - name: node-datadir + mountPath: /data/mojave + ports: + - name: http + containerPort: 8545 + protocol: TCP + - name: authrpc + containerPort: 8551 + protocol: TCP + - name: p2p + containerPort: 30303 + protocol: TCP + - name: p2p-udp + containerPort: 30303 + protocol: UDP + volumes: + - name: node-datadir + emptyDir: {} diff --git a/k8s/deploy.sequencer.yaml b/k8s/deploy.sequencer.yaml index 8a70896..aa453ed 100644 --- a/k8s/deploy.sequencer.yaml +++ b/k8s/deploy.sequencer.yaml @@ -1,11 +1,13 @@ apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: - name: mojave-sequencer-deployment + name: mojave-sequencer + namespace: 1sixtech labels: app: mojave-sequencer spec: - replicas: 3 + serviceName: mojave-sequencer + replicas: 2 selector: matchLabels: app: mojave-sequencer @@ -16,10 +18,29 @@ spec: spec: serviceAccountName: sequencer-sa terminationGracePeriodSeconds: 30 + initContainers: + - name: init-ordinal-file + image: busybox:1.36 + command: + - sh + - -c + - | + set -e + ordinal="${HOSTNAME##*-}" + echo "${ordinal}" > /data/mojave/statefulset-ordinal + mkdir -p /data/mojave/sequencer + if [ "${ordinal}" = "0" ]; then + echo "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" > /data/mojave/sequencer/node.key + else + echo "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaab" > /data/mojave/sequencer/node.key + fi + volumeMounts: + - name: sequencer-datadir + mountPath: /data/mojave containers: - name: mojave-sequencer - image: starkgiwook/mojave-sequencer:v0.1.9 - imagePullPolicy: Always + image: mojave-sequencer:latest + imagePullPolicy: IfNotPresent env: - name: POD_NAME valueFrom: @@ -44,22 +65,27 @@ spec: - "/data/testnet-genesis.json" - "--datadir" - "/data/mojave/sequencer" - - "--p2p.port" - - "30305" - "--discovery.port" - - "30305" + - "30303" - "--no-daemon" volumeMounts: - name: sequencer-datadir mountPath: /data/mojave ports: - name: p2p - containerPort: 30305 + containerPort: 30303 protocol: TCP - name: p2p-udp - containerPort: 30305 + containerPort: 30303 protocol: UDP - volumes: - - name: sequencer-datadir - persistentVolumeClaim: - claimName: sequencer-pvc + volumeClaimTemplates: + - metadata: + name: sequencer-datadir + labels: + app: mojave-sequencer + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi diff --git a/k8s/namespace.yaml b/k8s/namespace.yaml new file mode 100644 index 0000000..9156f8e --- /dev/null +++ b/k8s/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: 1sixtech diff --git a/k8s/pvc.yaml b/k8s/pvc.yaml deleted file mode 100644 index 79256b7..0000000 --- a/k8s/pvc.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: v1 -kind: PersistentVolume -metadata: - name: sequencer-pv -spec: - capacity: - storage: 20Gi - accessModes: - - ReadWriteMany - persistentVolumeReclaimPolicy: Retain - storageClassName: mojave-rwx - hostPath: - path: /data/mojave - type: DirectoryOrCreate ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: sequencer-pvc -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 20Gi - storageClassName: mojave-rwx - volumeName: sequencer-pv diff --git a/k8s/rbac.sequencer.yaml b/k8s/rbac.sequencer.yaml index 624cea4..192f899 100644 --- a/k8s/rbac.sequencer.yaml +++ b/k8s/rbac.sequencer.yaml @@ -2,11 +2,13 @@ apiVersion: v1 kind: ServiceAccount metadata: name: sequencer-sa + namespace: 1sixtech --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: sequencer-lease-role + namespace: 1sixtech rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] @@ -16,10 +18,12 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: sequencer-lease-rb + namespace: 1sixtech roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: sequencer-lease-role subjects: - kind: ServiceAccount - name: sequencer-sa \ No newline at end of file + name: sequencer-sa + namespace: 1sixtech diff --git a/k8s/secret.sequencer.yaml b/k8s/secret.sequencer.yaml new file mode 100644 index 0000000..4008707 --- /dev/null +++ b/k8s/secret.sequencer.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: sequencer-secrets + namespace: 1sixtech +type: Opaque +data: + private-key: MHhhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFh diff --git a/k8s/service.sequencer.yaml b/k8s/service.sequencer.yaml index ca56f7c..5ad37a8 100644 --- a/k8s/service.sequencer.yaml +++ b/k8s/service.sequencer.yaml @@ -1,9 +1,10 @@ apiVersion: v1 kind: Service metadata: - name: mojave-sequencer-service + name: mojave-sequencer + namespace: 1sixtech spec: - type: ClusterIP + clusterIP: None selector: app: mojave-sequencer ports: @@ -13,9 +14,9 @@ spec: targetPort: 18545 - name: p2p protocol: TCP - port: 30305 - targetPort: 30305 + port: 30303 + targetPort: 30303 - name: p2p-udp protocol: UDP - port: 30305 - targetPort: 30305 \ No newline at end of file + port: 30303 + targetPort: 30303 diff --git a/k8s/setup.sh b/k8s/setup.sh index e3c0524..7209152 100755 --- a/k8s/setup.sh +++ b/k8s/setup.sh @@ -1,8 +1,14 @@ #!/bin/bash +just docker-build mojave-sequencer "" +just docker-build mojave-node "" +just docker-build mojave-prover "" + kubectl delete -f k8s/ --ignore-not-found=true -kubectl apply -f k8s/pvc.yaml +kubectl apply -f k8s/namespace.yaml +kubectl apply -f k8s/secret.sequencer.yaml kubectl apply -f k8s/rbac.sequencer.yaml kubectl apply -f k8s/service.sequencer.yaml kubectl apply -f k8s/deploy.sequencer.yaml +kubectl apply -f k8s/deploy.node.yaml From d77b6f8f0a7c6aa4340176aa09c53af92215e667 Mon Sep 17 00:00:00 2001 From: Sacha Froment Date: Tue, 25 Nov 2025 16:25:00 +0100 Subject: [PATCH 16/29] chore: fix potential race Signed-off-by: Sacha Froment --- cmd/sequencer/src/k8s_leader.rs | 47 ++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/cmd/sequencer/src/k8s_leader.rs b/cmd/sequencer/src/k8s_leader.rs index e8d0667..6d2c7b7 100644 --- a/cmd/sequencer/src/k8s_leader.rs +++ b/cmd/sequencer/src/k8s_leader.rs @@ -1,4 +1,5 @@ -use kube::Client; +use k8s_openapi::api::coordination::v1::Lease; +use kube::{Api, Client}; use kube_leader_election::{LeaseLock, LeaseLockParams}; use std::{env, time::Duration}; use tokio::time::sleep; @@ -33,6 +34,7 @@ pub async fn run_with_k8s_coordination( proof_coordinator_options: ProofCoordinatorOptions, ) -> Result<(), Box> { let client = Client::try_default().await?; + let lease_client = client.clone(); let identity = env::var("POD_NAME").unwrap_or_else(|_| "sequencer-pod".to_string()); let namespace = env::var("POD_NAMESPACE").unwrap_or_else(|_| "default".to_string()); @@ -44,11 +46,11 @@ pub async fn run_with_k8s_coordination( let renew_every_secs = lease_ttl_sec / 5; // 1/5 of TTL let lease_lock = LeaseLock::new( - client, + lease_client, &namespace, LeaseLockParams { - lease_name, - holder_id: identity, + lease_name: lease_name.clone(), + holder_id: identity.clone(), lease_ttl: Duration::from_secs(lease_ttl_sec), }, ); @@ -80,13 +82,16 @@ pub async fn run_with_k8s_coordination( } res = lease_lock.try_acquire_or_renew() => { match res { - Ok(res) => { - let became_leader = res.acquired_lease; - - if !am_i_leader && became_leader { - // GW - Do I need to add Sleep here to wait all the leader task stop? - sleep(Duration::from_secs(2)).await; + Ok(_) => { + let currently_leader = match is_current_leader(&client, &namespace, &lease_name, &identity).await { + Ok(is_leader) => is_leader, + Err(err) => { + error!("Could not verify lease holder stepping down: {err:?}"); + false + } + }; + if !am_i_leader && currently_leader { info!("Became a leader. Start leader tasks"); leader_tasks = Some( start_leader_tasks( @@ -98,11 +103,14 @@ pub async fn run_with_k8s_coordination( .await?, ); am_i_leader = true; - } else if !became_leader && am_i_leader { - info!("Became a follower. Stop leader tasks"); + } else if am_i_leader && !currently_leader { + info!("Lost leadership. Stop leader tasks"); if let Some(lt) = leader_tasks.take() { stop_leader_tasks(lt).await?; } + if let Err(err) = lease_lock.step_down().await { + error!("Error while stepping down from leader: {err:?}"); + } am_i_leader = false; } @@ -180,3 +188,18 @@ pub fn is_k8s_env() -> bool { _ => false, } } + +async fn is_current_leader( + client: &Client, + namespace: &str, + lease_name: &str, + identity: &str, +) -> Result { + let leases: Api = Api::namespaced(client.clone(), namespace); + let lease = leases.get(lease_name).await?; + let holder = lease + .spec + .as_ref() + .and_then(|spec| spec.holder_identity.clone()); + Ok(holder.as_deref() == Some(identity)) +} From a39ef6b09a6727307b5e9f760d119c59a1c57569 Mon Sep 17 00:00:00 2001 From: Sacha Froment Date: Tue, 25 Nov 2025 16:26:24 +0100 Subject: [PATCH 17/29] chore: update docs Signed-off-by: Sacha Froment --- docs/k8s.md | 90 ++++++++++++++++++++++++++--------------------------- 1 file changed, 44 insertions(+), 46 deletions(-) diff --git a/docs/k8s.md b/docs/k8s.md index d731b73..d29a2a8 100644 --- a/docs/k8s.md +++ b/docs/k8s.md @@ -10,7 +10,7 @@ This document explains: #### 1.1 Scope - Kubernetes is currently used for **highly-available Sequencer deployment**. -- The provided manifests run **3 Sequencer pods** behind a `ClusterIP` service. +- The provided manifests run **multiple Sequencer pods** (2 by default) behind Services. - At any moment, **only one pod is the leader** and runs the "leader tasks"; the other pods stay ready to take over if the leader fails. - Other components (Node, Prover, etc.) can be run as regular processes or containers; this repository does not currently ship production-ready Kubernetes manifests for them. @@ -18,17 +18,18 @@ This document explains: All manifests live under `k8s/`: -- `k8s/deploy.sequencer.yaml`: `Deployment` with 3 replicas of `mojave-sequencer`, configured for Kubernetes leader election and persistent storage. -- `k8s/service.sequencer.yaml`: `ClusterIP` `Service` exposing: +- `k8s/namespace.yaml`: Namespace definition (`1sixtech`) for the Sequencer resources. +- `k8s/deploy.sequencer.yaml`: `StatefulSet` (2 replicas by default) of `mojave-sequencer`, with leader election and a `volumeClaimTemplate` (20Gi) per pod mounted at `/data/mojave`. +- `k8s/deploy.node.yaml`: `Deployment` (1 replica by default) of `mojave-node` with an init container that prepares `/data/mojave/node` and writes a random JWT secret for `authrpc`. +- `k8s/service.sequencer.yaml`: + - Headless `Service` (`mojave-sequencer-headless`) for the StatefulSet `serviceName`. + - `ClusterIP` `Service` exposing: - HTTP JSON-RPC on port `18545` (Currently not using). - P2P networking on port `30305` (TCP and UDP). - `k8s/rbac.sequencer.yaml`: - `ServiceAccount` (`sequencer-sa`). - `Role` and `RoleBinding` that grant access to `coordination.k8s.io/v1` `Lease` objects. -- `k8s/pvc.yaml`: - - `PersistentVolume` (`sequencer-pv`) that uses a `hostPath` directory **inside the Minikube node** at `/data/mojave`. - - `PersistentVolumeClaim` (`sequencer-pvc`) bound to that PV. -- `k8s/setup.sh`: Helper script that deletes everything under `k8s/` and then re-applies the core resources (PVC, RBAC, Service, Deployment). +- `k8s/setup.sh`: Helper script that deletes everything under `k8s/` and then re-applies the core resources (Namespace, Secret, RBAC, Services, StatefulSet). #### 1.3 Sequencer behavior on Kubernetes @@ -48,11 +49,12 @@ The Sequencer binary detects a Kubernetes environment by checking the `KUBERNETE - If the pod **loses** the lease, it steps down and stops the leader tasks. - On shutdown, the Sequencer writes a `node_config.json` under its data directory so that state can be reused on restart. -The data directory is backed by the PVC: +The data directory is backed by a per-pod `PersistentVolumeClaim` generated from the StatefulSet's `volumeClaimTemplate`: -- The PV on the Minikube node uses the host path `/data/mojave`. -- Inside the container, this PV is mounted at `/data/mojave`. -- The Sequencer is started with `--datadir /data/mojave/sequencer`, so the effective data directory is `/data/mojave/sequencer` on the Minikube node. +- Each pod gets its own 20Gi claim (using the cluster's default `StorageClass` unless you override `storageClassName`). +- The claim is mounted at `/data/mojave` in the container. +- The Sequencer is started with `--datadir /data/mojave/sequencer`, so each pod writes to its own `/data/mojave/sequencer`. +- An init container writes `/data/mojave/statefulset-ordinal` with the pod's ordinal (0, 1, …) and `/data/mojave/role` with `primary` for ordinal 0 and `secondary` otherwise. Adjust this script in `k8s/deploy.sequencer.yaml` if you want different per-pod config files. --- @@ -83,20 +85,7 @@ minikube addons enable metrics-server You only need to do this once per Minikube profile. -#### 2.3 Prepare persistent storage inside Minikube - -The `k8s/pvc.yaml` manifest expects a hostPath directory at `/data/mojave` on the Minikube node. Create it and make it writable: - -```bash -minikube ssh "sudo mkdir -p /data/mojave && sudo chmod 777 /data/mojave" -``` - -Notes: - -- This directory lives **inside the Minikube VM/container**, not on your host filesystem. -- The PV (`sequencer-pv`) references `/data/mojave`, and the Sequencer pod mounts that PV at `/data/mojave` inside the container. - -#### 2.4 Choose a Sequencer Docker image +#### 2.3 Choose a Sequencer Docker image For this HA test you need a Docker image that contains the `mojave-sequencer` binary. There are two main options: @@ -124,7 +113,7 @@ For this HA test you need a Docker image that contains the `mojave-sequencer` bi docker push /mojave-sequencer:latest ``` -3. Update the Deployment to use your image instead of the default: +3. Update the StatefulSet to use your image instead of the default: - Option 1: Edit `k8s/deploy.sequencer.yaml`: @@ -136,22 +125,24 @@ For this HA test you need a Docker image that contains the `mojave-sequencer` bi imagePullPolicy: IfNotPresent ``` - - Option 2: Patch the live Deployment after applying the manifests: + - Option 2: Patch the live StatefulSet after applying the manifests: ```bash - kubectl set image deployment/mojave-sequencer-deployment \ + kubectl set image statefulset/mojave-sequencer-deployment -n 1sixtech \ mojave-sequencer=/mojave-sequencer:latest ``` -#### 2.6 Apply the core Kubernetes resources +#### 2.4 Apply the core Kubernetes resources You can apply the manifests one by one: ```bash -kubectl apply -f k8s/pvc.yaml +kubectl apply -f k8s/namespace.yaml +kubectl apply -f k8s/secret.sequencer.yaml kubectl apply -f k8s/rbac.sequencer.yaml kubectl apply -f k8s/service.sequencer.yaml kubectl apply -f k8s/deploy.sequencer.yaml +kubectl apply -f k8s/deploy.node.yaml ``` Or use the helper script (note: this **first deletes** all manifests under `k8s/`): @@ -160,76 +151,83 @@ Or use the helper script (note: this **first deletes** all manifests under `k8s/ bash k8s/setup.sh ``` -#### 2.7 Verify the deployment +If you hit `pod has unbound immediate PersistentVolumeClaims` errors, make sure your cluster has a default `StorageClass` or set `storageClassName` in `k8s/deploy.sequencer.yaml` to one that exists in your cluster (e.g., `local-path` on k3d/k3s, `standard` on many managed clusters). + +#### 2.5 Verify the deployment Check that all resources are created: ```bash -kubectl get pods -kubectl get svc mojave-sequencer-service -kubectl get pvc sequencer-pvc -kubectl get lease sequencer-leader -o yaml +kubectl get sts mojave-sequencer-deployment -n 1sixtech +kubectl get pods -n 1sixtech +kubectl get svc mojave-sequencer-service -n 1sixtech +kubectl get pvc -n 1sixtech +kubectl get lease sequencer-leader -n 1sixtech -o yaml ``` You should see: -- 3 pods with `app=mojave-sequencer`. +- 2 pods (or however many replicas you configured) with `app=mojave-sequencer`. +- A StatefulSet `mojave-sequencer-deployment` reporting ready replicas. +- Per-pod PVCs named like `sequencer-datadir-mojave-sequencer-deployment-0`. - A single `Lease` named `sequencer-leader` with one of the pods listed as the current holder. -#### 2.8 Check that the current leader is producing blocks (via logs) +#### 2.6 Check that the current leader is producing blocks (via logs) Before testing failover, confirm that the current leader pod is actively producing blocks. 1. **Identify the current leader pod** (via the `Lease` holder identity): ```bash - LEADER_POD=$(kubectl get lease sequencer-leader -o jsonpath='{.spec.holderIdentity}') + LEADER_POD=$(kubectl get lease sequencer-leader -n 1sixtech -o jsonpath='{.spec.holderIdentity}') echo "Current leader pod: $LEADER_POD" ``` 2. **Print the last 100 log lines from the leader pod**: ```bash - kubectl logs "$LEADER_POD" -c mojave-sequencer | tail -n 100 + kubectl logs "$LEADER_POD" -c mojave-sequencer -n 1sixtech | tail -n 100 ``` Review the logs and confirm that the leader is producing blocks (or whatever periodic work you expect the Sequencer leader to perform). -#### 2.9 Leader failover test +#### 2.7 Leader failover test To see Kubernetes-based HA in action: 1. **Delete the current leader pod** (identified in the previous step): ```bash - kubectl delete pod "$LEADER_POD" + kubectl delete pod "$LEADER_POD" -n 1sixtech ``` 2. **Wait for a new leader to be elected and identify it**: ```bash # After some time (up to LEASE_TTL_SECONDS), check which pod is the new leader - NEW_LEADER_POD=$(kubectl get lease sequencer-leader -o jsonpath='{.spec.holderIdentity}') + NEW_LEADER_POD=$(kubectl get lease sequencer-leader -n 1sixtech -o jsonpath='{.spec.holderIdentity}') echo "New leader pod: $NEW_LEADER_POD" # (Optional) list all sequencer pods - kubectl get pods -l app=mojave-sequencer + kubectl get pods -l app=mojave-sequencer -n 1sixtech ``` 3. **Print the last 100 log lines from the new leader pod**: ```bash - kubectl logs "$NEW_LEADER_POD" -c mojave-sequencer | tail -n 100 + kubectl logs "$NEW_LEADER_POD" -c mojave-sequencer -n 1sixtech | tail -n 100 ``` Within approximately the lease TTL window (`LEASE_TTL_SECONDS`, default 15s), another pod should acquire the lease and start the leader tasks. By comparing the logs before and after the failover, you can verify that block production (or other leader activity) continues on the new leader. -#### 2.10 Cleanup +#### 2.8 Cleanup To remove all Mojave-related Kubernetes resources created from `k8s/`: ```bash kubectl delete -f k8s/ +# Delete the per-pod PVCs created by the StatefulSet +kubectl delete pvc -l app=mojave-sequencer -n 1sixtech ``` --- @@ -243,4 +241,4 @@ The manifests in this repository are designed for **local development and testin - Add liveness and readiness probes for the Sequencer container to improve reliability and rollout behavior. - Consider a `Headless Service` or separate Services if you need stable pod identities or direct pod-to-pod communication. - Configure `imagePullSecrets` if you use private container registries. -- Use a production-grade storage class instead of `hostPath` volumes. +- Use a production-grade storage class appropriate for your cluster/storage backend. From b4c895febab35663f42992a21de1f1545b870c69 Mon Sep 17 00:00:00 2001 From: Sacha Froment Date: Tue, 25 Nov 2025 16:29:42 +0100 Subject: [PATCH 18/29] chore: use anyhow error Signed-off-by: Sacha Froment --- cmd/node/src/cli.rs | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/cmd/node/src/cli.rs b/cmd/node/src/cli.rs index b1a3d95..31fce38 100644 --- a/cmd/node/src/cli.rs +++ b/cmd/node/src/cli.rs @@ -1,4 +1,4 @@ -use std::{fmt::format, str::FromStr}; +use std::str::FromStr; use clap::{ArgAction, Parser, Subcommand}; use mojave_node_lib::types::{Node, SyncMode}; @@ -6,16 +6,14 @@ use mojave_utils::network::Network; use std::net::ToSocketAddrs; use tracing::Level; -fn resolve_dns_host_port(addr: &str) -> Result { - let mut iter = addr.to_socket_addrs().map_err(|e| { - mojave_node_lib::error::Error::Custom(format!("to_socket failed {}", e).to_string()) - })?; +fn resolve_dns_host_port(addr: &str) -> Result { + let mut iter = addr + .to_socket_addrs() + .map_err(|e| anyhow::anyhow!("Failed to resolve DNS for `{addr}`: {e}"))?; - let socket_addr = iter.next().ok_or_else(|| { - mojave_node_lib::error::Error::Custom(format!( - "DNS resolution for `{addr}` returned no addresses" - )) - })?; + let socket_addr = iter + .next() + .ok_or_else(|| anyhow::anyhow!("DNS resolution for `{addr}` returned no addresses"))?; // This will be something like "10.42.0.12:3030" Ok(socket_addr.to_string()) @@ -27,7 +25,7 @@ pub struct DNSNode { } impl FromStr for DNSNode { - type Err = mojave_node_lib::error::Error; + type Err = anyhow::Error; fn from_str(enode: &str) -> Result { let at_pos = enode.find('@').ok_or_else(|| { From 99bd7adb0d000024ea89d17483eabc8e7fa87f89 Mon Sep 17 00:00:00 2001 From: Sacha Froment Date: Wed, 3 Dec 2025 09:53:48 +0100 Subject: [PATCH 19/29] refactor: change the way the main is started" (#275) * chore: add health check Signed-off-by: Sacha Froment * refactor: change the way the main is started Signed-off-by: Sacha Froment * chore: nitpick * mod: k8s yaml * chore: remove command in justfile * chore: fix just sequencer & node , fix kill process in start.sh * chore: rm * refactor: coordination k8s (#276) * refactor: coordination k8s Signed-off-by: Sacha Froment * chore: save Signed-off-by: Sacha Froment * chore: keep health alive Signed-off-by: Sacha Froment * fix: start Signed-off-by: Sacha Froment * Apply suggestion from @Copilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * chore: remove useless Signed-off-by: Sacha Froment * chore: review comment Signed-off-by: Sacha Froment * chore: add back save on shutdown Signed-off-by: Sacha Froment --------- Signed-off-by: Sacha Froment Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update justfile Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update cmd/sequencer/src/main.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update cmd/sequencer/src/cli.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update cmd/node/src/cli.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update cmd/sequencer/src/cli.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update justfile Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * chore: fix 1st call Signed-off-by: Sacha Froment * Refactor: Use idiomatic assert! in CLI tests (#277) * Initial plan * Replace if-panic patterns with assert! in CLI tests Co-authored-by: sfroment <7238385+sfroment@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: sfroment <7238385+sfroment@users.noreply.github.com> * Fix missing assert in CLI test for command parsing (#278) * Initial plan * Fix matches! macro usage in sequencer CLI tests Added assert! around matches! calls in parse_stop_and_get_pub_key test to properly verify command types instead of just evaluating the boolean. Co-authored-by: sfroment <7238385+sfroment@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: sfroment <7238385+sfroment@users.noreply.github.com> --------- Signed-off-by: Sacha Froment Co-authored-by: Giwook-Han Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com> Co-authored-by: sfroment <7238385+sfroment@users.noreply.github.com> --- Cargo.lock | 398 +++++++++--------- Cargo.toml | 3 +- cmd/node/src/cli.rs | 172 +++++--- cmd/node/src/main.rs | 152 ++++--- cmd/prover/src/main.rs | 8 +- cmd/sequencer/Cargo.toml | 8 +- cmd/sequencer/src/cli.rs | 232 +++++----- cmd/sequencer/src/k8s_leader.rs | 205 --------- cmd/sequencer/src/main.rs | 191 ++++----- crates/batch-producer/Cargo.toml | 1 - crates/coordination/Cargo.toml | 25 ++ crates/coordination/src/coordination_mode.rs | 6 + crates/coordination/src/k8s.rs | 183 ++++++++ crates/coordination/src/lib.rs | 4 + crates/coordination/src/sequencer.rs | 208 +++++++++ crates/coordination/src/utils.rs | 39 ++ crates/node/src/node.rs | 15 + crates/node/src/types.rs | 4 + crates/task/src/error.rs | 2 +- crates/task/src/traits.rs | 2 +- crates/utils/src/daemon.rs | 13 +- crates/utils/src/health.rs | 96 +++++ crates/utils/src/lib.rs | 1 + crates/utils/src/logging.rs | 6 +- justfile | 18 +- k8s/deploy.node.yaml | 18 +- k8s/setup.sh | 2 +- ...sequencer.yaml => stateful.sequencer.yaml} | 11 +- scripts/start.sh | 5 +- 29 files changed, 1255 insertions(+), 773 deletions(-) delete mode 100644 cmd/sequencer/src/k8s_leader.rs create mode 100644 crates/coordination/Cargo.toml create mode 100644 crates/coordination/src/coordination_mode.rs create mode 100644 crates/coordination/src/k8s.rs create mode 100644 crates/coordination/src/lib.rs create mode 100644 crates/coordination/src/sequencer.rs create mode 100644 crates/coordination/src/utils.rs create mode 100644 crates/utils/src/health.rs rename k8s/{deploy.sequencer.yaml => stateful.sequencer.yaml} (91%) diff --git a/Cargo.lock b/Cargo.lock index 8045985..3aa0d45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -139,22 +139,22 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "anstyle-wincon" -version = "3.0.10" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -222,7 +222,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62945a2f7e6de02a31fe400aa489f0e0f5b2502e69f95f853adb82a96c7a6b60" dependencies = [ "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -235,7 +235,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -274,7 +274,7 @@ checksum = "213888f660fddcca0d257e88e54ac05bca01885f258ccdf695bafd77031bb69d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -316,7 +316,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -357,7 +357,7 @@ checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -368,18 +368,18 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "axum" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18ed336352031311f4e0b4dd2ff392d4fbb370777c9d18d7fc9d7359f73871" +checksum = "5b098575ebe77cb6d14fc7f32749631a6e44edbef6b796f89b020e99ba20d425" dependencies = [ "axum-core", "bytes", "form_urlencoded", "futures-util", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-util", "itoa", "matchit", @@ -407,7 +407,7 @@ checksum = "59446ce19cd142f8833f856eb31f3eb097812d1479ab224f54d72428ca21ea22" dependencies = [ "bytes", "futures-core", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "http-body-util", "mime", @@ -429,7 +429,7 @@ dependencies = [ "bytes", "futures-util", "headers", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "http-body-util", "mime", @@ -532,7 +532,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -756,7 +756,7 @@ checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -773,9 +773,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" dependencies = [ "serde", ] @@ -864,9 +864,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.45" +version = "1.2.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35900b6c8d709fb1d854671ae27aeaa9eec2f8b01b364e1619a40da3e6fe2afe" +checksum = "c481bdbf0ed3b892f6f806287d72acd515b352a4ec27a208489b8c1bc839633a" dependencies = [ "find-msvc-tools", "jobserver", @@ -959,9 +959,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.51" +version = "4.5.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c26d721170e0295f191a69bd9a1f93efcdb0aff38684b61ab5750468972e5f5" +checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" dependencies = [ "clap_builder", "clap_derive", @@ -969,9 +969,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.51" +version = "4.5.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75835f0c7bf681bfd05abe44e965760fea999a5286c6eb2d59883634fd02011a" +checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" dependencies = [ "anstream", "anstyle", @@ -988,7 +988,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -1400,9 +1400,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", "typenum", @@ -1441,7 +1441,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -1484,7 +1484,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -1498,7 +1498,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -1509,7 +1509,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -1520,7 +1520,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -1593,7 +1593,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -1603,7 +1603,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -1633,7 +1633,7 @@ dependencies = [ "convert_case 0.6.0", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", "unicode-xid", ] @@ -1646,7 +1646,7 @@ dependencies = [ "convert_case 0.7.1", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -1733,7 +1733,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -1806,7 +1806,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -1894,7 +1894,7 @@ checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -2101,7 +2101,7 @@ dependencies = [ "reqwest 0.11.27", "serde", "serde_json", - "syn 2.0.110", + "syn 2.0.111", "toml", "walkdir", ] @@ -2119,7 +2119,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -2145,7 +2145,7 @@ dependencies = [ "serde", "serde_json", "strum 0.26.3", - "syn 2.0.110", + "syn 2.0.111", "tempfile", "thiserror 1.0.69", "tiny-keccak", @@ -2827,9 +2827,9 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "find-msvc-tools" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" [[package]] name = "fixed-hash" @@ -2983,7 +2983,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -3043,9 +3043,9 @@ checksum = "1d758ba1b47b00caf47f24925c0074ecb20d6dfcffe7f6d53395c0465674841a" [[package]] name = "generic-array" -version = "0.14.9" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -3158,7 +3158,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.12.0", + "indexmap 2.12.1", "slab", "tokio", "tokio-util", @@ -3176,8 +3176,8 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.3.1", - "indexmap 2.12.0", + "http 1.4.0", + "indexmap 2.12.1", "slab", "tokio", "tokio-util", @@ -3220,9 +3220,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" [[package]] name = "hashers" @@ -3242,7 +3242,7 @@ dependencies = [ "base64 0.22.1", "bytes", "headers-core", - "http 1.3.1", + "http 1.4.0", "httpdate", "mime", "sha1", @@ -3254,7 +3254,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http 1.3.1", + "http 1.4.0", ] [[package]] @@ -3277,9 +3277,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hex-conservative" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5313b072ce3c597065a808dbf612c4c8e8590bdbf8b579508bf7a762c5eae6cd" +checksum = "fda06d18ac606267c40c04e41b9947729bf8b9efe74bd4e82b61a5f26a510b9f" dependencies = [ "arrayvec", ] @@ -3321,12 +3321,11 @@ dependencies = [ [[package]] name = "http" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" dependencies = [ "bytes", - "fnv", "itoa", ] @@ -3348,7 +3347,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.3.1", + "http 1.4.0", ] [[package]] @@ -3359,7 +3358,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "pin-project-lite", ] @@ -3402,16 +3401,16 @@ dependencies = [ [[package]] name = "hyper" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", "h2 0.4.12", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "httparse", "httpdate", @@ -3443,8 +3442,8 @@ version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "http 1.3.1", - "hyper 1.7.0", + "http 1.4.0", + "hyper 1.8.1", "hyper-util", "log", "rustls 0.23.35", @@ -3461,7 +3460,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.7.0", + "hyper 1.8.1", "hyper-util", "pin-project-lite", "tokio", @@ -3476,7 +3475,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-util", "native-tls", "tokio", @@ -3486,18 +3485,18 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" +checksum = "52e9a2a24dc5c6821e71a7030e1e14b7b632acac55c40e9d2e082c621261bb56" dependencies = [ "base64 0.22.1", "bytes", "futures-channel", "futures-core", "futures-util", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", - "hyper 1.7.0", + "hyper 1.8.1", "ipnet", "libc", "percent-encoding", @@ -3704,7 +3703,7 @@ checksum = "a0eb5a3343abf848c0984fe4604b2b105da9539376e24fc0a3b0007411ae4fd9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -3732,12 +3731,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" +checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" dependencies = [ "equivalent", - "hashbrown 0.16.0", + "hashbrown 0.16.1", "serde", "serde_core", ] @@ -3770,7 +3769,7 @@ dependencies = [ "indoc", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -3858,9 +3857,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.82" +version = "0.3.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b011eec8cc36da2aab2d5cff675ec18454fad408585853910a202391cf9f8e65" +checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" dependencies = [ "once_cell", "wasm-bindgen", @@ -3978,10 +3977,10 @@ dependencies = [ "either", "futures", "home", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-rustls 0.27.7", "hyper-timeout", "hyper-util", @@ -4011,7 +4010,7 @@ dependencies = [ "chrono", "derive_more 2.0.1", "form_urlencoded", - "http 1.3.1", + "http 1.4.0", "k8s-openapi", "serde", "serde-value", @@ -4177,7 +4176,7 @@ dependencies = [ "bitflags 2.10.0", "derive_more 1.0.0", "impls", - "indexmap 2.12.0", + "indexmap 2.12.1", "libc", "mdbx-sys", "parking_lot", @@ -4210,9 +4209,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.22" +version = "1.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +checksum = "15d118bbf3771060e7311cc7bb0545b01d08a8b4a7de949198dec1fa0ca1c0f7" dependencies = [ "cc", "libc", @@ -4439,7 +4438,6 @@ dependencies = [ "ethrex-vm", "mojave-node-lib", "mojave-task", - "mojave-utils", "thiserror 2.0.17", "tokio", "tracing", @@ -4522,6 +4520,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "mojave-coordination" +version = "0.1.0" +dependencies = [ + "k8s-openapi", + "kube", + "kube-leader-election", + "mojave-batch-producer", + "mojave-block-producer", + "mojave-node-lib", + "mojave-proof-coordinator", + "mojave-task", + "mojave-utils", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "mojave-msgio" version = "0.1.0" @@ -4648,7 +4664,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -4672,16 +4688,10 @@ dependencies = [ "anyhow", "clap", "hex", - "k8s-openapi", - "kube", - "kube-leader-election", - "mojave-batch-producer", - "mojave-batch-submitter", "mojave-block-producer", - "mojave-msgio", + "mojave-coordination", "mojave-node-lib", "mojave-proof-coordinator", - "mojave-task", "mojave-utils", "tokio", "tracing", @@ -4756,7 +4766,7 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -4909,7 +4919,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -5009,7 +5019,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -5209,7 +5219,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -5316,9 +5326,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.3" +version = "2.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989e7521a040efde50c3ab6bbadafbe15ab6dc042686926be59ac35d74607df4" +checksum = "cbcfd20a6d4eeba40179f05735784ad32bdaef05ce8e8af05f180d45bb3e7e22" dependencies = [ "memchr", "ucd-trie", @@ -5326,9 +5336,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.8.3" +version = "2.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187da9a3030dbafabbbfb20cb323b976dc7b7ce91fcd84f2f74d6e31d378e2de" +checksum = "51f72981ade67b1ca6adc26ec221be9f463f2b5839c7508998daa17c23d94d7f" dependencies = [ "pest", "pest_generator", @@ -5336,22 +5346,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.3" +version = "2.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b401d98f5757ebe97a26085998d6c0eecec4995cad6ab7fc30ffdf4b052843" +checksum = "dee9efd8cdb50d719a80088b76f81aec7c41ed6d522ee750178f83883d271625" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] name = "pest_meta" -version = "2.8.3" +version = "2.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f27a2cfee9f9039c4d86faa5af122a0ac3851441a34865b8a043b46be0065a" +checksum = "bf1d70880e76bdc13ba52eafa6239ce793d85c8e43896507e43dd8984ff05b82" dependencies = [ "pest", "sha2", @@ -5364,7 +5374,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.12.0", + "indexmap 2.12.1", ] [[package]] @@ -5407,7 +5417,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -5436,7 +5446,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -5504,7 +5514,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -5628,7 +5638,7 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -5807,7 +5817,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -5900,10 +5910,10 @@ dependencies = [ "encoding_rs", "futures-core", "h2 0.4.12", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.1", "hyper-rustls 0.27.7", "hyper-tls", "hyper-util", @@ -5986,7 +5996,7 @@ dependencies = [ "bytecheck", "bytes", "hashbrown 0.15.5", - "indexmap 2.12.0", + "indexmap 2.12.1", "munge", "ptr_meta", "rancor", @@ -6004,7 +6014,7 @@ checksum = "bd83f5f173ff41e00337d97f6572e416d022ef8a19f371817259ae960324c482" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -6142,9 +6152,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94182ad936a0c91c324cd46c6511b9510ed16af436d7b5bab34beab0afd55f7a" +checksum = "708c0f9d5f54ba0272468c1d306a52c495b31fa155e91bc25371e6df7996908c" dependencies = [ "zeroize", ] @@ -6230,7 +6240,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -6302,7 +6312,7 @@ checksum = "22f968c5ea23d555e670b449c1c5e7b2fc399fdaec1d304a17cd48e288abc107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -6465,7 +6475,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -6500,7 +6510,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -6526,15 +6536,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.15.1" +version = "3.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa66c845eee442168b2c8134fec70ac50dc20e760769c8ba0ad1319ca1959b04" +checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.12.0", + "indexmap 2.12.1", "schemars 0.9.0", "schemars 1.1.0", "serde_core", @@ -6545,14 +6555,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.15.1" +version = "3.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91a903660542fced4e99881aa481bdbaec1634568ee02e0b8bd57c64cb38955" +checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c" dependencies = [ "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -6561,7 +6571,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.12.1", "itoa", "ryu", "serde", @@ -6644,9 +6654,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.6" +version = "1.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" +checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" dependencies = [ "libc", ] @@ -6745,9 +6755,9 @@ dependencies = [ [[package]] name = "sp1-lib" -version = "5.2.2" +version = "5.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fce8ad0f153443d09d398eccb650a0b2dcbf829470e394e4bf60ec4379c7af93" +checksum = "eb1a9935d58cb1dcd757a1b10d727090f5b718f1f03b512d48f0c1952e6ead00" dependencies = [ "bincode", "serde", @@ -6756,9 +6766,9 @@ dependencies = [ [[package]] name = "sp1-primitives" -version = "5.2.2" +version = "5.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0244dee3a7a0f88cf71c3edf518f4fc97794ae870a107cbe7c810ac3fbf879cb" +checksum = "a7d2a6187e394c30097ea7a975a4832f172918690dc89a979f0fad67422d3a8b" dependencies = [ "bincode", "blake3", @@ -6896,7 +6906,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -6908,7 +6918,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -6950,9 +6960,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.110" +version = "2.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea" +checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" dependencies = [ "proc-macro2", "quote", @@ -6982,7 +6992,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -7106,7 +7116,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -7117,7 +7127,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -7230,7 +7240,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -7355,7 +7365,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.12.1", "serde", "serde_spanned", "toml_datetime 0.6.11", @@ -7369,7 +7379,7 @@ version = "0.23.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d" dependencies = [ - "indexmap 2.12.0", + "indexmap 2.12.1", "toml_datetime 0.7.3", "toml_parser", "winnow", @@ -7409,15 +7419,15 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +checksum = "9cf146f99d442e8e68e585f5d798ccd3cad9a7835b917e09728880a862706456" dependencies = [ "base64 0.22.1", "bitflags 2.10.0", "bytes", "futures-util", - "http 1.3.1", + "http 1.4.0", "http-body 1.0.1", "iri-string", "mime", @@ -7442,9 +7452,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.41" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" dependencies = [ "log", "pin-project-lite", @@ -7454,20 +7464,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] name = "tracing-core" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" dependencies = [ "once_cell", "valuable", @@ -7506,9 +7516,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.20" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" +checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" dependencies = [ "matchers", "nu-ansi-term", @@ -7530,7 +7540,7 @@ checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -7607,7 +7617,7 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http 1.3.1", + "http 1.4.0", "httparse", "log", "native-tls", @@ -7861,9 +7871,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.105" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60" +checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" dependencies = [ "cfg-if 1.0.4", "once_cell", @@ -7874,9 +7884,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.55" +version = "0.4.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "551f88106c6d5e7ccc7cd9a16f312dd3b5d36ea8b4954304657d5dfba115d4a0" +checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" dependencies = [ "cfg-if 1.0.4", "js-sys", @@ -7887,9 +7897,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.105" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04264334509e04a7bf8690f2384ef5265f05143a4bff3889ab7a3269adab59c2" +checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -7897,31 +7907,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.105" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "420bc339d9f322e562942d52e115d57e950d12d88983a14c79b86859ee6c7ebc" +checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.105" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76f218a38c84bcb33c25ec7059b07847d465ce0e0a76b995e134a45adcb6af76" +checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.82" +version = "0.3.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a1f95c0d03a47f4ae1f7a64643a6bb97465d9b740f0fa8f90ea33915c99a9a1" +checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" dependencies = [ "js-sys", "wasm-bindgen", @@ -8041,7 +8051,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -8052,7 +8062,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -8079,13 +8089,13 @@ dependencies = [ [[package]] name = "windows-registry" -version = "0.5.3" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a9ed28765efc97bbc954883f4e6796c33a06546ebafacbabee9696967499e" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" dependencies = [ - "windows-link 0.1.3", - "windows-result 0.3.4", - "windows-strings 0.4.2", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", ] [[package]] @@ -8366,9 +8376,9 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" -version = "0.7.13" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" +checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" dependencies = [ "memchr", ] @@ -8448,28 +8458,28 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.27" +version = "0.8.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +checksum = "4ea879c944afe8a2b25fef16bb4ba234f47c694565e97383b36f3a878219065c" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.27" +version = "0.8.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +checksum = "cf955aa904d6040f70dc8e9384444cb1030aed272ba3cb09bbc4ab9e7c1f34f5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -8489,7 +8499,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", "synstructure", ] @@ -8510,7 +8520,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] @@ -8570,7 +8580,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.110", + "syn 2.0.111", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 6041de8..0b714da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ members = [ "crates/block-producer", "crates/btc-watcher", "crates/client", + "crates/coordination", "crates/msgio", "crates/node", "crates/proof-coordinator", @@ -44,9 +45,9 @@ resolver = "2" [workspace.dependencies] mojave-batch-producer = { path = "crates/batch-producer" } -mojave-batch-submitter = { path = "crates/batch-submitter" } mojave-block-producer = { path = "crates/block-producer" } mojave-client = { path = "crates/client" } +mojave-coordination = { path = "crates/coordination" } mojave-msgio = { path = "crates/msgio" } mojave-node-lib = { path = "crates/node" } mojave-proof-coordinator = { path = "crates/proof-coordinator" } diff --git a/cmd/node/src/cli.rs b/cmd/node/src/cli.rs index 31fce38..822b837 100644 --- a/cmd/node/src/cli.rs +++ b/cmd/node/src/cli.rs @@ -1,11 +1,16 @@ -use std::str::FromStr; +use std::{path::PathBuf, str::FromStr}; use clap::{ArgAction, Parser, Subcommand}; -use mojave_node_lib::types::{Node, SyncMode}; -use mojave_utils::network::Network; +use mojave_node_lib::{ + initializers::get_signer, + types::{Node, SyncMode}, +}; +use mojave_utils::{daemon::stop_daemonized, network::Network, p2p::public_key_from_signing_key}; use std::net::ToSocketAddrs; use tracing::Level; +use crate::PID_FILE_NAME; + fn resolve_dns_host_port(addr: &str) -> Result { let mut iter = addr .to_socket_addrs() @@ -15,7 +20,6 @@ fn resolve_dns_host_port(addr: &str) -> Result { .next() .ok_or_else(|| anyhow::anyhow!("DNS resolution for `{addr}` returned no addresses"))?; - // This will be something like "10.42.0.12:3030" Ok(socket_addr.to_string()) } @@ -54,6 +58,49 @@ impl From for Node { #[derive(Parser)] pub struct Options { + #[arg( + long = "log.level", + value_name = "LOG_LEVEL", + help = "The verbosity level used for logs.", + long_help = "Possible values: info, debug, trace, warn, error", + help_heading = "Node options", + global = true + )] + pub log_level: Option, + + #[arg( + long = "datadir", + value_name = "DATABASE_DIRECTORY", + help = "If the datadir is the word `memory`, ethrex will use the InMemory Engine", + default_value = ".mojave/node", + help = "Receives the name of the directory where the Database is located.", + long_help = "If the datadir is the word `memory`, ethrex will use the `InMemory Engine`.", + help_heading = "Node options", + env = "ETHREX_DATADIR", + global = true + )] + pub datadir: String, + + #[arg( + long = "health.port", + value_name = "HEALTH_PORT", + default_value = "9595", + help = "Port for the health check HTTP server.", + help_heading = "Node options", + env = "HEALTH_PORT" + )] + pub health_port: String, + + #[arg( + long = "health.addr", + value_name = "HEALTH_ADDR", + default_value = "0.0.0.0", + help = "Address for the health check HTTP server.", + help_heading = "Node options", + env = "HEALTH_ADDR" + )] + pub health_addr: String, + #[arg( long = "network", default_value_t = Network::default(), @@ -241,13 +288,15 @@ impl From<&Options> for mojave_node_lib::types::NodeOptions { .cloned() .map(|dn| dn.into()) .collect(), - datadir: Default::default(), + datadir: options.datadir.clone(), syncmode: options.syncmode.unwrap_or(SyncMode::Full), sponsorable_addresses_file_path: options.sponsorable_addresses_file_path.clone(), metrics_addr: options.metrics_addr.clone(), metrics_port: options.metrics_port.clone(), metrics_enabled: options.metrics_enabled, force: options.force, + health_addr: options.health_addr.clone(), + health_port: options.health_port.clone(), } } } @@ -258,33 +307,13 @@ impl From<&Options> for mojave_node_lib::types::NodeOptions { name = "mojave-node", author, version, - about = "mojave-node is the node implementation for the Mojave network.", - arg_required_else_help = true + about = "mojave-node is the node implementation for the Mojave network." )] pub struct Cli { - #[arg( - long = "log.level", - value_name = "LOG_LEVEL", - help = "The verbosity level used for logs.", - long_help = "Possible values: info, debug, trace, warn, error", - help_heading = "Node options", - global = true - )] - pub log_level: Option, - #[arg( - long = "datadir", - value_name = "DATABASE_DIRECTORY", - help = "If the datadir is the word `memory`, ethrex will use the InMemory Engine", - default_value = ".mojave/node", - help = "Receives the name of the directory where the Database is located.", - long_help = "If the datadir is the word `memory`, ethrex will use the `InMemory Engine`.", - help_heading = "Node options", - env = "ETHREX_DATADIR", - global = true - )] - pub datadir: String, + #[command(flatten)] + pub options: Options, #[command(subcommand)] - pub command: Command, + pub command: Option, } impl Cli { @@ -296,17 +325,26 @@ impl Cli { #[allow(clippy::large_enum_variant)] #[derive(Subcommand)] pub enum Command { - #[command(name = "init", about = "Run the node")] - Start { - #[command(flatten)] - options: Options, - }, #[command(name = "stop", about = "Stop the node")] Stop, #[command(name = "get-pub-key", about = "Display the public key of the node")] GetPubKey, } +impl Command { + pub async fn run(self, datadir: String) -> anyhow::Result<()> { + match self { + Command::Stop => stop_daemonized(PathBuf::from(datadir).join(PID_FILE_NAME)), + Command::GetPubKey => { + let signer = get_signer(&datadir).await.map_err(anyhow::Error::from)?; + let public_key = public_key_from_signing_key(&signer); + let public_key = hex::encode(public_key); + println!("{public_key}"); + Ok(()) + } + } + } +} #[cfg(test)] mod tests { use super::*; @@ -331,14 +369,12 @@ mod tests { #[test] fn parse_start_with_defaults() { - let cli = Cli::try_parse_from(["mojave-node", "init"]).unwrap(); + let Cli { command, options } = Cli::try_parse_from(["mojave-node"]).unwrap(); - assert_eq!(cli.datadir, ".mojave/node"); - assert!(cli.log_level.is_none()); + assert_eq!(options.datadir, ".mojave/node"); + assert!(options.log_level.is_none()); - let Command::Start { ref options } = cli.command else { - panic!("expected Start") - }; + assert!(command.is_none(), "expected None"); assert!(matches!(options.network, Network::DefaultNet)); assert!(options.bootnodes.is_empty()); @@ -365,9 +401,9 @@ mod tests { assert_eq!(options.metrics_port, "9090"); assert!(!options.metrics_enabled); assert!(!options.force); - assert_eq!(cli.datadir, ".mojave/node"); + assert_eq!(options.datadir, ".mojave/node"); - let node_opts: NodeOptions = options.into(); + let node_opts: NodeOptions = (&options).into(); assert_eq!(node_opts.http_addr, Some(options.http_addr.clone())); assert_eq!(node_opts.http_port, Some(options.http_port.clone())); assert_eq!(node_opts.authrpc_addr, Some(options.authrpc_addr.clone())); @@ -392,14 +428,13 @@ mod tests { assert_eq!(node_opts.metrics_port, options.metrics_port); assert_eq!(node_opts.metrics_enabled, options.metrics_enabled); assert_eq!(node_opts.force, options.force); - assert_eq!(node_opts.datadir, "".to_string()); // datadir is not set from Options. Override to default + assert_eq!(node_opts.datadir, ".mojave/node".to_string()); } #[test] fn parse_start_with_overrides() { let cli = Cli::try_parse_from([ "mojave-node", - "init", "--log.level", "debug", "--datadir", @@ -434,37 +469,34 @@ mod tests { ]) .unwrap(); - match cli.command { - Command::Start { options } => { - assert_eq!(cli.log_level, Some(Level::DEBUG)); - assert_eq!(cli.datadir, "/tmp/mojave-node"); - assert_eq!(options.http_addr, "127.0.0.1"); - assert_eq!(options.http_port, "18545"); - assert_eq!(options.authrpc_addr, "127.0.0.1"); - assert_eq!(options.authrpc_port, "18551"); - assert_eq!(options.authrpc_jwtsecret, "custom.jwt"); - assert_eq!(options.metrics_addr, "127.0.0.1"); - assert_eq!(options.metrics_port, "19090"); - assert!(options.metrics_enabled); - assert_eq!(options.p2p_addr, "0.0.0.0"); - assert_eq!(options.p2p_port, "30304"); - assert_eq!(options.discovery_addr, "0.0.0.0"); - assert_eq!(options.discovery_port, "30305"); - assert!(matches!(options.syncmode, Some(SyncMode::Snap))); - assert!(options.force); - assert!(options.no_daemon); - } - _ => panic!("expected Start"), - } + let options = cli.options; + + assert_eq!(options.log_level, Some(Level::DEBUG)); + assert_eq!(options.datadir, "/tmp/mojave-node"); + assert_eq!(options.http_addr, "127.0.0.1"); + assert_eq!(options.http_port, "18545"); + assert_eq!(options.authrpc_addr, "127.0.0.1"); + assert_eq!(options.authrpc_port, "18551"); + assert_eq!(options.authrpc_jwtsecret, "custom.jwt"); + assert_eq!(options.metrics_addr, "127.0.0.1"); + assert_eq!(options.metrics_port, "19090"); + assert!(options.metrics_enabled); + assert_eq!(options.p2p_addr, "0.0.0.0"); + assert_eq!(options.p2p_port, "30304"); + assert_eq!(options.discovery_addr, "0.0.0.0"); + assert_eq!(options.discovery_port, "30305"); + assert!(matches!(options.syncmode, Some(SyncMode::Snap))); + assert!(options.force); + assert!(options.no_daemon); } #[test] fn parse_stop_and_get_pub_key() { let cli = Cli::try_parse_from(["mojave-node", "stop"]).unwrap(); - assert!(matches!(cli.command, Command::Stop)); + assert!(matches!(cli.command, Some(Command::Stop))); let cli = Cli::try_parse_from(["mojave-node", "get-pub-key"]).unwrap(); - assert!(matches!(cli.command, Command::GetPubKey)); + assert!(matches!(cli.command, Some(Command::GetPubKey))); } #[test] @@ -475,8 +507,8 @@ mod tests { #[test] fn parse_log_level() { - let cli = Cli::try_parse_from(["mojave-node", "--log.level", "debug", "init"]).unwrap(); + let cli = Cli::try_parse_from(["mojave-node", "--log.level", "debug"]).unwrap(); - assert!(cli.log_level.is_some()); + assert!(cli.options.log_level.is_some()); } } diff --git a/cmd/node/src/main.rs b/cmd/node/src/main.rs index 379e7b0..0ddb7a4 100644 --- a/cmd/node/src/main.rs +++ b/cmd/node/src/main.rs @@ -1,79 +1,101 @@ pub mod cli; -use crate::cli::Command; -use anyhow::Result; -use mojave_node_lib::{initializers::get_signer, rpc::context::RpcApiContext, types::MojaveNode}; +use anyhow::{Context, Result}; +use mojave_node_lib::{rpc::context::RpcApiContext, types::MojaveNode}; +use mojave_rpc_core::types::Namespace; use mojave_rpc_server::RpcRegistry; -use mojave_utils::{ - block_on::block_on_current_thread, - daemon::{DaemonOptions, run_daemonized, stop_daemonized}, - p2p::public_key_from_signing_key, -}; +use mojave_utils::daemon::{DaemonOptions, run_daemonized}; use std::path::PathBuf; -use tracing::error; +use tracing::{error, info}; const PID_FILE_NAME: &str = "node.pid"; const LOG_FILE_NAME: &str = "node.log"; fn main() -> Result<()> { - mojave_utils::logging::init(); - let cli = cli::Cli::run(); + let cli::Cli { command, options } = cli::Cli::run(); - if let Some(log_level) = cli.log_level { - mojave_utils::logging::change_level(log_level); + mojave_utils::logging::init(options.log_level); + + let rt = build_runtime()?; + + if let Some(subcommand) = command { + return rt.block_on(async { subcommand.run(options.datadir.clone()).await }); } - match cli.command { - Command::Start { options } => { - let mut node_options: mojave_node_lib::types::NodeOptions = (&options).into(); - let rt = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build()?; - - if let Err(e) = - rt.block_on(async { MojaveNode::validate_node_options(&node_options).await }) - { - error!("Failed to validate node options: {}", e); - std::process::exit(1); - } - - node_options.datadir = cli.datadir.clone(); - let daemon_opts = DaemonOptions { - no_daemon: options.no_daemon, - pid_file_path: PathBuf::from(cli.datadir.clone()).join(PID_FILE_NAME), - log_file_path: PathBuf::from(cli.datadir).join(LOG_FILE_NAME), - }; - run_daemonized(daemon_opts, || async move { - let node = MojaveNode::init(&node_options) - .await - .unwrap_or_else(|error| { - error!("Failed to initialize the node: {}", error); - std::process::exit(1); - }); - - let registry = RpcRegistry::new().with_fallback( - mojave_rpc_core::types::Namespace::Eth, - |req, ctx: RpcApiContext| { - Box::pin(ethrex_rpc::map_eth_requests(req, ctx.l1_context)) - }, - ); - - node.run(&node_options, registry) - .await - .map_err(|e| Box::new(e) as Box) - }) - .unwrap_or_else(|err| { - error!(error = %err, "Failed to start daemonized node"); - }); - } - Command::Stop => stop_daemonized(PathBuf::from(cli.datadir.clone()).join(PID_FILE_NAME))?, - Command::GetPubKey => { - let signer = block_on_current_thread(|| async move { - get_signer(&cli.datadir).await.map_err(anyhow::Error::from) - })?; - let public_key = public_key_from_signing_key(&signer); - let public_key = hex::encode(public_key); - println!("{public_key}"); - } + + let node_options = build_node_options(&options); + + if let Err(e) = validate_node_options(&rt, &node_options) { + error!("Failed to validate node options: {}", e); + std::process::exit(1); } + + log_startup_config(&options); + info!("Starting Mojave Node..."); + + let daemon_opts = build_daemon_options(&options.datadir, options.no_daemon); + run_daemonized(daemon_opts, || async move { + let node = MojaveNode::init(&node_options) + .await + .context("initialize node") + .map_err(|e| Box::::from(e))?; + + let registry = build_registry(); + + node.run(&node_options, registry) + .await + .context("run node") + .map_err(|e| Box::::from(e)) + }) + .unwrap_or_else(|err| { + error!(error = %err, "Failed to start daemonized node"); + }); + Ok(()) } + +fn build_runtime() -> Result { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(Into::into) +} + +fn build_node_options(options: &cli::Options) -> mojave_node_lib::types::NodeOptions { + options.into() +} + +fn validate_node_options( + rt: &tokio::runtime::Runtime, + node_options: &mojave_node_lib::types::NodeOptions, +) -> Result<()> { + rt.block_on(async { MojaveNode::validate_node_options(node_options).await }) + .context("validate node options") +} + +fn build_daemon_options(datadir: &str, no_daemon: bool) -> DaemonOptions { + DaemonOptions { + no_daemon, + pid_file_path: PathBuf::from(datadir).join(PID_FILE_NAME), + log_file_path: PathBuf::from(datadir).join(LOG_FILE_NAME), + } +} + +fn build_registry() -> RpcRegistry { + RpcRegistry::new().with_fallback(Namespace::Eth, |req, ctx: RpcApiContext| { + Box::pin(ethrex_rpc::map_eth_requests(req, ctx.l1_context)) + }) +} + +fn log_startup_config(options: &cli::Options) { + info!( + datadir = %options.datadir, + network = %options.network, + http = %format!("{}:{}", options.http_addr, options.http_port), + authrpc = %format!("{}:{}", options.authrpc_addr, options.authrpc_port), + health = %format!("{}:{}", options.health_addr, options.health_port), + metrics = %format!("{}:{}", options.metrics_addr, options.metrics_port), + p2p = %format!("{}:{}", options.p2p_addr, options.p2p_port), + discovery = %format!("{}:{}", options.discovery_addr, options.discovery_port), + "Node startup configuration" + ); +} diff --git a/cmd/prover/src/main.rs b/cmd/prover/src/main.rs index 1928d4d..dff92be 100644 --- a/cmd/prover/src/main.rs +++ b/cmd/prover/src/main.rs @@ -10,13 +10,9 @@ const PID_FILE_NAME: &str = "prover.pid"; const LOG_FILE_NAME: &str = "prover.log"; fn main() -> Result<()> { - mojave_utils::logging::init(); - let cli = cli::Cli::run(); - if let Some(log_level) = cli.log_level { - mojave_utils::logging::change_level(log_level); - } + mojave_utils::logging::init(cli.log_level); match cli.command { Command::Start { prover_options } => { @@ -39,7 +35,7 @@ fn main() -> Result<()> { prover_options.queue_capacity, ) .await - .map_err(|e| Box::new(e) as Box) + .map_err(|e| Box::new(e) as Box) }) .unwrap_or_else(|err| tracing::error!("Failed to start daemonized prover: {}", err)); } diff --git a/cmd/sequencer/Cargo.toml b/cmd/sequencer/Cargo.toml index 55dab0b..8ddcf13 100644 --- a/cmd/sequencer/Cargo.toml +++ b/cmd/sequencer/Cargo.toml @@ -9,21 +9,15 @@ repository = { workspace = true } documentation = { workspace = true } [dependencies] -mojave-batch-producer = { workspace = true } -mojave-batch-submitter = { workspace = true } mojave-block-producer = { workspace = true } -mojave-msgio = { workspace = true } +mojave-coordination = { workspace = true } mojave-node-lib = { workspace = true } mojave-proof-coordinator = { workspace = true } -mojave-task = { workspace = true } mojave-utils = { workspace = true } anyhow = { workspace = true } clap = { workspace = true } hex = { workspace = true } -k8s-openapi = { workspace = true } -kube = { workspace = true } -kube-leader-election = { workspace = true } tokio = { workspace = true, features = [ "macros", "rt", diff --git a/cmd/sequencer/src/cli.rs b/cmd/sequencer/src/cli.rs index c299aff..73d495f 100644 --- a/cmd/sequencer/src/cli.rs +++ b/cmd/sequencer/src/cli.rs @@ -1,12 +1,62 @@ +use std::path::PathBuf; + use clap::{ArgAction, ArgGroup, Parser, Subcommand}; use mojave_block_producer::types::BlockProducerOptions; -use mojave_node_lib::types::{Node, SyncMode}; +use mojave_node_lib::{ + initializers::get_signer, + types::{Node, SyncMode}, +}; use mojave_proof_coordinator::types::ProofCoordinatorOptions; -use mojave_utils::network::Network; +use mojave_utils::{daemon::stop_daemonized, network::Network, p2p::public_key_from_signing_key}; use tracing::Level; +use crate::PID_FILE_NAME; + #[derive(Parser)] pub struct Options { + #[arg( + long = "log.level", + value_name = "LOG_LEVEL", + help = "The verbosity level used for logs.", + long_help = "Possible values: info, debug, trace, warn, error", + help_heading = "Node options", + global = true + )] + pub log_level: Option, + + #[arg( + long = "datadir", + value_name = "DATABASE_DIRECTORY", + help = "If the datadir is the word `memory`, ethrex will use the InMemory Engine", + default_value = ".mojave/sequencer", + help = "Receives the name of the directory where the Database is located.", + long_help = "If the datadir is the word `memory`, ethrex will use the `InMemory Engine`.", + help_heading = "Node options", + env = "ETHREX_DATADIR", + global = true + )] + pub datadir: String, + + #[arg( + long = "health.port", + value_name = "HEALTH_PORT", + default_value = "9595", + help = "Port for the health check HTTP server.", + help_heading = "Node options", + env = "HEALTH_PORT" + )] + pub health_port: String, + + #[arg( + long = "health.addr", + value_name = "HEALTH_ADDR", + default_value = "0.0.0.0", + help = "Address for the health check HTTP server.", + help_heading = "Node options", + env = "HEALTH_ADDR" + )] + pub health_addr: String, + #[arg( long = "network", default_value_t = Network::default(), @@ -125,6 +175,7 @@ pub struct Options { help_heading = "P2P options" )] pub discovery_port: String, + #[arg( long = "no-daemon", help = "If set, the sequencer will run in the foreground (not as a daemon). By default, the sequencer runs as a daemon.", @@ -149,13 +200,15 @@ impl From<&Options> for mojave_node_lib::types::NodeOptions { discovery_port: options.discovery_port.clone(), network: options.network.clone(), bootnodes: options.bootnodes.clone(), - datadir: Default::default(), + datadir: options.datadir.clone(), syncmode: options.syncmode.unwrap_or(SyncMode::Full), sponsorable_addresses_file_path: options.sponsorable_addresses_file_path.clone(), metrics_addr: options.metrics_addr.clone(), metrics_port: options.metrics_port.clone(), metrics_enabled: options.metrics_enabled, force: options.force, + health_addr: options.health_addr.clone(), + health_port: options.health_port.clone(), } } } @@ -170,29 +223,12 @@ impl From<&Options> for mojave_node_lib::types::NodeOptions { arg_required_else_help = true )] pub struct Cli { - #[arg( - long = "log.level", - value_name = "LOG_LEVEL", - help = "The verbosity level used for logs.", - long_help = "Possible values: info, debug, trace, warn, error", - help_heading = "Node options", - global = true - )] - pub log_level: Option, - #[arg( - long = "datadir", - value_name = "DATABASE_DIRECTORY", - help = "If the datadir is the word `memory`, ethrex will use the InMemory Engine", - default_value = ".mojave/sequencer", - help = "Receives the name of the directory where the Database is located.", - long_help = "If the datadir is the word `memory`, ethrex will use the `InMemory Engine`.", - help_heading = "Node options", - env = "ETHREX_DATADIR", - global = true - )] - pub datadir: String, + #[command(flatten)] + pub options: Options, + #[command(flatten)] + pub sequencer_options: SequencerOptions, #[command(subcommand)] - pub command: Command, + pub command: Option, } impl Cli { @@ -204,19 +240,27 @@ impl Cli { #[allow(clippy::large_enum_variant)] #[derive(Subcommand)] pub enum Command { - #[command(name = "init", about = "Run the sequencer")] - Start { - #[command(flatten)] - options: Options, - #[command(flatten)] - sequencer_options: SequencerOptions, - }, #[command(name = "stop", about = "Stop the sequencer")] Stop, #[command(name = "get-pub-key", about = "Display the public key of the node")] GetPubKey, } +impl Command { + pub async fn run(self, datadir: String) -> anyhow::Result<()> { + match self { + Command::Stop => stop_daemonized(PathBuf::from(datadir).join(PID_FILE_NAME)), + Command::GetPubKey => { + let signer = get_signer(&datadir).await.map_err(anyhow::Error::from)?; + let public_key = public_key_from_signing_key(&signer); + let public_key = hex::encode(public_key); + println!("{public_key}"); + Ok(()) + } + } + } +} + #[derive(Parser)] #[clap(group(ArgGroup::new("mojave::SequencerOptions")))] pub struct SequencerOptions { @@ -236,7 +280,8 @@ pub struct SequencerOptions { #[arg( long = "private_key", help = "Private key used for signing blocks", - env = "PRIVATE_KEY" + env = "PRIVATE_KEY", + default_value = "0xabc" )] pub private_key: String, } @@ -287,19 +332,16 @@ mod tests { #[test] fn parse_start_minimal_uses_defaults() { - let cli = - Cli::try_parse_from(["mojave-sequencer", "init", "--private_key", "0xabc"]).unwrap(); - - assert_eq!(cli.datadir, ".mojave/sequencer"); - assert!(cli.log_level.is_none()); - - let Command::Start { - ref options, - ref sequencer_options, - } = cli.command - else { - panic!("expected Start") - }; + let Cli { + command, + options, + sequencer_options, + } = Cli::try_parse_from(["mojave-sequencer", "--private_key", "0xabc"]).unwrap(); + + assert_eq!(options.datadir, ".mojave/sequencer"); + assert!(options.log_level.is_none()); + + assert!(command.is_none(), "expected None (default start)"); // Node Options defaults //assert_eq!(options.http_addr, "0.0.0.0"); @@ -324,9 +366,12 @@ mod tests { #[test] fn parse_start_with_overrides() { - let cli = Cli::try_parse_from([ + let Cli { + command, + options, + sequencer_options, + } = Cli::try_parse_from([ "mojave-sequencer", - "init", "--log.level", "debug", "--datadir", @@ -357,35 +402,29 @@ mod tests { ]) .unwrap(); - match cli.command { - Command::Start { - options, - sequencer_options, - } => { - assert_eq!(cli.log_level, Some(Level::DEBUG)); - assert_eq!(cli.datadir, "/tmp/sequencer"); - - assert_eq!(sequencer_options.prover_address, "http://127.0.0.1:3909"); - assert_eq!(sequencer_options.block_time, 2500); - assert_eq!(sequencer_options.private_key, "0xmojave"); - - //assert_eq!(options.http_addr, "127.0.0.1"); - //assert_eq!(options.http_port, "9000"); - //assert_eq!(options.authrpc_addr, "127.0.0.1"); - //assert_eq!(options.authrpc_port, "9001"); - //assert_eq!(options.authrpc_jwtsecret, "custom.jwt"); - assert_eq!(options.p2p_addr, "127.0.0.1"); - assert_eq!(options.p2p_port, "30304"); - assert_eq!(options.discovery_addr, "127.0.0.1"); - assert_eq!(options.discovery_port, "30305"); - assert_eq!(options.metrics_addr, "0.0.0.0"); - assert_eq!(options.metrics_port, "9393"); - assert!(options.metrics_enabled); - assert!(options.force); - assert!(matches!(options.syncmode, Some(SyncMode::Snap))); - } - _ => panic!("expected Start"), - } + assert!(command.is_none(), "expected None (default start)"); + + assert_eq!(options.log_level, Some(Level::DEBUG)); + assert_eq!(options.datadir, "/tmp/sequencer"); + + assert_eq!(sequencer_options.prover_address, "http://127.0.0.1:3909"); + assert_eq!(sequencer_options.block_time, 2500); + assert_eq!(sequencer_options.private_key, "0xmojave"); + + //assert_eq!(options.http_addr, "127.0.0.1"); + //assert_eq!(options.http_port, "9000"); + //assert_eq!(options.authrpc_addr, "127.0.0.1"); + //assert_eq!(options.authrpc_port, "9001"); + //assert_eq!(options.authrpc_jwtsecret, "custom.jwt"); + assert_eq!(options.p2p_addr, "127.0.0.1"); + assert_eq!(options.p2p_port, "30304"); + assert_eq!(options.discovery_addr, "127.0.0.1"); + assert_eq!(options.discovery_port, "30305"); + assert_eq!(options.metrics_addr, "0.0.0.0"); + assert_eq!(options.metrics_port, "9393"); + assert!(options.metrics_enabled); + assert!(options.force); + assert!(matches!(options.syncmode, Some(SyncMode::Snap))); } // #[test] @@ -425,9 +464,12 @@ mod tests { #[test] fn conversions_to_runtime_options_work() { // Options -> NodeOptions - let cli = Cli::try_parse_from([ + let Cli { + command, + options, + sequencer_options, + } = Cli::try_parse_from([ "mojave-sequencer", - "init", "--private_key", "0xabc", "--p2p.addr", @@ -448,16 +490,9 @@ mod tests { ]) .unwrap(); - let (node_opts, seq_opts) = match cli.command { - Command::Start { - options, - sequencer_options, - } => ( - mojave_node_lib::types::NodeOptions::from(&options), - sequencer_options, - ), - _ => panic!("expected Start"), - }; + assert!(command.is_none(), "expected None"); + + let node_opts = mojave_node_lib::types::NodeOptions::from(&options); //assert_eq!(node_opts.http_addr, "1.2.3.4"); //assert_eq!(node_opts.http_port, "9999"); @@ -474,13 +509,13 @@ mod tests { assert!(matches!(node_opts.syncmode, SyncMode::Full)); // SequencerOptions -> BlockProducerOptions - let bp: BlockProducerOptions = (&seq_opts).into(); - assert_eq!(bp.block_time, seq_opts.block_time); - assert_eq!(bp.private_key, seq_opts.private_key); + let bp: BlockProducerOptions = (&sequencer_options).into(); + assert_eq!(bp.block_time, sequencer_options.block_time); + assert_eq!(bp.private_key, sequencer_options.private_key); // SequencerOptions -> ProofCoordinatorOptions - let pc: ProofCoordinatorOptions = (&seq_opts).into(); - assert_eq!(pc.prover_address, seq_opts.prover_address); + let pc: ProofCoordinatorOptions = (&sequencer_options).into(); + assert_eq!(pc.prover_address, sequencer_options.prover_address); } #[test] @@ -500,15 +535,15 @@ mod tests { #[test] fn parse_stop_and_get_pub_key() { let cli = Cli::try_parse_from(["mojave-sequencer", "stop"]).unwrap(); - matches!(cli.command, Command::Stop); + assert!(matches!(cli.command, Some(Command::Stop))); let cli = Cli::try_parse_from(["mojave-sequencer", "get-pub-key"]).unwrap(); - matches!(cli.command, Command::GetPubKey); + assert!(matches!(cli.command, Some(Command::GetPubKey))); } #[test] fn invalid_bootnodes_string_rejected() { - let res = Cli::try_parse_from(["mojave-sequencer", "init", "--bootnodes", "not-enode-url"]); + let res = Cli::try_parse_from(["mojave-sequencer", "--bootnodes", "not-enode-url"]); assert!(res.is_err()); } @@ -518,12 +553,11 @@ mod tests { "mojave-sequencer", "--log.level", "debug", - "init", "--private_key", "0xabc", ]) .unwrap(); - assert!(cli.log_level.is_some()); + assert!(cli.options.log_level.is_some()); } } diff --git a/cmd/sequencer/src/k8s_leader.rs b/cmd/sequencer/src/k8s_leader.rs deleted file mode 100644 index 6d2c7b7..0000000 --- a/cmd/sequencer/src/k8s_leader.rs +++ /dev/null @@ -1,205 +0,0 @@ -use k8s_openapi::api::coordination::v1::Lease; -use kube::{Api, Client}; -use kube_leader_election::{LeaseLock, LeaseLockParams}; -use std::{env, time::Duration}; -use tokio::time::sleep; - -use mojave_batch_producer::{BatchProducer, types::Request as BatchProducerRequest}; -use mojave_batch_submitter::committer::Committer; -use mojave_block_producer::{ - BlockProducer, - types::{BlockProducerOptions, Request as BlockProducerRequest}, -}; -use mojave_node_lib::{ - types::{MojaveNode, NodeConfigFile}, - utils::store_node_config_file, -}; -use mojave_proof_coordinator::{ProofCoordinator, types::ProofCoordinatorOptions}; -use mojave_task::{Runner, Task, TaskHandle}; -use tracing::{error, info}; - -pub struct LeaderTasks { - batch: TaskHandle, - block: TaskHandle, - proof: TaskHandle, - committer: tokio::task::JoinHandle>, -} - -const BLOCK_PRODUCER_CAPACITY: usize = 100; - -pub async fn run_with_k8s_coordination( - node: MojaveNode, - node_options: mojave_node_lib::types::NodeOptions, - block_producer_options: BlockProducerOptions, - proof_coordinator_options: ProofCoordinatorOptions, -) -> Result<(), Box> { - let client = Client::try_default().await?; - let lease_client = client.clone(); - let identity = env::var("POD_NAME").unwrap_or_else(|_| "sequencer-pod".to_string()); - let namespace = env::var("POD_NAMESPACE").unwrap_or_else(|_| "default".to_string()); - - let lease_name = env::var("LEASE_NAME").unwrap_or_else(|_| "sequencer-leader".to_string()); - let lease_ttl_sec = env::var("LEASE_TTL_SECONDS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(15_u64); - let renew_every_secs = lease_ttl_sec / 5; // 1/5 of TTL - - let lease_lock = LeaseLock::new( - lease_client, - &namespace, - LeaseLockParams { - lease_name: lease_name.clone(), - holder_id: identity.clone(), - lease_ttl: Duration::from_secs(lease_ttl_sec), - }, - ); - - let mut am_i_leader = false; - let mut leader_tasks: Option = None; - - loop { - tokio::select! { - _ = mojave_utils::signal::wait_for_shutdown_signal() => { - info!("Termination signal received (K8s). Stopping leader tasks and exiting..."); - if am_i_leader { - if let Some(lt) = leader_tasks.take() { - stop_leader_tasks(lt).await?; - } - if let Err(err) = lease_lock.step_down().await { - error!("Error while stepping down from leader: {err:?}"); - } - } - - let (data_dir, _) = mojave_node_lib::utils::resolve_data_dir(&node_options.datadir).await?; - let node_config_path = data_dir.join("node_config.json"); - info!("Storing config at {:?}...", node_config_path); - let node_config = NodeConfigFile::new(node.peer_table.clone(), node.local_node_record.lock().await.clone()).await; - store_node_config_file(node_config, node_config_path).await; - - info!("Shutdown complete."); - break Ok(()); - } - res = lease_lock.try_acquire_or_renew() => { - match res { - Ok(_) => { - let currently_leader = match is_current_leader(&client, &namespace, &lease_name, &identity).await { - Ok(is_leader) => is_leader, - Err(err) => { - error!("Could not verify lease holder stepping down: {err:?}"); - false - } - }; - - if !am_i_leader && currently_leader { - info!("Became a leader. Start leader tasks"); - leader_tasks = Some( - start_leader_tasks( - node.clone(), - &node_options, - &block_producer_options, - &proof_coordinator_options, - ) - .await?, - ); - am_i_leader = true; - } else if am_i_leader && !currently_leader { - info!("Lost leadership. Stop leader tasks"); - if let Some(lt) = leader_tasks.take() { - stop_leader_tasks(lt).await?; - } - if let Err(err) = lease_lock.step_down().await { - error!("Error while stepping down from leader: {err:?}"); - } - am_i_leader = false; - } - - sleep(Duration::from_secs(renew_every_secs)).await; - } - Err(err) => { - error!("Error while k8s leader election: {err:?}"); - break Err(Box::new(err)); - } - } - } - } - } -} - -pub async fn start_leader_tasks( - node: MojaveNode, - node_options: &mojave_node_lib::types::NodeOptions, - block_producer_options: &BlockProducerOptions, - proof_coordinator_options: &ProofCoordinatorOptions, -) -> Result> { - let cancel_token = node.cancel_token.clone(); - - // TODO: replace by implementation backed by a real queue - let q = mojave_msgio::dummy::Dummy; - - let batch_counter = node.rollup_store.get_batch_number().await?.unwrap_or(0); - let batch_producer = BatchProducer::new(node.clone(), batch_counter); - let block_producer = BlockProducer::new(node.clone()); - let proof_coordinator = - ProofCoordinator::new(node.clone(), node_options, proof_coordinator_options)?; - - let batch = batch_producer - .clone() - .spawn_periodic(Duration::from_millis(100_000), || { - BatchProducerRequest::BuildBatch - }); - - let block = block_producer.spawn_with_capacity_periodic( - BLOCK_PRODUCER_CAPACITY, - Duration::from_millis(block_producer_options.block_time), - || BlockProducerRequest::BuildBlock, - ); - - let committer = Runner::new( - Committer::new(batch_producer.subscribe(), q, node.p2p_context.clone()), - cancel_token.clone(), - ) - .spawn(); - - let proof = proof_coordinator.spawn(); - - Ok(LeaderTasks { - batch, - block, - proof, - committer, - }) -} - -pub async fn stop_leader_tasks(lt: LeaderTasks) -> Result<(), Box> { - lt.batch.shutdown().await?; - lt.block.shutdown().await?; - lt.proof.shutdown().await?; - lt.committer.await??; - Ok(()) -} - -pub fn is_k8s_env() -> bool { - match std::env::var("KUBERNETES_SERVICE_HOST") { - Ok(_) => { - info!("Starting service as K8s version"); - true - } - _ => false, - } -} - -async fn is_current_leader( - client: &Client, - namespace: &str, - lease_name: &str, - identity: &str, -) -> Result { - let leases: Api = Api::namespaced(client.clone(), namespace); - let lease = leases.get(lease_name).await?; - let holder = lease - .spec - .as_ref() - .and_then(|spec| spec.holder_identity.clone()); - Ok(holder.as_deref() == Some(identity)) -} diff --git a/cmd/sequencer/src/main.rs b/cmd/sequencer/src/main.rs index 078cb80..7c93864 100644 --- a/cmd/sequencer/src/main.rs +++ b/cmd/sequencer/src/main.rs @@ -1,24 +1,12 @@ pub mod cli; -mod k8s_leader; -use crate::{ - cli::Command, - k8s_leader::{is_k8s_env, run_with_k8s_coordination, start_leader_tasks, stop_leader_tasks}, -}; -use anyhow::Result; +use anyhow::{Context, Result}; use mojave_block_producer::types::BlockProducerOptions; -use mojave_node_lib::{ - initializers::get_signer, - types::{MojaveNode, NodeConfigFile}, - utils::store_node_config_file, -}; +use mojave_coordination::sequencer::run_sequencer; +use mojave_node_lib::types::MojaveNode; use mojave_proof_coordinator::types::ProofCoordinatorOptions; -use mojave_utils::{ - block_on::block_on_current_thread, - daemon::{DaemonOptions, run_daemonized, stop_daemonized}, - p2p::public_key_from_signing_key, -}; +use mojave_utils::daemon::{DaemonOptions, run_daemonized}; use std::path::PathBuf; use tracing::{error, info}; @@ -26,94 +14,93 @@ const PID_FILE_NAME: &str = "sequencer.pid"; const LOG_FILE_NAME: &str = "sequencer.log"; fn main() -> Result<()> { - mojave_utils::logging::init(); - let cli = cli::Cli::run(); + let cli::Cli { + command, + options, + sequencer_options, + } = cli::Cli::run(); - if let Some(log_level) = cli.log_level { - mojave_utils::logging::change_level(log_level); + mojave_utils::logging::init(options.log_level); + + let rt = build_runtime()?; + + if let Some(subcommand) = command { + return rt.block_on(async { subcommand.run(options.datadir.clone()).await }); } - match cli.command { - Command::Start { - options, - sequencer_options, - } => { - let mut node_options: mojave_node_lib::types::NodeOptions = (&options).into(); - node_options.datadir = cli.datadir.clone(); - let rt = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build()?; - - if let Err(e) = - rt.block_on(async { MojaveNode::validate_node_options(&node_options).await }) - { - error!("Failed to validate node options: {}", e); - std::process::exit(1); - } - - let block_producer_options: BlockProducerOptions = (&sequencer_options).into(); - let proof_coordinator_options: ProofCoordinatorOptions = (&sequencer_options).into(); - let daemon_opts = DaemonOptions { - no_daemon: options.no_daemon, - pid_file_path: PathBuf::from(cli.datadir.clone()).join(PID_FILE_NAME), - log_file_path: PathBuf::from(cli.datadir).join(LOG_FILE_NAME), - }; - - run_daemonized(daemon_opts, || async move { - let node = MojaveNode::init(&node_options) - .await - .map_err(|e| Box::new(e) as Box)?; - - let use_k8s = is_k8s_env(); - - if use_k8s { - run_with_k8s_coordination( - node.clone(), - node_options.clone(), - block_producer_options, - proof_coordinator_options, - ) - .await?; - } else { - let lt = start_leader_tasks( - node.clone(), - &node_options, - &block_producer_options, - &proof_coordinator_options, - ) - .await?; - - tokio::select! { - _ = mojave_utils::signal::wait_for_shutdown_signal() => { - info!("Termination signal received, shutting down sequencer.."); - stop_leader_tasks(lt).await?; - - let node_config_path = PathBuf::from(node.data_dir).join("node_config.json"); - info!("Storing config at {:?}...", node_config_path); - let node_config = NodeConfigFile::new(node.peer_table.clone(), node.local_node_record.lock().await.clone()).await; - store_node_config_file(node_config, node_config_path).await; - - // TODO: wait for api to stop here - // if let Err(_elapsed) = tokio::time::timeout(std::time::Duration::from_secs(10), api_task).await { - // warn!("Timed out waiting for API to stop"); - // } - - info!("Successfully shut down the sequencer."); - } - } - } - Ok(()) - }) - .unwrap_or_else(|err| error!("Failed to start daemonized sequencer: {}", err)); - } - Command::Stop => stop_daemonized(PathBuf::from(cli.datadir.clone()).join(PID_FILE_NAME))?, - Command::GetPubKey => { - let signer = block_on_current_thread(|| async move { - get_signer(&cli.datadir).await.map_err(anyhow::Error::from) - })?; - let public_key = public_key_from_signing_key(&signer); - let public_key = hex::encode(public_key); - println!("{public_key}"); - } + + let node_options = build_node_options(&options); + if let Err(e) = validate_node_options(&rt, &node_options) { + error!("Failed to validate node options: {}", e); + std::process::exit(1); } + + log_startup_config(&options); + info!("Starting Sequencer..."); + + let block_producer_options: BlockProducerOptions = (&sequencer_options).into(); + let proof_coordinator_options: ProofCoordinatorOptions = (&sequencer_options).into(); + let daemon_opts = build_daemon_options(&options.datadir, options.no_daemon); + + run_daemonized(daemon_opts, || async move { + let node = MojaveNode::init(&node_options) + .await + .context("initialize sequencer node") + .map_err(|e| Box::::from(e))?; + + run_sequencer( + node, + &node_options, + &block_producer_options, + &proof_coordinator_options, + ) + .await + .map_err(|e| { + error!("Sequencer run failed: {e:?}"); + Box::::from(e) + }) + }) + .unwrap_or_else(|err| error!("Failed to start daemonized sequencer: {}", err)); + Ok(()) } + +fn build_runtime() -> Result { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(Into::into) +} + +fn build_node_options(options: &cli::Options) -> mojave_node_lib::types::NodeOptions { + let node_options: mojave_node_lib::types::NodeOptions = options.into(); + node_options +} + +fn validate_node_options( + rt: &tokio::runtime::Runtime, + node_options: &mojave_node_lib::types::NodeOptions, +) -> Result<()> { + rt.block_on(async { MojaveNode::validate_node_options(node_options).await }) + .context("validate node options") +} + +fn build_daemon_options(datadir: &str, no_daemon: bool) -> DaemonOptions { + DaemonOptions { + no_daemon, + pid_file_path: PathBuf::from(datadir).join(PID_FILE_NAME), + log_file_path: PathBuf::from(datadir).join(LOG_FILE_NAME), + } +} + +fn log_startup_config(options: &cli::Options) { + info!( + datadir = %options.datadir, + network = %options.network, + health = %format!("{}:{}", options.health_addr, options.health_port), + metrics = %format!("{}:{}", options.metrics_addr, options.metrics_port), + p2p_enabled = options.p2p_enabled, + p2p = %format!("{}:{}", options.p2p_addr, options.p2p_port), + discovery = %format!("{}:{}", options.discovery_addr, options.discovery_port), + "Sequencer startup configuration" + ); +} diff --git a/crates/batch-producer/Cargo.toml b/crates/batch-producer/Cargo.toml index f95066e..247aeab 100644 --- a/crates/batch-producer/Cargo.toml +++ b/crates/batch-producer/Cargo.toml @@ -11,7 +11,6 @@ documentation = { workspace = true } [dependencies] mojave-node-lib = { workspace = true } mojave-task = { workspace = true } -mojave-utils = { workspace = true } ethrex-blockchain = { workspace = true, default-features = false } ethrex-common = { workspace = true } diff --git a/crates/coordination/Cargo.toml b/crates/coordination/Cargo.toml new file mode 100644 index 0000000..4c01250 --- /dev/null +++ b/crates/coordination/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "mojave-coordination" +version = { workspace = true } +edition = { workspace = true } +authors = { workspace = true } +license = { workspace = true } +homepage = { workspace = true } +repository = { workspace = true } +documentation = { workspace = true } + +[dependencies] +mojave-batch-producer = { workspace = true } +mojave-block-producer = { workspace = true } +mojave-node-lib = { workspace = true } +mojave-proof-coordinator = { workspace = true } +mojave-task = { workspace = true } +mojave-utils = { workspace = true } + +k8s-openapi = { workspace = true } +kube = { workspace = true } +kube-leader-election = { workspace = true } +tracing = { workspace = true } + +tokio = { workspace = true } +tokio-util = { workspace = true } diff --git a/crates/coordination/src/coordination_mode.rs b/crates/coordination/src/coordination_mode.rs new file mode 100644 index 0000000..c36b82a --- /dev/null +++ b/crates/coordination/src/coordination_mode.rs @@ -0,0 +1,6 @@ +/// How the node coordinates leadership. +#[derive(Debug, Clone, Copy)] +pub enum CoordinationMode { + Kubernetes, + Standalone, +} diff --git a/crates/coordination/src/k8s.rs b/crates/coordination/src/k8s.rs new file mode 100644 index 0000000..f3a4a50 --- /dev/null +++ b/crates/coordination/src/k8s.rs @@ -0,0 +1,183 @@ +use std::{env, time::Duration}; + +use kube::Client; +use kube_leader_election::{LeaseLock, LeaseLockParams}; +use mojave_utils::signal::wait_for_shutdown_signal; +use tokio::{ + select, + time::{MissedTickBehavior, interval}, +}; +use tracing::{error, info}; + +use crate::utils::is_current_leader; + +/// Configuration for Kubernetes-based leader election, loaded from env vars. +struct K8sLeaderConfig { + identity: String, + namespace: String, + lease_lock: LeaseLock, + lease_name: String, + renew_every_secs: u64, +} + +impl K8sLeaderConfig { + fn from_env(client: Client) -> Self { + let identity = env::var("POD_NAME").unwrap_or_else(|_| "sequencer-pod".to_string()); + let namespace = env::var("POD_NAMESPACE").unwrap_or_else(|_| "default".to_string()); + let lease_name = env::var("LEASE_NAME").unwrap_or_else(|_| "sequencer-leader".to_string()); + + let lease_ttl_secs = env::var("LEASE_TTL_SECONDS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(15_u64); + + // Renew roughly every 1/5 of TTL, but never less than 1s. + let renew_every_secs = std::cmp::max(1, lease_ttl_secs / 5); + + let lease_lock_params = LeaseLockParams { + lease_name: lease_name.clone(), + holder_id: identity.clone(), + lease_ttl: Duration::from_secs(lease_ttl_secs), + }; + + let lease_lock = LeaseLock::new(client, &namespace, lease_lock_params); + + Self { + identity, + namespace, + lease_name, + renew_every_secs, + lease_lock, + } + } + + fn lease_lock(&self) -> &LeaseLock { + &self.lease_lock + } +} + +struct LeaderEpoch { + cancel_token: tokio_util::sync::CancellationToken, + handle: tokio::task::JoinHandle<()>, +} + +/// Run K8s leader election and drive a generic "leader task". +/// +/// - When this instance becomes leader, `spawn_leader_task` is called with a fresh +/// `CancellationToken`. +/// - When leadership is lost (or shutdown signal fires), the token is cancelled +/// and the leader task is awaited. +/// - This function returns when the **pod is shutting down** or on error. +/// +/// The K8s code does **not** know anything about `MojaveNode`, `NodeOptions`, etc. +/// It only knows how to start/stop some async work via a `CancellationToken`. +pub async fn run_with_k8s_coordination( + spawn_leader_task: F, +) -> Result<(), Box> +where + F: Fn(tokio_util::sync::CancellationToken) -> Fut, + Fut: std::future::Future + Send + 'static, +{ + let client = Client::try_default().await?; + let lease_client = client.clone(); + let k8s_config = K8sLeaderConfig::from_env(lease_client); + + let lease_lock = k8s_config.lease_lock(); + + let mut renew_interval = interval(Duration::from_secs(k8s_config.renew_every_secs)); + renew_interval.set_missed_tick_behavior(MissedTickBehavior::Delay); + + let mut am_i_leader = false; + let mut epoch: Option = None; + + // initial attempt to acquire leadership + match lease_lock.try_acquire_or_renew().await { + Ok(res) => { + if res.acquired_lease { + info!("This pod is now the leader (K8s). Starting leader task..."); + let cancel_token = tokio_util::sync::CancellationToken::new(); + let fut = spawn_leader_task(cancel_token.clone()); + let handle = tokio::spawn(async move { + fut.await; + }); + epoch = Some(LeaderEpoch { cancel_token, handle }); + am_i_leader = true; + } + } + Err(err) => { + error!("Error while k8s leader election: {err:?}"); + return Err(Box::new(err)); + } + } + + loop { + select! { + _ = wait_for_shutdown_signal() => { + shutdown(epoch, am_i_leader, lease_lock).await; + return Ok(()); + } + + _ = renew_interval.tick() => { + match lease_lock.try_acquire_or_renew().await { + Ok(res) => { + let currently_leader = res.acquired_lease; + + // becoming the leader + if currently_leader && !am_i_leader { + info!("This pod is now the leader (K8s). Starting leader task..."); + let cancel_token = tokio_util::sync::CancellationToken::new(); + let fut = spawn_leader_task(cancel_token.clone()); + let handle = tokio::spawn(async move { + fut.await; + }); + epoch = Some(LeaderEpoch { cancel_token, handle }); + am_i_leader = true; + } + // becoming a follower + else if !currently_leader && am_i_leader { + info!("This pod is no longer the leader (K8s). Stopping leader task..."); + + if let Some(LeaderEpoch { cancel_token, handle }) = epoch.take() { + cancel_token.cancel(); + match handle.await { + Ok(()) => {} + Err(join_err) => error!("Leader task panicked: {join_err:?}"), + } + } + if let Err(err) = lease_lock.step_down().await { + error!("Error while stepping down from leader: {err:?}"); + } + am_i_leader = false; + } + } + Err(err) => { + error!("Error while k8s leader election: {err:?}"); + return Err(Box::new(err)); + } + } + } + } + } +} + +async fn shutdown(epoch: Option, am_i_leader: bool, lease_lock: &LeaseLock) { + info!("Termination signal received (K8s). Stopping leader task and exiting..."); + + if let Some(LeaderEpoch { + cancel_token, + handle, + }) = epoch + { + cancel_token.cancel(); + match handle.await { + Ok(()) => {} + Err(join_err) => error!("Leader task panicked: {join_err:?}"), + } + } + + if am_i_leader && let Err(err) = lease_lock.step_down().await { + error!("Error while stepping down from leader: {err:?}"); + } + + info!("K8s coordination loop exiting."); +} diff --git a/crates/coordination/src/lib.rs b/crates/coordination/src/lib.rs new file mode 100644 index 0000000..41a6c10 --- /dev/null +++ b/crates/coordination/src/lib.rs @@ -0,0 +1,4 @@ +pub mod coordination_mode; +pub mod k8s; +pub mod sequencer; +pub mod utils; diff --git a/crates/coordination/src/sequencer.rs b/crates/coordination/src/sequencer.rs new file mode 100644 index 0000000..d012b63 --- /dev/null +++ b/crates/coordination/src/sequencer.rs @@ -0,0 +1,208 @@ +use std::{path::PathBuf, time::Duration}; + +use mojave_batch_producer::{BatchProducer, types::Request as BatchRequest}; +use mojave_block_producer::{ + BlockProducer, + types::{BlockProducerOptions, Request as BlockRequest}, +}; +use mojave_node_lib::{ + types::{MojaveNode, NodeConfigFile, NodeOptions}, + utils::store_node_config_file, +}; +use mojave_proof_coordinator::{ProofCoordinator, types::ProofCoordinatorOptions}; +use mojave_task::{Task, TaskHandle}; +use mojave_utils::{ + health::HealthProbeHandle, network::get_http_socket_addr, signal::wait_for_shutdown_signal, +}; +use tokio::select; +use tokio_util::sync::CancellationToken; +use tracing::{error, info}; + +use crate::{k8s::run_with_k8s_coordination, utils::is_k8s_env}; + +pub struct LeaderTasks { + batch: TaskHandle, + block: TaskHandle, + proof: TaskHandle, + health: HealthProbeHandle, +} + +const BLOCK_PRODUCER_CAPACITY: usize = 100; + +type BoxError = Box; + +async fn run_sequencer_leader_task( + node: MojaveNode, + options: &NodeOptions, + block_producer_options: &BlockProducerOptions, + proof_coordinator_options: &ProofCoordinatorOptions, + cancel_token: CancellationToken, +) -> Result<(), BoxError> { + info!("Starting Sequencer leader task..."); + + let leader_tasks = start_leader_tasks( + node, + options, + block_producer_options, + proof_coordinator_options, + cancel_token.clone(), + ) + .await?; + + cancel_token.cancelled().await; + info!("Shutdown token triggered, stopping sequencer leader tasks..."); + + stop_leader_tasks(leader_tasks).await?; + + info!("Sequencer leader tasks stopped."); + Ok(()) +} + +pub async fn run_sequencer( + node: MojaveNode, + options: &NodeOptions, + block_producer_options: &BlockProducerOptions, + proof_coordinator_options: &ProofCoordinatorOptions, +) -> Result<(), BoxError> { + let node_clone = node.clone(); + if is_k8s_env() { + run_with_k8s_coordination(move |shutdown_token: CancellationToken| { + let node_task = node.clone(); + let options_task = options.clone(); + let block_producer_options_task = block_producer_options.clone(); + let proof_coordinator_options_task = proof_coordinator_options.clone(); + + async move { + if let Err(err) = run_sequencer_leader_task( + node_task, + &options_task, + &block_producer_options_task, + &proof_coordinator_options_task, + shutdown_token, + ) + .await + { + error!("Sequencer leader task failed: {err:?}"); + } + } + }) + .await?; + } else { + info!("Starting Sequencer in standalone mode..."); + + let shutdown = CancellationToken::new(); + let shutdown_for_task = shutdown.clone(); + let node_task = node.clone(); + let options_task = options.clone(); + let block_producer_options_task = block_producer_options.clone(); + let proof_coordinator_options_task = proof_coordinator_options.clone(); + + let mut leader_task = tokio::spawn(async move { + run_sequencer_leader_task( + node_task, + &options_task, + &block_producer_options_task, + &proof_coordinator_options_task, + shutdown_for_task, + ) + .await + }); + + let mut leader_task_result = None; + + select! { + res = &mut leader_task => { + leader_task_result = Some(res); + } + _ = wait_for_shutdown_signal() => { + info!("Termination signal received, shutting down sequencer..."); + shutdown.cancel(); + } + } + + let result = match leader_task_result { + Some(res) => res, + None => leader_task.await, + }; + + match result { + Ok(Ok(())) => info!("Leader task shut down gracefully."), + Ok(Err(err)) => { + error!("Leader task returned error: {err:?}"); + return Err(err); + } + Err(err) => { + error!("Error while awaiting leader task shutdown: {err:?}"); + return Err(Box::new(err)); + } + }; + } + let node_config_path = PathBuf::from(node_clone.data_dir).join("node_config.json"); + info!("Storing config at {:?}...", node_config_path); + + let node_config = NodeConfigFile::new( + node_clone.peer_table.clone(), + node_clone.local_node_record.lock().await.clone(), + ) + .await; + store_node_config_file(node_config, node_config_path).await; + + Ok(()) +} + +async fn start_leader_tasks( + node: MojaveNode, + options: &NodeOptions, + block_producer_options: &BlockProducerOptions, + proof_coordinator_options: &ProofCoordinatorOptions, + cancel_token: CancellationToken, +) -> Result { + let batch_counter = node.rollup_store.get_batch_number().await?.unwrap_or(0); + let batch_producer = BatchProducer::new(node.clone(), batch_counter); + let block_producer = BlockProducer::new(node.clone()); + let proof_coordinator = + ProofCoordinator::new(node.clone(), options, proof_coordinator_options)?; + + let batch = batch_producer + .clone() + .spawn_periodic(Duration::from_millis(100_000), || BatchRequest::BuildBatch); + + let block = block_producer.spawn_with_capacity_periodic( + BLOCK_PRODUCER_CAPACITY, + Duration::from_millis(block_producer_options.block_time), + || BlockRequest::BuildBlock, + ); + + let proof = proof_coordinator.spawn(); + + // Health probe HTTP endpoint. + let health_socket_addr = + get_http_socket_addr(&options.health_addr, &options.health_port).await?; + let (_, health) = mojave_utils::health::spawn_health_probe( + health_socket_addr, + cancel_token.cancelled_owned(), + ) + .await?; + + Ok(LeaderTasks { + batch, + block, + proof, + health, + }) +} + +async fn stop_leader_tasks(lt: LeaderTasks) -> Result<(), BoxError> { + let LeaderTasks { + batch, + block, + proof, + health, + } = lt; + + batch.shutdown().await?; + block.shutdown().await?; + proof.shutdown().await?; + health.await??; + Ok(()) +} diff --git a/crates/coordination/src/utils.rs b/crates/coordination/src/utils.rs new file mode 100644 index 0000000..82c1119 --- /dev/null +++ b/crates/coordination/src/utils.rs @@ -0,0 +1,39 @@ +use std::env; + +use k8s_openapi::api::coordination::v1::Lease; +use kube::{Api, Client}; +use tracing::info; + +use crate::coordination_mode::CoordinationMode; + +/// Decide coordination mode based on environment (Kubernetes vs standalone). +pub fn detect_coordination_mode() -> CoordinationMode { + if env::var("KUBERNETES_SERVICE_HOST").is_ok() { + info!("Detected Kubernetes environment, using LeaseLock coordination"); + CoordinationMode::Kubernetes + } else { + info!("No Kubernetes detected, running in standalone mode"); + CoordinationMode::Standalone + } +} + +/// Backwards-compatible helper. +pub fn is_k8s_env() -> bool { + matches!(detect_coordination_mode(), CoordinationMode::Kubernetes) +} + +/// Check if the given identity is the current Lease holder. +pub async fn is_current_leader( + client: &Client, + namespace: &str, + lease_name: &str, + identity: &str, +) -> Result { + let leases: Api = Api::namespaced(client.clone(), namespace); + let lease = leases.get(lease_name).await?; + let holder = lease + .spec + .as_ref() + .and_then(|spec| spec.holder_identity.clone()); + Ok(holder.as_deref() == Some(identity)) +} diff --git a/crates/node/src/node.rs b/crates/node/src/node.rs index d7b3c94..0579371 100644 --- a/crates/node/src/node.rs +++ b/crates/node/src/node.rs @@ -168,6 +168,15 @@ impl MojaveNode { rpc_shutdown.clone(), registry, ); + + let health_socket_addr = + get_http_socket_addr(&options.health_addr, &options.health_port).await?; + let (_, health_handle) = mojave_utils::health::spawn_health_probe( + health_socket_addr, + self.cancel_token.clone().cancelled_owned(), + ) + .await?; + tokio::pin!(api_task); tokio::select! { res = &mut api_task => { @@ -175,6 +184,11 @@ impl MojaveNode { tracing::error!("API task returned error: {}", error); } } + res = health_handle => { + if let Err(error) = res { + tracing::error!("Health probe server returned error: {}", error); + } + } _ = mojave_utils::signal::wait_for_shutdown_signal() => { tracing::info!("Shutting down the full node.."); let node_config_path = PathBuf::from(self.data_dir).join("node_config.json"); @@ -202,6 +216,7 @@ impl MojaveNode { ensure_tcp_port_available(addr, port).await?; } ensure_udp_port_available(&options.discovery_addr, &options.discovery_port).await?; + ensure_tcp_port_available(&options.health_addr, &options.health_port).await?; if options.metrics_enabled { ensure_tcp_port_available(&options.metrics_addr, &options.metrics_port).await?; diff --git a/crates/node/src/types.rs b/crates/node/src/types.rs index 33421a4..b523070 100644 --- a/crates/node/src/types.rs +++ b/crates/node/src/types.rs @@ -61,6 +61,8 @@ pub struct NodeOptions { pub p2p_port: String, pub discovery_addr: String, pub discovery_port: String, + pub health_addr: String, + pub health_port: String, } impl Default for NodeOptions { @@ -85,6 +87,8 @@ impl Default for NodeOptions { metrics_port: Default::default(), metrics_enabled: Default::default(), force: false, + health_addr: Default::default(), + health_port: Default::default(), } } } diff --git a/crates/task/src/error.rs b/crates/task/src/error.rs index 59afcf1..d0b38fa 100644 --- a/crates/task/src/error.rs +++ b/crates/task/src/error.rs @@ -5,5 +5,5 @@ pub enum Error { #[error("Failed to receive a response: {0}")] Receive(#[from] tokio::sync::oneshot::error::RecvError), #[error("Task error: {0}")] - Task(Box), + Task(Box), } diff --git a/crates/task/src/traits.rs b/crates/task/src/traits.rs index 5c74e7b..4bb2826 100644 --- a/crates/task/src/traits.rs +++ b/crates/task/src/traits.rs @@ -10,7 +10,7 @@ use tokio::{ pub trait Task: Sized + 'static { type Request: Send + 'static; type Response: std::fmt::Debug + Send + 'static; - type Error: std::error::Error + Send + 'static; + type Error: std::error::Error + Send + Sync + 'static; fn name(&self) -> &'static str { std::any::type_name::() diff --git a/crates/utils/src/daemon.rs b/crates/utils/src/daemon.rs index 56287d9..31891cc 100644 --- a/crates/utils/src/daemon.rs +++ b/crates/utils/src/daemon.rs @@ -18,6 +18,8 @@ pub struct DaemonOptions { pub log_file_path: PathBuf, } +type DynError = Box; + #[derive(Debug, Error)] pub enum DaemonError { #[error("pid in pid file is already running. pid: {0}")] @@ -43,13 +45,10 @@ pub enum DaemonError { ParsePid(String), } -pub fn run_daemonized( - opts: DaemonOptions, - proc: F, -) -> Result<(), Box> +pub fn run_daemonized(opts: DaemonOptions, proc: F) -> Result<(), DynError> where F: FnOnce() -> Fut, - Fut: std::future::Future>>, + Fut: std::future::Future>, { if opts.no_daemon { return run_main_task(proc); @@ -173,10 +172,10 @@ fn is_pid_running(pid: Pid) -> bool { System::new_all().process(pid).is_some() } -fn run_main_task(proc: F) -> Result<(), Box> +fn run_main_task(proc: F) -> Result<(), DynError> where F: FnOnce() -> Fut, - Fut: std::future::Future>>, + Fut: std::future::Future>, { let rt = tokio::runtime::Builder::new_multi_thread() .enable_all() diff --git a/crates/utils/src/health.rs b/crates/utils/src/health.rs new file mode 100644 index 0000000..38df97d --- /dev/null +++ b/crates/utils/src/health.rs @@ -0,0 +1,96 @@ +use std::{future::Future, net::SocketAddr}; + +use tokio::{ + io::AsyncWriteExt, + net::{TcpListener, TcpStream}, + task::JoinHandle, +}; +use tracing::info; + +/// Background task handle for the health probe server. +pub type HealthProbeHandle = JoinHandle>; + +/// Spawn a lightweight HTTP server exposing a `/health` endpoint. +/// +/// The server binds the provided socket address (use port `0` to pick an ephemeral +/// port) and serves `GET /health` with a 200 OK response. The returned handle can +/// be awaited to surface server errors; the server stops when `shutdown_signal` +/// resolves. +pub async fn spawn_health_probe( + addr: SocketAddr, + shutdown_signal: F, +) -> Result<(SocketAddr, HealthProbeHandle), std::io::Error> +where + F: Future + Send + 'static, +{ + let listener = TcpListener::bind(addr).await?; + let bound_addr = listener.local_addr()?; + + info!("Health probe listening on {bound_addr}"); + + let handle = tokio::spawn(async move { + tokio::pin!(shutdown_signal); + + loop { + tokio::select! { + _ = &mut shutdown_signal => break, + accept_res = listener.accept() => { + let (mut stream, _) = accept_res?; + respond_ok(&mut stream).await?; + } + } + } + + Ok(()) + }); + + Ok((bound_addr, handle)) +} + +async fn respond_ok(stream: &mut TcpStream) -> std::io::Result<()> { + let mut buf = [0u8; 1024]; + let _ = stream.readable().await; + let _ = stream.try_read(&mut buf); + + const RESPONSE: &[u8] = b"HTTP/1.1 200 OK\r\ncontent-length: 2\r\ncontent-type: text/plain\r\nconnection: close\r\n\r\nOK"; + stream.write_all(RESPONSE).await?; + stream.shutdown().await +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::{io::AsyncReadExt, sync::oneshot}; + + #[tokio::test] + async fn health_probe_serves_ok() { + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let (addr, handle) = spawn_health_probe("127.0.0.1:0".parse().unwrap(), async { + let _ = shutdown_rx.await; + }) + .await + .expect("start health probe"); + + let mut stream = tokio::net::TcpStream::connect(addr) + .await + .expect("connect to health probe"); + stream + .write_all(b"GET /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + .await + .expect("write request"); + + let mut buf = Vec::new(); + stream.read_to_end(&mut buf).await.expect("read response"); + + let resp = String::from_utf8_lossy(&buf); + assert!( + resp.starts_with("HTTP/1.1 200 OK"), + "unexpected response: {resp}" + ); + assert!(resp.contains("\r\n\r\nOK"), "missing body: {resp}"); + + // Trigger graceful shutdown and surface any server errors + let _ = shutdown_tx.send(()); + handle.await.unwrap().unwrap(); + } +} diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 909f597..7a510aa 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -4,6 +4,7 @@ pub mod convert; pub mod daemon; pub mod error; pub mod hash; +pub mod health; pub mod logging; pub mod network; pub mod ordered_block; diff --git a/crates/utils/src/logging.rs b/crates/utils/src/logging.rs index fa4ccde..5b69799 100644 --- a/crates/utils/src/logging.rs +++ b/crates/utils/src/logging.rs @@ -6,7 +6,7 @@ use tracing_subscriber::{ static RELOAD_HANDLE: OnceLock> = OnceLock::new(); -pub fn init() { +pub fn init(log_level: Option) { let base_filter = EnvFilter::builder() .with_default_directive(Directive::from(Level::INFO)) .from_env_lossy(); @@ -22,6 +22,10 @@ pub fn init() { if let Err(e) = tracing::subscriber::set_global_default(subscriber) { eprintln!("Failed to set global tracing subscriber: {e}"); } + + if let Some(log_level) = log_level { + change_level(log_level); + } } pub fn change_level(log_level: Level) { diff --git a/justfile b/justfile index 9ce7d99..d44d2b6 100644 --- a/justfile +++ b/justfile @@ -34,9 +34,9 @@ node: else \ ifconfig | grep "inet " | grep -v 127.0.0.1 | awk '{print $2}' | head -n1; \ fi) && \ - if [ -z "$SKIP_BUILD" ]; then cargo build --bin mojave-node; fi && \ + if [ -z "${SKIP_BUILD:-}" ]; then cargo build --bin mojave-node; fi && \ ( \ - "${BIN_DIR:-target/debug}"/mojave-node init \ + "${BIN_DIR:-target/debug}"/mojave-node \ --network {{current-dir}}/data/testnet-genesis.json \ --bootnodes=enode://3e9c8a6bc193671ef87ea714ba2bcc979ae820672d5c93ff0ed265129b22180264eecebeae70ba947a6ffad76ab47eef41031838039f8f0ba84ea98b4d8734e5@$NODE_IP:30305 \ --no-daemon & \ @@ -56,9 +56,9 @@ node-release: else \ ifconfig | grep "inet " | grep -v 127.0.0.1 | awk '{print $2}' | head -n1; \ fi) && \ - if [ -z "$SKIP_BUILD" ]; then cargo build --release --bin mojave-node; fi && \ + if [ -z "${SKIP_BUILD:-}" ]; then cargo build --release --bin mojave-node; fi && \ ( \ - "${BIN_DIR:-target/release}"/mojave-node init \ + "${BIN_DIR:-target/release}"/mojave-node \ --network {{current-dir}}/data/testnet-genesis.json \ --bootnodes=enode://3e9c8a6bc193671ef87ea714ba2bcc979ae820672d5c93ff0ed265129b22180264eecebeae70ba947a6ffad76ab47eef41031838039f8f0ba84ea98b4d8734e5@$NODE_IP:30305 \ --no-daemon & \ @@ -74,9 +74,9 @@ sequencer: export $(cat .env | xargs) && \ mkdir -p {{home-dir}}/.mojave/sequencer && \ echo "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" > {{home-dir}}/.mojave/sequencer/node.key && \ - cargo build --bin mojave-sequencer && \ + if [ -z "${SKIP_BUILD:-}" ]; then cargo build --bin mojave-sequencer; fi && \ ( \ - "${BIN_DIR:-target/debug}"/mojave-sequencer init \ + "${BIN_DIR:-target/debug}"/mojave-sequencer \ --private_key 0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ --network {{current-dir}}/data/testnet-genesis.json \ --p2p.port 30305 \ @@ -95,9 +95,9 @@ sequencer-release: export $(cat .env | xargs) && \ mkdir -p {{home-dir}}/.mojave/sequencer && \ echo "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" > {{home-dir}}/.mojave/sequencer/node.key && \ - if [ -z "$SKIP_BUILD" ]; then cargo build --release --bin mojave-sequencer; fi && \ + if [ -z "${SKIP_BUILD:-}" ]; then cargo build --release --bin mojave-sequencer; fi && \ ( \ - "${BIN_DIR:-target/release}"/mojave-sequencer init \ + "${BIN_DIR:-target/release}"/mojave-sequencer \ --private_key 0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ --network {{current-dir}}/data/testnet-genesis.json \ --p2p.port 30305 \ @@ -189,7 +189,7 @@ registry := "1sixtech" docker-build bin registry=registry: role="{{bin}}"; \ role="${role#mojave-}"; \ - docker build --platform=linux/arm64 \ + docker build --platform=linux/arm64,linux/amd64 \ -f "docker/Dockerfile.target" \ -t {{ if registry == '' { bin } else { registry + '/' + bin } }} \ --build-arg "TARGET_BIN={{bin}}" \ diff --git a/k8s/deploy.node.yaml b/k8s/deploy.node.yaml index 72dccaf..31b55e7 100644 --- a/k8s/deploy.node.yaml +++ b/k8s/deploy.node.yaml @@ -1,3 +1,4 @@ +# yaml-language-server: $schema=https://kubernetesjsonschema.dev/v1.14.0/deployment-apps-v1.json apiVersion: apps/v1 kind: Deployment metadata: @@ -21,7 +22,6 @@ spec: image: mojave-node:latest imagePullPolicy: IfNotPresent args: - - "init" - "--datadir" - "/data/mojave/node" - "--http.addr" @@ -58,6 +58,22 @@ spec: - name: p2p-udp containerPort: 30303 protocol: UDP + - name: health + containerPort: 9595 + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: 9595 + initialDelaySeconds: 5 + periodSeconds: 3 + resources: + limits: + memory: "2Gi" + cpu: "1000m" + requests: + memory: "1Gi" + cpu: "500m" volumes: - name: node-datadir emptyDir: {} diff --git a/k8s/setup.sh b/k8s/setup.sh index 7209152..4aecd1f 100755 --- a/k8s/setup.sh +++ b/k8s/setup.sh @@ -10,5 +10,5 @@ kubectl apply -f k8s/namespace.yaml kubectl apply -f k8s/secret.sequencer.yaml kubectl apply -f k8s/rbac.sequencer.yaml kubectl apply -f k8s/service.sequencer.yaml -kubectl apply -f k8s/deploy.sequencer.yaml +kubectl apply -f k8s/stateful.sequencer.yaml kubectl apply -f k8s/deploy.node.yaml diff --git a/k8s/deploy.sequencer.yaml b/k8s/stateful.sequencer.yaml similarity index 91% rename from k8s/deploy.sequencer.yaml rename to k8s/stateful.sequencer.yaml index aa453ed..ff2fe5b 100644 --- a/k8s/deploy.sequencer.yaml +++ b/k8s/stateful.sequencer.yaml @@ -60,7 +60,6 @@ spec: name: sequencer-secrets key: private-key args: - - "init" - "--network" - "/data/testnet-genesis.json" - "--datadir" @@ -78,6 +77,16 @@ spec: - name: p2p-udp containerPort: 30303 protocol: UDP + - name: health + containerPort: 9595 + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: 9595 + initialDelaySeconds: 5 + periodSeconds: 3 + volumeClaimTemplates: - metadata: name: sequencer-datadir diff --git a/scripts/start.sh b/scripts/start.sh index 470a354..0af2ce0 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -60,8 +60,11 @@ cleanup() { printf "\nCaught Ctrl-C, stopping node and sequencer..." >&2 # Gracefully stop services via just recipes just kill-node kill-sequencer || true - # Stop log tailers + # Stop log tailers (sed processes) kill "$seq_tail" "$node_tail" 2>/dev/null || true + # Kill orphaned tail processes watching our log files + pkill -f "tail.*\.mojave/sequencer\.log" 2>/dev/null || true + pkill -f "tail.*\.mojave/node\.log" 2>/dev/null || true # Ensure background jobs are not left running kill "$seq_job" "$node_job" 2>/dev/null || true wait 2>/dev/null || true From 2e182a2b6b70efb67f36b67894bb8a2f60027a88 Mon Sep 17 00:00:00 2001 From: Sacha Froment Date: Wed, 3 Dec 2025 09:55:32 +0100 Subject: [PATCH 20/29] chore: lint Signed-off-by: Sacha Froment --- Cargo.lock | 61 +++++++++++++++++----------------- cmd/node/src/main.rs | 4 +-- cmd/sequencer/src/main.rs | 4 +-- crates/coordination/src/k8s.rs | 5 ++- 4 files changed, 39 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3aa0d45..d9a76f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -504,9 +504,9 @@ checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" [[package]] name = "bech32" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d965446196e3b7decd44aa7ee49e31d630118f90ef12f97900f262eb915c951d" +checksum = "32637268377fc7b10a8c6d51de3e7fba1ce5dd371a96e342b34e6078db558e7f" [[package]] name = "bincode" @@ -572,7 +572,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda569d741b895131a88ee5589a467e73e9c4718e958ac9308e4f7dc44b6945" dependencies = [ "base58ck", - "bech32 0.11.0", + "bech32 0.11.1", "bitcoin-internals", "bitcoin-io", "bitcoin-units", @@ -594,9 +594,9 @@ dependencies = [ [[package]] name = "bitcoin-io" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b47c4ab7a93edb0c7198c5535ed9b52b63095f4e9b45279c6736cec4b856baf" +checksum = "2dee39a0ee5b4095224a0cfc6bf4cc1baf0f9624b96b367e53b66d974e51d953" [[package]] name = "bitcoin-units" @@ -1179,9 +1179,9 @@ dependencies = [ [[package]] name = "convert_case" -version = "0.7.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb402b8d4c85569410425650ce3eddc7d698ed96d39a73f941b08fb63082f1e7" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" dependencies = [ "unicode-segmentation", ] @@ -1360,7 +1360,7 @@ checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ "bitflags 2.10.0", "crossterm_winapi", - "derive_more 2.0.1", + "derive_more 2.1.0", "document-features", "futures-core", "mio", @@ -1617,11 +1617,11 @@ dependencies = [ [[package]] name = "derive_more" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" +checksum = "10b768e943bed7bf2cab53df09f4bc34bfd217cdb57d971e769874c9a6710618" dependencies = [ - "derive_more-impl 2.0.1", + "derive_more-impl 2.1.0", ] [[package]] @@ -1639,13 +1639,14 @@ dependencies = [ [[package]] name = "derive_more-impl" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" +checksum = "6d286bfdaf75e988b4a78e013ecd79c581e06399ab53fbacd2d916c2f904f30b" dependencies = [ - "convert_case 0.7.1", + "convert_case 0.10.0", "proc-macro2", "quote", + "rustc_version", "syn 2.0.111", ] @@ -2639,7 +2640,7 @@ dependencies = [ "tower-http", "tracing", "tracing-subscriber", - "uuid 1.18.1", + "uuid 1.19.0", ] [[package]] @@ -3761,9 +3762,9 @@ dependencies = [ [[package]] name = "instability" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435d80800b936787d62688c927b6490e887c7ef5ff9ce922c6c6050fca75eb9a" +checksum = "6778b0196eefee7df739db78758e5cf9b37412268bfa5650bfeed028aed20d9c" dependencies = [ "darling 0.20.11", "indoc", @@ -4008,7 +4009,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "914bbb770e7bb721a06e3538c0edd2babed46447d128f7c21caa68747060ee73" dependencies = [ "chrono", - "derive_more 2.0.1", + "derive_more 2.1.0", "form_urlencoded", "http 1.4.0", "k8s-openapi", @@ -4132,9 +4133,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.177" +version = "0.2.178" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" [[package]] name = "libgit2-sys" @@ -4266,9 +4267,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.28" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "lru" @@ -6003,7 +6004,7 @@ dependencies = [ "rend", "rkyv_derive", "tinyvec", - "uuid 1.18.1", + "uuid 1.19.0", ] [[package]] @@ -7770,9 +7771,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.18.1" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" dependencies = [ "getrandom 0.3.4", "js-sys", @@ -8464,18 +8465,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.30" +version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea879c944afe8a2b25fef16bb4ba234f47c694565e97383b36f3a878219065c" +checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.30" +version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf955aa904d6040f70dc8e9384444cb1030aed272ba3cb09bbc4ab9e7c1f34f5" +checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" dependencies = [ "proc-macro2", "quote", @@ -8547,7 +8548,7 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tokio-util", - "uuid 1.18.1", + "uuid 1.19.0", ] [[package]] diff --git a/cmd/node/src/main.rs b/cmd/node/src/main.rs index 0ddb7a4..b5671af 100644 --- a/cmd/node/src/main.rs +++ b/cmd/node/src/main.rs @@ -37,14 +37,14 @@ fn main() -> Result<()> { let node = MojaveNode::init(&node_options) .await .context("initialize node") - .map_err(|e| Box::::from(e))?; + .map_err(Box::::from)?; let registry = build_registry(); node.run(&node_options, registry) .await .context("run node") - .map_err(|e| Box::::from(e)) + .map_err(Box::::from) }) .unwrap_or_else(|err| { error!(error = %err, "Failed to start daemonized node"); diff --git a/cmd/sequencer/src/main.rs b/cmd/sequencer/src/main.rs index 7c93864..03d8def 100644 --- a/cmd/sequencer/src/main.rs +++ b/cmd/sequencer/src/main.rs @@ -45,7 +45,7 @@ fn main() -> Result<()> { let node = MojaveNode::init(&node_options) .await .context("initialize sequencer node") - .map_err(|e| Box::::from(e))?; + .map_err(Box::::from)?; run_sequencer( node, @@ -56,7 +56,7 @@ fn main() -> Result<()> { .await .map_err(|e| { error!("Sequencer run failed: {e:?}"); - Box::::from(e) + e }) }) .unwrap_or_else(|err| error!("Failed to start daemonized sequencer: {}", err)); diff --git a/crates/coordination/src/k8s.rs b/crates/coordination/src/k8s.rs index f3a4a50..251e9d1 100644 --- a/crates/coordination/src/k8s.rs +++ b/crates/coordination/src/k8s.rs @@ -100,7 +100,10 @@ where let handle = tokio::spawn(async move { fut.await; }); - epoch = Some(LeaderEpoch { cancel_token, handle }); + epoch = Some(LeaderEpoch { + cancel_token, + handle, + }); am_i_leader = true; } } From b9dcca4bd9036e53552738eec4f0d284f918fc29 Mon Sep 17 00:00:00 2001 From: Sacha Froment Date: Wed, 3 Dec 2025 10:03:38 +0100 Subject: [PATCH 21/29] chore: lint Signed-off-by: Sacha Froment --- crates/coordination/src/k8s.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/crates/coordination/src/k8s.rs b/crates/coordination/src/k8s.rs index 251e9d1..3881880 100644 --- a/crates/coordination/src/k8s.rs +++ b/crates/coordination/src/k8s.rs @@ -9,19 +9,15 @@ use tokio::{ }; use tracing::{error, info}; -use crate::utils::is_current_leader; - /// Configuration for Kubernetes-based leader election, loaded from env vars. struct K8sLeaderConfig { - identity: String, - namespace: String, lease_lock: LeaseLock, - lease_name: String, renew_every_secs: u64, } impl K8sLeaderConfig { fn from_env(client: Client) -> Self { + // TODO: make this flags from NodeOptions or something similar let identity = env::var("POD_NAME").unwrap_or_else(|_| "sequencer-pod".to_string()); let namespace = env::var("POD_NAMESPACE").unwrap_or_else(|_| "default".to_string()); let lease_name = env::var("LEASE_NAME").unwrap_or_else(|_| "sequencer-leader".to_string()); @@ -43,9 +39,6 @@ impl K8sLeaderConfig { let lease_lock = LeaseLock::new(client, &namespace, lease_lock_params); Self { - identity, - namespace, - lease_name, renew_every_secs, lease_lock, } From ed8c4c459ad8acc9cd1904e577f44a8449cbfd6b Mon Sep 17 00:00:00 2001 From: Sacha Froment Date: Wed, 3 Dec 2025 10:20:02 +0100 Subject: [PATCH 22/29] chore: fix registry Signed-off-by: Sacha Froment --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index d44d2b6..03d3ad2 100644 --- a/justfile +++ b/justfile @@ -182,7 +182,7 @@ doc: doc-watch: cargo watch -x "doc --no-deps" -registry := "1sixtech" +registry := "ghcr.io/1sixtech" # Build the docker image for a specific binary # Binary name should be one of: mojave-node, mojave-sequencer, mojave-prover From 525c108c433dae37af196c3d858c0e7d9ce0a36a Mon Sep 17 00:00:00 2001 From: Sacha Froment Date: Wed, 3 Dec 2025 10:20:41 +0100 Subject: [PATCH 23/29] Update docs/k8s.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/k8s.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/k8s.md b/docs/k8s.md index d29a2a8..82ca21b 100644 --- a/docs/k8s.md +++ b/docs/k8s.md @@ -160,7 +160,7 @@ Check that all resources are created: ```bash kubectl get sts mojave-sequencer-deployment -n 1sixtech kubectl get pods -n 1sixtech -kubectl get svc mojave-sequencer-service -n 1sixtech +kubectl get svc mojave-sequencer -n 1sixtech kubectl get pvc -n 1sixtech kubectl get lease sequencer-leader -n 1sixtech -o yaml ``` From de5efe3cc5396fa076fc542014272e410190070e Mon Sep 17 00:00:00 2001 From: Sacha Froment Date: Wed, 3 Dec 2025 10:22:41 +0100 Subject: [PATCH 24/29] Update docs/k8s.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/k8s.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/k8s.md b/docs/k8s.md index 82ca21b..90a2189 100644 --- a/docs/k8s.md +++ b/docs/k8s.md @@ -128,7 +128,7 @@ For this HA test you need a Docker image that contains the `mojave-sequencer` bi - Option 2: Patch the live StatefulSet after applying the manifests: ```bash - kubectl set image statefulset/mojave-sequencer-deployment -n 1sixtech \ + kubectl set image statefulset/mojave-sequencer -n 1sixtech \ mojave-sequencer=/mojave-sequencer:latest ``` From 11f01832921da6eec10d8c78f069159df4b5238f Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 10:32:04 +0100 Subject: [PATCH 25/29] Fix StatefulSet name references in k8s documentation (#283) * Initial plan * Fix StatefulSet name references in k8s documentation Co-authored-by: sfroment <7238385+sfroment@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: sfroment <7238385+sfroment@users.noreply.github.com> --- docs/k8s.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/k8s.md b/docs/k8s.md index 90a2189..02cdbe1 100644 --- a/docs/k8s.md +++ b/docs/k8s.md @@ -158,7 +158,7 @@ If you hit `pod has unbound immediate PersistentVolumeClaims` errors, make sure Check that all resources are created: ```bash -kubectl get sts mojave-sequencer-deployment -n 1sixtech +kubectl get sts mojave-sequencer -n 1sixtech kubectl get pods -n 1sixtech kubectl get svc mojave-sequencer -n 1sixtech kubectl get pvc -n 1sixtech @@ -168,8 +168,8 @@ kubectl get lease sequencer-leader -n 1sixtech -o yaml You should see: - 2 pods (or however many replicas you configured) with `app=mojave-sequencer`. -- A StatefulSet `mojave-sequencer-deployment` reporting ready replicas. -- Per-pod PVCs named like `sequencer-datadir-mojave-sequencer-deployment-0`. +- A StatefulSet `mojave-sequencer` reporting ready replicas. +- Per-pod PVCs named like `sequencer-datadir-mojave-sequencer-0`. - A single `Lease` named `sequencer-leader` with one of the pods listed as the current holder. #### 2.6 Check that the current leader is producing blocks (via logs) From f80449709fdb0d47a8eb96d996bcba5212b17085 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 10:32:43 +0100 Subject: [PATCH 26/29] Fix documentation references to use correct StatefulSet manifest filename (#282) * Initial plan * Fix documentation: update references from deploy.sequencer.yaml to stateful.sequencer.yaml Co-authored-by: sfroment <7238385+sfroment@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: sfroment <7238385+sfroment@users.noreply.github.com> --- docs/k8s.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/k8s.md b/docs/k8s.md index 02cdbe1..ca87777 100644 --- a/docs/k8s.md +++ b/docs/k8s.md @@ -19,7 +19,7 @@ This document explains: All manifests live under `k8s/`: - `k8s/namespace.yaml`: Namespace definition (`1sixtech`) for the Sequencer resources. -- `k8s/deploy.sequencer.yaml`: `StatefulSet` (2 replicas by default) of `mojave-sequencer`, with leader election and a `volumeClaimTemplate` (20Gi) per pod mounted at `/data/mojave`. +- `k8s/stateful.sequencer.yaml`: `StatefulSet` (2 replicas by default) of `mojave-sequencer`, with leader election and a `volumeClaimTemplate` (20Gi) per pod mounted at `/data/mojave`. - `k8s/deploy.node.yaml`: `Deployment` (1 replica by default) of `mojave-node` with an init container that prepares `/data/mojave/node` and writes a random JWT secret for `authrpc`. - `k8s/service.sequencer.yaml`: - Headless `Service` (`mojave-sequencer-headless`) for the StatefulSet `serviceName`. @@ -35,7 +35,7 @@ All manifests live under `k8s/`: The Sequencer binary detects a Kubernetes environment by checking the `KUBERNETES_SERVICE_HOST` environment variable. When running in Kubernetes: -- The pod receives the following environment variables from `deploy.sequencer.yaml`: +- The pod receives the following environment variables from `stateful.sequencer.yaml`: - `POD_NAME`: the pod name (from the downward API). - `POD_NAMESPACE`: the namespace. - `LEASE_NAME`: the name of the `Lease` object used for leader election (default: `sequencer-leader`). @@ -54,7 +54,7 @@ The data directory is backed by a per-pod `PersistentVolumeClaim` generated from - Each pod gets its own 20Gi claim (using the cluster's default `StorageClass` unless you override `storageClassName`). - The claim is mounted at `/data/mojave` in the container. - The Sequencer is started with `--datadir /data/mojave/sequencer`, so each pod writes to its own `/data/mojave/sequencer`. -- An init container writes `/data/mojave/statefulset-ordinal` with the pod's ordinal (0, 1, …) and `/data/mojave/role` with `primary` for ordinal 0 and `secondary` otherwise. Adjust this script in `k8s/deploy.sequencer.yaml` if you want different per-pod config files. +- An init container writes `/data/mojave/statefulset-ordinal` with the pod's ordinal (0, 1, …) and `/data/mojave/role` with `primary` for ordinal 0 and `secondary` otherwise. Adjust this script in `k8s/stateful.sequencer.yaml` if you want different per-pod config files. --- @@ -91,7 +91,7 @@ For this HA test you need a Docker image that contains the `mojave-sequencer` bi **Option A (recommended for a quick test): Use the prebuilt Mojave image** -- `k8s/deploy.sequencer.yaml` already references a published image: +- `k8s/stateful.sequencer.yaml` already references a published image: - `starkgiwook/mojave-sequencer:latest` - If your Minikube cluster can pull from Docker Hub, you **do not need to build anything**. You can skip to **2.6 Apply the core Kubernetes resources**. @@ -115,10 +115,10 @@ For this HA test you need a Docker image that contains the `mojave-sequencer` bi 3. Update the StatefulSet to use your image instead of the default: - - Option 1: Edit `k8s/deploy.sequencer.yaml`: + - Option 1: Edit `k8s/stateful.sequencer.yaml`: ```yaml - # inside k8s/deploy.sequencer.yaml + # inside k8s/stateful.sequencer.yaml containers: - name: mojave-sequencer image: /mojave-sequencer:latest @@ -141,7 +141,7 @@ kubectl apply -f k8s/namespace.yaml kubectl apply -f k8s/secret.sequencer.yaml kubectl apply -f k8s/rbac.sequencer.yaml kubectl apply -f k8s/service.sequencer.yaml -kubectl apply -f k8s/deploy.sequencer.yaml +kubectl apply -f k8s/stateful.sequencer.yaml kubectl apply -f k8s/deploy.node.yaml ``` @@ -151,7 +151,7 @@ Or use the helper script (note: this **first deletes** all manifests under `k8s/ bash k8s/setup.sh ``` -If you hit `pod has unbound immediate PersistentVolumeClaims` errors, make sure your cluster has a default `StorageClass` or set `storageClassName` in `k8s/deploy.sequencer.yaml` to one that exists in your cluster (e.g., `local-path` on k3d/k3s, `standard` on many managed clusters). +If you hit `pod has unbound immediate PersistentVolumeClaims` errors, make sure your cluster has a default `StorageClass` or set `storageClassName` in `k8s/stateful.sequencer.yaml` to one that exists in your cluster (e.g., `local-path` on k3d/k3s, `standard` on many managed clusters). #### 2.5 Verify the deployment From 78d5276a36602e95ef4804d5f5e96e3c8d11f148 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 10:33:09 +0100 Subject: [PATCH 27/29] Fix incorrect filename references in Kubernetes documentation (#281) * Initial plan * Fix all references from deploy.sequencer.yaml to stateful.sequencer.yaml Co-authored-by: sfroment <7238385+sfroment@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: sfroment <7238385+sfroment@users.noreply.github.com> From c5d3457e469ae4abd4f76d64d704a4644b06002b Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Dec 2025 10:43:18 +0100 Subject: [PATCH 28/29] Consolidate shutdown logic in K8s leader election error handling (#284) * Initial plan * Call shutdown() on error paths to ensure consistent cleanup Co-authored-by: sfroment <7238385+sfroment@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: sfroment <7238385+sfroment@users.noreply.github.com> --- crates/coordination/src/k8s.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/coordination/src/k8s.rs b/crates/coordination/src/k8s.rs index 3881880..f2579fd 100644 --- a/crates/coordination/src/k8s.rs +++ b/crates/coordination/src/k8s.rs @@ -102,6 +102,7 @@ where } Err(err) => { error!("Error while k8s leader election: {err:?}"); + shutdown(epoch, am_i_leader, lease_lock).await; return Err(Box::new(err)); } } @@ -148,6 +149,7 @@ where } Err(err) => { error!("Error while k8s leader election: {err:?}"); + shutdown(epoch, am_i_leader, lease_lock).await; return Err(Box::new(err)); } } From 70ba46400ecb81d9f4ddd37c0cf76144b2bb1548 Mon Sep 17 00:00:00 2001 From: Sacha Froment Date: Wed, 3 Dec 2025 12:39:46 +0100 Subject: [PATCH 29/29] fix: healt.port Signed-off-by: Sacha Froment --- cmd/node/src/main.rs | 6 +++--- cmd/sequencer/src/main.rs | 6 +++--- justfile | 2 ++ 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/cmd/node/src/main.rs b/cmd/node/src/main.rs index b5671af..d16c691 100644 --- a/cmd/node/src/main.rs +++ b/cmd/node/src/main.rs @@ -25,7 +25,7 @@ fn main() -> Result<()> { let node_options = build_node_options(&options); if let Err(e) = validate_node_options(&rt, &node_options) { - error!("Failed to validate node options: {}", e); + error!("Failed to validate node options: {e}"); std::process::exit(1); } @@ -68,8 +68,8 @@ fn validate_node_options( rt: &tokio::runtime::Runtime, node_options: &mojave_node_lib::types::NodeOptions, ) -> Result<()> { - rt.block_on(async { MojaveNode::validate_node_options(node_options).await }) - .context("validate node options") + rt.block_on(MojaveNode::validate_node_options(node_options)) + .map_err(|e| anyhow::anyhow!("Node options validation failed: {e}")) } fn build_daemon_options(datadir: &str, no_daemon: bool) -> DaemonOptions { diff --git a/cmd/sequencer/src/main.rs b/cmd/sequencer/src/main.rs index 03d8def..86f29c1 100644 --- a/cmd/sequencer/src/main.rs +++ b/cmd/sequencer/src/main.rs @@ -30,7 +30,7 @@ fn main() -> Result<()> { let node_options = build_node_options(&options); if let Err(e) = validate_node_options(&rt, &node_options) { - error!("Failed to validate node options: {}", e); + error!("Failed to validate node options: {e}"); std::process::exit(1); } @@ -80,8 +80,8 @@ fn validate_node_options( rt: &tokio::runtime::Runtime, node_options: &mojave_node_lib::types::NodeOptions, ) -> Result<()> { - rt.block_on(async { MojaveNode::validate_node_options(node_options).await }) - .context("validate node options") + rt.block_on(MojaveNode::validate_node_options(node_options)) + .map_err(|e| anyhow::anyhow!("Node options validation failed: {e}")) } fn build_daemon_options(datadir: &str, no_daemon: bool) -> DaemonOptions { diff --git a/justfile b/justfile index 03d3ad2..db608b1 100644 --- a/justfile +++ b/justfile @@ -38,6 +38,7 @@ node: ( \ "${BIN_DIR:-target/debug}"/mojave-node \ --network {{current-dir}}/data/testnet-genesis.json \ + --health.port 9596 \ --bootnodes=enode://3e9c8a6bc193671ef87ea714ba2bcc979ae820672d5c93ff0ed265129b22180264eecebeae70ba947a6ffad76ab47eef41031838039f8f0ba84ea98b4d8734e5@$NODE_IP:30305 \ --no-daemon & \ pid=$!; \ @@ -60,6 +61,7 @@ node-release: ( \ "${BIN_DIR:-target/release}"/mojave-node \ --network {{current-dir}}/data/testnet-genesis.json \ + --health.port 9596 \ --bootnodes=enode://3e9c8a6bc193671ef87ea714ba2bcc979ae820672d5c93ff0ed265129b22180264eecebeae70ba947a6ffad76ab47eef41031838039f8f0ba84ea98b4d8734e5@$NODE_IP:30305 \ --no-daemon & \ pid=$!; \