From b25072fcf90d40f8353585cb501557554a5ac48c Mon Sep 17 00:00:00 2001 From: Daniel Sharifi Date: Fri, 6 Mar 2026 14:25:36 +0100 Subject: [PATCH 01/16] claude first pass --- crates/node/src/cli.rs | 714 ++++++++++---------------------- crates/node/src/config.rs | 3 + crates/node/src/config/start.rs | 80 ++++ crates/node/src/lib.rs | 1 + crates/node/src/run.rs | 321 ++++++++++++++ libs/nearcore | 2 +- 6 files changed, 629 insertions(+), 492 deletions(-) create mode 100644 crates/node/src/config/start.rs create mode 100644 crates/node/src/run.rs diff --git a/crates/node/src/cli.rs b/crates/node/src/cli.rs index 3f976a547..b4ca08405 100644 --- a/crates/node/src/cli.rs +++ b/crates/node/src/cli.rs @@ -1,60 +1,30 @@ use crate::{ config::{ - generate_and_write_backup_encryption_key_to_disk, load_config_file, BlockArgs, CKDConfig, - ConfigFile, ForeignChainsConfig, IndexerConfig, KeygenConfig, PersistentSecrets, - PresignatureConfig, RespondConfig, SecretsConfig, SignatureConfig, SyncMode, TripleConfig, - }, - coordinator::Coordinator, - db::SecretDB, - indexer::{ - real::spawn_real_indexer, tx_sender::TransactionSender, IndexerAPI, ReadForeignChainPolicy, + BlockArgs, CKDConfig, ConfigFile, ForeignChainsConfig, IndexerConfig, KeygenConfig, + PersistentSecrets, PresignatureConfig, SignatureConfig, StartConfig, SyncMode, + TeeAuthorityStartConfig, TripleConfig, }, keyshare::{ compat::legacy_ecdsa_key_from_keyshares, local::LocalPermanentKeyStorageBackend, permanent::{PermanentKeyStorage, PermanentKeyStorageBackend, PermanentKeyshareData}, - GcpPermanentKeyStorageConfig, KeyStorageConfig, KeyshareStorage, }, - migration_service::spawn_recovery_server_and_run_onboarding, p2p::testing::{generate_test_p2p_configs, PortSeed}, - profiler, - tracking::{self, start_root_task}, - web::{start_web_server, static_web_data, DebugRequest}, }; -use anyhow::{anyhow, Context}; use clap::{Args, Parser, Subcommand, ValueEnum}; use hex::FromHex; -use mpc_attestation::report_data::ReportDataV1; -use mpc_contract::state::ProtocolContractState; use near_account_id::AccountId; use near_indexer_primitives::types::Finality; -use near_time::Clock; use std::{ - collections::BTreeMap, net::{Ipv4Addr, SocketAddr}, - sync::Mutex, -}; -use std::{path::PathBuf, sync::Arc, sync::OnceLock, time::Duration}; -use tee_authority::tee_authority::{ - DstackTeeAuthorityConfig, LocalTeeAuthorityConfig, TeeAuthority, DEFAULT_DSTACK_ENDPOINT, - DEFAULT_PHALA_TDX_QUOTE_UPLOAD_URL, + path::PathBuf, }; -use tokio::sync::{broadcast, mpsc, oneshot, watch, RwLock}; -use tokio_util::sync::CancellationToken; +use tee_authority::tee_authority::{DEFAULT_DSTACK_ENDPOINT, DEFAULT_PHALA_TDX_QUOTE_UPLOAD_URL}; use url::Url; -use contract_interface::types::Ed25519PublicKey; -use { - crate::tee::{ - monitor_allowed_image_hashes, - remote_attestation::{monitor_attestation_removal, periodic_attestation_submission}, - AllowedImageHashesFile, - }, - mpc_contract::tee::proposal::MpcDockerImageHash, - tracing::info, -}; - -pub const ATTESTATION_RESUBMISSION_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour +// --------------------------------------------------------------------------- +// Top-level CLI +// --------------------------------------------------------------------------- #[derive(Parser, Debug)] #[command(name = "mpc-node")] @@ -77,6 +47,13 @@ pub enum LogFormat { #[derive(Subcommand, Debug)] pub enum CliCommand { + /// Starts the MPC node using a single JSON configuration file instead of + /// environment variables and CLI flags. + StartWithConfigFile { + /// Path to a JSON configuration file containing all settings needed to + /// start the MPC node. + config_path: PathBuf, + }, Start(StartCmd), /// Generates/downloads required files for Near node to run Init(InitConfigArgs), @@ -109,32 +86,9 @@ pub enum CliCommand { }, } -#[derive(Args, Debug)] -pub struct InitConfigArgs { - #[arg(long, env("MPC_HOME_DIR"))] - pub dir: std::path::PathBuf, - /// chain/network id (localnet, testnet, devnet, betanet) - #[arg(long)] - pub chain_id: Option, - /// Genesis file to use when initialize testnet (including downloading) - #[arg(long)] - pub genesis: Option, - /// Download the verified NEAR config file automatically. - #[arg(long)] - pub download_config: bool, - #[arg(long)] - pub download_config_url: Option, - /// Download the verified NEAR genesis file automatically. - #[arg(long)] - pub download_genesis: bool, - /// Specify a custom download URL for the genesis-file. - #[arg(long)] - pub download_genesis_url: Option, - #[arg(long)] - pub download_genesis_records_url: Option, - #[arg(long)] - pub boot_nodes: Option, -} +// --------------------------------------------------------------------------- +// Start subcommand (CLI flags / env vars) +// --------------------------------------------------------------------------- #[derive(Args, Debug)] pub struct StartCmd { @@ -153,17 +107,17 @@ pub struct StartCmd { pub gcp_project_id: Option, /// TEE authority config #[command(subcommand)] - pub tee_authority: TeeAuthorityConfig, + pub tee_authority: CliTeeAuthorityConfig, /// TEE related configuration settings. #[command(flatten)] - pub image_hash_config: MpcImageHashConfig, + pub image_hash_config: CliImageHashConfig, /// Hex-encoded 32 byte AES key for backup encryption. #[arg(env("MPC_BACKUP_ENCRYPTION_KEY_HEX"))] pub backup_encryption_key_hex: Option, } #[derive(Subcommand, Debug, Clone)] -pub enum TeeAuthorityConfig { +pub enum CliTeeAuthorityConfig { Local, Dstack { #[arg(long, env("DSTACK_ENDPOINT"), default_value = DEFAULT_DSTACK_ENDPOINT)] @@ -173,24 +127,8 @@ pub enum TeeAuthorityConfig { }, } -impl TryFrom for TeeAuthority { - type Error = anyhow::Error; - - fn try_from(cmd: TeeAuthorityConfig) -> Result { - let authority_config = match cmd { - TeeAuthorityConfig::Local => LocalTeeAuthorityConfig::default().into(), - TeeAuthorityConfig::Dstack { - dstack_endpoint, - quote_upload_url, - } => DstackTeeAuthorityConfig::new(dstack_endpoint, quote_upload_url).into(), - }; - - Ok(authority_config) - } -} - #[derive(Args, Debug)] -pub struct MpcImageHashConfig { +pub struct CliImageHashConfig { #[arg( long, env("MPC_IMAGE_HASH"), @@ -205,6 +143,65 @@ pub struct MpcImageHashConfig { pub latest_allowed_hash_file: Option, } +impl From for StartConfig { + fn from(cmd: StartCmd) -> Self { + StartConfig { + home_dir: cmd.home_dir, + secret_store_key_hex: cmd.secret_store_key_hex, + gcp_keyshare_secret_id: cmd.gcp_keyshare_secret_id, + gcp_project_id: cmd.gcp_project_id, + tee_authority: match cmd.tee_authority { + CliTeeAuthorityConfig::Local => TeeAuthorityStartConfig::Local, + CliTeeAuthorityConfig::Dstack { + dstack_endpoint, + quote_upload_url, + } => TeeAuthorityStartConfig::Dstack { + dstack_endpoint, + quote_upload_url: quote_upload_url.to_string(), + }, + }, + image_hash: cmd.image_hash_config.image_hash, + latest_allowed_hash_file: cmd.image_hash_config.latest_allowed_hash_file, + backup_encryption_key_hex: cmd.backup_encryption_key_hex, + } + } +} + +// --------------------------------------------------------------------------- +// Init subcommand +// --------------------------------------------------------------------------- + +#[derive(Args, Debug)] +pub struct InitConfigArgs { + #[arg(long, env("MPC_HOME_DIR"))] + pub dir: std::path::PathBuf, + /// chain/network id (localnet, testnet, devnet, betanet) + #[arg(long)] + pub chain_id: Option, + /// Genesis file to use when initialize testnet (including downloading) + #[arg(long)] + pub genesis: Option, + /// Download the verified NEAR config file automatically. + #[arg(long)] + pub download_config: bool, + #[arg(long)] + pub download_config_url: Option, + /// Download the verified NEAR genesis file automatically. + #[arg(long)] + pub download_genesis: bool, + /// Specify a custom download URL for the genesis-file. + #[arg(long)] + pub download_genesis_url: Option, + #[arg(long)] + pub download_genesis_records_url: Option, + #[arg(long)] + pub boot_nodes: Option, +} + +// --------------------------------------------------------------------------- +// Import/Export keyshare subcommands +// --------------------------------------------------------------------------- + #[derive(Args, Debug)] pub struct ImportKeyshareCmd { /// Path to home directory @@ -233,292 +230,17 @@ pub struct ExportKeyshareCmd { pub local_encryption_key_hex: String, } -impl StartCmd { - async fn run(self) -> anyhow::Result<()> { - let root_runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .worker_threads(1) - .build()?; - - let _tokio_enter_guard = root_runtime.enter(); - - // Load configuration and initialize persistent secrets - let home_dir = PathBuf::from(self.home_dir.clone()); - let config = load_config_file(&home_dir)?; - let persistent_secrets = PersistentSecrets::generate_or_get_existing( - &home_dir, - config.number_of_responder_keys, - )?; - - profiler::web_server::start_web_server(config.pprof_bind_address).await?; - root_runtime.spawn(crate::metrics::tokio_task_metrics::run_monitor_loop()); - - // TODO(#1296): Decide if the MPC responder account is actually needed - let respond_config = RespondConfig::from_parts(&config, &persistent_secrets); - - let backup_encryption_key_hex = match &self.backup_encryption_key_hex { - Some(key) => key.clone(), - None => generate_and_write_backup_encryption_key_to_disk(&home_dir)?, - }; - - // Load secrets from configuration and persistent storage - let secrets = SecretsConfig::from_parts( - &self.secret_store_key_hex, - persistent_secrets.clone(), - &backup_encryption_key_hex, - )?; - - // Generate attestation - let tee_authority = TeeAuthority::try_from(self.tee_authority.clone())?; - let tls_public_key = &secrets.persistent_secrets.p2p_private_key.verifying_key(); - - let account_public_key = &secrets.persistent_secrets.near_signer_key.verifying_key(); - - let report_data = ReportDataV1::new( - *Ed25519PublicKey::from(tls_public_key).as_bytes(), - *Ed25519PublicKey::from(account_public_key).as_bytes(), - ) - .into(); - - let attestation = tee_authority.generate_attestation(report_data).await?; - - // Create communication channels and runtime - let (debug_request_sender, _) = tokio::sync::broadcast::channel(10); - let root_task_handle = Arc::new(OnceLock::new()); - - let (protocol_state_sender, protocol_state_receiver) = - watch::channel(ProtocolContractState::NotInitialized); - - let (migration_state_sender, migration_state_receiver) = - watch::channel((0, BTreeMap::new())); - let web_server = root_runtime - .block_on(start_web_server( - root_task_handle.clone(), - debug_request_sender.clone(), - config.web_ui, - static_web_data(&secrets, Some(attestation)), - protocol_state_receiver, - migration_state_receiver, - )) - .context("Failed to create web server.")?; - - let _web_server_join_handle = root_runtime.spawn(web_server); - - // Create Indexer and wait for indexer to be synced. - let (indexer_exit_sender, indexer_exit_receiver) = oneshot::channel(); - let indexer_api = spawn_real_indexer( - home_dir.clone(), - config.indexer.clone(), - config.my_near_account_id.clone(), - persistent_secrets.near_signer_key.clone(), - respond_config, - indexer_exit_sender, - protocol_state_sender, - migration_state_sender, - *tls_public_key, - ); - - let (shutdown_signal_sender, mut shutdown_signal_receiver) = mpsc::channel(1); - let cancellation_token = CancellationToken::new(); - - let image_hash_watcher_handle = if let (Some(image_hash), Some(latest_allowed_hash_file)) = ( - &self.image_hash_config.image_hash, - &self.image_hash_config.latest_allowed_hash_file, - ) { - let current_image_hash_bytes: [u8; 32] = hex::decode(image_hash) - .expect("The currently running image is a hex string.") - .try_into() - .expect("The currently running image hash hex representation is 32 bytes."); - - let allowed_hashes_in_contract = indexer_api.allowed_docker_images_receiver.clone(); - let image_hash_storage = AllowedImageHashesFile::from(latest_allowed_hash_file.clone()); - - Some(root_runtime.spawn(monitor_allowed_image_hashes( - cancellation_token.child_token(), - MpcDockerImageHash::from(current_image_hash_bytes), - allowed_hashes_in_contract, - image_hash_storage, - shutdown_signal_sender.clone(), - ))) - } else { - tracing::info!( - "MPC_IMAGE_HASH and/or MPC_LATEST_ALLOWED_HASH_FILE not set, skipping TEE image hash monitoring" - ); - None - }; - - let root_future = self.create_root_future( - home_dir.clone(), - config.clone(), - secrets.clone(), - indexer_api, - debug_request_sender, - root_task_handle, - tee_authority, - ); - - let root_task = root_runtime.spawn(start_root_task("root", root_future).0); - - let exit_reason = tokio::select! { - root_task_result = root_task => { - root_task_result? - } - indexer_exit_response = indexer_exit_receiver => { - indexer_exit_response.context("Indexer thread dropped response channel.")? - } - Some(()) = shutdown_signal_receiver.recv() => { - Err(anyhow!("TEE allowed image hashes watcher is sending shutdown signal.")) - } - }; - - // Perform graceful shutdown - cancellation_token.cancel(); - - if let Some(handle) = image_hash_watcher_handle { - info!("Waiting for image hash watcher to gracefully exit."); - let exit_result = handle.await; - info!(?exit_result, "Image hash watcher exited."); - } - - exit_reason - } - - #[allow(clippy::too_many_arguments)] - async fn create_root_future( - self, - home_dir: PathBuf, - config: ConfigFile, - secrets: SecretsConfig, - indexer_api: IndexerAPI, - debug_request_sender: broadcast::Sender, - // Cloning a OnceLock returns a new cell, which is why we have to wrap it in an arc. - // Otherwise we would not write to the same cell/lock. - root_task_handle_once_lock: Arc>>, - tee_authority: TeeAuthority, - ) -> anyhow::Result<()> - where - TransactionSenderImpl: TransactionSender + 'static, - ForeignChainPolicyReader: ReadForeignChainPolicy + Clone + Send + Sync + 'static, - { - let root_task_handle = tracking::current_task(); - - root_task_handle_once_lock - .set(root_task_handle.clone()) - .map_err(|_| anyhow!("Root task handle was already set"))?; - - let tls_public_key = - Ed25519PublicKey::from(&secrets.persistent_secrets.p2p_private_key.verifying_key()); - let account_public_key = - Ed25519PublicKey::from(&secrets.persistent_secrets.near_signer_key.verifying_key()); - - let secret_db = SecretDB::new(&home_dir.join("assets"), secrets.local_storage_aes_key)?; - - let key_storage_config = KeyStorageConfig { - home_dir: home_dir.clone(), - local_encryption_key: secrets.local_storage_aes_key, - gcp: if let Some(secret_id) = self.gcp_keyshare_secret_id { - let project_id = self.gcp_project_id.ok_or_else(|| { - anyhow::anyhow!( - "GCP_PROJECT_ID must be specified to use GCP_KEYSHARE_SECRET_ID" - ) - })?; - Some(GcpPermanentKeyStorageConfig { - project_id, - secret_id, - }) - } else { - None - }, - }; - - // Spawn periodic attestation submission task - let tee_authority_clone = tee_authority.clone(); - let tx_sender_clone = indexer_api.txn_sender.clone(); - let tls_public_key_clone = tls_public_key.clone(); - let account_public_key_clone = account_public_key.clone(); - let allowed_docker_images_receiver_clone = - indexer_api.allowed_docker_images_receiver.clone(); - let allowed_launcher_compose_receiver_clone = - indexer_api.allowed_launcher_compose_receiver.clone(); - tokio::spawn(async move { - if let Err(e) = periodic_attestation_submission( - tee_authority_clone, - tx_sender_clone, - tls_public_key_clone, - account_public_key_clone, - allowed_docker_images_receiver_clone, - allowed_launcher_compose_receiver_clone, - tokio::time::interval(ATTESTATION_RESUBMISSION_INTERVAL), - ) - .await - { - tracing::error!( - error = ?e, - "periodic attestation submission task failed" - ); - } - }); - - // Spawn TEE attestation monitoring task - let tx_sender_clone = indexer_api.txn_sender.clone(); - let tee_accounts_receiver = indexer_api.attested_nodes_receiver.clone(); - let account_id_clone = config.my_near_account_id.clone(); - let allowed_docker_images_receiver_clone = - indexer_api.allowed_docker_images_receiver.clone(); - let allowed_launcher_compose_receiver_clone = - indexer_api.allowed_launcher_compose_receiver.clone(); - tokio::spawn(async move { - if let Err(e) = monitor_attestation_removal( - account_id_clone, - tee_authority, - tx_sender_clone, - tls_public_key, - account_public_key, - allowed_docker_images_receiver_clone, - allowed_launcher_compose_receiver_clone, - tee_accounts_receiver, - ) - .await - { - tracing::error!( - error = ?e, - "attestation removal monitoring task failed" - ); - } - }); - - let keyshare_storage: Arc> = - RwLock::new(key_storage_config.create().await?).into(); - - spawn_recovery_server_and_run_onboarding( - config.migration_web_ui, - (&secrets).into(), - config.my_near_account_id.clone(), - keyshare_storage.clone(), - indexer_api.my_migration_info_receiver.clone(), - indexer_api.contract_state_receiver.clone(), - indexer_api.txn_sender.clone(), - ) - .await?; - - let coordinator = Coordinator { - clock: Clock::real(), - config_file: config, - secrets, - secret_db, - keyshare_storage, - indexer: indexer_api, - currently_running_job_name: Arc::new(Mutex::new(String::new())), - debug_request_sender, - }; - coordinator.run().await - } -} +// --------------------------------------------------------------------------- +// Dispatch +// --------------------------------------------------------------------------- impl Cli { pub async fn run(self) -> anyhow::Result<()> { match self.command { - CliCommand::Start(start) => start.run().await, + CliCommand::StartWithConfigFile { config_path } => { + StartConfig::from_json_file(&config_path)?.run().await + } + CliCommand::Start(start) => StartConfig::from(start).run().await, CliCommand::Init(config) => { let (download_config_type, download_config_url) = if config.download_config { ( @@ -565,7 +287,7 @@ impl Cli { participants.len() == responders.len(), "Number of participants must match number of responders" ); - self.run_generate_test_configs( + run_generate_test_configs( output_dir, participants.clone(), responders.clone(), @@ -578,132 +300,12 @@ impl Cli { } } } - - fn duplicate_migrating_accounts( - mut accounts: Vec, - migrating_nodes: &[usize], - ) -> anyhow::Result> { - for migrating_node_idx in migrating_nodes { - let migrating_node_account: AccountId = accounts - .get(*migrating_node_idx) - .ok_or_else(|| { - anyhow::anyhow!("index {} out of bounds for accounts", migrating_node_idx) - })? - .clone(); - - accounts.push(migrating_node_account); - } - Ok(accounts) - } - - #[allow(clippy::too_many_arguments)] - fn run_generate_test_configs( - &self, - output_dir: &str, - participants: Vec, - responders: Vec, - threshold: usize, - desired_triples_to_buffer: usize, - desired_presignatures_to_buffer: usize, - desired_responder_keys_per_participant: usize, - migrating_nodes: &[usize], - ) -> anyhow::Result<()> { - let participants = Self::duplicate_migrating_accounts(participants, migrating_nodes)?; - let responders = Self::duplicate_migrating_accounts(responders, migrating_nodes)?; - - let p2p_key_pairs = participants - .iter() - .enumerate() - .map(|(idx, _account_id)| { - let subdir = PathBuf::from(output_dir).join(idx.to_string()); - PersistentSecrets::generate_or_get_existing( - &subdir, - desired_responder_keys_per_participant, - ) - .map(|secret| secret.p2p_private_key) - }) - .collect::, _>>()?; - let configs = generate_test_p2p_configs( - &participants, - threshold, - PortSeed::CLI_FOR_PYTEST, - Some(p2p_key_pairs), - )?; - let participants_config = configs[0].0.participants.clone(); - for (i, (_config, _p2p_private_key)) in configs.into_iter().enumerate() { - let subdir = format!("{}/{}", output_dir, i); - std::fs::create_dir_all(&subdir)?; - let file_config = self.create_file_config( - &participants[i], - &responders[i], - i, - desired_triples_to_buffer, - desired_presignatures_to_buffer, - )?; - std::fs::write( - format!("{}/config.yaml", subdir), - serde_yaml::to_string(&file_config)?, - )?; - } - std::fs::write( - format!("{}/participants.json", output_dir), - serde_json::to_string(&participants_config)?, - )?; - Ok(()) - } - - fn create_file_config( - &self, - participant: &AccountId, - responder: &AccountId, - index: usize, - desired_triples_to_buffer: usize, - desired_presignatures_to_buffer: usize, - ) -> anyhow::Result { - Ok(ConfigFile { - my_near_account_id: participant.clone(), - near_responder_account_id: responder.clone(), - number_of_responder_keys: 1, - web_ui: SocketAddr::new( - Ipv4Addr::LOCALHOST.into(), - PortSeed::CLI_FOR_PYTEST.web_port(index), - ), - migration_web_ui: SocketAddr::new( - Ipv4Addr::LOCALHOST.into(), - PortSeed::CLI_FOR_PYTEST.migration_web_port(index), - ), - pprof_bind_address: SocketAddr::new( - Ipv4Addr::LOCALHOST.into(), - PortSeed::CLI_FOR_PYTEST.pprof_web_port(index), - ), - indexer: IndexerConfig { - validate_genesis: true, - sync_mode: SyncMode::Block(BlockArgs { height: 0 }), - concurrency: 1.try_into().unwrap(), - mpc_contract_id: "test0".parse().unwrap(), - finality: Finality::None, - port_override: None, - }, - triple: TripleConfig { - concurrency: 2, - desired_triples_to_buffer, - timeout_sec: 60, - parallel_triple_generation_stagger_time_sec: 1, - }, - presignature: PresignatureConfig { - concurrency: 2, - desired_presignatures_to_buffer, - timeout_sec: 60, - }, - signature: SignatureConfig { timeout_sec: 60 }, - ckd: CKDConfig { timeout_sec: 60 }, - keygen: KeygenConfig { timeout_sec: 60 }, - foreign_chains: ForeignChainsConfig::default(), - cores: Some(4), - }) - } } +// --------------------------------------------------------------------------- +// Import/Export keyshare implementations +// --------------------------------------------------------------------------- + impl ImportKeyshareCmd { pub async fn run(&self) -> anyhow::Result<()> { let runtime = tokio::runtime::Runtime::new()?; @@ -801,6 +403,136 @@ impl ExportKeyshareCmd { } } +// --------------------------------------------------------------------------- +// Test config generation +// --------------------------------------------------------------------------- + +fn duplicate_migrating_accounts( + mut accounts: Vec, + migrating_nodes: &[usize], +) -> anyhow::Result> { + for migrating_node_idx in migrating_nodes { + let migrating_node_account: AccountId = accounts + .get(*migrating_node_idx) + .ok_or_else(|| { + anyhow::anyhow!("index {} out of bounds for accounts", migrating_node_idx) + })? + .clone(); + + accounts.push(migrating_node_account); + } + Ok(accounts) +} + +#[allow(clippy::too_many_arguments)] +fn run_generate_test_configs( + output_dir: &str, + participants: Vec, + responders: Vec, + threshold: usize, + desired_triples_to_buffer: usize, + desired_presignatures_to_buffer: usize, + desired_responder_keys_per_participant: usize, + migrating_nodes: &[usize], +) -> anyhow::Result<()> { + let participants = duplicate_migrating_accounts(participants, migrating_nodes)?; + let responders = duplicate_migrating_accounts(responders, migrating_nodes)?; + + let p2p_key_pairs = participants + .iter() + .enumerate() + .map(|(idx, _account_id)| { + let subdir = PathBuf::from(output_dir).join(idx.to_string()); + PersistentSecrets::generate_or_get_existing( + &subdir, + desired_responder_keys_per_participant, + ) + .map(|secret| secret.p2p_private_key) + }) + .collect::, _>>()?; + let configs = generate_test_p2p_configs( + &participants, + threshold, + PortSeed::CLI_FOR_PYTEST, + Some(p2p_key_pairs), + )?; + let participants_config = configs[0].0.participants.clone(); + for (i, (_config, _p2p_private_key)) in configs.into_iter().enumerate() { + let subdir = format!("{}/{}", output_dir, i); + std::fs::create_dir_all(&subdir)?; + let file_config = create_file_config( + &participants[i], + &responders[i], + i, + desired_triples_to_buffer, + desired_presignatures_to_buffer, + ); + std::fs::write( + format!("{}/config.yaml", subdir), + serde_yaml::to_string(&file_config)?, + )?; + } + std::fs::write( + format!("{}/participants.json", output_dir), + serde_json::to_string(&participants_config)?, + )?; + Ok(()) +} + +fn create_file_config( + participant: &AccountId, + responder: &AccountId, + index: usize, + desired_triples_to_buffer: usize, + desired_presignatures_to_buffer: usize, +) -> ConfigFile { + ConfigFile { + my_near_account_id: participant.clone(), + near_responder_account_id: responder.clone(), + number_of_responder_keys: 1, + web_ui: SocketAddr::new( + Ipv4Addr::LOCALHOST.into(), + PortSeed::CLI_FOR_PYTEST.web_port(index), + ), + migration_web_ui: SocketAddr::new( + Ipv4Addr::LOCALHOST.into(), + PortSeed::CLI_FOR_PYTEST.migration_web_port(index), + ), + pprof_bind_address: SocketAddr::new( + Ipv4Addr::LOCALHOST.into(), + PortSeed::CLI_FOR_PYTEST.pprof_web_port(index), + ), + indexer: IndexerConfig { + validate_genesis: true, + sync_mode: SyncMode::Block(BlockArgs { height: 0 }), + concurrency: 1.try_into().unwrap(), + mpc_contract_id: "test0".parse().unwrap(), + finality: Finality::None, + port_override: None, + }, + triple: TripleConfig { + concurrency: 2, + desired_triples_to_buffer, + timeout_sec: 60, + parallel_triple_generation_stagger_time_sec: 1, + }, + presignature: PresignatureConfig { + concurrency: 2, + desired_presignatures_to_buffer, + timeout_sec: 60, + }, + signature: SignatureConfig { timeout_sec: 60 }, + ckd: CKDConfig { timeout_sec: 60 }, + keygen: KeygenConfig { timeout_sec: 60 }, + foreign_chains: ForeignChainsConfig::default(), + cores: Some(4), + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + #[cfg(test)] mod tests { use super::*; diff --git a/crates/node/src/config.rs b/crates/node/src/config.rs index fdabd98bf..9969e1464 100644 --- a/crates/node/src/config.rs +++ b/crates/node/src/config.rs @@ -13,6 +13,9 @@ use std::{ path::Path, }; +mod start; +pub use start::{StartConfig, TeeAuthorityStartConfig}; + mod foreign_chains; pub use foreign_chains::{ AbstractApiVariant, AbstractChainConfig, AbstractProviderConfig, AuthConfig, BitcoinApiVariant, diff --git a/crates/node/src/config/start.rs b/crates/node/src/config/start.rs new file mode 100644 index 000000000..29a6b3a9f --- /dev/null +++ b/crates/node/src/config/start.rs @@ -0,0 +1,80 @@ +use anyhow::Context; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; +use tee_authority::tee_authority::{ + DstackTeeAuthorityConfig, LocalTeeAuthorityConfig, TeeAuthority, DEFAULT_DSTACK_ENDPOINT, + DEFAULT_PHALA_TDX_QUOTE_UPLOAD_URL, +}; +use url::Url; + +/// Configuration for starting the MPC node. This is the canonical type used +/// by the run logic. Both `StartCmd` (CLI flags) and `StartWithConfigFileCmd` +/// (JSON file) convert into this type. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StartConfig { + pub home_dir: String, + /// Hex-encoded 16 byte AES key for local storage encryption. + pub secret_store_key_hex: String, + /// If provided, the root keyshare is stored on GCP. + #[serde(default)] + pub gcp_keyshare_secret_id: Option, + #[serde(default)] + pub gcp_project_id: Option, + /// TEE authority configuration. + pub tee_authority: TeeAuthorityStartConfig, + /// Hex representation of the hash of the running image. Only required in TEE. + #[serde(default)] + pub image_hash: Option, + /// Path to the file where the node writes the latest allowed hash. + /// If not set, assumes running outside of TEE and skips image hash monitoring. + #[serde(default)] + pub latest_allowed_hash_file: Option, + /// Hex-encoded 32 byte AES key for backup encryption. + #[serde(default)] + pub backup_encryption_key_hex: Option, +} + +/// TEE authority configuration for JSON deserialization. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum TeeAuthorityStartConfig { + Local, + Dstack { + #[serde(default = "default_dstack_endpoint")] + dstack_endpoint: String, + #[serde(default = "default_quote_upload_url")] + quote_upload_url: String, + }, +} + +fn default_dstack_endpoint() -> String { + DEFAULT_DSTACK_ENDPOINT.to_string() +} + +fn default_quote_upload_url() -> String { + DEFAULT_PHALA_TDX_QUOTE_UPLOAD_URL.to_string() +} + +impl TeeAuthorityStartConfig { + pub fn into_tee_authority(self) -> anyhow::Result { + Ok(match self { + TeeAuthorityStartConfig::Local => LocalTeeAuthorityConfig::default().into(), + TeeAuthorityStartConfig::Dstack { + dstack_endpoint, + quote_upload_url, + } => { + let url: Url = quote_upload_url.parse().context("invalid quote_upload_url")?; + DstackTeeAuthorityConfig::new(dstack_endpoint, url).into() + } + }) + } +} + +impl StartConfig { + pub fn from_json_file(path: &std::path::Path) -> anyhow::Result { + let content = std::fs::read_to_string(path) + .with_context(|| format!("failed to read config file: {}", path.display()))?; + serde_json::from_str(&content) + .with_context(|| format!("failed to parse config file: {}", path.display())) + } +} diff --git a/crates/node/src/lib.rs b/crates/node/src/lib.rs index 71778f6fe..dffd498fe 100644 --- a/crates/node/src/lib.rs +++ b/crates/node/src/lib.rs @@ -39,6 +39,7 @@ mod protocol; mod protocol_version; mod providers; pub mod requests; +mod run; mod runtime; mod storage; pub mod tracing; diff --git a/crates/node/src/run.rs b/crates/node/src/run.rs new file mode 100644 index 000000000..127c749ef --- /dev/null +++ b/crates/node/src/run.rs @@ -0,0 +1,321 @@ +use crate::{ + config::{ + generate_and_write_backup_encryption_key_to_disk, load_config_file, ConfigFile, + PersistentSecrets, RespondConfig, SecretsConfig, StartConfig, + }, + coordinator::Coordinator, + db::SecretDB, + indexer::{ + real::spawn_real_indexer, tx_sender::TransactionSender, IndexerAPI, ReadForeignChainPolicy, + }, + keyshare::{GcpPermanentKeyStorageConfig, KeyStorageConfig, KeyshareStorage}, + migration_service::spawn_recovery_server_and_run_onboarding, + profiler, + tracking::{self, start_root_task}, + web::{start_web_server, static_web_data, DebugRequest}, +}; +use anyhow::{anyhow, Context}; +use contract_interface::types::Ed25519PublicKey; +use mpc_attestation::report_data::ReportDataV1; +use mpc_contract::state::ProtocolContractState; +use mpc_contract::tee::proposal::MpcDockerImageHash; +use near_time::Clock; +use std::{ + collections::BTreeMap, + path::PathBuf, + sync::{Arc, Mutex, OnceLock}, + time::Duration, +}; +use tee_authority::tee_authority::TeeAuthority; +use tokio::sync::{broadcast, mpsc, oneshot, watch, RwLock}; +use tokio_util::sync::CancellationToken; +use tracing::info; + +use crate::tee::{ + monitor_allowed_image_hashes, + remote_attestation::{monitor_attestation_removal, periodic_attestation_submission}, + AllowedImageHashesFile, +}; + +pub const ATTESTATION_RESUBMISSION_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour + +impl StartConfig { + pub async fn run(self) -> anyhow::Result<()> { + let root_runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .worker_threads(1) + .build()?; + + let _tokio_enter_guard = root_runtime.enter(); + + // Load configuration and initialize persistent secrets + let home_dir = PathBuf::from(self.home_dir.clone()); + let config = load_config_file(&home_dir)?; + let persistent_secrets = PersistentSecrets::generate_or_get_existing( + &home_dir, + config.number_of_responder_keys, + )?; + + profiler::web_server::start_web_server(config.pprof_bind_address).await?; + root_runtime.spawn(crate::metrics::tokio_task_metrics::run_monitor_loop()); + + // TODO(#1296): Decide if the MPC responder account is actually needed + let respond_config = RespondConfig::from_parts(&config, &persistent_secrets); + + let backup_encryption_key_hex = match &self.backup_encryption_key_hex { + Some(key) => key.clone(), + None => generate_and_write_backup_encryption_key_to_disk(&home_dir)?, + }; + + // Load secrets from configuration and persistent storage + let secrets = SecretsConfig::from_parts( + &self.secret_store_key_hex, + persistent_secrets.clone(), + &backup_encryption_key_hex, + )?; + + // Generate attestation + let tee_authority = self.tee_authority.clone().into_tee_authority()?; + let tls_public_key = &secrets.persistent_secrets.p2p_private_key.verifying_key(); + + let account_public_key = &secrets.persistent_secrets.near_signer_key.verifying_key(); + + let report_data = ReportDataV1::new( + *Ed25519PublicKey::from(tls_public_key).as_bytes(), + *Ed25519PublicKey::from(account_public_key).as_bytes(), + ) + .into(); + + let attestation = tee_authority.generate_attestation(report_data).await?; + + // Create communication channels and runtime + let (debug_request_sender, _) = tokio::sync::broadcast::channel(10); + let root_task_handle = Arc::new(OnceLock::new()); + + let (protocol_state_sender, protocol_state_receiver) = + watch::channel(ProtocolContractState::NotInitialized); + + let (migration_state_sender, migration_state_receiver) = + watch::channel((0, BTreeMap::new())); + let web_server = root_runtime + .block_on(start_web_server( + root_task_handle.clone(), + debug_request_sender.clone(), + config.web_ui, + static_web_data(&secrets, Some(attestation)), + protocol_state_receiver, + migration_state_receiver, + )) + .context("Failed to create web server.")?; + + let _web_server_join_handle = root_runtime.spawn(web_server); + + // Create Indexer and wait for indexer to be synced. + let (indexer_exit_sender, indexer_exit_receiver) = oneshot::channel(); + let indexer_api = spawn_real_indexer( + home_dir.clone(), + config.indexer.clone(), + config.my_near_account_id.clone(), + persistent_secrets.near_signer_key.clone(), + respond_config, + indexer_exit_sender, + protocol_state_sender, + migration_state_sender, + *tls_public_key, + ); + + let (shutdown_signal_sender, mut shutdown_signal_receiver) = mpsc::channel(1); + let cancellation_token = CancellationToken::new(); + + let image_hash_watcher_handle = + if let (Some(image_hash), Some(latest_allowed_hash_file)) = + (&self.image_hash, &self.latest_allowed_hash_file) + { + let current_image_hash_bytes: [u8; 32] = hex::decode(image_hash) + .expect("The currently running image is a hex string.") + .try_into() + .expect("The currently running image hash hex representation is 32 bytes."); + + let allowed_hashes_in_contract = + indexer_api.allowed_docker_images_receiver.clone(); + let image_hash_storage = + AllowedImageHashesFile::from(latest_allowed_hash_file.clone()); + + Some(root_runtime.spawn(monitor_allowed_image_hashes( + cancellation_token.child_token(), + MpcDockerImageHash::from(current_image_hash_bytes), + allowed_hashes_in_contract, + image_hash_storage, + shutdown_signal_sender.clone(), + ))) + } else { + tracing::info!( + "MPC_IMAGE_HASH and/or MPC_LATEST_ALLOWED_HASH_FILE not set, skipping TEE image hash monitoring" + ); + None + }; + + let root_future = create_root_future( + self, + home_dir.clone(), + config.clone(), + secrets.clone(), + indexer_api, + debug_request_sender, + root_task_handle, + tee_authority, + ); + + let root_task = root_runtime.spawn(start_root_task("root", root_future).0); + + let exit_reason = tokio::select! { + root_task_result = root_task => { + root_task_result? + } + indexer_exit_response = indexer_exit_receiver => { + indexer_exit_response.context("Indexer thread dropped response channel.")? + } + Some(()) = shutdown_signal_receiver.recv() => { + Err(anyhow!("TEE allowed image hashes watcher is sending shutdown signal.")) + } + }; + + // Perform graceful shutdown + cancellation_token.cancel(); + + if let Some(handle) = image_hash_watcher_handle { + info!("Waiting for image hash watcher to gracefully exit."); + let exit_result = handle.await; + info!(?exit_result, "Image hash watcher exited."); + } + + exit_reason + } +} + +#[allow(clippy::too_many_arguments)] +async fn create_root_future( + start_config: StartConfig, + home_dir: PathBuf, + config: ConfigFile, + secrets: SecretsConfig, + indexer_api: IndexerAPI, + debug_request_sender: broadcast::Sender, + // Cloning a OnceLock returns a new cell, which is why we have to wrap it in an arc. + // Otherwise we would not write to the same cell/lock. + root_task_handle_once_lock: Arc>>, + tee_authority: TeeAuthority, +) -> anyhow::Result<()> +where + TransactionSenderImpl: TransactionSender + 'static, + ForeignChainPolicyReader: ReadForeignChainPolicy + Clone + Send + Sync + 'static, +{ + let root_task_handle = tracking::current_task(); + + root_task_handle_once_lock + .set(root_task_handle.clone()) + .map_err(|_| anyhow!("Root task handle was already set"))?; + + let tls_public_key = + Ed25519PublicKey::from(&secrets.persistent_secrets.p2p_private_key.verifying_key()); + let account_public_key = + Ed25519PublicKey::from(&secrets.persistent_secrets.near_signer_key.verifying_key()); + + let secret_db = SecretDB::new(&home_dir.join("assets"), secrets.local_storage_aes_key)?; + + let key_storage_config = KeyStorageConfig { + home_dir: home_dir.clone(), + local_encryption_key: secrets.local_storage_aes_key, + gcp: if let Some(secret_id) = start_config.gcp_keyshare_secret_id { + let project_id = start_config.gcp_project_id.ok_or_else(|| { + anyhow::anyhow!("GCP_PROJECT_ID must be specified to use GCP_KEYSHARE_SECRET_ID") + })?; + Some(GcpPermanentKeyStorageConfig { + project_id, + secret_id, + }) + } else { + None + }, + }; + + // Spawn periodic attestation submission task + let tee_authority_clone = tee_authority.clone(); + let tx_sender_clone = indexer_api.txn_sender.clone(); + let tls_public_key_clone = tls_public_key.clone(); + let account_public_key_clone = account_public_key.clone(); + let allowed_docker_images_receiver_clone = indexer_api.allowed_docker_images_receiver.clone(); + let allowed_launcher_compose_receiver_clone = + indexer_api.allowed_launcher_compose_receiver.clone(); + tokio::spawn(async move { + if let Err(e) = periodic_attestation_submission( + tee_authority_clone, + tx_sender_clone, + tls_public_key_clone, + account_public_key_clone, + allowed_docker_images_receiver_clone, + allowed_launcher_compose_receiver_clone, + tokio::time::interval(ATTESTATION_RESUBMISSION_INTERVAL), + ) + .await + { + tracing::error!( + error = ?e, + "periodic attestation submission task failed" + ); + } + }); + + // Spawn TEE attestation monitoring task + let tx_sender_clone = indexer_api.txn_sender.clone(); + let tee_accounts_receiver = indexer_api.attested_nodes_receiver.clone(); + let account_id_clone = config.my_near_account_id.clone(); + let allowed_docker_images_receiver_clone = indexer_api.allowed_docker_images_receiver.clone(); + let allowed_launcher_compose_receiver_clone = + indexer_api.allowed_launcher_compose_receiver.clone(); + tokio::spawn(async move { + if let Err(e) = monitor_attestation_removal( + account_id_clone, + tee_authority, + tx_sender_clone, + tls_public_key, + account_public_key, + allowed_docker_images_receiver_clone, + allowed_launcher_compose_receiver_clone, + tee_accounts_receiver, + ) + .await + { + tracing::error!( + error = ?e, + "attestation removal monitoring task failed" + ); + } + }); + + let keyshare_storage: Arc> = + RwLock::new(key_storage_config.create().await?).into(); + + spawn_recovery_server_and_run_onboarding( + config.migration_web_ui, + (&secrets).into(), + config.my_near_account_id.clone(), + keyshare_storage.clone(), + indexer_api.my_migration_info_receiver.clone(), + indexer_api.contract_state_receiver.clone(), + indexer_api.txn_sender.clone(), + ) + .await?; + + let coordinator = Coordinator { + clock: Clock::real(), + config_file: config, + secrets, + secret_db, + keyshare_storage, + indexer: indexer_api, + currently_running_job_name: Arc::new(Mutex::new(String::new())), + debug_request_sender, + }; + coordinator.run().await +} diff --git a/libs/nearcore b/libs/nearcore index 8a8c21bc8..3def2f7eb 160000 --- a/libs/nearcore +++ b/libs/nearcore @@ -1 +1 @@ -Subproject commit 8a8c21bc81999af93edd1b6bca5b7c6c6337aa63 +Subproject commit 3def2f7ebb7455199e7b3f7b371e3735c23e2930 From af78d2c518e5f01c1f346081b08856d688937eac Mon Sep 17 00:00:00 2001 From: Daniel Sharifi Date: Fri, 6 Mar 2026 14:53:35 +0100 Subject: [PATCH 02/16] testing manually with localnet works --- crates/node/src/cli.rs | 58 ++-- crates/node/src/config.rs | 4 +- crates/node/src/config/foreign_chains/auth.rs | 4 + crates/node/src/config/start.rs | 50 +++- crates/node/src/run.rs | 31 +- docs/localnet/localnet.md | 276 ++++++++++-------- 6 files changed, 255 insertions(+), 168 deletions(-) diff --git a/crates/node/src/cli.rs b/crates/node/src/cli.rs index b4ca08405..a21dc1f01 100644 --- a/crates/node/src/cli.rs +++ b/crates/node/src/cli.rs @@ -1,8 +1,9 @@ use crate::{ config::{ - BlockArgs, CKDConfig, ConfigFile, ForeignChainsConfig, IndexerConfig, KeygenConfig, - PersistentSecrets, PresignatureConfig, SignatureConfig, StartConfig, SyncMode, - TeeAuthorityStartConfig, TripleConfig, + load_config_file, BlockArgs, CKDConfig, ConfigFile, ForeignChainsConfig, GcpStartConfig, + IndexerConfig, KeygenConfig, PersistentSecrets, PresignatureConfig, SecretsStartConfig, + SignatureConfig, StartConfig, SyncMode, TeeAuthorityStartConfig, TeeStartConfig, + TripleConfig, }, keyshare::{ compat::legacy_ecdsa_key_from_keyshares, @@ -143,26 +144,37 @@ pub struct CliImageHashConfig { pub latest_allowed_hash_file: Option, } -impl From for StartConfig { - fn from(cmd: StartCmd) -> Self { +impl StartCmd { + fn into_start_config(self, config: ConfigFile) -> StartConfig { + let gcp = match (self.gcp_keyshare_secret_id, self.gcp_project_id) { + (Some(keyshare_secret_id), Some(project_id)) => Some(GcpStartConfig { + keyshare_secret_id, + project_id, + }), + _ => None, + }; StartConfig { - home_dir: cmd.home_dir, - secret_store_key_hex: cmd.secret_store_key_hex, - gcp_keyshare_secret_id: cmd.gcp_keyshare_secret_id, - gcp_project_id: cmd.gcp_project_id, - tee_authority: match cmd.tee_authority { - CliTeeAuthorityConfig::Local => TeeAuthorityStartConfig::Local, - CliTeeAuthorityConfig::Dstack { - dstack_endpoint, - quote_upload_url, - } => TeeAuthorityStartConfig::Dstack { - dstack_endpoint, - quote_upload_url: quote_upload_url.to_string(), + home_dir: self.home_dir, + secrets: SecretsStartConfig { + secret_store_key_hex: self.secret_store_key_hex, + backup_encryption_key_hex: self.backup_encryption_key_hex, + }, + tee: TeeStartConfig { + authority: match self.tee_authority { + CliTeeAuthorityConfig::Local => TeeAuthorityStartConfig::Local, + CliTeeAuthorityConfig::Dstack { + dstack_endpoint, + quote_upload_url, + } => TeeAuthorityStartConfig::Dstack { + dstack_endpoint, + quote_upload_url: quote_upload_url.to_string(), + }, }, + image_hash: self.image_hash_config.image_hash, + latest_allowed_hash_file: self.image_hash_config.latest_allowed_hash_file, }, - image_hash: cmd.image_hash_config.image_hash, - latest_allowed_hash_file: cmd.image_hash_config.latest_allowed_hash_file, - backup_encryption_key_hex: cmd.backup_encryption_key_hex, + gcp, + node: config, } } } @@ -240,7 +252,11 @@ impl Cli { CliCommand::StartWithConfigFile { config_path } => { StartConfig::from_json_file(&config_path)?.run().await } - CliCommand::Start(start) => StartConfig::from(start).run().await, + CliCommand::Start(start) => { + let home_dir = std::path::Path::new(&start.home_dir); + let config_file = load_config_file(home_dir)?; + start.into_start_config(config_file).run().await + } CliCommand::Init(config) => { let (download_config_type, download_config_url) = if config.download_config { ( diff --git a/crates/node/src/config.rs b/crates/node/src/config.rs index 9969e1464..f48c9f922 100644 --- a/crates/node/src/config.rs +++ b/crates/node/src/config.rs @@ -14,7 +14,9 @@ use std::{ }; mod start; -pub use start::{StartConfig, TeeAuthorityStartConfig}; +pub use start::{ + GcpStartConfig, SecretsStartConfig, StartConfig, TeeAuthorityStartConfig, TeeStartConfig, +}; mod foreign_chains; pub use foreign_chains::{ diff --git a/crates/node/src/config/foreign_chains/auth.rs b/crates/node/src/config/foreign_chains/auth.rs index 5269b3130..48d916acd 100644 --- a/crates/node/src/config/foreign_chains/auth.rs +++ b/crates/node/src/config/foreign_chains/auth.rs @@ -52,6 +52,10 @@ pub enum TokenConfig { impl TokenConfig { pub fn resolve(&self) -> anyhow::Result { match self { + // TODO: do not resolve env variables this deep in the binary. + // Should be resolved at start, preferably in the config so we can kill env configs + // + // One option is to have a separate secrets config file. TokenConfig::Env { env } => { std::env::var(env).with_context(|| format!("environment variable {env} is not set")) } diff --git a/crates/node/src/config/start.rs b/crates/node/src/config/start.rs index 29a6b3a9f..c5ed3d802 100644 --- a/crates/node/src/config/start.rs +++ b/crates/node/src/config/start.rs @@ -1,3 +1,4 @@ +use super::ConfigFile; use anyhow::Context; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -13,15 +14,33 @@ use url::Url; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StartConfig { pub home_dir: String, + /// Encryption keys and backup settings. + pub secrets: SecretsStartConfig, + /// TEE authority and image hash monitoring settings. + pub tee: TeeStartConfig, + /// GCP keyshare storage settings. Optional — omit if not using GCP. + #[serde(default)] + pub gcp: Option, + /// Node configuration (indexer, protocol parameters, etc.). + pub node: ConfigFile, +} + +/// Encryption keys needed at startup. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SecretsStartConfig { /// Hex-encoded 16 byte AES key for local storage encryption. pub secret_store_key_hex: String, - /// If provided, the root keyshare is stored on GCP. - #[serde(default)] - pub gcp_keyshare_secret_id: Option, + /// Hex-encoded 32 byte AES key for backup encryption. + /// If not provided, a key is generated and written to disk. #[serde(default)] - pub gcp_project_id: Option, + pub backup_encryption_key_hex: Option, +} + +/// TEE-related configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TeeStartConfig { /// TEE authority configuration. - pub tee_authority: TeeAuthorityStartConfig, + pub authority: TeeAuthorityStartConfig, /// Hex representation of the hash of the running image. Only required in TEE. #[serde(default)] pub image_hash: Option, @@ -29,9 +48,15 @@ pub struct StartConfig { /// If not set, assumes running outside of TEE and skips image hash monitoring. #[serde(default)] pub latest_allowed_hash_file: Option, - /// Hex-encoded 32 byte AES key for backup encryption. - #[serde(default)] - pub backup_encryption_key_hex: Option, +} + +/// GCP keyshare storage configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GcpStartConfig { + /// GCP secret ID for storing the root keyshare. + pub keyshare_secret_id: String, + /// GCP project ID. + pub project_id: String, } /// TEE authority configuration for JSON deserialization. @@ -74,7 +99,12 @@ impl StartConfig { pub fn from_json_file(path: &std::path::Path) -> anyhow::Result { let content = std::fs::read_to_string(path) .with_context(|| format!("failed to read config file: {}", path.display()))?; - serde_json::from_str(&content) - .with_context(|| format!("failed to parse config file: {}", path.display())) + let config: Self = serde_json::from_str(&content) + .with_context(|| format!("failed to parse config file: {}", path.display()))?; + config + .node + .validate() + .context("invalid node config in config file")?; + Ok(config) } } diff --git a/crates/node/src/run.rs b/crates/node/src/run.rs index 127c749ef..263b30a56 100644 --- a/crates/node/src/run.rs +++ b/crates/node/src/run.rs @@ -1,7 +1,7 @@ use crate::{ config::{ - generate_and_write_backup_encryption_key_to_disk, load_config_file, ConfigFile, - PersistentSecrets, RespondConfig, SecretsConfig, StartConfig, + generate_and_write_backup_encryption_key_to_disk, ConfigFile, PersistentSecrets, + RespondConfig, SecretsConfig, StartConfig, }, coordinator::Coordinator, db::SecretDB, @@ -50,7 +50,7 @@ impl StartConfig { // Load configuration and initialize persistent secrets let home_dir = PathBuf::from(self.home_dir.clone()); - let config = load_config_file(&home_dir)?; + let config = self.node.clone(); let persistent_secrets = PersistentSecrets::generate_or_get_existing( &home_dir, config.number_of_responder_keys, @@ -62,20 +62,20 @@ impl StartConfig { // TODO(#1296): Decide if the MPC responder account is actually needed let respond_config = RespondConfig::from_parts(&config, &persistent_secrets); - let backup_encryption_key_hex = match &self.backup_encryption_key_hex { + let backup_encryption_key_hex = match &self.secrets.backup_encryption_key_hex { Some(key) => key.clone(), None => generate_and_write_backup_encryption_key_to_disk(&home_dir)?, }; // Load secrets from configuration and persistent storage let secrets = SecretsConfig::from_parts( - &self.secret_store_key_hex, + &self.secrets.secret_store_key_hex, persistent_secrets.clone(), &backup_encryption_key_hex, )?; // Generate attestation - let tee_authority = self.tee_authority.clone().into_tee_authority()?; + let tee_authority = self.tee.authority.clone().into_tee_authority()?; let tls_public_key = &secrets.persistent_secrets.p2p_private_key.verifying_key(); let account_public_key = &secrets.persistent_secrets.near_signer_key.verifying_key(); @@ -129,7 +129,7 @@ impl StartConfig { let image_hash_watcher_handle = if let (Some(image_hash), Some(latest_allowed_hash_file)) = - (&self.image_hash, &self.latest_allowed_hash_file) + (&self.tee.image_hash, &self.tee.latest_allowed_hash_file) { let current_image_hash_bytes: [u8; 32] = hex::decode(image_hash) .expect("The currently running image is a hex string.") @@ -150,7 +150,7 @@ impl StartConfig { ))) } else { tracing::info!( - "MPC_IMAGE_HASH and/or MPC_LATEST_ALLOWED_HASH_FILE not set, skipping TEE image hash monitoring" + "image_hash and/or latest_allowed_hash_file not set, skipping TEE image hash monitoring" ); None }; @@ -226,17 +226,10 @@ where let key_storage_config = KeyStorageConfig { home_dir: home_dir.clone(), local_encryption_key: secrets.local_storage_aes_key, - gcp: if let Some(secret_id) = start_config.gcp_keyshare_secret_id { - let project_id = start_config.gcp_project_id.ok_or_else(|| { - anyhow::anyhow!("GCP_PROJECT_ID must be specified to use GCP_KEYSHARE_SECRET_ID") - })?; - Some(GcpPermanentKeyStorageConfig { - project_id, - secret_id, - }) - } else { - None - }, + gcp: start_config.gcp.map(|gcp| GcpPermanentKeyStorageConfig { + project_id: gcp.project_id, + secret_id: gcp.keyshare_secret_id, + }), }; // Spawn periodic attestation submission task diff --git a/docs/localnet/localnet.md b/docs/localnet/localnet.md index 24dc69ce8..fd55b62e4 100644 --- a/docs/localnet/localnet.md +++ b/docs/localnet/localnet.md @@ -191,64 +191,86 @@ Since this is not a validator node, we can remove `validator_key.json` rm ~/.near/mpc-frodo/validator_key.json ``` -Next we'll create a `config.yaml` for the MPC-indexer: - -```shell -cat > ~/.near/mpc-frodo/config.yaml << 'EOF' -my_near_account_id: frodo.test.near -near_responder_account_id: frodo.test.near -number_of_responder_keys: 1 -web_ui: 127.0.0.1:8081 -migration_web_ui: 127.0.0.1:8079 -pprof_bind_address: 127.0.0.1:34001 -triple: - concurrency: 2 - desired_triples_to_buffer: 128 - timeout_sec: 60 - parallel_triple_generation_stagger_time_sec: 1 -presignature: - concurrency: 4 - desired_presignatures_to_buffer: 64 - timeout_sec: 60 -signature: - timeout_sec: 60 -indexer: - validate_genesis: false - sync_mode: Latest - concurrency: 1 - mpc_contract_id: mpc-contract.test.near - finality: optimistic -ckd: - timeout_sec: 60 -cores: 4 -foreign_chains: - bitcoin: - timeout_sec: 30 - max_retries: 3 - providers: - public: - api_variant: esplora - rpc_url: "https://bitcoin-rpc.publicnode.com" - auth: - kind: none - abstract: - timeout_sec: 30 - max_retries: 3 - providers: - public: - api_variant: standard - rpc_url: "https://api.testnet.abs.xyz" - auth: - kind: none - starknet: - timeout_sec: 30 - max_retries: 3 - providers: - public: - api_variant: standard - rpc_url: "https://starknet-rpc.publicnode.com" - auth: - kind: none +Next we'll create a JSON configuration file for Frodo's MPC node. This single file +contains all settings (secrets, TEE config, and node parameters): + +```shell +cat > ~/.near/mpc-frodo/mpc-config.json << EOF +{ + "home_dir": "$HOME/.near/mpc-frodo", + "secrets": { + "secret_store_key_hex": "11111111111111111111111111111111" + }, + "tee": { + "authority": { "type": "local" }, + "image_hash": "8b40f81f77b8c22d6c777a6e14d307a1d11cb55ab83541fbb8575d02d86a74b0", + "latest_allowed_hash_file": "/tmp/LATEST_ALLOWED_HASH_FILE.txt" + }, + "node": { + "my_near_account_id": "frodo.test.near", + "near_responder_account_id": "frodo.test.near", + "number_of_responder_keys": 1, + "web_ui": "127.0.0.1:8081", + "migration_web_ui": "127.0.0.1:8079", + "pprof_bind_address": "127.0.0.1:34001", + "triple": { + "concurrency": 2, + "desired_triples_to_buffer": 128, + "timeout_sec": 60, + "parallel_triple_generation_stagger_time_sec": 1 + }, + "presignature": { + "concurrency": 4, + "desired_presignatures_to_buffer": 64, + "timeout_sec": 60 + }, + "signature": { "timeout_sec": 60 }, + "indexer": { + "validate_genesis": false, + "sync_mode": "Latest", + "concurrency": 1, + "mpc_contract_id": "mpc-contract.test.near", + "finality": "optimistic" + }, + "ckd": { "timeout_sec": 60 }, + "cores": 4, + "foreign_chains": { + "bitcoin": { + "timeout_sec": 30, + "max_retries": 3, + "providers": { + "public": { + "api_variant": "esplora", + "rpc_url": "https://bitcoin-rpc.publicnode.com", + "auth": { "kind": "none" } + } + } + }, + "abstract": { + "timeout_sec": 30, + "max_retries": 3, + "providers": { + "public": { + "api_variant": "standard", + "rpc_url": "https://api.testnet.abs.xyz", + "auth": { "kind": "none" } + } + } + }, + "starknet": { + "timeout_sec": 30, + "max_retries": 3, + "providers": { + "public": { + "api_variant": "standard", + "rpc_url": "https://starknet-rpc.publicnode.com", + "auth": { "kind": "none" } + } + } + } + } + } +} EOF ``` @@ -273,79 +295,99 @@ rm ~/.near/mpc-sam/validator_key.json ``` ```shell -cat > ~/.near/mpc-sam/config.yaml << 'EOF' -my_near_account_id: sam.test.near -near_responder_account_id: sam.test.near -number_of_responder_keys: 1 -web_ui: 127.0.0.1:8082 -migration_web_ui: 127.0.0.1:8078 -pprof_bind_address: 127.0.0.1:34002 -triple: - concurrency: 2 - desired_triples_to_buffer: 128 - timeout_sec: 60 - parallel_triple_generation_stagger_time_sec: 1 -presignature: - concurrency: 4 - desired_presignatures_to_buffer: 64 - timeout_sec: 60 -signature: - timeout_sec: 60 -indexer: - validate_genesis: false - sync_mode: Latest - concurrency: 1 - mpc_contract_id: mpc-contract.test.near - finality: optimistic -ckd: - timeout_sec: 60 -cores: 4 -foreign_chains: - bitcoin: - timeout_sec: 30 - max_retries: 3 - providers: - public: - api_variant: esplora - rpc_url: "https://bitcoin-rpc.publicnode.com" - auth: - kind: none - abstract: - timeout_sec: 30 - max_retries: 3 - providers: - public: - api_variant: standard - rpc_url: "https://api.testnet.abs.xyz" - auth: - kind: none - starknet: - timeout_sec: 30 - max_retries: 3 - providers: - public: - api_variant: standard - rpc_url: "https://starknet-rpc.publicnode.com" - auth: - kind: none +cat > ~/.near/mpc-sam/mpc-config.json << EOF +{ + "home_dir": "$HOME/.near/mpc-sam", + "secrets": { + "secret_store_key_hex": "11111111111111111111111111111111" + }, + "tee": { + "authority": { "type": "local" }, + "image_hash": "8b40f81f77b8c22d6c777a6e14d307a1d11cb55ab83541fbb8575d02d86a74b0", + "latest_allowed_hash_file": "/tmp/LATEST_ALLOWED_HASH_FILE.txt" + }, + "node": { + "my_near_account_id": "sam.test.near", + "near_responder_account_id": "sam.test.near", + "number_of_responder_keys": 1, + "web_ui": "127.0.0.1:8082", + "migration_web_ui": "127.0.0.1:8078", + "pprof_bind_address": "127.0.0.1:34002", + "triple": { + "concurrency": 2, + "desired_triples_to_buffer": 128, + "timeout_sec": 60, + "parallel_triple_generation_stagger_time_sec": 1 + }, + "presignature": { + "concurrency": 4, + "desired_presignatures_to_buffer": 64, + "timeout_sec": 60 + }, + "signature": { "timeout_sec": 60 }, + "indexer": { + "validate_genesis": false, + "sync_mode": "Latest", + "concurrency": 1, + "mpc_contract_id": "mpc-contract.test.near", + "finality": "optimistic" + }, + "ckd": { "timeout_sec": 60 }, + "cores": 4, + "foreign_chains": { + "bitcoin": { + "timeout_sec": 30, + "max_retries": 3, + "providers": { + "public": { + "api_variant": "esplora", + "rpc_url": "https://bitcoin-rpc.publicnode.com", + "auth": { "kind": "none" } + } + } + }, + "abstract": { + "timeout_sec": 30, + "max_retries": 3, + "providers": { + "public": { + "api_variant": "standard", + "rpc_url": "https://api.testnet.abs.xyz", + "auth": { "kind": "none" } + } + } + }, + "starknet": { + "timeout_sec": 30, + "max_retries": 3, + "providers": { + "public": { + "api_variant": "standard", + "rpc_url": "https://starknet-rpc.publicnode.com", + "auth": { "kind": "none" } + } + } + } + } + } +} EOF ``` ### Run the MPC binary -In two separate shells run the MPC binary for frodo and sam. Note the last argument repeating (`11111111111111111111111111111111`) is the encryption key for the secret storage, and can be any arbitrary value. +In two separate shells run the MPC binary for Frodo and Sam using their JSON config files: ```shell -RUST_LOG=info mpc-node start --home-dir ~/.near/mpc-sam/ 11111111111111111111111111111111 --image-hash "8b40f81f77b8c22d6c777a6e14d307a1d11cb55ab83541fbb8575d02d86a74b0" --latest-allowed-hash-file /temp/LATEST_ALLOWED_HASH_FILE.txt local +RUST_LOG=info mpc-node start-with-config-file ~/.near/mpc-sam/mpc-config.json ``` ```shell -RUST_LOG=info mpc-node start --home-dir ~/.near/mpc-frodo/ 11111111111111111111111111111111 --image-hash "8b40f81f77b8c22d6c777a6e14d307a1d11cb55ab83541fbb8575d02d86a74b0" --latest-allowed-hash-file /temp/LATEST_ALLOWED_HASH_FILE.txt local +RUST_LOG=info mpc-node start-with-config-file ~/.near/mpc-frodo/mpc-config.json ``` Notes: -- `8b40f81f77b8c22d6c777a6e14d307a1d11cb55ab83541fbb8575d02d86a74b0` is just an arbitrary hash. - If you get the following error: ```console From ae8e459a986bc3e9745880c555e3e0ef929d3898 Mon Sep 17 00:00:00 2001 From: Daniel Sharifi Date: Fri, 6 Mar 2026 14:59:38 +0100 Subject: [PATCH 03/16] use envsubst --- docs/localnet/localnet.md | 161 +------------------------ docs/localnet/mpc-config.template.json | 75 ++++++++++++ 2 files changed, 81 insertions(+), 155 deletions(-) create mode 100644 docs/localnet/mpc-config.template.json diff --git a/docs/localnet/localnet.md b/docs/localnet/localnet.md index fd55b62e4..e9c0674ef 100644 --- a/docs/localnet/localnet.md +++ b/docs/localnet/localnet.md @@ -191,87 +191,13 @@ Since this is not a validator node, we can remove `validator_key.json` rm ~/.near/mpc-frodo/validator_key.json ``` -Next we'll create a JSON configuration file for Frodo's MPC node. This single file +Next we'll create a JSON configuration file for Frodo's MPC node using the +shared template at `docs/localnet/mpc-config.template.json`. This single file contains all settings (secrets, TEE config, and node parameters): ```shell -cat > ~/.near/mpc-frodo/mpc-config.json << EOF -{ - "home_dir": "$HOME/.near/mpc-frodo", - "secrets": { - "secret_store_key_hex": "11111111111111111111111111111111" - }, - "tee": { - "authority": { "type": "local" }, - "image_hash": "8b40f81f77b8c22d6c777a6e14d307a1d11cb55ab83541fbb8575d02d86a74b0", - "latest_allowed_hash_file": "/tmp/LATEST_ALLOWED_HASH_FILE.txt" - }, - "node": { - "my_near_account_id": "frodo.test.near", - "near_responder_account_id": "frodo.test.near", - "number_of_responder_keys": 1, - "web_ui": "127.0.0.1:8081", - "migration_web_ui": "127.0.0.1:8079", - "pprof_bind_address": "127.0.0.1:34001", - "triple": { - "concurrency": 2, - "desired_triples_to_buffer": 128, - "timeout_sec": 60, - "parallel_triple_generation_stagger_time_sec": 1 - }, - "presignature": { - "concurrency": 4, - "desired_presignatures_to_buffer": 64, - "timeout_sec": 60 - }, - "signature": { "timeout_sec": 60 }, - "indexer": { - "validate_genesis": false, - "sync_mode": "Latest", - "concurrency": 1, - "mpc_contract_id": "mpc-contract.test.near", - "finality": "optimistic" - }, - "ckd": { "timeout_sec": 60 }, - "cores": 4, - "foreign_chains": { - "bitcoin": { - "timeout_sec": 30, - "max_retries": 3, - "providers": { - "public": { - "api_variant": "esplora", - "rpc_url": "https://bitcoin-rpc.publicnode.com", - "auth": { "kind": "none" } - } - } - }, - "abstract": { - "timeout_sec": 30, - "max_retries": 3, - "providers": { - "public": { - "api_variant": "standard", - "rpc_url": "https://api.testnet.abs.xyz", - "auth": { "kind": "none" } - } - } - }, - "starknet": { - "timeout_sec": 30, - "max_retries": 3, - "providers": { - "public": { - "api_variant": "standard", - "rpc_url": "https://starknet-rpc.publicnode.com", - "auth": { "kind": "none" } - } - } - } - } - } -} -EOF +MPC_NODE_ID=mpc-frodo NEAR_ACCOUNT_ID=frodo.test.near WEB_UI_PORT=8081 MIGRATION_WEB_UI_PORT=8079 PPROF_PORT=34001 \ + envsubst < docs/localnet/mpc-config.template.json > ~/.near/mpc-frodo/mpc-config.json ``` ### Initialize Sam's node @@ -295,83 +221,8 @@ rm ~/.near/mpc-sam/validator_key.json ``` ```shell -cat > ~/.near/mpc-sam/mpc-config.json << EOF -{ - "home_dir": "$HOME/.near/mpc-sam", - "secrets": { - "secret_store_key_hex": "11111111111111111111111111111111" - }, - "tee": { - "authority": { "type": "local" }, - "image_hash": "8b40f81f77b8c22d6c777a6e14d307a1d11cb55ab83541fbb8575d02d86a74b0", - "latest_allowed_hash_file": "/tmp/LATEST_ALLOWED_HASH_FILE.txt" - }, - "node": { - "my_near_account_id": "sam.test.near", - "near_responder_account_id": "sam.test.near", - "number_of_responder_keys": 1, - "web_ui": "127.0.0.1:8082", - "migration_web_ui": "127.0.0.1:8078", - "pprof_bind_address": "127.0.0.1:34002", - "triple": { - "concurrency": 2, - "desired_triples_to_buffer": 128, - "timeout_sec": 60, - "parallel_triple_generation_stagger_time_sec": 1 - }, - "presignature": { - "concurrency": 4, - "desired_presignatures_to_buffer": 64, - "timeout_sec": 60 - }, - "signature": { "timeout_sec": 60 }, - "indexer": { - "validate_genesis": false, - "sync_mode": "Latest", - "concurrency": 1, - "mpc_contract_id": "mpc-contract.test.near", - "finality": "optimistic" - }, - "ckd": { "timeout_sec": 60 }, - "cores": 4, - "foreign_chains": { - "bitcoin": { - "timeout_sec": 30, - "max_retries": 3, - "providers": { - "public": { - "api_variant": "esplora", - "rpc_url": "https://bitcoin-rpc.publicnode.com", - "auth": { "kind": "none" } - } - } - }, - "abstract": { - "timeout_sec": 30, - "max_retries": 3, - "providers": { - "public": { - "api_variant": "standard", - "rpc_url": "https://api.testnet.abs.xyz", - "auth": { "kind": "none" } - } - } - }, - "starknet": { - "timeout_sec": 30, - "max_retries": 3, - "providers": { - "public": { - "api_variant": "standard", - "rpc_url": "https://starknet-rpc.publicnode.com", - "auth": { "kind": "none" } - } - } - } - } - } -} -EOF +MPC_NODE_ID=mpc-sam NEAR_ACCOUNT_ID=sam.test.near WEB_UI_PORT=8082 MIGRATION_WEB_UI_PORT=8078 PPROF_PORT=34002 \ + envsubst < docs/localnet/mpc-config.template.json > ~/.near/mpc-sam/mpc-config.json ``` ### Run the MPC binary diff --git a/docs/localnet/mpc-config.template.json b/docs/localnet/mpc-config.template.json new file mode 100644 index 000000000..07e15ff27 --- /dev/null +++ b/docs/localnet/mpc-config.template.json @@ -0,0 +1,75 @@ +{ + "home_dir": "$HOME/.near/$MPC_NODE_ID", + "secrets": { + "secret_store_key_hex": "11111111111111111111111111111111" + }, + "tee": { + "authority": { "type": "local" }, + "image_hash": "8b40f81f77b8c22d6c777a6e14d307a1d11cb55ab83541fbb8575d02d86a74b0", + "latest_allowed_hash_file": "/tmp/LATEST_ALLOWED_HASH_FILE.txt" + }, + "node": { + "my_near_account_id": "$NEAR_ACCOUNT_ID", + "near_responder_account_id": "$NEAR_ACCOUNT_ID", + "number_of_responder_keys": 1, + "web_ui": "127.0.0.1:$WEB_UI_PORT", + "migration_web_ui": "127.0.0.1:$MIGRATION_WEB_UI_PORT", + "pprof_bind_address": "127.0.0.1:$PPROF_PORT", + "triple": { + "concurrency": 2, + "desired_triples_to_buffer": 128, + "timeout_sec": 60, + "parallel_triple_generation_stagger_time_sec": 1 + }, + "presignature": { + "concurrency": 4, + "desired_presignatures_to_buffer": 64, + "timeout_sec": 60 + }, + "signature": { "timeout_sec": 60 }, + "indexer": { + "validate_genesis": false, + "sync_mode": "Latest", + "concurrency": 1, + "mpc_contract_id": "mpc-contract.test.near", + "finality": "optimistic" + }, + "ckd": { "timeout_sec": 60 }, + "cores": 4, + "foreign_chains": { + "bitcoin": { + "timeout_sec": 30, + "max_retries": 3, + "providers": { + "public": { + "api_variant": "esplora", + "rpc_url": "https://bitcoin-rpc.publicnode.com", + "auth": { "kind": "none" } + } + } + }, + "abstract": { + "timeout_sec": 30, + "max_retries": 3, + "providers": { + "public": { + "api_variant": "standard", + "rpc_url": "https://api.testnet.abs.xyz", + "auth": { "kind": "none" } + } + } + }, + "starknet": { + "timeout_sec": 30, + "max_retries": 3, + "providers": { + "public": { + "api_variant": "standard", + "rpc_url": "https://starknet-rpc.publicnode.com", + "auth": { "kind": "none" } + } + } + } + } + } +} From a989b042101605fc016cf44d55af3500414ea498 Mon Sep 17 00:00:00 2001 From: Daniel Sharifi Date: Fri, 6 Mar 2026 14:59:52 +0100 Subject: [PATCH 04/16] make it portable for fish --- docs/localnet/localnet.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/localnet/localnet.md b/docs/localnet/localnet.md index e9c0674ef..90045af21 100644 --- a/docs/localnet/localnet.md +++ b/docs/localnet/localnet.md @@ -196,7 +196,7 @@ shared template at `docs/localnet/mpc-config.template.json`. This single file contains all settings (secrets, TEE config, and node parameters): ```shell -MPC_NODE_ID=mpc-frodo NEAR_ACCOUNT_ID=frodo.test.near WEB_UI_PORT=8081 MIGRATION_WEB_UI_PORT=8079 PPROF_PORT=34001 \ +env MPC_NODE_ID=mpc-frodo NEAR_ACCOUNT_ID=frodo.test.near WEB_UI_PORT=8081 MIGRATION_WEB_UI_PORT=8079 PPROF_PORT=34001 \ envsubst < docs/localnet/mpc-config.template.json > ~/.near/mpc-frodo/mpc-config.json ``` @@ -221,7 +221,7 @@ rm ~/.near/mpc-sam/validator_key.json ``` ```shell -MPC_NODE_ID=mpc-sam NEAR_ACCOUNT_ID=sam.test.near WEB_UI_PORT=8082 MIGRATION_WEB_UI_PORT=8078 PPROF_PORT=34002 \ +env MPC_NODE_ID=mpc-sam NEAR_ACCOUNT_ID=sam.test.near WEB_UI_PORT=8082 MIGRATION_WEB_UI_PORT=8078 PPROF_PORT=34002 \ envsubst < docs/localnet/mpc-config.template.json > ~/.near/mpc-sam/mpc-config.json ``` From 62efec6044f63143c3b1192dd00fb4b2749e8857 Mon Sep 17 00:00:00 2001 From: Daniel Sharifi Date: Fri, 6 Mar 2026 15:05:55 +0100 Subject: [PATCH 05/16] remove section comments --- crates/node/src/cli.rs | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/crates/node/src/cli.rs b/crates/node/src/cli.rs index a21dc1f01..0ea6c4811 100644 --- a/crates/node/src/cli.rs +++ b/crates/node/src/cli.rs @@ -22,11 +22,6 @@ use std::{ }; use tee_authority::tee_authority::{DEFAULT_DSTACK_ENDPOINT, DEFAULT_PHALA_TDX_QUOTE_UPLOAD_URL}; use url::Url; - -// --------------------------------------------------------------------------- -// Top-level CLI -// --------------------------------------------------------------------------- - #[derive(Parser, Debug)] #[command(name = "mpc-node")] #[command(about = "MPC Node for Near Protocol")] @@ -86,11 +81,6 @@ pub enum CliCommand { migrating_nodes: Vec, }, } - -// --------------------------------------------------------------------------- -// Start subcommand (CLI flags / env vars) -// --------------------------------------------------------------------------- - #[derive(Args, Debug)] pub struct StartCmd { #[arg(long, env("MPC_HOME_DIR"))] @@ -178,11 +168,6 @@ impl StartCmd { } } } - -// --------------------------------------------------------------------------- -// Init subcommand -// --------------------------------------------------------------------------- - #[derive(Args, Debug)] pub struct InitConfigArgs { #[arg(long, env("MPC_HOME_DIR"))] @@ -209,11 +194,6 @@ pub struct InitConfigArgs { #[arg(long)] pub boot_nodes: Option, } - -// --------------------------------------------------------------------------- -// Import/Export keyshare subcommands -// --------------------------------------------------------------------------- - #[derive(Args, Debug)] pub struct ImportKeyshareCmd { /// Path to home directory @@ -241,11 +221,6 @@ pub struct ExportKeyshareCmd { #[arg(help = "Hex-encoded 16 byte AES key for local storage encryption")] pub local_encryption_key_hex: String, } - -// --------------------------------------------------------------------------- -// Dispatch -// --------------------------------------------------------------------------- - impl Cli { pub async fn run(self) -> anyhow::Result<()> { match self.command { @@ -317,11 +292,6 @@ impl Cli { } } } - -// --------------------------------------------------------------------------- -// Import/Export keyshare implementations -// --------------------------------------------------------------------------- - impl ImportKeyshareCmd { pub async fn run(&self) -> anyhow::Result<()> { let runtime = tokio::runtime::Runtime::new()?; @@ -418,11 +388,6 @@ impl ExportKeyshareCmd { }) } } - -// --------------------------------------------------------------------------- -// Test config generation -// --------------------------------------------------------------------------- - fn duplicate_migrating_accounts( mut accounts: Vec, migrating_nodes: &[usize], @@ -544,11 +509,6 @@ fn create_file_config( cores: Some(4), } } - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - #[cfg(test)] mod tests { use super::*; From 50f27e069a2e3b32b365336002d260b86be6a7dd Mon Sep 17 00:00:00 2001 From: Daniel Sharifi Date: Fri, 6 Mar 2026 15:06:15 +0100 Subject: [PATCH 06/16] cargo fmt --- crates/node/src/config/start.rs | 4 ++- crates/node/src/run.rs | 47 +++++++++++++++------------------ 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/crates/node/src/config/start.rs b/crates/node/src/config/start.rs index c5ed3d802..8ee8ea9a7 100644 --- a/crates/node/src/config/start.rs +++ b/crates/node/src/config/start.rs @@ -88,7 +88,9 @@ impl TeeAuthorityStartConfig { dstack_endpoint, quote_upload_url, } => { - let url: Url = quote_upload_url.parse().context("invalid quote_upload_url")?; + let url: Url = quote_upload_url + .parse() + .context("invalid quote_upload_url")?; DstackTeeAuthorityConfig::new(dstack_endpoint, url).into() } }) diff --git a/crates/node/src/run.rs b/crates/node/src/run.rs index 263b30a56..9c033f305 100644 --- a/crates/node/src/run.rs +++ b/crates/node/src/run.rs @@ -127,33 +127,30 @@ impl StartConfig { let (shutdown_signal_sender, mut shutdown_signal_receiver) = mpsc::channel(1); let cancellation_token = CancellationToken::new(); - let image_hash_watcher_handle = - if let (Some(image_hash), Some(latest_allowed_hash_file)) = - (&self.tee.image_hash, &self.tee.latest_allowed_hash_file) - { - let current_image_hash_bytes: [u8; 32] = hex::decode(image_hash) - .expect("The currently running image is a hex string.") - .try_into() - .expect("The currently running image hash hex representation is 32 bytes."); - - let allowed_hashes_in_contract = - indexer_api.allowed_docker_images_receiver.clone(); - let image_hash_storage = - AllowedImageHashesFile::from(latest_allowed_hash_file.clone()); - - Some(root_runtime.spawn(monitor_allowed_image_hashes( - cancellation_token.child_token(), - MpcDockerImageHash::from(current_image_hash_bytes), - allowed_hashes_in_contract, - image_hash_storage, - shutdown_signal_sender.clone(), - ))) - } else { - tracing::info!( + let image_hash_watcher_handle = if let (Some(image_hash), Some(latest_allowed_hash_file)) = + (&self.tee.image_hash, &self.tee.latest_allowed_hash_file) + { + let current_image_hash_bytes: [u8; 32] = hex::decode(image_hash) + .expect("The currently running image is a hex string.") + .try_into() + .expect("The currently running image hash hex representation is 32 bytes."); + + let allowed_hashes_in_contract = indexer_api.allowed_docker_images_receiver.clone(); + let image_hash_storage = AllowedImageHashesFile::from(latest_allowed_hash_file.clone()); + + Some(root_runtime.spawn(monitor_allowed_image_hashes( + cancellation_token.child_token(), + MpcDockerImageHash::from(current_image_hash_bytes), + allowed_hashes_in_contract, + image_hash_storage, + shutdown_signal_sender.clone(), + ))) + } else { + tracing::info!( "image_hash and/or latest_allowed_hash_file not set, skipping TEE image hash monitoring" ); - None - }; + None + }; let root_future = create_root_future( self, From b5aedee8bddcaeffeecd002c2fc7832ca805cd1b Mon Sep 17 00:00:00 2001 From: Daniel Sharifi Date: Fri, 6 Mar 2026 15:20:15 +0100 Subject: [PATCH 07/16] Have run as standalone function --- crates/node/src/cli.rs | 8 +- crates/node/src/run.rs | 285 ++++++++++++++++++++--------------------- 2 files changed, 147 insertions(+), 146 deletions(-) diff --git a/crates/node/src/cli.rs b/crates/node/src/cli.rs index 0ea6c4811..2b1aa8abf 100644 --- a/crates/node/src/cli.rs +++ b/crates/node/src/cli.rs @@ -11,6 +11,7 @@ use crate::{ permanent::{PermanentKeyStorage, PermanentKeyStorageBackend, PermanentKeyshareData}, }, p2p::testing::{generate_test_p2p_configs, PortSeed}, + run::run, }; use clap::{Args, Parser, Subcommand, ValueEnum}; use hex::FromHex; @@ -225,12 +226,15 @@ impl Cli { pub async fn run(self) -> anyhow::Result<()> { match self.command { CliCommand::StartWithConfigFile { config_path } => { - StartConfig::from_json_file(&config_path)?.run().await + let node_configuration = StartConfig::from_json_file(&config_path)?; + run(node_configuration).await } CliCommand::Start(start) => { let home_dir = std::path::Path::new(&start.home_dir); let config_file = load_config_file(home_dir)?; - start.into_start_config(config_file).run().await + + let node_configuration = start.into_start_config(config_file); + run(node_configuration).await } CliCommand::Init(config) => { let (download_config_type, download_config_url) = if config.download_config { diff --git a/crates/node/src/run.rs b/crates/node/src/run.rs index 9c033f305..f403a299f 100644 --- a/crates/node/src/run.rs +++ b/crates/node/src/run.rs @@ -39,155 +39,152 @@ use crate::tee::{ pub const ATTESTATION_RESUBMISSION_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour -impl StartConfig { - pub async fn run(self) -> anyhow::Result<()> { - let root_runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .worker_threads(1) - .build()?; - - let _tokio_enter_guard = root_runtime.enter(); - - // Load configuration and initialize persistent secrets - let home_dir = PathBuf::from(self.home_dir.clone()); - let config = self.node.clone(); - let persistent_secrets = PersistentSecrets::generate_or_get_existing( - &home_dir, - config.number_of_responder_keys, - )?; - - profiler::web_server::start_web_server(config.pprof_bind_address).await?; - root_runtime.spawn(crate::metrics::tokio_task_metrics::run_monitor_loop()); - - // TODO(#1296): Decide if the MPC responder account is actually needed - let respond_config = RespondConfig::from_parts(&config, &persistent_secrets); - - let backup_encryption_key_hex = match &self.secrets.backup_encryption_key_hex { - Some(key) => key.clone(), - None => generate_and_write_backup_encryption_key_to_disk(&home_dir)?, - }; - - // Load secrets from configuration and persistent storage - let secrets = SecretsConfig::from_parts( - &self.secrets.secret_store_key_hex, - persistent_secrets.clone(), - &backup_encryption_key_hex, - )?; - - // Generate attestation - let tee_authority = self.tee.authority.clone().into_tee_authority()?; - let tls_public_key = &secrets.persistent_secrets.p2p_private_key.verifying_key(); - - let account_public_key = &secrets.persistent_secrets.near_signer_key.verifying_key(); - - let report_data = ReportDataV1::new( - *Ed25519PublicKey::from(tls_public_key).as_bytes(), - *Ed25519PublicKey::from(account_public_key).as_bytes(), - ) - .into(); - - let attestation = tee_authority.generate_attestation(report_data).await?; - - // Create communication channels and runtime - let (debug_request_sender, _) = tokio::sync::broadcast::channel(10); - let root_task_handle = Arc::new(OnceLock::new()); - - let (protocol_state_sender, protocol_state_receiver) = - watch::channel(ProtocolContractState::NotInitialized); - - let (migration_state_sender, migration_state_receiver) = - watch::channel((0, BTreeMap::new())); - let web_server = root_runtime - .block_on(start_web_server( - root_task_handle.clone(), - debug_request_sender.clone(), - config.web_ui, - static_web_data(&secrets, Some(attestation)), - protocol_state_receiver, - migration_state_receiver, - )) - .context("Failed to create web server.")?; - - let _web_server_join_handle = root_runtime.spawn(web_server); - - // Create Indexer and wait for indexer to be synced. - let (indexer_exit_sender, indexer_exit_receiver) = oneshot::channel(); - let indexer_api = spawn_real_indexer( - home_dir.clone(), - config.indexer.clone(), - config.my_near_account_id.clone(), - persistent_secrets.near_signer_key.clone(), - respond_config, - indexer_exit_sender, - protocol_state_sender, - migration_state_sender, - *tls_public_key, - ); - - let (shutdown_signal_sender, mut shutdown_signal_receiver) = mpsc::channel(1); - let cancellation_token = CancellationToken::new(); - - let image_hash_watcher_handle = if let (Some(image_hash), Some(latest_allowed_hash_file)) = - (&self.tee.image_hash, &self.tee.latest_allowed_hash_file) - { - let current_image_hash_bytes: [u8; 32] = hex::decode(image_hash) - .expect("The currently running image is a hex string.") - .try_into() - .expect("The currently running image hash hex representation is 32 bytes."); - - let allowed_hashes_in_contract = indexer_api.allowed_docker_images_receiver.clone(); - let image_hash_storage = AllowedImageHashesFile::from(latest_allowed_hash_file.clone()); - - Some(root_runtime.spawn(monitor_allowed_image_hashes( - cancellation_token.child_token(), - MpcDockerImageHash::from(current_image_hash_bytes), - allowed_hashes_in_contract, - image_hash_storage, - shutdown_signal_sender.clone(), - ))) - } else { - tracing::info!( +pub async fn run(config: StartConfig) -> anyhow::Result<()> { + let root_runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .worker_threads(1) + .build()?; + + let _tokio_enter_guard = root_runtime.enter(); + + // Load configuration and initialize persistent secrets + let home_dir = PathBuf::from(config.home_dir.clone()); + let node_config = config.node.clone(); + let persistent_secrets = PersistentSecrets::generate_or_get_existing( + &home_dir, + node_config.number_of_responder_keys, + )?; + + profiler::web_server::start_web_server(node_config.pprof_bind_address).await?; + root_runtime.spawn(crate::metrics::tokio_task_metrics::run_monitor_loop()); + + // TODO(#1296): Decide if the MPC responder account is actually needed + let respond_config = RespondConfig::from_parts(&node_config, &persistent_secrets); + + let backup_encryption_key_hex = match &config.secrets.backup_encryption_key_hex { + Some(key) => key.clone(), + None => generate_and_write_backup_encryption_key_to_disk(&home_dir)?, + }; + + // Load secrets from configuration and persistent storage + let secrets = SecretsConfig::from_parts( + &config.secrets.secret_store_key_hex, + persistent_secrets.clone(), + &backup_encryption_key_hex, + )?; + + // Generate attestation + let tee_authority = config.tee.authority.clone().into_tee_authority()?; + let tls_public_key = &secrets.persistent_secrets.p2p_private_key.verifying_key(); + + let account_public_key = &secrets.persistent_secrets.near_signer_key.verifying_key(); + + let report_data = ReportDataV1::new( + *Ed25519PublicKey::from(tls_public_key).as_bytes(), + *Ed25519PublicKey::from(account_public_key).as_bytes(), + ) + .into(); + + let attestation = tee_authority.generate_attestation(report_data).await?; + + // Create communication channels and runtime + let (debug_request_sender, _) = tokio::sync::broadcast::channel(10); + let root_task_handle = Arc::new(OnceLock::new()); + + let (protocol_state_sender, protocol_state_receiver) = + watch::channel(ProtocolContractState::NotInitialized); + + let (migration_state_sender, migration_state_receiver) = watch::channel((0, BTreeMap::new())); + let web_server = root_runtime + .block_on(start_web_server( + root_task_handle.clone(), + debug_request_sender.clone(), + node_config.web_ui, + static_web_data(&secrets, Some(attestation)), + protocol_state_receiver, + migration_state_receiver, + )) + .context("Failed to create web server.")?; + + let _web_server_join_handle = root_runtime.spawn(web_server); + + // Create Indexer and wait for indexer to be synced. + let (indexer_exit_sender, indexer_exit_receiver) = oneshot::channel(); + let indexer_api = spawn_real_indexer( + home_dir.clone(), + node_config.indexer.clone(), + node_config.my_near_account_id.clone(), + persistent_secrets.near_signer_key.clone(), + respond_config, + indexer_exit_sender, + protocol_state_sender, + migration_state_sender, + *tls_public_key, + ); + + let (shutdown_signal_sender, mut shutdown_signal_receiver) = mpsc::channel(1); + let cancellation_token = CancellationToken::new(); + + let image_hash_watcher_handle = if let (Some(image_hash), Some(latest_allowed_hash_file)) = + (&config.tee.image_hash, &config.tee.latest_allowed_hash_file) + { + let current_image_hash_bytes: [u8; 32] = hex::decode(image_hash) + .expect("The currently running image is a hex string.") + .try_into() + .expect("The currently running image hash hex representation is 32 bytes."); + + let allowed_hashes_in_contract = indexer_api.allowed_docker_images_receiver.clone(); + let image_hash_storage = AllowedImageHashesFile::from(latest_allowed_hash_file.clone()); + + Some(root_runtime.spawn(monitor_allowed_image_hashes( + cancellation_token.child_token(), + MpcDockerImageHash::from(current_image_hash_bytes), + allowed_hashes_in_contract, + image_hash_storage, + shutdown_signal_sender.clone(), + ))) + } else { + tracing::info!( "image_hash and/or latest_allowed_hash_file not set, skipping TEE image hash monitoring" ); - None - }; - - let root_future = create_root_future( - self, - home_dir.clone(), - config.clone(), - secrets.clone(), - indexer_api, - debug_request_sender, - root_task_handle, - tee_authority, - ); - - let root_task = root_runtime.spawn(start_root_task("root", root_future).0); - - let exit_reason = tokio::select! { - root_task_result = root_task => { - root_task_result? - } - indexer_exit_response = indexer_exit_receiver => { - indexer_exit_response.context("Indexer thread dropped response channel.")? - } - Some(()) = shutdown_signal_receiver.recv() => { - Err(anyhow!("TEE allowed image hashes watcher is sending shutdown signal.")) - } - }; - - // Perform graceful shutdown - cancellation_token.cancel(); - - if let Some(handle) = image_hash_watcher_handle { - info!("Waiting for image hash watcher to gracefully exit."); - let exit_result = handle.await; - info!(?exit_result, "Image hash watcher exited."); + None + }; + + let root_future = create_root_future( + config, + home_dir.clone(), + node_config.clone(), + secrets.clone(), + indexer_api, + debug_request_sender, + root_task_handle, + tee_authority, + ); + + let root_task = root_runtime.spawn(start_root_task("root", root_future).0); + + let exit_reason = tokio::select! { + root_task_result = root_task => { + root_task_result? + } + indexer_exit_response = indexer_exit_receiver => { + indexer_exit_response.context("Indexer thread dropped response channel.")? } + Some(()) = shutdown_signal_receiver.recv() => { + Err(anyhow!("TEE allowed image hashes watcher is sending shutdown signal.")) + } + }; - exit_reason + // Perform graceful shutdown + cancellation_token.cancel(); + + if let Some(handle) = image_hash_watcher_handle { + info!("Waiting for image hash watcher to gracefully exit."); + let exit_result = handle.await; + info!(?exit_result, "Image hash watcher exited."); } + + exit_reason } #[allow(clippy::too_many_arguments)] From 0636c9292618b5210a3780c971943b4718603f97 Mon Sep 17 00:00:00 2001 From: Daniel Sharifi Date: Fri, 6 Mar 2026 15:20:41 +0100 Subject: [PATCH 08/16] rename to run_mpc_node --- crates/node/src/cli.rs | 6 +++--- crates/node/src/run.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/node/src/cli.rs b/crates/node/src/cli.rs index 2b1aa8abf..5062cc7dd 100644 --- a/crates/node/src/cli.rs +++ b/crates/node/src/cli.rs @@ -11,7 +11,7 @@ use crate::{ permanent::{PermanentKeyStorage, PermanentKeyStorageBackend, PermanentKeyshareData}, }, p2p::testing::{generate_test_p2p_configs, PortSeed}, - run::run, + run::run_mpc_node, }; use clap::{Args, Parser, Subcommand, ValueEnum}; use hex::FromHex; @@ -227,14 +227,14 @@ impl Cli { match self.command { CliCommand::StartWithConfigFile { config_path } => { let node_configuration = StartConfig::from_json_file(&config_path)?; - run(node_configuration).await + run_mpc_node(node_configuration).await } CliCommand::Start(start) => { let home_dir = std::path::Path::new(&start.home_dir); let config_file = load_config_file(home_dir)?; let node_configuration = start.into_start_config(config_file); - run(node_configuration).await + run_mpc_node(node_configuration).await } CliCommand::Init(config) => { let (download_config_type, download_config_url) = if config.download_config { diff --git a/crates/node/src/run.rs b/crates/node/src/run.rs index f403a299f..dcde5b1db 100644 --- a/crates/node/src/run.rs +++ b/crates/node/src/run.rs @@ -39,7 +39,7 @@ use crate::tee::{ pub const ATTESTATION_RESUBMISSION_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour -pub async fn run(config: StartConfig) -> anyhow::Result<()> { +pub async fn run_mpc_node(config: StartConfig) -> anyhow::Result<()> { let root_runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() .worker_threads(1) From 599d99900f3c3ff8892fdd03554346088cfc6e58 Mon Sep 17 00:00:00 2001 From: Daniel Sharifi Date: Fri, 6 Mar 2026 15:26:29 +0100 Subject: [PATCH 09/16] use pathbuf --- crates/node/src/cli.rs | 1 + crates/node/src/config/start.rs | 3 ++- crates/node/src/run.rs | 3 +-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/node/src/cli.rs b/crates/node/src/cli.rs index 5062cc7dd..cf141edfb 100644 --- a/crates/node/src/cli.rs +++ b/crates/node/src/cli.rs @@ -229,6 +229,7 @@ impl Cli { let node_configuration = StartConfig::from_json_file(&config_path)?; run_mpc_node(node_configuration).await } + // TODO: deprecate this CliCommand::Start(start) => { let home_dir = std::path::Path::new(&start.home_dir); let config_file = load_config_file(home_dir)?; diff --git a/crates/node/src/config/start.rs b/crates/node/src/config/start.rs index 8ee8ea9a7..ffda2122b 100644 --- a/crates/node/src/config/start.rs +++ b/crates/node/src/config/start.rs @@ -13,7 +13,7 @@ use url::Url; /// (JSON file) convert into this type. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StartConfig { - pub home_dir: String, + pub home_dir: PathBuf, /// Encryption keys and backup settings. pub secrets: SecretsStartConfig, /// TEE authority and image hash monitoring settings. @@ -68,6 +68,7 @@ pub enum TeeAuthorityStartConfig { #[serde(default = "default_dstack_endpoint")] dstack_endpoint: String, #[serde(default = "default_quote_upload_url")] + // TODO: use URL type for this type quote_upload_url: String, }, } diff --git a/crates/node/src/run.rs b/crates/node/src/run.rs index dcde5b1db..7dd40d392 100644 --- a/crates/node/src/run.rs +++ b/crates/node/src/run.rs @@ -48,10 +48,9 @@ pub async fn run_mpc_node(config: StartConfig) -> anyhow::Result<()> { let _tokio_enter_guard = root_runtime.enter(); // Load configuration and initialize persistent secrets - let home_dir = PathBuf::from(config.home_dir.clone()); let node_config = config.node.clone(); let persistent_secrets = PersistentSecrets::generate_or_get_existing( - &home_dir, + &config.home_dir, node_config.number_of_responder_keys, )?; From ffa3497cd8a1f356043840ba857ef750888c1466 Mon Sep 17 00:00:00 2001 From: Daniel Sharifi Date: Fri, 6 Mar 2026 15:28:41 +0100 Subject: [PATCH 10/16] add todo issue links --- crates/node/src/cli.rs | 2 +- crates/node/src/config/foreign_chains/auth.rs | 2 +- crates/node/src/config/start.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/node/src/cli.rs b/crates/node/src/cli.rs index cf141edfb..365729816 100644 --- a/crates/node/src/cli.rs +++ b/crates/node/src/cli.rs @@ -229,7 +229,7 @@ impl Cli { let node_configuration = StartConfig::from_json_file(&config_path)?; run_mpc_node(node_configuration).await } - // TODO: deprecate this + // TODO(#2334): deprecate this CliCommand::Start(start) => { let home_dir = std::path::Path::new(&start.home_dir); let config_file = load_config_file(home_dir)?; diff --git a/crates/node/src/config/foreign_chains/auth.rs b/crates/node/src/config/foreign_chains/auth.rs index 48d916acd..3540fe837 100644 --- a/crates/node/src/config/foreign_chains/auth.rs +++ b/crates/node/src/config/foreign_chains/auth.rs @@ -52,7 +52,7 @@ pub enum TokenConfig { impl TokenConfig { pub fn resolve(&self) -> anyhow::Result { match self { - // TODO: do not resolve env variables this deep in the binary. + // TODO(#2335): do not resolve env variables this deep in the binary. // Should be resolved at start, preferably in the config so we can kill env configs // // One option is to have a separate secrets config file. diff --git a/crates/node/src/config/start.rs b/crates/node/src/config/start.rs index ffda2122b..78aa8be3d 100644 --- a/crates/node/src/config/start.rs +++ b/crates/node/src/config/start.rs @@ -68,7 +68,7 @@ pub enum TeeAuthorityStartConfig { #[serde(default = "default_dstack_endpoint")] dstack_endpoint: String, #[serde(default = "default_quote_upload_url")] - // TODO: use URL type for this type + // TODO(#2333): use URL type for this type quote_upload_url: String, }, } From 0b53d1c0a01a7885c6da1fed6c2ed08bcff3636c Mon Sep 17 00:00:00 2001 From: Daniel Sharifi Date: Fri, 6 Mar 2026 15:47:40 +0100 Subject: [PATCH 11/16] fix pathbuf issue --- crates/node/src/cli.rs | 2 +- crates/node/src/run.rs | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/node/src/cli.rs b/crates/node/src/cli.rs index 365729816..47cefd480 100644 --- a/crates/node/src/cli.rs +++ b/crates/node/src/cli.rs @@ -85,7 +85,7 @@ pub enum CliCommand { #[derive(Args, Debug)] pub struct StartCmd { #[arg(long, env("MPC_HOME_DIR"))] - pub home_dir: String, + pub home_dir: PathBuf, /// Hex-encoded 16 byte AES key for local storage encryption. /// This key should come from a secure secret storage. /// TODO(#444): After TEE integration decide on what to do with AES encryption key diff --git a/crates/node/src/run.rs b/crates/node/src/run.rs index 7dd40d392..93b9c99fe 100644 --- a/crates/node/src/run.rs +++ b/crates/node/src/run.rs @@ -62,7 +62,7 @@ pub async fn run_mpc_node(config: StartConfig) -> anyhow::Result<()> { let backup_encryption_key_hex = match &config.secrets.backup_encryption_key_hex { Some(key) => key.clone(), - None => generate_and_write_backup_encryption_key_to_disk(&home_dir)?, + None => generate_and_write_backup_encryption_key_to_disk(&config.home_dir)?, }; // Load secrets from configuration and persistent storage @@ -110,7 +110,7 @@ pub async fn run_mpc_node(config: StartConfig) -> anyhow::Result<()> { // Create Indexer and wait for indexer to be synced. let (indexer_exit_sender, indexer_exit_receiver) = oneshot::channel(); let indexer_api = spawn_real_indexer( - home_dir.clone(), + config.home_dir.clone(), node_config.indexer.clone(), node_config.my_near_account_id.clone(), persistent_secrets.near_signer_key.clone(), @@ -149,6 +149,7 @@ pub async fn run_mpc_node(config: StartConfig) -> anyhow::Result<()> { None }; + let home_dir = config.home_dir.clone(); let root_future = create_root_future( config, home_dir.clone(), From 34515c211401108e4f61945a72419051a1d9f4ab Mon Sep 17 00:00:00 2001 From: Daniel Sharifi Date: Fri, 6 Mar 2026 15:49:20 +0100 Subject: [PATCH 12/16] update pytests --- pytest/common_lib/shared/__init__.py | 4 ++++ pytest/common_lib/shared/mpc_node.py | 34 +++++++++++++++++++++------- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/pytest/common_lib/shared/__init__.py b/pytest/common_lib/shared/__init__.py index 48cb7cb3a..27c391764 100644 --- a/pytest/common_lib/shared/__init__.py +++ b/pytest/common_lib/shared/__init__.py @@ -230,6 +230,7 @@ class ConfigValues: migration_address: str pprof_address: str backup_key: bytes + node_config: dict # JSON-serializable dict matching Rust ConfigFile def generate_mpc_configs( @@ -331,6 +332,7 @@ def generate_mpc_configs( ] backup_key = os.urandom(32) + configs.append( ConfigValues( signer_key, @@ -341,6 +343,7 @@ def generate_mpc_configs( migration_address, pprof_address, backup_key, + node_config=config, ) ) return configs @@ -531,6 +534,7 @@ def start_cluster_with_mpc( pytest_signer_keys=pytest_signer_keys, backup_key=config.backup_key, pprof_address=config.pprof_address, + node_config=config.node_config, ) mpc_node.init_nonces(validators[0]) mpc_node.set_block_ingestion(True) diff --git a/pytest/common_lib/shared/mpc_node.py b/pytest/common_lib/shared/mpc_node.py index 436ded22a..c8b87d5ce 100644 --- a/pytest/common_lib/shared/mpc_node.py +++ b/pytest/common_lib/shared/mpc_node.py @@ -61,6 +61,7 @@ def __init__( p2p_public_key: str, pytest_signer_keys: list[Key], backup_key: bytes, + node_config: dict, ): super().__init__(near_node, signer_key, pytest_signer_keys) self.p2p_url: str = p2p_url @@ -74,6 +75,7 @@ def __init__( self.is_running = False self.metrics = MetricsTracker(near_node) self.backup_key = backup_key + self.node_config = node_config def print(self): if not self.is_running: @@ -127,22 +129,38 @@ def reset_mpc_data(self): for file_path in pathlib.Path(self.home_dir).glob(pattern): file_path.unlink() + def _write_start_config(self) -> str: + """Build a StartConfig JSON file and write it to the node's home dir. + Returns the path to the written config file.""" + start_config = { + "home_dir": self.home_dir, + "secrets": { + "secret_store_key_hex": self.secret_store_key, + "backup_encryption_key_hex": self.backup_key.hex(), + }, + "tee": { + "authority": {"type": "local"}, + "image_hash": DUMMY_MPC_IMAGE_HASH, + "latest_allowed_hash_file": "latest_allowed_hash.txt", + }, + "node": self.node_config, + } + config_path = str(pathlib.Path(self.home_dir) / "start_config.json") + with open(config_path, "w") as f: + json.dump(start_config, f, indent=2) + return config_path + def run(self): assert not self.is_running self.is_running = True + config_path = self._write_start_config() extra_env = { "RUST_LOG": "INFO", # mpc-node produces too much output on DEBUG - "MPC_SECRET_STORE_KEY": self.secret_store_key, - "MPC_IMAGE_HASH": DUMMY_MPC_IMAGE_HASH, - "MPC_LATEST_ALLOWED_HASH_FILE": "latest_allowed_hash.txt", - "MPC_BACKUP_ENCRYPTION_KEY_HEX": self.backup_key.hex(), } cmd = ( MPC_BINARY_PATH, - "start", - "--home-dir", - self.home_dir, - "local", + "start-with-config-file", + config_path, ) self.near_node.run_cmd(cmd=cmd, extra_env=extra_env) From ca80bca948cf4b07981a40fc90daf9b599685189 Mon Sep 17 00:00:00 2001 From: Daniel Sharifi Date: Fri, 6 Mar 2026 16:08:20 +0100 Subject: [PATCH 13/16] fix: test config was overwriting neard config --- crates/node/src/cli.rs | 4 ++-- pytest/common_lib/shared/__init__.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/node/src/cli.rs b/crates/node/src/cli.rs index 47cefd480..f5529b72b 100644 --- a/crates/node/src/cli.rs +++ b/crates/node/src/cli.rs @@ -454,8 +454,8 @@ fn run_generate_test_configs( desired_presignatures_to_buffer, ); std::fs::write( - format!("{}/config.yaml", subdir), - serde_yaml::to_string(&file_config)?, + format!("{}/mpc_node_config.json", subdir), + serde_json::to_string_pretty(&file_config)?, )?; } std::fs::write( diff --git a/pytest/common_lib/shared/__init__.py b/pytest/common_lib/shared/__init__.py index 27c391764..8fdfdad44 100644 --- a/pytest/common_lib/shared/__init__.py +++ b/pytest/common_lib/shared/__init__.py @@ -42,7 +42,7 @@ dot_near = pathlib.Path.home() / ".near" SECRETS_JSON = "secrets.json" NUMBER_OF_VALIDATORS = 1 -CONFIG_YAML = "config.yaml" +MPC_NODE_CONFIG_JSON = "mpc_node_config.json" def create_function_call_access_key_action( @@ -308,9 +308,9 @@ def generate_mpc_configs( my_port = participant["port"] p2p_url = f"http://{my_addr}:{my_port}" - config_file_path = os.path.join(dot_near, str(idx), CONFIG_YAML) + config_file_path = os.path.join(dot_near, str(idx), MPC_NODE_CONFIG_JSON) with open(config_file_path, "r") as f: - config = yaml.load(f, Loader=SafeLoaderIgnoreUnknown) + config = json.load(f) web_address = config.get("web_ui") migration_address = config.get("migration_web_ui") From 44b9a0e765b9f9937169136b0f975e4caae320f4 Mon Sep 17 00:00:00 2001 From: Daniel Sharifi Date: Fri, 6 Mar 2026 16:09:10 +0100 Subject: [PATCH 14/16] reset nearcore change --- libs/nearcore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/nearcore b/libs/nearcore index 3def2f7eb..8a8c21bc8 160000 --- a/libs/nearcore +++ b/libs/nearcore @@ -1 +1 @@ -Subproject commit 3def2f7ebb7455199e7b3f7b371e3735c23e2930 +Subproject commit 8a8c21bc81999af93edd1b6bca5b7c6c6337aa63 From bc9b2fcd462946a775b97f1087f4a2ecfd71b024 Mon Sep 17 00:00:00 2001 From: Daniel Sharifi Date: Fri, 6 Mar 2026 16:23:25 +0100 Subject: [PATCH 15/16] fix yml failure --- pytest/common_lib/shared/foreign_chains.py | 25 +++------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/pytest/common_lib/shared/foreign_chains.py b/pytest/common_lib/shared/foreign_chains.py index 5d71cb30c..fa6193b87 100644 --- a/pytest/common_lib/shared/foreign_chains.py +++ b/pytest/common_lib/shared/foreign_chains.py @@ -1,32 +1,13 @@ """Shared helpers for foreign chain configuration and policy tests.""" -import pathlib -import re from typing import Any -import yaml - - -def node_config_path(node) -> pathlib.Path: - return pathlib.Path(node.home_dir) / "config.yaml" - def set_foreign_chains_config(node, foreign_chains: dict[str, Any] | None) -> None: - config_path = node_config_path(node) - - config_text = config_path.read_text(encoding="utf-8") - # Keep generated YAML tags intact by editing only the trailing `foreign_chains` section. - config_text = ( - re.sub(r"\nforeign_chains:[\s\S]*\Z", "\n", config_text).rstrip() + "\n" - ) - if foreign_chains is not None: - foreign_chains_text = yaml.safe_dump( - {"foreign_chains": foreign_chains}, sort_keys=False - ) - config_text += "\n" + foreign_chains_text - - config_path.write_text(config_text, encoding="utf-8") + node.node_config["foreign_chains"] = foreign_chains + else: + node.node_config["foreign_chains"] = {} def normalize_policy(policy: dict[str, Any]) -> list[tuple[str, tuple[str, ...]]]: From f414f2d60011fb9642da45a0350580e7c6e297c6 Mon Sep 17 00:00:00 2001 From: Daniel Sharifi Date: Fri, 6 Mar 2026 16:23:59 +0100 Subject: [PATCH 16/16] . --- pytest/common_lib/shared/__init__.py | 10 ++----- pytest/common_lib/shared/foreign_chains.py | 25 ++++++++++++++-- pytest/common_lib/shared/mpc_node.py | 34 +++++----------------- 3 files changed, 33 insertions(+), 36 deletions(-) diff --git a/pytest/common_lib/shared/__init__.py b/pytest/common_lib/shared/__init__.py index 8fdfdad44..48cb7cb3a 100644 --- a/pytest/common_lib/shared/__init__.py +++ b/pytest/common_lib/shared/__init__.py @@ -42,7 +42,7 @@ dot_near = pathlib.Path.home() / ".near" SECRETS_JSON = "secrets.json" NUMBER_OF_VALIDATORS = 1 -MPC_NODE_CONFIG_JSON = "mpc_node_config.json" +CONFIG_YAML = "config.yaml" def create_function_call_access_key_action( @@ -230,7 +230,6 @@ class ConfigValues: migration_address: str pprof_address: str backup_key: bytes - node_config: dict # JSON-serializable dict matching Rust ConfigFile def generate_mpc_configs( @@ -308,9 +307,9 @@ def generate_mpc_configs( my_port = participant["port"] p2p_url = f"http://{my_addr}:{my_port}" - config_file_path = os.path.join(dot_near, str(idx), MPC_NODE_CONFIG_JSON) + config_file_path = os.path.join(dot_near, str(idx), CONFIG_YAML) with open(config_file_path, "r") as f: - config = json.load(f) + config = yaml.load(f, Loader=SafeLoaderIgnoreUnknown) web_address = config.get("web_ui") migration_address = config.get("migration_web_ui") @@ -332,7 +331,6 @@ def generate_mpc_configs( ] backup_key = os.urandom(32) - configs.append( ConfigValues( signer_key, @@ -343,7 +341,6 @@ def generate_mpc_configs( migration_address, pprof_address, backup_key, - node_config=config, ) ) return configs @@ -534,7 +531,6 @@ def start_cluster_with_mpc( pytest_signer_keys=pytest_signer_keys, backup_key=config.backup_key, pprof_address=config.pprof_address, - node_config=config.node_config, ) mpc_node.init_nonces(validators[0]) mpc_node.set_block_ingestion(True) diff --git a/pytest/common_lib/shared/foreign_chains.py b/pytest/common_lib/shared/foreign_chains.py index fa6193b87..5d71cb30c 100644 --- a/pytest/common_lib/shared/foreign_chains.py +++ b/pytest/common_lib/shared/foreign_chains.py @@ -1,13 +1,32 @@ """Shared helpers for foreign chain configuration and policy tests.""" +import pathlib +import re from typing import Any +import yaml + + +def node_config_path(node) -> pathlib.Path: + return pathlib.Path(node.home_dir) / "config.yaml" + def set_foreign_chains_config(node, foreign_chains: dict[str, Any] | None) -> None: + config_path = node_config_path(node) + + config_text = config_path.read_text(encoding="utf-8") + # Keep generated YAML tags intact by editing only the trailing `foreign_chains` section. + config_text = ( + re.sub(r"\nforeign_chains:[\s\S]*\Z", "\n", config_text).rstrip() + "\n" + ) + if foreign_chains is not None: - node.node_config["foreign_chains"] = foreign_chains - else: - node.node_config["foreign_chains"] = {} + foreign_chains_text = yaml.safe_dump( + {"foreign_chains": foreign_chains}, sort_keys=False + ) + config_text += "\n" + foreign_chains_text + + config_path.write_text(config_text, encoding="utf-8") def normalize_policy(policy: dict[str, Any]) -> list[tuple[str, tuple[str, ...]]]: diff --git a/pytest/common_lib/shared/mpc_node.py b/pytest/common_lib/shared/mpc_node.py index c8b87d5ce..436ded22a 100644 --- a/pytest/common_lib/shared/mpc_node.py +++ b/pytest/common_lib/shared/mpc_node.py @@ -61,7 +61,6 @@ def __init__( p2p_public_key: str, pytest_signer_keys: list[Key], backup_key: bytes, - node_config: dict, ): super().__init__(near_node, signer_key, pytest_signer_keys) self.p2p_url: str = p2p_url @@ -75,7 +74,6 @@ def __init__( self.is_running = False self.metrics = MetricsTracker(near_node) self.backup_key = backup_key - self.node_config = node_config def print(self): if not self.is_running: @@ -129,38 +127,22 @@ def reset_mpc_data(self): for file_path in pathlib.Path(self.home_dir).glob(pattern): file_path.unlink() - def _write_start_config(self) -> str: - """Build a StartConfig JSON file and write it to the node's home dir. - Returns the path to the written config file.""" - start_config = { - "home_dir": self.home_dir, - "secrets": { - "secret_store_key_hex": self.secret_store_key, - "backup_encryption_key_hex": self.backup_key.hex(), - }, - "tee": { - "authority": {"type": "local"}, - "image_hash": DUMMY_MPC_IMAGE_HASH, - "latest_allowed_hash_file": "latest_allowed_hash.txt", - }, - "node": self.node_config, - } - config_path = str(pathlib.Path(self.home_dir) / "start_config.json") - with open(config_path, "w") as f: - json.dump(start_config, f, indent=2) - return config_path - def run(self): assert not self.is_running self.is_running = True - config_path = self._write_start_config() extra_env = { "RUST_LOG": "INFO", # mpc-node produces too much output on DEBUG + "MPC_SECRET_STORE_KEY": self.secret_store_key, + "MPC_IMAGE_HASH": DUMMY_MPC_IMAGE_HASH, + "MPC_LATEST_ALLOWED_HASH_FILE": "latest_allowed_hash.txt", + "MPC_BACKUP_ENCRYPTION_KEY_HEX": self.backup_key.hex(), } cmd = ( MPC_BINARY_PATH, - "start-with-config-file", - config_path, + "start", + "--home-dir", + self.home_dir, + "local", ) self.near_node.run_cmd(cmd=cmd, extra_env=extra_env)