diff --git a/crates/node/src/cli.rs b/crates/node/src/cli.rs index 3f976a547..f5529b72b 100644 --- a/crates/node/src/cli.rs +++ b/crates/node/src/cli.rs @@ -1,61 +1,28 @@ use crate::{ config::{ - generate_and_write_backup_encryption_key_to_disk, load_config_file, BlockArgs, CKDConfig, - ConfigFile, ForeignChainsConfig, IndexerConfig, KeygenConfig, PersistentSecrets, - PresignatureConfig, RespondConfig, SecretsConfig, SignatureConfig, SyncMode, TripleConfig, - }, - coordinator::Coordinator, - db::SecretDB, - indexer::{ - real::spawn_real_indexer, tx_sender::TransactionSender, IndexerAPI, ReadForeignChainPolicy, + load_config_file, BlockArgs, CKDConfig, ConfigFile, ForeignChainsConfig, GcpStartConfig, + IndexerConfig, KeygenConfig, PersistentSecrets, PresignatureConfig, SecretsStartConfig, + SignatureConfig, StartConfig, SyncMode, TeeAuthorityStartConfig, TeeStartConfig, + TripleConfig, }, keyshare::{ compat::legacy_ecdsa_key_from_keyshares, local::LocalPermanentKeyStorageBackend, permanent::{PermanentKeyStorage, PermanentKeyStorageBackend, PermanentKeyshareData}, - GcpPermanentKeyStorageConfig, KeyStorageConfig, KeyshareStorage, }, - migration_service::spawn_recovery_server_and_run_onboarding, p2p::testing::{generate_test_p2p_configs, PortSeed}, - profiler, - tracking::{self, start_root_task}, - web::{start_web_server, static_web_data, DebugRequest}, + run::run_mpc_node, }; -use anyhow::{anyhow, Context}; use clap::{Args, Parser, Subcommand, ValueEnum}; use hex::FromHex; -use mpc_attestation::report_data::ReportDataV1; -use mpc_contract::state::ProtocolContractState; use near_account_id::AccountId; use near_indexer_primitives::types::Finality; -use near_time::Clock; use std::{ - collections::BTreeMap, net::{Ipv4Addr, SocketAddr}, - sync::Mutex, -}; -use std::{path::PathBuf, sync::Arc, sync::OnceLock, time::Duration}; -use tee_authority::tee_authority::{ - DstackTeeAuthorityConfig, LocalTeeAuthorityConfig, TeeAuthority, DEFAULT_DSTACK_ENDPOINT, - DEFAULT_PHALA_TDX_QUOTE_UPLOAD_URL, + path::PathBuf, }; -use tokio::sync::{broadcast, mpsc, oneshot, watch, RwLock}; -use tokio_util::sync::CancellationToken; +use tee_authority::tee_authority::{DEFAULT_DSTACK_ENDPOINT, DEFAULT_PHALA_TDX_QUOTE_UPLOAD_URL}; use url::Url; - -use contract_interface::types::Ed25519PublicKey; -use { - crate::tee::{ - monitor_allowed_image_hashes, - remote_attestation::{monitor_attestation_removal, periodic_attestation_submission}, - AllowedImageHashesFile, - }, - mpc_contract::tee::proposal::MpcDockerImageHash, - tracing::info, -}; - -pub const ATTESTATION_RESUBMISSION_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour - #[derive(Parser, Debug)] #[command(name = "mpc-node")] #[command(about = "MPC Node for Near Protocol")] @@ -77,6 +44,13 @@ pub enum LogFormat { #[derive(Subcommand, Debug)] pub enum CliCommand { + /// Starts the MPC node using a single JSON configuration file instead of + /// environment variables and CLI flags. + StartWithConfigFile { + /// Path to a JSON configuration file containing all settings needed to + /// start the MPC node. + config_path: PathBuf, + }, Start(StartCmd), /// Generates/downloads required files for Near node to run Init(InitConfigArgs), @@ -108,38 +82,10 @@ pub enum CliCommand { migrating_nodes: Vec, }, } - -#[derive(Args, Debug)] -pub struct InitConfigArgs { - #[arg(long, env("MPC_HOME_DIR"))] - pub dir: std::path::PathBuf, - /// chain/network id (localnet, testnet, devnet, betanet) - #[arg(long)] - pub chain_id: Option, - /// Genesis file to use when initialize testnet (including downloading) - #[arg(long)] - pub genesis: Option, - /// Download the verified NEAR config file automatically. - #[arg(long)] - pub download_config: bool, - #[arg(long)] - pub download_config_url: Option, - /// Download the verified NEAR genesis file automatically. - #[arg(long)] - pub download_genesis: bool, - /// Specify a custom download URL for the genesis-file. - #[arg(long)] - pub download_genesis_url: Option, - #[arg(long)] - pub download_genesis_records_url: Option, - #[arg(long)] - pub boot_nodes: Option, -} - #[derive(Args, Debug)] pub struct StartCmd { #[arg(long, env("MPC_HOME_DIR"))] - pub home_dir: String, + pub home_dir: PathBuf, /// Hex-encoded 16 byte AES key for local storage encryption. /// This key should come from a secure secret storage. /// TODO(#444): After TEE integration decide on what to do with AES encryption key @@ -153,17 +99,17 @@ pub struct StartCmd { pub gcp_project_id: Option, /// TEE authority config #[command(subcommand)] - pub tee_authority: TeeAuthorityConfig, + pub tee_authority: CliTeeAuthorityConfig, /// TEE related configuration settings. #[command(flatten)] - pub image_hash_config: MpcImageHashConfig, + pub image_hash_config: CliImageHashConfig, /// Hex-encoded 32 byte AES key for backup encryption. #[arg(env("MPC_BACKUP_ENCRYPTION_KEY_HEX"))] pub backup_encryption_key_hex: Option, } #[derive(Subcommand, Debug, Clone)] -pub enum TeeAuthorityConfig { +pub enum CliTeeAuthorityConfig { Local, Dstack { #[arg(long, env("DSTACK_ENDPOINT"), default_value = DEFAULT_DSTACK_ENDPOINT)] @@ -173,24 +119,8 @@ pub enum TeeAuthorityConfig { }, } -impl TryFrom for TeeAuthority { - type Error = anyhow::Error; - - fn try_from(cmd: TeeAuthorityConfig) -> Result { - let authority_config = match cmd { - TeeAuthorityConfig::Local => LocalTeeAuthorityConfig::default().into(), - TeeAuthorityConfig::Dstack { - dstack_endpoint, - quote_upload_url, - } => DstackTeeAuthorityConfig::new(dstack_endpoint, quote_upload_url).into(), - }; - - Ok(authority_config) - } -} - #[derive(Args, Debug)] -pub struct MpcImageHashConfig { +pub struct CliImageHashConfig { #[arg( long, env("MPC_IMAGE_HASH"), @@ -205,6 +135,66 @@ pub struct MpcImageHashConfig { pub latest_allowed_hash_file: Option, } +impl StartCmd { + fn into_start_config(self, config: ConfigFile) -> StartConfig { + let gcp = match (self.gcp_keyshare_secret_id, self.gcp_project_id) { + (Some(keyshare_secret_id), Some(project_id)) => Some(GcpStartConfig { + keyshare_secret_id, + project_id, + }), + _ => None, + }; + StartConfig { + home_dir: self.home_dir, + secrets: SecretsStartConfig { + secret_store_key_hex: self.secret_store_key_hex, + backup_encryption_key_hex: self.backup_encryption_key_hex, + }, + tee: TeeStartConfig { + authority: match self.tee_authority { + CliTeeAuthorityConfig::Local => TeeAuthorityStartConfig::Local, + CliTeeAuthorityConfig::Dstack { + dstack_endpoint, + quote_upload_url, + } => TeeAuthorityStartConfig::Dstack { + dstack_endpoint, + quote_upload_url: quote_upload_url.to_string(), + }, + }, + image_hash: self.image_hash_config.image_hash, + latest_allowed_hash_file: self.image_hash_config.latest_allowed_hash_file, + }, + gcp, + node: config, + } + } +} +#[derive(Args, Debug)] +pub struct InitConfigArgs { + #[arg(long, env("MPC_HOME_DIR"))] + pub dir: std::path::PathBuf, + /// chain/network id (localnet, testnet, devnet, betanet) + #[arg(long)] + pub chain_id: Option, + /// Genesis file to use when initialize testnet (including downloading) + #[arg(long)] + pub genesis: Option, + /// Download the verified NEAR config file automatically. + #[arg(long)] + pub download_config: bool, + #[arg(long)] + pub download_config_url: Option, + /// Download the verified NEAR genesis file automatically. + #[arg(long)] + pub download_genesis: bool, + /// Specify a custom download URL for the genesis-file. + #[arg(long)] + pub download_genesis_url: Option, + #[arg(long)] + pub download_genesis_records_url: Option, + #[arg(long)] + pub boot_nodes: Option, +} #[derive(Args, Debug)] pub struct ImportKeyshareCmd { /// Path to home directory @@ -232,293 +222,21 @@ pub struct ExportKeyshareCmd { #[arg(help = "Hex-encoded 16 byte AES key for local storage encryption")] pub local_encryption_key_hex: String, } - -impl StartCmd { - async fn run(self) -> anyhow::Result<()> { - let root_runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .worker_threads(1) - .build()?; - - let _tokio_enter_guard = root_runtime.enter(); - - // Load configuration and initialize persistent secrets - let home_dir = PathBuf::from(self.home_dir.clone()); - let config = load_config_file(&home_dir)?; - let persistent_secrets = PersistentSecrets::generate_or_get_existing( - &home_dir, - config.number_of_responder_keys, - )?; - - profiler::web_server::start_web_server(config.pprof_bind_address).await?; - root_runtime.spawn(crate::metrics::tokio_task_metrics::run_monitor_loop()); - - // TODO(#1296): Decide if the MPC responder account is actually needed - let respond_config = RespondConfig::from_parts(&config, &persistent_secrets); - - let backup_encryption_key_hex = match &self.backup_encryption_key_hex { - Some(key) => key.clone(), - None => generate_and_write_backup_encryption_key_to_disk(&home_dir)?, - }; - - // Load secrets from configuration and persistent storage - let secrets = SecretsConfig::from_parts( - &self.secret_store_key_hex, - persistent_secrets.clone(), - &backup_encryption_key_hex, - )?; - - // Generate attestation - let tee_authority = TeeAuthority::try_from(self.tee_authority.clone())?; - let tls_public_key = &secrets.persistent_secrets.p2p_private_key.verifying_key(); - - let account_public_key = &secrets.persistent_secrets.near_signer_key.verifying_key(); - - let report_data = ReportDataV1::new( - *Ed25519PublicKey::from(tls_public_key).as_bytes(), - *Ed25519PublicKey::from(account_public_key).as_bytes(), - ) - .into(); - - let attestation = tee_authority.generate_attestation(report_data).await?; - - // Create communication channels and runtime - let (debug_request_sender, _) = tokio::sync::broadcast::channel(10); - let root_task_handle = Arc::new(OnceLock::new()); - - let (protocol_state_sender, protocol_state_receiver) = - watch::channel(ProtocolContractState::NotInitialized); - - let (migration_state_sender, migration_state_receiver) = - watch::channel((0, BTreeMap::new())); - let web_server = root_runtime - .block_on(start_web_server( - root_task_handle.clone(), - debug_request_sender.clone(), - config.web_ui, - static_web_data(&secrets, Some(attestation)), - protocol_state_receiver, - migration_state_receiver, - )) - .context("Failed to create web server.")?; - - let _web_server_join_handle = root_runtime.spawn(web_server); - - // Create Indexer and wait for indexer to be synced. - let (indexer_exit_sender, indexer_exit_receiver) = oneshot::channel(); - let indexer_api = spawn_real_indexer( - home_dir.clone(), - config.indexer.clone(), - config.my_near_account_id.clone(), - persistent_secrets.near_signer_key.clone(), - respond_config, - indexer_exit_sender, - protocol_state_sender, - migration_state_sender, - *tls_public_key, - ); - - let (shutdown_signal_sender, mut shutdown_signal_receiver) = mpsc::channel(1); - let cancellation_token = CancellationToken::new(); - - let image_hash_watcher_handle = if let (Some(image_hash), Some(latest_allowed_hash_file)) = ( - &self.image_hash_config.image_hash, - &self.image_hash_config.latest_allowed_hash_file, - ) { - let current_image_hash_bytes: [u8; 32] = hex::decode(image_hash) - .expect("The currently running image is a hex string.") - .try_into() - .expect("The currently running image hash hex representation is 32 bytes."); - - let allowed_hashes_in_contract = indexer_api.allowed_docker_images_receiver.clone(); - let image_hash_storage = AllowedImageHashesFile::from(latest_allowed_hash_file.clone()); - - Some(root_runtime.spawn(monitor_allowed_image_hashes( - cancellation_token.child_token(), - MpcDockerImageHash::from(current_image_hash_bytes), - allowed_hashes_in_contract, - image_hash_storage, - shutdown_signal_sender.clone(), - ))) - } else { - tracing::info!( - "MPC_IMAGE_HASH and/or MPC_LATEST_ALLOWED_HASH_FILE not set, skipping TEE image hash monitoring" - ); - None - }; - - let root_future = self.create_root_future( - home_dir.clone(), - config.clone(), - secrets.clone(), - indexer_api, - debug_request_sender, - root_task_handle, - tee_authority, - ); - - let root_task = root_runtime.spawn(start_root_task("root", root_future).0); - - let exit_reason = tokio::select! { - root_task_result = root_task => { - root_task_result? - } - indexer_exit_response = indexer_exit_receiver => { - indexer_exit_response.context("Indexer thread dropped response channel.")? - } - Some(()) = shutdown_signal_receiver.recv() => { - Err(anyhow!("TEE allowed image hashes watcher is sending shutdown signal.")) - } - }; - - // Perform graceful shutdown - cancellation_token.cancel(); - - if let Some(handle) = image_hash_watcher_handle { - info!("Waiting for image hash watcher to gracefully exit."); - let exit_result = handle.await; - info!(?exit_result, "Image hash watcher exited."); - } - - exit_reason - } - - #[allow(clippy::too_many_arguments)] - async fn create_root_future( - self, - home_dir: PathBuf, - config: ConfigFile, - secrets: SecretsConfig, - indexer_api: IndexerAPI, - debug_request_sender: broadcast::Sender, - // Cloning a OnceLock returns a new cell, which is why we have to wrap it in an arc. - // Otherwise we would not write to the same cell/lock. - root_task_handle_once_lock: Arc>>, - tee_authority: TeeAuthority, - ) -> anyhow::Result<()> - where - TransactionSenderImpl: TransactionSender + 'static, - ForeignChainPolicyReader: ReadForeignChainPolicy + Clone + Send + Sync + 'static, - { - let root_task_handle = tracking::current_task(); - - root_task_handle_once_lock - .set(root_task_handle.clone()) - .map_err(|_| anyhow!("Root task handle was already set"))?; - - let tls_public_key = - Ed25519PublicKey::from(&secrets.persistent_secrets.p2p_private_key.verifying_key()); - let account_public_key = - Ed25519PublicKey::from(&secrets.persistent_secrets.near_signer_key.verifying_key()); - - let secret_db = SecretDB::new(&home_dir.join("assets"), secrets.local_storage_aes_key)?; - - let key_storage_config = KeyStorageConfig { - home_dir: home_dir.clone(), - local_encryption_key: secrets.local_storage_aes_key, - gcp: if let Some(secret_id) = self.gcp_keyshare_secret_id { - let project_id = self.gcp_project_id.ok_or_else(|| { - anyhow::anyhow!( - "GCP_PROJECT_ID must be specified to use GCP_KEYSHARE_SECRET_ID" - ) - })?; - Some(GcpPermanentKeyStorageConfig { - project_id, - secret_id, - }) - } else { - None - }, - }; - - // Spawn periodic attestation submission task - let tee_authority_clone = tee_authority.clone(); - let tx_sender_clone = indexer_api.txn_sender.clone(); - let tls_public_key_clone = tls_public_key.clone(); - let account_public_key_clone = account_public_key.clone(); - let allowed_docker_images_receiver_clone = - indexer_api.allowed_docker_images_receiver.clone(); - let allowed_launcher_compose_receiver_clone = - indexer_api.allowed_launcher_compose_receiver.clone(); - tokio::spawn(async move { - if let Err(e) = periodic_attestation_submission( - tee_authority_clone, - tx_sender_clone, - tls_public_key_clone, - account_public_key_clone, - allowed_docker_images_receiver_clone, - allowed_launcher_compose_receiver_clone, - tokio::time::interval(ATTESTATION_RESUBMISSION_INTERVAL), - ) - .await - { - tracing::error!( - error = ?e, - "periodic attestation submission task failed" - ); - } - }); - - // Spawn TEE attestation monitoring task - let tx_sender_clone = indexer_api.txn_sender.clone(); - let tee_accounts_receiver = indexer_api.attested_nodes_receiver.clone(); - let account_id_clone = config.my_near_account_id.clone(); - let allowed_docker_images_receiver_clone = - indexer_api.allowed_docker_images_receiver.clone(); - let allowed_launcher_compose_receiver_clone = - indexer_api.allowed_launcher_compose_receiver.clone(); - tokio::spawn(async move { - if let Err(e) = monitor_attestation_removal( - account_id_clone, - tee_authority, - tx_sender_clone, - tls_public_key, - account_public_key, - allowed_docker_images_receiver_clone, - allowed_launcher_compose_receiver_clone, - tee_accounts_receiver, - ) - .await - { - tracing::error!( - error = ?e, - "attestation removal monitoring task failed" - ); - } - }); - - let keyshare_storage: Arc> = - RwLock::new(key_storage_config.create().await?).into(); - - spawn_recovery_server_and_run_onboarding( - config.migration_web_ui, - (&secrets).into(), - config.my_near_account_id.clone(), - keyshare_storage.clone(), - indexer_api.my_migration_info_receiver.clone(), - indexer_api.contract_state_receiver.clone(), - indexer_api.txn_sender.clone(), - ) - .await?; - - let coordinator = Coordinator { - clock: Clock::real(), - config_file: config, - secrets, - secret_db, - keyshare_storage, - indexer: indexer_api, - currently_running_job_name: Arc::new(Mutex::new(String::new())), - debug_request_sender, - }; - coordinator.run().await - } -} - impl Cli { pub async fn run(self) -> anyhow::Result<()> { match self.command { - CliCommand::Start(start) => start.run().await, + CliCommand::StartWithConfigFile { config_path } => { + let node_configuration = StartConfig::from_json_file(&config_path)?; + run_mpc_node(node_configuration).await + } + // TODO(#2334): deprecate this + CliCommand::Start(start) => { + let home_dir = std::path::Path::new(&start.home_dir); + let config_file = load_config_file(home_dir)?; + + let node_configuration = start.into_start_config(config_file); + run_mpc_node(node_configuration).await + } CliCommand::Init(config) => { let (download_config_type, download_config_url) = if config.download_config { ( @@ -565,7 +283,7 @@ impl Cli { participants.len() == responders.len(), "Number of participants must match number of responders" ); - self.run_generate_test_configs( + run_generate_test_configs( output_dir, participants.clone(), responders.clone(), @@ -578,132 +296,7 @@ impl Cli { } } } - - fn duplicate_migrating_accounts( - mut accounts: Vec, - migrating_nodes: &[usize], - ) -> anyhow::Result> { - for migrating_node_idx in migrating_nodes { - let migrating_node_account: AccountId = accounts - .get(*migrating_node_idx) - .ok_or_else(|| { - anyhow::anyhow!("index {} out of bounds for accounts", migrating_node_idx) - })? - .clone(); - - accounts.push(migrating_node_account); - } - Ok(accounts) - } - - #[allow(clippy::too_many_arguments)] - fn run_generate_test_configs( - &self, - output_dir: &str, - participants: Vec, - responders: Vec, - threshold: usize, - desired_triples_to_buffer: usize, - desired_presignatures_to_buffer: usize, - desired_responder_keys_per_participant: usize, - migrating_nodes: &[usize], - ) -> anyhow::Result<()> { - let participants = Self::duplicate_migrating_accounts(participants, migrating_nodes)?; - let responders = Self::duplicate_migrating_accounts(responders, migrating_nodes)?; - - let p2p_key_pairs = participants - .iter() - .enumerate() - .map(|(idx, _account_id)| { - let subdir = PathBuf::from(output_dir).join(idx.to_string()); - PersistentSecrets::generate_or_get_existing( - &subdir, - desired_responder_keys_per_participant, - ) - .map(|secret| secret.p2p_private_key) - }) - .collect::, _>>()?; - let configs = generate_test_p2p_configs( - &participants, - threshold, - PortSeed::CLI_FOR_PYTEST, - Some(p2p_key_pairs), - )?; - let participants_config = configs[0].0.participants.clone(); - for (i, (_config, _p2p_private_key)) in configs.into_iter().enumerate() { - let subdir = format!("{}/{}", output_dir, i); - std::fs::create_dir_all(&subdir)?; - let file_config = self.create_file_config( - &participants[i], - &responders[i], - i, - desired_triples_to_buffer, - desired_presignatures_to_buffer, - )?; - std::fs::write( - format!("{}/config.yaml", subdir), - serde_yaml::to_string(&file_config)?, - )?; - } - std::fs::write( - format!("{}/participants.json", output_dir), - serde_json::to_string(&participants_config)?, - )?; - Ok(()) - } - - fn create_file_config( - &self, - participant: &AccountId, - responder: &AccountId, - index: usize, - desired_triples_to_buffer: usize, - desired_presignatures_to_buffer: usize, - ) -> anyhow::Result { - Ok(ConfigFile { - my_near_account_id: participant.clone(), - near_responder_account_id: responder.clone(), - number_of_responder_keys: 1, - web_ui: SocketAddr::new( - Ipv4Addr::LOCALHOST.into(), - PortSeed::CLI_FOR_PYTEST.web_port(index), - ), - migration_web_ui: SocketAddr::new( - Ipv4Addr::LOCALHOST.into(), - PortSeed::CLI_FOR_PYTEST.migration_web_port(index), - ), - pprof_bind_address: SocketAddr::new( - Ipv4Addr::LOCALHOST.into(), - PortSeed::CLI_FOR_PYTEST.pprof_web_port(index), - ), - indexer: IndexerConfig { - validate_genesis: true, - sync_mode: SyncMode::Block(BlockArgs { height: 0 }), - concurrency: 1.try_into().unwrap(), - mpc_contract_id: "test0".parse().unwrap(), - finality: Finality::None, - port_override: None, - }, - triple: TripleConfig { - concurrency: 2, - desired_triples_to_buffer, - timeout_sec: 60, - parallel_triple_generation_stagger_time_sec: 1, - }, - presignature: PresignatureConfig { - concurrency: 2, - desired_presignatures_to_buffer, - timeout_sec: 60, - }, - signature: SignatureConfig { timeout_sec: 60 }, - ckd: CKDConfig { timeout_sec: 60 }, - keygen: KeygenConfig { timeout_sec: 60 }, - foreign_chains: ForeignChainsConfig::default(), - cores: Some(4), - }) - } } - impl ImportKeyshareCmd { pub async fn run(&self) -> anyhow::Result<()> { let runtime = tokio::runtime::Runtime::new()?; @@ -800,7 +393,127 @@ impl ExportKeyshareCmd { }) } } +fn duplicate_migrating_accounts( + mut accounts: Vec, + migrating_nodes: &[usize], +) -> anyhow::Result> { + for migrating_node_idx in migrating_nodes { + let migrating_node_account: AccountId = accounts + .get(*migrating_node_idx) + .ok_or_else(|| { + anyhow::anyhow!("index {} out of bounds for accounts", migrating_node_idx) + })? + .clone(); + + accounts.push(migrating_node_account); + } + Ok(accounts) +} + +#[allow(clippy::too_many_arguments)] +fn run_generate_test_configs( + output_dir: &str, + participants: Vec, + responders: Vec, + threshold: usize, + desired_triples_to_buffer: usize, + desired_presignatures_to_buffer: usize, + desired_responder_keys_per_participant: usize, + migrating_nodes: &[usize], +) -> anyhow::Result<()> { + let participants = duplicate_migrating_accounts(participants, migrating_nodes)?; + let responders = duplicate_migrating_accounts(responders, migrating_nodes)?; + + let p2p_key_pairs = participants + .iter() + .enumerate() + .map(|(idx, _account_id)| { + let subdir = PathBuf::from(output_dir).join(idx.to_string()); + PersistentSecrets::generate_or_get_existing( + &subdir, + desired_responder_keys_per_participant, + ) + .map(|secret| secret.p2p_private_key) + }) + .collect::, _>>()?; + let configs = generate_test_p2p_configs( + &participants, + threshold, + PortSeed::CLI_FOR_PYTEST, + Some(p2p_key_pairs), + )?; + let participants_config = configs[0].0.participants.clone(); + for (i, (_config, _p2p_private_key)) in configs.into_iter().enumerate() { + let subdir = format!("{}/{}", output_dir, i); + std::fs::create_dir_all(&subdir)?; + let file_config = create_file_config( + &participants[i], + &responders[i], + i, + desired_triples_to_buffer, + desired_presignatures_to_buffer, + ); + std::fs::write( + format!("{}/mpc_node_config.json", subdir), + serde_json::to_string_pretty(&file_config)?, + )?; + } + std::fs::write( + format!("{}/participants.json", output_dir), + serde_json::to_string(&participants_config)?, + )?; + Ok(()) +} +fn create_file_config( + participant: &AccountId, + responder: &AccountId, + index: usize, + desired_triples_to_buffer: usize, + desired_presignatures_to_buffer: usize, +) -> ConfigFile { + ConfigFile { + my_near_account_id: participant.clone(), + near_responder_account_id: responder.clone(), + number_of_responder_keys: 1, + web_ui: SocketAddr::new( + Ipv4Addr::LOCALHOST.into(), + PortSeed::CLI_FOR_PYTEST.web_port(index), + ), + migration_web_ui: SocketAddr::new( + Ipv4Addr::LOCALHOST.into(), + PortSeed::CLI_FOR_PYTEST.migration_web_port(index), + ), + pprof_bind_address: SocketAddr::new( + Ipv4Addr::LOCALHOST.into(), + PortSeed::CLI_FOR_PYTEST.pprof_web_port(index), + ), + indexer: IndexerConfig { + validate_genesis: true, + sync_mode: SyncMode::Block(BlockArgs { height: 0 }), + concurrency: 1.try_into().unwrap(), + mpc_contract_id: "test0".parse().unwrap(), + finality: Finality::None, + port_override: None, + }, + triple: TripleConfig { + concurrency: 2, + desired_triples_to_buffer, + timeout_sec: 60, + parallel_triple_generation_stagger_time_sec: 1, + }, + presignature: PresignatureConfig { + concurrency: 2, + desired_presignatures_to_buffer, + timeout_sec: 60, + }, + signature: SignatureConfig { timeout_sec: 60 }, + ckd: CKDConfig { timeout_sec: 60 }, + keygen: KeygenConfig { timeout_sec: 60 }, + foreign_chains: ForeignChainsConfig::default(), + cores: Some(4), + } +} #[cfg(test)] mod tests { use super::*; diff --git a/crates/node/src/config.rs b/crates/node/src/config.rs index fdabd98bf..f48c9f922 100644 --- a/crates/node/src/config.rs +++ b/crates/node/src/config.rs @@ -13,6 +13,11 @@ use std::{ path::Path, }; +mod start; +pub use start::{ + GcpStartConfig, SecretsStartConfig, StartConfig, TeeAuthorityStartConfig, TeeStartConfig, +}; + mod foreign_chains; pub use foreign_chains::{ AbstractApiVariant, AbstractChainConfig, AbstractProviderConfig, AuthConfig, BitcoinApiVariant, diff --git a/crates/node/src/config/foreign_chains/auth.rs b/crates/node/src/config/foreign_chains/auth.rs index 5269b3130..3540fe837 100644 --- a/crates/node/src/config/foreign_chains/auth.rs +++ b/crates/node/src/config/foreign_chains/auth.rs @@ -52,6 +52,10 @@ pub enum TokenConfig { impl TokenConfig { pub fn resolve(&self) -> anyhow::Result { match self { + // TODO(#2335): do not resolve env variables this deep in the binary. + // Should be resolved at start, preferably in the config so we can kill env configs + // + // One option is to have a separate secrets config file. TokenConfig::Env { env } => { std::env::var(env).with_context(|| format!("environment variable {env} is not set")) } diff --git a/crates/node/src/config/start.rs b/crates/node/src/config/start.rs new file mode 100644 index 000000000..b2a26e565 --- /dev/null +++ b/crates/node/src/config/start.rs @@ -0,0 +1,128 @@ +use super::ConfigFile; +use anyhow::Context; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; +use tee_authority::tee_authority::{ + DstackTeeAuthorityConfig, LocalTeeAuthorityConfig, TeeAuthority, DEFAULT_DSTACK_ENDPOINT, + DEFAULT_PHALA_TDX_QUOTE_UPLOAD_URL, +}; +use url::Url; + +/// Configuration for starting the MPC node. This is the canonical type used +/// by the run logic. Both `StartCmd` (CLI flags) and `StartWithConfigFileCmd` +/// (JSON file) convert into this type. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StartConfig { + pub home_dir: PathBuf, + /// Encryption keys and backup settings. + pub secrets: SecretsStartConfig, + /// TEE authority and image hash monitoring settings. + pub tee: TeeStartConfig, + /// GCP keyshare storage settings. Optional — omit if not using GCP. + #[serde(default)] + pub gcp: Option, + /// Node configuration (indexer, protocol parameters, etc.). + pub node: ConfigFile, +} + +/// Encryption keys needed at startup. +#[derive(Clone, Serialize, Deserialize)] +pub struct SecretsStartConfig { + /// Hex-encoded 16 byte AES key for local storage encryption. + pub secret_store_key_hex: String, + /// Hex-encoded 32 byte AES key for backup encryption. + /// If not provided, a key is generated and written to disk. + #[serde(default)] + pub backup_encryption_key_hex: Option, +} + +impl std::fmt::Debug for SecretsStartConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SecretsStartConfig") + .field("secret_store_key_hex", &"[REDACTED]") + .field( + "backup_encryption_key_hex", + &self + .backup_encryption_key_hex + .as_ref() + .map(|_| "[REDACTED]"), + ) + .finish() + } +} + +/// TEE-related configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TeeStartConfig { + /// TEE authority configuration. + pub authority: TeeAuthorityStartConfig, + /// Hex representation of the hash of the running image. Only required in TEE. + #[serde(default)] + pub image_hash: Option, + /// Path to the file where the node writes the latest allowed hash. + /// If not set, assumes running outside of TEE and skips image hash monitoring. + #[serde(default)] + pub latest_allowed_hash_file: Option, +} + +/// GCP keyshare storage configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GcpStartConfig { + /// GCP secret ID for storing the root keyshare. + pub keyshare_secret_id: String, + /// GCP project ID. + pub project_id: String, +} + +/// TEE authority configuration for JSON deserialization. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum TeeAuthorityStartConfig { + Local, + Dstack { + #[serde(default = "default_dstack_endpoint")] + dstack_endpoint: String, + #[serde(default = "default_quote_upload_url")] + // TODO(#2333): use URL type for this type + quote_upload_url: String, + }, +} + +fn default_dstack_endpoint() -> String { + DEFAULT_DSTACK_ENDPOINT.to_string() +} + +fn default_quote_upload_url() -> String { + DEFAULT_PHALA_TDX_QUOTE_UPLOAD_URL.to_string() +} + +impl TeeAuthorityStartConfig { + pub fn into_tee_authority(self) -> anyhow::Result { + Ok(match self { + TeeAuthorityStartConfig::Local => LocalTeeAuthorityConfig::default().into(), + TeeAuthorityStartConfig::Dstack { + dstack_endpoint, + quote_upload_url, + } => { + let url: Url = quote_upload_url + .parse() + .context("invalid quote_upload_url")?; + DstackTeeAuthorityConfig::new(dstack_endpoint, url).into() + } + }) + } +} + +impl StartConfig { + pub fn from_json_file(path: &std::path::Path) -> anyhow::Result { + let content = std::fs::read_to_string(path) + .with_context(|| format!("failed to read config file: {}", path.display()))?; + let config: Self = serde_json::from_str(&content) + .with_context(|| format!("failed to parse config file: {}", path.display()))?; + config + .node + .validate() + .context("invalid node config in config file")?; + Ok(config) + } +} diff --git a/crates/node/src/lib.rs b/crates/node/src/lib.rs index 71778f6fe..dffd498fe 100644 --- a/crates/node/src/lib.rs +++ b/crates/node/src/lib.rs @@ -39,6 +39,7 @@ mod protocol; mod protocol_version; mod providers; pub mod requests; +mod run; mod runtime; mod storage; pub mod tracing; diff --git a/crates/node/src/run.rs b/crates/node/src/run.rs new file mode 100644 index 000000000..93b9c99fe --- /dev/null +++ b/crates/node/src/run.rs @@ -0,0 +1,308 @@ +use crate::{ + config::{ + generate_and_write_backup_encryption_key_to_disk, ConfigFile, PersistentSecrets, + RespondConfig, SecretsConfig, StartConfig, + }, + coordinator::Coordinator, + db::SecretDB, + indexer::{ + real::spawn_real_indexer, tx_sender::TransactionSender, IndexerAPI, ReadForeignChainPolicy, + }, + keyshare::{GcpPermanentKeyStorageConfig, KeyStorageConfig, KeyshareStorage}, + migration_service::spawn_recovery_server_and_run_onboarding, + profiler, + tracking::{self, start_root_task}, + web::{start_web_server, static_web_data, DebugRequest}, +}; +use anyhow::{anyhow, Context}; +use contract_interface::types::Ed25519PublicKey; +use mpc_attestation::report_data::ReportDataV1; +use mpc_contract::state::ProtocolContractState; +use mpc_contract::tee::proposal::MpcDockerImageHash; +use near_time::Clock; +use std::{ + collections::BTreeMap, + path::PathBuf, + sync::{Arc, Mutex, OnceLock}, + time::Duration, +}; +use tee_authority::tee_authority::TeeAuthority; +use tokio::sync::{broadcast, mpsc, oneshot, watch, RwLock}; +use tokio_util::sync::CancellationToken; +use tracing::info; + +use crate::tee::{ + monitor_allowed_image_hashes, + remote_attestation::{monitor_attestation_removal, periodic_attestation_submission}, + AllowedImageHashesFile, +}; + +pub const ATTESTATION_RESUBMISSION_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour + +pub async fn run_mpc_node(config: StartConfig) -> anyhow::Result<()> { + let root_runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .worker_threads(1) + .build()?; + + let _tokio_enter_guard = root_runtime.enter(); + + // Load configuration and initialize persistent secrets + let node_config = config.node.clone(); + let persistent_secrets = PersistentSecrets::generate_or_get_existing( + &config.home_dir, + node_config.number_of_responder_keys, + )?; + + profiler::web_server::start_web_server(node_config.pprof_bind_address).await?; + root_runtime.spawn(crate::metrics::tokio_task_metrics::run_monitor_loop()); + + // TODO(#1296): Decide if the MPC responder account is actually needed + let respond_config = RespondConfig::from_parts(&node_config, &persistent_secrets); + + let backup_encryption_key_hex = match &config.secrets.backup_encryption_key_hex { + Some(key) => key.clone(), + None => generate_and_write_backup_encryption_key_to_disk(&config.home_dir)?, + }; + + // Load secrets from configuration and persistent storage + let secrets = SecretsConfig::from_parts( + &config.secrets.secret_store_key_hex, + persistent_secrets.clone(), + &backup_encryption_key_hex, + )?; + + // Generate attestation + let tee_authority = config.tee.authority.clone().into_tee_authority()?; + let tls_public_key = &secrets.persistent_secrets.p2p_private_key.verifying_key(); + + let account_public_key = &secrets.persistent_secrets.near_signer_key.verifying_key(); + + let report_data = ReportDataV1::new( + *Ed25519PublicKey::from(tls_public_key).as_bytes(), + *Ed25519PublicKey::from(account_public_key).as_bytes(), + ) + .into(); + + let attestation = tee_authority.generate_attestation(report_data).await?; + + // Create communication channels and runtime + let (debug_request_sender, _) = tokio::sync::broadcast::channel(10); + let root_task_handle = Arc::new(OnceLock::new()); + + let (protocol_state_sender, protocol_state_receiver) = + watch::channel(ProtocolContractState::NotInitialized); + + let (migration_state_sender, migration_state_receiver) = watch::channel((0, BTreeMap::new())); + let web_server = root_runtime + .block_on(start_web_server( + root_task_handle.clone(), + debug_request_sender.clone(), + node_config.web_ui, + static_web_data(&secrets, Some(attestation)), + protocol_state_receiver, + migration_state_receiver, + )) + .context("Failed to create web server.")?; + + let _web_server_join_handle = root_runtime.spawn(web_server); + + // Create Indexer and wait for indexer to be synced. + let (indexer_exit_sender, indexer_exit_receiver) = oneshot::channel(); + let indexer_api = spawn_real_indexer( + config.home_dir.clone(), + node_config.indexer.clone(), + node_config.my_near_account_id.clone(), + persistent_secrets.near_signer_key.clone(), + respond_config, + indexer_exit_sender, + protocol_state_sender, + migration_state_sender, + *tls_public_key, + ); + + let (shutdown_signal_sender, mut shutdown_signal_receiver) = mpsc::channel(1); + let cancellation_token = CancellationToken::new(); + + let image_hash_watcher_handle = if let (Some(image_hash), Some(latest_allowed_hash_file)) = + (&config.tee.image_hash, &config.tee.latest_allowed_hash_file) + { + let current_image_hash_bytes: [u8; 32] = hex::decode(image_hash) + .expect("The currently running image is a hex string.") + .try_into() + .expect("The currently running image hash hex representation is 32 bytes."); + + let allowed_hashes_in_contract = indexer_api.allowed_docker_images_receiver.clone(); + let image_hash_storage = AllowedImageHashesFile::from(latest_allowed_hash_file.clone()); + + Some(root_runtime.spawn(monitor_allowed_image_hashes( + cancellation_token.child_token(), + MpcDockerImageHash::from(current_image_hash_bytes), + allowed_hashes_in_contract, + image_hash_storage, + shutdown_signal_sender.clone(), + ))) + } else { + tracing::info!( + "image_hash and/or latest_allowed_hash_file not set, skipping TEE image hash monitoring" + ); + None + }; + + let home_dir = config.home_dir.clone(); + let root_future = create_root_future( + config, + home_dir.clone(), + node_config.clone(), + secrets.clone(), + indexer_api, + debug_request_sender, + root_task_handle, + tee_authority, + ); + + let root_task = root_runtime.spawn(start_root_task("root", root_future).0); + + let exit_reason = tokio::select! { + root_task_result = root_task => { + root_task_result? + } + indexer_exit_response = indexer_exit_receiver => { + indexer_exit_response.context("Indexer thread dropped response channel.")? + } + Some(()) = shutdown_signal_receiver.recv() => { + Err(anyhow!("TEE allowed image hashes watcher is sending shutdown signal.")) + } + }; + + // Perform graceful shutdown + cancellation_token.cancel(); + + if let Some(handle) = image_hash_watcher_handle { + info!("Waiting for image hash watcher to gracefully exit."); + let exit_result = handle.await; + info!(?exit_result, "Image hash watcher exited."); + } + + exit_reason +} + +#[allow(clippy::too_many_arguments)] +async fn create_root_future( + start_config: StartConfig, + home_dir: PathBuf, + config: ConfigFile, + secrets: SecretsConfig, + indexer_api: IndexerAPI, + debug_request_sender: broadcast::Sender, + // Cloning a OnceLock returns a new cell, which is why we have to wrap it in an arc. + // Otherwise we would not write to the same cell/lock. + root_task_handle_once_lock: Arc>>, + tee_authority: TeeAuthority, +) -> anyhow::Result<()> +where + TransactionSenderImpl: TransactionSender + 'static, + ForeignChainPolicyReader: ReadForeignChainPolicy + Clone + Send + Sync + 'static, +{ + let root_task_handle = tracking::current_task(); + + root_task_handle_once_lock + .set(root_task_handle.clone()) + .map_err(|_| anyhow!("Root task handle was already set"))?; + + let tls_public_key = + Ed25519PublicKey::from(&secrets.persistent_secrets.p2p_private_key.verifying_key()); + let account_public_key = + Ed25519PublicKey::from(&secrets.persistent_secrets.near_signer_key.verifying_key()); + + let secret_db = SecretDB::new(&home_dir.join("assets"), secrets.local_storage_aes_key)?; + + let key_storage_config = KeyStorageConfig { + home_dir: home_dir.clone(), + local_encryption_key: secrets.local_storage_aes_key, + gcp: start_config.gcp.map(|gcp| GcpPermanentKeyStorageConfig { + project_id: gcp.project_id, + secret_id: gcp.keyshare_secret_id, + }), + }; + + // Spawn periodic attestation submission task + let tee_authority_clone = tee_authority.clone(); + let tx_sender_clone = indexer_api.txn_sender.clone(); + let tls_public_key_clone = tls_public_key.clone(); + let account_public_key_clone = account_public_key.clone(); + let allowed_docker_images_receiver_clone = indexer_api.allowed_docker_images_receiver.clone(); + let allowed_launcher_compose_receiver_clone = + indexer_api.allowed_launcher_compose_receiver.clone(); + tokio::spawn(async move { + if let Err(e) = periodic_attestation_submission( + tee_authority_clone, + tx_sender_clone, + tls_public_key_clone, + account_public_key_clone, + allowed_docker_images_receiver_clone, + allowed_launcher_compose_receiver_clone, + tokio::time::interval(ATTESTATION_RESUBMISSION_INTERVAL), + ) + .await + { + tracing::error!( + error = ?e, + "periodic attestation submission task failed" + ); + } + }); + + // Spawn TEE attestation monitoring task + let tx_sender_clone = indexer_api.txn_sender.clone(); + let tee_accounts_receiver = indexer_api.attested_nodes_receiver.clone(); + let account_id_clone = config.my_near_account_id.clone(); + let allowed_docker_images_receiver_clone = indexer_api.allowed_docker_images_receiver.clone(); + let allowed_launcher_compose_receiver_clone = + indexer_api.allowed_launcher_compose_receiver.clone(); + tokio::spawn(async move { + if let Err(e) = monitor_attestation_removal( + account_id_clone, + tee_authority, + tx_sender_clone, + tls_public_key, + account_public_key, + allowed_docker_images_receiver_clone, + allowed_launcher_compose_receiver_clone, + tee_accounts_receiver, + ) + .await + { + tracing::error!( + error = ?e, + "attestation removal monitoring task failed" + ); + } + }); + + let keyshare_storage: Arc> = + RwLock::new(key_storage_config.create().await?).into(); + + spawn_recovery_server_and_run_onboarding( + config.migration_web_ui, + (&secrets).into(), + config.my_near_account_id.clone(), + keyshare_storage.clone(), + indexer_api.my_migration_info_receiver.clone(), + indexer_api.contract_state_receiver.clone(), + indexer_api.txn_sender.clone(), + ) + .await?; + + let coordinator = Coordinator { + clock: Clock::real(), + config_file: config, + secrets, + secret_db, + keyshare_storage, + indexer: indexer_api, + currently_running_job_name: Arc::new(Mutex::new(String::new())), + debug_request_sender, + }; + coordinator.run().await +} diff --git a/docs/localnet/localnet.md b/docs/localnet/localnet.md index 24dc69ce8..90045af21 100644 --- a/docs/localnet/localnet.md +++ b/docs/localnet/localnet.md @@ -191,65 +191,13 @@ Since this is not a validator node, we can remove `validator_key.json` rm ~/.near/mpc-frodo/validator_key.json ``` -Next we'll create a `config.yaml` for the MPC-indexer: - -```shell -cat > ~/.near/mpc-frodo/config.yaml << 'EOF' -my_near_account_id: frodo.test.near -near_responder_account_id: frodo.test.near -number_of_responder_keys: 1 -web_ui: 127.0.0.1:8081 -migration_web_ui: 127.0.0.1:8079 -pprof_bind_address: 127.0.0.1:34001 -triple: - concurrency: 2 - desired_triples_to_buffer: 128 - timeout_sec: 60 - parallel_triple_generation_stagger_time_sec: 1 -presignature: - concurrency: 4 - desired_presignatures_to_buffer: 64 - timeout_sec: 60 -signature: - timeout_sec: 60 -indexer: - validate_genesis: false - sync_mode: Latest - concurrency: 1 - mpc_contract_id: mpc-contract.test.near - finality: optimistic -ckd: - timeout_sec: 60 -cores: 4 -foreign_chains: - bitcoin: - timeout_sec: 30 - max_retries: 3 - providers: - public: - api_variant: esplora - rpc_url: "https://bitcoin-rpc.publicnode.com" - auth: - kind: none - abstract: - timeout_sec: 30 - max_retries: 3 - providers: - public: - api_variant: standard - rpc_url: "https://api.testnet.abs.xyz" - auth: - kind: none - starknet: - timeout_sec: 30 - max_retries: 3 - providers: - public: - api_variant: standard - rpc_url: "https://starknet-rpc.publicnode.com" - auth: - kind: none -EOF +Next we'll create a JSON configuration file for Frodo's MPC node using the +shared template at `docs/localnet/mpc-config.template.json`. This single file +contains all settings (secrets, TEE config, and node parameters): + +```shell +env MPC_NODE_ID=mpc-frodo NEAR_ACCOUNT_ID=frodo.test.near WEB_UI_PORT=8081 MIGRATION_WEB_UI_PORT=8079 PPROF_PORT=34001 \ + envsubst < docs/localnet/mpc-config.template.json > ~/.near/mpc-frodo/mpc-config.json ``` ### Initialize Sam's node @@ -273,79 +221,24 @@ rm ~/.near/mpc-sam/validator_key.json ``` ```shell -cat > ~/.near/mpc-sam/config.yaml << 'EOF' -my_near_account_id: sam.test.near -near_responder_account_id: sam.test.near -number_of_responder_keys: 1 -web_ui: 127.0.0.1:8082 -migration_web_ui: 127.0.0.1:8078 -pprof_bind_address: 127.0.0.1:34002 -triple: - concurrency: 2 - desired_triples_to_buffer: 128 - timeout_sec: 60 - parallel_triple_generation_stagger_time_sec: 1 -presignature: - concurrency: 4 - desired_presignatures_to_buffer: 64 - timeout_sec: 60 -signature: - timeout_sec: 60 -indexer: - validate_genesis: false - sync_mode: Latest - concurrency: 1 - mpc_contract_id: mpc-contract.test.near - finality: optimistic -ckd: - timeout_sec: 60 -cores: 4 -foreign_chains: - bitcoin: - timeout_sec: 30 - max_retries: 3 - providers: - public: - api_variant: esplora - rpc_url: "https://bitcoin-rpc.publicnode.com" - auth: - kind: none - abstract: - timeout_sec: 30 - max_retries: 3 - providers: - public: - api_variant: standard - rpc_url: "https://api.testnet.abs.xyz" - auth: - kind: none - starknet: - timeout_sec: 30 - max_retries: 3 - providers: - public: - api_variant: standard - rpc_url: "https://starknet-rpc.publicnode.com" - auth: - kind: none -EOF +env MPC_NODE_ID=mpc-sam NEAR_ACCOUNT_ID=sam.test.near WEB_UI_PORT=8082 MIGRATION_WEB_UI_PORT=8078 PPROF_PORT=34002 \ + envsubst < docs/localnet/mpc-config.template.json > ~/.near/mpc-sam/mpc-config.json ``` ### Run the MPC binary -In two separate shells run the MPC binary for frodo and sam. Note the last argument repeating (`11111111111111111111111111111111`) is the encryption key for the secret storage, and can be any arbitrary value. +In two separate shells run the MPC binary for Frodo and Sam using their JSON config files: ```shell -RUST_LOG=info mpc-node start --home-dir ~/.near/mpc-sam/ 11111111111111111111111111111111 --image-hash "8b40f81f77b8c22d6c777a6e14d307a1d11cb55ab83541fbb8575d02d86a74b0" --latest-allowed-hash-file /temp/LATEST_ALLOWED_HASH_FILE.txt local +RUST_LOG=info mpc-node start-with-config-file ~/.near/mpc-sam/mpc-config.json ``` ```shell -RUST_LOG=info mpc-node start --home-dir ~/.near/mpc-frodo/ 11111111111111111111111111111111 --image-hash "8b40f81f77b8c22d6c777a6e14d307a1d11cb55ab83541fbb8575d02d86a74b0" --latest-allowed-hash-file /temp/LATEST_ALLOWED_HASH_FILE.txt local +RUST_LOG=info mpc-node start-with-config-file ~/.near/mpc-frodo/mpc-config.json ``` Notes: -- `8b40f81f77b8c22d6c777a6e14d307a1d11cb55ab83541fbb8575d02d86a74b0` is just an arbitrary hash. - If you get the following error: ```console diff --git a/docs/localnet/mpc-config.template.json b/docs/localnet/mpc-config.template.json new file mode 100644 index 000000000..07e15ff27 --- /dev/null +++ b/docs/localnet/mpc-config.template.json @@ -0,0 +1,75 @@ +{ + "home_dir": "$HOME/.near/$MPC_NODE_ID", + "secrets": { + "secret_store_key_hex": "11111111111111111111111111111111" + }, + "tee": { + "authority": { "type": "local" }, + "image_hash": "8b40f81f77b8c22d6c777a6e14d307a1d11cb55ab83541fbb8575d02d86a74b0", + "latest_allowed_hash_file": "/tmp/LATEST_ALLOWED_HASH_FILE.txt" + }, + "node": { + "my_near_account_id": "$NEAR_ACCOUNT_ID", + "near_responder_account_id": "$NEAR_ACCOUNT_ID", + "number_of_responder_keys": 1, + "web_ui": "127.0.0.1:$WEB_UI_PORT", + "migration_web_ui": "127.0.0.1:$MIGRATION_WEB_UI_PORT", + "pprof_bind_address": "127.0.0.1:$PPROF_PORT", + "triple": { + "concurrency": 2, + "desired_triples_to_buffer": 128, + "timeout_sec": 60, + "parallel_triple_generation_stagger_time_sec": 1 + }, + "presignature": { + "concurrency": 4, + "desired_presignatures_to_buffer": 64, + "timeout_sec": 60 + }, + "signature": { "timeout_sec": 60 }, + "indexer": { + "validate_genesis": false, + "sync_mode": "Latest", + "concurrency": 1, + "mpc_contract_id": "mpc-contract.test.near", + "finality": "optimistic" + }, + "ckd": { "timeout_sec": 60 }, + "cores": 4, + "foreign_chains": { + "bitcoin": { + "timeout_sec": 30, + "max_retries": 3, + "providers": { + "public": { + "api_variant": "esplora", + "rpc_url": "https://bitcoin-rpc.publicnode.com", + "auth": { "kind": "none" } + } + } + }, + "abstract": { + "timeout_sec": 30, + "max_retries": 3, + "providers": { + "public": { + "api_variant": "standard", + "rpc_url": "https://api.testnet.abs.xyz", + "auth": { "kind": "none" } + } + } + }, + "starknet": { + "timeout_sec": 30, + "max_retries": 3, + "providers": { + "public": { + "api_variant": "standard", + "rpc_url": "https://starknet-rpc.publicnode.com", + "auth": { "kind": "none" } + } + } + } + } + } +} diff --git a/pytest/common_lib/shared/__init__.py b/pytest/common_lib/shared/__init__.py index 48cb7cb3a..8fdfdad44 100644 --- a/pytest/common_lib/shared/__init__.py +++ b/pytest/common_lib/shared/__init__.py @@ -42,7 +42,7 @@ dot_near = pathlib.Path.home() / ".near" SECRETS_JSON = "secrets.json" NUMBER_OF_VALIDATORS = 1 -CONFIG_YAML = "config.yaml" +MPC_NODE_CONFIG_JSON = "mpc_node_config.json" def create_function_call_access_key_action( @@ -230,6 +230,7 @@ class ConfigValues: migration_address: str pprof_address: str backup_key: bytes + node_config: dict # JSON-serializable dict matching Rust ConfigFile def generate_mpc_configs( @@ -307,9 +308,9 @@ def generate_mpc_configs( my_port = participant["port"] p2p_url = f"http://{my_addr}:{my_port}" - config_file_path = os.path.join(dot_near, str(idx), CONFIG_YAML) + config_file_path = os.path.join(dot_near, str(idx), MPC_NODE_CONFIG_JSON) with open(config_file_path, "r") as f: - config = yaml.load(f, Loader=SafeLoaderIgnoreUnknown) + config = json.load(f) web_address = config.get("web_ui") migration_address = config.get("migration_web_ui") @@ -331,6 +332,7 @@ def generate_mpc_configs( ] backup_key = os.urandom(32) + configs.append( ConfigValues( signer_key, @@ -341,6 +343,7 @@ def generate_mpc_configs( migration_address, pprof_address, backup_key, + node_config=config, ) ) return configs @@ -531,6 +534,7 @@ def start_cluster_with_mpc( pytest_signer_keys=pytest_signer_keys, backup_key=config.backup_key, pprof_address=config.pprof_address, + node_config=config.node_config, ) mpc_node.init_nonces(validators[0]) mpc_node.set_block_ingestion(True) diff --git a/pytest/common_lib/shared/foreign_chains.py b/pytest/common_lib/shared/foreign_chains.py index 5d71cb30c..fa6193b87 100644 --- a/pytest/common_lib/shared/foreign_chains.py +++ b/pytest/common_lib/shared/foreign_chains.py @@ -1,32 +1,13 @@ """Shared helpers for foreign chain configuration and policy tests.""" -import pathlib -import re from typing import Any -import yaml - - -def node_config_path(node) -> pathlib.Path: - return pathlib.Path(node.home_dir) / "config.yaml" - def set_foreign_chains_config(node, foreign_chains: dict[str, Any] | None) -> None: - config_path = node_config_path(node) - - config_text = config_path.read_text(encoding="utf-8") - # Keep generated YAML tags intact by editing only the trailing `foreign_chains` section. - config_text = ( - re.sub(r"\nforeign_chains:[\s\S]*\Z", "\n", config_text).rstrip() + "\n" - ) - if foreign_chains is not None: - foreign_chains_text = yaml.safe_dump( - {"foreign_chains": foreign_chains}, sort_keys=False - ) - config_text += "\n" + foreign_chains_text - - config_path.write_text(config_text, encoding="utf-8") + node.node_config["foreign_chains"] = foreign_chains + else: + node.node_config["foreign_chains"] = {} def normalize_policy(policy: dict[str, Any]) -> list[tuple[str, tuple[str, ...]]]: diff --git a/pytest/common_lib/shared/mpc_node.py b/pytest/common_lib/shared/mpc_node.py index 436ded22a..c8b87d5ce 100644 --- a/pytest/common_lib/shared/mpc_node.py +++ b/pytest/common_lib/shared/mpc_node.py @@ -61,6 +61,7 @@ def __init__( p2p_public_key: str, pytest_signer_keys: list[Key], backup_key: bytes, + node_config: dict, ): super().__init__(near_node, signer_key, pytest_signer_keys) self.p2p_url: str = p2p_url @@ -74,6 +75,7 @@ def __init__( self.is_running = False self.metrics = MetricsTracker(near_node) self.backup_key = backup_key + self.node_config = node_config def print(self): if not self.is_running: @@ -127,22 +129,38 @@ def reset_mpc_data(self): for file_path in pathlib.Path(self.home_dir).glob(pattern): file_path.unlink() + def _write_start_config(self) -> str: + """Build a StartConfig JSON file and write it to the node's home dir. + Returns the path to the written config file.""" + start_config = { + "home_dir": self.home_dir, + "secrets": { + "secret_store_key_hex": self.secret_store_key, + "backup_encryption_key_hex": self.backup_key.hex(), + }, + "tee": { + "authority": {"type": "local"}, + "image_hash": DUMMY_MPC_IMAGE_HASH, + "latest_allowed_hash_file": "latest_allowed_hash.txt", + }, + "node": self.node_config, + } + config_path = str(pathlib.Path(self.home_dir) / "start_config.json") + with open(config_path, "w") as f: + json.dump(start_config, f, indent=2) + return config_path + def run(self): assert not self.is_running self.is_running = True + config_path = self._write_start_config() extra_env = { "RUST_LOG": "INFO", # mpc-node produces too much output on DEBUG - "MPC_SECRET_STORE_KEY": self.secret_store_key, - "MPC_IMAGE_HASH": DUMMY_MPC_IMAGE_HASH, - "MPC_LATEST_ALLOWED_HASH_FILE": "latest_allowed_hash.txt", - "MPC_BACKUP_ENCRYPTION_KEY_HEX": self.backup_key.hex(), } cmd = ( MPC_BINARY_PATH, - "start", - "--home-dir", - self.home_dir, - "local", + "start-with-config-file", + config_path, ) self.near_node.run_cmd(cmd=cmd, extra_env=extra_env) diff --git a/tee_launcher/launcher.py b/tee_launcher/launcher.py index a0dd34958..42373b657 100644 --- a/tee_launcher/launcher.py +++ b/tee_launcher/launcher.py @@ -80,12 +80,20 @@ class Platform(Enum): DEFAULT_MPC_REGISTRY = "registry.hub.docker.com" DEFAULT_MPC_IMAGE_TAG = "latest" +# --- MPC node configuration file --- +# Key in dstack user config that specifies the path to the MPC node JSON config file. +# This path must be accessible inside the container (e.g. under /tapp which is mounted read-only). +DSTACK_USER_CONFIG_MPC_CONFIG_PATH = "MPC_CONFIG_PATH" +# Default path for the MPC node config file (inside /tapp, provided by the operator). +DEFAULT_MPC_CONFIG_PATH = "/tapp/mpc_config.json" + # Environment variables that configure the launcher itself. # These are read from the user config file but should NEVER be passed to the MPC container. ALLOWED_LAUNCHER_ENV_VARS = { DSTACK_USER_CONFIG_MPC_IMAGE_TAGS, DSTACK_USER_CONFIG_MPC_IMAGE_NAME, DSTACK_USER_CONFIG_MPC_IMAGE_REGISTRY, + DSTACK_USER_CONFIG_MPC_CONFIG_PATH, ENV_VAR_MPC_HASH_OVERRIDE, ENV_VAR_RPC_REQUEST_TIMEOUT_SECS, ENV_VAR_RPC_REQUEST_INTERVAL_SECS, @@ -105,59 +113,6 @@ class Platform(Enum): JSON_KEY_APPROVED_HASHES = "approved_hashes" -# -------------------------------------------------------------------------------------- -# Security policy for env passthrough -# -------------------------------------------------------------------------------------- - -# Allow all MPC_* keys, but keep validation strict. -MPC_ENV_KEY_RE = re.compile(r"^MPC_[A-Z0-9_]{1,64}$") - - -# Hard caps to prevent DoS via huge env payloads. -MAX_PASSTHROUGH_ENV_VARS = 64 -MAX_ENV_VALUE_LEN = 1024 -MAX_TOTAL_ENV_BYTES = 32 * 1024 # 32KB total across passed envs - -# Never pass raw private keys via launcher (any platform) -DENIED_CONTAINER_ENV_KEYS = { - "MPC_P2P_PRIVATE_KEY", - "MPC_ACCOUNT_SK", -} - -# Example of .user-config file format: -# -# MPC_ACCOUNT_ID=mpc-user-123 -# MPC_LOCAL_ADDRESS=127.0.0.1 -# MPC_SECRET_STORE_KEY=secret -# MPC_CONTRACT_ID=mpc-contract -# MPC_ENV=testnet -# MPC_HOME_DIR=/data -# NEAR_BOOT_NODES=boot1,boot2 -# RUST_BACKTRACE=1 -# RUST_LOG=info -# MPC_RESPONDER_ID=responder-xyz -# EXTRA_HOSTS=host1:192.168.0.1,host2:192.168.0.2 -# PORTS=11780:11780,2200:2200 - -# Define an allow-list of permitted environment variables that will be passed to MPC container. -# Note - extra hosts and port forwarding are explicitly defined in the docker run command generation. -# NOTE: Kept for backwards compatibility and for documentation purposes; the effective policy is: -# - allow MPC_* keys that match MPC_ENV_KEY_RE -# - plus existing non-MPC keys below (RUST_LOG / RUST_BACKTRACE / NEAR_BOOT_NODES) -ALLOWED_MPC_ENV_VARS = { - "MPC_ACCOUNT_ID", # ID of the MPC account on the network - "MPC_LOCAL_ADDRESS", # Local IP address or hostname used by the MPC node - "MPC_SECRET_STORE_KEY", # Key used to encrypt/decrypt secrets - "MPC_CONTRACT_ID", # Contract ID associated with the MPC node - "MPC_ENV", # Environment (e.g., 'testnet', 'mainnet') - "MPC_HOME_DIR", # Home directory for the MPC node - "NEAR_BOOT_NODES", # Comma-separated list of boot nodes - "RUST_BACKTRACE", # Enables backtraces for Rust errors - "RUST_LOG", # Logging level for Rust code - "MPC_RESPONDER_ID", # Unique responder ID for MPC communication - "MPC_BACKUP_ENCRYPTION_KEY_HEX", # encryption key for backups -} - # Regex: hostnames must be alphanum + dash/dot, IPs must be valid IPv4 HOST_ENTRY_RE = re.compile(r"^[a-zA-Z0-9\-\.]+:\d{1,3}(\.\d{1,3}){3}$") PORT_MAPPING_RE = re.compile(r"^(\d{1,5}):(\d{1,5})$") @@ -166,33 +121,6 @@ class Platform(Enum): INVALID_HOST_ENTRY_PATTERN = re.compile(r"^[;&|`$\\<>-]|^--") -def _has_control_chars(s: str) -> bool: - # Disallow NUL + CR/LF at minimum; also block other ASCII control chars (< 0x20) except tab. - for ch in s: - oc = ord(ch) - if ch in ("\n", "\r", "\x00"): - return True - if oc < 0x20 and ch != "\t": - return True - return False - - -def is_safe_env_value(value: str) -> bool: - """ - Validates that an env value contains no unsafe control characters (CR/LF/NUL), - does not include LD_PRELOAD, and is within size limits to prevent injection or DoS. - """ - if not isinstance(value, str): - return False - if len(value) > MAX_ENV_VALUE_LEN: - return False - if _has_control_chars(value): - return False - if "LD_PRELOAD" in value: - return False - return True - - def is_valid_ip(ip: str) -> bool: try: ipaddress.ip_address(ip) @@ -549,7 +477,19 @@ def extend_rtmr3(platform: Platform, valid_hash: str) -> None: raise RuntimeError("EmitEvent failed while extending RTMR3") -def launch_mpc_container(platform: Platform, valid_hash: str, user_env: dict) -> None: +def get_mpc_config_path(dstack_config: dict[str, str]) -> str: + """ + Returns the path to the MPC node JSON config file. + The path must be accessible inside the container (e.g. under /tapp). + """ + return dstack_config.get( + DSTACK_USER_CONFIG_MPC_CONFIG_PATH, DEFAULT_MPC_CONFIG_PATH + ) + + +def launch_mpc_container( + platform: Platform, valid_hash: str, user_env: dict +) -> None: logging.info(f"Launching MPC node with validated hash: {valid_hash}") remove_existing_container() @@ -778,43 +718,23 @@ def get_bare_digest(full_digest: str) -> str: return full_digest.split(":", 1)[1] -def is_allowed_container_env_key(key: str) -> bool: - if key in DENIED_CONTAINER_ENV_KEYS: - return False - # Allow MPC_* keys with strict regex - if MPC_ENV_KEY_RE.match(key): - return True - # Keep allowlist - if key in ALLOWED_MPC_ENV_VARS: - return True - return False - - def build_docker_cmd( - platform: Platform, user_env: dict[str, str], image_digest: str + platform: Platform, + user_env: dict[str, str], + image_digest: str, ) -> list[str]: - bare_digest = get_bare_digest(image_digest) + mpc_config_path = get_mpc_config_path(user_env) docker_cmd = ["docker", "run"] - # Required environment variables - docker_cmd += ["--env", f"MPC_IMAGE_HASH={bare_digest}"] - docker_cmd += ["--env", f"MPC_LATEST_ALLOWED_HASH_FILE={IMAGE_DIGEST_FILE}"] - if platform is Platform.TEE: - docker_cmd += ["--env", f"DSTACK_ENDPOINT={DSTACK_UNIX_SOCKET}"] docker_cmd += ["-v", f"{DSTACK_UNIX_SOCKET}:{DSTACK_UNIX_SOCKET}"] - # Track env passthrough size/caps - passed_env_count = 0 - total_env_bytes = 0 - - # Deterministic iteration for stable logs/behavior + # Handle EXTRA_HOSTS and PORTS from user config (docker networking, not MPC binary config) for key in sorted(user_env.keys()): value = user_env[key] if key in ALLOWED_LAUNCHER_ENV_VARS: - # launcher-only env vars: never pass to container continue if key == "EXTRA_HOSTS": @@ -841,30 +761,6 @@ def build_docker_cmd( ) continue - if not is_allowed_container_env_key(key): - logging.warning(f"Ignoring unknown or unapproved env var: {key}") - continue - - if not is_safe_env_value(value): - logging.warning(f"Ignoring env var with unsafe value: {key}") - continue - - # Enforce caps - passed_env_count += 1 - if passed_env_count > MAX_PASSTHROUGH_ENV_VARS: - raise RuntimeError( - f"Too many env vars to pass through (>{MAX_PASSTHROUGH_ENV_VARS})." - ) - - # Approximate byte accounting (key=value plus overhead) - total_env_bytes += len(key) + 1 + len(value) - if total_env_bytes > MAX_TOTAL_ENV_BYTES: - raise RuntimeError( - f"Total env payload too large (>{MAX_TOTAL_ENV_BYTES} bytes)." - ) - - docker_cmd += ["--env", f"{key}={value}"] - # Container run configuration docker_cmd += [ "--security-opt", @@ -879,6 +775,9 @@ def build_docker_cmd( MPC_CONTAINER_NAME, "--detach", image_digest, # IMPORTANT: Docker must get the FULL digest + # Command: use config file instead of env vars + "start-with-config-file", + mpc_config_path, ] logging.info("docker cmd %s", " ".join(docker_cmd)) diff --git a/tee_launcher/test_launcher_config.py b/tee_launcher/test_launcher_config.py index 281993e8d..534135594 100644 --- a/tee_launcher/test_launcher_config.py +++ b/tee_launcher/test_launcher_config.py @@ -2,6 +2,7 @@ import inspect import json +import os import tempfile import tee_launcher.launcher as launcher @@ -17,11 +18,7 @@ is_valid_host_entry, is_valid_port_mapping, Platform, - is_safe_env_value, - _has_control_chars, - is_allowed_container_env_key, - MAX_ENV_VALUE_LEN, - MAX_PASSTHROUGH_ENV_VARS, + get_mpc_config_path, ) from tee_launcher.launcher import ( JSON_KEY_APPROVED_HASHES, @@ -139,15 +136,12 @@ def test_build_docker_cmd_sanitizes_ports_and_hosts(): env = { "PORTS": TEST_PORTS_WITH_INJECTION, "EXTRA_HOSTS": TEST_EXTRA_HOSTS_WITH_INJECTION, - "MPC_ACCOUNT_ID": TEST_MPC_ACCOUNT_ID, } cmd = build_docker_cmd(launcher.Platform.TEE, env, "sha256:abc123") func_name = inspect.currentframe().f_code.co_name print(f"[{func_name}] CMD:", " ".join(cmd)) - assert "--env" in cmd - assert f"MPC_ACCOUNT_ID={TEST_MPC_ACCOUNT_ID}" in cmd assert "-p" in cmd assert "11780:11780" in cmd assert "--add-host" in cmd @@ -157,11 +151,13 @@ def test_build_docker_cmd_sanitizes_ports_and_hosts(): assert not any("BAD=1" in arg for arg in cmd) assert not any("/:/mnt" in arg for arg in cmd) + # Verify config file is passed as command arg + assert "start-with-config-file" in cmd + def test_extra_hosts_does_not_allow_ld_preload(): env = { "EXTRA_HOSTS": "host:1.2.3.4,--env LD_PRELOAD=/evil.so", - "MPC_ACCOUNT_ID": "safe", } cmd = build_docker_cmd(launcher.Platform.TEE, env, "sha256:abc123") @@ -172,7 +168,6 @@ def test_extra_hosts_does_not_allow_ld_preload(): def test_ports_does_not_allow_volume_injection(): env = { "PORTS": "2200:2200,--volume /:/mnt", - "MPC_ACCOUNT_ID": "safe", } cmd = build_docker_cmd(launcher.Platform.TEE, env, "sha256:abc123") @@ -180,67 +175,28 @@ def test_ports_does_not_allow_volume_injection(): assert not any("/:/mnt" in arg for arg in cmd) -def test_invalid_env_key_is_ignored(): - env = { - "BAD_KEY": "should_not_be_used", - "MPC_ACCOUNT_ID": "safe", - } - cmd = build_docker_cmd(launcher.Platform.TEE, env, "sha256:abc123") - - assert "should_not_be_used" not in " ".join(cmd) - assert "MPC_ACCOUNT_ID=safe" in cmd - - -def test_mpc_backup_encryption_key_is_allowed(): - env = { - "MPC_BACKUP_ENCRYPTION_KEY_HEX": "0000000000000000000000000000000000000000000000000000000000000000", - } - cmd = build_docker_cmd(launcher.Platform.TEE, env, "sha256:abc123") - - assert ( - "MPC_BACKUP_ENCRYPTION_KEY_HEX=0000000000000000000000000000000000000000000000000000000000000000" - in " ".join(cmd) - ) - - def test_malformed_extra_host_is_ignored(): env = { "EXTRA_HOSTS": "badhostentry,no-colon,also--bad", - "MPC_ACCOUNT_ID": "safe", } cmd = build_docker_cmd(launcher.Platform.TEE, env, "sha256:abc123") assert "--add-host" not in cmd # All malformed entries should be skipped -def test_env_value_with_shell_injection_is_handled_safely(): - env = { - "MPC_ACCOUNT_ID": "safe; rm -rf /", - } - cmd = build_docker_cmd(launcher.Platform.TEE, env, "sha256:abc123") - - assert "--env" in cmd - assert "MPC_ACCOUNT_ID=safe; rm -rf /" in cmd - - def test_parse_and_build_docker_cmd_full_flow(): config_str = """ # Valid entries - MPC_ACCOUNT_ID=test-user PORTS=11780:11780, --env BAD=oops EXTRA_HOSTS=host1:192.168.1.1, --volume /:/mnt - IMAGE_HASH=sha256:abc123 """ env = parse_env_string(config_str) - image_hash = env.get("IMAGE_HASH", "sha256:default") - cmd = build_docker_cmd(launcher.Platform.TEE, env, image_hash) + cmd = build_docker_cmd(launcher.Platform.TEE, env, "sha256:abc123") print(f"[{inspect.currentframe().f_code.co_name}] CMD: {' '.join(cmd)}") - assert "--env" in cmd - assert "MPC_ACCOUNT_ID=test-user" in cmd assert "-p" in cmd assert "11780:11780" in cmd assert "--add-host" in cmd @@ -253,146 +209,71 @@ def test_parse_and_build_docker_cmd_full_flow(): # Test that ensures LD_PRELOAD cannot be injected into the docker command def test_ld_preload_injection_blocked1(): - # Set up the environment variable with a dangerous LD_PRELOAD value malicious_env = { - "MPC_ACCOUNT_ID": TEST_MPC_ACCOUNT_ID, - "--env LD_PRELOAD": "/path/to/my/malloc.so", # The dangerous value + "--env LD_PRELOAD": "/path/to/my/malloc.so", } - # Call build_docker_cmd to generate the docker command docker_cmd = build_docker_cmd(launcher.Platform.TEE, malicious_env, "sha256:abc123") - # Check that LD_PRELOAD is not included in the command - assert "--env" in docker_cmd # Ensure there is an env var assert ( "LD_PRELOAD" not in docker_cmd - ) # Make sure LD_PRELOAD is not in the generated command - - # Alternatively, if you're using a regex to ensure safe environment variables - assert not any( - "-e " in arg for arg in docker_cmd - ) # Ensure no CLI injection for LD_PRELOAD - - -# Additional tests can go here for host/port validation + ) -# Test that ensures LD_PRELOAD cannot be injected through extra hosts def test_ld_preload_in_extra_hosts1(): - # Set up environment with malicious EXRA_HOSTS containing LD_PRELOAD malicious_env = { - "MPC_ACCOUNT_ID": TEST_MPC_ACCOUNT_ID, "EXTRA_HOSTS": "host1:192.168.0.1,host2:192.168.0.2,--env LD_PRELOAD=/path/to/my/malloc.so", } - # Call build_docker_cmd to generate the docker command docker_cmd = build_docker_cmd(launcher.Platform.TEE, malicious_env, "sha256:abc123") - # Check that LD_PRELOAD is not part of the extra hosts in the docker command - assert "--add-host" in docker_cmd # Ensure extra hosts are included - assert "LD_PRELOAD" not in docker_cmd # Ensure LD_PRELOAD is NOT in the command - - # Check that there are no malicious injections - assert not any( - "--env LD_PRELOAD" in arg for arg in docker_cmd - ) # No environment injection + assert "--add-host" in docker_cmd + assert "LD_PRELOAD" not in docker_cmd -# Test that ensures LD_PRELOAD cannot be injected through ports def test_ld_preload_in_ports1(): - # Set up environment with malicious PORTS containing LD_PRELOAD malicious_env = { - "MPC_ACCOUNT_ID": TEST_MPC_ACCOUNT_ID, "PORTS": "11780:11780,--env LD_PRELOAD=/path/to/my/malloc.so", } - # Call build_docker_cmd to generate the docker command - docker_cmd = build_docker_cmd(launcher.Platform.TEE, malicious_env, "sha256:abc123") - - # Check that LD_PRELOAD is not part of the port mappings in the docker command - assert "-p" in docker_cmd # Ensure port mappings are included - assert "LD_PRELOAD" not in docker_cmd # Ensure LD_PRELOAD is NOT in the command - - # Check that there are no malicious injections - assert not any( - "--env LD_PRELOAD" in arg for arg in docker_cmd - ) # No environment injection - - -# Additional tests could go here to check other edge cases - - -# Test that ensures LD_PRELOAD cannot be injected through mpc account id -def test_ld_preload_in_mpc_account_id(): - # Set up environment with malicious EXRA_HOSTS containing LD_PRELOAD - malicious_env = { - "MPC_ACCOUNT_ID": f"{TEST_MPC_ACCOUNT_ID}, --env LD_PRELOAD=/path/to/my/malloc.so", - "EXTRA_HOSTS": "host1:192.168.0.1,host2:192.168.0.2", - } - - # Call build_docker_cmd to generate the docker command docker_cmd = build_docker_cmd(launcher.Platform.TEE, malicious_env, "sha256:abc123") - # Check that LD_PRELOAD is not part of the extra hosts in the docker command - assert "--add-host" in docker_cmd # Ensure extra hosts are included - assert "LD_PRELOAD" not in docker_cmd # Ensure LD_PRELOAD is NOT in the command + assert "-p" in docker_cmd + assert "LD_PRELOAD" not in docker_cmd - # Check that there are no malicious injections - print(docker_cmd) - assert not any( - "--env LD_PRELOAD" in arg for arg in docker_cmd - ) # No environment injection - -# Test that ensures LD_PRELOAD cannot be injected into the docker command def test_ld_preload_injection_blocked2(): - # Set up the environment variable with a dangerous LD_PRELOAD value malicious_env = { - "MPC_ACCOUNT_ID": TEST_MPC_ACCOUNT_ID, - "-e LD_PRELOAD": "/path/to/my/malloc.so", # The dangerous value + "-e LD_PRELOAD": "/path/to/my/malloc.so", } - # Call build_docker_cmd to generate the docker command docker_cmd = build_docker_cmd(launcher.Platform.TEE, malicious_env, "sha256:abc123") assert ( "-e LD_PRELOAD" not in docker_cmd - ) # Make sure LD_PRELOAD is not in the generated command - - -# Additional tests can go here for host/port validation + ) -# Test that ensures LD_PRELOAD cannot be injected through extra hosts def test_ld_preload_in_extra_hosts2(): - # Set up environment with malicious EXRA_HOSTS containing LD_PRELOAD malicious_env = { - "MPC_ACCOUNT_ID": TEST_MPC_ACCOUNT_ID, "EXTRA_HOSTS": "host1:192.168.0.1,host2:192.168.0.2,-e LD_PRELOAD=/path/to/my/malloc.so", } - # Call build_docker_cmd to generate the docker command docker_cmd = build_docker_cmd(launcher.Platform.TEE, malicious_env, "sha256:abc123") - # Check that LD_PRELOAD is not part of the extra hosts in the docker command - assert "--add-host" in docker_cmd # Ensure extra hosts are included - assert "LD_PRELOAD" not in docker_cmd # Ensure LD_PRELOAD is NOT in the command + assert "--add-host" in docker_cmd + assert "LD_PRELOAD" not in docker_cmd -# Test that ensures LD_PRELOAD cannot be injected through ports def test_ld_preload_in_ports2(): - # Set up environment with malicious PORTS containing LD_PRELOAD malicious_env = { - "MPC_ACCOUNT_ID": TEST_MPC_ACCOUNT_ID, "PORTS": "11780:11780,-e LD_PRELOAD=/path/to/my/malloc.so", } - # Call build_docker_cmd to generate the docker command docker_cmd = build_docker_cmd(launcher.Platform.TEE, malicious_env, "sha256:abc123") - # Check that LD_PRELOAD is not part of the port mappings in the docker command - assert "-p" in docker_cmd # Ensure port mappings are included - assert "LD_PRELOAD" not in docker_cmd # Ensure LD_PRELOAD is NOT in the command + assert "-p" in docker_cmd + assert "LD_PRELOAD" not in docker_cmd def test_json_key_matches_node(): @@ -598,24 +479,61 @@ def fake_curl(endpoint, payload, capture_output=False): def test_build_docker_cmd_nontee_no_dstack_mount(base_env): env = { - "MPC_ACCOUNT_ID": "x", # launcher-only env should be ignored launcher.ENV_VAR_RPC_MAX_ATTEMPTS: "5", } cmd = launcher.build_docker_cmd(launcher.Platform.NONTEE, env, "sha256:" + "c" * 64) s = " ".join(cmd) - assert "DSTACK_ENDPOINT=" not in s assert f"{launcher.DSTACK_UNIX_SOCKET}:{launcher.DSTACK_UNIX_SOCKET}" not in s + # Config file path should be passed as command arg + assert "start-with-config-file" in cmd def test_build_docker_cmd_tee_has_dstack_mount(base_env): - env = {"MPC_ACCOUNT_ID": "x"} + env = {} cmd = launcher.build_docker_cmd(launcher.Platform.TEE, env, "sha256:" + "c" * 64) s = " ".join(cmd) - assert f"DSTACK_ENDPOINT={launcher.DSTACK_UNIX_SOCKET}" in s assert f"{launcher.DSTACK_UNIX_SOCKET}:{launcher.DSTACK_UNIX_SOCKET}" in s + assert "start-with-config-file" in cmd + + +def test_build_docker_cmd_no_env_passthrough(): + """Ensure no --env flags are emitted (all config via file now).""" + env = { + "MPC_ACCOUNT_ID": "test", + "MPC_CONTRACT_ID": "contract.near", + "RUST_LOG": "info", + } + cmd = build_docker_cmd(Platform.NONTEE, env, "sha256:" + "a" * 64) + assert "--env" not in cmd + + +def test_build_docker_cmd_passes_config_file_path(): + env = {} + cmd = build_docker_cmd(Platform.NONTEE, env, "sha256:" + "a" * 64) + + # Should end with: start-with-config-file + assert cmd[-2] == "start-with-config-file" + assert cmd[-1] == launcher.DEFAULT_MPC_CONFIG_PATH + + +def test_build_docker_cmd_custom_config_path(): + env = {launcher.DSTACK_USER_CONFIG_MPC_CONFIG_PATH: "/tapp/custom_config.json"} + cmd = build_docker_cmd(Platform.NONTEE, env, "sha256:" + "a" * 64) + + assert cmd[-2] == "start-with-config-file" + assert cmd[-1] == "/tapp/custom_config.json" + + +def test_get_mpc_config_path_default(): + assert get_mpc_config_path({}) == launcher.DEFAULT_MPC_CONFIG_PATH + + +def test_get_mpc_config_path_override(): + config = {launcher.DSTACK_USER_CONFIG_MPC_CONFIG_PATH: "/tapp/my_config.json"} + assert get_mpc_config_path(config) == "/tapp/my_config.json" def test_main_tee_fails_closed_before_launch(monkeypatch, base_env): @@ -679,20 +597,7 @@ def assert_subsequence(seq, subseq): def test_main_nontee_builds_expected_mpc_docker_cmd(monkeypatch, tmp_path): """ Verify that launcher.main() builds the correct MPC docker command in NONTEE mode. - - Steps: - 1. Configure the launcher to run with PLATFORM=NONTEE. - 2. Set required environment variables (DOCKER_CONTENT_TRUST, DEFAULT_IMAGE_DIGEST). - 3. Create a temporary user_config file with MPC env vars, ports, and extra hosts. - 4. Simulate a missing IMAGE_DIGEST_FILE so the launcher falls back to DEFAULT_IMAGE_DIGEST. - 5. Stub image validation and docker interactions to avoid real network or docker usage. - 6. Invoke launcher.main(). - 7. Capture the docker run command used to start the MPC container. - 8. Assert that the command: - - Includes expected MPC configuration (env vars, ports, hosts, volumes). - - Does NOT include dstack socket mounts or DSTACK_ENDPOINT. - - Filters out injection attempts in ports and hosts. - - Uses the expected full image digest. + Config is now passed as a file path argument, not via env vars. """ # --- Arrange: environment (NONTEE) --- monkeypatch.setenv(launcher.ENV_VAR_PLATFORM, launcher.Platform.NONTEE.value) @@ -701,12 +606,11 @@ def test_main_nontee_builds_expected_mpc_docker_cmd(monkeypatch, tmp_path): default_digest = "sha256:" + "a" * 64 monkeypatch.setenv(launcher.ENV_VAR_DEFAULT_IMAGE_DIGEST, default_digest) - # Provide a temp user config file so main() passes env into build_docker_cmd() + # Provide a temp user config file with ports and extra hosts user_config = tmp_path / "user_config" user_config.write_text( "\n".join( [ - f"MPC_ACCOUNT_ID={TEST_MPC_ACCOUNT_ID}", f"PORTS={TEST_PORTS_WITH_INJECTION}", # injection should be ignored f"EXTRA_HOSTS={TEST_EXTRA_HOSTS_WITH_INJECTION}", # injection should be ignored ] @@ -759,12 +663,13 @@ def fake_run(cmd, *args, **kwargs): cmd_str = " ".join(cmd) - # NONTEE invariants - assert "DSTACK_ENDPOINT=" not in cmd_str + # NONTEE invariants: no dstack socket mount assert f"{launcher.DSTACK_UNIX_SOCKET}:{launcher.DSTACK_UNIX_SOCKET}" not in cmd_str - # Expected env propagation + sanitization - assert f"MPC_ACCOUNT_ID={TEST_MPC_ACCOUNT_ID}" in cmd_str + # No env passthrough (all config is via file now) + assert "--env" not in cmd + + # Docker networking still works assert "-p" in cmd and "11780:11780" in cmd_str assert "--add-host" in cmd and TEST_EXTRA_HOSTS_WITH_IP in cmd_str @@ -780,8 +685,10 @@ def fake_run(cmd, *args, **kwargs): assert "mpc-data:/data" in cmd_str assert f"--name {launcher.MPC_CONTAINER_NAME}" in cmd_str - # Image digest should be the final argument and should be the FULL digest - assert cmd[-1] == default_digest + # Config file is passed as command arg after the image digest + digest_idx = cmd.index(default_digest) + assert cmd[digest_idx + 1] == "start-with-config-file" + assert cmd[digest_idx + 2] == launcher.DEFAULT_MPC_CONFIG_PATH expected_core = [ "docker", @@ -799,124 +706,3 @@ def fake_run(cmd, *args, **kwargs): "--detach", ] assert_subsequence(cmd, expected_core) - - -def _base_env(): - # Minimal env for build_docker_cmd (launcher will add required MPC_IMAGE_HASH etc itself) - return { - "MPC_ACCOUNT_ID": "mpc-user-123", - "MPC_CONTRACT_ID": "contract.near", - "MPC_ENV": "testnet", - "MPC_HOME_DIR": "/data", - "NEAR_BOOT_NODES": "boot1,boot2", - "RUST_LOG": "info", - } - - -def test_has_control_chars_rejects_newline_and_cr(): - assert _has_control_chars("a\nb") is True - assert _has_control_chars("a\rb") is True - - -def test_has_control_chars_rejects_other_control_chars_but_allows_tab(): - # tab is allowed by the Python helper in the patched launcher - assert _has_control_chars("a\tb") is False - # ASCII control char 0x1F should be rejected - assert _has_control_chars("a" + chr(0x1F) + "b") is True - - -def test_is_safe_env_value_rejects_control_chars(): - assert is_safe_env_value("ok\nno") is False - assert is_safe_env_value("ok\rno") is False - assert is_safe_env_value("ok" + chr(0x1F) + "no") is False - - -def test_is_safe_env_value_rejects_ld_preload_substring(): - assert is_safe_env_value("LD_PRELOAD=/tmp/x.so") is False - assert is_safe_env_value("foo LD_PRELOAD bar") is False - - -def test_is_safe_env_value_rejects_too_long_value(): - assert is_safe_env_value("a" * (MAX_ENV_VALUE_LEN + 1)) is False - assert is_safe_env_value("a" * MAX_ENV_VALUE_LEN) is True - - -def testis_allowed_container_env_key_allows_mpc_prefix_uppercase(): - assert is_allowed_container_env_key("MPC_FOO") is True - assert is_allowed_container_env_key("MPC_FOO_123") is True - assert is_allowed_container_env_key("MPC_A_B_C") is True - - -def testis_allowed_container_env_key_rejects_lowercase_or_invalid_chars(): - assert is_allowed_container_env_key("MPC_foo") is False - assert is_allowed_container_env_key("MPC-FOO") is False - assert is_allowed_container_env_key("MPC.FOO") is False - assert is_allowed_container_env_key("MPC_") is False - - -def testis_allowed_container_env_key_allows_compat_non_mpc_keys(): - assert is_allowed_container_env_key("RUST_LOG") is True - assert is_allowed_container_env_key("RUST_BACKTRACE") is True - assert is_allowed_container_env_key("NEAR_BOOT_NODES") is True - - -def testis_allowed_container_env_key_denies_sensitive_keys(): - assert is_allowed_container_env_key("MPC_P2P_PRIVATE_KEY") is False - assert is_allowed_container_env_key("MPC_ACCOUNT_SK") is False - - -def test_build_docker_cmd_allows_arbitrary_mpc_prefix_env_vars(): - env = _base_env() - env["MPC_NEW_FEATURE_FLAG"] = "1" - env["MPC_SOME_CONFIG"] = "value" - - cmd = build_docker_cmd(Platform.NONTEE, env, "sha256:" + "a" * 64) - - cmd_str = " ".join(cmd) - assert "--env MPC_NEW_FEATURE_FLAG=1" in cmd_str - assert "--env MPC_SOME_CONFIG=value" in cmd_str - - -def test_build_docker_cmd_blocks_sensitive_mpc_private_keys(): - env = _base_env() - env["MPC_P2P_PRIVATE_KEY"] = "supersecret" - env["MPC_ACCOUNT_SK"] = "supersecret2" - - cmd = build_docker_cmd(Platform.NONTEE, env, "sha256:" + "a" * 64) - cmd_str = " ".join(cmd) - - assert "MPC_P2P_PRIVATE_KEY" not in cmd_str - assert "MPC_ACCOUNT_SK" not in cmd_str - - -def test_build_docker_cmd_rejects_env_value_with_newline(): - env = _base_env() - env["MPC_NEW_FEATURE_FLAG"] = "ok\nbad" - - cmd = build_docker_cmd(Platform.NONTEE, env, "sha256:" + "a" * 64) - cmd_str = " ".join(cmd) - - # It should be ignored (not passed) - assert "MPC_NEW_FEATURE_FLAG" not in cmd_str - - -def test_build_docker_cmd_enforces_max_env_count_cap(): - env = _base_env() - # add many MPC_* keys to exceed cap - for i in range(MAX_PASSTHROUGH_ENV_VARS + 1): - env[f"MPC_X_{i}"] = "1" - - with pytest.raises(RuntimeError, match="Too many env vars"): - build_docker_cmd(Platform.NONTEE, env, "sha256:" + "a" * 64) - - -def test_build_docker_cmd_enforces_total_env_bytes_cap(): - env = _base_env() - - # Each env contributes ~ len(key)+1+MAX_ENV_VALUE_LEN bytes. - # With MAX_ENV_VALUE_LEN=1024 and MAX_TOTAL_ENV_BYTES=32768, ~35-40 vars will exceed the cap. - for i in range(40): - env[f"MPC_BIG_{i}"] = "a" * MAX_ENV_VALUE_LEN - - with pytest.raises(RuntimeError, match="Total env payload too large"): - build_docker_cmd(Platform.NONTEE, env, "sha256:" + "a" * 64) diff --git a/tee_launcher/user-config.conf b/tee_launcher/user-config.conf index 711b67c41..5b6b49457 100644 --- a/tee_launcher/user-config.conf +++ b/tee_launcher/user-config.conf @@ -3,16 +3,10 @@ MPC_IMAGE_NAME=nearone/mpc-node MPC_IMAGE_TAGS=3.3.2 MPC_REGISTRY=registry.hub.docker.com -MPC_ACCOUNT_ID=mpc-3-barak-launch1-b654bfa0a52e.5035bf56abb0.testnet -MPC_LOCAL_ADDRESS=127.0.0.1 -MPC_SECRET_STORE_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -MPC_CONTRACT_ID=mpc-contract-barak-launch1-4c5e2fe1fb42.5035bf56abb0.testnet -MPC_ENV=testnet -MPC_HOME_DIR=/data -RUST_BACKTRACE=full -RUST_LOG=mpc=debug,info +# Path to the MPC node JSON config file (must be accessible inside the container). +# Defaults to /tapp/mpc_config.json if not set. +# MPC_CONFIG_PATH=/tapp/mpc_config.json -NEAR_BOOT_NODES=ed25519:9qyu1RaJ5shX6UEb7UooPQYVXCC1tNHCiDPPxJ8Pv1UJ@116.202.220.238:34567,ed25519:8mzYnfuT5zQYqV99CfYAX6XoRmNxVJ1nAZHXXW4GrFD@34.221.144.70:24567,ed25519:B87Qq34LbWadFx2dq5bwUEtB5KBgr8ZhsoEpAiSP2qVX@142.132.203.80:24567,ed25519:EufXMhFVixgFpg2bBaHGL4Zrks1DDrhAZTQYwbjRTAUX@65.109.25.109:24567,ed25519:HJJde5skATXLA4wGk8P9awvfzaW47tCU2EsRXnMoFRA9@129.150.39.19:24567,ed25519:BavpjuYrnXRFQVWjLdx9vx9vAvanit9NhhcPeM6gjAkE@95.217.198.233:24567,ed25519:81zk9MvvoxB1AzTW721o9m2NeYx3pDFDZyRJUQej65uc@195.14.6.172:24567,ed25519:E4gQXBovauvqxx85TdemezhkDDsAsqEL7ZJ4cp5Cdhsb@129.80.119.109:24567,ed25519:6cWtXFAzqpZ8D7EpLGYBmkw95oKYkzN8i99UcRgsyRMy@164.132.247.155:24567,ed25519:CLnWy9xv2GUqfgepzLwpv4bozj3H3kgzjbVREyS6wcqq@47.242.112.172:24567,ed25519:2NmT9Wy9HGBmH8sTWSq2QfaMk4R8ZHBEhk8ZH4g4f1Qk@65.109.88.175:24567,ed25519:9dhPYd1ArZ6mTMP7nnRzm8JBPwKCaBxiYontS5KfXz5h@34.239.1.54:24567,ed25519:8iiQH4vtqsqWgsm4ypCJQQwqJR3AGp9o7F69YRaCHKxA@141.95.204.11:24567,ed25519:4L97JnFFFVbfE8M3tY9bRtgV5376y5dFH8cSaoBDRWnK@5.199.170.103:24567,ed25519:DGJ91V2wJ8NFpkqZvphtSeM4CBeiLsrHGdinTugiRoFF@52.35.74.212:24567,ed25519:B9LSvCTimoEUtuUvpfu1S54an54uTetVabmkT5dELUCN@91.134.22.129:24567,ed25519:cRGmtzkkSZT6wXNjbthSXMD6dHrEgSeDtiEJAcnLLxH@15.204.213.166:24567 # needed: Port forwarding - telemetry. PORTS=8080:8080,3030:3030,80:80,24567:24567