[telemetry] Add structured logging with span-based tracing

.github/workflows/ci.yml

@@ -10,6 +10,7 @@ permissions:
 
 env:
   RUSTFLAGS: -Dwarnings
+  RUST_LOG_JSON: "0"
 
 jobs:
   test:

Cargo.toml

@@ -43,7 +43,7 @@ tabled = "0.16"
 anyhow = "1.0.98"
 toml = "0.9.2"
 futures = "0.3.31"
-tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }
+tracing-subscriber = { version = "0.3.19", features = ["env-filter", "json"] }
 # TODO: Switch back to a crates.io release once age 0.12+ is published.
 # We pin to this specific commit because age 0.11.x depends on i18n-embed-fl 0.9.x,
 # which has a non-determinism bug (HashMap iteration in the fl!() proc macro) that

crates/e2e-tests/src/main.rs

@@ -24,6 +24,10 @@ use std::path::Path;
     about = "Manage a local Hashi dev environment"
 )]
 struct Cli {
+    /// Enable verbose tracing output (INFO level).
+    #[clap(long, short, global = true)]
+    verbose: bool,
+
     #[command(subcommand)]
     command: Commands,
 }
@@ -73,10 +77,6 @@ enum Commands {
         #[clap(long, default_value = "18443")]
         btc_rpc_port: u16,
 
-        /// Enable verbose tracing output
-        #[clap(long, short)]
-        verbose: bool,
-
         #[command(flatten)]
         opts: LocalnetOpts,
     },
@@ -215,23 +215,24 @@ fn print_warning(msg: &str) {
 async fn main() -> Result<()> {
     let cli = Cli::parse();
 
+    let default_level = if cli.verbose {
+        tracing::level_filters::LevelFilter::INFO
+    } else {
+        tracing::level_filters::LevelFilter::OFF
+    };
+    hashi_types::telemetry::TelemetryConfig::new()
+        .with_default_level(default_level)
+        .with_target(false)
+        .with_env()
+        .init();
+
     match cli.command {
         Commands::Start {
             num_validators,
             sui_rpc_port,
             btc_rpc_port,
-            verbose,
             opts,
-        } => {
-            cmd_start(
-                num_validators,
-                sui_rpc_port,
-                btc_rpc_port,
-                verbose,
-                &opts.data_dir,
-            )
-            .await
-        }
+        } => cmd_start(num_validators, sui_rpc_port, btc_rpc_port, &opts.data_dir).await,
         Commands::Stop { opts } => cmd_stop(&opts.data_dir).await,
         Commands::Status { opts } => cmd_status(&opts.data_dir),
         Commands::Info { opts } => cmd_info(&opts.data_dir),
@@ -259,7 +260,6 @@ async fn cmd_start(
     num_validators: usize,
     sui_rpc_port: u16,
     btc_rpc_port: u16,
-    verbose: bool,
     data_dir: &Path,
 ) -> Result<()> {
     // Check for existing running instance
@@ -273,21 +273,6 @@ async fn cmd_start(
         print_warning("Found stale state file, cleaning up...");
     }
 
-    let default_level = if verbose {
-        tracing::level_filters::LevelFilter::INFO
-    } else {
-        tracing::level_filters::LevelFilter::OFF
-    };
-
-    tracing_subscriber::fmt()
-        .with_env_filter(
-            tracing_subscriber::EnvFilter::builder()
-                .with_default_directive(default_level.into())
-                .from_env_lossy(),
-        )
-        .with_target(false)
-        .init();
-
     use std::io::Write;
     print!(
         "{} Starting localnet with {} validators...",

crates/hashi-guardian/src/main.rs

@@ -102,7 +102,10 @@ pub struct EphemeralKeyPairs {
 /// SETUP_MODE=false: all endpoints except setup_new_key are enabled.
 #[tokio::main]
 async fn main() -> Result<()> {
-    init_tracing_subscriber(true);
+    hashi_types::telemetry::TelemetryConfig::new()
+        .with_file_line(true)
+        .with_env()
+        .init();
 
     // Check if SETUP_MODE is enabled (defaults to false)
     let setup_mode = std::env::var("SETUP_MODE")
@@ -537,27 +540,6 @@ impl Enclave {
     }
 }
 
-// ---------------------------------
-//    Tracing utilities
-// ---------------------------------
-
-/// Initialize tracing subscriber with optional file/line number logging
-pub fn init_tracing_subscriber(with_file_line: bool) {
-    let mut builder = tracing_subscriber::FmtSubscriber::builder().with_env_filter(
-        tracing_subscriber::EnvFilter::builder()
-            .with_default_directive(tracing::level_filters::LevelFilter::INFO.into())
-            .from_env_lossy(),
-    );
-
-    if with_file_line {
-        builder = builder.with_file(true).with_line_number(true);
-    }
-
-    let subscriber = builder.finish();
-    tracing::subscriber::set_global_default(subscriber)
-        .expect("unable to initialize tracing subscriber");
-}
-
 // ---------------------------------
 //    Tests and related utilities
 // ---------------------------------

crates/hashi-monitor/src/main.rs

@@ -51,7 +51,10 @@ enum Command {
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
-    init_tracing_subscriber(false);
+    hashi_types::telemetry::TelemetryConfig::new()
+        .with_target(false)
+        .with_env()
+        .init();
 
     let cli = Cli::parse();
 
@@ -78,22 +81,3 @@ async fn main() -> anyhow::Result<()> {
 
     Ok(())
 }
-
-pub fn init_tracing_subscriber(with_file_line: bool) {
-    let mut builder = tracing_subscriber::FmtSubscriber::builder().with_env_filter(
-        tracing_subscriber::EnvFilter::builder()
-            .with_default_directive(tracing::level_filters::LevelFilter::INFO.into())
-            .from_env_lossy(),
-    );
-
-    if with_file_line {
-        builder = builder
-            .with_file(true)
-            .with_line_number(true)
-            .with_target(false);
-    }
-
-    let subscriber = builder.finish();
-    tracing::subscriber::set_global_default(subscriber)
-        .expect("unable to initialize tracing subscriber");
-}

crates/hashi-screener/src/main.rs

@@ -226,20 +226,6 @@ impl ScreenerService for ScreenerServiceImpl {
     }
 }
 
-fn init_tracing_subscriber() {
-    let subscriber = tracing_subscriber::FmtSubscriber::builder()
-        .with_env_filter(
-            tracing_subscriber::EnvFilter::builder()
-                .with_default_directive(tracing::level_filters::LevelFilter::INFO.into())
-                .from_env_lossy(),
-        )
-        .with_file(true)
-        .with_line_number(true)
-        .finish();
-    tracing::subscriber::set_global_default(subscriber)
-        .expect("unable to initialize tracing subscriber");
-}
-
 fn start_metrics_server(registry: prometheus::Registry) -> sui_http::ServerHandle {
     let addr: SocketAddr = "0.0.0.0:9184".parse().unwrap();
     info!("Prometheus metrics server listening on {}", addr);
@@ -266,7 +252,10 @@ async fn metrics_handler(
 
 #[tokio::main]
 async fn main() -> Result<()> {
-    init_tracing_subscriber();
+    hashi_types::telemetry::TelemetryConfig::new()
+        .with_file_line(true)
+        .with_env()
+        .init();
 
     let api_key = env::var("MERKLE_SCIENCE_API_KEY")
         .map_err(|_| anyhow::anyhow!("MERKLE_SCIENCE_API_KEY environment variable is not set"))?;

crates/hashi-types/Cargo.toml

@@ -37,6 +37,7 @@ ed25519-consensus = "2.1"
 miniscript = "13.0.0"
 serde_json.workspace = true
 tracing.workspace = true
+tracing-subscriber.workspace = true
 time = "0.3"
 hex.workspace = true
 

crates/hashi-types/src/lib.rs

@@ -6,4 +6,5 @@ pub mod committee;
 pub mod guardian;
 pub mod move_types;
 pub mod proto;
+pub mod telemetry;
 pub mod utils;

crates/hashi-types/src/telemetry.rs

@@ -0,0 +1,102 @@
+// Copyright (c) Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Shared `tracing` subscriber initialization for all hashi binaries.
+
+use std::io::IsTerminal;
+use std::io::stderr;
+
+use tracing::level_filters::LevelFilter;
+use tracing_subscriber::EnvFilter;
+use tracing_subscriber::Layer;
+use tracing_subscriber::layer::SubscriberExt;
+use tracing_subscriber::util::SubscriberInitExt;
+
+pub struct TelemetryConfig {
+    default_level: LevelFilter,
+    json: Option<bool>,
+    file_line: bool,
+    target: bool,
+}
+
+impl Default for TelemetryConfig {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl TelemetryConfig {
+    pub fn new() -> Self {
+        Self {
+            default_level: LevelFilter::INFO,
+            json: None,
+            file_line: false,
+            target: true,
+        }
+    }
+
+    pub fn with_default_level(mut self, level: LevelFilter) -> Self {
+        self.default_level = level;
+        self
+    }
+
+    pub fn with_json(mut self, json: bool) -> Self {
+        self.json = Some(json);
+        self
+    }
+
+    pub fn with_file_line(mut self, enabled: bool) -> Self {
+        self.file_line = enabled;
+        self
+    }
+
+    pub fn with_target(mut self, enabled: bool) -> Self {
+        self.target = enabled;
+        self
+    }
+
+    /// `RUST_LOG_JSON=0`/`false`/`no` forces TTY; any other value forces JSON.
+    pub fn with_env(mut self) -> Self {
+        if let Ok(value) = std::env::var("RUST_LOG_JSON") {
+            self.json = match value.trim().to_ascii_lowercase().as_str() {
+                "0" | "false" | "no" => Some(false),
+                _ => Some(true),
+            };
+        }
+        self
+    }
+
+    pub fn init(self) {
+        let use_json = match self.json {
+            Some(true) => true,
+            Some(false) => false,
+            None => !stderr().is_terminal(),
+        };
+
+        let env_filter = EnvFilter::builder()
+            .with_default_directive(self.default_level.into())
+            .from_env_lossy();
+
+        if use_json {
+            let fmt_layer = tracing_subscriber::fmt::layer()
+                .with_writer(stderr)
+                .with_file(true)
+                .with_line_number(true)
+                .with_target(self.target)
+                .json()
+                .with_filter(env_filter);
+
+            tracing_subscriber::registry().with(fmt_layer).init();
+        } else {
+            let fmt_layer = tracing_subscriber::fmt::layer()
+                .with_writer(stderr)
+                .with_file(self.file_line)
+                .with_line_number(self.file_line)
+                .with_target(self.target)
+                .with_ansi(stderr().is_terminal())
+                .with_filter(env_filter);
+
+            tracing_subscriber::registry().with(fmt_layer).init();
+        }
+    }
+}

crates/hashi/src/btc_monitor/monitor.rs

@@ -226,6 +226,7 @@ impl Monitor {
     }
 
     /// Run the main event loop, returning the reason it exited.
+    #[tracing::instrument(name = "btc_monitor", skip_all)]
     async fn run_event_loop(
         &mut self,
         kyoto_client: &mut kyoto::Client,

crates/hashi/src/cli/mod.rs

@@ -668,19 +668,16 @@ fn parse_metadata(args: Vec<String>) -> Vec<(String, String)> {
 }
 
 fn init_tracing(verbose: bool) {
-    let filter = if verbose {
-        tracing_subscriber::EnvFilter::builder()
-            .with_default_directive(tracing::level_filters::LevelFilter::DEBUG.into())
-            .from_env_lossy()
+    let level = if verbose {
+        tracing::level_filters::LevelFilter::DEBUG
     } else {
-        tracing_subscriber::EnvFilter::builder()
-            .with_default_directive(tracing::level_filters::LevelFilter::WARN.into())
-            .from_env_lossy()
+        tracing::level_filters::LevelFilter::WARN
     };
 
-    tracing_subscriber::fmt()
-        .with_env_filter(filter)
+    hashi_types::telemetry::TelemetryConfig::new()
+        .with_default_level(level)
         .with_target(false)
+        .with_env()
         .init();
 }