From faf768d2f11b0123254ebb65e20c6997c1e0c9de Mon Sep 17 00:00:00 2001 From: Nick Patilsen Date: Tue, 20 Jan 2026 23:55:41 +0000 Subject: [PATCH 1/3] add timeout to readers --- src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 2cc0cef..245b379 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -307,6 +307,7 @@ fn init_metrics(config: Config) -> (Option, SdkMeterProvider let reader = PeriodicReader::builder(exporter, runtime::Tokio) .with_interval(Duration::from_secs(export_target.interval_secs)) + .with_timeout(Duration::from_secs(export_target.timeout)) .build(); meter_provider_builder = meter_provider_builder.with_reader(reader); } @@ -322,7 +323,9 @@ fn init_metrics(config: Config) -> (Option, SdkMeterProvider }) .build(); - let reader = PeriodicReader::builder(exporter, runtime::Tokio).build(); + let reader = PeriodicReader::builder(exporter, runtime::Tokio) + .with_timeout(Duration::from_secs(30)) + .build(); meter_provider_builder = meter_provider_builder.with_reader(reader); } From 54b5c0069d4e47cd034dd4744fba021138c51fd5 Mon Sep 17 00:00:00 2001 From: Nick Patilsen Date: Wed, 21 Jan 2026 19:02:43 +0000 Subject: [PATCH 2/3] add connection timeouts and shutdown timeouts --- src/lib.rs | 58 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index d2290dd..cfd73cc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -160,16 +160,61 @@ impl Otel { // Graceful shutdown that flushes any pending metrics and logs to the exporter. info!("shutting down otel component"); - if let Err(metrics_error) = self.meter_provider.force_flush() { - warn!("encountered error while flushing metrics: {metrics_error:?}"); + // Use a timeout for flush/shutdown operations to prevent hanging + // when the server is unavailable + let shutdown_timeout = Duration::from_secs(10); + + let flush_result = tokio::time::timeout( + shutdown_timeout, + tokio::task::spawn_blocking({ + let meter_provider = self.meter_provider.clone(); + move || meter_provider.force_flush() + }) + ).await; + + match flush_result { + Err(_) => warn!("meter provider force_flush timed out"), + Ok(Err(e)) => warn!("meter provider force_flush task failed: {e:?}"), + Ok(Ok(Err(e))) => warn!("encountered error while flushing metrics: {e:?}"), + Ok(Ok(Ok(()))) => debug!("meter provider force_flush completed"), } - if let Err(metrics_error) = self.meter_provider.shutdown() { - warn!("encountered error while shutting down meter provider: {metrics_error:?}"); + + let shutdown_result = tokio::time::timeout( + shutdown_timeout, + tokio::task::spawn_blocking({ + let meter_provider = self.meter_provider.clone(); + move || meter_provider.shutdown() + }) + ).await; + + match shutdown_result { + Err(_) => warn!("meter provider shutdown timed out"), + Ok(Err(e)) => warn!("meter provider shutdown task failed: {e:?}"), + Ok(Ok(Err(e))) => warn!("encountered error while shutting down meter provider: {e:?}"), + Ok(Ok(Ok(()))) => debug!("meter provider shutdown completed"), } if let Some(logger_provider) = self.logger_provider.clone() { - logger_provider.force_flush(); - let _ = logger_provider.shutdown(); + let flush_result = tokio::time::timeout( + shutdown_timeout, + tokio::task::spawn_blocking({ + let lp = logger_provider.clone(); + move || lp.force_flush() + }) + ).await; + + if let Err(_) | Ok(Err(_)) = flush_result { + warn!("logger provider force_flush timed out or failed"); + } + + let shutdown_result = tokio::time::timeout( + shutdown_timeout, + tokio::task::spawn_blocking(move || logger_provider.shutdown()) + ).await; + + if let Err(_) | Ok(Err(_)) = shutdown_result { + warn!("logger provider shutdown timed out or failed"); + } } } @@ -474,6 +519,7 @@ fn handle_tls( } }); let channel = tonic_endpoint + .connect_timeout(timeout) .timeout(timeout) .connect_with_connector_lazy(custom_connector); Ok(exporter_builder.with_channel(channel)) From 054a23337af404adc136554d3a0627312aa8ed21 Mon Sep 17 00:00:00 2001 From: Nick Patilsen Date: Wed, 21 Jan 2026 22:09:50 +0000 Subject: [PATCH 3/3] clippy --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index cfd73cc..b6a8a9a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -288,7 +288,7 @@ impl TemporalitySelector for DeltaTemporalitySelector { /// setup the stdout metrics writer if enabled, and initializes STATIC Metrics. /// /// Returns the Prometheus Registry or None if Prometheus was disabled. -/// +#[allow(clippy::too_many_lines)] fn init_metrics(config: Config) -> (Option, SdkMeterProvider) { let mut keys = vec![KeyValue::new(SERVICE_NAME_KEY, config.service_name.clone())]; if let Some(resource_attributes) = config.resource_attributes {