From f10323885dd209a0bcd78439a34203c73f32f7df Mon Sep 17 00:00:00 2001 From: Nick Patilsen Date: Thu, 22 Jan 2026 00:19:14 +0000 Subject: [PATCH 1/3] take logger to avoid race condition --- src/lib.rs | 46 +++++++++++++--------------------------------- 1 file changed, 13 insertions(+), 33 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b6a8a9a..9250421 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -160,25 +160,10 @@ impl Otel { // Graceful shutdown that flushes any pending metrics and logs to the exporter. info!("shutting down otel component"); - // Use a timeout for flush/shutdown operations to prevent hanging - // when the server is unavailable + // Use a timeout for shutdown operations to prevent hanging + // when the server is unavailable. let shutdown_timeout = Duration::from_secs(10); - let flush_result = tokio::time::timeout( - shutdown_timeout, - tokio::task::spawn_blocking({ - let meter_provider = self.meter_provider.clone(); - move || meter_provider.force_flush() - }) - ).await; - - match flush_result { - Err(_) => warn!("meter provider force_flush timed out"), - Ok(Err(e)) => warn!("meter provider force_flush task failed: {e:?}"), - Ok(Ok(Err(e))) => warn!("encountered error while flushing metrics: {e:?}"), - Ok(Ok(Ok(()))) => debug!("meter provider force_flush completed"), - } - let shutdown_result = tokio::time::timeout( shutdown_timeout, tokio::task::spawn_blocking({ @@ -190,30 +175,25 @@ impl Otel { match shutdown_result { Err(_) => warn!("meter provider shutdown timed out"), Ok(Err(e)) => warn!("meter provider shutdown task failed: {e:?}"), - Ok(Ok(Err(e))) => warn!("encountered error while shutting down meter provider: {e:?}"), + Ok(Ok(Err(e))) => { + debug!("meter provider shutdown completed with error (often expected during shutdown): {e:?}"); + } Ok(Ok(Ok(()))) => debug!("meter provider shutdown completed"), } - if let Some(logger_provider) = self.logger_provider.clone() { - let flush_result = tokio::time::timeout( - shutdown_timeout, - tokio::task::spawn_blocking({ - let lp = logger_provider.clone(); - move || lp.force_flush() - }) - ).await; - - if let Err(_) | Ok(Err(_)) = flush_result { - warn!("logger provider force_flush timed out or failed"); - } - + if let Some(logger_provider) = self.logger_provider.take() { let shutdown_result = tokio::time::timeout( shutdown_timeout, tokio::task::spawn_blocking(move || logger_provider.shutdown()) ).await; - if let Err(_) | Ok(Err(_)) = shutdown_result { - warn!("logger provider shutdown timed out or failed"); + match shutdown_result { + Err(_) => warn!("logger provider shutdown timed out"), + Ok(Err(e)) => warn!("logger provider shutdown task failed: {e:?}"), + Ok(Ok(Err(e))) => { + debug!("logger provider shutdown completed with error (often expected during shutdown): {e:?}"); + } + Ok(Ok(Ok(()))) => debug!("logger provider shutdown completed"), } } From b30a26d890888ae11d3ca919d1bf58681ca1e06b Mon Sep 17 00:00:00 2001 From: Nick Patilsen Date: Thu, 22 Jan 2026 17:59:32 +0000 Subject: [PATCH 2/3] try increase timeout --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 9250421..9d1fc9a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -162,7 +162,7 @@ impl Otel { // Use a timeout for shutdown operations to prevent hanging // when the server is unavailable. - let shutdown_timeout = Duration::from_secs(10); + let shutdown_timeout = Duration::from_secs(30); let shutdown_result = tokio::time::timeout( shutdown_timeout, From 75a5f394bf046d1588f5ee3b35ebaff5dfd9e173 Mon Sep 17 00:00:00 2001 From: Nick Patilsen Date: Thu, 22 Jan 2026 23:19:16 +0000 Subject: [PATCH 3/3] const --- src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 9d1fc9a..7b891fa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -45,6 +45,7 @@ pub mod loggers; pub mod syslog_writer; pub(crate) const SERVICE_NAME_KEY: &str = "service.name"; +pub const OTEL_SHUTDOWN_TIMEOUT_SECS: u64 = 30; struct PrometheusRegistry { registry: Registry, @@ -162,7 +163,7 @@ impl Otel { // Use a timeout for shutdown operations to prevent hanging // when the server is unavailable. - let shutdown_timeout = Duration::from_secs(30); + let shutdown_timeout = Duration::from_secs(OTEL_SHUTDOWN_TIMEOUT_SECS); let shutdown_result = tokio::time::timeout( shutdown_timeout,