fix(obs): shut down global telemetry guard (#4430)

Co-authored-by: heihutu <heihutu@gmail.com>

crates/obs/src/error.rs

@@ -27,9 +27,11 @@ pub enum GlobalError {
     #[error("Failed to set a global recorder: {0}")]
     SetRecorder(#[from] metrics::SetRecorderError<crate::Recorder>),
     #[error("Failed to set global guard: {0}")]
-    SetError(#[from] SetError<Arc<Mutex<OtelGuard>>>),
+    SetError(#[from] SetError<Arc<Mutex<Option<OtelGuard>>>>),
     #[error("Global guard not initialized")]
     NotInitialized,
+    #[error("Global guard lock poisoned: {0}")]
+    GuardPoisoned(&'static str),
     #[error("Global system metrics err: {0}")]
     MetricsError(String),
     #[error("Failed to get current PID: {0}")]

crates/obs/src/global.rs

@@ -22,7 +22,7 @@ const LOG_SUBSYSTEM_GLOBAL: &str = "global";
 const EVENT_OBS_GLOBAL_STATE: &str = "obs_global_state";
 
 /// Global guard for OpenTelemetry tracing
-static GLOBAL_GUARD: OnceCell<Arc<Mutex<OtelGuard>>> = OnceCell::const_new();
+static GLOBAL_GUARD: OnceCell<Arc<Mutex<Option<OtelGuard>>>> = OnceCell::const_new();
 
 /// Flag indicating if observability metric is enabled
 pub(crate) static OBSERVABILITY_METRIC_ENABLED: OnceCell<bool> = OnceCell::const_new();
@@ -158,13 +158,15 @@ pub fn set_global_guard(guard: OtelGuard) -> Result<(), GlobalError> {
         state = "guard_initializing",
         "obs global state changed"
     );
-    GLOBAL_GUARD.set(Arc::new(Mutex::new(guard))).map_err(GlobalError::SetError)
+    GLOBAL_GUARD
+        .set(Arc::new(Mutex::new(Some(guard))))
+        .map_err(GlobalError::SetError)
 }
 
 /// Get the global guard for OtelGuard
 ///
 /// # Returns
-/// * `Ok(Arc<Mutex<OtelGuard>>)` if guard exists
+/// * `Ok(Arc<Mutex<Option<OtelGuard>>>>)` if guard exists
 /// * `Err(GuardError)` if guard not initialized
 ///
 /// # Example
@@ -178,10 +180,23 @@ pub fn set_global_guard(guard: OtelGuard) -> Result<(), GlobalError> {
 /// #    Ok(())
 /// # }
 /// ```
-pub fn get_global_guard() -> Result<Arc<Mutex<OtelGuard>>, GlobalError> {
+pub fn get_global_guard() -> Result<Arc<Mutex<Option<OtelGuard>>>, GlobalError> {
     GLOBAL_GUARD.get().cloned().ok_or(GlobalError::NotInitialized)
 }
 
+/// Explicitly drop the global guard so telemetry providers can flush before exit.
+pub fn shutdown_global_guard() -> Result<(), GlobalError> {
+    let guard_handle = get_global_guard()?;
+    let guard = {
+        let mut slot = guard_handle
+            .lock()
+            .map_err(|_| GlobalError::GuardPoisoned("global observability guard"))?;
+        slot.take()
+    };
+    drop(guard);
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/obs/src/telemetry/local.rs

@@ -213,14 +213,14 @@ pub(super) fn init_local_logging(
         match init_file_logging_internal(config, log_directory, logger_level, is_production) {
             Ok(guard) => Ok(guard),
             Err(error) if should_fallback_to_stdout(&error) => {
-                let guard = init_stdout_only(config, logger_level, is_production);
+                let guard = init_stdout_only(config, logger_level, is_production)?;
                 emit_file_logging_fallback_warning(log_directory, &error);
                 Ok(guard)
             }
             Err(error) => Err(error),
         }
     } else {
-        Ok(init_stdout_only(config, logger_level, is_production))
+        init_stdout_only(config, logger_level, is_production)
     }
 }
 
@@ -236,7 +236,7 @@ pub(super) fn init_local_logging(
 /// * `_config` - Unused at the moment; reserved for future configuration.
 /// * `logger_level` - Effective log level string.
 /// * `is_production` - Controls span event verbosity.
-fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: bool) -> OtelGuard {
+fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: bool) -> Result<OtelGuard, TelemetryError> {
     let env_filter = build_env_filter(logger_level, None);
     let (nb, guard) = tracing_appender::non_blocking(std::io::stdout());
     let span_events = if is_production { FmtSpan::CLOSE } else { FmtSpan::FULL };
@@ -249,7 +249,8 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
         .with(env_filter)
         .with(ErrorLayer::default())
         .with(fmt_layer)
-        .init();
+        .try_init()
+        .map_err(|err| TelemetryError::SubscriberInit(err.to_string()))?;
 
     set_observability_metric_enabled(false);
     counter!("rustfs_start_total").increment(1);
@@ -266,7 +267,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
         "local logging state"
     );
 
-    OtelGuard {
+    Ok(OtelGuard {
         tracer_provider: None,
         meter_provider: None,
         logger_provider: None,
@@ -274,7 +275,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
         tracing_guard: Some(guard),
         stdout_guard: None,
         cleanup_handle: None,
-    }
+    })
 }
 
 // ─── File-rolling ─────────────────────────────────────────────────────────────
@@ -359,7 +360,8 @@ fn init_file_logging_internal(
         .with(ErrorLayer::default())
         .with(file_layer)
         .with(stdout_layer)
-        .init();
+        .try_init()
+        .map_err(|err| TelemetryError::SubscriberInit(err.to_string()))?;
 
     set_observability_metric_enabled(false);
 

crates/obs/src/telemetry/otel.rs

@@ -295,7 +295,8 @@ pub(super) fn init_observability_http(
         .with(tracer_layer)
         .with(otel_bridge)
         .with(metrics_layer)
-        .init();
+        .try_init()
+        .map_err(|err| TelemetryError::SubscriberInit(err.to_string()))?;
 
     counter!("rustfs_start_total").increment(1);
     info!(