feat: preserve request ids across async recovery logs (#3451)

* feat(obs): promote request ids in structured logs

* refactor(tracing): propagate spans into request tasks

* test(ecstore): baseline recovery monitor log chains

* fix(replication): reduce startup resync log noise

* chore(docs): stop tracking local recovery baseline

* chore(obs): polish request id logging cleanup

rustfs/src/admin/console.rs

@@ -15,7 +15,11 @@
 use crate::admin::handlers::health::{HealthProbe, build_health_response_parts, collect_dependency_readiness};
 use crate::license::has_valid_license;
 use crate::server::has_path_prefix;
-use crate::server::{CONSOLE_PREFIX, FAVICON_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, LICENSE, RUSTFS_ADMIN_PREFIX, VERSION};
+use crate::server::{
+    CONSOLE_PREFIX, FAVICON_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, HeaderMapCarrier, LICENSE, RUSTFS_ADMIN_PREFIX,
+    RequestContextLayer, VERSION,
+};
+use crate::storage::request_context::RequestContext;
 use crate::version::build;
 use axum::{
     Json, Router,
@@ -27,6 +31,7 @@ use axum::{
 };
 use http::{HeaderMap, HeaderName, HeaderValue, Method, StatusCode, Uri};
 use mime_guess::from_path;
+use opentelemetry::global;
 use rust_embed::RustEmbed;
 use serde::Serialize;
 use std::{
@@ -41,7 +46,8 @@ use tower_http::limit::RequestBodyLimitLayer;
 use tower_http::request_id::{MakeRequestUuid, PropagateRequestIdLayer, SetRequestIdLayer};
 use tower_http::timeout::TimeoutLayer;
 use tower_http::trace::TraceLayer;
-use tracing::{debug, error, info, instrument, warn};
+use tracing::{Span, debug, error, info, instrument, warn};
+use tracing_opentelemetry::OpenTelemetrySpanExt;
 
 #[derive(RustEmbed)]
 #[folder = "$CARGO_MANIFEST_DIR/static"]
@@ -491,9 +497,45 @@ fn setup_console_middleware_stack(
     // Add comprehensive middleware layers using tower-http features
     app = app
         .layer(CatchPanicLayer::new())
+        .layer(
+            TraceLayer::new_for_http()
+                .make_span_with(|request: &Request| {
+                    let request_context = request.extensions().get::<RequestContext>();
+                    let request_id = request_context.map(|ctx| ctx.request_id.as_str()).unwrap_or("unknown");
+                    let trace_id = request_context.and_then(|ctx| ctx.trace_id.as_deref()).unwrap_or("unknown");
+                    let span_id = request_context.and_then(|ctx| ctx.span_id.as_deref()).unwrap_or("unknown");
+
+                    let parent_context = global::get_text_map_propagator(|propagator| {
+                        propagator.extract(&HeaderMapCarrier::new(request.headers()))
+                    });
+
+                    let span = tracing::info_span!(
+                        "console-request",
+                        request_id = %request_id,
+                        trace_id = %trace_id,
+                        span_id = %span_id,
+                        method = %request.method(),
+                        uri = %request.uri(),
+                        status_code = tracing::field::Empty,
+                    );
+
+                    if span.is_disabled() {
+                        return span;
+                    }
+
+                    if let Err(err) = span.set_parent(parent_context) {
+                        debug!(error = ?err, "Failed to propagate tracing context for console request");
+                    }
+
+                    span
+                })
+                .on_response(|response: &Response, _latency: Duration, span: &Span| {
+                    span.record("status_code", tracing::field::display(response.status()));
+                }),
+        )
         .layer(PropagateRequestIdLayer::x_request_id())
+        .layer(RequestContextLayer)
         .layer(SetRequestIdLayer::x_request_id(MakeRequestUuid))
-        .layer(TraceLayer::new_for_http())
         // Compress responses
         .layer(CompressionLayer::new())
         .layer(middleware::from_fn(console_logging_middleware))
@@ -684,12 +726,47 @@ pub(crate) fn make_console_server() -> Router {
 mod tests {
     use super::*;
     use axum::body::Body;
+    use axum::routing::get;
     use http::{Request, StatusCode};
     use http_body_util::BodyExt;
     use serial_test::serial;
+    use std::io;
     use std::net::{IpAddr, Ipv4Addr};
+    use std::sync::{Arc, Mutex};
     use temp_env::async_with_vars;
     use tower::ServiceExt;
+    use tracing_subscriber::{Registry, layer::SubscriberExt};
+
+    #[derive(Clone, Default)]
+    struct SharedWriter {
+        inner: Arc<Mutex<Vec<u8>>>,
+    }
+
+    struct SharedWriterGuard {
+        inner: Arc<Mutex<Vec<u8>>>,
+    }
+
+    impl<'writer> tracing_subscriber::fmt::MakeWriter<'writer> for SharedWriter {
+        type Writer = SharedWriterGuard;
+
+        fn make_writer(&'writer self) -> Self::Writer {
+            SharedWriterGuard {
+                inner: Arc::clone(&self.inner),
+            }
+        }
+    }
+
+    impl io::Write for SharedWriterGuard {
+        fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
+            let mut inner = self.inner.lock().expect("shared writer lock should not be poisoned");
+            inner.extend_from_slice(buf);
+            Ok(buf.len())
+        }
+
+        fn flush(&mut self) -> io::Result<()> {
+            Ok(())
+        }
+    }
 
     #[test]
     fn console_api_base_url_keeps_rustfs_admin_prefix() {
@@ -739,6 +816,112 @@ mod tests {
         );
     }
 
+    #[tokio::test(flavor = "current_thread")]
+    async fn console_trace_layer_records_request_id_on_current_span() {
+        let writer = SharedWriter::default();
+        let subscriber = Registry::default().with(
+            tracing_subscriber::fmt::layer()
+                .without_time()
+                .with_target(false)
+                .with_level(false)
+                .with_ansi(false)
+                .json()
+                .flatten_event(true)
+                .with_current_span(true)
+                .with_span_list(true)
+                .with_writer(writer.clone()),
+        );
+        let _guard = tracing::subscriber::set_default(subscriber);
+
+        let app = Router::new()
+            .route(
+                "/trace-test",
+                get(|| async {
+                    tracing::info!("console handler log");
+                    StatusCode::OK
+                }),
+            )
+            .layer(
+                TraceLayer::new_for_http()
+                    .make_span_with(|request: &http::Request<_>| {
+                        let request_context = request.extensions().get::<RequestContext>();
+                        let request_id = request_context.map(|ctx| ctx.request_id.as_str()).unwrap_or("unknown");
+                        let trace_id = request_context.and_then(|ctx| ctx.trace_id.as_deref()).unwrap_or("unknown");
+                        let span_id = request_context.and_then(|ctx| ctx.span_id.as_deref()).unwrap_or("unknown");
+
+                        let parent_context = global::get_text_map_propagator(|propagator| {
+                            propagator.extract(&HeaderMapCarrier::new(request.headers()))
+                        });
+
+                        let span = tracing::info_span!(
+                            "console-request",
+                            request_id = %request_id,
+                            trace_id = %trace_id,
+                            span_id = %span_id,
+                            method = %request.method(),
+                            uri = %request.uri(),
+                            status_code = tracing::field::Empty,
+                        );
+
+                        if span.is_disabled() {
+                            return span;
+                        }
+
+                        if let Err(err) = span.set_parent(parent_context) {
+                            debug!(error = ?err, "Failed to propagate tracing context for console request");
+                        }
+
+                        span
+                    })
+                    .on_response(|response: &Response, _latency: Duration, span: &Span| {
+                        span.record("status_code", tracing::field::display(response.status()));
+                    }),
+            )
+            .layer(RequestContextLayer)
+            .layer(SetRequestIdLayer::x_request_id(MakeRequestUuid));
+
+        app.oneshot(
+            Request::builder()
+                .uri("/trace-test")
+                .body(Body::empty())
+                .expect("failed to build trace test request"),
+        )
+        .await
+        .expect("trace test request should complete");
+
+        let output = String::from_utf8(
+            writer
+                .inner
+                .lock()
+                .expect("shared writer lock should not be poisoned")
+                .clone(),
+        )
+        .expect("console trace log output should be valid UTF-8");
+
+        let log = output
+            .lines()
+            .map(|line| serde_json::from_str::<serde_json::Value>(line).expect("console log line should be valid JSON"))
+            .find(|value| value.get("message").and_then(serde_json::Value::as_str) == Some("console handler log"))
+            .expect("expected console handler log entry");
+
+        assert_eq!(log["span"]["method"], serde_json::Value::String("GET".to_string()));
+        assert_eq!(log["span"]["name"], serde_json::Value::String("console-request".to_string()));
+        assert!(
+            log.get("span")
+                .and_then(|span| span.get("request_id"))
+                .and_then(serde_json::Value::as_str)
+                .is_some(),
+            "{output}"
+        );
+        assert_ne!(
+            log.get("span")
+                .and_then(|span| span.get("request_id"))
+                .and_then(serde_json::Value::as_str),
+            Some("unknown"),
+            "{output}"
+        );
+    }
+
     /// Regression: when no console CORS origins are configured (the new
     /// default), the layer must NOT emit `Access-Control-Allow-Origin`, so
     /// browsers treat responses as same-origin only.

rustfs/src/admin/handlers/heal.rs

@@ -17,6 +17,7 @@ use crate::admin::router::{AdminOperation, Operation, S3Router};
 use crate::app::context::resolve_object_store_handle;
 use crate::server::ADMIN_PREFIX;
 use crate::server::RemoteAddr;
+use crate::storage::request_context::spawn_traced;
 use bytes::Bytes;
 use http::{HeaderMap, HeaderValue, Uri};
 use hyper::{Method, StatusCode};
@@ -34,7 +35,6 @@ use s3s::{Body, S3Request, S3Response, S3Result, s3_error};
 use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
 use time::{OffsetDateTime, format_description::well_known::Rfc3339};
-use tokio::spawn;
 use tokio::sync::mpsc;
 use tracing::{info, warn};
 
@@ -395,7 +395,7 @@ impl Operation for HealHandler {
             let tx_clone = tx.clone();
             let heal_path_str = heal_path.to_str().unwrap_or_default().to_string();
             let client_token = hip.client_token.clone();
-            spawn(async move {
+            spawn_traced(async move {
                 match rustfs_common::heal_channel::query_heal_status(heal_path_str, client_token).await {
                     Ok(response) if response.success => {
                         let (summary, items) = heal_channel_response_status(&response);
@@ -445,7 +445,7 @@ impl Operation for HealHandler {
             let client_token = hip.client_token.clone();
             let client_address = client_address.clone();
             let heal_settings = hip.hs;
-            spawn(async move {
+            spawn_traced(async move {
                 match rustfs_common::heal_channel::cancel_heal_task(heal_path_str, client_token.clone()).await {
                     Ok(response) if response.success => {
                         let resp_bytes = if client_token.is_empty() {
@@ -495,7 +495,7 @@ impl Operation for HealHandler {
             // Use new heal channel mechanism
             let tx_clone = tx.clone();
             let client_address = client_address.clone();
-            spawn(async move {
+            spawn_traced(async move {
                 // Create heal request through channel
                 let heal_request = build_heal_channel_request(&hip);
                 let client_token = heal_request.id.clone();

rustfs/src/admin/handlers/metrics.rs

@@ -19,6 +19,7 @@
 //! exposition endpoint.
 
 use crate::admin::router::Operation;
+use crate::storage::request_context::spawn_traced;
 use bytes::Bytes;
 use futures::{Stream, StreamExt};
 use http::{HeaderMap, HeaderValue, Uri};
@@ -35,9 +36,9 @@ use std::collections::{HashMap, HashSet};
 use std::pin::Pin;
 use std::task::{Context, Poll};
 use std::time::Duration as StdDuration;
+use tokio::select;
 use tokio::sync::mpsc;
 use tokio::time::interval;
-use tokio::{select, spawn};
 use tokio_stream::wrappers::ReceiverStream;
 use tracing::{debug, error, warn};
 
@@ -217,7 +218,7 @@ impl Operation for MetricsHandler {
         });
         let body = Body::from(in_stream);
 
-        spawn(async move {
+        spawn_traced(async move {
             while n > 0 {
                 let mut metrics = RealtimeMetrics::default();
                 let local_metrics = collect_local_metrics(types, &opts).await;

rustfs/src/admin/handlers/tier.rs

@@ -21,6 +21,7 @@ use crate::{
     app::context::{resolve_object_store_handle, resolve_tier_config_handle},
     auth::{check_key_valid, get_session_token},
     server::{ADMIN_PREFIX, RemoteAddr},
+    storage::request_context::spawn_traced,
 };
 use http::Uri;
 use http::{HeaderMap, StatusCode};
@@ -54,7 +55,6 @@ use s3s::{
 use serde_urlencoded::from_bytes;
 use std::collections::HashMap;
 use time::OffsetDateTime;
-use tokio::spawn;
 use tracing::{debug, warn};
 
 const LOG_COMPONENT_ADMIN: &str = "admin";
@@ -91,7 +91,7 @@ pub struct AddTier {}
 
 fn spawn_transition_tier_config_propagation(action: &'static str) {
     if let Some(notification_sys) = get_global_notification_sys() {
-        spawn(async move {
+        spawn_traced(async move {
             for peer_result in notification_sys.load_transition_tier_config().await {
                 if let Some(err) = peer_result.err {
                     warn!(

rustfs/src/admin/router.rs

@@ -23,6 +23,7 @@ use crate::server::{
     ADMIN_PREFIX, HEALTH_PREFIX, HEALTH_READY_PATH, MINIO_ADMIN_PREFIX, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, is_admin_path,
 };
 use crate::storage::access::{ReqInfo, authorize_request};
+use crate::storage::request_context::spawn_traced;
 use aws_sdk_s3::primitives::ByteStream as AwsByteStream;
 use bytes::Bytes;
 use futures::{Stream, StreamExt};
@@ -1310,7 +1311,7 @@ fn build_listen_notification_response(uri: &Uri, bucket: Option<&str>) -> S3Resu
         inner: ReceiverStream::new(rx),
     });
 
-    tokio::spawn(async move {
+    spawn_traced(async move {
         let mut ticker = tokio::time::interval(interval_duration);
         let mut peer_ticker = tokio::time::interval(LISTEN_NOTIFICATION_PEER_POLL_INTERVAL);
         // Skip the immediate first tick so behavior starts after interval duration.

rustfs/src/server/mod.rs

@@ -41,7 +41,9 @@ pub use service_state::ShutdownSignal;
 pub use service_state::wait_for_shutdown;
 
 // Items only used within the library crate (admin handlers, server/http.rs, etc.).
+pub(crate) use http::HeaderMapCarrier;
 pub(crate) use http::active_http_requests;
+pub(crate) use layer::RequestContextLayer;
 pub(crate) use module_switch::{
     ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches, current_module_switch_snapshot,
     refresh_persisted_module_switches_from_store, save_persisted_module_switches_to_store, validate_module_switch_update,

rustfs/src/storage/rpc/http_service.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use crate::server::RPC_PREFIX;
+use crate::storage::request_context::spawn_traced;
 use bytes::{Bytes, BytesMut};
 use futures_util::TryStreamExt;
 use http::{HeaderMap, Method, Request, Response, StatusCode, Uri};
@@ -243,7 +244,7 @@ async fn handle_walk_dir(req: Request<Incoming>) -> Response<Body> {
     };
 
     let (rd, mut wd) = tokio::io::duplex(DEFAULT_READ_BUFFER_SIZE);
-    tokio::spawn(async move {
+    spawn_traced(async move {
         if let Err(e) = disk.walk_dir(args, &mut wd).await {
             warn!(error = %e, "walk_dir failed");
         }