feat: preserve request ids across async recovery logs (#3451)

* feat(obs): promote request ids in structured logs

* refactor(tracing): propagate spans into request tasks

* test(ecstore): baseline recovery monitor log chains

* fix(replication): reduce startup resync log noise

* chore(docs): stop tracking local recovery baseline

* chore(obs): polish request id logging cleanup

crates/obs/src/telemetry/local.rs

@@ -48,13 +48,19 @@ use rustfs_config::observability::{
     DEFAULT_OBS_LOG_ZSTD_WORKERS,
 };
 use rustfs_config::{APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOG_STDOUT_ENABLED};
+use serde_json::Value as JsonValue;
 use std::sync::Arc;
+use std::{collections::BTreeMap, fmt};
 use std::{fs, io::IsTerminal, time::Duration};
 use tracing::Subscriber;
 use tracing::{info, warn};
 use tracing_error::ErrorLayer;
 use tracing_subscriber::{
-    fmt::{format::FmtSpan, time::LocalTime},
+    fmt::{
+        FormatEvent,
+        format::{FmtSpan, Format, Json, JsonFields, Writer},
+        time::LocalTime,
+    },
     layer::SubscriberExt,
     registry::LookupSpan,
     util::SubscriberInitExt,
@@ -65,6 +71,73 @@ const LOG_SUBSYSTEM_LOCAL_LOGGING: &str = "local_logging";
 const EVENT_LOCAL_LOGGING_STATE: &str = "local_logging_state";
 const EVENT_LOG_CLEANER_STATE: &str = "log_cleaner_state";
 const STDERR_WARNING_PREFIX: &str = "[WARN]";
+const REQUEST_ID: &str = "request-id";
+
+#[derive(Clone, Debug)]
+struct RequestIdJsonFormat<T> {
+    inner: Format<Json, T>,
+}
+
+impl<T> RequestIdJsonFormat<T> {
+    fn new(inner: Format<Json, T>) -> Self {
+        Self { inner }
+    }
+}
+
+impl<S, T> FormatEvent<S, JsonFields> for RequestIdJsonFormat<T>
+where
+    S: Subscriber + for<'lookup> LookupSpan<'lookup>,
+    T: tracing_subscriber::fmt::time::FormatTime,
+{
+    fn format_event(
+        &self,
+        ctx: &tracing_subscriber::fmt::FmtContext<'_, S, JsonFields>,
+        mut writer: Writer<'_>,
+        event: &tracing::Event<'_>,
+    ) -> fmt::Result {
+        let mut buffer = String::new();
+        self.inner.format_event(ctx, Writer::new(&mut buffer), event)?;
+
+        let request_id = span_scope_request_id(ctx, event);
+        if request_id.is_none() {
+            return writer.write_str(&buffer);
+        }
+
+        let trimmed = buffer.trim_end();
+        let mut payload: JsonValue = serde_json::from_str(trimmed).map_err(|_| fmt::Error)?;
+        if let Some(object) = payload.as_object_mut() {
+            object
+                .entry(REQUEST_ID.to_string())
+                .or_insert_with(|| JsonValue::String(request_id.expect("checked is_some")));
+        }
+
+        let serialized = serde_json::to_string(&payload).map_err(|_| fmt::Error)?;
+        writer.write_str(&serialized)?;
+        writer.write_char('\n')
+    }
+}
+
+fn span_scope_request_id<S>(
+    ctx: &tracing_subscriber::fmt::FmtContext<'_, S, JsonFields>,
+    event: &tracing::Event<'_>,
+) -> Option<String>
+where
+    S: Subscriber + for<'lookup> LookupSpan<'lookup>,
+{
+    let current_span = event.parent().and_then(|id| ctx.span(id)).or_else(|| ctx.lookup_current())?;
+    let mut request_id = None;
+
+    for span in current_span.scope().from_root() {
+        let extensions = span.extensions();
+        let formatted_fields = extensions.get::<tracing_subscriber::fmt::FormattedFields<JsonFields>>()?;
+        let fields: BTreeMap<String, JsonValue> = serde_json::from_str(&formatted_fields.fields).ok()?;
+        if let Some(value) = fields.get(REQUEST_ID).and_then(JsonValue::as_str) {
+            request_id = Some(value.to_string());
+        }
+    }
+
+    request_id
+}
 
 pub(super) fn build_json_log_layer<S, W>(writer: W, enable_ansi: bool, span_events: FmtSpan) -> impl tracing_subscriber::Layer<S>
 where
@@ -81,9 +154,11 @@ where
         .with_line_number(true)
         .with_writer(writer)
         .json()
+        .flatten_event(true)
         .with_current_span(true)
         .with_span_list(true)
         .with_span_events(span_events)
+        .map_event_format(RequestIdJsonFormat::new)
 }
 
 /// Initialize local logging (stdout-only or file-rolling).
@@ -520,7 +595,7 @@ pub fn spawn_cleanup_task(
                 }
                 Ok(Err(e)) => {
                     counter!(METRIC_LOG_CLEANER_RUN_FAILURES_TOTAL).increment(1);
-                    tracing::warn!(
+                    warn!(
                         event = EVENT_LOG_CLEANER_STATE,
                         component = LOG_COMPONENT_OBS,
                         subsystem = LOG_SUBSYSTEM_LOCAL_LOGGING,
@@ -531,7 +606,7 @@ pub fn spawn_cleanup_task(
                 }
                 Err(e) => {
                     counter!(METRIC_LOG_CLEANER_RUN_FAILURES_TOTAL).increment(1);
-                    tracing::warn!(
+                    warn!(
                         event = EVENT_LOG_CLEANER_STATE,
                         component = LOG_COMPONENT_OBS,
                         subsystem = LOG_SUBSYSTEM_LOCAL_LOGGING,
@@ -593,7 +668,7 @@ mod tests {
         let subscriber = Registry::default().with(layer);
 
         tracing::subscriber::with_default(subscriber, || {
-            tracing::info!(component = "obs_contract_test", sink = "stdout", "hello world");
+            info!(component = "obs_contract_test", sink = "stdout", "hello world");
         });
 
         let raw = String::from_utf8(
@@ -609,6 +684,55 @@ mod tests {
         (raw, parsed)
     }
 
+    fn render_json_log_with_request_span() -> Value {
+        let writer = SharedWriter::default();
+        let layer = build_json_log_layer(writer.clone(), false, FmtSpan::NONE);
+        let subscriber = Registry::default().with(layer);
+
+        tracing::subscriber::with_default(subscriber, || {
+            let span = tracing::info_span!("http-request", request_id = "req-123", method = "GET");
+            let _guard = span.enter();
+            info!(component = "obs_contract_test", message = "inside request span");
+        });
+
+        let raw = String::from_utf8(
+            writer
+                .inner
+                .lock()
+                .expect("shared writer lock should not be poisoned")
+                .clone(),
+        )
+        .expect("log output should be valid UTF-8");
+        let first_line = raw.lines().next().expect("expected one JSON log line");
+        serde_json::from_str(first_line).expect("log line should be valid JSON")
+    }
+
+    fn render_json_log_with_recovery_monitor_child_span() -> Value {
+        let writer = SharedWriter::default();
+        let layer = build_json_log_layer(writer.clone(), false, FmtSpan::NONE);
+        let subscriber = Registry::default().with(layer);
+
+        tracing::subscriber::with_default(subscriber, || {
+            let request_span = tracing::info_span!("request-span", request_id = "req-parent", method = "GET");
+            let _request_guard = request_span.enter();
+            let recovery_span =
+                tracing::info_span!("recovery-monitor", kind = "remote_disk", endpoint = "http://127.0.0.1:9000/data");
+            let _recovery_guard = recovery_span.enter();
+            warn!(component = "obs_contract_test", message = "inside recovery monitor");
+        });
+
+        let raw = String::from_utf8(
+            writer
+                .inner
+                .lock()
+                .expect("shared writer lock should not be poisoned")
+                .clone(),
+        )
+        .expect("log output should be valid UTF-8");
+        let first_line = raw.lines().next().expect("expected one JSON log line");
+        serde_json::from_str(first_line).expect("log line should be valid JSON")
+    }
+
     #[test]
     /// Invalid file names should be reported as errors instead of panicking.
     fn test_init_file_logging_invalid_filename_does_not_panic() {
@@ -661,9 +785,9 @@ mod tests {
 
         assert!(!raw.contains('\u{1b}'), "JSON log output must not contain ANSI escape sequences: {raw}");
         assert_eq!(parsed.get("level").and_then(Value::as_str), Some("INFO"));
-        assert_eq!(parsed["fields"]["message"], Value::String("hello world".to_string()));
-        assert_eq!(parsed["fields"]["component"], Value::String("obs_contract_test".to_string()));
-        assert_eq!(parsed["fields"]["sink"], Value::String("stdout".to_string()));
+        assert_eq!(parsed["message"], Value::String("hello world".to_string()));
+        assert_eq!(parsed["component"], Value::String("obs_contract_test".to_string()));
+        assert_eq!(parsed["sink"], Value::String("stdout".to_string()));
         assert!(parsed.get("target").is_some(), "expected target field in JSON log output");
     }
 
@@ -686,18 +810,54 @@ mod tests {
             .collect::<BTreeSet<_>>();
         assert_eq!(plain_keys, color_keys, "top-level JSON log keys must stay stable across sinks");
 
-        let plain_field_keys = plain["fields"]
+        let plain_field_keys = plain
             .as_object()
-            .expect("plain JSON log fields should be an object")
+            .expect("plain JSON log should be an object")
             .keys()
+            .filter(|key| {
+                !matches!(
+                    key.as_str(),
+                    "timestamp" | "level" | "target" | "filename" | "line_number" | "threadName" | "threadId"
+                )
+            })
             .cloned()
             .collect::<BTreeSet<_>>();
-        let color_field_keys = color["fields"]
+        let color_field_keys = color
             .as_object()
-            .expect("color JSON log fields should be an object")
+            .expect("color JSON log should be an object")
             .keys()
+            .filter(|key| {
+                !matches!(
+                    key.as_str(),
+                    "timestamp" | "level" | "target" | "filename" | "line_number" | "threadName" | "threadId"
+                )
+            })
             .cloned()
             .collect::<BTreeSet<_>>();
         assert_eq!(plain_field_keys, color_field_keys, "structured field keys must stay stable across sinks");
     }
+
+    #[test]
+    fn test_json_log_layer_promotes_request_id_from_current_span() {
+        let parsed = render_json_log_with_request_span();
+
+        assert_eq!(parsed["request_id"], Value::String("req-123".to_string()));
+        assert_eq!(parsed["message"], Value::String("inside request span".to_string()));
+        assert_eq!(parsed["span"]["request_id"], Value::String("req-123".to_string()));
+    }
+
+    #[test]
+    fn test_json_log_layer_promotes_parent_request_id_for_recovery_monitor_child_span() {
+        let parsed = render_json_log_with_recovery_monitor_child_span();
+
+        assert_eq!(parsed["request_id"], Value::String("req-parent".to_string()));
+        assert_eq!(parsed["message"], Value::String("inside recovery monitor".to_string()));
+        assert_eq!(parsed["span"]["name"], Value::String("recovery-monitor".to_string()));
+        assert_eq!(parsed["span"]["kind"], Value::String("remote_disk".to_string()));
+        let spans = parsed["spans"].as_array().expect("spans should be present");
+        assert!(spans.iter().any(|span| {
+            span.get("name").and_then(Value::as_str) == Some("request-span")
+                && span.get("request_id").and_then(Value::as_str) == Some("req-parent")
+        }));
+    }
 }