refactor(obs): unify local and otlp log output (#3352)

wip(obs): start lg-002 output unification

crates/obs/src/telemetry/otel.rs

@@ -41,7 +41,7 @@ use crate::config::OtelConfig;
 use crate::global::set_observability_metric_enabled;
 use crate::telemetry::filter::build_env_filter;
 use crate::telemetry::guard::{MemoryProfilingAgent, OtelGuard, ProfilingAgent};
-use crate::telemetry::local::spawn_cleanup_task;
+use crate::telemetry::local::{build_json_log_layer, spawn_cleanup_task};
 use crate::telemetry::recorder::Recorder;
 use crate::telemetry::resource::build_resource;
 use crate::telemetry::rolling::{RollingAppender, Rotation};
@@ -68,12 +68,7 @@ use std::{fs, io::IsTerminal, time::Duration};
 use tracing::info;
 use tracing_error::ErrorLayer;
 use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer};
-use tracing_subscriber::{
-    Layer,
-    fmt::{format::FmtSpan, time::LocalTime},
-    layer::SubscriberExt,
-    util::SubscriberInitExt,
-};
+use tracing_subscriber::{Layer, fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt};
 
 /// Initialize the full OpenTelemetry HTTP pipeline (traces + metrics + logs).
 ///
@@ -231,20 +226,7 @@ pub(super) fn init_observability_http(
                 RollingAppender::new(log_directory, log_filename.to_string(), rotation, max_single_file_size, match_mode)?;
 
             let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
-            let file_layer = tracing_subscriber::fmt::layer()
-                .with_timer(LocalTime::rfc_3339())
-                .with_target(true)
-                .with_ansi(false)
-                .with_thread_names(true)
-                .with_thread_ids(true)
-                .with_file(true)
-                .with_line_number(true)
-                .with_writer(non_blocking)
-                .json()
-                .with_current_span(true)
-                .with_span_list(true)
-                .with_span_events(span_events.clone())
-                .with_filter(build_env_filter(logger_level, None));
+            let file_layer = build_json_log_layer(non_blocking, false, span_events.clone());
             let cleanup_handle = spawn_cleanup_task(config, log_directory, log_filename, keep_files);
             Ok((file_layer, guard, cleanup_handle, rotation_str))
         })();
@@ -256,8 +238,17 @@ pub(super) fn init_observability_http(
                 cleanup_handle = Some(new_cleanup_handle);
 
                 info!(
-                    "Init file logging at '{}', rotation: {}, keep {} files",
-                    log_directory, rotation_str, keep_files
+                    backend = "local",
+                    sink = "file",
+                    output_format = "json",
+                    log_directory,
+                    rotation = %rotation_str,
+                    keep_files,
+                    stdout_mirror_enabled = config.log_stdout_enabled.unwrap_or(DEFAULT_OBS_LOG_STDOUT_ENABLED)
+                        || !is_production,
+                    logger_level,
+                    is_production,
+                    "Initialized local logging fallback for observability"
                 );
             }
             Err(error) if crate::telemetry::local::should_fallback_to_stdout(&error) => {
@@ -279,26 +270,16 @@ pub(super) fn init_observability_http(
     if force_stdout_logging || config.log_stdout_enabled.unwrap_or(DEFAULT_OBS_LOG_STDOUT_ENABLED) || !is_production {
         let (stdout_nb, stdout_g) = tracing_appender::non_blocking(std::io::stdout());
         stdout_guard = Some(stdout_g);
-        stdout_layer_opt = Some(
-            tracing_subscriber::fmt::layer()
-                .with_timer(LocalTime::rfc_3339())
-                .with_target(true)
-                .with_ansi(std::io::stdout().is_terminal())
-                .with_thread_names(true)
-                .with_thread_ids(true)
-                .with_file(true)
-                .with_line_number(true)
-                .with_writer(stdout_nb)
-                .with_span_events(span_events)
-                .with_filter(build_env_filter(logger_level, None)),
-        );
+        stdout_layer_opt = Some(build_json_log_layer(stdout_nb, std::io::stdout().is_terminal(), span_events));
     }
+    let local_file_fallback_enabled = file_layer_opt.is_some();
+    let stdout_mirror_enabled = stdout_guard.is_some();
     let filter = build_env_filter(logger_level, None);
     tracing_subscriber::registry()
         .with(filter)
         .with(ErrorLayer::default())
-        .with(file_layer_opt) // File
-        .with(stdout_layer_opt) // Stdout (only if file logging enabled it)
+        .with(file_layer_opt)
+        .with(stdout_layer_opt)
         .with(tracer_layer)
         .with(otel_bridge)
         .with(metrics_layer)
@@ -306,8 +287,16 @@ pub(super) fn init_observability_http(
 
     counter!("rustfs_start_total").increment(1);
     info!(
-        "Init observability (HTTP): trace='{}', metric='{}', log='{}'",
-        trace_ep, metric_ep, log_ep
+        backend = "otlp_http",
+        trace_endpoint = %trace_ep,
+        metric_endpoint = %metric_ep,
+        log_endpoint = %log_ep,
+        local_file_fallback_enabled,
+        stdout_mirror_enabled,
+        output_format = "json",
+        logger_level,
+        is_production,
+        "Initialized observability"
     );
 
     Ok(OtelGuard {