feat(obs): enhance observability with tracing spans and metrics integration

crates/obs/src/lib.rs

@@ -65,6 +65,7 @@ mod global;
 mod logging;
 pub mod metrics;
 pub mod semconv;
+mod task;
 mod telemetry;
 
 pub use cleaner::*;
@@ -79,7 +80,8 @@ pub use metrics::{
     MetricsRuntimeShutdownHandle, MetricsRuntimeStatusSnapshot, MetricsRuntimeWorkerMutation, init_metrics_runtime,
     metrics_runtime_controller_snapshot, metrics_runtime_status_snapshot,
 };
-pub use semconv::{ErrorClass, Operation, ResultClass, Stage, observe_operation, stage_span};
+pub use semconv::{ErrorClass, Operation, ResultClass, Stage, StreamDirection, observe_operation, stage_span, stream_span};
+pub use task::spawn_traced;
 pub use telemetry::{OtelGuard, Recorder};
 
 // Dial9 Tokio runtime telemetry

crates/obs/src/semconv.rs

@@ -6,6 +6,7 @@
 
 use metrics::{Gauge, counter, gauge, histogram};
 use opentelemetry::trace::Status;
+use rustfs_utils::trace_attributes;
 use std::future::Future;
 use std::time::Instant;
 use tracing::Instrument;
@@ -41,6 +42,7 @@ pub enum Operation {
     DeleteObject,
     CreateMultipartUpload,
     UploadPart,
+    UploadPartCopy,
     CompleteMultipartUpload,
     AbortMultipartUpload,
 }
@@ -60,6 +62,7 @@ impl Operation {
             Self::DeleteObject => "delete_object",
             Self::CreateMultipartUpload => "create_multipart_upload",
             Self::UploadPart => "upload_part",
+            Self::UploadPartCopy => "upload_part_copy",
             Self::CompleteMultipartUpload => "complete_multipart_upload",
             Self::AbortMultipartUpload => "abort_multipart_upload",
         }
@@ -79,6 +82,22 @@ pub enum Stage {
     ReplicationRemote,
 }
 
+/// Bounded direction for a stream traversing the object pipeline.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum StreamDirection {
+    Ingress,
+    Copy,
+}
+
+impl StreamDirection {
+    const fn as_str(self) -> &'static str {
+        match self {
+            Self::Ingress => trace_attributes::stream_direction::INGRESS,
+            Self::Copy => trace_attributes::stream_direction::COPY,
+        }
+    }
+}
+
 impl Stage {
     pub const fn as_str(self) -> &'static str {
         match self {
@@ -146,7 +165,25 @@ impl ErrorClass {
 
 /// Create a child span for a bounded internal stage.
 pub fn stage_span(stage: Stage) -> tracing::Span {
-    tracing::info_span!("rustfs.stage", event = "rustfs_stage", component = "storage", stage = stage.as_str())
+    tracing::info_span!(
+        "rustfs.stage",
+        event = "rustfs_stage",
+        component = "storage",
+        stage = stage.as_str(),
+        "rustfs.stage" = stage.as_str()
+    )
+}
+
+/// Create a child span for a request stream without exposing payload contents.
+pub fn stream_span(direction: StreamDirection, expected_bytes: i64, buffer_bytes: usize) -> tracing::Span {
+    tracing::info_span!(
+        "rustfs.stream",
+        event = "rustfs_stream",
+        component = "storage",
+        "rustfs.stream.direction" = direction.as_str(),
+        "rustfs.stream.expected_bytes" = expected_bytes,
+        "rustfs.stream.buffer_bytes" = buffer_bytes
+    )
 }
 
 /// Observe a complete S3 operation without exposing request data.
@@ -159,7 +196,8 @@ where
         "rustfs.s3.operation",
         event = "s3_operation",
         component = "storage",
-        operation = operation_name
+        operation = operation_name,
+        "rustfs.operation" = operation_name
     );
     let in_flight = gauge!(METRIC_REQUESTS_IN_FLIGHT, "operation" => operation_name);
     in_flight.increment(1.0);
@@ -218,8 +256,10 @@ mod tests {
     #[test]
     fn operation_names_are_stable_and_bounded() {
         assert_eq!(Operation::PutObject.as_str(), "put_object");
+        assert_eq!(Operation::UploadPartCopy.as_str(), "upload_part_copy");
         assert_eq!(Operation::CompleteMultipartUpload.as_str(), "complete_multipart_upload");
         assert_eq!(Stage::ReplicationRemote.as_str(), "replication_remote");
+        assert_eq!(StreamDirection::Ingress.as_str(), trace_attributes::stream_direction::INGRESS);
         assert_eq!(ErrorClass::Quorum.as_str(), "quorum_failed");
     }
 

crates/obs/src/task.rs

@@ -0,0 +1,52 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Task helpers for preserving tracing context across Tokio task boundaries.
+
+use tracing::Instrument;
+
+/// Spawn a Tokio task that inherits the current tracing span.
+///
+/// Tokio does not propagate [`tracing::Span::current`] into spawned tasks.
+/// Use this for work that remains part of the caller's operation. Detached
+/// background side effects should start a new span instead.
+pub fn spawn_traced<F>(future: F) -> tokio::task::JoinHandle<F::Output>
+where
+    F: std::future::Future + Send + 'static,
+    F::Output: Send + 'static,
+{
+    tokio::spawn(future.instrument(tracing::Span::current()))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::spawn_traced;
+
+    #[tokio::test]
+    async fn spawned_task_inherits_current_span() {
+        tracing::subscriber::set_global_default(tracing_subscriber::Registry::default())
+            .expect("task tracing test should install its subscriber");
+
+        let (parent_id, task) = {
+            let parent = tracing::info_span!("parent");
+            let parent_id = parent.id().expect("enabled parent span should have an id");
+            let task = parent
+                .in_scope(|| spawn_traced(async { tracing::Span::current().id().expect("task should retain its parent span") }));
+            (parent_id, task)
+        };
+
+        let task_span_id = task.await.expect("task should complete");
+        assert_eq!(task_span_id, parent_id);
+    }
+}

crates/obs/src/telemetry/filter.rs

@@ -228,6 +228,16 @@ pub(super) fn build_env_filter(logger_level: &str, default_level: Option<&str>)
     filter
 }
 
+/// Build the filter used exclusively by the OTLP trace exporter.
+///
+/// Trace collection must not depend on the log sink's severity: production
+/// deployments commonly run with `RUST_LOG=warn` or `error` while still
+/// expecting request spans to be exported. RustFS operation spans are emitted
+/// at `info`, so retain that level regardless of `RUST_LOG`.
+pub(super) fn build_trace_filter() -> EnvFilter {
+    EnvFilter::new("info")
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -344,6 +354,13 @@ mod tests {
         });
     }
 
+    #[test]
+    fn trace_filter_ignores_rust_log_severity() {
+        temp_env::with_var("RUST_LOG", Some("error"), || {
+            assert_eq!(build_trace_filter().to_string(), "info");
+        });
+    }
+
     #[test]
     fn test_build_env_filter_target_only_rust_log_keeps_target_verbose() {
         // `RUST_LOG=hyper` is a target-only directive and should not be treated

crates/obs/src/telemetry/otel.rs

@@ -39,7 +39,7 @@
 use crate::cleaner::types::FileMatchMode;
 use crate::config::OtelConfig;
 use crate::global::set_observability_metric_enabled;
-use crate::telemetry::filter::build_env_filter;
+use crate::telemetry::filter::{build_env_filter, build_trace_filter};
 use crate::telemetry::guard::{OtelGuard, ProfilingAgent};
 use crate::telemetry::local::{build_json_log_layer, spawn_cleanup_task};
 use crate::telemetry::recorder::{Recorder, install_process_global_recorder};
@@ -241,8 +241,10 @@ pub(super) fn init_observability_http(
     // ── Tracing subscriber registry ───────────────────────────────────────────
     let tracer_layer = tracer_provider
         .as_ref()
-        .map(|p| OpenTelemetryLayer::new(p.tracer(service_name.to_string())));
-    let metrics_layer = meter_provider.as_ref().map(|p| MetricsLayer::new(p.clone()));
+        .map(|p| OpenTelemetryLayer::new(p.tracer(service_name.to_string())).with_filter(build_trace_filter()));
+    let metrics_layer = meter_provider
+        .as_ref()
+        .map(|p| MetricsLayer::new(p.clone()).with_filter(build_env_filter(logger_level, None)));
 
     // Optional stdout mirror (matching init_file_logging_internal logic)
     // This is separate from OTLP stdout logic. If file logging is enabled, we honor its stdout rules.
@@ -253,12 +255,11 @@ pub(super) fn init_observability_http(
     }
     let local_file_fallback_enabled = file_layer_opt.is_some();
     let stdout_mirror_enabled = stdout_guard.is_some();
-    let filter = build_env_filter(logger_level, None);
     tracing_subscriber::registry()
-        .with(filter)
+        .with(build_trace_filter())
         .with(ErrorLayer::default())
-        .with(file_layer_opt)
-        .with(stdout_layer_opt)
+        .with(file_layer_opt.map(|layer| layer.with_filter(build_env_filter(logger_level, None))))
+        .with(stdout_layer_opt.map(|layer| layer.with_filter(build_env_filter(logger_level, None))))
         .with(tracer_layer)
         .with(otel_bridge)
         .with(metrics_layer)