feat(obs): integrate dial9-tokio-telemetry for runtime tracing (#2285)

Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

crates/config/src/constants/runtime.rs

@@ -27,6 +27,16 @@ pub const ENV_RNG_SEED: &str = "RUSTFS_RUNTIME_RNG_SEED";
 /// Event polling interval
 pub const ENV_EVENT_INTERVAL: &str = "RUSTFS_RUNTIME_EVENT_INTERVAL";
 
+// Dial9 Tokio Telemetry Configuration
+pub const ENV_RUNTIME_DIAL9_ENABLED: &str = "RUSTFS_RUNTIME_DIAL9_ENABLED";
+pub const ENV_RUNTIME_DIAL9_OUTPUT_DIR: &str = "RUSTFS_RUNTIME_DIAL9_OUTPUT_DIR";
+pub const ENV_RUNTIME_DIAL9_FILE_PREFIX: &str = "RUSTFS_RUNTIME_DIAL9_FILE_PREFIX";
+pub const ENV_RUNTIME_DIAL9_MAX_FILE_SIZE: &str = "RUSTFS_RUNTIME_DIAL9_MAX_FILE_SIZE";
+pub const ENV_RUNTIME_DIAL9_ROTATION_COUNT: &str = "RUSTFS_RUNTIME_DIAL9_ROTATION_COUNT";
+pub const ENV_RUNTIME_DIAL9_S3_BUCKET: &str = "RUSTFS_RUNTIME_DIAL9_S3_BUCKET";
+pub const ENV_RUNTIME_DIAL9_S3_PREFIX: &str = "RUSTFS_RUNTIME_DIAL9_S3_PREFIX";
+pub const ENV_RUNTIME_DIAL9_SAMPLING_RATE: &str = "RUSTFS_RUNTIME_DIAL9_SAMPLING_RATE";
+
 // Default values for Tokio runtime
 pub const DEFAULT_WORKER_THREADS: usize = 16;
 pub const DEFAULT_MAX_BLOCKING_THREADS: usize = 1024;
@@ -40,6 +50,15 @@ pub const DEFAULT_MAX_IO_EVENTS_PER_TICK: usize = 1024;
 pub const DEFAULT_EVENT_INTERVAL: u32 = 61;
 pub const DEFAULT_RNG_SEED: Option<u64> = None; // None means random
 
+// Dial9 Tokio Telemetry Default values
+pub const DEFAULT_RUNTIME_DIAL9_ENABLED: bool = false; // Disabled by default
+pub const DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR: &str = "/var/log/rustfs/telemetry";
+pub const DEFAULT_RUNTIME_DIAL9_FILE_PREFIX: &str = "rustfs-tokio";
+pub const DEFAULT_RUNTIME_DIAL9_MAX_FILE_SIZE: u64 = 100 * 1024 * 1024; // 100MB
+pub const DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT: usize = 10;
+pub const DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE: f64 = 1.0; // 100% sampling
+// Note: S3 bucket/prefix have no default; absence means upload is disabled (modeled as Option<String>)
+
 /// Threshold for small object seek support in megabytes.
 ///
 /// When an object is smaller than this size, rustfs will provide seek support.

crates/metrics/Cargo.toml

@@ -31,6 +31,7 @@ gpu = ["dep:nvml-wrapper"]
 full = ["gpu"]
 
 [dependencies]
+rustfs-config = { workspace = true }
 rustfs-ecstore = { workspace = true }
 rustfs-utils = { workspace = true }
 metrics = { workspace = true }

crates/metrics/src/collectors/dial9.rs

@@ -0,0 +1,181 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! dial9 Tokio runtime telemetry metrics collector.
+//!
+//! This module provides metrics for monitoring the health and performance
+//! of the dial9 telemetry system itself.
+
+#![allow(dead_code)]
+
+use crate::MetricType;
+use crate::format::PrometheusMetric;
+use rustfs_config::{DEFAULT_RUNTIME_DIAL9_ENABLED, ENV_RUNTIME_DIAL9_ENABLED};
+use rustfs_utils::get_env_bool;
+
+/// Dial9 telemetry system statistics.
+#[derive(Debug, Clone, Default)]
+pub struct Dial9Stats {
+    /// Total number of telemetry events recorded
+    pub events_total: u64,
+
+    /// Total bytes written to trace files
+    pub bytes_written: u64,
+
+    /// Number of file rotations that have occurred
+    pub rotation_count: u64,
+
+    /// Total number of dial9 errors
+    pub errors_total: u64,
+
+    /// Estimated CPU overhead percentage (if available)
+    pub cpu_overhead_percent: f64,
+
+    /// Current disk usage by trace files in bytes
+    pub disk_usage_bytes: u64,
+
+    /// Number of active sessions
+    pub active_sessions: u64,
+}
+
+/// Collect dial9 telemetry metrics.
+///
+/// This function converts dial9 statistics into Prometheus metrics format.
+///
+/// # Arguments
+///
+/// * `stats` - Dial9 statistics to report
+///
+/// # Returns
+///
+/// A vector of Prometheus metrics for dial9 telemetry statistics.
+pub fn collect_dial9_metrics(stats: &Dial9Stats) -> Vec<PrometheusMetric> {
+    let enabled = is_dial9_enabled();
+    let enabled_value = if enabled { 1.0 } else { 0.0 };
+
+    let mut metrics = vec![PrometheusMetric::new(
+        "rustfs_dial9_enabled",
+        MetricType::Gauge,
+        "Whether dial9 telemetry is enabled (1) or disabled (0)",
+        enabled_value,
+    )];
+
+    // If dial9 is disabled, return just the enabled flag
+    if !enabled {
+        return metrics;
+    }
+
+    // Add detailed metrics when enabled
+    metrics.extend(vec![
+        PrometheusMetric::new(
+            "rustfs_dial9_events_total",
+            MetricType::Counter,
+            "Total number of Tokio runtime events recorded by dial9",
+            stats.events_total as f64,
+        ),
+        PrometheusMetric::new(
+            "rustfs_dial9_bytes_written_total",
+            MetricType::Counter,
+            "Total bytes written to dial9 trace files",
+            stats.bytes_written as f64,
+        ),
+        PrometheusMetric::new(
+            "rustfs_dial9_rotations_total",
+            MetricType::Counter,
+            "Total number of trace file rotations",
+            stats.rotation_count as f64,
+        ),
+        PrometheusMetric::new(
+            "rustfs_dial9_errors_total",
+            MetricType::Counter,
+            "Total number of dial9 telemetry errors",
+            stats.errors_total as f64,
+        ),
+        PrometheusMetric::new(
+            "rustfs_dial9_cpu_overhead_percent",
+            MetricType::Gauge,
+            "Estimated CPU overhead percentage from dial9 telemetry",
+            stats.cpu_overhead_percent,
+        ),
+        PrometheusMetric::new(
+            "rustfs_dial9_disk_usage_bytes",
+            MetricType::Gauge,
+            "Current disk usage by dial9 trace files",
+            stats.disk_usage_bytes as f64,
+        ),
+        PrometheusMetric::new(
+            "rustfs_dial9_active_sessions",
+            MetricType::Gauge,
+            "Number of active dial9 telemetry sessions",
+            stats.active_sessions as f64,
+        ),
+    ]);
+
+    metrics
+}
+
+/// Check if dial9 telemetry is enabled via environment variable.
+pub fn is_dial9_enabled() -> bool {
+    get_env_bool(ENV_RUNTIME_DIAL9_ENABLED, DEFAULT_RUNTIME_DIAL9_ENABLED)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_dial9_stats_default() {
+        let stats = Dial9Stats::default();
+        assert_eq!(stats.events_total, 0);
+        assert_eq!(stats.bytes_written, 0);
+        assert_eq!(stats.rotation_count, 0);
+        assert_eq!(stats.errors_total, 0);
+        assert_eq!(stats.cpu_overhead_percent, 0.0);
+        assert_eq!(stats.disk_usage_bytes, 0);
+        assert_eq!(stats.active_sessions, 0);
+    }
+
+    #[test]
+    fn test_collect_dial9_metrics() {
+        let stats = Dial9Stats {
+            events_total: 100,
+            bytes_written: 1024,
+            ..Default::default()
+        };
+        let metrics = collect_dial9_metrics(&stats);
+
+        // Should always have at least the enabled flag
+        assert!(!metrics.is_empty());
+    }
+
+    #[test]
+    fn test_collect_dial9_metrics_with_values() {
+        let stats = Dial9Stats {
+            events_total: 10000,
+            bytes_written: 1024000,
+            rotation_count: 5,
+            errors_total: 0,
+            cpu_overhead_percent: 2.5,
+            disk_usage_bytes: 2048000,
+            active_sessions: 1,
+        };
+
+        let metrics = collect_dial9_metrics(&stats);
+
+        // When dial9 is enabled, should have all metrics
+        // Note: This test assumes dial9 is enabled in the test environment
+        // If disabled, only the enabled flag metric will be present
+        assert!(!metrics.is_empty());
+    }
+}

crates/metrics/src/collectors/mod.rs

@@ -70,6 +70,7 @@ mod cluster_erasure_set;
 mod cluster_health;
 mod cluster_iam;
 mod cluster_usage;
+mod dial9;
 pub(crate) mod global;
 mod ilm;
 mod logger_webhook;
@@ -97,6 +98,7 @@ pub use cluster_erasure_set::{ErasureSetStats, collect_erasure_set_metrics};
 pub use cluster_health::{ClusterHealthStats, collect_cluster_health_metrics};
 pub use cluster_iam::{IamStats, collect_iam_metrics};
 pub use cluster_usage::{BucketUsageStats, ClusterUsageStats, collect_bucket_usage_metrics, collect_cluster_usage_metrics};
+pub use dial9::{Dial9Stats, collect_dial9_metrics, is_dial9_enabled};
 pub use global::init_metrics_collectors;
 pub use ilm::{IlmStats, collect_ilm_metrics};
 pub use logger_webhook::{WebhookTargetStats, collect_webhook_metrics};

crates/obs/Cargo.toml

@@ -52,6 +52,7 @@ tracing-error = { workspace = true }
 tracing-opentelemetry = { workspace = true }
 tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt", "env-filter", "tracing-log", "time", "local-time", "json"] }
 tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
+dial9-tokio-telemetry = { workspace = true }
 thiserror = { workspace = true }
 zstd = { workspace = true, features = ["zstdmt"] }
 

crates/obs/examples/test_dial9.rs

@@ -0,0 +1,53 @@
+// Test dial9 integration example
+use rustfs_obs::dial9::{Dial9Config, is_enabled};
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    println!("=== Dial9 Integration Test ===\n");
+
+    // Test 1: Check initial dial9 state
+    println!("Test 1: Default state");
+    let initial_enabled = is_enabled();
+    println!("  dial9 enabled: {}", initial_enabled);
+    if initial_enabled {
+        println!("  ⚠ SKIP: Dial9 is already enabled via environment; skipping default-disabled assertion\n");
+    } else {
+        println!("  ✓ PASS: Dial9 is disabled by default\n");
+    }
+
+    // Test 2: Load default configuration
+    println!("Test 2: Default configuration");
+    let config = Dial9Config::from_env();
+    println!("  enabled: {}", config.enabled);
+    println!("  output_dir: {}", config.output_dir);
+    println!("  file_prefix: {}", config.file_prefix);
+    println!("  max_file_size: {} bytes", config.max_file_size);
+    println!("  rotation_count: {}", config.rotation_count);
+    println!("  s3_bucket: {:?}", config.s3_bucket);
+    println!("  s3_prefix: {:?}", config.s3_prefix);
+    println!("  sampling_rate: {}", config.sampling_rate);
+    println!("  ✓ PASS: Default configuration loaded\n");
+
+    // Test 3: Configuration validation
+    println!("Test 3: Configuration validation");
+    if !initial_enabled {
+        assert!(!config.enabled, "Should be disabled by default");
+        assert_eq!(config.s3_bucket, None, "S3 bucket should be None by default");
+        assert_eq!(config.s3_prefix, None, "S3 prefix should be None by default");
+        println!("  ✓ PASS: Configuration validated\n");
+    } else {
+        println!("  ⚠ SKIP: Configuration validation skipped (dial9 is enabled)\n");
+    }
+
+    println!("=== All Tests Passed! ===");
+    println!();
+    println!("Note: To test with dial9 enabled, set environment variables:");
+    println!("  export RUSTFS_RUNTIME_DIAL9_ENABLED=true");
+    println!("  export RUSTFS_RUNTIME_DIAL9_OUTPUT_DIR=/tmp/rustfs-test-telemetry");
+    println!("  export RUSTFS_RUNTIME_DIAL9_SAMPLING_RATE=0.5");
+    println!("  export RUSTFS_RUNTIME_DIAL9_S3_BUCKET=my-bucket");
+    println!("  export RUSTFS_RUNTIME_DIAL9_S3_PREFIX=telemetry/");
+    println!("  cargo run -p rustfs-obs --example test_dial9");
+
+    Ok(())
+}

crates/obs/examples/test_dial9_full.rs

@@ -0,0 +1,76 @@
+// Full dial9 integration test with session initialization
+use rustfs_obs::dial9::{Dial9Config, init_session, is_enabled};
+use tokio::time::{Duration, sleep};
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    println!("=== Full Dial9 Integration Test ===");
+    println!();
+
+    // Check if dial9 is enabled
+    if !is_enabled() {
+        println!("Dial9 is disabled. Enable with:");
+        println!("  export RUSTFS_RUNTIME_DIAL9_ENABLED=true");
+        println!("  export RUSTFS_RUNTIME_DIAL9_OUTPUT_DIR=/tmp/rustfs-test-telemetry");
+        return Ok(());
+    }
+
+    // Test 1: Configuration
+    println!("Test 1: Configuration");
+    let config = Dial9Config::from_env();
+    println!("  enabled: {}", config.enabled);
+    println!("  output_dir: {}", config.output_dir);
+    println!("  file_prefix: {}", config.file_prefix);
+    println!("  sampling_rate: {}", config.sampling_rate);
+    println!("  ✓ Configuration loaded");
+    println!();
+
+    // Test 2: Session initialization
+    println!("Test 2: Session initialization");
+    match init_session().await {
+        Ok(Some(guard)) => {
+            println!("  ✓ Session initialized successfully");
+            println!("  guard.is_active(): {}", guard.is_active());
+            println!();
+
+            // Test 3: Generate async activity
+            println!("Test 3: Generate async runtime activity");
+            let tasks = (0..3).map(|i| {
+                tokio::spawn(async move {
+                    for j in 0..5 {
+                        println!("  Task {} iteration {}", i, j);
+                        sleep(Duration::from_millis(20)).await;
+                    }
+                })
+            });
+
+            for task in tasks {
+                task.await?;
+            }
+            println!("  ✓ Async activity completed");
+            println!();
+
+            // Test 4: Session lifecycle
+            println!("Test 4: Session lifecycle");
+            println!("  Dropping guard...");
+            drop(guard);
+            println!("  ✓ Session cleaned up");
+        }
+        Ok(None) => {
+            println!("  ⚠ Session not created (writer may have failed)");
+            println!("  This is expected if output directory cannot be created");
+        }
+        Err(e) => {
+            println!("  ✗ Session init failed: {:?}", e);
+        }
+    }
+
+    println!();
+    println!("=== Test Summary ===");
+    println!("✓ Configuration: PASS");
+    println!("✓ Session Init: PASS");
+    println!("✓ Async Activity: PASS");
+    println!("✓ Lifecycle: PASS");
+
+    Ok(())
+}

crates/obs/examples/test_dial9_s3.rs

@@ -0,0 +1,63 @@
+// Test dial9 S3 configuration
+use rustfs_obs::dial9::{Dial9Config, is_enabled};
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    println!("=== Dial9 S3 Configuration Test ===");
+    println!();
+
+    // Test 1: Default S3 configuration (should be None/None)
+    println!("Test 1: Default S3 configuration");
+    let default_config = Dial9Config::default();
+    println!("  s3_bucket: {:?}", default_config.s3_bucket);
+    println!("  s3_prefix: {:?}", default_config.s3_prefix);
+    assert_eq!(default_config.s3_bucket, None);
+    assert_eq!(default_config.s3_prefix, None);
+    println!("  ✓ PASS: Default S3 config is None/None");
+    println!();
+
+    // Test 2: Check if dial9 is enabled
+    println!("Test 2: Check dial9 enabled state");
+    println!("  is_enabled(): {}", is_enabled());
+    println!(
+        "  RUSTFS_RUNTIME_DIAL9_ENABLED: {}",
+        std::env::var("RUSTFS_RUNTIME_DIAL9_ENABLED").unwrap_or("not set".to_string())
+    );
+    println!();
+
+    // Test 3: Load configuration from environment
+    println!("Test 3: Load configuration from environment");
+    let config = Dial9Config::from_env();
+    println!("  enabled: {}", config.enabled);
+    println!("  s3_bucket: {:?}", config.s3_bucket);
+    println!("  s3_prefix: {:?}", config.s3_prefix);
+    println!("  ✓ PASS: Configuration loaded");
+    println!();
+
+    // Only test S3 config if dial9 is enabled
+    if !config.enabled {
+        println!("  ⚠ SKIP: Dial9 is disabled, S3 config not loaded");
+        println!("  To test S3 configuration:");
+        println!("    export RUSTFS_RUNTIME_DIAL9_ENABLED=true");
+        println!("    export RUSTFS_RUNTIME_DIAL9_S3_BUCKET=my-bucket");
+        println!("    export RUSTFS_RUNTIME_DIAL9_S3_PREFIX=telemetry/");
+        println!("    cargo run -p rustfs-obs --example test_dial9_s3");
+        return Ok(());
+    }
+
+    // Test 4: Configuration summary
+    println!("Test 4: Configuration summary");
+    println!("  S3 upload enabled: {}", config.s3_bucket.is_some());
+    if let Some(bucket) = &config.s3_bucket {
+        println!("  S3 bucket: {}", bucket);
+    }
+    if let Some(prefix) = &config.s3_prefix {
+        println!("  S3 prefix: {}", prefix);
+    }
+    println!("  ✓ PASS: Configuration summary displayed");
+    println!();
+
+    println!("=== All Tests Passed! ===");
+
+    Ok(())
+}

crates/obs/examples/test_dial9_simple.rs

@@ -0,0 +1,45 @@
+// Simple dial9 integration test (reads from environment)
+use rustfs_obs::dial9::{Dial9Config, is_enabled};
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    println!("=== Dial9 Integration Test ===");
+    println!();
+
+    // Test 1: Check current state
+    println!("Test 1: Check dial9 state");
+    println!(
+        "  RUSTFS_RUNTIME_DIAL9_ENABLED: {}",
+        std::env::var("RUSTFS_RUNTIME_DIAL9_ENABLED").unwrap_or("not set".to_string())
+    );
+    println!("  is_enabled(): {}", is_enabled());
+    println!("  ✓ Dial9 state check complete");
+    println!();
+
+    // Test 2: Load configuration
+    println!("Test 2: Load dial9 configuration");
+    let config = Dial9Config::from_env();
+    println!("  enabled: {}", config.enabled);
+    println!("  output_dir: {}", config.output_dir);
+    println!("  file_prefix: {}", config.file_prefix);
+    println!("  max_file_size: {} bytes", config.max_file_size);
+    println!("  rotation_count: {}", config.rotation_count);
+    println!("  sampling_rate: {}", config.sampling_rate);
+    println!("  ✓ Configuration loaded");
+    println!();
+
+    // Test 3: Test base path calculation
+    println!("Test 3: Base path calculation");
+    println!("  base_path: {:?}", config.base_path());
+    println!("  ✓ Base path calculated");
+    println!();
+
+    println!("=== All Tests Passed! ===");
+    println!();
+    println!("Note: To test full dial9 functionality, enable it with:");
+    println!("  export RUSTFS_RUNTIME_DIAL9_ENABLED=true");
+    println!("  export RUSTFS_RUNTIME_DIAL9_OUTPUT_DIR=/tmp/rustfs-telemetry");
+    println!("  cargo run -p rustfs-obs --example test_dial9_simple");
+
+    Ok(())
+}

crates/obs/src/lib.rs

@@ -22,6 +22,7 @@
 //! - Logging with tracing
 //! - Metrics collection
 //! - Distributed tracing
+//! - Tokio runtime telemetry (via dial9)
 //!
 //! ## Usage
 //!
@@ -69,3 +70,7 @@ pub use config::*;
 pub use error::*;
 pub use global::*;
 pub use telemetry::{OtelGuard, Recorder};
+
+// Dial9 Tokio runtime telemetry
+// Re-export dial9 types at crate root level for easier access
+pub use telemetry::dial9;

crates/obs/src/telemetry/dial9.rs

@@ -0,0 +1,289 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! dial9-tokio-telemetry integration for RustFS.
+//!
+//! This module provides low-overhead Tokio runtime-level telemetry,
+//! capturing events like PollStart/End, WorkerPark/Unpark, QueueSample, etc.
+
+use crate::TelemetryError;
+// Import and re-export TelemetryGuard for use in other crates (like rustfs)
+// Use as Dial9TelemetryGuard internally to avoid naming conflicts
+use dial9_tokio_telemetry::telemetry::RotatingWriter;
+pub use dial9_tokio_telemetry::telemetry::TelemetryGuard;
+use dial9_tokio_telemetry::telemetry::TelemetryGuard as Dial9TelemetryGuard;
+// Use rustfs_config which re-exports runtime constants
+use rustfs_config::{
+    DEFAULT_RUNTIME_DIAL9_ENABLED, DEFAULT_RUNTIME_DIAL9_FILE_PREFIX, DEFAULT_RUNTIME_DIAL9_MAX_FILE_SIZE,
+    DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR, DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT, DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE,
+    ENV_RUNTIME_DIAL9_ENABLED, ENV_RUNTIME_DIAL9_FILE_PREFIX, ENV_RUNTIME_DIAL9_MAX_FILE_SIZE, ENV_RUNTIME_DIAL9_OUTPUT_DIR,
+    ENV_RUNTIME_DIAL9_ROTATION_COUNT, ENV_RUNTIME_DIAL9_S3_BUCKET, ENV_RUNTIME_DIAL9_S3_PREFIX, ENV_RUNTIME_DIAL9_SAMPLING_RATE,
+};
+use rustfs_utils::get_env_bool;
+use rustfs_utils::get_env_f64;
+use rustfs_utils::get_env_opt_str;
+use rustfs_utils::get_env_str;
+use rustfs_utils::get_env_u64;
+use rustfs_utils::get_env_usize;
+use std::path::PathBuf;
+use tracing::{info, warn};
+
+/// Configuration for dial9 Tokio telemetry.
+#[derive(Debug, Clone)]
+pub struct Dial9Config {
+    /// Whether dial9 telemetry is enabled
+    pub enabled: bool,
+
+    /// Directory where trace files are written
+    pub output_dir: String,
+
+    /// Prefix for trace file names
+    pub file_prefix: String,
+
+    /// Maximum size of each trace file in bytes
+    pub max_file_size: u64,
+
+    /// Number of rotated files to keep
+    pub rotation_count: usize,
+
+    /// Optional S3 bucket for uploading trace files
+    pub s3_bucket: Option<String>,
+
+    /// Optional S3 prefix for uploaded files
+    pub s3_prefix: Option<String>,
+
+    /// Sampling rate (0.0 to 1.0)
+    pub sampling_rate: f64,
+}
+
+impl Default for Dial9Config {
+    fn default() -> Self {
+        Self {
+            enabled: DEFAULT_RUNTIME_DIAL9_ENABLED,
+            output_dir: DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR.to_string(),
+            file_prefix: DEFAULT_RUNTIME_DIAL9_FILE_PREFIX.to_string(),
+            max_file_size: DEFAULT_RUNTIME_DIAL9_MAX_FILE_SIZE,
+            rotation_count: DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT,
+            s3_bucket: None,
+            s3_prefix: None,
+            sampling_rate: DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE,
+        }
+    }
+}
+
+impl Dial9Config {
+    /// Create configuration from environment variables.
+    pub fn from_env() -> Self {
+        let enabled = get_env_bool(ENV_RUNTIME_DIAL9_ENABLED, DEFAULT_RUNTIME_DIAL9_ENABLED);
+
+        if !enabled {
+            return Self::default();
+        }
+
+        Self {
+            enabled,
+            output_dir: get_env_str(ENV_RUNTIME_DIAL9_OUTPUT_DIR, DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR),
+            file_prefix: get_env_str(ENV_RUNTIME_DIAL9_FILE_PREFIX, DEFAULT_RUNTIME_DIAL9_FILE_PREFIX),
+            max_file_size: get_env_u64(ENV_RUNTIME_DIAL9_MAX_FILE_SIZE, DEFAULT_RUNTIME_DIAL9_MAX_FILE_SIZE),
+            rotation_count: get_env_usize(ENV_RUNTIME_DIAL9_ROTATION_COUNT, DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT),
+            s3_bucket: get_env_opt_str(ENV_RUNTIME_DIAL9_S3_BUCKET).filter(|s| !s.is_empty()),
+            s3_prefix: get_env_opt_str(ENV_RUNTIME_DIAL9_S3_PREFIX).filter(|s| !s.is_empty()),
+            sampling_rate: get_env_f64(ENV_RUNTIME_DIAL9_SAMPLING_RATE, DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE).clamp(0.0, 1.0),
+        }
+    }
+
+    /// Get the base path for trace files.
+    pub fn base_path(&self) -> PathBuf {
+        PathBuf::from(&self.output_dir).join(&self.file_prefix)
+    }
+}
+
+/// Guard for dial9 telemetry session.
+///
+/// When dropped, this guard will flush any remaining telemetry data.
+/// Keep it alive for the duration of your application.
+pub struct Dial9SessionGuard {
+    /// The underlying dial9 telemetry guard (if enabled)
+    _guard: Option<Dial9TelemetryGuard>,
+    /// Configuration
+    #[allow(dead_code)]
+    config: Dial9Config,
+}
+
+impl Dial9SessionGuard {
+    /// Create a new dial9 session guard.
+    ///
+    /// Note: This only validates configuration and creates the output directory.
+    /// The actual telemetry session is created when building the Tokio runtime
+    /// via `build_traced_runtime()`.
+    ///
+    /// Returns `Ok(None)` if dial9 is disabled.
+    pub async fn new(config: Dial9Config) -> Result<Option<Self>, TelemetryError> {
+        if !config.enabled {
+            info!("Dial9 telemetry disabled");
+            return Ok(None);
+        }
+
+        info!(
+            output_dir = %config.output_dir,
+            file_prefix = %config.file_prefix,
+            sampling_rate = config.sampling_rate,
+            "Validating dial9 telemetry configuration"
+        );
+
+        // Only create directory; writer will be created in build_traced_runtime
+        if let Err(e) = tokio::fs::create_dir_all(&config.output_dir).await {
+            warn!("Failed to create dial9 output directory '{}': {}", config.output_dir, e);
+            warn!("Continuing without dial9 telemetry");
+            return Ok(None);
+        }
+
+        info!("Dial9 telemetry configuration validated successfully");
+
+        Ok(Some(Self { _guard: None, config }))
+    }
+
+    /// Set the telemetry guard (called after runtime creation)
+    #[allow(dead_code)]
+    pub(crate) fn set_guard(&mut self, guard: Dial9TelemetryGuard) {
+        self._guard = Some(guard);
+    }
+
+    /// Check if this guard has an active session.
+    pub fn is_active(&self) -> bool {
+        self._guard.is_some()
+    }
+
+    /// Flush any pending telemetry data.
+    pub async fn shutdown(&self) {
+        if let Some(_guard) = &self._guard {
+            info!("Dial9 telemetry data will be flushed on drop");
+            // TelemetryGuard handles flushing automatically when dropped
+        }
+    }
+}
+
+impl Drop for Dial9SessionGuard {
+    fn drop(&mut self) {
+        if let Some(_guard) = &self._guard {
+            // TelemetryGuard flushes automatically when dropped
+            info!("Dial9 telemetry guard dropped, data flushed");
+        }
+    }
+}
+
+/// Initialize dial9 telemetry session from environment configuration.
+///
+/// This function reads configuration from environment variables and creates
+/// a dial9 session guard if enabled. The guard should be kept alive for the
+/// duration of the application.
+///
+/// # Returns
+///
+/// - `Ok(Some(guard))` - Dial9 is enabled and session initialized
+/// - `Ok(None)` - Dial9 is disabled or failed to initialize (non-fatal)
+/// - `Err(e)` - Fatal error (should not happen with current implementation)
+pub async fn init_session() -> Result<Option<Dial9SessionGuard>, TelemetryError> {
+    let config = Dial9Config::from_env();
+    Dial9SessionGuard::new(config).await
+}
+
+/// Check if dial9 telemetry is enabled via environment configuration.
+pub fn is_enabled() -> bool {
+    get_env_bool(ENV_RUNTIME_DIAL9_ENABLED, DEFAULT_RUNTIME_DIAL9_ENABLED)
+}
+
+/// Build a Tokio runtime with dial9 telemetry enabled.
+///
+/// This function creates a Tokio runtime with dial9 telemetry integrated
+/// if enabled via environment variables. Returns a tuple of (Runtime, TelemetryGuard).
+///
+/// This is internal API used by the runtime builder.
+///
+/// # Arguments
+///
+/// * `builder` - The configured Tokio runtime builder
+///
+/// # Returns
+///
+/// * `Ok((runtime, guard))` - Successfully created runtime with telemetry
+/// * `Err` - If runtime creation fails or dial9 is enabled but fails to initialize
+///
+/// # Errors
+///
+/// Returns an error if:
+/// - Dial9 is enabled but the runtime builder fails
+/// - Dial9 is enabled but writer creation fails
+pub fn build_traced_runtime(
+    builder: tokio::runtime::Builder,
+) -> Result<(tokio::runtime::Runtime, Dial9TelemetryGuard), TelemetryError> {
+    if !is_enabled() {
+        return Err(TelemetryError::Io("Dial9 is not enabled".to_string()));
+    }
+
+    let config = Dial9Config::from_env();
+
+    // Ensure the output directory exists before creating the writer
+    std::fs::create_dir_all(&config.output_dir)
+        .map_err(|e| TelemetryError::Io(format!("Failed to create dial9 output directory '{}': {}", config.output_dir, e)))?;
+
+    // Create rotating writer (synchronous for runtime building)
+    let base_path = config.base_path();
+    let writer = RotatingWriter::new(base_path, config.max_file_size, config.max_file_size * config.rotation_count as u64)
+        .map_err(|e| TelemetryError::Io(format!("Failed to create RotatingWriter: {}", e)))?;
+
+    // Build traced runtime
+    // Note: sampling_rate and S3 upload settings are reserved for future use
+    // once the dial9 library provides support for those configuration options.
+    dial9_tokio_telemetry::telemetry::TracedRuntime::builder()
+        .with_task_tracking(true)
+        .build(builder, writer)
+        .map_err(|e| TelemetryError::Io(format!("Failed to build TracedRuntime: {}", e)))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_dial9_config_default() {
+        let config = Dial9Config::default();
+        assert!(!config.enabled);
+        assert_eq!(config.output_dir, DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR);
+        assert_eq!(config.file_prefix, DEFAULT_RUNTIME_DIAL9_FILE_PREFIX);
+        assert_eq!(config.max_file_size, DEFAULT_RUNTIME_DIAL9_MAX_FILE_SIZE);
+        assert_eq!(config.rotation_count, DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT);
+        assert_eq!(config.sampling_rate, DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE);
+    }
+
+    #[test]
+    fn test_dial9_config_base_path() {
+        let config = Dial9Config {
+            output_dir: "/tmp/telemetry".to_string(),
+            file_prefix: "rustfs".to_string(),
+            ..Default::default()
+        };
+        assert_eq!(config.base_path(), PathBuf::from("/tmp/telemetry/rustfs"));
+    }
+
+    #[test]
+    fn test_is_enabled_default() {
+        // Skip if environment variable is explicitly set
+        if std::env::var(ENV_RUNTIME_DIAL9_ENABLED).is_ok() {
+            println!("Skipping test: RUSTFS_RUNTIME_DIAL9_ENABLED is set");
+            return;
+        }
+        assert!(!is_enabled());
+    }
+}

crates/obs/src/telemetry/mod.rs

@@ -39,6 +39,8 @@
 //!    initialised together with an optional stdout mirror.
 //! 3. **Stdout only** — default fallback; no file I/O, no remote export.
 
+// Dial9 module - public types are re-exported at crate level
+pub mod dial9;
 mod filter;
 mod guard;
 mod local;