feat(obs): integrate dial9-tokio-telemetry for runtime tracing (#2285)

Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

rustfs/src/main.rs

@@ -107,9 +107,8 @@ fn main() {
         eprintln!("[WARN] Failed to bootstrap external-prefix compatibility: {err}");
     }
 
-    let runtime = server::tokio_runtime_builder()
-        .build()
-        .expect("Failed to build Tokio runtime");
+    // Build Tokio runtime with optional dial9 telemetry support
+    let runtime = server::build_tokio_runtime().expect("Failed to build Tokio runtime");
     let result = runtime.block_on(async_main());
     if let Err(ref e) = result {
         // Use eprintln as tracing may not be initialized at this point
@@ -202,6 +201,15 @@ async fn async_main() -> Result<()> {
         }
     }
 
+    // Check dial9 Tokio runtime telemetry status
+    // Note: The actual telemetry session is created in build_tokio_runtime()
+    // which stores the TelemetryGuard globally for the program duration.
+    if rustfs_obs::dial9::is_enabled() {
+        info!(target: "rustfs::main", "Dial9 Tokio telemetry is configured as enabled; runtime guard was installed during startup.");
+    } else {
+        info!(target: "rustfs::main", "Dial9 Tokio telemetry is not configured (set RUSTFS_RUNTIME_DIAL9_ENABLED=true to enable).");
+    }
+
     info!("license status: {}", license_status());
     if let Some(token) = current_license() {
         info!("runtime license loaded: {}", token.name);

rustfs/src/server/mod.rs

@@ -31,7 +31,7 @@ pub(crate) use event::{init_event_notifier, shutdown_event_notifier};
 pub(crate) use http::start_http_server;
 pub(crate) use prefix::*;
 pub(crate) use readiness::ReadinessGateLayer;
-pub(crate) use runtime::tokio_runtime_builder;
+pub(crate) use runtime::build_tokio_runtime;
 pub(crate) use service_state::SHUTDOWN_TIMEOUT;
 pub(crate) use service_state::ServiceState;
 pub(crate) use service_state::ServiceStateManager;

rustfs/src/server/runtime.rs

@@ -12,9 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::OnceLock;
 use std::time::Duration;
 use sysinfo::{RefreshKind, System};
 
+// Import TelemetryGuard from rustfs_obs re-export
+use rustfs_obs::dial9::TelemetryGuard;
+
+// Global storage for TelemetryGuard to keep it alive for the program duration
+static DIAL9_TELEMETRY_GUARD: OnceLock<TelemetryGuard> = OnceLock::new();
+
 #[inline]
 fn compute_default_thread_stack_size() -> usize {
     // Baseline: Release 1 MiB，Debug 2 MiB；macOS at least 2 MiB
@@ -80,9 +87,9 @@ fn compute_default_max_blocking_threads() -> usize {
 /// Panics if environment variable values are invalid
 /// # Examples
 /// ```no_run
-/// use rustfs_server::tokio_runtime_builder;
-/// let builder = tokio_runtime_builder();
-/// let runtime = builder.build().unwrap();
+/// // tokio_runtime_builder is pub(crate) - call it from within the rustfs binary:
+/// // let builder = tokio_runtime_builder();
+/// // let runtime = builder.build().unwrap();
 /// ```
 pub(crate) fn tokio_runtime_builder() -> tokio::runtime::Builder {
     let mut builder = tokio::runtime::Builder::new_multi_thread();
@@ -158,3 +165,78 @@ pub(crate) fn tokio_runtime_builder() -> tokio::runtime::Builder {
 fn print_tokio_thread_enable() -> bool {
     rustfs_utils::get_env_bool(rustfs_config::ENV_THREAD_PRINT_ENABLED, rustfs_config::DEFAULT_THREAD_PRINT_ENABLED)
 }
+
+/// Build Tokio runtime with optional dial9 telemetry support.
+///
+/// If dial9 is enabled via environment variables, creates a TracedRuntime
+/// and stores the TelemetryGuard globally to keep it alive for the
+/// duration of the program.
+///
+/// # Returns
+///
+/// * `Ok(runtime)` - Successfully created runtime
+/// * `Err(e)` - Failed to create runtime
+///
+/// # Errors
+///
+/// Returns an error if:
+/// - The Tokio runtime builder fails
+/// - Dial9 is enabled but fails to initialize (falls back to standard runtime)
+///
+/// # Examples
+///
+/// ```no_run
+/// // build_tokio_runtime is pub(crate) - call it from within the rustfs binary:
+/// // let runtime = build_tokio_runtime().expect("Failed to build runtime");
+/// // runtime.block_on(async { /* ... */ })
+/// ```
+pub(crate) fn build_tokio_runtime() -> Result<tokio::runtime::Runtime, BuildError> {
+    let mut builder = tokio_runtime_builder();
+
+    // Check if dial9 is enabled
+    if rustfs_obs::dial9::is_enabled() {
+        tracing::info!("Dial9 telemetry enabled, building TracedRuntime");
+
+        return match rustfs_obs::dial9::build_traced_runtime(builder) {
+            Ok((runtime, guard)) => {
+                // Store guard in global static to keep it alive for the program duration
+                let _ = DIAL9_TELEMETRY_GUARD.set(guard);
+                tracing::info!("TracedRuntime created successfully, guard stored globally");
+                Ok(runtime)
+            }
+            Err(e) => {
+                tracing::warn!("Failed to build TracedRuntime: {}", e);
+                tracing::warn!("Falling back to standard Tokio runtime");
+                // Rebuild the builder for standard runtime
+                let mut builder = tokio_runtime_builder();
+                builder.build().map_err(BuildError::Runtime)
+            }
+        };
+    }
+
+    // Standard runtime
+    builder.build().map_err(BuildError::Runtime)
+}
+
+/// Error type for runtime building failures.
+#[derive(Debug)]
+pub enum BuildError {
+    /// Tokio runtime creation failed
+    Runtime(std::io::Error),
+}
+
+impl std::fmt::Display for BuildError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            BuildError::Runtime(e) => write!(f, "Failed to build Tokio runtime: {}", e),
+        }
+    }
+}
+
+impl std::error::Error for BuildError {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        match self {
+            BuildError::Runtime(e) => Some(e),
+        }
+    }
+}