diff --git a/.cargo/config.toml b/.cargo/config.toml index 32993a64d..ee48b9247 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -14,13 +14,20 @@ # RustFS Cargo configuration -# Enable tokio_unstable cfg for dial9-tokio-telemetry support -# This allows dial9 to hook into Tokio's internal runtime events -[build] -# Enable Tokio unstable features required by dial9-tokio-telemetry for runtime tracing. -# See: https://docs.rs/tokio/latest/tokio/#unstable-features -rustflags = ["--cfg", "tokio_unstable"] - -# Enable frame pointers for CPU profiling (Linux only, optional but recommended) -# Uncomment the following line for better CPU profiling data -# rustflags = ["--cfg", "tokio_unstable", "-C", "force-frame-pointers=yes"] +# NOTE: `--cfg tokio_unstable` is deliberately NOT set here. +# +# It used to be a global `[build] rustflags` entry so that the (default-off) +# `dial9` telemetry feature could compile. That made every build depend on +# Tokio's unstable, non-semver API, and it broke silently whenever a caller +# exported their own RUSTFLAGS — an environment RUSTFLAGS replaces the value +# from this file rather than appending to it. +# +# The flag is now scoped to telemetry builds, which opt in explicitly: +# +# make build-profiling +# RUSTFLAGS="--cfg tokio_unstable" cargo build --features dial9 +# +# `crates/obs/build.rs` fails the compile if the `dial9` feature is on without +# the flag, so the two can no longer drift apart unnoticed. +# +# For CPU profiling, add `-C force-frame-pointers=yes` to that RUSTFLAGS value. diff --git a/.config/make/build.mak b/.config/make/build.mak index a38016144..8c0638d2f 100644 --- a/.config/make/build.mak +++ b/.config/make/build.mak @@ -35,6 +35,26 @@ build-gnu-arm64: ## Build aarch64 GNU version ./build-rustfs.sh --platform aarch64-unknown-linux-gnu +## —— Profiling build (dial9 Tokio runtime telemetry) ------------------------------------------ + +# dial9 hooks Tokio's unstable runtime instrumentation, so it needs +# `--cfg tokio_unstable`. That flag is deliberately absent from +# .cargo/config.toml: it is not free, and release binaries do not carry it. +# Setting RUSTFLAGS here replaces (never appends to) the config-file value, and +# crates/obs/build.rs fails the build if the feature and the flag disagree. +# +# DIAL9_FEATURES can add `dial9-taskdump` (async backtraces of stalled tasks; +# Linux x86_64/aarch64, and also needs --cfg tokio_taskdump). There is no S3 +# upload feature — see the note in crates/obs/Cargo.toml. +DIAL9_FEATURES ?= dial9 +DIAL9_RUSTFLAGS ?= --cfg tokio_unstable + +.PHONY: build-profiling +build-profiling: ## Build RustFS with dial9 Tokio runtime telemetry (diagnostic builds only) + @echo "🔬 Building RustFS with dial9 telemetry (features: $(DIAL9_FEATURES))..." + @echo "⚠️ Diagnostic build: telemetry writes trace segments to disk continuously." + RUSTFLAGS="$(DIAL9_RUSTFLAGS)" cargo build --release --bin rustfs --features $(DIAL9_FEATURES) + .PHONY: build-cross-all build-cross-all: core-deps ## Build binaries for all architectures @echo "🔧 Building all target architectures..." diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8008c4cfd..df867fd09 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -224,11 +224,11 @@ jobs: timeout-minutes: 90 env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" - # Always enable Tokio unstable features (required by dial9-tokio-telemetry). - # The RUSTFLAGS env var takes precedence over .cargo/config.toml [build] rustflags, - # so we must include --cfg tokio_unstable here explicitly; otherwise an empty - # RUSTFLAGS value would shadow the config-file flag and silently break tracing. - RUSTFLAGS: "--cfg tokio_unstable ${{ matrix.rustflags }}" + # Release binaries ship without dial9 telemetry and therefore do not need + # --cfg tokio_unstable. Telemetry builds opt in explicitly (see + # `make build-profiling`); crates/obs/build.rs fails the compile if the + # `dial9` feature is enabled without the flag. + RUSTFLAGS: "${{ matrix.rustflags }}" strategy: fail-fast: false matrix: ${{ fromJson(needs.prepare-platform-matrix.outputs.matrix) }} diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index e22e8e830..7b1acef4b 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -48,7 +48,8 @@ concurrency: env: CARGO_TERM_COLOR: always CARGO_BUILD_TARGET: x86_64-unknown-linux-gnu - RUSTFLAGS: "--cfg tokio_unstable -C target-feature=-crt-static" + # Fuzz targets build without the dial9 feature, so tokio_unstable is not needed. + RUSTFLAGS: "-C target-feature=-crt-static" jobs: cancel-closed-pr-runs: diff --git a/Cargo.lock b/Cargo.lock index 693502e8e..62ea8cb74 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10031,7 +10031,7 @@ dependencies = [ [[package]] name = "rustfs-uring" version = "0.1.0" -source = "git+https://github.com/rustfs/uring?rev=f577ba0bfd6c3d74f9bdc54b573b8fcb45d6bc22#f577ba0bfd6c3d74f9bdc54b573b8fcb45d6bc22" +source = "git+https://github.com/rustfs/uring?rev=719b245f9bb24e838de8639c19858df2690f3ae1#719b245f9bb24e838de8639c19858df2690f3ae1" dependencies = [ "io-uring", "libc", @@ -11616,6 +11616,7 @@ version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ + "backtrace", "bytes", "libc", "mio", diff --git a/crates/config/src/constants/runtime.rs b/crates/config/src/constants/runtime.rs index 1a50bd146..287b86394 100644 --- a/crates/config/src/constants/runtime.rs +++ b/crates/config/src/constants/runtime.rs @@ -33,9 +33,15 @@ pub const ENV_RUNTIME_DIAL9_OUTPUT_DIR: &str = "RUSTFS_RUNTIME_DIAL9_OUTPUT_DIR" pub const ENV_RUNTIME_DIAL9_FILE_PREFIX: &str = "RUSTFS_RUNTIME_DIAL9_FILE_PREFIX"; pub const ENV_RUNTIME_DIAL9_MAX_FILE_SIZE: &str = "RUSTFS_RUNTIME_DIAL9_MAX_FILE_SIZE"; pub const ENV_RUNTIME_DIAL9_ROTATION_COUNT: &str = "RUSTFS_RUNTIME_DIAL9_ROTATION_COUNT"; +/// Accepted but not honoured: dial9's S3 uploader depends on a vulnerable +/// rustls-webpki. Setting this logs a warning. See rustfs/backlog#1157. pub const ENV_RUNTIME_DIAL9_S3_BUCKET: &str = "RUSTFS_RUNTIME_DIAL9_S3_BUCKET"; +/// Accepted but not honoured; see [`ENV_RUNTIME_DIAL9_S3_BUCKET`]. pub const ENV_RUNTIME_DIAL9_S3_PREFIX: &str = "RUSTFS_RUNTIME_DIAL9_S3_PREFIX"; -pub const ENV_RUNTIME_DIAL9_SAMPLING_RATE: &str = "RUSTFS_RUNTIME_DIAL9_SAMPLING_RATE"; +/// Capture async backtraces for tasks that stall on a worker. +pub const ENV_RUNTIME_DIAL9_TASK_DUMP_ENABLED: &str = "RUSTFS_RUNTIME_DIAL9_TASK_DUMP_ENABLED"; +/// Mean idle duration for task-dump Poisson sampling, in milliseconds. +pub const ENV_RUNTIME_DIAL9_TASK_DUMP_IDLE_THRESHOLD_MS: &str = "RUSTFS_RUNTIME_DIAL9_TASK_DUMP_IDLE_THRESHOLD_MS"; // Default values for Tokio runtime pub const DEFAULT_WORKER_THREADS: usize = 16; @@ -56,7 +62,9 @@ pub const DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR: &str = "/var/log/rustfs/telemetry"; pub const DEFAULT_RUNTIME_DIAL9_FILE_PREFIX: &str = "rustfs-tokio"; pub const DEFAULT_RUNTIME_DIAL9_MAX_FILE_SIZE: u64 = 100 * 1024 * 1024; // 100MB pub const DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT: usize = 10; -pub const DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE: f64 = 1.0; // 100% sampling +pub const DEFAULT_RUNTIME_DIAL9_TASK_DUMP_ENABLED: bool = false; +/// Matches dial9's own default mean idle duration for task-dump sampling. +pub const DEFAULT_RUNTIME_DIAL9_TASK_DUMP_IDLE_THRESHOLD_MS: u64 = 10; // Note: S3 bucket/prefix have no default; absence means upload is disabled (modeled as Option) /// Maximum transition workers used as a local fallback when runtime env is unset. diff --git a/crates/obs/Cargo.toml b/crates/obs/Cargo.toml index 2376d806e..1b49a0a69 100644 --- a/crates/obs/Cargo.toml +++ b/crates/obs/Cargo.toml @@ -27,9 +27,29 @@ documentation = "https://docs.rs/rustfs-obs/latest/rustfs_obs/" [features] default = [] +# Tokio runtime-level telemetry. Requires a `--cfg tokio_unstable` build; the +# build script fails the compile when that flag is missing. Off by default so +# ordinary builds neither pay for nor depend on Tokio's unstable API. +dial9 = ["dep:dial9-tokio-telemetry"] +# Capture async backtraces of stalled tasks. Enables `tokio/taskdump`, which +# hard-errors at compile time on anything other than linux/{aarch64,x86,x86_64}, +# and additionally needs `--cfg tokio_taskdump` to record anything. Linux-only, +# and not a silent no-op elsewhere: enabling it on macOS fails the build. +dial9-taskdump = ["dial9", "dial9-tokio-telemetry/taskdump"] +# +# NOTE: there is deliberately no `dial9-s3` feature. dial9's `worker-s3` feature +# pulls aws-sdk-s3-transfer-manager 0.1.3 (its latest), which pins +# aws-smithy-http-client onto hyper-rustls 0.24 / rustls-webpki 0.101.7. That +# webpki carries RUSTSEC-2026-0098, -0099 and -0104, and `cargo deny` rejects it. +# Feature unification cannot drop a transitive dependency, so this can only be +# fixed upstream. Tracked in rustfs/backlog#1157 (D9-14). gpu = ["dep:nvml-wrapper"] pyroscope = ["dep:pyroscope"] +[[example]] +name = "dial9_smoke" +required-features = ["dial9"] + [lints] workspace = true @@ -76,7 +96,7 @@ tracing-opentelemetry = { workspace = true } tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt", "env-filter", "tracing-log", "time", "local-time", "json"] } tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] } tokio-util = { workspace = true } -dial9-tokio-telemetry = { workspace = true } +dial9-tokio-telemetry = { workspace = true, optional = true } thiserror = { workspace = true } zstd = { workspace = true, features = ["zstdmt"] } sysinfo = { workspace = true } diff --git a/crates/obs/build.rs b/crates/obs/build.rs new file mode 100644 index 000000000..aaa13037c --- /dev/null +++ b/crates/obs/build.rs @@ -0,0 +1,42 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Enforces that the `dial9` feature is paired with `--cfg tokio_unstable`. +//! +//! `dial9-tokio-telemetry` hooks Tokio's unstable runtime instrumentation. The +//! flag lives in `RUSTFLAGS`, which a caller can silently clobber by exporting +//! their own value — that used to turn telemetry off with no diagnostic. Fail +//! the build instead. + +fn main() { + println!("cargo::rerun-if-changed=build.rs"); + println!("cargo::rerun-if-env-changed=RUSTFLAGS"); + + let dial9_enabled = std::env::var_os("CARGO_FEATURE_DIAL9").is_some(); + let tokio_unstable = std::env::var_os("CARGO_CFG_TOKIO_UNSTABLE").is_some(); + + if dial9_enabled && !tokio_unstable { + panic!( + "\n\ + rustfs-obs: the `dial9` feature requires a `--cfg tokio_unstable` build.\n\ + \n\ + Build telemetry-enabled binaries with:\n\ + \n RUSTFLAGS=\"--cfg tokio_unstable\" cargo build --features dial9\n\ + \n\ + or run `make build-profiling`. Note that setting RUSTFLAGS in the\n\ + environment overrides `.cargo/config.toml`, so the flag must be\n\ + present in the value you export.\n" + ); + } +} diff --git a/crates/obs/examples/dial9_smoke.rs b/crates/obs/examples/dial9_smoke.rs new file mode 100644 index 000000000..348b8a750 --- /dev/null +++ b/crates/obs/examples/dial9_smoke.rs @@ -0,0 +1,110 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! End-to-end check that a traced runtime actually records events to disk. +//! +//! Guards against the regression where `TracedRuntime::builder()...build(..)` +//! was used instead of `build_and_start(..)`: that returns a guard with +//! recording disabled, so telemetry appeared healthy and produced empty traces. +//! +//! Configuration comes from the environment (the crate forbids `unsafe`, so this +//! cannot call `set_var`). Run with: +//! +//! ```bash +//! RUSTFLAGS="--cfg tokio_unstable" \ +//! RUSTFS_RUNTIME_DIAL9_ENABLED=true \ +//! RUSTFS_RUNTIME_DIAL9_OUTPUT_DIR=/tmp/rustfs-dial9-smoke \ +//! RUSTFS_RUNTIME_DIAL9_MAX_FILE_SIZE=65536 \ +//! RUSTFS_RUNTIME_DIAL9_ROTATION_COUNT=4 \ +//! cargo run -p rustfs-obs --features dial9 --example dial9_smoke +//! ``` + +use rustfs_obs::dial9::{Dial9Config, build_traced_runtime}; +use std::path::Path; + +const USAGE: &str = "\ +set RUSTFS_RUNTIME_DIAL9_ENABLED=true and RUSTFS_RUNTIME_DIAL9_OUTPUT_DIR= \ +before running this example (see the module docs for a full command line)"; + +/// A segment that was opened but never recorded an event still carries a header +/// (~292 bytes when this was written). A workload like the one below produces +/// several KiB. Anything under this threshold means events are not being +/// recorded, even though the file exists and the guard looks healthy. +const MIN_EXPECTED_TRACE_BYTES: u64 = 2048; + +fn segment_bytes(dir: &Path, prefix: &str) -> u64 { + let Ok(entries) = std::fs::read_dir(dir) else { + return 0; + }; + entries + .filter_map(Result::ok) + .filter(|e| e.file_name().to_str().is_some_and(|n| n.starts_with(prefix))) + .filter_map(|e| e.metadata().ok()) + .map(|m| m.len()) + .sum() +} + +fn main() { + let config = Dial9Config::from_env(); + if !config.enabled { + eprintln!("{USAGE}"); + std::process::exit(2); + } + + let output_dir = std::path::PathBuf::from(&config.output_dir); + println!("output_dir = {}", config.output_dir); + println!("disk budget = {} bytes", config.total_disk_budget()); + + let mut builder = tokio::runtime::Builder::new_multi_thread(); + builder.worker_threads(2).enable_all(); + + let (runtime, guard) = build_traced_runtime(builder).expect("traced runtime should build"); + // Necessary but not sufficient: a guard built without `build_and_start` also + // reports active. Only the byte count below proves events were recorded. + assert!(guard.is_active(), "telemetry session must be live"); + println!("session active = {}", guard.is_active()); + + // Generate poll/park/spawn traffic, including a deliberately long poll. + runtime.block_on(async { + let mut handles = Vec::new(); + for i in 0..64_u64 { + handles.push(tokio::spawn(async move { + if i % 16 == 0 { + // Block the worker: exactly the fault dial9 exists to surface. + std::thread::sleep(std::time::Duration::from_millis(5)); + } + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + i * 2 + })); + } + for h in handles { + h.await.expect("task should not panic"); + } + }); + + // Dropping the guard flushes buffered events and seals the active segment. + drop(guard); + drop(runtime); + + let bytes = segment_bytes(&output_dir, &config.file_prefix); + println!("trace bytes = {bytes}"); + + assert!( + bytes >= MIN_EXPECTED_TRACE_BYTES, + "dial9 wrote only {bytes} bytes to {} (expected >= {MIN_EXPECTED_TRACE_BYTES}); \ + the segment holds a header but no events, so recording never started", + output_dir.display() + ); + println!("\nOK: dial9 recorded {bytes} bytes of trace data."); +} diff --git a/crates/obs/examples/test_dial9.rs b/crates/obs/examples/test_dial9.rs deleted file mode 100644 index 70c17689c..000000000 --- a/crates/obs/examples/test_dial9.rs +++ /dev/null @@ -1,53 +0,0 @@ -// Test dial9 integration example -use rustfs_obs::dial9::{Dial9Config, is_enabled}; - -#[tokio::main] -async fn main() -> Result<(), Box> { - println!("=== Dial9 Integration Test ===\n"); - - // Test 1: Check initial dial9 state - println!("Test 1: Default state"); - let initial_enabled = is_enabled(); - println!(" dial9 enabled: {}", initial_enabled); - if initial_enabled { - println!(" ⚠ SKIP: Dial9 is already enabled via environment; skipping default-disabled assertion\n"); - } else { - println!(" ✓ PASS: Dial9 is disabled by default\n"); - } - - // Test 2: Load default configuration - println!("Test 2: Default configuration"); - let config = Dial9Config::from_env(); - println!(" enabled: {}", config.enabled); - println!(" output_dir: {}", config.output_dir); - println!(" file_prefix: {}", config.file_prefix); - println!(" max_file_size: {} bytes", config.max_file_size); - println!(" rotation_count: {}", config.rotation_count); - println!(" s3_bucket: {:?}", config.s3_bucket); - println!(" s3_prefix: {:?}", config.s3_prefix); - println!(" sampling_rate: {}", config.sampling_rate); - println!(" ✓ PASS: Default configuration loaded\n"); - - // Test 3: Configuration validation - println!("Test 3: Configuration validation"); - if !initial_enabled { - assert!(!config.enabled, "Should be disabled by default"); - assert_eq!(config.s3_bucket, None, "S3 bucket should be None by default"); - assert_eq!(config.s3_prefix, None, "S3 prefix should be None by default"); - println!(" ✓ PASS: Configuration validated\n"); - } else { - println!(" ⚠ SKIP: Configuration validation skipped (dial9 is enabled)\n"); - } - - println!("=== All Tests Passed! ==="); - println!(); - println!("Note: To test with dial9 enabled, set environment variables:"); - println!(" export RUSTFS_RUNTIME_DIAL9_ENABLED=true"); - println!(" export RUSTFS_RUNTIME_DIAL9_OUTPUT_DIR=/tmp/rustfs-test-telemetry"); - println!(" export RUSTFS_RUNTIME_DIAL9_SAMPLING_RATE=0.5"); - println!(" export RUSTFS_RUNTIME_DIAL9_S3_BUCKET=my-bucket"); - println!(" export RUSTFS_RUNTIME_DIAL9_S3_PREFIX=telemetry/"); - println!(" cargo run -p rustfs-obs --example test_dial9"); - - Ok(()) -} diff --git a/crates/obs/examples/test_dial9_full.rs b/crates/obs/examples/test_dial9_full.rs deleted file mode 100644 index e4c4306ae..000000000 --- a/crates/obs/examples/test_dial9_full.rs +++ /dev/null @@ -1,76 +0,0 @@ -// Full dial9 integration test with session initialization -use rustfs_obs::dial9::{Dial9Config, init_session, is_enabled}; -use tokio::time::{Duration, sleep}; - -#[tokio::main] -async fn main() -> Result<(), Box> { - println!("=== Full Dial9 Integration Test ==="); - println!(); - - // Check if dial9 is enabled - if !is_enabled() { - println!("Dial9 is disabled. Enable with:"); - println!(" export RUSTFS_RUNTIME_DIAL9_ENABLED=true"); - println!(" export RUSTFS_RUNTIME_DIAL9_OUTPUT_DIR=/tmp/rustfs-test-telemetry"); - return Ok(()); - } - - // Test 1: Configuration - println!("Test 1: Configuration"); - let config = Dial9Config::from_env(); - println!(" enabled: {}", config.enabled); - println!(" output_dir: {}", config.output_dir); - println!(" file_prefix: {}", config.file_prefix); - println!(" sampling_rate: {}", config.sampling_rate); - println!(" ✓ Configuration loaded"); - println!(); - - // Test 2: Session initialization - println!("Test 2: Session initialization"); - match init_session().await { - Ok(Some(guard)) => { - println!(" ✓ Session initialized successfully"); - println!(" guard.is_active(): {}", guard.is_active()); - println!(); - - // Test 3: Generate async activity - println!("Test 3: Generate async runtime activity"); - let tasks = (0..3).map(|i| { - tokio::spawn(async move { - for j in 0..5 { - println!(" Task {} iteration {}", i, j); - sleep(Duration::from_millis(20)).await; - } - }) - }); - - for task in tasks { - task.await?; - } - println!(" ✓ Async activity completed"); - println!(); - - // Test 4: Session lifecycle - println!("Test 4: Session lifecycle"); - println!(" Dropping guard..."); - drop(guard); - println!(" ✓ Session cleaned up"); - } - Ok(None) => { - println!(" ⚠ Session not created (writer may have failed)"); - println!(" This is expected if output directory cannot be created"); - } - Err(e) => { - println!(" ✗ Session init failed: {:?}", e); - } - } - - println!(); - println!("=== Test Summary ==="); - println!("✓ Configuration: PASS"); - println!("✓ Session Init: PASS"); - println!("✓ Async Activity: PASS"); - println!("✓ Lifecycle: PASS"); - - Ok(()) -} diff --git a/crates/obs/examples/test_dial9_s3.rs b/crates/obs/examples/test_dial9_s3.rs deleted file mode 100644 index bd7dbbf45..000000000 --- a/crates/obs/examples/test_dial9_s3.rs +++ /dev/null @@ -1,63 +0,0 @@ -// Test dial9 S3 configuration -use rustfs_obs::dial9::{Dial9Config, is_enabled}; - -#[tokio::main] -async fn main() -> Result<(), Box> { - println!("=== Dial9 S3 Configuration Test ==="); - println!(); - - // Test 1: Default S3 configuration (should be None/None) - println!("Test 1: Default S3 configuration"); - let default_config = Dial9Config::default(); - println!(" s3_bucket: {:?}", default_config.s3_bucket); - println!(" s3_prefix: {:?}", default_config.s3_prefix); - assert_eq!(default_config.s3_bucket, None); - assert_eq!(default_config.s3_prefix, None); - println!(" ✓ PASS: Default S3 config is None/None"); - println!(); - - // Test 2: Check if dial9 is enabled - println!("Test 2: Check dial9 enabled state"); - println!(" is_enabled(): {}", is_enabled()); - println!( - " RUSTFS_RUNTIME_DIAL9_ENABLED: {}", - std::env::var("RUSTFS_RUNTIME_DIAL9_ENABLED").unwrap_or_else(|_| "not set".to_string()) - ); - println!(); - - // Test 3: Load configuration from environment - println!("Test 3: Load configuration from environment"); - let config = Dial9Config::from_env(); - println!(" enabled: {}", config.enabled); - println!(" s3_bucket: {:?}", config.s3_bucket); - println!(" s3_prefix: {:?}", config.s3_prefix); - println!(" ✓ PASS: Configuration loaded"); - println!(); - - // Only test S3 config if dial9 is enabled - if !config.enabled { - println!(" ⚠ SKIP: Dial9 is disabled, S3 config not loaded"); - println!(" To test S3 configuration:"); - println!(" export RUSTFS_RUNTIME_DIAL9_ENABLED=true"); - println!(" export RUSTFS_RUNTIME_DIAL9_S3_BUCKET=my-bucket"); - println!(" export RUSTFS_RUNTIME_DIAL9_S3_PREFIX=telemetry/"); - println!(" cargo run -p rustfs-obs --example test_dial9_s3"); - return Ok(()); - } - - // Test 4: Configuration summary - println!("Test 4: Configuration summary"); - println!(" S3 upload enabled: {}", config.s3_bucket.is_some()); - if let Some(bucket) = &config.s3_bucket { - println!(" S3 bucket: {}", bucket); - } - if let Some(prefix) = &config.s3_prefix { - println!(" S3 prefix: {}", prefix); - } - println!(" ✓ PASS: Configuration summary displayed"); - println!(); - - println!("=== All Tests Passed! ==="); - - Ok(()) -} diff --git a/crates/obs/examples/test_dial9_simple.rs b/crates/obs/examples/test_dial9_simple.rs deleted file mode 100644 index 8e2b7b0ac..000000000 --- a/crates/obs/examples/test_dial9_simple.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Simple dial9 integration test (reads from environment) -use rustfs_obs::dial9::{Dial9Config, is_enabled}; - -#[tokio::main] -async fn main() -> Result<(), Box> { - println!("=== Dial9 Integration Test ==="); - println!(); - - // Test 1: Check current state - println!("Test 1: Check dial9 state"); - println!( - " RUSTFS_RUNTIME_DIAL9_ENABLED: {}", - std::env::var("RUSTFS_RUNTIME_DIAL9_ENABLED").unwrap_or_else(|_| "not set".to_string()) - ); - println!(" is_enabled(): {}", is_enabled()); - println!(" ✓ Dial9 state check complete"); - println!(); - - // Test 2: Load configuration - println!("Test 2: Load dial9 configuration"); - let config = Dial9Config::from_env(); - println!(" enabled: {}", config.enabled); - println!(" output_dir: {}", config.output_dir); - println!(" file_prefix: {}", config.file_prefix); - println!(" max_file_size: {} bytes", config.max_file_size); - println!(" rotation_count: {}", config.rotation_count); - println!(" sampling_rate: {}", config.sampling_rate); - println!(" ✓ Configuration loaded"); - println!(); - - // Test 3: Test base path calculation - println!("Test 3: Base path calculation"); - println!(" base_path: {:?}", config.base_path()); - println!(" ✓ Base path calculated"); - println!(); - - println!("=== All Tests Passed! ==="); - println!(); - println!("Note: To test full dial9 functionality, enable it with:"); - println!(" export RUSTFS_RUNTIME_DIAL9_ENABLED=true"); - println!(" export RUSTFS_RUNTIME_DIAL9_OUTPUT_DIR=/tmp/rustfs-telemetry"); - println!(" cargo run -p rustfs-obs --example test_dial9_simple"); - - Ok(()) -} diff --git a/crates/obs/src/metrics/collectors/dial9.rs b/crates/obs/src/metrics/collectors/dial9.rs index f21c54e36..351a6e5c2 100644 --- a/crates/obs/src/metrics/collectors/dial9.rs +++ b/crates/obs/src/metrics/collectors/dial9.rs @@ -14,100 +14,65 @@ //! dial9 Tokio runtime telemetry metrics collector. //! -//! This module provides metrics for monitoring the health and performance -//! of the dial9 telemetry system itself. - -#![allow(dead_code)] +//! Reports the health of the telemetry system itself, not the runtime events it +//! records — those live in the trace segments on disk. +//! +//! Every metric here is backed by a value the process actually observes. A +//! counter that cannot be sourced is not exported at all: a series pinned at +//! zero reads as "nothing happened", which is worse than a missing series. use crate::MetricType; use crate::metrics::report::PrometheusMetric; -use crate::telemetry::dial9::runtime_stats_snapshot; -use rustfs_config::{DEFAULT_RUNTIME_DIAL9_ENABLED, ENV_RUNTIME_DIAL9_ENABLED}; -use rustfs_utils::get_env_bool; +use crate::telemetry::dial9::{is_configured, is_enabled, is_supported, runtime_stats_snapshot}; -/// Dial9 telemetry system statistics. +/// Health of the dial9 telemetry system. #[derive(Debug, Clone, Default)] pub struct Dial9Stats { - /// Total number of telemetry events recorded - pub events_total: u64, - - /// Total bytes written to trace files - pub bytes_written: u64, - - /// Number of file rotations that have occurred - pub rotation_count: u64, - - /// Total number of dial9 errors + /// Cumulative count of telemetry setup failures. pub errors_total: u64, - /// Estimated CPU overhead percentage (if available) - pub cpu_overhead_percent: f64, - - /// Current disk usage by trace files in bytes + /// Bytes on disk held by trace segments, as of the last background refresh. pub disk_usage_bytes: u64, - /// Number of active sessions + /// Number of active telemetry sessions (0 or 1). pub active_sessions: u64, } -/// Collect dial9 telemetry metrics. +/// Convert dial9 health statistics into Prometheus metrics. /// -/// This function converts dial9 statistics into Prometheus metrics format. -/// -/// # Arguments -/// -/// * `stats` - Dial9 statistics to report -/// -/// # Returns -/// -/// A vector of Prometheus metrics for dial9 telemetry statistics. +/// `rustfs_dial9_supported` reflects compile-time support, `rustfs_dial9_configured` +/// reflects operator intent, and `rustfs_dial9_active_sessions` reflects reality. +/// They disagree when a build lacks the feature, or when the traced runtime +/// failed to build and the process fell back to a standard runtime — read all +/// three before concluding that telemetry is running. pub fn collect_dial9_metrics(stats: &Dial9Stats) -> Vec { - let enabled = is_dial9_enabled(); - let enabled_value = if enabled { 1.0 } else { 0.0 }; + let mut metrics = vec![ + PrometheusMetric::new( + "rustfs_dial9_supported", + MetricType::Gauge, + "Whether this binary was compiled with dial9 telemetry support (1) or not (0)", + bool_gauge(is_supported()), + ), + PrometheusMetric::new( + "rustfs_dial9_configured", + MetricType::Gauge, + "Whether dial9 telemetry is requested via environment configuration (1) or not (0)", + bool_gauge(is_configured()), + ), + ]; - let mut metrics = vec![PrometheusMetric::new( - "rustfs_dial9_enabled", - MetricType::Gauge, - "Whether dial9 telemetry is enabled (1) or disabled (0)", - enabled_value, - )]; - - // If dial9 is disabled, return just the enabled flag - if !enabled { + // Without a running session the remaining series carry no information; + // exporting them would only publish zeros. + if !is_enabled() { return metrics; } - // Add detailed metrics when enabled - metrics.extend(vec![ + metrics.extend([ PrometheusMetric::new( - "rustfs_dial9_events_total", - MetricType::Counter, - "Total number of Tokio runtime events recorded by dial9", - stats.events_total as f64, - ), - PrometheusMetric::new( - "rustfs_dial9_bytes_written_total", - MetricType::Counter, - "Total bytes written to dial9 trace files", - stats.bytes_written as f64, - ), - PrometheusMetric::new( - "rustfs_dial9_rotations_total", - MetricType::Counter, - "Total number of trace file rotations", - stats.rotation_count as f64, - ), - PrometheusMetric::new( - "rustfs_dial9_errors_total", - MetricType::Counter, - "Total number of dial9 telemetry errors", - stats.errors_total as f64, - ), - PrometheusMetric::new( - "rustfs_dial9_cpu_overhead_percent", + "rustfs_dial9_active_sessions", MetricType::Gauge, - "Estimated CPU overhead percentage from dial9 telemetry", - stats.cpu_overhead_percent, + "Number of active dial9 telemetry sessions", + stats.active_sessions as f64, ), PrometheusMetric::new( "rustfs_dial9_disk_usage_bytes", @@ -116,80 +81,82 @@ pub fn collect_dial9_metrics(stats: &Dial9Stats) -> Vec { stats.disk_usage_bytes as f64, ), PrometheusMetric::new( - "rustfs_dial9_active_sessions", - MetricType::Gauge, - "Number of active dial9 telemetry sessions", - stats.active_sessions as f64, + "rustfs_dial9_errors_total", + MetricType::Counter, + "Total number of dial9 telemetry setup errors", + stats.errors_total as f64, ), ]); metrics } +fn bool_gauge(value: bool) -> f64 { + if value { 1.0 } else { 0.0 } +} + /// Collect dial9 metrics from the current runtime snapshot. +/// +/// Reads cached atomics only; performs no I/O. pub fn collect_current_dial9_metrics() -> Vec { let snapshot = runtime_stats_snapshot(); let stats = Dial9Stats { errors_total: snapshot.errors_total, disk_usage_bytes: snapshot.disk_usage_bytes, active_sessions: snapshot.active_sessions, - ..Dial9Stats::default() }; collect_dial9_metrics(&stats) } -/// Check if dial9 telemetry is enabled via environment variable. +/// Whether dial9 telemetry is actually running: compiled in *and* configured. pub fn is_dial9_enabled() -> bool { - get_env_bool(ENV_RUNTIME_DIAL9_ENABLED, DEFAULT_RUNTIME_DIAL9_ENABLED) + is_enabled() } #[cfg(test)] mod tests { use super::*; - #[test] - fn test_dial9_stats_default() { - let stats = Dial9Stats::default(); - assert_eq!(stats.events_total, 0); - assert_eq!(stats.bytes_written, 0); - assert_eq!(stats.rotation_count, 0); - assert_eq!(stats.errors_total, 0); - assert_eq!(stats.cpu_overhead_percent, 0.0); - assert_eq!(stats.disk_usage_bytes, 0); - assert_eq!(stats.active_sessions, 0); + fn metric_names(metrics: &[PrometheusMetric]) -> Vec { + metrics.iter().map(|m| m.name.to_string()).collect() } #[test] - fn test_collect_dial9_metrics() { - let stats = Dial9Stats { - events_total: 100, - bytes_written: 1024, - ..Default::default() - }; - let metrics = collect_dial9_metrics(&stats); - - // Should always have at least the enabled flag - assert!(!metrics.is_empty()); + fn always_reports_support_and_intent() { + let names = metric_names(&collect_dial9_metrics(&Dial9Stats::default())); + assert!(names.iter().any(|n| n == "rustfs_dial9_supported")); + assert!(names.iter().any(|n| n == "rustfs_dial9_configured")); } #[test] - fn test_collect_dial9_metrics_with_values() { - let stats = Dial9Stats { - events_total: 10000, - bytes_written: 1024000, - rotation_count: 5, - errors_total: 0, - cpu_overhead_percent: 2.5, - disk_usage_bytes: 2048000, - active_sessions: 1, - }; + fn omits_session_metrics_when_telemetry_is_not_running() { + // Under `cargo test` dial9 is neither compiled in nor configured, so the + // collector must stop after the two compile-time/intent gauges rather + // than publish zeroed session series. + if is_enabled() { + return; + } + let metrics = collect_dial9_metrics(&Dial9Stats::default()); + assert_eq!(metrics.len(), 2); + let names = metric_names(&metrics); + assert!(!names.iter().any(|n| n == "rustfs_dial9_disk_usage_bytes")); + assert!(!names.iter().any(|n| n == "rustfs_dial9_active_sessions")); + } - let metrics = collect_dial9_metrics(&stats); + #[test] + fn supported_gauge_tracks_compile_time_feature() { + let metrics = collect_dial9_metrics(&Dial9Stats::default()); + let supported = metrics + .iter() + .find(|m| m.name == "rustfs_dial9_supported") + .expect("supported gauge is always exported"); + assert_eq!(supported.value, bool_gauge(cfg!(feature = "dial9"))); + } - // When dial9 is enabled, should have all metrics - // Note: This test assumes dial9 is enabled in the test environment - // If disabled, only the enabled flag metric will be present - assert!(!metrics.is_empty()); + #[test] + fn bool_gauge_maps_to_one_and_zero() { + assert_eq!(bool_gauge(true), 1.0); + assert_eq!(bool_gauge(false), 0.0); } } diff --git a/crates/obs/src/telemetry/dial9.rs b/crates/obs/src/telemetry/dial9.rs deleted file mode 100644 index a1166b17f..000000000 --- a/crates/obs/src/telemetry/dial9.rs +++ /dev/null @@ -1,640 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! dial9-tokio-telemetry integration for RustFS. -//! -//! This module provides low-overhead Tokio runtime-level telemetry, -//! capturing events like PollStart/End, WorkerPark/Unpark, QueueSample, etc. - -use crate::TelemetryError; -// Import and re-export TelemetryGuard for use in other crates (like rustfs) -// Use as Dial9TelemetryGuard internally to avoid naming conflicts -use dial9_tokio_telemetry::telemetry::RotatingWriter; -pub use dial9_tokio_telemetry::telemetry::TelemetryGuard; -use dial9_tokio_telemetry::telemetry::TelemetryGuard as Dial9TelemetryGuard; -// Use rustfs_config which re-exports runtime constants -use rustfs_config::{ - DEFAULT_RUNTIME_DIAL9_ENABLED, DEFAULT_RUNTIME_DIAL9_FILE_PREFIX, DEFAULT_RUNTIME_DIAL9_MAX_FILE_SIZE, - DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR, DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT, DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE, - ENV_RUNTIME_DIAL9_ENABLED, ENV_RUNTIME_DIAL9_FILE_PREFIX, ENV_RUNTIME_DIAL9_MAX_FILE_SIZE, ENV_RUNTIME_DIAL9_OUTPUT_DIR, - ENV_RUNTIME_DIAL9_ROTATION_COUNT, ENV_RUNTIME_DIAL9_S3_BUCKET, ENV_RUNTIME_DIAL9_S3_PREFIX, ENV_RUNTIME_DIAL9_SAMPLING_RATE, -}; -use rustfs_utils::get_env_bool; -use rustfs_utils::get_env_opt_str; -use rustfs_utils::get_env_opt_u64; -use rustfs_utils::get_env_opt_usize; -use rustfs_utils::get_env_str; -use std::path::PathBuf; -use std::sync::OnceLock; -use std::sync::RwLock; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use tracing::{info, warn}; - -const LOG_COMPONENT_OBS: &str = "obs"; -const LOG_SUBSYSTEM_DIAL9: &str = "dial9"; -const EVENT_DIAL9_STATE: &str = "dial9_state"; - -#[derive(Debug, Clone, Default)] -pub(crate) struct Dial9RuntimeSnapshot { - pub active_sessions: u64, - pub errors_total: u64, - pub disk_usage_bytes: u64, -} - -#[derive(Debug)] -struct Dial9RuntimeState { - enabled: AtomicBool, - active_sessions: AtomicU64, - errors_total: AtomicU64, - output_dir: RwLock>, - file_prefix: RwLock>, -} - -impl Dial9RuntimeState { - fn new() -> Self { - Self { - enabled: AtomicBool::new(false), - active_sessions: AtomicU64::new(0), - errors_total: AtomicU64::new(0), - output_dir: RwLock::new(None), - file_prefix: RwLock::new(None), - } - } - - fn record_config(&self, config: &Dial9Config) { - self.enabled.store(config.enabled, Ordering::Relaxed); - *self.output_dir.write().expect("dial9 output_dir lock should not be poisoned") = Some(PathBuf::from(&config.output_dir)); - *self - .file_prefix - .write() - .expect("dial9 file_prefix lock should not be poisoned") = Some(config.file_prefix.clone()); - if !config.enabled { - self.active_sessions.store(0, Ordering::Relaxed); - } - } - - fn record_runtime_started(&self, config: &Dial9Config) { - self.record_config(config); - self.active_sessions.store(1, Ordering::Relaxed); - } - - fn record_runtime_error(&self, config: &Dial9Config) { - self.record_config(config); - self.active_sessions.store(0, Ordering::Relaxed); - self.errors_total.fetch_add(1, Ordering::Relaxed); - } - - fn snapshot(&self) -> Dial9RuntimeSnapshot { - Dial9RuntimeSnapshot { - active_sessions: self.active_sessions.load(Ordering::Relaxed), - errors_total: self.errors_total.load(Ordering::Relaxed), - disk_usage_bytes: self.current_disk_usage_bytes(), - } - } - - fn current_disk_usage_bytes(&self) -> u64 { - let output_dir = self - .output_dir - .read() - .expect("dial9 output_dir lock should not be poisoned") - .clone(); - let file_prefix = self - .file_prefix - .read() - .expect("dial9 file_prefix lock should not be poisoned") - .clone(); - - let Some(output_dir) = output_dir else { - return 0; - }; - let Some(file_prefix) = file_prefix else { - return 0; - }; - - std::fs::read_dir(output_dir) - .ok() - .into_iter() - .flatten() - .filter_map(Result::ok) - .filter_map(|entry| { - let filename = entry.file_name(); - let filename = filename.to_str()?; - filename - .starts_with(&file_prefix) - .then(|| entry.metadata().ok().map(|meta| meta.len())) - }) - .flatten() - .sum() - } -} - -fn dial9_runtime_state() -> &'static Dial9RuntimeState { - static STATE: OnceLock = OnceLock::new(); - STATE.get_or_init(Dial9RuntimeState::new) -} - -fn resolve_sampling_rate() -> (f64, bool) { - match std::env::var(ENV_RUNTIME_DIAL9_SAMPLING_RATE) { - Ok(raw) => { - let trimmed = raw.trim(); - if trimmed.is_empty() { - return (DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE, false); - } - - match trimmed.parse::() { - Ok(value) if value.is_finite() => (value.clamp(0.0, 1.0), true), - _ => { - warn!( - event = EVENT_DIAL9_STATE, - component = LOG_COMPONENT_OBS, - subsystem = LOG_SUBSYSTEM_DIAL9, - result = "invalid_sampling_rate", - provided = trimmed, - fallback = DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE, - "dial9 state changed" - ); - (DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE, true) - } - } - } - Err(_) => (DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE, false), - } -} - -fn warn_ignored_runtime_knobs(config: &Dial9Config, sampling_rate_explicit: bool) { - if config.s3_bucket.is_some() || config.s3_prefix.is_some() { - warn!( - event = EVENT_DIAL9_STATE, - component = LOG_COMPONENT_OBS, - subsystem = LOG_SUBSYSTEM_DIAL9, - result = "unsupported_s3_upload_knobs_ignored", - s3_bucket = config.s3_bucket.as_deref().unwrap_or(""), - s3_prefix = config.s3_prefix.as_deref().unwrap_or(""), - "dial9 state changed" - ); - } - - if sampling_rate_explicit { - warn!( - event = EVENT_DIAL9_STATE, - component = LOG_COMPONENT_OBS, - subsystem = LOG_SUBSYSTEM_DIAL9, - result = "unsupported_sampling_rate_ignored", - sampling_rate = config.sampling_rate, - "dial9 state changed" - ); - } -} - -fn sanitize_dial9_max_file_size(bytes: u64) -> u64 { - bytes.max(1) -} - -fn sanitize_dial9_rotation_count(count: usize) -> usize { - count.max(1) -} - -fn dial9_total_rotation_size(max_file_size: u64, rotation_count: usize) -> u64 { - max_file_size.saturating_mul(u64::try_from(rotation_count).unwrap_or(u64::MAX)) -} - -/// Configuration for dial9 Tokio telemetry. -#[derive(Debug, Clone)] -pub struct Dial9Config { - /// Whether dial9 telemetry is enabled - pub enabled: bool, - - /// Directory where trace files are written - pub output_dir: String, - - /// Prefix for trace file names - pub file_prefix: String, - - /// Maximum size of each trace file in bytes - pub max_file_size: u64, - - /// Number of rotated files to keep - pub rotation_count: usize, - - /// Optional S3 bucket for uploading trace files - pub s3_bucket: Option, - - /// Optional S3 prefix for uploaded files - pub s3_prefix: Option, - - /// Sampling rate (0.0 to 1.0) - pub sampling_rate: f64, -} - -impl Default for Dial9Config { - fn default() -> Self { - Self { - enabled: DEFAULT_RUNTIME_DIAL9_ENABLED, - output_dir: DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR.to_string(), - file_prefix: DEFAULT_RUNTIME_DIAL9_FILE_PREFIX.to_string(), - max_file_size: DEFAULT_RUNTIME_DIAL9_MAX_FILE_SIZE, - rotation_count: DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT, - s3_bucket: None, - s3_prefix: None, - sampling_rate: DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE, - } - } -} - -impl Dial9Config { - /// Create configuration from environment variables. - pub fn from_env() -> Self { - let enabled = get_env_bool(ENV_RUNTIME_DIAL9_ENABLED, DEFAULT_RUNTIME_DIAL9_ENABLED); - - if !enabled { - let config = Self::default(); - dial9_runtime_state().record_config(&config); - return config; - } - - let raw_max_file_size = get_env_opt_u64(ENV_RUNTIME_DIAL9_MAX_FILE_SIZE); - let raw_rotation_count = get_env_opt_usize(ENV_RUNTIME_DIAL9_ROTATION_COUNT); - let (sampling_rate, sampling_rate_explicit) = resolve_sampling_rate(); - let max_file_size = sanitize_dial9_max_file_size(raw_max_file_size.unwrap_or(DEFAULT_RUNTIME_DIAL9_MAX_FILE_SIZE)); - let rotation_count = sanitize_dial9_rotation_count(raw_rotation_count.unwrap_or(DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT)); - if raw_max_file_size == Some(0) { - warn!( - event = EVENT_DIAL9_STATE, - component = LOG_COMPONENT_OBS, - subsystem = LOG_SUBSYSTEM_DIAL9, - result = "invalid_max_file_size", - fallback_bytes = 1_u64, - "dial9 state changed" - ); - } - if raw_rotation_count == Some(0) { - warn!( - event = EVENT_DIAL9_STATE, - component = LOG_COMPONENT_OBS, - subsystem = LOG_SUBSYSTEM_DIAL9, - result = "invalid_rotation_count", - fallback_count = 1_usize, - "dial9 state changed" - ); - } - - let config = Self { - enabled, - output_dir: get_env_str(ENV_RUNTIME_DIAL9_OUTPUT_DIR, DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR), - file_prefix: get_env_str(ENV_RUNTIME_DIAL9_FILE_PREFIX, DEFAULT_RUNTIME_DIAL9_FILE_PREFIX), - max_file_size, - rotation_count, - s3_bucket: get_env_opt_str(ENV_RUNTIME_DIAL9_S3_BUCKET).filter(|s| !s.is_empty()), - s3_prefix: get_env_opt_str(ENV_RUNTIME_DIAL9_S3_PREFIX).filter(|s| !s.is_empty()), - sampling_rate, - }; - warn_ignored_runtime_knobs(&config, sampling_rate_explicit); - dial9_runtime_state().record_config(&config); - config - } - - /// Get the base path for trace files. - pub fn base_path(&self) -> PathBuf { - PathBuf::from(&self.output_dir).join(&self.file_prefix) - } -} - -/// Guard for dial9 telemetry session. -/// -/// When dropped, this guard will flush any remaining telemetry data. -/// Keep it alive for the duration of your application. -pub struct Dial9SessionGuard { - /// The underlying dial9 telemetry guard (if enabled) - _guard: Option, - /// Configuration - #[allow(dead_code)] - config: Dial9Config, -} - -impl Dial9SessionGuard { - /// Create a new dial9 session guard. - /// - /// Note: This only validates configuration and creates the output directory. - /// The actual telemetry session is created when building the Tokio runtime - /// via `build_traced_runtime()`. - /// - /// Returns `Ok(None)` if dial9 is disabled. - pub async fn new(config: Dial9Config) -> Result, TelemetryError> { - if !config.enabled { - info!( - event = EVENT_DIAL9_STATE, - component = LOG_COMPONENT_OBS, - subsystem = LOG_SUBSYSTEM_DIAL9, - state = "disabled", - "dial9 state changed" - ); - return Ok(None); - } - - info!( - event = EVENT_DIAL9_STATE, - component = LOG_COMPONENT_OBS, - subsystem = LOG_SUBSYSTEM_DIAL9, - state = "validating", - output_dir = %config.output_dir, - file_prefix = %config.file_prefix, - sampling_rate = config.sampling_rate, - "dial9 state changed" - ); - - // Only create directory; writer will be created in build_traced_runtime - if let Err(e) = tokio::fs::create_dir_all(&config.output_dir).await { - warn!( - event = EVENT_DIAL9_STATE, - component = LOG_COMPONENT_OBS, - subsystem = LOG_SUBSYSTEM_DIAL9, - result = "output_dir_create_failed", - output_dir = %config.output_dir, - error = %e, - fallback = "disabled", - "dial9 state changed" - ); - return Ok(None); - } - - info!( - event = EVENT_DIAL9_STATE, - component = LOG_COMPONENT_OBS, - subsystem = LOG_SUBSYSTEM_DIAL9, - state = "validated", - output_dir = %config.output_dir, - file_prefix = %config.file_prefix, - "dial9 state changed" - ); - - dial9_runtime_state().record_config(&config); - Ok(Some(Self { _guard: None, config })) - } - - /// Set the telemetry guard (called after runtime creation) - #[allow(dead_code)] - pub(crate) fn set_guard(&mut self, guard: Dial9TelemetryGuard) { - self._guard = Some(guard); - dial9_runtime_state().record_runtime_started(&self.config); - } - - /// Check if this guard has an active session. - pub fn is_active(&self) -> bool { - dial9_runtime_state().snapshot().active_sessions > 0 - } - - /// Flush any pending telemetry data. - pub async fn shutdown(&self) { - if self.is_active() { - info!( - event = EVENT_DIAL9_STATE, - component = LOG_COMPONENT_OBS, - subsystem = LOG_SUBSYSTEM_DIAL9, - state = "shutdown_requested", - result = "runtime_guard_managed_elsewhere", - "dial9 state" - ); - } - } -} - -impl Drop for Dial9SessionGuard { - fn drop(&mut self) { - if let Some(_guard) = &self._guard { - // TelemetryGuard flushes automatically when dropped - info!( - event = EVENT_DIAL9_STATE, - component = LOG_COMPONENT_OBS, - subsystem = LOG_SUBSYSTEM_DIAL9, - state = "flushed", - "dial9 state" - ); - } - } -} - -/// Initialize dial9 telemetry session from environment configuration. -/// -/// This function reads configuration from environment variables and creates -/// a dial9 session guard if enabled. The guard should be kept alive for the -/// duration of the application. -/// -/// # Returns -/// -/// - `Ok(Some(guard))` - Dial9 is enabled and session initialized -/// - `Ok(None)` - Dial9 is disabled or failed to initialize (non-fatal) -/// - `Err(e)` - Fatal error (should not happen with current implementation) -pub async fn init_session() -> Result, TelemetryError> { - let config = Dial9Config::from_env(); - Dial9SessionGuard::new(config).await -} - -/// Check if dial9 telemetry is enabled via environment configuration. -pub fn is_enabled() -> bool { - get_env_bool(ENV_RUNTIME_DIAL9_ENABLED, DEFAULT_RUNTIME_DIAL9_ENABLED) -} - -/// Build a Tokio runtime with dial9 telemetry enabled. -/// -/// This function creates a Tokio runtime with dial9 telemetry integrated -/// if enabled via environment variables. Returns a tuple of (Runtime, TelemetryGuard). -/// -/// This is internal API used by the runtime builder. -/// -/// # Arguments -/// -/// * `builder` - The configured Tokio runtime builder -/// -/// # Returns -/// -/// * `Ok((runtime, guard))` - Successfully created runtime with telemetry -/// * `Err` - If runtime creation fails or dial9 is enabled but fails to initialize -/// -/// # Errors -/// -/// Returns an error if: -/// - Dial9 is enabled but the runtime builder fails -/// - Dial9 is enabled but writer creation fails -pub fn build_traced_runtime( - builder: tokio::runtime::Builder, -) -> Result<(tokio::runtime::Runtime, Dial9TelemetryGuard), TelemetryError> { - if !is_enabled() { - return Err(TelemetryError::Io("Dial9 is not enabled".to_string())); - } - - let config = Dial9Config::from_env(); - dial9_runtime_state().record_config(&config); - - // Ensure the output directory exists before creating the writer - std::fs::create_dir_all(&config.output_dir).map_err(|e| { - dial9_runtime_state().record_runtime_error(&config); - TelemetryError::Io(format!("Failed to create dial9 output directory '{}': {}", config.output_dir, e)) - })?; - - // Create rotating writer (synchronous for runtime building) - let base_path = config.base_path(); - let writer = RotatingWriter::new( - base_path, - config.max_file_size, - dial9_total_rotation_size(config.max_file_size, config.rotation_count), - ) - .map_err(|e| { - dial9_runtime_state().record_runtime_error(&config); - TelemetryError::Io(format!("Failed to create RotatingWriter: {}", e)) - })?; - - // Build traced runtime - // Note: sampling_rate and S3 upload settings are reserved for future use - // once the dial9 library provides support for those configuration options. - let (runtime, guard) = dial9_tokio_telemetry::telemetry::TracedRuntime::builder() - .with_task_tracking(true) - .build(builder, writer) - .map_err(|e| { - dial9_runtime_state().record_runtime_error(&config); - TelemetryError::Io(format!("Failed to build TracedRuntime: {}", e)) - })?; - - dial9_runtime_state().record_runtime_started(&config); - Ok((runtime, guard)) -} - -pub(crate) fn runtime_stats_snapshot() -> Dial9RuntimeSnapshot { - dial9_runtime_state().snapshot() -} - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::{Mutex, OnceLock}; - use tempfile::tempdir; - - static DIAL9_ENV_TEST_LOCK: OnceLock> = OnceLock::new(); - - fn with_dial9_env_lock(f: F) - where - F: FnOnce(), - { - let _guard = DIAL9_ENV_TEST_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("dial9 env test lock should not be poisoned"); - f(); - } - - fn reset_dial9_runtime_state_for_test() { - let state = dial9_runtime_state(); - state.enabled.store(false, Ordering::Relaxed); - state.active_sessions.store(0, Ordering::Relaxed); - state.errors_total.store(0, Ordering::Relaxed); - *state - .output_dir - .write() - .expect("dial9 output_dir lock should not be poisoned") = None; - *state - .file_prefix - .write() - .expect("dial9 file_prefix lock should not be poisoned") = None; - } - - #[test] - fn test_dial9_config_default() { - let config = Dial9Config::default(); - assert!(!config.enabled); - assert_eq!(config.output_dir, DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR); - assert_eq!(config.file_prefix, DEFAULT_RUNTIME_DIAL9_FILE_PREFIX); - assert_eq!(config.max_file_size, DEFAULT_RUNTIME_DIAL9_MAX_FILE_SIZE); - assert_eq!(config.rotation_count, DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT); - assert_eq!(config.sampling_rate, DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE); - } - - #[test] - fn test_dial9_config_base_path() { - let config = Dial9Config { - output_dir: "/tmp/telemetry".to_string(), - file_prefix: "rustfs".to_string(), - ..Default::default() - }; - assert_eq!(config.base_path(), PathBuf::from("/tmp/telemetry/rustfs")); - } - - #[test] - fn test_dial9_sanitizers_reject_zero_values() { - assert_eq!(sanitize_dial9_max_file_size(0), 1); - assert_eq!(sanitize_dial9_rotation_count(0), 1); - } - - #[test] - fn test_dial9_total_rotation_size_saturates() { - assert_eq!(dial9_total_rotation_size(u64::MAX, 2), u64::MAX); - } - - #[test] - fn test_is_enabled_default() { - // Skip if environment variable is explicitly set - if std::env::var(ENV_RUNTIME_DIAL9_ENABLED).is_ok() { - println!("Skipping test: RUSTFS_RUNTIME_DIAL9_ENABLED is set"); - return; - } - assert!(!is_enabled()); - } - - #[test] - fn sampling_rate_rejects_nan_and_falls_back_to_default() { - reset_dial9_runtime_state_for_test(); - with_dial9_env_lock(|| { - temp_env::with_var(ENV_RUNTIME_DIAL9_ENABLED, Some("true"), || { - temp_env::with_var(ENV_RUNTIME_DIAL9_SAMPLING_RATE, Some("NaN"), || { - let config = Dial9Config::from_env(); - assert_eq!(config.sampling_rate, DEFAULT_RUNTIME_DIAL9_SAMPLING_RATE); - }); - }); - }); - } - - #[test] - fn session_guard_is_active_when_runtime_state_reports_an_active_session() { - reset_dial9_runtime_state_for_test(); - let config = Dial9Config { - enabled: true, - ..Dial9Config::default() - }; - dial9_runtime_state().record_runtime_started(&config); - - let guard = Dial9SessionGuard { _guard: None, config }; - - assert!(guard.is_active()); - } - - #[test] - fn runtime_stats_snapshot_reports_disk_usage_and_errors() { - reset_dial9_runtime_state_for_test(); - let dir = tempdir().expect("create temp dir"); - let trace_path = dir.path().join("dial9-trace.segment"); - std::fs::write(&trace_path, vec![0_u8; 128]).expect("write fake dial9 trace"); - - let config = Dial9Config { - enabled: true, - output_dir: dir.path().to_string_lossy().into_owned(), - file_prefix: "dial9-trace".to_string(), - ..Dial9Config::default() - }; - dial9_runtime_state().record_runtime_started(&config); - dial9_runtime_state().record_runtime_error(&config); - - let snapshot = runtime_stats_snapshot(); - assert_eq!(snapshot.active_sessions, 0); - assert_eq!(snapshot.errors_total, 1); - assert_eq!(snapshot.disk_usage_bytes, 128); - } -} diff --git a/crates/obs/src/telemetry/dial9/config.rs b/crates/obs/src/telemetry/dial9/config.rs new file mode 100644 index 000000000..a044d0948 --- /dev/null +++ b/crates/obs/src/telemetry/dial9/config.rs @@ -0,0 +1,276 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Configuration for dial9 Tokio runtime telemetry. +//! +//! This module carries no dependency on the `dial9-tokio-telemetry` crate, so +//! it compiles identically whether or not the `dial9` feature is enabled. That +//! lets callers read the configured state (and export metrics about it) from a +//! binary that was built without telemetry support. + +use super::state::dial9_runtime_state; +use rustfs_config::{ + DEFAULT_RUNTIME_DIAL9_ENABLED, DEFAULT_RUNTIME_DIAL9_FILE_PREFIX, DEFAULT_RUNTIME_DIAL9_MAX_FILE_SIZE, + DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR, DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT, DEFAULT_RUNTIME_DIAL9_TASK_DUMP_ENABLED, + DEFAULT_RUNTIME_DIAL9_TASK_DUMP_IDLE_THRESHOLD_MS, ENV_RUNTIME_DIAL9_ENABLED, ENV_RUNTIME_DIAL9_FILE_PREFIX, + ENV_RUNTIME_DIAL9_MAX_FILE_SIZE, ENV_RUNTIME_DIAL9_OUTPUT_DIR, ENV_RUNTIME_DIAL9_ROTATION_COUNT, ENV_RUNTIME_DIAL9_S3_BUCKET, + ENV_RUNTIME_DIAL9_S3_PREFIX, ENV_RUNTIME_DIAL9_TASK_DUMP_ENABLED, ENV_RUNTIME_DIAL9_TASK_DUMP_IDLE_THRESHOLD_MS, +}; +use rustfs_utils::{get_env_bool, get_env_opt_str, get_env_opt_u64, get_env_opt_usize, get_env_str}; +use std::path::PathBuf; +use std::time::Duration; +use tracing::warn; + +use super::{EVENT_DIAL9_STATE, LOG_COMPONENT_OBS, LOG_SUBSYSTEM_DIAL9}; + +/// Whether this binary was compiled with dial9 telemetry support. +/// +/// Telemetry requires both the `dial9` cargo feature and `--cfg tokio_unstable`; +/// the crate's build script enforces that pairing, so this single flag is +/// sufficient to describe compile-time support. +pub const fn is_supported() -> bool { + cfg!(feature = "dial9") +} + +/// Whether dial9 telemetry is requested via environment configuration. +/// +/// This reports operator intent and is independent of whether the binary can +/// honour it. Use [`is_enabled`] to test whether telemetry will actually run. +pub fn is_configured() -> bool { + get_env_bool(ENV_RUNTIME_DIAL9_ENABLED, DEFAULT_RUNTIME_DIAL9_ENABLED) +} + +/// Whether dial9 telemetry will actually run: requested *and* compiled in. +pub fn is_enabled() -> bool { + is_supported() && is_configured() +} + +fn sanitize_max_file_size(bytes: u64) -> u64 { + bytes.max(1) +} + +fn sanitize_rotation_count(count: usize) -> usize { + count.max(1) +} + +/// Total on-disk byte budget for the trace family, saturating on overflow. +pub(super) fn total_rotation_size(max_file_size: u64, rotation_count: usize) -> u64 { + max_file_size.saturating_mul(u64::try_from(rotation_count).unwrap_or(u64::MAX)) +} + +/// Configuration for dial9 Tokio telemetry. +#[derive(Debug, Clone)] +pub struct Dial9Config { + /// Whether dial9 telemetry is enabled + pub enabled: bool, + + /// Directory where trace files are written + pub output_dir: String, + + /// Prefix for trace file names + pub file_prefix: String, + + /// Maximum size of each trace file in bytes + pub max_file_size: u64, + + /// Number of rotated files to keep + pub rotation_count: usize, + + /// Optional S3 bucket for uploading sealed trace segments + pub s3_bucket: Option, + + /// Optional key prefix for uploaded segments + pub s3_prefix: Option, + + /// Whether to capture async backtraces for tasks that stall + pub task_dump_enabled: bool, + + /// Mean idle duration for task-dump Poisson sampling + pub task_dump_idle_threshold: Duration, +} + +impl Default for Dial9Config { + fn default() -> Self { + Self { + enabled: DEFAULT_RUNTIME_DIAL9_ENABLED, + output_dir: DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR.to_string(), + file_prefix: DEFAULT_RUNTIME_DIAL9_FILE_PREFIX.to_string(), + max_file_size: DEFAULT_RUNTIME_DIAL9_MAX_FILE_SIZE, + rotation_count: DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT, + s3_bucket: None, + s3_prefix: None, + task_dump_enabled: DEFAULT_RUNTIME_DIAL9_TASK_DUMP_ENABLED, + task_dump_idle_threshold: Duration::from_millis(DEFAULT_RUNTIME_DIAL9_TASK_DUMP_IDLE_THRESHOLD_MS), + } + } +} + +impl Dial9Config { + /// Create configuration from environment variables. + pub fn from_env() -> Self { + let enabled = is_enabled(); + + if !enabled { + let config = Self::default(); + dial9_runtime_state().record_config(&config); + return config; + } + + let raw_max_file_size = get_env_opt_u64(ENV_RUNTIME_DIAL9_MAX_FILE_SIZE); + let raw_rotation_count = get_env_opt_usize(ENV_RUNTIME_DIAL9_ROTATION_COUNT); + let max_file_size = sanitize_max_file_size(raw_max_file_size.unwrap_or(DEFAULT_RUNTIME_DIAL9_MAX_FILE_SIZE)); + let rotation_count = sanitize_rotation_count(raw_rotation_count.unwrap_or(DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT)); + + if raw_max_file_size == Some(0) { + warn!( + event = EVENT_DIAL9_STATE, + component = LOG_COMPONENT_OBS, + subsystem = LOG_SUBSYSTEM_DIAL9, + result = "invalid_max_file_size", + fallback_bytes = 1_u64, + "dial9 state changed" + ); + } + if raw_rotation_count == Some(0) { + warn!( + event = EVENT_DIAL9_STATE, + component = LOG_COMPONENT_OBS, + subsystem = LOG_SUBSYSTEM_DIAL9, + result = "invalid_rotation_count", + fallback_count = 1_usize, + "dial9 state changed" + ); + } + + let s3_bucket = get_env_opt_str(ENV_RUNTIME_DIAL9_S3_BUCKET).filter(|s| !s.is_empty()); + let s3_prefix = get_env_opt_str(ENV_RUNTIME_DIAL9_S3_PREFIX).filter(|s| !s.is_empty()); + warn_unusable_s3_upload(s3_bucket.as_deref(), s3_prefix.as_deref()); + + let config = Self { + enabled, + output_dir: get_env_str(ENV_RUNTIME_DIAL9_OUTPUT_DIR, DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR), + file_prefix: get_env_str(ENV_RUNTIME_DIAL9_FILE_PREFIX, DEFAULT_RUNTIME_DIAL9_FILE_PREFIX), + max_file_size, + rotation_count, + s3_bucket, + s3_prefix, + task_dump_enabled: get_env_bool(ENV_RUNTIME_DIAL9_TASK_DUMP_ENABLED, DEFAULT_RUNTIME_DIAL9_TASK_DUMP_ENABLED), + task_dump_idle_threshold: Duration::from_millis( + get_env_opt_u64(ENV_RUNTIME_DIAL9_TASK_DUMP_IDLE_THRESHOLD_MS) + .filter(|ms| *ms > 0) + .unwrap_or(DEFAULT_RUNTIME_DIAL9_TASK_DUMP_IDLE_THRESHOLD_MS), + ), + }; + + dial9_runtime_state().record_config(&config); + config + } + + /// Get the base path for trace files. + pub fn base_path(&self) -> PathBuf { + PathBuf::from(&self.output_dir).join(&self.file_prefix) + } + + /// Total on-disk byte budget across all retained segments. + pub fn total_disk_budget(&self) -> u64 { + total_rotation_size(self.max_file_size, self.rotation_count) + } + + /// Whether S3 upload of sealed segments is both requested and usable. + pub fn s3_upload_requested(&self) -> bool { + self.s3_bucket.is_some() + } +} + +/// S3 upload of sealed segments is not available in any build. +/// +/// dial9's `worker-s3` feature depends on aws-sdk-s3-transfer-manager 0.1.3, +/// which pins hyper-rustls 0.24 / rustls-webpki 0.101.7 — a webpki carrying +/// RUSTSEC-2026-0098, -0099 and -0104. Cargo cannot drop a transitive +/// dependency, so this needs an upstream fix (rustfs/backlog#1157, D9-14). +/// +/// Warn loudly rather than silently discarding a configured bucket: an operator +/// who set these expects their traces to be uploaded somewhere. +fn warn_unusable_s3_upload(s3_bucket: Option<&str>, s3_prefix: Option<&str>) { + if s3_bucket.is_none() && s3_prefix.is_none() { + return; + } + + warn!( + event = EVENT_DIAL9_STATE, + component = LOG_COMPONENT_OBS, + subsystem = LOG_SUBSYSTEM_DIAL9, + result = "s3_upload_unsupported", + s3_bucket = s3_bucket.unwrap_or(""), + s3_prefix = s3_prefix.unwrap_or(""), + reason = "dial9 worker-s3 depends on a vulnerable rustls-webpki (RUSTSEC-2026-0098/0099/0104)", + remedy = "collect trace segments from the output directory instead", + "dial9 state changed" + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sanitizers_reject_zero_values() { + assert_eq!(sanitize_max_file_size(0), 1); + assert_eq!(sanitize_rotation_count(0), 1); + } + + #[test] + fn total_rotation_size_saturates() { + assert_eq!(total_rotation_size(u64::MAX, 2), u64::MAX); + } + + #[test] + fn default_config_is_disabled() { + let config = Dial9Config::default(); + assert!(!config.enabled); + assert_eq!(config.output_dir, DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR); + assert_eq!(config.file_prefix, DEFAULT_RUNTIME_DIAL9_FILE_PREFIX); + assert_eq!(config.max_file_size, DEFAULT_RUNTIME_DIAL9_MAX_FILE_SIZE); + assert_eq!(config.rotation_count, DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT); + } + + #[test] + fn base_path_joins_prefix() { + let config = Dial9Config { + output_dir: "/tmp/telemetry".to_string(), + file_prefix: "rustfs".to_string(), + ..Default::default() + }; + assert_eq!(config.base_path(), PathBuf::from("/tmp/telemetry/rustfs")); + } + + #[test] + fn total_disk_budget_multiplies_file_size_by_rotation_count() { + let config = Dial9Config { + max_file_size: 100, + rotation_count: 10, + ..Default::default() + }; + assert_eq!(config.total_disk_budget(), 1_000); + } + + #[test] + fn is_enabled_requires_compile_time_support() { + // `is_enabled` is the conjunction of the two; when support is absent it + // must stay false no matter what the environment requests. + if !is_supported() { + assert!(!is_enabled()); + } + } +} diff --git a/crates/obs/src/telemetry/dial9/disabled.rs b/crates/obs/src/telemetry/dial9/disabled.rs new file mode 100644 index 000000000..de2b299e2 --- /dev/null +++ b/crates/obs/src/telemetry/dial9/disabled.rs @@ -0,0 +1,63 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Stub implementation used when the `dial9` feature is off. +//! +//! Mirrors the surface of [`super::enabled`] so that callers never need a +//! `#[cfg]`. Building a traced runtime always fails here, which drives callers +//! down their standard-runtime fallback path. + +use crate::TelemetryError; + +/// Placeholder for `dial9_tokio_telemetry::telemetry::TelemetryGuard`. +/// +/// Never constructed: [`build_traced_runtime`] always fails without the +/// `dial9` feature. +#[derive(Debug)] +pub struct TelemetryGuard(()); + +/// Placeholder session guard. Never constructed without the `dial9` feature. +#[derive(Debug)] +pub struct Dial9SessionGuard(()); + +impl Dial9SessionGuard { + /// Always false: telemetry support is not compiled in. + pub fn is_active(&self) -> bool { + false + } +} + +/// Always fails: telemetry support is not compiled in. +/// +/// # Errors +/// +/// Always returns [`TelemetryError::Io`] describing the missing feature. +pub fn build_traced_runtime( + _builder: tokio::runtime::Builder, +) -> Result<(tokio::runtime::Runtime, Dial9SessionGuard), TelemetryError> { + Err(TelemetryError::Io( + "dial9 telemetry support is not compiled in; rebuild with --features dial9".to_string(), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn build_traced_runtime_always_fails_without_the_feature() { + let result = build_traced_runtime(tokio::runtime::Builder::new_current_thread()); + assert!(result.is_err()); + } +} diff --git a/crates/obs/src/telemetry/dial9/enabled.rs b/crates/obs/src/telemetry/dial9/enabled.rs new file mode 100644 index 000000000..e5d286c00 --- /dev/null +++ b/crates/obs/src/telemetry/dial9/enabled.rs @@ -0,0 +1,236 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! `dial9-tokio-telemetry` integration, compiled when the `dial9` feature is on. +//! +//! Captures Tokio runtime-level events (poll start/end, worker park/unpark, +//! task spawn/terminate) into rotating binary trace segments. This is an +//! on-demand profiler rather than always-on telemetry: it is disabled by +//! default and requires a `--cfg tokio_unstable` build. + +use super::config::Dial9Config; +use super::state::{dial9_runtime_state, measure_disk_usage_bytes}; +use super::{EVENT_DIAL9_STATE, LOG_COMPONENT_OBS, LOG_SUBSYSTEM_DIAL9}; +use crate::TelemetryError; +use dial9_tokio_telemetry::telemetry::{ProcessResourceUsageConfig, RotatingWriter, TaskDumpConfig, TracedRuntime}; +use std::time::Duration; +use tracing::{info, warn}; + +pub use dial9_tokio_telemetry::telemetry::TelemetryGuard; + +/// How often the background refresher restates trace-file disk usage. +const DISK_USAGE_REFRESH_INTERVAL: Duration = Duration::from_secs(60); + +/// Name recorded in segment metadata so the trace viewer can label workers. +const RUNTIME_NAME: &str = "rustfs-worker"; + +/// Owns a live dial9 telemetry session. +/// +/// Dropping this guard flushes buffered events and seals the active segment. +/// It must be dropped before the process exits: a guard parked in a `static` is +/// never dropped, and the final buffered events — usually the interesting ones — +/// are lost. +pub struct Dial9SessionGuard { + guard: TelemetryGuard, + config: Dial9Config, +} + +impl Dial9SessionGuard { + /// Whether the underlying telemetry session is recording. + pub fn is_active(&self) -> bool { + self.guard.is_enabled() + } + + /// The configuration this session was built from. + pub fn config(&self) -> &Dial9Config { + &self.config + } +} + +impl std::fmt::Debug for Dial9SessionGuard { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Dial9SessionGuard") + .field("active", &self.is_active()) + .field("output_dir", &self.config.output_dir) + .finish() + } +} + +impl Drop for Dial9SessionGuard { + fn drop(&mut self) { + dial9_runtime_state().record_runtime_stopped(); + info!( + event = EVENT_DIAL9_STATE, + component = LOG_COMPONENT_OBS, + subsystem = LOG_SUBSYSTEM_DIAL9, + state = "flushed", + "dial9 state changed" + ); + // `TelemetryGuard`'s own `Drop` flushes buffered events and seals the + // active segment; it runs immediately after this body. + } +} + +/// Build a Tokio runtime with dial9 telemetry attached and recording started. +/// +/// # Errors +/// +/// Returns an error when dial9 is not enabled, when the output directory cannot +/// be created, or when the traced runtime fails to build. Callers are expected +/// to fall back to a standard runtime. +pub fn build_traced_runtime( + builder: tokio::runtime::Builder, +) -> Result<(tokio::runtime::Runtime, Dial9SessionGuard), TelemetryError> { + let config = Dial9Config::from_env(); + if !config.enabled { + return Err(TelemetryError::Io("dial9 telemetry is not enabled".to_string())); + } + + std::fs::create_dir_all(&config.output_dir).map_err(|e| { + dial9_runtime_state().record_runtime_error(&config); + TelemetryError::Io(format!("Failed to create dial9 output directory '{}': {e}", config.output_dir)) + })?; + + let writer = RotatingWriter::new(config.base_path(), config.max_file_size, config.total_disk_budget()).map_err(|e| { + dial9_runtime_state().record_runtime_error(&config); + TelemetryError::Io(format!("Failed to create dial9 RotatingWriter: {e}")) + })?; + + // `with_trace_path` transitions the builder into the state that spawns the + // background worker, which drives the segment pipeline. + let traced = TracedRuntime::builder() + .with_trace_path(&config.output_dir) + .with_task_tracking(true) + .with_runtime_name(RUNTIME_NAME) + .with_process_resource_usage(ProcessResourceUsageConfig::default()); + + let traced = if config.task_dump_enabled { + traced.with_task_dumps( + TaskDumpConfig::builder() + .idle_threshold(config.task_dump_idle_threshold) + .build(), + ) + } else { + traced + }; + + // `build_and_start` rather than `build`: `build` returns a live guard that + // never records, writing segments that contain only a header. + // + // No `with_s3_uploader` here: dial9's `worker-s3` feature carries a + // vulnerable TLS stack. See the note in `crates/obs/Cargo.toml`. + finish_traced_runtime(traced.build_and_start(builder, writer), config) +} + +/// Publish the outcome of a traced-runtime build and start the background +/// disk-usage refresher. +fn finish_traced_runtime( + started: std::io::Result<(tokio::runtime::Runtime, TelemetryGuard)>, + config: Dial9Config, +) -> Result<(tokio::runtime::Runtime, Dial9SessionGuard), TelemetryError> { + let (runtime, guard) = started.map_err(|e| { + dial9_runtime_state().record_runtime_error(&config); + TelemetryError::Io(format!("Failed to build dial9 TracedRuntime: {e}")) + })?; + + // `is_enabled` distinguishes a live guard from the inert one a lenient + // config produces after a build failure. It does NOT mean recording has + // started — a guard from `build` (rather than `build_and_start`) reports + // `true` while writing segments that contain only a header. Recording is + // guaranteed by the `build_and_start` call above, not by this check. + if !guard.is_enabled() { + dial9_runtime_state().record_runtime_error(&config); + return Err(TelemetryError::Io("dial9 TracedRuntime built with telemetry disabled".to_string())); + } + + dial9_runtime_state().record_runtime_started(&config); + runtime.spawn(refresh_disk_usage_loop()); + + info!( + event = EVENT_DIAL9_STATE, + component = LOG_COMPONENT_OBS, + subsystem = LOG_SUBSYSTEM_DIAL9, + state = "recording", + output_dir = %config.output_dir, + file_prefix = %config.file_prefix, + disk_budget_bytes = config.total_disk_budget(), + task_dumps = config.task_dump_enabled, + "dial9 state changed" + ); + + Ok((runtime, Dial9SessionGuard { guard, config })) +} + +/// Periodically restate trace-file disk usage so the metrics collector can read +/// it from an atomic instead of walking the directory on its own thread. +async fn refresh_disk_usage_loop() { + let mut ticker = tokio::time::interval(DISK_USAGE_REFRESH_INTERVAL); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + ticker.tick().await; + + let Some(location) = dial9_runtime_state().trace_location() else { + continue; + }; + + // `measure_disk_usage_bytes` walks a directory and stats every entry, + // so it must stay off the async workers. + let measured = + tokio::task::spawn_blocking(move || measure_disk_usage_bytes(&location.output_dir, &location.file_prefix)).await; + + match measured { + Ok(bytes) => dial9_runtime_state().set_disk_usage_bytes(bytes), + Err(e) => warn!( + event = EVENT_DIAL9_STATE, + component = LOG_COMPONENT_OBS, + subsystem = LOG_SUBSYSTEM_DIAL9, + result = "disk_usage_refresh_failed", + error = %e, + "dial9 state changed" + ), + } + } +} + +#[cfg(test)] +mod tests { + use super::super::state::{reset_for_test, runtime_stats_snapshot}; + use super::*; + + #[test] + fn build_traced_runtime_refuses_when_disabled() { + reset_for_test(); + // dial9 is not configured in the unit-test environment, so + // `Dial9Config::from_env()` yields a disabled config. + let result = build_traced_runtime(tokio::runtime::Builder::new_current_thread()); + assert!(result.is_err(), "disabled dial9 must not build a traced runtime"); + } + + #[test] + fn recording_session_reports_active_until_stopped() { + reset_for_test(); + let config = Dial9Config { + enabled: true, + ..Dial9Config::default() + }; + + dial9_runtime_state().record_runtime_started(&config); + assert_eq!(runtime_stats_snapshot().active_sessions, 1); + + // Mirrors what `Dial9SessionGuard::drop` publishes. + dial9_runtime_state().record_runtime_stopped(); + assert_eq!(runtime_stats_snapshot().active_sessions, 0); + } +} diff --git a/crates/obs/src/telemetry/dial9/mod.rs b/crates/obs/src/telemetry/dial9/mod.rs new file mode 100644 index 000000000..ca30c729a --- /dev/null +++ b/crates/obs/src/telemetry/dial9/mod.rs @@ -0,0 +1,66 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Tokio runtime-level telemetry via `dial9-tokio-telemetry`. +//! +//! This is an on-demand profiler for executor-level faults — long polls that +//! stall a worker, park/unpark storms, tasks that never yield — which are +//! invisible to request-level metrics and spans. +//! +//! # Compile-time support +//! +//! Telemetry requires the `dial9` cargo feature *and* a `--cfg tokio_unstable` +//! build; `build.rs` enforces that pairing. Without the feature this module +//! still exposes its full API, backed by [`disabled`] stubs, so no caller needs +//! a `#[cfg]`. Use [`is_supported`] to test compile-time support, +//! [`is_configured`] to test operator intent, and [`is_enabled`] for the +//! conjunction that decides whether telemetry actually runs. +//! +//! # Cost +//! +//! Trace segments are written to disk continuously and evicted oldest-first +//! once `RUSTFS_RUNTIME_DIAL9_MAX_FILE_SIZE * RUSTFS_RUNTIME_DIAL9_ROTATION_COUNT` +//! bytes are retained. Under a high poll rate the budget can wrap in minutes, +//! so size it against the window you need to capture. +//! +//! # Known observability gap +//! +//! `dial9`'s `RotatingWriter` stops accepting writes (its internal `Finished` +//! state) when the output directory disappears or a segment cannot be sealed, +//! and it exposes no way to observe that from outside. `TelemetryGuard::is_enabled` +//! reports how the session was *built*, not whether it is still writing. There +//! is therefore no `writer_healthy` metric: it could only ever be hard-coded to +//! `1`. Watch `rustfs_dial9_disk_usage_bytes` — a session that is recording but +//! whose disk usage stops growing has most likely hit this state. + +mod config; +mod state; + +#[cfg(not(feature = "dial9"))] +mod disabled; +#[cfg(feature = "dial9")] +mod enabled; + +pub(super) const LOG_COMPONENT_OBS: &str = "obs"; +pub(super) const LOG_SUBSYSTEM_DIAL9: &str = "dial9"; +pub(super) const EVENT_DIAL9_STATE: &str = "dial9_state"; + +pub use config::{Dial9Config, is_configured, is_enabled, is_supported}; +pub(crate) use state::runtime_stats_snapshot; + +#[cfg(feature = "dial9")] +pub use enabled::{Dial9SessionGuard, TelemetryGuard, build_traced_runtime}; + +#[cfg(not(feature = "dial9"))] +pub use disabled::{Dial9SessionGuard, TelemetryGuard, build_traced_runtime}; diff --git a/crates/obs/src/telemetry/dial9/state.rs b/crates/obs/src/telemetry/dial9/state.rs new file mode 100644 index 000000000..644fc3bce --- /dev/null +++ b/crates/obs/src/telemetry/dial9/state.rs @@ -0,0 +1,212 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Process-global observable state for dial9 telemetry. +//! +//! Every field is an atomic so that [`runtime_stats_snapshot`] is allocation +//! free and performs no I/O: it runs on the metrics collection path, which must +//! never block. Disk usage is sampled by a background refresher (see +//! `enabled::refresh_disk_usage_loop`) and published here. + +use super::config::Dial9Config; +#[cfg(any(feature = "dial9", test))] +use std::path::Path; +use std::path::PathBuf; +use std::sync::OnceLock; +use std::sync::RwLock; +use std::sync::atomic::{AtomicU64, Ordering}; + +/// Point-in-time view of dial9 runtime state. +#[derive(Debug, Clone, Default)] +pub(crate) struct Dial9RuntimeSnapshot { + /// 1 while a traced runtime is installed, 0 otherwise. + pub active_sessions: u64, + /// Cumulative count of telemetry setup failures. + pub errors_total: u64, + /// Bytes on disk for this trace family, as of the last refresh. + pub disk_usage_bytes: u64, +} + +#[derive(Debug)] +pub(super) struct Dial9RuntimeState { + active_sessions: AtomicU64, + errors_total: AtomicU64, + disk_usage_bytes: AtomicU64, + trace_dir: RwLock>, +} + +#[derive(Debug, Clone)] +pub(super) struct TraceLocation { + #[cfg_attr(not(feature = "dial9"), allow(dead_code))] + pub output_dir: PathBuf, + #[cfg_attr(not(feature = "dial9"), allow(dead_code))] + pub file_prefix: String, +} + +impl Dial9RuntimeState { + fn new() -> Self { + Self { + active_sessions: AtomicU64::new(0), + errors_total: AtomicU64::new(0), + disk_usage_bytes: AtomicU64::new(0), + trace_dir: RwLock::new(None), + } + } + + pub(super) fn record_config(&self, config: &Dial9Config) { + *self.trace_dir.write().expect("dial9 trace_dir lock should not be poisoned") = Some(TraceLocation { + output_dir: PathBuf::from(&config.output_dir), + file_prefix: config.file_prefix.clone(), + }); + if !config.enabled { + self.active_sessions.store(0, Ordering::Relaxed); + } + } + + #[cfg(any(feature = "dial9", test))] + pub(super) fn record_runtime_started(&self, config: &Dial9Config) { + self.record_config(config); + self.active_sessions.store(1, Ordering::Relaxed); + } + + #[cfg(any(feature = "dial9", test))] + pub(super) fn record_runtime_error(&self, config: &Dial9Config) { + self.record_config(config); + self.active_sessions.store(0, Ordering::Relaxed); + self.errors_total.fetch_add(1, Ordering::Relaxed); + } + + #[cfg(feature = "dial9")] + pub(super) fn record_runtime_stopped(&self) { + self.active_sessions.store(0, Ordering::Relaxed); + } + + #[cfg(any(feature = "dial9", test))] + pub(super) fn set_disk_usage_bytes(&self, bytes: u64) { + self.disk_usage_bytes.store(bytes, Ordering::Relaxed); + } + + #[cfg(feature = "dial9")] + pub(super) fn trace_location(&self) -> Option { + self.trace_dir + .read() + .expect("dial9 trace_dir lock should not be poisoned") + .clone() + } + + fn snapshot(&self) -> Dial9RuntimeSnapshot { + Dial9RuntimeSnapshot { + active_sessions: self.active_sessions.load(Ordering::Relaxed), + errors_total: self.errors_total.load(Ordering::Relaxed), + disk_usage_bytes: self.disk_usage_bytes.load(Ordering::Relaxed), + } + } +} + +pub(super) fn dial9_runtime_state() -> &'static Dial9RuntimeState { + static STATE: OnceLock = OnceLock::new(); + STATE.get_or_init(Dial9RuntimeState::new) +} + +/// Reset the process-global state. Tests share one process, so any test that +/// mutates the state must call this first. +#[cfg(test)] +pub(super) fn reset_for_test() { + let state = dial9_runtime_state(); + state.active_sessions.store(0, Ordering::Relaxed); + state.errors_total.store(0, Ordering::Relaxed); + state.disk_usage_bytes.store(0, Ordering::Relaxed); + *state.trace_dir.write().expect("dial9 trace_dir lock should not be poisoned") = None; +} + +/// Sum the on-disk size of every segment belonging to this trace family. +/// +/// This walks a directory and stats each entry, so it must never run on an +/// async worker or on the metrics collection path. Callers are responsible for +/// scheduling it on a blocking thread. +#[cfg(any(feature = "dial9", test))] +pub(super) fn measure_disk_usage_bytes(output_dir: &Path, file_prefix: &str) -> u64 { + let Ok(entries) = std::fs::read_dir(output_dir) else { + return 0; + }; + + entries + .filter_map(Result::ok) + .filter(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|filename| filename.starts_with(file_prefix)) + }) + .filter_map(|entry| entry.metadata().ok()) + .map(|meta| meta.len()) + .sum() +} + +/// Snapshot dial9 runtime state. Lock-free apart from a read lock on the trace +/// location, and performs no I/O. +pub(crate) fn runtime_stats_snapshot() -> Dial9RuntimeSnapshot { + dial9_runtime_state().snapshot() +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + fn measure_disk_usage_sums_only_matching_prefix() { + let dir = tempdir().expect("create temp dir"); + std::fs::write(dir.path().join("rustfs-tokio.0.bin"), vec![0_u8; 128]).expect("write segment"); + std::fs::write(dir.path().join("rustfs-tokio.1.bin"), vec![0_u8; 64]).expect("write segment"); + std::fs::write(dir.path().join("unrelated.log"), vec![0_u8; 4096]).expect("write unrelated"); + + assert_eq!(measure_disk_usage_bytes(dir.path(), "rustfs-tokio"), 192); + } + + #[test] + fn measure_disk_usage_of_missing_dir_is_zero() { + assert_eq!(measure_disk_usage_bytes(Path::new("/nonexistent/rustfs/dial9"), "rustfs-tokio"), 0); + } + + #[test] + fn snapshot_reports_cached_disk_usage_without_touching_disk() { + reset_for_test(); + let config = Dial9Config { + enabled: true, + ..Dial9Config::default() + }; + dial9_runtime_state().record_runtime_started(&config); + dial9_runtime_state().set_disk_usage_bytes(4096); + + let snapshot = runtime_stats_snapshot(); + assert_eq!(snapshot.active_sessions, 1); + assert_eq!(snapshot.disk_usage_bytes, 4096); + } + + #[test] + fn runtime_error_clears_session_and_counts_the_failure() { + reset_for_test(); + let config = Dial9Config { + enabled: true, + ..Dial9Config::default() + }; + dial9_runtime_state().record_runtime_started(&config); + dial9_runtime_state().record_runtime_error(&config); + + let snapshot = runtime_stats_snapshot(); + assert_eq!(snapshot.active_sessions, 0); + assert_eq!(snapshot.errors_total, 1); + } +} diff --git a/docs/architecture/global-state-inventory.md b/docs/architecture/global-state-inventory.md index 1b8b7839c..eacb3e708 100644 --- a/docs/architecture/global-state-inventory.md +++ b/docs/architecture/global-state-inventory.md @@ -95,7 +95,6 @@ inventory. Generic function-local names such as `CACHE`, `LOCK`, `INIT`, and | `LICENSE_STATE`, `LICENSE_VERIFIER` | `rustfs/src/license.rs` | Process-global owner-local state | License state and verifier selection stay private to the license owner; callers use license helper functions. | | `CPU_CONT_GUARD`, `PROFILING_CANCEL_TOKEN` | `rustfs/src/profiling.rs` | Process-global owner-local guard | CPU profiling guard and cancellation state stay private to the profiling owner. | | `MEMORY_SYSTEM` | `rustfs/src/memory_observability.rs` | Process-global owner-local cache | Memory sampling keeps the `sysinfo::System` cache private to the memory observability owner. | -| `DIAL9_TELEMETRY_GUARD` | `rustfs/src/server/runtime.rs` | Process-global owner-local guard | Dial9 telemetry lifetime state stays private to runtime setup. | | `DISPLAY_CONFIG_SNAPSHOT`, `GLOBAL_CONFIG_SNAPSHOT` | `rustfs/src/config/snapshot.rs` | Process-global owner-local state | Config snapshots stay private to the config snapshot owner. | | `BUFFER_CONFIG_SINGLETON`, `BUFFER_PROFILE_ENABLED` | `rustfs/src/config/workload_profiles.rs` | Process-global owner-local state | Workload buffer profile configuration stays private to workload profile helpers. | | `LEGACY_CREDENTIAL_WARNED_KEYS` | `rustfs/src/config/config_struct.rs` | Process-global owner-local cache | Legacy credential warning de-duplication stays private to config parsing. | diff --git a/docs/operations/dial9-runtime-profiling.md b/docs/operations/dial9-runtime-profiling.md new file mode 100644 index 000000000..2a15a9057 --- /dev/null +++ b/docs/operations/dial9-runtime-profiling.md @@ -0,0 +1,138 @@ +# dial9 Tokio Runtime Profiling + +`dial9-tokio-telemetry` records Tokio runtime-level events — poll start/end, +worker park/unpark, task spawn/terminate, and optionally async backtraces of +stalled tasks — into binary trace segments. + +It answers questions that Prometheus metrics and `tracing` spans cannot: + +- A request took 200 ms. Was a worker thread blocked, or was it waiting on I/O? +- Which task held a worker for 40 ms without yielding? +- Are workers parking because there is no work, or because the queue is starved? + +These are the failure modes behind drive stalls, `io_uring`/`O_DIRECT` regressions, +and object-data-cache fill contention. They are invisible to the rest of the obs stack. + +## This is a profiler, not telemetry + +Three properties follow from that, and all three matter operationally. + +**The stock binary cannot run it.** dial9 hooks Tokio's unstable runtime API, which +requires `--cfg tokio_unstable`. Release binaries are built without it, so they neither +depend on Tokio's non-semver surface nor pay its cost. Setting +`RUSTFS_RUNTIME_DIAL9_ENABLED=true` on a stock binary logs a warning and records nothing. + +**Traces are written continuously and evicted.** The retained budget is +`MAX_FILE_SIZE × ROTATION_COUNT` (1 GB by default). Once exceeded, the *oldest* +segments are deleted. Under a high poll rate that budget can wrap in minutes, +which means the incident you are chasing may already have been overwritten. +Size the budget against the window you need, and prefer short, targeted runs. + +**There is no runtime toggle.** Telemetry is installed when the Tokio runtime is +constructed, so enabling or disabling it requires a process restart. + +## Building + +```bash +make build-profiling +``` + +That is `cargo build --release --bin rustfs --features dial9` with +`RUSTFLAGS="--cfg tokio_unstable"`. One optional feature layers on top: + +```bash +# Async backtraces of stalled tasks. Linux aarch64/x86/x86_64 only — +# `tokio/taskdump` hard-errors at compile time on other targets, so this +# will not build on macOS. +DIAL9_RUSTFLAGS="--cfg tokio_unstable --cfg tokio_taskdump" \ + DIAL9_FEATURES="dial9-taskdump" make build-profiling +``` + +`crates/obs/build.rs` fails the build if the `dial9` feature is enabled without +`--cfg tokio_unstable`. This is deliberate: an environment `RUSTFLAGS` *replaces* +the value from `.cargo/config.toml` rather than appending to it, so the flag used +to disappear silently whenever anything else set `RUSTFLAGS`. + +For CPU profiling with usable stacks, add `-C force-frame-pointers=yes`. + +## Running an investigation + +```bash +export RUSTFS_RUNTIME_DIAL9_ENABLED=true +export RUSTFS_RUNTIME_DIAL9_OUTPUT_DIR=/tmp/rustfs-telemetry-investigation +export RUSTFS_RUNTIME_DIAL9_ROTATION_COUNT=3 # 300 MB total, ~short window +export RUSTFS_RUNTIME_DIAL9_TASK_DUMP_ENABLED=true # if built with dial9-taskdump +``` + +Reproduce the fault, then stop the process **gracefully**. The final buffered +events are flushed when the telemetry guard drops during shutdown; a `SIGKILL` +loses them, and they are usually the interesting ones. + +Segments land in `$OUTPUT_DIR/$FILE_PREFIX.N.bin`. Analyse them with upstream's +`TRACE_ANALYSIS_GUIDE.md`. + +Turn `RUSTFS_RUNTIME_DIAL9_ENABLED` back off when you are done. + +## Configuration + +| Variable | Default | Notes | +|---|---|---| +| `RUSTFS_RUNTIME_DIAL9_ENABLED` | `false` | Needs a `dial9` build to take effect | +| `RUSTFS_RUNTIME_DIAL9_OUTPUT_DIR` | `/var/log/rustfs/telemetry` | Must be writable | +| `RUSTFS_RUNTIME_DIAL9_FILE_PREFIX` | `rustfs-tokio` | | +| `RUSTFS_RUNTIME_DIAL9_MAX_FILE_SIZE` | `104857600` | Bytes per segment | +| `RUSTFS_RUNTIME_DIAL9_ROTATION_COUNT` | `10` | Total budget = size × count | +| `RUSTFS_RUNTIME_DIAL9_TASK_DUMP_ENABLED` | `false` | Needs `dial9-taskdump` (Linux only) | +| `RUSTFS_RUNTIME_DIAL9_TASK_DUMP_IDLE_THRESHOLD_MS` | `10` | Mean idle for Poisson sampling | +| `RUSTFS_RUNTIME_DIAL9_S3_BUCKET` | unset | **Not honoured**, see below | +| `RUSTFS_RUNTIME_DIAL9_S3_PREFIX` | unset | **Not honoured**, see below | + +Variables whose build feature is absent are ignored, with a warning naming the +feature to rebuild with. + +### S3 upload is unavailable + +dial9's `worker-s3` feature depends on `aws-sdk-s3-transfer-manager` 0.1.3 (its +latest release), which pins `aws-smithy-http-client` onto `hyper-rustls` 0.24 and +`rustls-webpki` 0.101.7. That webpki carries RUSTSEC-2026-0098, -0099 and -0104, +and the repository's `cargo deny` gate rejects it. + +Cargo's feature unification can add features but cannot drop a transitive +dependency, so this cannot be worked around downstream — it needs an upstream +release. RustFS therefore builds no S3 uploader at all. Setting the two variables +above logs a warning and changes nothing; retrieve trace segments from +`OUTPUT_DIR` directly. Tracked as D9-14 in rustfs/backlog#1157. + +## Metrics + +| Metric | Meaning | +|---|---| +| `rustfs_dial9_supported` | Binary was compiled with telemetry support | +| `rustfs_dial9_configured` | Operator asked for telemetry via the environment | +| `rustfs_dial9_active_sessions` | A session is actually recording (0 or 1) | +| `rustfs_dial9_disk_usage_bytes` | Trace segment bytes on disk, refreshed every 60 s | +| `rustfs_dial9_errors_total` | Telemetry setup failures | + +The first three are separate on purpose, because they disagree in exactly the +cases you need to diagnose: + +- `configured=1, supported=0` — wrong binary. Rebuild with `make build-profiling`. +- `configured=1, supported=1, active_sessions=0` — telemetry failed to start + (usually an unwritable `OUTPUT_DIR`) and the process fell back to a standard + runtime. Check `rustfs_dial9_errors_total` and the startup logs. + +The session metrics are not exported at all unless telemetry is running. A counter +pinned at zero reads as "nothing happened", which is worse than a missing series. + +### Known gap: writer death is not directly observable + +dial9's `RotatingWriter` stops accepting writes (its internal `Finished` state) if +the output directory is removed underneath it or a segment cannot be sealed — +a real risk in containers where `/var/log` gets cleaned. Upstream exposes no way +to observe this: `TelemetryGuard::is_enabled()` reports how the session was *built*, +not whether it is still writing. + +There is therefore no `writer_healthy` gauge, because it could only ever be +hard-coded to `1`. Instead, watch `rustfs_dial9_disk_usage_bytes`: a session with +`active_sessions=1` whose disk usage has stopped growing has most likely hit this +state, and needs a restart. diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 47a324d00..bb41235ed 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -35,12 +35,6 @@ path = "src/lib.rs" name = "rustfs" path = "src/main.rs" -[[bin]] -name = "manual-test-dial9" -path = "tests/manual/test_dial9.rs" -test = false -bench = false - [[bench]] name = "s3_operations" harness = false @@ -60,6 +54,9 @@ full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp", "pyroscope"] manual-test-runners = [] rio-v2 = ["rustfs-ecstore/rio-v2"] pyroscope = ["rustfs-obs/pyroscope"] +# Tokio runtime telemetry. Requires `--cfg tokio_unstable`; use `make build-profiling`. +dial9 = ["rustfs-obs/dial9"] +dial9-taskdump = ["dial9", "rustfs-obs/dial9-taskdump"] hotpath = [ "dep:hotpath", "hotpath/hotpath", diff --git a/rustfs/src/runtime_capabilities.rs b/rustfs/src/runtime_capabilities.rs index dd0fc9987..d038e76d1 100644 --- a/rustfs/src/runtime_capabilities.rs +++ b/rustfs/src/runtime_capabilities.rs @@ -24,6 +24,7 @@ const EBPF_LINUX_ONLY: &str = "eBPF support is only available on linux targets"; const STORAGE_MEDIA_NOT_REPORTED: &str = "storage media not reported by endpoints"; const FAILURE_DOMAIN_NOT_REPORTED: &str = "failure domain labels not reported by endpoints"; const NUMA_NOT_WIRED: &str = "NUMA topology not wired into runtime"; +const RUNTIME_TELEMETRY_NOT_COMPILED_IN: &str = "dial9 telemetry not compiled into this binary"; const NUMA_LINUX_ONLY: &str = "NUMA topology reporting currently targets linux runtimes"; #[derive(Debug, Default, Clone, Copy)] @@ -99,7 +100,10 @@ pub fn topology_snapshot_from_endpoint_pools(endpoint_pools: &EndpointServerPool } fn runtime_telemetry_status() -> CapabilityStatus { - if rustfs_obs::dial9::is_enabled() { + if !rustfs_obs::dial9::is_supported() { + return CapabilityStatus::unsupported().with_reason(RUNTIME_TELEMETRY_NOT_COMPILED_IN); + } + if rustfs_obs::dial9::is_configured() { CapabilityStatus::supported() } else { CapabilityStatus::disabled() @@ -190,10 +194,16 @@ mod tests { assert_eq!(snapshot.platform.os.as_deref(), Some(std::env::consts::OS)); assert_eq!(snapshot.platform.arch.as_deref(), Some(std::env::consts::ARCH)); assert_eq!(snapshot.platform.target_triple.as_deref(), Some(compiled_target_triple().as_str())); + // A binary built without the `dial9` feature reports `Unsupported` + // rather than `Disabled`: telemetry cannot be turned on by configuration + // alone, and reporting it as merely disabled would imply that it can. assert!(matches!( snapshot.runtime_telemetry.state, - CapabilityState::Supported | CapabilityState::Disabled + CapabilityState::Supported | CapabilityState::Disabled | CapabilityState::Unsupported )); + if !rustfs_obs::dial9::is_supported() { + assert!(matches!(snapshot.runtime_telemetry.state, CapabilityState::Unsupported)); + } assert!( snapshot .platform diff --git a/rustfs/src/server/runtime.rs b/rustfs/src/server/runtime.rs index 3f2d7c638..1619d194f 100644 --- a/rustfs/src/server/runtime.rs +++ b/rustfs/src/server/runtime.rs @@ -12,15 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::OnceLock; use std::time::Duration; use sysinfo::{RefreshKind, System}; -// Import TelemetryGuard from rustfs_obs re-export -use rustfs_obs::dial9::TelemetryGuard; - -// Global storage for TelemetryGuard to keep it alive for the program duration -static DIAL9_TELEMETRY_GUARD: OnceLock = OnceLock::new(); +use rustfs_obs::dial9::Dial9SessionGuard; #[inline] fn compute_default_thread_stack_size() -> usize { @@ -166,56 +161,43 @@ fn print_tokio_thread_enable() -> bool { rustfs_utils::get_env_bool(rustfs_config::ENV_THREAD_PRINT_ENABLED, rustfs_config::DEFAULT_THREAD_PRINT_ENABLED) } -/// Build Tokio runtime with optional dial9 telemetry support. +/// Build the Tokio runtime, attaching dial9 telemetry when it is enabled. /// -/// If dial9 is enabled via environment variables, creates a TracedRuntime -/// and stores the TelemetryGuard globally to keep it alive for the -/// duration of the program. +/// The returned [`Dial9SessionGuard`] owns the telemetry session. It **must** +/// be dropped before the process exits: dropping it flushes buffered events and +/// seals the final trace segment. Parking it in a `static` would mean it is +/// never dropped and the last events — usually the ones that explain a stall — +/// are lost. +/// +/// When telemetry is requested but cannot be built, this falls back to a +/// standard runtime and returns `None` for the guard rather than failing +/// startup. The `rustfs_dial9_active_sessions` metric reports that outcome. /// /// # Returns /// -/// * `Ok(runtime)` - Successfully created runtime -/// * `Err(e)` - Failed to create runtime +/// The runtime, plus `Some(guard)` when a telemetry session is recording. /// /// # Errors /// -/// Returns an error if: -/// - The Tokio runtime builder fails -/// - Dial9 is enabled but fails to initialize (falls back to standard runtime) -/// -/// # Examples -/// -/// ```no_run -/// // build_tokio_runtime is pub(crate) - call it from within the rustfs binary: -/// // let runtime = build_tokio_runtime().expect("Failed to build runtime"); -/// // runtime.block_on(async { /* ... */ }) -/// ``` -pub fn build_tokio_runtime() -> Result { - let mut builder = tokio_runtime_builder(); - - // Check if dial9 is enabled - if rustfs_obs::dial9::is_enabled() { - tracing::info!("Dial9 telemetry enabled, building TracedRuntime"); - - return match rustfs_obs::dial9::build_traced_runtime(builder) { - Ok((runtime, guard)) => { - // Store guard in global static to keep it alive for the program duration - let _ = DIAL9_TELEMETRY_GUARD.set(guard); - tracing::info!("TracedRuntime created successfully, guard stored globally"); - Ok(runtime) - } - Err(e) => { - tracing::warn!("Failed to build TracedRuntime: {}", e); - tracing::warn!("Falling back to standard Tokio runtime"); - // Rebuild the builder for standard runtime - let mut builder = tokio_runtime_builder(); - builder.build().map_err(BuildError::Runtime) - } - }; +/// Returns [`BuildError::Runtime`] if the Tokio runtime builder itself fails. +pub fn build_tokio_runtime() -> Result<(tokio::runtime::Runtime, Option), BuildError> { + if !rustfs_obs::dial9::is_enabled() { + let runtime = tokio_runtime_builder().build().map_err(BuildError::Runtime)?; + return Ok((runtime, None)); } - // Standard runtime - builder.build().map_err(BuildError::Runtime) + tracing::info!("Dial9 telemetry enabled, building TracedRuntime"); + match rustfs_obs::dial9::build_traced_runtime(tokio_runtime_builder()) { + Ok((runtime, guard)) => { + tracing::info!("TracedRuntime created and recording"); + Ok((runtime, Some(guard))) + } + Err(e) => { + tracing::warn!("Failed to build TracedRuntime: {e}; falling back to standard Tokio runtime"); + let runtime = tokio_runtime_builder().build().map_err(BuildError::Runtime)?; + Ok((runtime, None)) + } + } } /// Error type for runtime building failures. diff --git a/rustfs/src/startup_entrypoint.rs b/rustfs/src/startup_entrypoint.rs index f11fac12f..ac710031c 100644 --- a/rustfs/src/startup_entrypoint.rs +++ b/rustfs/src/startup_entrypoint.rs @@ -32,8 +32,14 @@ const OBSERVABILITY_INIT_FATAL_ALREADY_REPORTED: &str = "observability initializ pub fn run_process() { // Building the process runtime is a startup fatal boundary. - let runtime = crate::server::build_tokio_runtime().expect("Failed to build Tokio runtime"); + let (runtime, dial9_guard) = crate::server::build_tokio_runtime().expect("Failed to build Tokio runtime"); let result = runtime.block_on(async_main()); + + // Flush and seal the trace segment before any exit path. `process::exit` + // below does not run destructors, and neither does returning from `main` + // for a guard held in a `static`. + drop(dial9_guard); + if let Err(ref e) = result { if e.to_string() != OBSERVABILITY_INIT_FATAL_ALREADY_REPORTED { // Tracing may not be initialized when startup fails this early. diff --git a/rustfs/src/startup_runtime_hooks.rs b/rustfs/src/startup_runtime_hooks.rs index a472b34d9..90bc1d402 100644 --- a/rustfs/src/startup_runtime_hooks.rs +++ b/rustfs/src/startup_runtime_hooks.rs @@ -17,7 +17,7 @@ use crate::startup_runtime_sources; use rustls::crypto::aws_lc_rs::default_provider; use std::future::Future; use std::io::Result; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; const LOG_COMPONENT_EMBEDDED: &str = "embedded"; const LOG_COMPONENT_MAIN: &str = "main"; @@ -35,24 +35,39 @@ pub(crate) fn log_startup_runtime_diagnostics() { } fn log_dial9_runtime_status() { - if rustfs_obs::dial9::is_enabled() { - info!( + let supported = rustfs_obs::dial9::is_supported(); + let configured = rustfs_obs::dial9::is_configured(); + + match (supported, configured) { + (true, true) => info!( target: "rustfs::main", event = EVENT_DIAL9_RUNTIME_STATUS, component = LOG_COMPONENT_MAIN, subsystem = LOG_SUBSYSTEM_STARTUP, enabled = true, "Dial9 Tokio runtime telemetry is enabled" - ); - } else { - debug!( + ), + // Requested but unavailable. This is the case an operator most needs to + // see: the process otherwise runs to completion recording nothing. + (false, true) => warn!( target: "rustfs::main", event = EVENT_DIAL9_RUNTIME_STATUS, component = LOG_COMPONENT_MAIN, subsystem = LOG_SUBSYSTEM_STARTUP, enabled = false, + supported = false, + remedy = "rebuild with --features dial9 (see `make build-profiling`)", + "Dial9 Tokio runtime telemetry is configured but not compiled into this binary" + ), + (_, false) => debug!( + target: "rustfs::main", + event = EVENT_DIAL9_RUNTIME_STATUS, + component = LOG_COMPONENT_MAIN, + subsystem = LOG_SUBSYSTEM_STARTUP, + enabled = false, + supported, "Dial9 Tokio runtime telemetry is disabled" - ); + ), } } diff --git a/rustfs/tests/manual/test_dial9.rs b/rustfs/tests/manual/test_dial9.rs deleted file mode 100644 index bdfeda2c3..000000000 --- a/rustfs/tests/manual/test_dial9.rs +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Manual Dial9 integration runner. -// -// Run with: -// `cargo run -p rustfs --features manual-test-runners --bin manual-test-dial9` -// -// This file lives under `rustfs/tests/manual` and is registered explicitly in -// `rustfs/Cargo.toml` so it stays out of `cargo test` auto-discovery. -use rustfs_obs::dial9::{Dial9Config, Dial9SessionGuard}; -use tokio::time::{Duration, sleep}; - -#[tokio::main] -async fn main() -> Result<(), Box> { - println!("=== Dial9 Integration Test ===\n"); - - // Test 1: Check default dial9 configuration - println!("Test 1: Default configuration"); - let default_config = Dial9Config::default(); - println!(" default enabled: {}", default_config.enabled); - println!(" default output_dir: {}", default_config.output_dir); - println!(" default file_prefix: {}", default_config.file_prefix); - println!(" ✓ PASS: Default configuration loaded\n"); - - // Test 2: Create explicit dial9 configuration - println!("Test 2: Explicit dial9 configuration"); - let config = Dial9Config { - enabled: true, - output_dir: "/tmp/rustfs-test-telemetry".to_string(), - sampling_rate: 0.5, - ..Dial9Config::default() - }; - println!(" config.enabled: {}", config.enabled); - println!(" config.output_dir: {}", config.output_dir); - println!(" config.file_prefix: {}", config.file_prefix); - println!(" config.sampling_rate: {}", config.sampling_rate); - - assert!(config.enabled); - assert_eq!(config.output_dir, "/tmp/rustfs-test-telemetry"); - assert_eq!(config.sampling_rate, 0.5); - println!(" ✓ PASS: Configuration loaded correctly\n"); - - // Test 3: Initialize dial9 session - println!("Test 3: Initialize dial9 session"); - match Dial9SessionGuard::new(config.clone()).await { - Ok(Some(guard)) => { - println!(" Dial9 session initialized successfully"); - println!(" guard.is_active(): {}", guard.is_active()); - println!(" ✓ PASS: Session initialized\n"); - - // Test 4: Generate some async activity - println!("Test 4: Generate async activity for tracing"); - let handle = tokio::spawn(async { - for i in 1..=5 { - println!(" Task iteration {}", i); - sleep(Duration::from_millis(50)).await; - } - }); - handle.await?; - println!(" ✓ PASS: Async activity completed\n"); - - // Test 5: Session shutdown - println!("Test 5: Session cleanup"); - drop(guard); - println!(" ✓ PASS: Session cleaned up\n"); - } - Ok(None) => { - println!(" ⚠ SKIP: Dial9 session not created (configuration validation may have failed)\n"); - } - Err(e) => { - println!(" ✗ FAIL: {:?}", e); - return Err(e.into()); - } - } - - // Cleanup - if let Err(err) = tokio::fs::remove_dir_all(&config.output_dir).await { - println!(" ⚠ SKIP: Failed to remove output directory: {}", err); - } - - println!("=== All Tests Passed! ==="); - Ok(()) -} diff --git a/scripts/check_logging_guardrails.sh b/scripts/check_logging_guardrails.sh index 2c3462693..1d82e6dbf 100755 --- a/scripts/check_logging_guardrails.sh +++ b/scripts/check_logging_guardrails.sh @@ -94,7 +94,8 @@ checked_files=( "crates/protocols/src/swift/quota.rs" "crates/protocols/src/swift/symlink.rs" "crates/protocols/src/common/gateway.rs" - "crates/obs/src/telemetry/dial9.rs" + "crates/obs/src/telemetry/dial9/config.rs" + "crates/obs/src/telemetry/dial9/enabled.rs" "crates/obs/src/telemetry/local.rs" "crates/obs/src/metrics/scheduler.rs" "crates/obs/src/cleaner/core.rs" diff --git a/scripts/run.sh b/scripts/run.sh index 987d6afee..bcddc11e8 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -115,18 +115,24 @@ export RUSTFS_RUNTIME_GLOBAL_QUEUE_INTERVAL=31 # ============================================================================ # dial9 Tokio Runtime Telemetry Configuration # ============================================================================ -# dial9 provides low-overhead Tokio runtime-level telemetry for performance diagnostics. -# It captures events like PollStart/End, WorkerPark/Unpark, QueueSample, TaskSpawn. +# dial9 captures Tokio runtime-level events (poll start/end, worker park/unpark, +# task spawn/terminate) into binary trace segments. It sees executor-level faults +# — a long poll stalling a worker, a task that never yields — that request-level +# metrics and spans cannot. # -# Features: -# - CPU overhead < 5% (with sampling rate 1.0) -# - Automatic file rotation (configurable size and count) -# - Graceful degradation if initialization fails +# This is an on-demand profiler, not always-on telemetry: # -# Note: Disabled by default. Enable only when needed for runtime diagnostics. -# Note: Requires build flag --cfg tokio_unstable (set in .cargo/config.toml). +# - The stock binary does NOT support it. Build with `make build-profiling`, +# which enables the `dial9` feature and `--cfg tokio_unstable`. Setting the +# variables below on a stock binary logs a warning and changes nothing. +# - Trace segments are written continuously, and the oldest are deleted once +# MAX_FILE_SIZE * ROTATION_COUNT bytes are retained. Under a high poll rate +# that budget can wrap in minutes — size it against the window you need. +# - Turn it off again when the investigation is over. +# +# See docs/operations/dial9-runtime-profiling.md. -# Enable dial9 telemetry (default: false) +# Enable dial9 telemetry (default: false; requires a `make build-profiling` binary) #export RUSTFS_RUNTIME_DIAL9_ENABLED=true # Output directory for trace files (default: /var/log/rustfs/telemetry) @@ -138,33 +144,31 @@ export RUSTFS_RUNTIME_GLOBAL_QUEUE_INTERVAL=31 # Maximum trace file size in bytes (default: 104857600 = 100MB) #export RUSTFS_RUNTIME_DIAL9_MAX_FILE_SIZE=104857600 -# Number of rotated files to keep (default: 10) +# Number of rotated files to keep (default: 10). Total disk budget is +# MAX_FILE_SIZE * ROTATION_COUNT; older segments are evicted, not kept. #export RUSTFS_RUNTIME_DIAL9_ROTATION_COUNT=10 -# Sampling rate: 0.0 to 1.0 (default: 1.0 = 100% sampling) -# Lower values reduce CPU overhead. Recommended: 0.1-0.5 for production. -#export RUSTFS_RUNTIME_DIAL9_SAMPLING_RATE=1.0 +# Capture async backtraces of tasks that stall (default: false). +# Needs a binary built with the `dial9-taskdump` feature (Linux only). +#export RUSTFS_RUNTIME_DIAL9_TASK_DUMP_ENABLED=true +# Mean idle duration for task-dump Poisson sampling, in ms (default: 10) +#export RUSTFS_RUNTIME_DIAL9_TASK_DUMP_IDLE_THRESHOLD_MS=10 -# S3 upload settings (not yet implemented; reserved for future use): +# S3 upload is NOT available: dial9's uploader depends on a rustls-webpki with +# known CVEs, so the feature is not built. These are parsed and warned about, +# never honoured. Collect segments from OUTPUT_DIR instead. #export RUSTFS_RUNTIME_DIAL9_S3_BUCKET=my-trace-bucket #export RUSTFS_RUNTIME_DIAL9_S3_PREFIX=telemetry/ -# --- Scenario 1: Development / Debugging --- -# Full tracing with local storage, high sampling rate +# --- Scenario 1: local development --- #export RUSTFS_RUNTIME_DIAL9_ENABLED=true #export RUSTFS_RUNTIME_DIAL9_OUTPUT_DIR="$current_dir/deploy/telemetry" -#export RUSTFS_RUNTIME_DIAL9_SAMPLING_RATE=1.0 -# --- Scenario 2: Production Diagnostics --- -# Reduced sampling rate to minimize overhead -#export RUSTFS_RUNTIME_DIAL9_ENABLED=true -#export RUSTFS_RUNTIME_DIAL9_SAMPLING_RATE=0.1 - -# --- Scenario 3: Performance Investigation --- -# Short-term tracing with high detail, manual cleanup +# --- Scenario 2: investigating a worker stall --- +# Task dumps show where stalled tasks are parked. Keep the run short. #export RUSTFS_RUNTIME_DIAL9_ENABLED=true #export RUSTFS_RUNTIME_DIAL9_OUTPUT_DIR=/tmp/rustfs-telemetry-investigation -#export RUSTFS_RUNTIME_DIAL9_SAMPLING_RATE=1.0 +#export RUSTFS_RUNTIME_DIAL9_TASK_DUMP_ENABLED=true #export RUSTFS_RUNTIME_DIAL9_ROTATION_COUNT=3 export OTEL_INSTRUMENTATION_NAME="rustfs"