diff --git a/.config/make/build.mak b/.config/make/build.mak index 8c0638d2f..db3a38919 100644 --- a/.config/make/build.mak +++ b/.config/make/build.mak @@ -43,9 +43,8 @@ build-gnu-arm64: ## Build aarch64 GNU version # Setting RUSTFLAGS here replaces (never appends to) the config-file value, and # crates/obs/build.rs fails the build if the feature and the flag disagree. # -# DIAL9_FEATURES can add `dial9-taskdump` (async backtraces of stalled tasks; -# Linux x86_64/aarch64, and also needs --cfg tokio_taskdump). There is no S3 -# upload feature — see the note in crates/obs/Cargo.toml. +# There are no task-dump or S3-upload features — see the notes in +# crates/obs/Cargo.toml for why. DIAL9_FEATURES ?= dial9 DIAL9_RUSTFLAGS ?= --cfg tokio_unstable diff --git a/Cargo.lock b/Cargo.lock index b2722a925..445e4d4ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11617,7 +11617,6 @@ version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ - "backtrace", "bytes", "libc", "mio", diff --git a/crates/config/src/constants/runtime.rs b/crates/config/src/constants/runtime.rs index 287b86394..d63ad22d2 100644 --- a/crates/config/src/constants/runtime.rs +++ b/crates/config/src/constants/runtime.rs @@ -38,10 +38,11 @@ pub const ENV_RUNTIME_DIAL9_ROTATION_COUNT: &str = "RUSTFS_RUNTIME_DIAL9_ROTATIO pub const ENV_RUNTIME_DIAL9_S3_BUCKET: &str = "RUSTFS_RUNTIME_DIAL9_S3_BUCKET"; /// Accepted but not honoured; see [`ENV_RUNTIME_DIAL9_S3_BUCKET`]. pub const ENV_RUNTIME_DIAL9_S3_PREFIX: &str = "RUSTFS_RUNTIME_DIAL9_S3_PREFIX"; -/// Capture async backtraces for tasks that stall on a worker. -pub const ENV_RUNTIME_DIAL9_TASK_DUMP_ENABLED: &str = "RUSTFS_RUNTIME_DIAL9_TASK_DUMP_ENABLED"; -/// Mean idle duration for task-dump Poisson sampling, in milliseconds. -pub const ENV_RUNTIME_DIAL9_TASK_DUMP_IDLE_THRESHOLD_MS: &str = "RUSTFS_RUNTIME_DIAL9_TASK_DUMP_IDLE_THRESHOLD_MS"; +// Note: there are deliberately no task-dump knobs. dial9 only captures task +// dumps for futures spawned through `dial9_tokio_telemetry::spawn`, and RustFS +// spawns with `tokio::spawn` throughout, so the switch could never do anything. +// Measured: 0 dumps via tokio::spawn vs 14709 via dial9::spawn on an identical +// workload. See rustfs/backlog#1157 (D9-16). // Default values for Tokio runtime pub const DEFAULT_WORKER_THREADS: usize = 16; @@ -62,9 +63,6 @@ pub const DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR: &str = "/var/log/rustfs/telemetry"; pub const DEFAULT_RUNTIME_DIAL9_FILE_PREFIX: &str = "rustfs-tokio"; pub const DEFAULT_RUNTIME_DIAL9_MAX_FILE_SIZE: u64 = 100 * 1024 * 1024; // 100MB pub const DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT: usize = 10; -pub const DEFAULT_RUNTIME_DIAL9_TASK_DUMP_ENABLED: bool = false; -/// Matches dial9's own default mean idle duration for task-dump sampling. -pub const DEFAULT_RUNTIME_DIAL9_TASK_DUMP_IDLE_THRESHOLD_MS: u64 = 10; // Note: S3 bucket/prefix have no default; absence means upload is disabled (modeled as Option) /// Maximum transition workers used as a local fallback when runtime env is unset. diff --git a/crates/obs/Cargo.toml b/crates/obs/Cargo.toml index 1b49a0a69..281cadff8 100644 --- a/crates/obs/Cargo.toml +++ b/crates/obs/Cargo.toml @@ -31,11 +31,15 @@ default = [] # build script fails the compile when that flag is missing. Off by default so # ordinary builds neither pay for nor depend on Tokio's unstable API. dial9 = ["dep:dial9-tokio-telemetry"] -# Capture async backtraces of stalled tasks. Enables `tokio/taskdump`, which -# hard-errors at compile time on anything other than linux/{aarch64,x86,x86_64}, -# and additionally needs `--cfg tokio_taskdump` to record anything. Linux-only, -# and not a silent no-op elsewhere: enabling it on macOS fails the build. -dial9-taskdump = ["dial9", "dial9-tokio-telemetry/taskdump"] +# +# NOTE: there is deliberately no `dial9-taskdump` feature. dial9 only captures a +# task dump for futures it wrapped itself, i.e. those spawned via +# `dial9_tokio_telemetry::spawn`. RustFS spawns with `tokio::spawn` throughout, +# so enabling `tokio/taskdump` would cost a Linux-only build constraint and +# record nothing. Measured on an identical workload: 0 dumps via `tokio::spawn`, +# 14709 via `dial9::spawn`. Re-adding this feature only makes sense together +# with migrating the paths under investigation to dial9's spawner. +# See rustfs/backlog#1157 (D9-16) and dial9-rs/dial9#477. # # NOTE: there is deliberately no `dial9-s3` feature. dial9's `worker-s3` feature # pulls aws-sdk-s3-transfer-manager 0.1.3 (its latest), which pins diff --git a/crates/obs/src/telemetry/dial9/config.rs b/crates/obs/src/telemetry/dial9/config.rs index a044d0948..9f2057e3f 100644 --- a/crates/obs/src/telemetry/dial9/config.rs +++ b/crates/obs/src/telemetry/dial9/config.rs @@ -22,14 +22,12 @@ use super::state::dial9_runtime_state; use rustfs_config::{ DEFAULT_RUNTIME_DIAL9_ENABLED, DEFAULT_RUNTIME_DIAL9_FILE_PREFIX, DEFAULT_RUNTIME_DIAL9_MAX_FILE_SIZE, - DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR, DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT, DEFAULT_RUNTIME_DIAL9_TASK_DUMP_ENABLED, - DEFAULT_RUNTIME_DIAL9_TASK_DUMP_IDLE_THRESHOLD_MS, ENV_RUNTIME_DIAL9_ENABLED, ENV_RUNTIME_DIAL9_FILE_PREFIX, - ENV_RUNTIME_DIAL9_MAX_FILE_SIZE, ENV_RUNTIME_DIAL9_OUTPUT_DIR, ENV_RUNTIME_DIAL9_ROTATION_COUNT, ENV_RUNTIME_DIAL9_S3_BUCKET, - ENV_RUNTIME_DIAL9_S3_PREFIX, ENV_RUNTIME_DIAL9_TASK_DUMP_ENABLED, ENV_RUNTIME_DIAL9_TASK_DUMP_IDLE_THRESHOLD_MS, + DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR, DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT, ENV_RUNTIME_DIAL9_ENABLED, + ENV_RUNTIME_DIAL9_FILE_PREFIX, ENV_RUNTIME_DIAL9_MAX_FILE_SIZE, ENV_RUNTIME_DIAL9_OUTPUT_DIR, + ENV_RUNTIME_DIAL9_ROTATION_COUNT, ENV_RUNTIME_DIAL9_S3_BUCKET, ENV_RUNTIME_DIAL9_S3_PREFIX, }; use rustfs_utils::{get_env_bool, get_env_opt_str, get_env_opt_u64, get_env_opt_usize, get_env_str}; use std::path::PathBuf; -use std::time::Duration; use tracing::warn; use super::{EVENT_DIAL9_STATE, LOG_COMPONENT_OBS, LOG_SUBSYSTEM_DIAL9}; @@ -92,12 +90,6 @@ pub struct Dial9Config { /// Optional key prefix for uploaded segments pub s3_prefix: Option, - - /// Whether to capture async backtraces for tasks that stall - pub task_dump_enabled: bool, - - /// Mean idle duration for task-dump Poisson sampling - pub task_dump_idle_threshold: Duration, } impl Default for Dial9Config { @@ -110,8 +102,6 @@ impl Default for Dial9Config { rotation_count: DEFAULT_RUNTIME_DIAL9_ROTATION_COUNT, s3_bucket: None, s3_prefix: None, - task_dump_enabled: DEFAULT_RUNTIME_DIAL9_TASK_DUMP_ENABLED, - task_dump_idle_threshold: Duration::from_millis(DEFAULT_RUNTIME_DIAL9_TASK_DUMP_IDLE_THRESHOLD_MS), } } } @@ -165,12 +155,6 @@ impl Dial9Config { rotation_count, s3_bucket, s3_prefix, - task_dump_enabled: get_env_bool(ENV_RUNTIME_DIAL9_TASK_DUMP_ENABLED, DEFAULT_RUNTIME_DIAL9_TASK_DUMP_ENABLED), - task_dump_idle_threshold: Duration::from_millis( - get_env_opt_u64(ENV_RUNTIME_DIAL9_TASK_DUMP_IDLE_THRESHOLD_MS) - .filter(|ms| *ms > 0) - .unwrap_or(DEFAULT_RUNTIME_DIAL9_TASK_DUMP_IDLE_THRESHOLD_MS), - ), }; dial9_runtime_state().record_config(&config); diff --git a/crates/obs/src/telemetry/dial9/enabled.rs b/crates/obs/src/telemetry/dial9/enabled.rs index e5d286c00..6d1cd40cb 100644 --- a/crates/obs/src/telemetry/dial9/enabled.rs +++ b/crates/obs/src/telemetry/dial9/enabled.rs @@ -23,7 +23,7 @@ use super::config::Dial9Config; use super::state::{dial9_runtime_state, measure_disk_usage_bytes}; use super::{EVENT_DIAL9_STATE, LOG_COMPONENT_OBS, LOG_SUBSYSTEM_DIAL9}; use crate::TelemetryError; -use dial9_tokio_telemetry::telemetry::{ProcessResourceUsageConfig, RotatingWriter, TaskDumpConfig, TracedRuntime}; +use dial9_tokio_telemetry::telemetry::{ProcessResourceUsageConfig, RotatingWriter, TracedRuntime}; use std::time::Duration; use tracing::{info, warn}; @@ -115,19 +115,16 @@ pub fn build_traced_runtime( .with_runtime_name(RUNTIME_NAME) .with_process_resource_usage(ProcessResourceUsageConfig::default()); - let traced = if config.task_dump_enabled { - traced.with_task_dumps( - TaskDumpConfig::builder() - .idle_threshold(config.task_dump_idle_threshold) - .build(), - ) - } else { - traced - }; - // `build_and_start` rather than `build`: `build` returns a live guard that // never records, writing segments that contain only a header. // + // No `with_task_dumps` here. dial9 captures a task dump only for futures it + // wrapped itself, i.e. those spawned via `dial9_tokio_telemetry::spawn`; + // `tokio::spawn` gets no wrapper. RustFS spawns with `tokio::spawn` + // throughout, so calling `with_task_dumps` records nothing. Measured on an + // identical workload: 0 dumps via `tokio::spawn`, 14709 via `dial9::spawn`. + // See rustfs/backlog#1157 (D9-16) and dial9-rs/dial9#477. + // // No `with_s3_uploader` here: dial9's `worker-s3` feature carries a // vulnerable TLS stack. See the note in `crates/obs/Cargo.toml`. finish_traced_runtime(traced.build_and_start(builder, writer), config) @@ -165,7 +162,6 @@ fn finish_traced_runtime( output_dir = %config.output_dir, file_prefix = %config.file_prefix, disk_budget_bytes = config.total_disk_budget(), - task_dumps = config.task_dump_enabled, "dial9 state changed" ); diff --git a/crates/obs/src/telemetry/dial9/mod.rs b/crates/obs/src/telemetry/dial9/mod.rs index ca30c729a..08c294c33 100644 --- a/crates/obs/src/telemetry/dial9/mod.rs +++ b/crates/obs/src/telemetry/dial9/mod.rs @@ -31,8 +31,21 @@ //! //! Trace segments are written to disk continuously and evicted oldest-first //! once `RUSTFS_RUNTIME_DIAL9_MAX_FILE_SIZE * RUSTFS_RUNTIME_DIAL9_ROTATION_COUNT` -//! bytes are retained. Under a high poll rate the budget can wrap in minutes, -//! so size it against the window you need to capture. +//! bytes are retained. Measured at ~0.16 MiB/s (13k events/s) under a 66 MiB/s +//! warp workload, so the default 1 GiB budget wraps after roughly 108 minutes. +//! Scale that by your own event rate before relying on a long capture. +//! +//! # What it does not see +//! +//! A drive stall does not show up here. RustFS performs disk I/O on the blocking +//! pool and through io_uring, never on an async worker, so a slow drive does not +//! lengthen any poll. Injecting 200 ms of drive latency cut throughput by 64% +//! and left the poll-duration distribution unchanged. Use the `rustfs_io_*` +//! metrics and the drive-stall budget for that; dial9 answers a different +//! question — which task held a worker, and for how long. +//! +//! Task dumps would answer *where* it was stuck, but dial9 only captures them +//! for futures spawned through its own `spawn`. See rustfs/backlog#1157 (D9-16). //! //! # Known observability gap //! @@ -43,6 +56,7 @@ //! is therefore no `writer_healthy` metric: it could only ever be hard-coded to //! `1`. Watch `rustfs_dial9_disk_usage_bytes` — a session that is recording but //! whose disk usage stops growing has most likely hit this state. +//! Reported upstream as dial9-rs/dial9#658. mod config; mod state; diff --git a/docs/operations/dial9-runtime-profiling.md b/docs/operations/dial9-runtime-profiling.md index 2a15a9057..eff939932 100644 --- a/docs/operations/dial9-runtime-profiling.md +++ b/docs/operations/dial9-runtime-profiling.md @@ -6,12 +6,29 @@ stalled tasks — into binary trace segments. It answers questions that Prometheus metrics and `tracing` spans cannot: -- A request took 200 ms. Was a worker thread blocked, or was it waiting on I/O? - Which task held a worker for 40 ms without yielding? - Are workers parking because there is no work, or because the queue is starved? -These are the failure modes behind drive stalls, `io_uring`/`O_DIRECT` regressions, -and object-data-cache fill contention. They are invisible to the rest of the obs stack. +On a plain warp workload with no injected fault it recorded single polls of +418–625 ms: real worker stalls that nothing else in the obs stack would surface. + +## What it does not see + +**A drive stall is invisible to dial9.** RustFS performs disk I/O on the blocking +pool (`spawn_blocking`) and through io_uring, never on an async worker, so a slow +drive does not lengthen any poll. Injecting 200 ms of latency on one of four +drives cut throughput by 64% and left the poll-duration distribution unchanged +(polls ≥ 5 ms: 49 → 56; p999: 2.67 ms → 2.75 ms). Enabling dial9's CPU and sched +profilers does not help either: sched events are captured per-worker only, and +the CPU profiler samples on-CPU, while a stalled drive is an off-CPU wait. + +For drive stalls use the `rustfs_io_*` metrics and the drive-stall budget. dial9 +answers a different question: which task held a worker, and for how long. + +**It cannot tell you where a task was stuck.** That would need a task dump, and +dial9 only captures those for futures spawned through `dial9_tokio_telemetry::spawn`. +RustFS spawns with `tokio::spawn` throughout, so no task dump is ever recorded and +no configuration exposes one. Tracked as D9-16 in rustfs/backlog#1157. ## This is a profiler, not telemetry @@ -23,10 +40,11 @@ depend on Tokio's non-semver surface nor pay its cost. Setting `RUSTFS_RUNTIME_DIAL9_ENABLED=true` on a stock binary logs a warning and records nothing. **Traces are written continuously and evicted.** The retained budget is -`MAX_FILE_SIZE × ROTATION_COUNT` (1 GB by default). Once exceeded, the *oldest* -segments are deleted. Under a high poll rate that budget can wrap in minutes, -which means the incident you are chasing may already have been overwritten. -Size the budget against the window you need, and prefer short, targeted runs. +`MAX_FILE_SIZE × ROTATION_COUNT` (1 GiB by default). Once exceeded, the *oldest* +segments are deleted. Measured on a single-node 4-drive cluster under warp mixed +(66 MiB/s, 110 obj/s, 32 concurrent): **13023 events/s, 0.16 MiB/s**, so the +default budget wraps after roughly **108 minutes**. Scale that by your own event +rate — it tracks poll rate, not object throughput — and prefer short, targeted runs. **There is no runtime toggle.** Telemetry is installed when the Tokio runtime is constructed, so enabling or disabling it requires a process restart. @@ -38,15 +56,8 @@ make build-profiling ``` That is `cargo build --release --bin rustfs --features dial9` with -`RUSTFLAGS="--cfg tokio_unstable"`. One optional feature layers on top: - -```bash -# Async backtraces of stalled tasks. Linux aarch64/x86/x86_64 only — -# `tokio/taskdump` hard-errors at compile time on other targets, so this -# will not build on macOS. -DIAL9_RUSTFLAGS="--cfg tokio_unstable --cfg tokio_taskdump" \ - DIAL9_FEATURES="dial9-taskdump" make build-profiling -``` +`RUSTFLAGS="--cfg tokio_unstable"`. There are no other telemetry features: task +dumps and S3 upload are both unavailable, for the reasons given above and below. `crates/obs/build.rs` fails the build if the `dial9` feature is enabled without `--cfg tokio_unstable`. This is deliberate: an environment `RUSTFLAGS` *replaces* @@ -60,8 +71,7 @@ For CPU profiling with usable stacks, add `-C force-frame-pointers=yes`. ```bash export RUSTFS_RUNTIME_DIAL9_ENABLED=true export RUSTFS_RUNTIME_DIAL9_OUTPUT_DIR=/tmp/rustfs-telemetry-investigation -export RUSTFS_RUNTIME_DIAL9_ROTATION_COUNT=3 # 300 MB total, ~short window -export RUSTFS_RUNTIME_DIAL9_TASK_DUMP_ENABLED=true # if built with dial9-taskdump +export RUSTFS_RUNTIME_DIAL9_ROTATION_COUNT=3 # 300 MiB total, ~short window ``` Reproduce the fault, then stop the process **gracefully**. The final buffered @@ -82,8 +92,6 @@ Turn `RUSTFS_RUNTIME_DIAL9_ENABLED` back off when you are done. | `RUSTFS_RUNTIME_DIAL9_FILE_PREFIX` | `rustfs-tokio` | | | `RUSTFS_RUNTIME_DIAL9_MAX_FILE_SIZE` | `104857600` | Bytes per segment | | `RUSTFS_RUNTIME_DIAL9_ROTATION_COUNT` | `10` | Total budget = size × count | -| `RUSTFS_RUNTIME_DIAL9_TASK_DUMP_ENABLED` | `false` | Needs `dial9-taskdump` (Linux only) | -| `RUSTFS_RUNTIME_DIAL9_TASK_DUMP_IDLE_THRESHOLD_MS` | `10` | Mean idle for Poisson sampling | | `RUSTFS_RUNTIME_DIAL9_S3_BUCKET` | unset | **Not honoured**, see below | | `RUSTFS_RUNTIME_DIAL9_S3_PREFIX` | unset | **Not honoured**, see below | @@ -101,7 +109,8 @@ Cargo's feature unification can add features but cannot drop a transitive dependency, so this cannot be worked around downstream — it needs an upstream release. RustFS therefore builds no S3 uploader at all. Setting the two variables above logs a warning and changes nothing; retrieve trace segments from -`OUTPUT_DIR` directly. Tracked as D9-14 in rustfs/backlog#1157. +`OUTPUT_DIR` directly. Tracked as D9-14 in rustfs/backlog#1157 and reported +upstream as [dial9-rs/dial9#659](https://github.com/dial9-rs/dial9/issues/659). ## Metrics @@ -136,3 +145,5 @@ There is therefore no `writer_healthy` gauge, because it could only ever be hard-coded to `1`. Instead, watch `rustfs_dial9_disk_usage_bytes`: a session with `active_sessions=1` whose disk usage has stopped growing has most likely hit this state, and needs a restart. + +Reported upstream as [dial9-rs/dial9#658](https://github.com/dial9-rs/dial9/issues/658). diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index bb41235ed..fbe1c748e 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -56,7 +56,6 @@ rio-v2 = ["rustfs-ecstore/rio-v2"] pyroscope = ["rustfs-obs/pyroscope"] # Tokio runtime telemetry. Requires `--cfg tokio_unstable`; use `make build-profiling`. dial9 = ["rustfs-obs/dial9"] -dial9-taskdump = ["dial9", "rustfs-obs/dial9-taskdump"] hotpath = [ "dep:hotpath", "hotpath/hotpath", diff --git a/scripts/run.sh b/scripts/run.sh index bcddc11e8..55c6a3d6b 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -116,9 +116,13 @@ export RUSTFS_RUNTIME_GLOBAL_QUEUE_INTERVAL=31 # dial9 Tokio Runtime Telemetry Configuration # ============================================================================ # dial9 captures Tokio runtime-level events (poll start/end, worker park/unpark, -# task spawn/terminate) into binary trace segments. It sees executor-level faults -# — a long poll stalling a worker, a task that never yields — that request-level -# metrics and spans cannot. +# task spawn/terminate) into binary trace segments. It sees long polls that stall +# a worker — something request-level metrics and spans cannot show. +# +# It does NOT see a drive stall: RustFS does its disk I/O on the blocking pool +# and via io_uring, never on an async worker, so a slow drive does not lengthen +# any poll. Measured: injecting 200ms of drive latency cut throughput by 64% and +# left the poll-duration distribution unchanged. # # This is an on-demand profiler, not always-on telemetry: # @@ -126,8 +130,9 @@ export RUSTFS_RUNTIME_GLOBAL_QUEUE_INTERVAL=31 # which enables the `dial9` feature and `--cfg tokio_unstable`. Setting the # variables below on a stock binary logs a warning and changes nothing. # - Trace segments are written continuously, and the oldest are deleted once -# MAX_FILE_SIZE * ROTATION_COUNT bytes are retained. Under a high poll rate -# that budget can wrap in minutes — size it against the window you need. +# MAX_FILE_SIZE * ROTATION_COUNT bytes are retained. Measured at ~0.16 MiB/s +# under a 66 MiB/s warp workload, so the default 1 GiB budget wraps after +# roughly 108 minutes. Scale that by your event rate. # - Turn it off again when the investigation is over. # # See docs/operations/dial9-runtime-profiling.md. @@ -148,12 +153,6 @@ export RUSTFS_RUNTIME_GLOBAL_QUEUE_INTERVAL=31 # MAX_FILE_SIZE * ROTATION_COUNT; older segments are evicted, not kept. #export RUSTFS_RUNTIME_DIAL9_ROTATION_COUNT=10 -# Capture async backtraces of tasks that stall (default: false). -# Needs a binary built with the `dial9-taskdump` feature (Linux only). -#export RUSTFS_RUNTIME_DIAL9_TASK_DUMP_ENABLED=true -# Mean idle duration for task-dump Poisson sampling, in ms (default: 10) -#export RUSTFS_RUNTIME_DIAL9_TASK_DUMP_IDLE_THRESHOLD_MS=10 - # S3 upload is NOT available: dial9's uploader depends on a rustls-webpki with # known CVEs, so the feature is not built. These are parsed and warned about, # never honoured. Collect segments from OUTPUT_DIR instead. @@ -165,10 +164,10 @@ export RUSTFS_RUNTIME_GLOBAL_QUEUE_INTERVAL=31 #export RUSTFS_RUNTIME_DIAL9_OUTPUT_DIR="$current_dir/deploy/telemetry" # --- Scenario 2: investigating a worker stall --- -# Task dumps show where stalled tasks are parked. Keep the run short. +# Trace shows which task held a worker and for how long, but not where it was +# stuck: task dumps need dial9's own spawner (see backlog#1157 D9-16). #export RUSTFS_RUNTIME_DIAL9_ENABLED=true #export RUSTFS_RUNTIME_DIAL9_OUTPUT_DIR=/tmp/rustfs-telemetry-investigation -#export RUSTFS_RUNTIME_DIAL9_TASK_DUMP_ENABLED=true #export RUSTFS_RUNTIME_DIAL9_ROTATION_COUNT=3 export OTEL_INSTRUMENTATION_NAME="rustfs"