From be22175035fd176bb94c6806070072f24a872391 Mon Sep 17 00:00:00 2001 From: Henry Guo Date: Wed, 26 Aug 2026 10:03:34 +0800 Subject: [PATCH] fix(capacity): honor high-latency timeout profile (#6611) Co-authored-by: Henry Guo --- crates/config/README.md | 3 +- crates/object-capacity/README.md | 7 +- .../object-capacity/src/capacity_manager.rs | 116 +++++++++++++++--- docs/operations/drive-timeout-tuning.md | 2 + 4 files changed, 106 insertions(+), 22 deletions(-) diff --git a/crates/config/README.md b/crates/config/README.md index 338b85dbf..c450c05b0 100644 --- a/crates/config/README.md +++ b/crates/config/README.md @@ -150,9 +150,10 @@ Drive timeout health-action policy: Drive timeout profile preset: - `RUSTFS_DRIVE_TIMEOUT_PROFILE` - `default` (default): keep current timeout defaults. - - `high_latency`: use 60s default timeout for scanner-sensitive operations when no per-operation timeout override is set (`read_metadata`, `disk_info`, `list_dir`, `walk_dir`, `walk_dir_stall`). + - `high_latency`: use 60s default timeout for scanner-sensitive operations when no operation-specific override is set (`read_metadata`, `disk_info`, `list_dir`, `walk_dir`, `walk_dir_stall`, and object-capacity scan base/maximum budgets). - Precedence: - Explicit per-operation timeout env (`RUSTFS_DRIVE_*_TIMEOUT_SECS`) takes highest precedence. + - Explicit object-capacity timeout env (`RUSTFS_CAPACITY_STAT_TIMEOUT`, `RUSTFS_CAPACITY_MAX_TIMEOUT`) takes precedence for capacity scans. - Then `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION` legacy fallback. - Then the profile-derived default (`default` or `high_latency`). diff --git a/crates/object-capacity/README.md b/crates/object-capacity/README.md index 4762d4ae6..4a32dd24c 100644 --- a/crates/object-capacity/README.md +++ b/crates/object-capacity/README.md @@ -287,15 +287,18 @@ The configuration constants are defined in `crates/config/src/constants/capacity | `RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD` | `5` | Recent 60-second write-frequency threshold | | `RUSTFS_CAPACITY_FAST_UPDATE_THRESHOLD` | `30s` | Cache age required before fast refresh is considered | | `RUSTFS_CAPACITY_MAX_FILES_THRESHOLD` | `200000` | Exact-count file threshold | -| `RUSTFS_CAPACITY_STAT_TIMEOUT` | `3s` | Base scan timeout | +| `RUSTFS_CAPACITY_STAT_TIMEOUT` | `3s` (`60s` with `RUSTFS_DRIVE_TIMEOUT_PROFILE=high_latency`) | Base scan timeout | | `RUSTFS_CAPACITY_SAMPLE_RATE` | `200` | Overflow-file sampling interval | | `RUSTFS_CAPACITY_METRICS_INTERVAL` | `600s` | Runtime summary emission interval | | `RUSTFS_CAPACITY_FOLLOW_SYMLINKS` | `false` | Whether to follow symlinks | | `RUSTFS_CAPACITY_ENABLE_DYNAMIC_TIMEOUT` | `true` | Whether to enable dynamic timeout scaling | | `RUSTFS_CAPACITY_MIN_TIMEOUT` | `2s` | Dynamic-timeout lower bound | -| `RUSTFS_CAPACITY_MAX_TIMEOUT` | `15s` | Dynamic-timeout upper bound | +| `RUSTFS_CAPACITY_MAX_TIMEOUT` | `15s` (`60s` with `RUSTFS_DRIVE_TIMEOUT_PROFILE=high_latency`) | Dynamic-timeout upper bound | | `RUSTFS_CAPACITY_STALL_TIMEOUT` | `20s` | Stall-detection threshold | +Explicit `RUSTFS_CAPACITY_STAT_TIMEOUT` and `RUSTFS_CAPACITY_MAX_TIMEOUT` values +take precedence over the drive-timeout profile. + ### Configuration-Caching Note In non-test builds, configuration is cached behind `OnceLock`: diff --git a/crates/object-capacity/src/capacity_manager.rs b/crates/object-capacity/src/capacity_manager.rs index 8d61ba90c..207a18fe8 100644 --- a/crates/object-capacity/src/capacity_manager.rs +++ b/crates/object-capacity/src/capacity_manager.rs @@ -20,19 +20,21 @@ use crate::capacity_scope::{CapacityScope, CapacityScopeDisk, drain_global_dirty use futures::FutureExt; use rustfs_config::{ DEFAULT_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, DEFAULT_CAPACITY_FOLLOW_SYMLINKS, DEFAULT_CAPACITY_MAX_TIMEOUT_SECS, - DEFAULT_CAPACITY_METRICS_INTERVAL_SECS, DEFAULT_CAPACITY_MIN_TIMEOUT_SECS, DEFAULT_FAST_UPDATE_THRESHOLD_SECS, - DEFAULT_MAX_FILES_THRESHOLD, DEFAULT_SAMPLE_RATE, DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS, DEFAULT_STAT_TIMEOUT_SECS, - DEFAULT_WRITE_FREQUENCY_THRESHOLD, DEFAULT_WRITE_TRIGGER_DELAY_SECS, ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, + DEFAULT_CAPACITY_METRICS_INTERVAL_SECS, DEFAULT_CAPACITY_MIN_TIMEOUT_SECS, DEFAULT_DRIVE_TIMEOUT_PROFILE, + DEFAULT_FAST_UPDATE_THRESHOLD_SECS, DEFAULT_MAX_FILES_THRESHOLD, DEFAULT_SAMPLE_RATE, DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS, + DEFAULT_STAT_TIMEOUT_SECS, DEFAULT_WRITE_FREQUENCY_THRESHOLD, DEFAULT_WRITE_TRIGGER_DELAY_SECS, + DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY, DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS, ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, ENV_CAPACITY_FAST_UPDATE_THRESHOLD, ENV_CAPACITY_FOLLOW_SYMLINKS, ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_MAX_TIMEOUT, ENV_CAPACITY_METRICS_INTERVAL, ENV_CAPACITY_MIN_TIMEOUT, ENV_CAPACITY_SAMPLE_RATE, ENV_CAPACITY_SCHEDULED_INTERVAL, ENV_CAPACITY_STAT_TIMEOUT, ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD, ENV_CAPACITY_WRITE_TRIGGER_DELAY, + ENV_DRIVE_TIMEOUT_PROFILE, }; use rustfs_io_metrics::capacity_metrics::{ record_capacity_current_bytes, record_capacity_degraded_reading, record_capacity_dirty_disk_count, record_capacity_refresh_inflight, record_capacity_refresh_joiner, record_capacity_refresh_result, record_capacity_update_completed, record_capacity_update_failed, record_capacity_write_operation, }; -use rustfs_utils::{get_env_bool, get_env_u64, get_env_usize}; +use rustfs_utils::{get_env_bool, get_env_str, get_env_u64, get_env_usize}; use std::collections::{HashMap, HashSet}; use std::future::Future; use std::panic::AssertUnwindSafe; @@ -112,6 +114,15 @@ fn env_u64_at_least(env: &'static str, default: u64, min: u64) -> u64 { } } +fn capacity_timeout_profile_default(default: u64) -> u64 { + let profile = get_env_str(ENV_DRIVE_TIMEOUT_PROFILE, DEFAULT_DRIVE_TIMEOUT_PROFILE); + if profile.trim().eq_ignore_ascii_case(DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY) { + DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS + } else { + default + } +} + impl CachedCapacityConfig { /// Build configuration from environment variables fn from_env() -> Self { @@ -131,7 +142,11 @@ impl CachedCapacityConfig { )), max_files_threshold: env_u64_at_least(ENV_CAPACITY_MAX_FILES_THRESHOLD, DEFAULT_MAX_FILES_THRESHOLD as u64, 1) as usize, - stat_timeout: Duration::from_secs(env_u64_at_least(ENV_CAPACITY_STAT_TIMEOUT, DEFAULT_STAT_TIMEOUT_SECS, 1)), + stat_timeout: Duration::from_secs(env_u64_at_least( + ENV_CAPACITY_STAT_TIMEOUT, + capacity_timeout_profile_default(DEFAULT_STAT_TIMEOUT_SECS), + 1, + )), sample_rate: get_env_usize(ENV_CAPACITY_SAMPLE_RATE, DEFAULT_SAMPLE_RATE), metrics_interval: Duration::from_secs(get_env_u64( ENV_CAPACITY_METRICS_INTERVAL, @@ -140,7 +155,11 @@ impl CachedCapacityConfig { follow_symlinks: get_env_bool(ENV_CAPACITY_FOLLOW_SYMLINKS, DEFAULT_CAPACITY_FOLLOW_SYMLINKS), enable_dynamic_timeout: get_env_bool(ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, DEFAULT_CAPACITY_ENABLE_DYNAMIC_TIMEOUT), min_timeout: Duration::from_secs(env_u64_at_least(ENV_CAPACITY_MIN_TIMEOUT, DEFAULT_CAPACITY_MIN_TIMEOUT_SECS, 1)), - max_timeout: Duration::from_secs(env_u64_at_least(ENV_CAPACITY_MAX_TIMEOUT, DEFAULT_CAPACITY_MAX_TIMEOUT_SECS, 1)), + max_timeout: Duration::from_secs(env_u64_at_least( + ENV_CAPACITY_MAX_TIMEOUT, + capacity_timeout_profile_default(DEFAULT_CAPACITY_MAX_TIMEOUT_SECS), + 1, + )), } } } @@ -1497,9 +1516,10 @@ mod tests { use super::*; use crate::capacity_scope::{CapacityScope, CapacityScopeDisk, record_capacity_scope, record_global_dirty_scope}; use rustfs_config::{ - ENV_CAPACITY_FAST_UPDATE_THRESHOLD, ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_METRICS_INTERVAL, - ENV_CAPACITY_SAMPLE_RATE, ENV_CAPACITY_STAT_TIMEOUT, ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD, - ENV_CAPACITY_WRITE_TRIGGER_DELAY, + DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY, ENV_CAPACITY_FAST_UPDATE_THRESHOLD, ENV_CAPACITY_MAX_FILES_THRESHOLD, + ENV_CAPACITY_MAX_TIMEOUT, ENV_CAPACITY_METRICS_INTERVAL, ENV_CAPACITY_MIN_TIMEOUT, ENV_CAPACITY_SAMPLE_RATE, + ENV_CAPACITY_STAT_TIMEOUT, ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD, ENV_CAPACITY_WRITE_TRIGGER_DELAY, + ENV_DRIVE_TIMEOUT_PROFILE, }; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -1719,11 +1739,13 @@ mod tests { #[test] fn test_config_getter_defaults() { - for (env_var, getter, default, _, _) in config_getter_cases() { - temp_env::with_var(env_var, None::<&str>, || { - assert_eq!(getter(), default, "{env_var}: unexpected default value"); - }); - } + temp_env::with_var_unset(ENV_DRIVE_TIMEOUT_PROFILE, || { + for (env_var, getter, default, _, _) in config_getter_cases() { + temp_env::with_var(env_var, None::<&str>, || { + assert_eq!(getter(), default, "{env_var}: unexpected default value"); + }); + } + }); } #[test] @@ -1747,11 +1769,67 @@ mod tests { (ENV_CAPACITY_MIN_TIMEOUT, || get_min_timeout().as_secs(), 2), (ENV_CAPACITY_MAX_TIMEOUT, || get_max_timeout().as_secs(), 15), ]; - for (env_var, getter, default) in zero_cases { - temp_env::with_var(env_var, Some("0"), || { - assert_eq!(getter(), default, "{env_var}: zero must clamp to default"); - }); - } + temp_env::with_var_unset(ENV_DRIVE_TIMEOUT_PROFILE, || { + for (env_var, getter, default) in zero_cases { + temp_env::with_var(env_var, Some("0"), || { + assert_eq!(getter(), default, "{env_var}: zero must clamp to default"); + }); + } + }); + } + + #[test] + fn capacity_timeouts_use_high_latency_drive_profile_defaults() { + temp_env::with_vars( + [ + (ENV_DRIVE_TIMEOUT_PROFILE, Some(DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY)), + (ENV_CAPACITY_STAT_TIMEOUT, None), + (ENV_CAPACITY_MIN_TIMEOUT, None), + (ENV_CAPACITY_MAX_TIMEOUT, None), + ], + || { + let config = CachedCapacityConfig::from_env(); + assert_eq!(config.stat_timeout, Duration::from_secs(60)); + assert_eq!(config.min_timeout, Duration::from_secs(2)); + assert_eq!(config.max_timeout, Duration::from_secs(60)); + }, + ); + } + + #[test] + fn explicit_capacity_timeouts_override_high_latency_drive_profile() { + temp_env::with_vars( + [ + (ENV_DRIVE_TIMEOUT_PROFILE, Some(DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY)), + (ENV_CAPACITY_STAT_TIMEOUT, Some("7")), + (ENV_CAPACITY_MIN_TIMEOUT, Some("4")), + (ENV_CAPACITY_MAX_TIMEOUT, Some("11")), + ], + || { + let config = CachedCapacityConfig::from_env(); + assert_eq!(config.stat_timeout, Duration::from_secs(7)); + assert_eq!(config.min_timeout, Duration::from_secs(4)); + assert_eq!(config.max_timeout, Duration::from_secs(11)); + }, + ); + } + + #[test] + fn invalid_drive_profile_preserves_capacity_defaults() { + temp_env::with_vars( + [ + (ENV_DRIVE_TIMEOUT_PROFILE, Some("invalid")), + (ENV_CAPACITY_STAT_TIMEOUT, None), + (ENV_CAPACITY_MIN_TIMEOUT, None), + (ENV_CAPACITY_MAX_TIMEOUT, None), + ], + || { + let config = CachedCapacityConfig::from_env(); + assert_eq!(config.stat_timeout, Duration::from_secs(DEFAULT_STAT_TIMEOUT_SECS)); + assert_eq!(config.min_timeout, Duration::from_secs(DEFAULT_CAPACITY_MIN_TIMEOUT_SECS)); + assert_eq!(config.max_timeout, Duration::from_secs(DEFAULT_CAPACITY_MAX_TIMEOUT_SECS)); + }, + ); } #[tokio::test] diff --git a/docs/operations/drive-timeout-tuning.md b/docs/operations/drive-timeout-tuning.md index 1901dfc97..2976db091 100644 --- a/docs/operations/drive-timeout-tuning.md +++ b/docs/operations/drive-timeout-tuning.md @@ -62,6 +62,8 @@ Explicit per-operation overrides always win over the profile, so you can select | `RUSTFS_DRIVE_METADATA_TIMEOUT_SECS` | `5` | `60` | Timeout for metadata reads such as `read_metadata`. | | `RUSTFS_DRIVE_DISK_INFO_TIMEOUT_SECS` | `5` | `60` | Timeout for `disk_info()` calls. | | `RUSTFS_OBJECT_DISK_READ_TIMEOUT` | `10` | `60` | Per-read stall budget while streaming an object body from disk. | +| `RUSTFS_CAPACITY_STAT_TIMEOUT` | `3` | `60` | Base cooperative budget for an object-capacity disk scan. | +| `RUSTFS_CAPACITY_MAX_TIMEOUT` | `15` | `60` | Upper bound for the dynamically adjusted object-capacity scan budget. | | `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION` | `30` | — | Deprecated global fallback for every per-operation knob without an explicit override. Prefer the per-operation knobs. | The health-transition and probe knobs (`RUSTFS_DRIVE_TIMEOUT_HEALTH_ACTION`,