fix(object-capacity): drive scan progress checks by visited entries and remove unreachable stall detector (#4557)

Progress/timeout checks only ran when file_count advanced, so trees of
directory-only or error-dense entries (dirs, traversal errors, symlinks
never increment file_count) bypassed the entire cooperative time budget
and held the blocking thread for unbounded time, while each error entry
logged its own warn — permission-dense trees flooded the log (S13).
The stall detector was structurally unreachable: it fired on 'no file
progress between checks', but checks themselves only ran when file
progress happened, so RUSTFS_CAPACITY_STALL_TIMEOUT never did anything
since its introduction (S09).

- Progress checks (timeout + early-sampling entry) are now driven by a
  visited-entry counter that also advances on directories and errors, so
  every tree shape reaches the budget checks.
- Remove the unreachable stall detector and its plumbing end to end
  (ProgressMonitor fields, ScanLimits, config getter, env const, stall
  metric). Genuine walker wedges are handled by the hard outer
  wall-clock budget from backlog#1017; no decorative protection is left.
- Cap per-entry error warns at 10 per scan with an explicit suppression
  notice; had_partial_errors still records the condition.

Ref: rustfs/backlog#1016 (S09+S13 from audit rustfs/backlog#1010)
This commit is contained in:
Zhengchao An
2026-07-09 02:47:17 +08:00
committed by GitHub
parent d232a46b4d
commit 0ebe00b383
5 changed files with 145 additions and 167 deletions
+6 -22
View File
@@ -21,13 +21,12 @@ use futures::FutureExt;
use rustfs_config::{
DEFAULT_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, DEFAULT_CAPACITY_FOLLOW_SYMLINKS, DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH,
DEFAULT_CAPACITY_MAX_TIMEOUT_SECS, DEFAULT_CAPACITY_METRICS_INTERVAL_SECS, DEFAULT_CAPACITY_MIN_TIMEOUT_SECS,
DEFAULT_CAPACITY_STALL_TIMEOUT_SECS, DEFAULT_FAST_UPDATE_THRESHOLD_SECS, DEFAULT_MAX_FILES_THRESHOLD, DEFAULT_SAMPLE_RATE,
DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS, DEFAULT_STAT_TIMEOUT_SECS, DEFAULT_WRITE_FREQUENCY_THRESHOLD,
DEFAULT_WRITE_TRIGGER_DELAY_SECS, ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, ENV_CAPACITY_FAST_UPDATE_THRESHOLD,
ENV_CAPACITY_FOLLOW_SYMLINKS, ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_MAX_SYMLINK_DEPTH, ENV_CAPACITY_MAX_TIMEOUT,
ENV_CAPACITY_METRICS_INTERVAL, ENV_CAPACITY_MIN_TIMEOUT, ENV_CAPACITY_SAMPLE_RATE, ENV_CAPACITY_SCHEDULED_INTERVAL,
ENV_CAPACITY_STALL_TIMEOUT, ENV_CAPACITY_STAT_TIMEOUT, ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD,
ENV_CAPACITY_WRITE_TRIGGER_DELAY,
DEFAULT_FAST_UPDATE_THRESHOLD_SECS, DEFAULT_MAX_FILES_THRESHOLD, DEFAULT_SAMPLE_RATE, DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS,
DEFAULT_STAT_TIMEOUT_SECS, DEFAULT_WRITE_FREQUENCY_THRESHOLD, DEFAULT_WRITE_TRIGGER_DELAY_SECS,
ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, ENV_CAPACITY_FAST_UPDATE_THRESHOLD, ENV_CAPACITY_FOLLOW_SYMLINKS,
ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_MAX_SYMLINK_DEPTH, ENV_CAPACITY_MAX_TIMEOUT, ENV_CAPACITY_METRICS_INTERVAL,
ENV_CAPACITY_MIN_TIMEOUT, ENV_CAPACITY_SAMPLE_RATE, ENV_CAPACITY_SCHEDULED_INTERVAL, ENV_CAPACITY_STAT_TIMEOUT,
ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD, ENV_CAPACITY_WRITE_TRIGGER_DELAY,
};
use rustfs_io_metrics::capacity_metrics::{
record_capacity_current_bytes, record_capacity_degraded_reading, record_capacity_dirty_disk_count,
@@ -90,8 +89,6 @@ struct CachedCapacityConfig {
min_timeout: Duration,
/// Max timeout
max_timeout: Duration,
/// Stall timeout
stall_timeout: Duration,
}
impl CachedCapacityConfig {
@@ -123,7 +120,6 @@ impl CachedCapacityConfig {
enable_dynamic_timeout: get_env_bool(ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, DEFAULT_CAPACITY_ENABLE_DYNAMIC_TIMEOUT),
min_timeout: Duration::from_secs(get_env_u64(ENV_CAPACITY_MIN_TIMEOUT, DEFAULT_CAPACITY_MIN_TIMEOUT_SECS)),
max_timeout: Duration::from_secs(get_env_u64(ENV_CAPACITY_MAX_TIMEOUT, DEFAULT_CAPACITY_MAX_TIMEOUT_SECS)),
stall_timeout: Duration::from_secs(get_env_u64(ENV_CAPACITY_STALL_TIMEOUT, DEFAULT_CAPACITY_STALL_TIMEOUT_SECS)),
}
}
}
@@ -297,18 +293,6 @@ pub fn get_max_timeout() -> Duration {
get_cached_config().max_timeout
}
/// Get stall timeout from environment or default
#[cfg(not(test))]
pub fn get_stall_timeout() -> Duration {
get_cached_config().stall_timeout
}
/// Get stall timeout from environment or default (test mode)
#[cfg(test)]
pub fn get_stall_timeout() -> Duration {
get_cached_config().stall_timeout
}
// ============================================================================
// Data Structures
// ============================================================================