mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-31 09:18:28 +00:00
fix(object-capacity): drive scan progress checks by visited entries and remove unreachable stall detector (#4557)
Progress/timeout checks only ran when file_count advanced, so trees of directory-only or error-dense entries (dirs, traversal errors, symlinks never increment file_count) bypassed the entire cooperative time budget and held the blocking thread for unbounded time, while each error entry logged its own warn — permission-dense trees flooded the log (S13). The stall detector was structurally unreachable: it fired on 'no file progress between checks', but checks themselves only ran when file progress happened, so RUSTFS_CAPACITY_STALL_TIMEOUT never did anything since its introduction (S09). - Progress checks (timeout + early-sampling entry) are now driven by a visited-entry counter that also advances on directories and errors, so every tree shape reaches the budget checks. - Remove the unreachable stall detector and its plumbing end to end (ProgressMonitor fields, ScanLimits, config getter, env const, stall metric). Genuine walker wedges are handled by the hard outer wall-clock budget from backlog#1017; no decorative protection is left. - Cap per-entry error warns at 10 per scan with an explicit suppression notice; had_partial_errors still records the condition. Ref: rustfs/backlog#1016 (S09+S13 from audit rustfs/backlog#1010)
This commit is contained in:
@@ -21,13 +21,12 @@ use futures::FutureExt;
|
||||
use rustfs_config::{
|
||||
DEFAULT_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, DEFAULT_CAPACITY_FOLLOW_SYMLINKS, DEFAULT_CAPACITY_MAX_SYMLINK_DEPTH,
|
||||
DEFAULT_CAPACITY_MAX_TIMEOUT_SECS, DEFAULT_CAPACITY_METRICS_INTERVAL_SECS, DEFAULT_CAPACITY_MIN_TIMEOUT_SECS,
|
||||
DEFAULT_CAPACITY_STALL_TIMEOUT_SECS, DEFAULT_FAST_UPDATE_THRESHOLD_SECS, DEFAULT_MAX_FILES_THRESHOLD, DEFAULT_SAMPLE_RATE,
|
||||
DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS, DEFAULT_STAT_TIMEOUT_SECS, DEFAULT_WRITE_FREQUENCY_THRESHOLD,
|
||||
DEFAULT_WRITE_TRIGGER_DELAY_SECS, ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, ENV_CAPACITY_FAST_UPDATE_THRESHOLD,
|
||||
ENV_CAPACITY_FOLLOW_SYMLINKS, ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_MAX_SYMLINK_DEPTH, ENV_CAPACITY_MAX_TIMEOUT,
|
||||
ENV_CAPACITY_METRICS_INTERVAL, ENV_CAPACITY_MIN_TIMEOUT, ENV_CAPACITY_SAMPLE_RATE, ENV_CAPACITY_SCHEDULED_INTERVAL,
|
||||
ENV_CAPACITY_STALL_TIMEOUT, ENV_CAPACITY_STAT_TIMEOUT, ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD,
|
||||
ENV_CAPACITY_WRITE_TRIGGER_DELAY,
|
||||
DEFAULT_FAST_UPDATE_THRESHOLD_SECS, DEFAULT_MAX_FILES_THRESHOLD, DEFAULT_SAMPLE_RATE, DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS,
|
||||
DEFAULT_STAT_TIMEOUT_SECS, DEFAULT_WRITE_FREQUENCY_THRESHOLD, DEFAULT_WRITE_TRIGGER_DELAY_SECS,
|
||||
ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, ENV_CAPACITY_FAST_UPDATE_THRESHOLD, ENV_CAPACITY_FOLLOW_SYMLINKS,
|
||||
ENV_CAPACITY_MAX_FILES_THRESHOLD, ENV_CAPACITY_MAX_SYMLINK_DEPTH, ENV_CAPACITY_MAX_TIMEOUT, ENV_CAPACITY_METRICS_INTERVAL,
|
||||
ENV_CAPACITY_MIN_TIMEOUT, ENV_CAPACITY_SAMPLE_RATE, ENV_CAPACITY_SCHEDULED_INTERVAL, ENV_CAPACITY_STAT_TIMEOUT,
|
||||
ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD, ENV_CAPACITY_WRITE_TRIGGER_DELAY,
|
||||
};
|
||||
use rustfs_io_metrics::capacity_metrics::{
|
||||
record_capacity_current_bytes, record_capacity_degraded_reading, record_capacity_dirty_disk_count,
|
||||
@@ -90,8 +89,6 @@ struct CachedCapacityConfig {
|
||||
min_timeout: Duration,
|
||||
/// Max timeout
|
||||
max_timeout: Duration,
|
||||
/// Stall timeout
|
||||
stall_timeout: Duration,
|
||||
}
|
||||
|
||||
impl CachedCapacityConfig {
|
||||
@@ -123,7 +120,6 @@ impl CachedCapacityConfig {
|
||||
enable_dynamic_timeout: get_env_bool(ENV_CAPACITY_ENABLE_DYNAMIC_TIMEOUT, DEFAULT_CAPACITY_ENABLE_DYNAMIC_TIMEOUT),
|
||||
min_timeout: Duration::from_secs(get_env_u64(ENV_CAPACITY_MIN_TIMEOUT, DEFAULT_CAPACITY_MIN_TIMEOUT_SECS)),
|
||||
max_timeout: Duration::from_secs(get_env_u64(ENV_CAPACITY_MAX_TIMEOUT, DEFAULT_CAPACITY_MAX_TIMEOUT_SECS)),
|
||||
stall_timeout: Duration::from_secs(get_env_u64(ENV_CAPACITY_STALL_TIMEOUT, DEFAULT_CAPACITY_STALL_TIMEOUT_SECS)),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -297,18 +293,6 @@ pub fn get_max_timeout() -> Duration {
|
||||
get_cached_config().max_timeout
|
||||
}
|
||||
|
||||
/// Get stall timeout from environment or default
|
||||
#[cfg(not(test))]
|
||||
pub fn get_stall_timeout() -> Duration {
|
||||
get_cached_config().stall_timeout
|
||||
}
|
||||
|
||||
/// Get stall timeout from environment or default (test mode)
|
||||
#[cfg(test)]
|
||||
pub fn get_stall_timeout() -> Duration {
|
||||
get_cached_config().stall_timeout
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Data Structures
|
||||
// ============================================================================
|
||||
|
||||
Reference in New Issue
Block a user