mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-11 07:36:53 +00:00
fix(lifecycle): stop tier free-version recovery walk-timeout loop (#5194)
The background tier free-version recovery walk pinned a hardcoded 60s total wall-clock timeout that overrides every operator knob, so any bucket whose healthy full walk exceeds 60s fails forever; the failed run's duration was also subtracted from the next 60s tick, restarting the walk immediately and pinning CPU and disk I/O. - Drop the total wall-clock budget on the recovery walk (walkdir_timeout: Duration::ZERO) and inherit the operator-tunable drive stall budget (RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS) for per-call progress, so hung disks still fail fast. - Back off failed runs from completion time: 60s doubling to a 600s cap, reset on success; never subtract the failed run's duration. - Add RUSTFS_TIER_FREE_VERSION_RECOVERY_ENABLED (default true) to opt out of the recovery worker on deployments with no remote tiers; invalid values warn and fail open. Fixes #5130 Co-authored-by: claude <claude@ehdtn.com>
This commit is contained in:
@@ -2063,7 +2063,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn walk_dir_skip_total_timeout_keeps_stream_pending() {
|
||||
async fn walk_dir_total_timeout_disable_modes_keep_stream_pending() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS, Some("1"))], async {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
let endpoint =
|
||||
@@ -2086,24 +2086,34 @@ mod tests {
|
||||
.await
|
||||
.expect("object metadata should be written");
|
||||
|
||||
let mut writer = PendingWriter;
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_millis(20),
|
||||
wrapper.walk_dir(
|
||||
for (reason, options) in [
|
||||
(
|
||||
"skip_total_timeout",
|
||||
WalkDirOptions {
|
||||
bucket: bucket.to_string(),
|
||||
recursive: true,
|
||||
skip_total_timeout: true,
|
||||
..Default::default()
|
||||
},
|
||||
&mut writer,
|
||||
),
|
||||
)
|
||||
.await;
|
||||
(
|
||||
"zero per-request timeout",
|
||||
WalkDirOptions {
|
||||
bucket: bucket.to_string(),
|
||||
recursive: true,
|
||||
timeout_ms: Some(0),
|
||||
stall_timeout_ms: None,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
] {
|
||||
let mut writer = PendingWriter;
|
||||
let result = tokio::time::timeout(Duration::from_millis(1_100), wrapper.walk_dir(options, &mut writer)).await;
|
||||
|
||||
assert!(result.is_err(), "skip_total_timeout should leave backpressured walk pending");
|
||||
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online);
|
||||
assert!(!wrapper.health.is_faulty());
|
||||
assert!(result.is_err(), "{reason} should leave backpressured walk pending");
|
||||
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online);
|
||||
assert!(!wrapper.health.is_faulty());
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user