fix(lifecycle): stop tier free-version recovery walk-timeout loop (#5194)

The background tier free-version recovery walk pinned a hardcoded 60s
total wall-clock timeout that overrides every operator knob, so any
bucket whose healthy full walk exceeds 60s fails forever; the failed
run's duration was also subtracted from the next 60s tick, restarting
the walk immediately and pinning CPU and disk I/O.

- Drop the total wall-clock budget on the recovery walk
  (walkdir_timeout: Duration::ZERO) and inherit the operator-tunable
  drive stall budget (RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS) for
  per-call progress, so hung disks still fail fast.
- Back off failed runs from completion time: 60s doubling to a 600s
  cap, reset on success; never subtract the failed run's duration.
- Add RUSTFS_TIER_FREE_VERSION_RECOVERY_ENABLED (default true) to opt
  out of the recovery worker on deployments with no remote tiers;
  invalid values warn and fail open.

Fixes #5130

Co-authored-by: claude <claude@ehdtn.com>
This commit is contained in:
harry han
2026-07-26 11:54:33 +09:00
committed by GitHub
parent 42fc840630
commit 03af8e472b
4 changed files with 187 additions and 70 deletions
+21 -11
View File
@@ -2063,7 +2063,7 @@ mod tests {
}
#[tokio::test]
async fn walk_dir_skip_total_timeout_keeps_stream_pending() {
async fn walk_dir_total_timeout_disable_modes_keep_stream_pending() {
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS, Some("1"))], async {
let dir = tempfile::tempdir().expect("temp dir should be created");
let endpoint =
@@ -2086,24 +2086,34 @@ mod tests {
.await
.expect("object metadata should be written");
let mut writer = PendingWriter;
let result = tokio::time::timeout(
Duration::from_millis(20),
wrapper.walk_dir(
for (reason, options) in [
(
"skip_total_timeout",
WalkDirOptions {
bucket: bucket.to_string(),
recursive: true,
skip_total_timeout: true,
..Default::default()
},
&mut writer,
),
)
.await;
(
"zero per-request timeout",
WalkDirOptions {
bucket: bucket.to_string(),
recursive: true,
timeout_ms: Some(0),
stall_timeout_ms: None,
..Default::default()
},
),
] {
let mut writer = PendingWriter;
let result = tokio::time::timeout(Duration::from_millis(1_100), wrapper.walk_dir(options, &mut writer)).await;
assert!(result.is_err(), "skip_total_timeout should leave backpressured walk pending");
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online);
assert!(!wrapper.health.is_faulty());
assert!(result.is_err(), "{reason} should leave backpressured walk pending");
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online);
assert!(!wrapper.health.is_faulty());
}
})
.await;
}