fix(lifecycle): stop tier free-version recovery walk-timeout loop (#5194)

The background tier free-version recovery walk pinned a hardcoded 60s
total wall-clock timeout that overrides every operator knob, so any
bucket whose healthy full walk exceeds 60s fails forever; the failed
run's duration was also subtracted from the next 60s tick, restarting
the walk immediately and pinning CPU and disk I/O.

- Drop the total wall-clock budget on the recovery walk
  (walkdir_timeout: Duration::ZERO) and inherit the operator-tunable
  drive stall budget (RUSTFS_DRIVE_WALKDIR_STALL_TIMEOUT_SECS) for
  per-call progress, so hung disks still fail fast.
- Back off failed runs from completion time: 60s doubling to a 600s
  cap, reset on success; never subtract the failed run's duration.
- Add RUSTFS_TIER_FREE_VERSION_RECOVERY_ENABLED (default true) to opt
  out of the recovery worker on deployments with no remote tiers;
  invalid values warn and fail open.

Fixes #5130

Co-authored-by: claude <claude@ehdtn.com>

crates/ecstore/src/cache_value/metacache_set.rs

@@ -267,6 +267,23 @@ impl Clone for ListPathRawOptions {
     }
 }
 
+fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
+    WalkDirOptions {
+        bucket: opts.bucket.clone(),
+        base_dir: opts.path.clone(),
+        recursive: opts.recursive,
+        incl_deleted: opts.incl_deleted,
+        report_notfound: opts.report_not_found,
+        filter_prefix: opts.filter_prefix.clone(),
+        forward_to: opts.forward_to.clone(),
+        limit: opts.per_disk_limit,
+        skip_total_timeout: opts.skip_walkdir_total_timeout,
+        timeout_ms: opts.walkdir_timeout.map(duration_millis),
+        stall_timeout_ms: opts.walkdir_stall_timeout.map(duration_millis),
+        ..Default::default()
+    }
+}
+
 pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
     let rx = rx.child_token();
     let _cancel_guard = rx.clone().drop_guard();
@@ -373,20 +390,7 @@ async fn list_path_raw_inner(
                 None
             };
 
-            let wakl_opts = WalkDirOptions {
-                bucket: opts_clone.bucket.clone(),
-                base_dir: opts_clone.path.clone(),
-                recursive: opts_clone.recursive,
-                incl_deleted: opts_clone.incl_deleted,
-                report_notfound: opts_clone.report_not_found,
-                filter_prefix: opts_clone.filter_prefix.clone(),
-                forward_to: opts_clone.forward_to.clone(),
-                limit: opts_clone.per_disk_limit,
-                skip_total_timeout: opts_clone.skip_walkdir_total_timeout,
-                timeout_ms: opts_clone.walkdir_timeout.map(duration_millis),
-                stall_timeout_ms: opts_clone.walkdir_stall_timeout.map(duration_millis),
-                ..Default::default()
-            };
+            let wakl_opts = walk_dir_options(&opts_clone);
 
             let mut need_fallback = false;
             let mut last_err = None;
@@ -559,27 +563,7 @@ async fn list_path_raw_inner(
                 }
 
                 let fallback_walk_started = std::time::Instant::now();
-                match disk
-                    .as_ref()
-                    .walk_dir(
-                        WalkDirOptions {
-                            bucket: opts_clone.bucket.clone(),
-                            base_dir: opts_clone.path.clone(),
-                            recursive: opts_clone.recursive,
-                            incl_deleted: opts_clone.incl_deleted,
-                            report_notfound: opts_clone.report_not_found,
-                            filter_prefix: opts_clone.filter_prefix.clone(),
-                            forward_to: opts_clone.forward_to.clone(),
-                            limit: opts_clone.per_disk_limit,
-                            skip_total_timeout: opts_clone.skip_walkdir_total_timeout,
-                            timeout_ms: opts_clone.walkdir_timeout.map(duration_millis),
-                            stall_timeout_ms: opts_clone.walkdir_stall_timeout.map(duration_millis),
-                            ..Default::default()
-                        },
-                        &mut wr,
-                    )
-                    .await
-                {
+                match disk.as_ref().walk_dir(walk_dir_options(&opts_clone), &mut wr).await {
                     Ok(_r) => {
                         rustfs_io_metrics::record_stage_duration(
                             "metacache_walk_dir_fallback",
@@ -1091,6 +1075,19 @@ mod tests {
         assert!(!is_benign_not_found_listing_failure(&[DiskError::DiskNotFound]));
     }
 
+    #[test]
+    fn walk_dir_options_preserve_zero_total_and_inherited_stall_timeouts() {
+        let options = walk_dir_options(&ListPathRawOptions {
+            walkdir_timeout: Some(Duration::ZERO),
+            walkdir_stall_timeout: None,
+            ..Default::default()
+        });
+
+        assert_eq!(options.timeout_ms, Some(0));
+        assert_eq!(options.stall_timeout_ms, None);
+        assert!(!options.skip_total_timeout);
+    }
+
     #[tokio::test]
     async fn list_path_raw_empty_disks_returns_read_quorum() {
         let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())