fix(storage): harden offline drive fail-fast paths (#2564)

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: houseme <4829346+houseme@users.noreply.github.com>

crates/ecstore/src/cache_value/metacache_set.rs

@@ -12,12 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::disk::disk_store::get_drive_walkdir_stall_timeout;
 use crate::disk::error::DiskError;
 use crate::disk::{self, DiskAPI, DiskStore, WalkDirOptions};
 use futures::future::join_all;
+use metrics::counter;
 use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetacacheReader, is_io_eof};
-use std::{future::Future, pin::Pin};
+use std::{future::Future, pin::Pin, time::Duration};
+use tokio::io::AsyncRead;
 use tokio::spawn;
+use tokio::time::timeout;
 use tokio_util::sync::CancellationToken;
 use tracing::{error, info, warn};
 
@@ -26,6 +30,21 @@ pub type PartialFn =
     Box<dyn Fn(MetaCacheEntries, &[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
 type FinishedFn = Box<dyn Fn(&[Option<DiskError>]) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + 'static>;
 
+#[derive(Debug)]
+enum PeekOutcome {
+    Ready(Option<MetaCacheEntry>),
+    Error(rustfs_filemeta::Error),
+    TimedOut,
+}
+
+async fn peek_with_timeout<R: AsyncRead + Unpin>(reader: &mut MetacacheReader<R>, timeout_duration: Duration) -> PeekOutcome {
+    match timeout(timeout_duration, reader.peek()).await {
+        Ok(Ok(entry)) => PeekOutcome::Ready(entry),
+        Ok(Err(err)) => PeekOutcome::Error(err),
+        Err(_) => PeekOutcome::TimedOut,
+    }
+}
+
 #[derive(Default)]
 pub struct ListPathRawOptions {
     pub disks: Vec<Option<DiskStore>>,
@@ -160,6 +179,7 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
     }
 
     let revjob = spawn(async move {
+        let peek_timeout = get_drive_walkdir_stall_timeout();
         let mut errs: Vec<Option<DiskError>> = Vec::with_capacity(readers.len());
         for _ in 0..readers.len() {
             errs.push(None);
@@ -191,8 +211,8 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
                     continue;
                 }
 
-                let entry = match r.peek().await {
-                    Ok(res) => {
+                let entry = match peek_with_timeout(r, peek_timeout).await {
+                    PeekOutcome::Ready(res) => {
                         if let Some(entry) = res {
                             // info!("read entry disk: {}, name: {}", i, entry.name);
                             entry
@@ -203,7 +223,7 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
                             continue;
                         }
                     }
-                    Err(err) => {
+                    PeekOutcome::Error(err) => {
                         if err == rustfs_filemeta::Error::Unexpected {
                             at_eof += 1;
                             // warn!("list_path_raw: peek err eof, disk: {}", i);
@@ -236,6 +256,31 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
                             continue;
                         }
                     }
+                    PeekOutcome::TimedOut => {
+                        has_err += 1;
+                        errs[i] = Some(DiskError::Timeout);
+                        let endpoint = opts
+                            .disks
+                            .get(i)
+                            .and_then(|disk| disk.as_ref().map(|disk| disk.endpoint().to_string()))
+                            .unwrap_or_else(|| "missing".to_string());
+                        counter!(
+                            "rustfs_list_path_raw_stall_total",
+                            "drive" => endpoint.clone()
+                        )
+                        .increment(1);
+                        warn!(
+                            drive = %endpoint,
+                            bucket = %opts.bucket,
+                            path = %opts.path,
+                            timeout_ms = peek_timeout.as_millis(),
+                            "list_path_raw reader peek timed out; excluding drive from current merge"
+                        );
+                        let (detached_rd, write_half) = tokio::io::duplex(1);
+                        drop(write_half);
+                        *r = MetacacheReader::new(detached_rd);
+                        continue;
+                    }
                 };
 
                 // warn!("list_path_raw: loop entry: {:?}, disk: {}", &entry.name, i);
@@ -364,3 +409,42 @@ pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> d
     // warn!("list_path_raw: done");
     Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rustfs_filemeta::MetacacheWriter;
+
+    #[tokio::test]
+    async fn peek_with_timeout_times_out_on_silent_reader() {
+        let (_writer, reader) = tokio::io::duplex(64);
+        let mut reader = MetacacheReader::new(reader);
+
+        let outcome = peek_with_timeout(&mut reader, Duration::from_millis(20)).await;
+        assert!(matches!(outcome, PeekOutcome::TimedOut));
+    }
+
+    #[tokio::test]
+    async fn peek_with_timeout_reads_entry_before_deadline() {
+        let (reader, writer) = tokio::io::duplex(256);
+        let mut metacache_reader = MetacacheReader::new(reader);
+
+        tokio::spawn(async move {
+            let mut writer = MetacacheWriter::new(writer);
+            let entry = MetaCacheEntry {
+                name: "bucket/object".to_string(),
+                metadata: vec![1, 2, 3],
+                cached: None,
+                reusable: false,
+            };
+            writer.write(&[entry]).await.expect("entry should be written");
+            writer.close().await.expect("writer should close");
+        });
+
+        let outcome = peek_with_timeout(&mut metacache_reader, Duration::from_secs(1)).await;
+        match outcome {
+            PeekOutcome::Ready(Some(entry)) => assert_eq!(entry.name, "bucket/object"),
+            other => panic!("expected ready entry, got {other:?}"),
+        }
+    }
+}

crates/ecstore/src/disk/disk_store.rs

@@ -16,10 +16,16 @@ use crate::disk::{
     CheckPartsResp, DeleteOptions, DiskAPI, DiskError, DiskInfo, DiskInfoOptions, DiskLocation, Endpoint, Error,
     FileInfoVersions, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, Result, UpdateMetadataOpts, VolumeInfo,
     WalkDirOptions,
+    health_state::{
+        RuntimeDriveHealthState, classify_drive_recovery, get_drive_returning_probe_interval,
+        get_drive_returning_success_threshold, get_drive_suspect_failure_threshold, record_drive_offline_duration,
+        record_drive_recovery_class, record_drive_runtime_state, record_drive_state_transition,
+    },
     local::{LocalDisk, ScanGuard},
 };
 use crate::global::GLOBAL_LOCAL_DISK_ID_MAP;
 use bytes::Bytes;
+use metrics::counter;
 use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
 use std::{
     path::PathBuf,
@@ -40,7 +46,6 @@ const DISK_HEALTH_FAULTY: u32 = 1;
 
 pub const ENV_RUSTFS_DRIVE_ACTIVE_MONITORING: &str = "RUSTFS_DRIVE_ACTIVE_MONITORING";
 pub const DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING: bool = true;
-pub const ENV_RUSTFS_DRIVE_MAX_TIMEOUT_DURATION: &str = "RUSTFS_DRIVE_MAX_TIMEOUT_DURATION";
 pub const CHECK_EVERY: Duration = Duration::from_secs(15);
 pub const SKIP_IF_SUCCESS_BEFORE: Duration = Duration::from_secs(5);
 pub const CHECK_TIMEOUT_DURATION: Duration = Duration::from_secs(5);
@@ -51,9 +56,52 @@ lazy_static::lazy_static! {
 }
 
 pub fn get_max_timeout_duration() -> Duration {
-    std::env::var(ENV_RUSTFS_DRIVE_MAX_TIMEOUT_DURATION)
-        .map(|v| Duration::from_secs(v.parse::<u64>().unwrap_or(30)))
-        .unwrap_or(Duration::from_secs(30))
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION,
+        rustfs_config::DEFAULT_DRIVE_MAX_TIMEOUT_DURATION_SECS,
+    ))
+}
+
+fn get_drive_timeout_duration(env_key: &str, default_secs: u64) -> Duration {
+    Duration::from_secs(
+        rustfs_utils::get_env_opt_u64_with_aliases(env_key, &[rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION])
+            .unwrap_or(default_secs),
+    )
+}
+
+pub fn get_drive_metadata_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS,
+    )
+}
+
+pub fn get_drive_disk_info_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_DISK_INFO_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_DISK_INFO_TIMEOUT_SECS,
+    )
+}
+
+pub fn get_drive_list_dir_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_LIST_DIR_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_LIST_DIR_TIMEOUT_SECS,
+    )
+}
+
+pub fn get_drive_walkdir_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_WALKDIR_TIMEOUT_SECS,
+    )
+}
+
+pub fn get_drive_walkdir_stall_timeout() -> Duration {
+    get_drive_timeout_duration(
+        rustfs_config::ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_WALKDIR_STALL_TIMEOUT_SECS,
+    )
 }
 
 /// DiskHealthTracker tracks the health status of a disk.
@@ -68,6 +116,16 @@ pub struct DiskHealthTracker {
     pub status: AtomicU32,
     /// Atomic number of waiting operations
     pub waiting: AtomicU32,
+    /// Runtime drive health state
+    pub runtime_state: AtomicU32,
+    /// Consecutive failures while transitioning away from online
+    pub consecutive_failures: AtomicU32,
+    /// Consecutive successes while returning online
+    pub consecutive_successes: AtomicU32,
+    /// When the drive first left the online state
+    pub offline_since_unix_secs: AtomicI64,
+    /// Last runtime state transition timestamp
+    pub last_transition_unix_secs: AtomicI64,
 }
 
 impl DiskHealthTracker {
@@ -83,6 +141,11 @@ impl DiskHealthTracker {
             last_started: AtomicI64::new(now),
             status: AtomicU32::new(DISK_HEALTH_OK),
             waiting: AtomicU32::new(0),
+            runtime_state: AtomicU32::new(RuntimeDriveHealthState::Online as u32),
+            consecutive_failures: AtomicU32::new(0),
+            consecutive_successes: AtomicU32::new(0),
+            offline_since_unix_secs: AtomicI64::new(0),
+            last_transition_unix_secs: AtomicI64::new(now / 1_000_000_000),
         }
     }
 
@@ -110,12 +173,139 @@ impl DiskHealthTracker {
         self.status.store(DISK_HEALTH_OK, Ordering::Release);
     }
 
+    #[cfg(test)]
+    pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
+        self.runtime_state.store(state as u32, Ordering::Release);
+        match state {
+            RuntimeDriveHealthState::Offline => self.set_faulty(),
+            RuntimeDriveHealthState::Online | RuntimeDriveHealthState::Suspect | RuntimeDriveHealthState::Returning => {
+                self.set_ok();
+            }
+        }
+    }
+
     pub fn swap_ok_to_faulty(&self) -> bool {
         self.status
             .compare_exchange(DISK_HEALTH_OK, DISK_HEALTH_FAULTY, Ordering::AcqRel, Ordering::Relaxed)
             .is_ok()
     }
 
+    pub fn runtime_state(&self) -> RuntimeDriveHealthState {
+        RuntimeDriveHealthState::from_u32(self.runtime_state.load(Ordering::Acquire))
+    }
+
+    pub fn offline_duration(&self) -> Option<Duration> {
+        let offline_since = self.offline_since_unix_secs.load(Ordering::Acquire);
+        if offline_since <= 0 {
+            return None;
+        }
+        let now = current_unix_secs();
+        Some(Duration::from_secs(now.saturating_sub(offline_since as u64)))
+    }
+
+    pub fn mark_failure(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
+        let current = self.runtime_state();
+        let now = current_unix_secs();
+        let next = match current {
+            RuntimeDriveHealthState::Online => {
+                self.consecutive_failures.store(1, Ordering::Release);
+                self.consecutive_successes.store(0, Ordering::Release);
+                self.offline_since_unix_secs
+                    .compare_exchange(0, now as i64, Ordering::AcqRel, Ordering::Relaxed)
+                    .ok();
+                RuntimeDriveHealthState::Suspect
+            }
+            RuntimeDriveHealthState::Suspect => {
+                let failures = self.consecutive_failures.fetch_add(1, Ordering::AcqRel) + 1;
+                if failures >= get_drive_suspect_failure_threshold() {
+                    RuntimeDriveHealthState::Offline
+                } else {
+                    RuntimeDriveHealthState::Suspect
+                }
+            }
+            RuntimeDriveHealthState::Returning => {
+                self.consecutive_failures.store(0, Ordering::Release);
+                self.consecutive_successes.store(0, Ordering::Release);
+                RuntimeDriveHealthState::Offline
+            }
+            RuntimeDriveHealthState::Offline => RuntimeDriveHealthState::Offline,
+        };
+
+        self.status.store(DISK_HEALTH_FAULTY, Ordering::Release);
+        self.transition_state(endpoint, current, next, reason);
+        current == RuntimeDriveHealthState::Online
+    }
+
+    pub fn mark_recovery_success(&self, endpoint: &Endpoint, reason: &'static str) -> bool {
+        let current = self.runtime_state();
+        let next = match current {
+            RuntimeDriveHealthState::Online => RuntimeDriveHealthState::Online,
+            RuntimeDriveHealthState::Suspect => RuntimeDriveHealthState::Online,
+            RuntimeDriveHealthState::Offline => {
+                self.consecutive_successes.store(1, Ordering::Release);
+                RuntimeDriveHealthState::Returning
+            }
+            RuntimeDriveHealthState::Returning => {
+                let successes = self.consecutive_successes.fetch_add(1, Ordering::AcqRel) + 1;
+                if successes >= get_drive_returning_success_threshold() {
+                    RuntimeDriveHealthState::Online
+                } else {
+                    RuntimeDriveHealthState::Returning
+                }
+            }
+        };
+
+        let became_online = next == RuntimeDriveHealthState::Online;
+        if became_online {
+            self.status.store(DISK_HEALTH_OK, Ordering::Release);
+            self.consecutive_failures.store(0, Ordering::Release);
+            self.consecutive_successes.store(0, Ordering::Release);
+        }
+        self.transition_state(endpoint, current, next, reason);
+        if became_online {
+            self.log_success();
+        }
+        became_online
+    }
+
+    fn transition_state(
+        &self,
+        endpoint: &Endpoint,
+        current: RuntimeDriveHealthState,
+        next: RuntimeDriveHealthState,
+        reason: &'static str,
+    ) {
+        if current == next {
+            return;
+        }
+
+        self.runtime_state.store(next as u32, Ordering::Release);
+        self.last_transition_unix_secs
+            .store(current_unix_secs() as i64, Ordering::Release);
+
+        if matches!(
+            next,
+            RuntimeDriveHealthState::Suspect | RuntimeDriveHealthState::Offline | RuntimeDriveHealthState::Returning
+        ) && self.offline_since_unix_secs.load(Ordering::Acquire) == 0
+        {
+            self.offline_since_unix_secs
+                .store(current_unix_secs() as i64, Ordering::Release);
+        }
+
+        if next == RuntimeDriveHealthState::Online {
+            if let Some(duration) = self.offline_duration() {
+                record_drive_offline_duration(endpoint, duration);
+                record_drive_recovery_class(classify_drive_recovery(duration));
+            }
+            self.offline_since_unix_secs.store(0, Ordering::Release);
+        } else if let Some(duration) = self.offline_duration() {
+            record_drive_offline_duration(endpoint, duration);
+        }
+
+        record_drive_state_transition(endpoint, current, next, reason);
+        record_drive_runtime_state(endpoint, next);
+    }
+
     /// Increment waiting operations counter
     pub fn increment_waiting(&self) {
         self.waiting.fetch_add(1, Ordering::Relaxed);
@@ -137,6 +327,13 @@ impl DiskHealthTracker {
     }
 }
 
+fn current_unix_secs() -> u64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap()
+        .as_secs()
+}
+
 impl Default for DiskHealthTracker {
     fn default() -> Self {
         Self::new()
@@ -186,19 +383,34 @@ impl LocalDiskWrapper {
         let env_health_check =
             rustfs_utils::get_env_bool(ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING);
 
-        Self {
+        let wrapper = Self {
             disk,
             health: Arc::new(DiskHealthTracker::new()),
             health_check: health_check && env_health_check,
             cancel_token: CancellationToken::new(),
             disk_id: Arc::new(RwLock::new(None)),
-        }
+        };
+        record_drive_runtime_state(&wrapper.disk.endpoint(), RuntimeDriveHealthState::Online);
+        wrapper
     }
 
     pub fn get_disk(&self) -> Arc<LocalDisk> {
         self.disk.clone()
     }
 
+    pub fn runtime_state(&self) -> RuntimeDriveHealthState {
+        self.health.runtime_state()
+    }
+
+    pub fn offline_duration_secs(&self) -> Option<u64> {
+        self.health.offline_duration().map(|duration| duration.as_secs())
+    }
+
+    #[cfg(test)]
+    pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
+        self.health.force_runtime_state_for_test(state);
+    }
+
     /// Enable health monitoring after disk creation.
     /// Used to defer health checks until after startup format loading completes.
     pub fn enable_health_check(&self) {
@@ -219,6 +431,20 @@ impl LocalDiskWrapper {
         self.cancel_token.cancel();
     }
 
+    fn spawn_recovery_monitor_if_needed(&self) {
+        if !self.health_check {
+            return;
+        }
+
+        self.health.increment_waiting();
+        let health = Arc::clone(&self.health);
+        let disk = Arc::clone(&self.disk);
+        let cancel_token = self.cancel_token.clone();
+        tokio::spawn(async move {
+            Self::monitor_disk_status(disk, health, cancel_token).await;
+        });
+    }
+
     /// Monitor disk writability periodically
     async fn monitor_disk_writable(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
         // TODO: config interval
@@ -256,7 +482,9 @@ impl LocalDiskWrapper {
 
 
                     let test_obj = format!("health-check-{}", Uuid::new_v4());
-                    if Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, true, CHECK_TIMEOUT_DURATION).await.is_err() && health.swap_ok_to_faulty() {
+                    if Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, true, CHECK_TIMEOUT_DURATION).await.is_err()
+                        && health.mark_failure(&disk.endpoint(), "active_health_check_failed")
+                    {
                         // Health check failed, disk is considered faulty
                         warn!("health check: failed, disk is considered faulty");
 
@@ -345,9 +573,9 @@ impl LocalDiskWrapper {
 
     /// Monitor disk status and try to bring it back online
     async fn monitor_disk_status(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
-        const CHECK_EVERY: Duration = Duration::from_secs(5);
+        let check_every = get_drive_returning_probe_interval();
 
-        let mut interval = time::interval(CHECK_EVERY);
+        let mut interval = time::interval(check_every);
 
         loop {
             tokio::select! {
@@ -362,12 +590,18 @@ impl LocalDiskWrapper {
                     let test_obj = format!("health-check-{}", Uuid::new_v4());
                     match Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, false, CHECK_TIMEOUT_DURATION).await {
                         Ok(_) => {
+                            let state_before = health.runtime_state();
+                            let is_online = health.mark_recovery_success(&disk.endpoint(), "recovery_probe_success");
+                            info!("Disk {} recovery probe succeeded; state={:?}", disk.to_string(), state_before);
+                            if !is_online {
+                                continue;
+                            }
                             info!("Disk {} is back online", disk.to_string());
-                            health.set_ok();
                             health.decrement_waiting();
                             return;
                         }
                         Err(e) => {
+                            health.mark_failure(&disk.endpoint(), "recovery_probe_failed");
                             warn!("Disk {} still faulty: {:?}", disk.to_string(), e);
                         }
                     }
@@ -435,6 +669,19 @@ impl LocalDiskWrapper {
     /// Track disk health for an operation.
     /// This method should wrap disk operations to ensure health checking.
     pub async fn track_disk_health<T, F, Fut>(&self, operation: F, timeout_duration: Duration) -> Result<T>
+    where
+        F: FnOnce() -> Fut,
+        Fut: std::future::Future<Output = Result<T>>,
+    {
+        self.track_disk_health_with_op("unknown", operation, timeout_duration).await
+    }
+
+    pub async fn track_disk_health_with_op<T, F, Fut>(
+        &self,
+        op: &'static str,
+        operation: F,
+        timeout_duration: Duration,
+    ) -> Result<T>
     where
         F: FnOnce() -> Fut,
         Fut: std::future::Future<Output = Result<T>>,
@@ -479,7 +726,21 @@ impl LocalDiskWrapper {
             Err(_) => {
                 // Timeout occurred, mark disk as potentially faulty and decrement waiting counter
                 self.health.decrement_waiting();
-                warn!("disk operation timeout after {:?}", timeout_duration);
+                if self.health.mark_failure(&self.endpoint(), "operation_timeout") {
+                    self.spawn_recovery_monitor_if_needed();
+                }
+                counter!(
+                    "rustfs_drive_op_timeout_total",
+                    "endpoint" => self.endpoint().to_string(),
+                    "op" => op.to_string()
+                )
+                .increment(1);
+                warn!(
+                    endpoint = %self.endpoint(),
+                    op,
+                    timeout_ms = timeout_duration.as_millis(),
+                    "Local disk operation timed out"
+                );
                 Err(DiskError::other(format!("disk operation timeout after {timeout_duration:?}")))
             }
         }
@@ -489,8 +750,12 @@ impl LocalDiskWrapper {
 #[async_trait::async_trait]
 impl DiskAPI for LocalDiskWrapper {
     async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
-        self.track_disk_health(|| async { self.disk.read_metadata(volume, path).await }, Duration::ZERO)
-            .await
+        self.track_disk_health_with_op(
+            "read_metadata",
+            || async { self.disk.read_metadata(volume, path).await },
+            get_drive_metadata_timeout(),
+        )
+        .await
     }
 
     fn start_scan(&self) -> ScanGuard {
@@ -565,15 +830,22 @@ impl DiskAPI for LocalDiskWrapper {
             return Err(DiskError::FaultyDisk);
         }
 
-        let result = self.disk.disk_info(opts).await?;
+        self.track_disk_health_with_op(
+            "disk_info",
+            || async {
+                let result = self.disk.disk_info(opts).await?;
 
-        if let Some(current_disk_id) = *self.disk_id.read().await
-            && Some(current_disk_id) != result.id
-        {
-            return Err(DiskError::DiskNotFound);
-        };
+                if let Some(current_disk_id) = *self.disk_id.read().await
+                    && Some(current_disk_id) != result.id
+                {
+                    return Err(DiskError::DiskNotFound);
+                };
 
-        Ok(result)
+                Ok(result)
+            },
+            get_drive_disk_info_timeout(),
+        )
+        .await
     }
 
     async fn make_volume(&self, volume: &str) -> Result<()> {
@@ -587,7 +859,7 @@ impl DiskAPI for LocalDiskWrapper {
     }
 
     async fn list_volumes(&self) -> Result<Vec<VolumeInfo>> {
-        self.track_disk_health(|| async { self.disk.list_volumes().await }, Duration::ZERO)
+        self.track_disk_health_with_op("list_volumes", || async { self.disk.list_volumes().await }, Duration::ZERO)
             .await
     }
 
@@ -602,7 +874,7 @@ impl DiskAPI for LocalDiskWrapper {
     }
 
     async fn walk_dir<W: tokio::io::AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
-        self.track_disk_health(|| async { self.disk.walk_dir(opts, wr).await }, Duration::ZERO)
+        self.track_disk_health_with_op("walk_dir", || async { self.disk.walk_dir(opts, wr).await }, get_drive_walkdir_timeout())
             .await
     }
 
@@ -710,9 +982,10 @@ impl DiskAPI for LocalDiskWrapper {
     }
 
     async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
-        self.track_disk_health(
+        self.track_disk_health_with_op(
+            "list_dir",
             || async { self.disk.list_dir(origvolume, volume, dir_path, count).await },
-            get_max_timeout_duration(),
+            get_drive_list_dir_timeout(),
         )
         .await
     }
@@ -802,3 +1075,74 @@ impl DiskAPI for LocalDiskWrapper {
             .await
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::disk::endpoint::Endpoint;
+    use crate::disk::health_state::RuntimeDriveHealthState;
+
+    #[test]
+    fn drive_metadata_timeout_uses_default_when_unset() {
+        temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
+            temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
+                assert_eq!(
+                    get_drive_metadata_timeout(),
+                    Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS)
+                );
+            });
+        });
+    }
+
+    #[test]
+    fn drive_metadata_timeout_uses_legacy_fallback_when_canonical_unset() {
+        temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
+            temp_env::with_var(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("17"), || {
+                assert_eq!(get_drive_metadata_timeout(), Duration::from_secs(17));
+            });
+        });
+    }
+
+    #[test]
+    fn drive_metadata_timeout_prefers_canonical_over_legacy() {
+        temp_env::with_var(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, Some("7"), || {
+            temp_env::with_var(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("17"), || {
+                assert_eq!(get_drive_metadata_timeout(), Duration::from_secs(7));
+            });
+        });
+    }
+
+    #[test]
+    fn runtime_state_transitions_from_online_to_suspect_then_offline() {
+        let endpoint = Endpoint::try_from("/tmp/runtime-state-disk").expect("endpoint should parse");
+        let health = DiskHealthTracker::new();
+
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
+        assert!(health.mark_failure(&endpoint, "timeout"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Suspect);
+
+        assert!(!health.mark_failure(&endpoint, "timeout"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
+        assert!(health.offline_duration().is_some());
+    }
+
+    #[test]
+    fn runtime_state_transitions_back_online_after_recovery_threshold() {
+        let endpoint = Endpoint::try_from("/tmp/runtime-state-recovery").expect("endpoint should parse");
+        let health = DiskHealthTracker::new();
+
+        health.mark_failure(&endpoint, "timeout");
+        health.mark_failure(&endpoint, "timeout");
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Offline);
+
+        assert!(!health.mark_recovery_success(&endpoint, "probe"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Returning);
+
+        assert!(!health.mark_recovery_success(&endpoint, "probe"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Returning);
+
+        assert!(health.mark_recovery_success(&endpoint, "probe"));
+        assert_eq!(health.runtime_state(), RuntimeDriveHealthState::Online);
+        assert!(health.offline_duration().is_none());
+    }
+}

crates/ecstore/src/disk/health_state.rs

@@ -0,0 +1,237 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use super::{DiskAPI, DiskStore};
+use crate::disk::endpoint::Endpoint;
+use metrics::{counter, gauge};
+use std::time::Duration;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[repr(u32)]
+pub enum RuntimeDriveHealthState {
+    Online = 0,
+    Suspect = 1,
+    Offline = 2,
+    Returning = 3,
+}
+
+impl RuntimeDriveHealthState {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::Online => "online",
+            Self::Suspect => "suspect",
+            Self::Offline => "offline",
+            Self::Returning => "returning",
+        }
+    }
+
+    pub fn from_u32(value: u32) -> Self {
+        match value {
+            1 => Self::Suspect,
+            2 => Self::Offline,
+            3 => Self::Returning,
+            _ => Self::Online,
+        }
+    }
+
+    pub fn is_snapshot_eligible(self) -> bool {
+        matches!(self, Self::Online | Self::Suspect | Self::Returning)
+    }
+
+    pub fn is_strictly_online(self) -> bool {
+        matches!(self, Self::Online)
+    }
+
+    pub fn should_probe_for_admin(self) -> bool {
+        matches!(self, Self::Online | Self::Returning)
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum DriveRecoveryClass {
+    ShortOffline,
+    MediumOffline,
+    LongOffline,
+}
+
+impl DriveRecoveryClass {
+    pub fn as_str(self) -> &'static str {
+        match self {
+            Self::ShortOffline => "short_offline",
+            Self::MediumOffline => "medium_offline",
+            Self::LongOffline => "long_offline",
+        }
+    }
+}
+
+pub fn get_drive_suspect_failure_threshold() -> u32 {
+    rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD,
+        rustfs_config::DEFAULT_DRIVE_SUSPECT_FAILURE_THRESHOLD,
+    ) as u32
+}
+
+pub fn get_drive_returning_success_threshold() -> u32 {
+    rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD,
+        rustfs_config::DEFAULT_DRIVE_RETURNING_SUCCESS_THRESHOLD,
+    ) as u32
+}
+
+pub fn get_drive_returning_probe_interval() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_RETURNING_PROBE_INTERVAL_SECS,
+        rustfs_config::DEFAULT_DRIVE_RETURNING_PROBE_INTERVAL_SECS,
+    ))
+}
+
+pub fn get_drive_offline_grace_period() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_OFFLINE_GRACE_PERIOD_SECS,
+        rustfs_config::DEFAULT_DRIVE_OFFLINE_GRACE_PERIOD_SECS,
+    ))
+}
+
+pub fn get_drive_long_offline_threshold() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_LONG_OFFLINE_THRESHOLD_SECS,
+        rustfs_config::DEFAULT_DRIVE_LONG_OFFLINE_THRESHOLD_SECS,
+    ))
+}
+
+pub fn classify_drive_recovery(duration: Duration) -> DriveRecoveryClass {
+    if duration <= get_drive_offline_grace_period() {
+        DriveRecoveryClass::ShortOffline
+    } else if duration >= get_drive_long_offline_threshold() {
+        DriveRecoveryClass::LongOffline
+    } else {
+        DriveRecoveryClass::MediumOffline
+    }
+}
+
+pub fn record_drive_runtime_state(endpoint: &Endpoint, state: RuntimeDriveHealthState) {
+    let endpoint_label = endpoint.to_string();
+    let pool_label = endpoint.pool_idx.to_string();
+    let set_label = endpoint.set_idx.to_string();
+    let disk_label = endpoint.disk_idx.to_string();
+
+    for candidate in [
+        RuntimeDriveHealthState::Online,
+        RuntimeDriveHealthState::Suspect,
+        RuntimeDriveHealthState::Offline,
+        RuntimeDriveHealthState::Returning,
+    ] {
+        gauge!(
+            "rustfs_drive_runtime_state",
+            "endpoint" => endpoint_label.clone(),
+            "pool" => pool_label.clone(),
+            "set" => set_label.clone(),
+            "disk" => disk_label.clone(),
+            "state" => candidate.as_str().to_string()
+        )
+        .set(if candidate == state { 1.0 } else { 0.0 });
+    }
+}
+
+pub fn record_drive_state_transition(
+    endpoint: &Endpoint,
+    from: RuntimeDriveHealthState,
+    to: RuntimeDriveHealthState,
+    reason: &'static str,
+) {
+    counter!(
+        "rustfs_drive_state_transition_total",
+        "endpoint" => endpoint.to_string(),
+        "pool" => endpoint.pool_idx.to_string(),
+        "set" => endpoint.set_idx.to_string(),
+        "disk" => endpoint.disk_idx.to_string(),
+        "from" => from.as_str().to_string(),
+        "to" => to.as_str().to_string(),
+        "reason" => reason.to_string()
+    )
+    .increment(1);
+}
+
+pub fn record_drive_recovery_class(class: DriveRecoveryClass) {
+    counter!(
+        "rustfs_drive_recovery_class_total",
+        "class" => class.as_str().to_string()
+    )
+    .increment(1);
+}
+
+pub fn record_drive_offline_duration(endpoint: &Endpoint, duration: Duration) {
+    gauge!(
+        "rustfs_drive_offline_duration_seconds",
+        "endpoint" => endpoint.to_string(),
+        "pool" => endpoint.pool_idx.to_string(),
+        "set" => endpoint.set_idx.to_string(),
+        "disk" => endpoint.disk_idx.to_string()
+    )
+    .set(duration.as_secs_f64());
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct DriveMembershipSnapshot {
+    pub online: Vec<DiskStore>,
+    pub suspect: Vec<DiskStore>,
+    pub returning: Vec<DiskStore>,
+    pub offline: Vec<DiskStore>,
+}
+
+impl DriveMembershipSnapshot {
+    pub fn from_optional_disks(disks: &[Option<DiskStore>]) -> Self {
+        let mut snapshot = Self::default();
+
+        for disk in disks.iter().flatten() {
+            match disk.runtime_state() {
+                RuntimeDriveHealthState::Online => snapshot.online.push(disk.clone()),
+                RuntimeDriveHealthState::Suspect => snapshot.suspect.push(disk.clone()),
+                RuntimeDriveHealthState::Returning => snapshot.returning.push(disk.clone()),
+                RuntimeDriveHealthState::Offline => snapshot.offline.push(disk.clone()),
+            }
+        }
+
+        snapshot
+    }
+
+    pub fn scanner_heal_candidates(&self) -> Vec<DiskStore> {
+        let mut disks = Vec::with_capacity(self.online.len() + self.suspect.len() + self.returning.len());
+        disks.extend(self.online.iter().cloned());
+        disks.extend(self.suspect.iter().cloned());
+        disks.extend(self.returning.iter().cloned());
+        disks
+    }
+
+    pub fn strict_online_candidates(&self) -> Vec<DiskStore> {
+        self.online.clone()
+    }
+
+    pub fn strict_online_local_candidates(&self) -> Vec<DiskStore> {
+        self.online.iter().filter(|disk| disk.is_local()).cloned().collect()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn runtime_drive_health_state_snapshot_eligibility_matches_membership_policy() {
+        assert!(RuntimeDriveHealthState::Online.is_snapshot_eligible());
+        assert!(RuntimeDriveHealthState::Suspect.is_snapshot_eligible());
+        assert!(RuntimeDriveHealthState::Returning.is_snapshot_eligible());
+        assert!(!RuntimeDriveHealthState::Offline.is_snapshot_eligible());
+    }
+}

crates/ecstore/src/disk/mod.rs

@@ -19,6 +19,7 @@ pub mod error_conv;
 pub mod error_reduce;
 pub mod format;
 pub mod fs;
+pub mod health_state;
 pub mod local;
 pub mod os;
 
@@ -33,6 +34,7 @@ pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
 pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
 
 use crate::disk::disk_store::LocalDiskWrapper;
+use crate::disk::health_state::RuntimeDriveHealthState;
 use crate::disk::local::ScanGuard;
 use crate::rpc::RemoteDisk;
 use bytes::Bytes;
@@ -410,6 +412,30 @@ impl DiskAPI for Disk {
     }
 }
 
+impl Disk {
+    pub fn runtime_state(&self) -> RuntimeDriveHealthState {
+        match self {
+            Disk::Local(local_disk) => local_disk.runtime_state(),
+            Disk::Remote(remote_disk) => remote_disk.runtime_state(),
+        }
+    }
+
+    pub fn offline_duration_secs(&self) -> Option<u64> {
+        match self {
+            Disk::Local(local_disk) => local_disk.offline_duration_secs(),
+            Disk::Remote(remote_disk) => remote_disk.offline_duration_secs(),
+        }
+    }
+
+    #[cfg(test)]
+    pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
+        match self {
+            Disk::Local(local_disk) => local_disk.force_runtime_state_for_test(state),
+            Disk::Remote(remote_disk) => remote_disk.force_runtime_state_for_test(state),
+        }
+    }
+}
+
 impl Disk {
     /// Enable health monitoring on this disk.
     /// Called after startup format loading completes so that remote peers

crates/ecstore/src/rpc/remote_disk.rs

@@ -17,9 +17,11 @@ use crate::disk::{
     FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
     disk_store::{
         CHECK_EVERY, CHECK_TIMEOUT_DURATION, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING,
-        SKIP_IF_SUCCESS_BEFORE, get_max_timeout_duration,
+        SKIP_IF_SUCCESS_BEFORE, get_drive_disk_info_timeout, get_drive_list_dir_timeout, get_drive_metadata_timeout,
+        get_drive_walkdir_stall_timeout, get_drive_walkdir_timeout, get_max_timeout_duration,
     },
     endpoint::Endpoint,
+    health_state::{RuntimeDriveHealthState, get_drive_returning_probe_interval, record_drive_runtime_state},
 };
 use crate::disk::{disk_store::DiskHealthTracker, error::DiskError, local::ScanGuard};
 use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
@@ -31,7 +33,9 @@ use crate::{
 use bytes::Bytes;
 use futures::lock::Mutex;
 use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE};
+use metrics::counter;
 use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
+use rustfs_protos::evict_failed_connection;
 use rustfs_protos::proto_gen::node_service::RenamePartRequest;
 use rustfs_protos::proto_gen::node_service::{
     CheckPartsRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest,
@@ -43,7 +47,7 @@ use rustfs_protos::proto_gen::node_service::{
 use rustfs_rio::{HttpReader, HttpWriter};
 use serde::{Serialize, de::DeserializeOwned};
 use std::{
-    io::Cursor,
+    io::{Cursor, ErrorKind},
     path::PathBuf,
     sync::{
         Arc,
@@ -116,10 +120,38 @@ impl RemoteDisk {
             health: Arc::new(DiskHealthTracker::new()),
             cancel_token: CancellationToken::new(),
         };
+        record_drive_runtime_state(ep, RuntimeDriveHealthState::Online);
 
         Ok(disk)
     }
 
+    pub fn runtime_state(&self) -> RuntimeDriveHealthState {
+        self.health.runtime_state()
+    }
+
+    pub fn offline_duration_secs(&self) -> Option<u64> {
+        self.health.offline_duration().map(|duration| duration.as_secs())
+    }
+
+    #[cfg(test)]
+    pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
+        self.health.force_runtime_state_for_test(state);
+    }
+
+    fn spawn_recovery_monitor_if_needed(&self) {
+        if !self.health_check {
+            return;
+        }
+
+        let addr = self.addr.clone();
+        let endpoint = self.endpoint.clone();
+        let health = Arc::clone(&self.health);
+        let cancel_token = self.cancel_token.clone();
+        tokio::spawn(async move {
+            Self::monitor_remote_disk_recovery(addr, endpoint, health, cancel_token).await;
+        });
+    }
+
     /// Enable health monitoring after disk creation.
     /// Used to defer health checks until after startup format loading completes,
     /// so that remote peers have time to come online.
@@ -130,27 +162,34 @@ impl RemoteDisk {
         let health = Arc::clone(&self.health);
         let cancel_token = self.cancel_token.clone();
         let addr = self.addr.clone();
+        let endpoint = self.endpoint.clone();
 
         tokio::spawn(async move {
-            Self::monitor_remote_disk_health(addr, health, cancel_token).await;
+            Self::monitor_remote_disk_health(addr, endpoint, health, cancel_token).await;
         });
     }
 
     /// Monitor remote disk health periodically
-    async fn monitor_remote_disk_health(addr: String, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
+    async fn monitor_remote_disk_health(
+        addr: String,
+        endpoint: Endpoint,
+        health: Arc<DiskHealthTracker>,
+        cancel_token: CancellationToken,
+    ) {
         let mut interval = time::interval(CHECK_EVERY);
 
         // Perform basic connectivity check
-        if Self::perform_connectivity_check(&addr).await.is_err() && health.swap_ok_to_faulty() {
+        if Self::perform_connectivity_check(&addr).await.is_err() && health.mark_failure(&endpoint, "connectivity_probe_failed") {
             warn!("Remote disk health check failed for {}: marking as faulty", addr);
 
             // Start recovery monitoring
             let health_clone = Arc::clone(&health);
             let addr_clone = addr.clone();
+            let endpoint_clone = endpoint.clone();
             let cancel_clone = cancel_token.clone();
 
             tokio::spawn(async move {
-                Self::monitor_remote_disk_recovery(addr_clone, health_clone, cancel_clone).await;
+                Self::monitor_remote_disk_recovery(addr_clone, endpoint_clone, health_clone, cancel_clone).await;
             });
         }
 
@@ -183,16 +222,17 @@ impl RemoteDisk {
                     }
 
                     // Perform basic connectivity check
-                    if Self::perform_connectivity_check(&addr).await.is_err() && health.swap_ok_to_faulty() {
+                    if Self::perform_connectivity_check(&addr).await.is_err() && health.mark_failure(&endpoint, "connectivity_probe_failed") {
                         warn!("Remote disk health check failed for {}: marking as faulty", addr);
 
                         // Start recovery monitoring
                         let health_clone = Arc::clone(&health);
                         let addr_clone = addr.clone();
+                        let endpoint_clone = endpoint.clone();
                         let cancel_clone = cancel_token.clone();
 
                         tokio::spawn(async move {
-                            Self::monitor_remote_disk_recovery(addr_clone, health_clone, cancel_clone).await;
+                            Self::monitor_remote_disk_recovery(addr_clone, endpoint_clone, health_clone, cancel_clone).await;
                         });
                     }
                 }
@@ -201,8 +241,13 @@ impl RemoteDisk {
     }
 
     /// Monitor remote disk recovery and mark as healthy when recovered
-    async fn monitor_remote_disk_recovery(addr: String, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
-        let mut interval = time::interval(CHECK_EVERY);
+    async fn monitor_remote_disk_recovery(
+        addr: String,
+        endpoint: Endpoint,
+        health: Arc<DiskHealthTracker>,
+        cancel_token: CancellationToken,
+    ) {
+        let mut interval = time::interval(get_drive_returning_probe_interval());
 
         loop {
             tokio::select! {
@@ -211,9 +256,14 @@ impl RemoteDisk {
                 }
                 _ = interval.tick() => {
                     if Self::perform_connectivity_check(&addr).await.is_ok() {
-                        info!("Remote disk recovered: {}", addr);
-                        health.set_ok();
-                        return;
+                        let became_online = health.mark_recovery_success(&endpoint, "connectivity_probe_success");
+                        info!("Remote disk recovery probe succeeded: {}", addr);
+                        if became_online {
+                            info!("Remote disk recovered: {}", addr);
+                            return;
+                        }
+                    } else {
+                        health.mark_failure(&endpoint, "connectivity_probe_failed");
                     }
                 }
             }
@@ -242,6 +292,19 @@ impl RemoteDisk {
 
     /// Execute operation with timeout and health tracking
     async fn execute_with_timeout<T, F, Fut>(&self, operation: F, timeout_duration: Duration) -> Result<T>
+    where
+        F: FnOnce() -> Fut,
+        Fut: std::future::Future<Output = Result<T>>,
+    {
+        self.execute_with_timeout_for_op("unknown", operation, timeout_duration).await
+    }
+
+    async fn execute_with_timeout_for_op<T, F, Fut>(
+        &self,
+        op: &'static str,
+        operation: F,
+        timeout_duration: Duration,
+    ) -> Result<T>
     where
         F: FnOnce() -> Fut,
         Fut: std::future::Future<Output = Result<T>>,
@@ -270,17 +333,81 @@ impl RemoteDisk {
                     self.health.log_success();
                 }
                 self.health.decrement_waiting();
+                if let Err(err) = &operation_result
+                    && Self::is_timeout_like_error(err)
+                {
+                    counter!(
+                        "rustfs_drive_op_timeout_total",
+                        "endpoint" => self.endpoint.to_string(),
+                        "op" => op.to_string()
+                    )
+                    .increment(1);
+                    warn!(
+                        endpoint = %self.endpoint,
+                        addr = %self.addr,
+                        op,
+                        timeout_ms = timeout_duration.as_millis(),
+                        "Remote disk operation returned a timeout-like error"
+                    );
+                    self.mark_faulty_and_evict("operation_timeout_error").await;
+                }
                 operation_result
             }
             Err(_) => {
                 // Timeout occurred, mark disk as potentially faulty
                 self.health.decrement_waiting();
-                warn!("Remote disk operation timeout after {:?}", timeout_duration);
+                counter!(
+                    "rustfs_drive_op_timeout_total",
+                    "endpoint" => self.endpoint.to_string(),
+                    "op" => op.to_string()
+                )
+                .increment(1);
+                self.mark_faulty_and_evict("operation_timeout").await;
+                warn!(
+                    endpoint = %self.endpoint,
+                    addr = %self.addr,
+                    op,
+                    timeout_ms = timeout_duration.as_millis(),
+                    "Remote disk operation timed out"
+                );
                 Err(Error::other(format!("Remote disk operation timeout after {timeout_duration:?}")))
             }
         }
     }
 
+    fn is_timeout_like_error(err: &Error) -> bool {
+        matches!(err, DiskError::Timeout) || matches!(err, DiskError::Io(io_err) if io_err.kind() == ErrorKind::TimedOut)
+    }
+
+    async fn mark_faulty_and_evict(&self, reason: &'static str) {
+        if self.health.mark_failure(&self.endpoint, reason) {
+            self.spawn_recovery_monitor_if_needed();
+            counter!(
+                "rustfs_drive_faulty_mark_total",
+                "endpoint" => self.endpoint.to_string(),
+                "reason" => reason.to_string()
+            )
+            .increment(1);
+            warn!(
+                "Remote disk marked faulty after timeout: endpoint={}, addr={}, reason={}",
+                self.endpoint, self.addr, reason
+            );
+            counter!(
+                "rustfs_drive_connection_evict_total",
+                "endpoint" => self.endpoint.to_string(),
+                "reason" => reason.to_string()
+            )
+            .increment(1);
+            info!(
+                endpoint = %self.endpoint,
+                addr = %self.addr,
+                reason,
+                "Evicting cached remote disk connection after fault transition"
+            );
+            evict_failed_connection(&self.addr).await;
+        }
+    }
+
     async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
         node_service_time_out_client(&self.addr, TonicInterceptor::Signature(gen_tonic_signature_interceptor()))
             .await
@@ -757,7 +884,8 @@ impl DiskAPI for RemoteDisk {
         let file_info = serde_json::to_string(&fi)?;
         let file_info_bin = encode_msgpack(&fi)?;
 
-        self.execute_with_timeout(
+        self.execute_with_timeout_for_op(
+            "write_metadata",
             || async {
                 let disk = self.disk_ref().await;
                 let mut client = self
@@ -786,24 +914,31 @@ impl DiskAPI for RemoteDisk {
     }
 
     async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
-        let disk = self.disk_ref().await;
-        let mut client = self
-            .get_client()
-            .await
-            .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
-        let request = Request::new(ReadMetadataRequest {
-            volume: volume.to_string(),
-            path: path.to_string(),
-            disk,
-        });
+        self.execute_with_timeout_for_op(
+            "read_metadata",
+            || async {
+                let disk = self.disk_ref().await;
+                let mut client = self
+                    .get_client()
+                    .await
+                    .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
+                let request = Request::new(ReadMetadataRequest {
+                    volume: volume.to_string(),
+                    path: path.to_string(),
+                    disk,
+                });
 
-        let response = client.read_metadata(request).await?.into_inner();
+                let response = client.read_metadata(request).await?.into_inner();
 
-        if !response.success {
-            return Err(response.error.unwrap_or_default().into());
-        }
+                if !response.success {
+                    return Err(response.error.unwrap_or_default().into());
+                }
 
-        Ok(response.data)
+                Ok(response.data)
+            },
+            get_drive_metadata_timeout(),
+        )
+        .await
     }
 
     #[tracing::instrument(skip(self))]
@@ -814,7 +949,8 @@ impl DiskAPI for RemoteDisk {
         let file_info_bin = encode_msgpack(&fi)?;
         let opts_bin = encode_msgpack(opts)?;
 
-        self.execute_with_timeout(
+        self.execute_with_timeout_for_op(
+            "update_metadata",
             || async {
                 let disk = self.disk_ref().await;
                 let mut client = self
@@ -967,53 +1103,66 @@ impl DiskAPI for RemoteDisk {
     async fn list_dir(&self, _origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
         debug!("list_dir {}/{}", volume, dir_path);
 
-        if self.health.is_faulty() {
-            return Err(DiskError::FaultyDisk);
-        }
-        let disk = self.disk_ref().await;
+        self.execute_with_timeout(
+            || async {
+                let disk = self.disk_ref().await;
 
-        let mut client = self
-            .get_client()
-            .await
-            .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
-        let request = Request::new(ListDirRequest {
-            disk,
-            volume: volume.to_string(),
-            dir_path: dir_path.to_string(),
-            count,
-        });
+                let mut client = self
+                    .get_client()
+                    .await
+                    .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
+                let request = Request::new(ListDirRequest {
+                    disk,
+                    volume: volume.to_string(),
+                    dir_path: dir_path.to_string(),
+                    count,
+                });
 
-        let response = client.list_dir(request).await?.into_inner();
+                let response = client.list_dir(request).await?.into_inner();
 
-        if !response.success {
-            return Err(response.error.unwrap_or_default().into());
-        }
+                if !response.success {
+                    return Err(response.error.unwrap_or_default().into());
+                }
 
-        Ok(response.volumes)
+                Ok(response.volumes)
+            },
+            get_drive_list_dir_timeout(),
+        )
+        .await
     }
 
     #[tracing::instrument(skip(self, wr))]
     async fn walk_dir<W: AsyncWrite + Unpin + Send>(&self, opts: WalkDirOptions, wr: &mut W) -> Result<()> {
         info!("walk_dir {}", self.endpoint.to_string());
 
-        if self.health.is_faulty() {
-            return Err(DiskError::FaultyDisk);
-        }
-        let disk = self.disk_ref().await;
+        self.execute_with_timeout(
+            || async {
+                let disk = self.disk_ref().await;
 
-        let url = format!("{}/rustfs/rpc/walk_dir?disk={}", self.endpoint.grid_host(), urlencoding::encode(&disk),);
+                let url = format!("{}/rustfs/rpc/walk_dir?disk={}", self.endpoint.grid_host(), urlencoding::encode(&disk),);
 
-        let opts = serde_json::to_vec(&opts)?;
+                let opts = serde_json::to_vec(&opts)?;
 
-        let mut headers = HeaderMap::new();
-        headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
-        build_auth_headers(&url, &Method::GET, &mut headers);
+                let mut headers = HeaderMap::new();
+                headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
+                build_auth_headers(&url, &Method::GET, &mut headers);
 
-        let mut reader = HttpReader::new(url, Method::GET, headers, Some(opts)).await?;
+                let mut reader = HttpReader::new_with_stall_timeout(
+                    url,
+                    Method::GET,
+                    headers,
+                    Some(opts),
+                    Some(get_drive_walkdir_stall_timeout()),
+                )
+                .await?;
 
-        copy_stream_with_buffer(&mut reader, wr, DEFAULT_READ_BUFFER_SIZE).await?;
+                copy_stream_with_buffer(&mut reader, wr, DEFAULT_READ_BUFFER_SIZE).await?;
 
-        Ok(())
+                Ok(())
+            },
+            get_drive_walkdir_timeout(),
+        )
+        .await
     }
 
     #[tracing::instrument(level = "debug", skip(self))]
@@ -1424,29 +1573,32 @@ impl DiskAPI for RemoteDisk {
 
     #[tracing::instrument(skip(self))]
     async fn disk_info(&self, opts: &DiskInfoOptions) -> Result<DiskInfo> {
-        if self.health.is_faulty() {
-            return Err(DiskError::FaultyDisk);
-        }
+        self.execute_with_timeout_for_op(
+            "disk_info",
+            || async {
+                let opts = serde_json::to_string(&opts)?;
+                let mut client = self
+                    .get_client()
+                    .await
+                    .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
+                let request = Request::new(DiskInfoRequest {
+                    disk: self.endpoint.to_string(),
+                    opts,
+                });
 
-        let opts = serde_json::to_string(&opts)?;
-        let mut client = self
-            .get_client()
-            .await
-            .map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
-        let request = Request::new(DiskInfoRequest {
-            disk: self.endpoint.to_string(),
-            opts,
-        });
+                let response = client.disk_info(request).await?.into_inner();
 
-        let response = client.disk_info(request).await?.into_inner();
+                if !response.success {
+                    return Err(response.error.unwrap_or_default().into());
+                }
 
-        if !response.success {
-            return Err(response.error.unwrap_or_default().into());
-        }
+                let disk_info = serde_json::from_str::<DiskInfo>(&response.disk_info)?;
 
-        let disk_info = serde_json::from_str::<DiskInfo>(&response.disk_info)?;
-
-        Ok(disk_info)
+                Ok(disk_info)
+            },
+            get_drive_disk_info_timeout(),
+        )
+        .await
     }
 
     #[tracing::instrument(skip(self))]
@@ -1459,9 +1611,11 @@ impl DiskAPI for RemoteDisk {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use rustfs_common::GLOBAL_CONN_MAP;
     use std::sync::Once;
     use tokio::io::duplex;
     use tokio::net::TcpListener;
+    use tonic::transport::Endpoint as TonicEndpoint;
     use tracing::Level;
     use uuid::Uuid;
 
@@ -1793,6 +1947,132 @@ mod tests {
         assert!(result.is_ok());
     }
 
+    #[tokio::test]
+    async fn test_execute_with_timeout_marks_remote_disk_faulty() {
+        let url = url::Url::parse("http://remote-timeout:9000").unwrap();
+        let endpoint = Endpoint {
+            url,
+            is_local: false,
+            pool_idx: 0,
+            set_idx: 0,
+            disk_idx: 0,
+        };
+
+        let remote_disk = RemoteDisk::new(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .unwrap();
+
+        let err = remote_disk
+            .execute_with_timeout(
+                || async {
+                    tokio::time::sleep(Duration::from_millis(50)).await;
+                    Ok::<(), Error>(())
+                },
+                Duration::from_millis(10),
+            )
+            .await
+            .expect_err("timeout should fail");
+
+        assert!(err.to_string().contains("timeout"));
+        assert!(!remote_disk.is_online().await, "remote disk should be marked faulty after timeout");
+    }
+
+    #[tokio::test]
+    async fn test_execute_with_timeout_evicts_cached_connection() {
+        let addr = "http://127.0.0.1:59991".to_string();
+        let url = url::Url::parse(&format!("{addr}/data")).unwrap();
+        let endpoint = Endpoint {
+            url,
+            is_local: false,
+            pool_idx: 0,
+            set_idx: 0,
+            disk_idx: 0,
+        };
+
+        let remote_disk = RemoteDisk::new(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .unwrap();
+
+        let channel = TonicEndpoint::from_shared(addr.clone()).unwrap().connect_lazy();
+        GLOBAL_CONN_MAP.write().await.insert(addr.clone(), channel);
+        assert!(GLOBAL_CONN_MAP.read().await.contains_key(&addr));
+
+        let _ = remote_disk
+            .execute_with_timeout(
+                || async {
+                    tokio::time::sleep(Duration::from_millis(50)).await;
+                    Ok::<(), Error>(())
+                },
+                Duration::from_millis(10),
+            )
+            .await
+            .expect_err("timeout should fail");
+
+        assert!(
+            !GLOBAL_CONN_MAP.read().await.contains_key(&addr),
+            "timeout should evict cached connection"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_execute_with_timeout_marks_faulty_on_timeout_like_error() {
+        let addr = "http://127.0.0.1:59992".to_string();
+        let url = url::Url::parse(&format!("{addr}/data")).unwrap();
+        let endpoint = Endpoint {
+            url,
+            is_local: false,
+            pool_idx: 0,
+            set_idx: 0,
+            disk_idx: 0,
+        };
+
+        let remote_disk = RemoteDisk::new(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .unwrap();
+
+        let channel = TonicEndpoint::from_shared(addr.clone()).unwrap().connect_lazy();
+        GLOBAL_CONN_MAP.write().await.insert(addr.clone(), channel);
+
+        let err = remote_disk
+            .execute_with_timeout(
+                || async { Err::<(), Error>(DiskError::Io(std::io::Error::new(std::io::ErrorKind::TimedOut, "stall timeout"))) },
+                Duration::from_secs(1),
+            )
+            .await
+            .expect_err("timeout-like operation error should fail");
+
+        assert_eq!(
+            match &err {
+                DiskError::Io(io_err) => io_err.kind(),
+                other => panic!("expected io timeout error, got {other:?}"),
+            },
+            std::io::ErrorKind::TimedOut
+        );
+        assert!(!remote_disk.is_online().await, "timeout-like errors should mark remote disk faulty");
+        assert!(
+            !GLOBAL_CONN_MAP.read().await.contains_key(&addr),
+            "timeout-like errors should evict cached connection"
+        );
+    }
+
     #[test]
     fn test_remote_disk_sync_properties() {
         let url = url::Url::parse("https://secure-remote:9000/data").unwrap();

crates/ecstore/src/rpc/remote_locker.rs

@@ -18,8 +18,10 @@ use rustfs_lock::{
     LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
     types::{LockId, LockMetadata, LockPriority},
 };
-use rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient;
 use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, GenerallyLockRequest, PingRequest};
+use rustfs_protos::{evict_failed_connection, proto_gen::node_service::node_service_client::NodeServiceClient};
+use std::time::Duration;
+use tokio::time::timeout;
 use tonic::Request;
 use tonic::service::interceptor::InterceptedService;
 use tonic::transport::Channel;
@@ -62,6 +64,77 @@ impl RemoteClient {
             .map_err(|err| LockError::internal(format!("can not get client, err: {err}")))
     }
 
+    async fn evict_connection(&self, op: &'static str, reason: &str) {
+        warn!(
+            addr = %self.addr,
+            op,
+            reason,
+            "Evicting cached remote lock connection after RPC failure"
+        );
+        evict_failed_connection(&self.addr).await;
+    }
+
+    fn rpc_timeout(timeout_duration: Duration) -> Duration {
+        if timeout_duration.is_zero() {
+            Duration::from_millis(1)
+        } else {
+            timeout_duration
+        }
+    }
+
+    async fn execute_rpc<T, F>(
+        &self,
+        op: &'static str,
+        timeout_duration: Duration,
+        future: F,
+    ) -> std::result::Result<T, LockError>
+    where
+        F: std::future::Future<Output = std::result::Result<T, tonic::Status>>,
+    {
+        let timeout_duration = Self::rpc_timeout(timeout_duration);
+        match timeout(timeout_duration, future).await {
+            Ok(Ok(response)) => Ok(response),
+            Ok(Err(err)) => {
+                let reason = err.to_string();
+                self.evict_connection(op, &reason).await;
+                Err(LockError::internal(format!("{op} RPC failed: {reason}")))
+            }
+            Err(_) => {
+                let reason = format!("RPC timed out after {:?}", timeout_duration);
+                self.evict_connection(op, &reason).await;
+                Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), timeout_duration))
+            }
+        }
+    }
+
+    fn timeout_failure_response(request: &LockRequest) -> LockResponse {
+        LockResponse::failure("Lock acquisition timeout", request.acquire_timeout)
+    }
+
+    fn rpc_failure_response(_request: &LockRequest, err: &LockError) -> LockResponse {
+        LockResponse::failure(format!("Remote lock RPC failed: {err}"), Duration::ZERO)
+    }
+
+    fn timeout_failure_batch(requests: &[LockRequest]) -> Vec<LockResponse> {
+        requests.iter().map(Self::timeout_failure_response).collect()
+    }
+
+    fn rpc_failure_batch(requests: &[LockRequest], err: &LockError) -> Vec<LockResponse> {
+        requests
+            .iter()
+            .map(|request| Self::rpc_failure_response(request, err))
+            .collect()
+    }
+
+    fn batch_rpc_timeout(requests: &[LockRequest]) -> Duration {
+        requests
+            .iter()
+            .map(|request| request.acquire_timeout)
+            .max()
+            .map(Self::rpc_timeout)
+            .unwrap_or_else(|| Duration::from_millis(1))
+    }
+
     fn build_lock_info(request: &LockRequest, lock_info_json: Option<String>) -> LockInfo {
         if let Some(lock_info_json) = lock_info_json {
             match serde_json::from_str::<LockInfo>(&lock_info_json) {
@@ -111,11 +184,11 @@ impl LockClient for RemoteClient {
                 .map_err(|e| LockError::internal(format!("Failed to serialize request: {e}")))?,
         });
 
-        let resp = client
-            .lock(req)
-            .await
-            .map_err(|e| LockError::internal(e.to_string()))?
-            .into_inner();
+        let resp = match self.execute_rpc("lock", request.acquire_timeout, client.lock(req)).await {
+            Ok(resp) => resp.into_inner(),
+            Err(LockError::Timeout { .. }) => return Ok(Self::timeout_failure_response(request)),
+            Err(err) => return Ok(Self::rpc_failure_response(request, &err)),
+        };
 
         // Check for explicit error first
         if let Some(error_info) = resp.error_info {
@@ -138,6 +211,10 @@ impl LockClient for RemoteClient {
     }
 
     async fn acquire_locks_batch(&self, requests: &[LockRequest]) -> Result<Vec<LockResponse>> {
+        if requests.is_empty() {
+            return Ok(Vec::new());
+        }
+
         let mut client = self.get_client().await?;
         let req = Request::new(BatchGenerallyLockRequest {
             args: requests
@@ -148,11 +225,14 @@ impl LockClient for RemoteClient {
                 .collect::<Result<Vec<_>>>()?,
         });
 
-        let resp = client
-            .lock_batch(req)
+        let resp = match self
+            .execute_rpc("lock_batch", Self::batch_rpc_timeout(requests), client.lock_batch(req))
             .await
-            .map_err(|e| LockError::internal(e.to_string()))?
-            .into_inner();
+        {
+            Ok(resp) => resp.into_inner(),
+            Err(LockError::Timeout { .. }) => return Ok(Self::timeout_failure_batch(requests)),
+            Err(err) => return Ok(Self::rpc_failure_batch(requests, &err)),
+        };
 
         Ok(requests
             .iter()
@@ -380,3 +460,95 @@ impl LockClient for RemoteClient {
         false
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rustfs_common::GLOBAL_CONN_MAP;
+    use rustfs_lock::{ObjectKey, types::LockPriority};
+    use tokio::net::TcpListener;
+    use tokio::task::JoinHandle;
+    use tonic::transport::Endpoint as TonicEndpoint;
+
+    async fn spawn_hanging_listener() -> (String, JoinHandle<()>) {
+        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let addr = format!("http://{}", listener.local_addr().unwrap());
+        let task = tokio::spawn(async move {
+            if let Ok((stream, _)) = listener.accept().await {
+                let _stream = stream;
+                tokio::time::sleep(Duration::from_secs(2)).await;
+            }
+        });
+        (addr, task)
+    }
+
+    async fn cache_lazy_channel(addr: &str) {
+        let channel = TonicEndpoint::from_shared(addr.to_string()).unwrap().connect_lazy();
+        GLOBAL_CONN_MAP.write().await.insert(addr.to_string(), channel);
+    }
+
+    fn test_lock_request(timeout_duration: Duration) -> LockRequest {
+        LockRequest::new(ObjectKey::new("bucket", "object"), LockType::Exclusive, "owner-a")
+            .with_acquire_timeout(timeout_duration)
+            .with_priority(LockPriority::Normal)
+    }
+
+    #[tokio::test]
+    async fn test_remote_client_acquire_lock_respects_request_timeout_and_evicts_connection() {
+        let (addr, accept_task) = spawn_hanging_listener().await;
+        cache_lazy_channel(&addr).await;
+        assert!(GLOBAL_CONN_MAP.read().await.contains_key(&addr));
+
+        let client = RemoteClient::new(addr.clone());
+        let request = test_lock_request(Duration::from_millis(50));
+        let started_at = tokio::time::Instant::now();
+
+        let response = client.acquire_lock(&request).await.unwrap();
+
+        assert!(
+            started_at.elapsed() < Duration::from_secs(1),
+            "remote lock RPC should honor request timeout"
+        );
+        assert!(!response.success, "timed out lock acquisition should fail");
+        assert_eq!(response.error.as_deref(), Some("Lock acquisition timeout"));
+        assert!(
+            !GLOBAL_CONN_MAP.read().await.contains_key(&addr),
+            "timeout should evict cached connection"
+        );
+
+        accept_task.abort();
+    }
+
+    #[tokio::test]
+    async fn test_remote_client_acquire_locks_batch_respects_request_timeout_and_evicts_connection() {
+        let (addr, accept_task) = spawn_hanging_listener().await;
+        cache_lazy_channel(&addr).await;
+        assert!(GLOBAL_CONN_MAP.read().await.contains_key(&addr));
+
+        let client = RemoteClient::new(addr.clone());
+        let requests = vec![test_lock_request(Duration::from_millis(50))];
+        let started_at = tokio::time::Instant::now();
+
+        let responses = client.acquire_locks_batch(&requests).await.unwrap();
+
+        assert!(
+            started_at.elapsed() < Duration::from_secs(1),
+            "remote batch lock RPC should honor request timeout"
+        );
+        assert_eq!(responses.len(), 1);
+        assert!(!responses[0].success, "timed out batch lock acquisition should fail");
+        assert_eq!(responses[0].error.as_deref(), Some("Lock acquisition timeout"));
+        assert!(
+            !GLOBAL_CONN_MAP.read().await.contains_key(&addr),
+            "batch timeout should evict cached connection"
+        );
+
+        accept_task.abort();
+    }
+
+    #[test]
+    fn test_remote_client_zero_timeout_is_clamped() {
+        assert_eq!(RemoteClient::rpc_timeout(Duration::ZERO), Duration::from_millis(1));
+        assert_eq!(RemoteClient::rpc_timeout(Duration::from_millis(25)), Duration::from_millis(25));
+    }
+}

crates/ecstore/src/set_disk.rs

@@ -1032,57 +1032,163 @@ impl SetDisks {
             1
         };
 
-        let client_results = join_all(self.lockers.iter().cloned().enumerate().map(|(client_idx, client)| {
-            let requests = requests.clone();
-            async move { (client_idx, client.acquire_locks_batch(&requests).await) }
-        }))
-        .await;
-
         let mut lock_ids_by_object: Vec<Vec<(usize, rustfs_lock::LockId)>> = vec![Vec::new(); requests.len()];
         let mut errors_by_object: Vec<Option<String>> = vec![None; requests.len()];
+        #[derive(Clone, Copy, Debug, PartialEq, Eq)]
+        enum ObjectLockResolution {
+            Pending,
+            Succeeded,
+            Failed,
+        }
 
-        for (client_idx, result) in client_results {
-            match result {
-                Ok(responses) => {
+        let mut resolution_by_object = vec![ObjectLockResolution::Pending; requests.len()];
+        let mut pending_clients = self.lockers.len();
+        let mut unresolved_objects = requests.len();
+        let mut cleanup_lock_ids_by_client = vec![Vec::new(); self.lockers.len()];
+
+        let mut pending = tokio::task::JoinSet::new();
+        for (client_idx, client) in self.lockers.iter().cloned().enumerate() {
+            let requests = requests.clone();
+            pending.spawn(async move { (client_idx, client.acquire_locks_batch(&requests).await) });
+        }
+
+        while unresolved_objects > 0 {
+            let Some(join_result) = pending.join_next().await else {
+                break;
+            };
+            pending_clients = pending_clients.saturating_sub(1);
+
+            match join_result {
+                Ok((client_idx, Ok(responses))) => {
                     for (req_idx, request) in requests.iter().enumerate() {
-                        match responses.get(req_idx) {
-                            Some(response) if response.success => {
-                                if let Some(lock_info) = response.lock_info.as_ref() {
-                                    lock_ids_by_object[req_idx].push((client_idx, lock_info.id.clone()));
-                                } else if errors_by_object[req_idx].is_none() {
-                                    errors_by_object[req_idx] = Some(format!(
-                                        "missing distributed lock id for {}/{}",
-                                        request.resource.bucket, request.resource.object
-                                    ));
+                        let response = responses.get(req_idx);
+                        match resolution_by_object[req_idx] {
+                            ObjectLockResolution::Pending => match response {
+                                Some(response) if response.success => {
+                                    let lock_id = response
+                                        .lock_info
+                                        .as_ref()
+                                        .map(|lock_info| lock_info.id.clone())
+                                        .unwrap_or_else(|| request.lock_id.clone());
+                                    lock_ids_by_object[req_idx].push((client_idx, lock_id));
                                 }
-                            }
-                            Some(response) => {
-                                if errors_by_object[req_idx].is_none() {
-                                    errors_by_object[req_idx] = Some(
-                                        response
-                                            .error
-                                            .clone()
-                                            .unwrap_or_else(|| "distributed lock acquisition failed".to_string()),
-                                    );
+                                Some(response) => {
+                                    if errors_by_object[req_idx].is_none() {
+                                        errors_by_object[req_idx] = Some(
+                                            response
+                                                .error
+                                                .clone()
+                                                .unwrap_or_else(|| "distributed lock acquisition failed".to_string()),
+                                        );
+                                    }
                                 }
-                            }
-                            None => {
-                                if errors_by_object[req_idx].is_none() {
-                                    errors_by_object[req_idx] =
-                                        Some(format!("client {client_idx} returned incomplete batch lock response"));
+                                None => {
+                                    if errors_by_object[req_idx].is_none() {
+                                        errors_by_object[req_idx] =
+                                            Some(format!("client {client_idx} returned incomplete batch lock response"));
+                                    }
+                                }
+                            },
+                            ObjectLockResolution::Succeeded | ObjectLockResolution::Failed => {
+                                if let Some(response) = response
+                                    && response.success
+                                {
+                                    let lock_id = response
+                                        .lock_info
+                                        .as_ref()
+                                        .map(|lock_info| lock_info.id.clone())
+                                        .unwrap_or_else(|| request.lock_id.clone());
+                                    cleanup_lock_ids_by_client[client_idx].push(lock_id);
                                 }
                             }
                         }
                     }
                 }
-                Err(err) => {
-                    for error in errors_by_object.iter_mut().take(requests.len()) {
-                        if error.is_none() {
+                Ok((client_idx, Err(err))) => {
+                    for (req_idx, error) in errors_by_object.iter_mut().enumerate().take(requests.len()) {
+                        if resolution_by_object[req_idx] == ObjectLockResolution::Pending && error.is_none() {
                             *error = Some(format!("client {client_idx} batch lock request failed: {err}"));
                         }
                     }
                 }
+                Err(err) => {
+                    for (req_idx, error) in errors_by_object.iter_mut().enumerate().take(requests.len()) {
+                        if resolution_by_object[req_idx] == ObjectLockResolution::Pending && error.is_none() {
+                            *error = Some(format!("batch lock task join failed: {err}"));
+                        }
+                    }
+                }
             }
+
+            for req_idx in 0..requests.len() {
+                if resolution_by_object[req_idx] != ObjectLockResolution::Pending {
+                    continue;
+                }
+
+                let success_count = lock_ids_by_object[req_idx].len();
+                if success_count >= write_quorum {
+                    resolution_by_object[req_idx] = ObjectLockResolution::Succeeded;
+                    unresolved_objects -= 1;
+                } else if success_count + pending_clients < write_quorum {
+                    resolution_by_object[req_idx] = ObjectLockResolution::Failed;
+                    unresolved_objects -= 1;
+                }
+            }
+        }
+
+        if !pending.is_empty() {
+            let cleanup_requests = requests.clone();
+            let lockers = self.lockers.clone();
+            let handle = tokio::spawn(async move {
+                let mut late_lock_ids_by_client = vec![Vec::new(); lockers.len()];
+                let mut pending = pending;
+                while let Some(join_result) = pending.join_next().await {
+                    match join_result {
+                        Ok((client_idx, Ok(responses))) => {
+                            for (req_idx, request) in cleanup_requests.iter().enumerate() {
+                                if let Some(response) = responses.get(req_idx)
+                                    && response.success
+                                {
+                                    let lock_id = response
+                                        .lock_info
+                                        .as_ref()
+                                        .map(|lock_info| lock_info.id.clone())
+                                        .unwrap_or_else(|| request.lock_id.clone());
+                                    if let Some(client_locks) = late_lock_ids_by_client.get_mut(client_idx) {
+                                        client_locks.push(lock_id);
+                                    }
+                                }
+                            }
+                        }
+                        Ok((_client_idx, Err(err))) => {
+                            tracing::warn!("late distributed delete lock batch request failed: {}", err);
+                        }
+                        Err(err) => {
+                            tracing::warn!("late distributed delete lock batch task join failed: {}", err);
+                        }
+                    }
+                }
+
+                join_all(lockers.iter().cloned().enumerate().filter_map(|(client_idx, client)| {
+                    let lock_ids = late_lock_ids_by_client.get(client_idx).cloned().unwrap_or_default();
+                    if lock_ids.is_empty() {
+                        None
+                    } else {
+                        Some(async move {
+                            if let Err(err) = client.release_locks_batch(&lock_ids).await {
+                                tracing::warn!(
+                                    client_idx,
+                                    lock_count = lock_ids.len(),
+                                    "failed to cleanup late distributed delete locks in batch: {}",
+                                    err
+                                );
+                            }
+                        })
+                    }
+                }))
+                .await;
+            });
+            drop(handle);
         }
 
         let mut failed_map = HashMap::new();
@@ -1092,24 +1198,31 @@ impl SetDisks {
 
         for (req_idx, req) in batch.requests.iter().enumerate() {
             let success_count = lock_ids_by_object[req_idx].len();
-            if success_count >= write_quorum {
-                for (client_idx, lock_id) in lock_ids_by_object[req_idx].drain(..) {
-                    held_lock_ids_by_client[client_idx].push(lock_id);
+            match resolution_by_object[req_idx] {
+                ObjectLockResolution::Succeeded => {
+                    for (client_idx, lock_id) in lock_ids_by_object[req_idx].drain(..) {
+                        held_lock_ids_by_client[client_idx].push(lock_id);
+                    }
+                    locked_objects.insert(req.key.object.as_ref().to_string());
                 }
-                locked_objects.insert(req.key.object.as_ref().to_string());
-            } else {
-                for (client_idx, lock_id) in lock_ids_by_object[req_idx].drain(..) {
-                    rollback_lock_ids_by_client[client_idx].push(lock_id);
+                ObjectLockResolution::Pending | ObjectLockResolution::Failed => {
+                    for (client_idx, lock_id) in lock_ids_by_object[req_idx].drain(..) {
+                        rollback_lock_ids_by_client[client_idx].push(lock_id);
+                    }
+                    failed_map.insert(
+                        (req.key.bucket.as_ref().to_string(), req.key.object.as_ref().to_string()),
+                        errors_by_object[req_idx].clone().unwrap_or_else(|| {
+                            format!("failed to acquire distributed delete lock quorum: {success_count}/{write_quorum}")
+                        }),
+                    );
                 }
-                failed_map.insert(
-                    (req.key.bucket.as_ref().to_string(), req.key.object.as_ref().to_string()),
-                    errors_by_object[req_idx].clone().unwrap_or_else(|| {
-                        format!("failed to acquire distributed delete lock quorum: {success_count}/{write_quorum}")
-                    }),
-                );
             }
         }
 
+        for (client_idx, cleanup_ids) in cleanup_lock_ids_by_client.into_iter().enumerate() {
+            rollback_lock_ids_by_client[client_idx].extend(cleanup_ids);
+        }
+
         self.release_dist_delete_object_locks_batch(rollback_lock_ids_by_client).await;
 
         (failed_map, locked_objects, held_lock_ids_by_client)
@@ -3975,47 +4088,57 @@ async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<ru
 
     for (i, pool) in disks.iter().enumerate() {
         if let Some(disk) = pool {
-            match disk.disk_info(&DiskInfoOptions::default()).await {
-                Ok(res) => ret.push(rustfs_madmin::Disk {
-                    endpoint: eps[i].to_string(),
-                    local: eps[i].is_local,
-                    pool_index: eps[i].pool_idx,
-                    set_index: eps[i].set_idx,
-                    disk_index: eps[i].disk_idx,
-                    state: "ok".to_owned(),
+            let runtime_state = disk.runtime_state();
+            let offline_duration_seconds = disk.offline_duration_secs();
+            if runtime_state.should_probe_for_admin() {
+                match disk.disk_info(&DiskInfoOptions::default()).await {
+                    Ok(res) => ret.push(rustfs_madmin::Disk {
+                        endpoint: eps[i].to_string(),
+                        local: eps[i].is_local,
+                        pool_index: eps[i].pool_idx,
+                        set_index: eps[i].set_idx,
+                        disk_index: eps[i].disk_idx,
+                        state: "ok".to_owned(),
 
-                    root_disk: res.root_disk,
-                    drive_path: res.mount_path.clone(),
-                    healing: res.healing,
-                    scanning: res.scanning,
+                        root_disk: res.root_disk,
+                        drive_path: res.mount_path.clone(),
+                        healing: res.healing,
+                        scanning: res.scanning,
+                        runtime_state: Some(runtime_state.as_str().to_string()),
+                        offline_duration_seconds,
 
-                    uuid: res.id.map_or_else(|| "".to_string(), |id| id.to_string()),
-                    major: res.major as u32,
-                    minor: res.minor as u32,
-                    model: None,
-                    total_space: res.total,
-                    used_space: res.used,
-                    available_space: res.free,
-                    utilization: {
-                        if res.total > 0 {
-                            res.used as f64 / res.total as f64 * 100_f64
-                        } else {
-                            0_f64
-                        }
-                    },
-                    used_inodes: res.used_inodes,
-                    free_inodes: res.free_inodes,
-                    ..Default::default()
-                }),
-                Err(err) => ret.push(rustfs_madmin::Disk {
-                    state: err.to_string(),
-                    endpoint: eps[i].to_string(),
-                    local: eps[i].is_local,
-                    pool_index: eps[i].pool_idx,
-                    set_index: eps[i].set_idx,
-                    disk_index: eps[i].disk_idx,
-                    ..Default::default()
-                }),
+                        uuid: res.id.map_or_else(|| "".to_string(), |id| id.to_string()),
+                        major: res.major as u32,
+                        minor: res.minor as u32,
+                        model: None,
+                        total_space: res.total,
+                        used_space: res.used,
+                        available_space: res.free,
+                        utilization: {
+                            if res.total > 0 {
+                                res.used as f64 / res.total as f64 * 100_f64
+                            } else {
+                                0_f64
+                            }
+                        },
+                        used_inodes: res.used_inodes,
+                        free_inodes: res.free_inodes,
+                        ..Default::default()
+                    }),
+                    Err(err) => ret.push(rustfs_madmin::Disk {
+                        state: err.to_string(),
+                        endpoint: eps[i].to_string(),
+                        local: eps[i].is_local,
+                        pool_index: eps[i].pool_idx,
+                        set_index: eps[i].set_idx,
+                        disk_index: eps[i].disk_idx,
+                        runtime_state: Some(runtime_state.as_str().to_string()),
+                        offline_duration_seconds,
+                        ..Default::default()
+                    }),
+                }
+            } else {
+                ret.push(build_runtime_snapshot_disk(&eps[i], runtime_state, offline_duration_seconds));
             }
         } else {
             ret.push(rustfs_madmin::Disk {
@@ -4024,6 +4147,8 @@ async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<ru
                 pool_index: eps[i].pool_idx,
                 set_index: eps[i].set_idx,
                 disk_index: eps[i].disk_idx,
+                runtime_state: None,
+                offline_duration_seconds: None,
                 state: DiskError::DiskNotFound.to_string(),
                 ..Default::default()
             })
@@ -4032,6 +4157,24 @@ async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<ru
 
     ret
 }
+
+fn build_runtime_snapshot_disk(
+    endpoint: &Endpoint,
+    runtime_state: crate::disk::health_state::RuntimeDriveHealthState,
+    offline_duration_seconds: Option<u64>,
+) -> rustfs_madmin::Disk {
+    rustfs_madmin::Disk {
+        endpoint: endpoint.to_string(),
+        local: endpoint.is_local,
+        pool_index: endpoint.pool_idx,
+        set_index: endpoint.set_idx,
+        disk_index: endpoint.disk_idx,
+        state: runtime_state.as_str().to_string(),
+        runtime_state: Some(runtime_state.as_str().to_string()),
+        offline_duration_seconds,
+        ..Default::default()
+    }
+}
 async fn get_storage_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> rustfs_madmin::StorageInfo {
     // let mut disks = get_disks_info(disks, eps).await;
     // disks.sort_by(|a, b| a.total_space.cmp(&b.total_space));
@@ -4188,14 +4331,17 @@ mod tests {
     use crate::disk::CHECK_PART_VOLUME_NOT_FOUND;
     use crate::disk::endpoint::Endpoint;
     use crate::disk::error::DiskError;
+    use crate::disk::health_state::RuntimeDriveHealthState;
     use crate::endpoints::SetupType;
     use crate::global::{is_dist_erasure, is_erasure, is_erasure_sd, update_erasure_type};
     use crate::store_api::{CompletePart, ObjectInfo};
+    use crate::store_init::save_format_file;
     use rustfs_filemeta::ErasureInfo;
     use rustfs_lock::client::local::LocalClient;
     use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats};
     use serial_test::serial;
     use std::collections::HashMap;
+    use tempfile::TempDir;
     use time::OffsetDateTime;
 
     #[derive(Debug, Default)]
@@ -4240,6 +4386,60 @@ mod tests {
         }
     }
 
+    #[derive(Debug)]
+    struct DelayedBatchClient {
+        inner: Arc<dyn LockClient>,
+        delay: Duration,
+    }
+
+    #[async_trait::async_trait]
+    impl LockClient for DelayedBatchClient {
+        async fn acquire_lock(&self, request: &rustfs_lock::LockRequest) -> rustfs_lock::Result<LockResponse> {
+            self.inner.acquire_lock(request).await
+        }
+
+        async fn acquire_locks_batch(&self, requests: &[rustfs_lock::LockRequest]) -> rustfs_lock::Result<Vec<LockResponse>> {
+            tokio::time::sleep(self.delay).await;
+            self.inner.acquire_locks_batch(requests).await
+        }
+
+        async fn release(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
+            self.inner.release(lock_id).await
+        }
+
+        async fn release_locks_batch(&self, lock_ids: &[rustfs_lock::LockId]) -> rustfs_lock::Result<Vec<bool>> {
+            self.inner.release_locks_batch(lock_ids).await
+        }
+
+        async fn refresh(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
+            self.inner.refresh(lock_id).await
+        }
+
+        async fn force_release(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
+            self.inner.force_release(lock_id).await
+        }
+
+        async fn check_status(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<Option<LockInfo>> {
+            self.inner.check_status(lock_id).await
+        }
+
+        async fn get_stats(&self) -> rustfs_lock::Result<LockStats> {
+            self.inner.get_stats().await
+        }
+
+        async fn close(&self) -> rustfs_lock::Result<()> {
+            self.inner.close().await
+        }
+
+        async fn is_online(&self) -> bool {
+            self.inner.is_online().await
+        }
+
+        async fn is_local(&self) -> bool {
+            self.inner.is_local().await
+        }
+    }
+
     async fn make_test_set_disks(lockers: Vec<Arc<dyn LockClient>>) -> Arc<SetDisks> {
         let endpoints = vec![
             Endpoint::try_from("http://127.0.0.1:9000/data").expect("first endpoint should parse"),
@@ -4296,6 +4496,33 @@ mod tests {
         }
     }
 
+    async fn make_formatted_local_disk_for_info_test(disk_idx: usize, format: &FormatV3) -> (TempDir, Endpoint, DiskStore) {
+        let dir = tempfile::tempdir().expect("tempdir should be created");
+        let mut endpoint =
+            Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
+        endpoint.set_pool_index(0);
+        endpoint.set_set_index(0);
+        endpoint.set_disk_index(disk_idx);
+
+        let disk = new_disk(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .expect("disk should be created");
+
+        let mut disk_format = format.clone();
+        disk_format.erasure.this = format.erasure.sets[0][disk_idx];
+        save_format_file(&Some(disk.clone()), &Some(disk_format))
+            .await
+            .expect("format should be saved");
+
+        (dir, endpoint, disk)
+    }
+
     #[test]
     fn disk_health_entry_returns_cached_value_within_ttl() {
         let entry = DiskHealthEntry {
@@ -4542,6 +4769,113 @@ mod tests {
         drop(guard);
     }
 
+    #[tokio::test(flavor = "multi_thread")]
+    #[serial]
+    async fn test_acquire_dist_delete_object_locks_batch_returns_after_quorum_without_waiting_for_slow_lockers() {
+        let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
+
+        let manager_fast_1 = Arc::new(rustfs_lock::GlobalLockManager::new());
+        let manager_fast_2 = Arc::new(rustfs_lock::GlobalLockManager::new());
+        let manager_fast_3 = Arc::new(rustfs_lock::GlobalLockManager::new());
+        let manager_slow = Arc::new(rustfs_lock::GlobalLockManager::new());
+
+        let client_fast_1: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager_fast_1));
+        let client_fast_2: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager_fast_2));
+        let client_fast_3: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager_fast_3));
+        let client_slow: Arc<dyn LockClient> = Arc::new(DelayedBatchClient {
+            inner: Arc::new(LocalClient::with_manager(manager_slow.clone())),
+            delay: Duration::from_millis(250),
+        });
+
+        let set_disks = make_test_set_disks(vec![client_fast_1, client_fast_2, client_fast_3, client_slow]).await;
+
+        let batch = rustfs_lock::BatchLockRequest::new(set_disks.locker_owner.as_str())
+            .with_all_or_nothing(false)
+            .add_write_lock(ObjectKey::new("bucket", "object-a"))
+            .add_write_lock(ObjectKey::new("bucket", "object-b"));
+
+        let started = Instant::now();
+        let (failed_map, locked_objects, held_lock_ids_by_client) =
+            set_disks.acquire_dist_delete_object_locks_batch(&batch).await;
+
+        assert!(
+            started.elapsed() < Duration::from_millis(150),
+            "batch distributed delete locks should return once quorum is satisfied"
+        );
+        assert!(failed_map.is_empty());
+        assert_eq!(locked_objects.len(), 2);
+
+        set_disks
+            .release_dist_delete_object_locks_batch(held_lock_ids_by_client)
+            .await;
+
+        tokio::time::sleep(Duration::from_millis(350)).await;
+
+        let slow_lock = NamespaceLock::with_local_manager("slow-node".to_string(), manager_slow);
+        let guard_a = slow_lock
+            .get_write_lock(ObjectKey::new("bucket", "object-a"), "owner-b", Duration::from_millis(100))
+            .await
+            .expect("late successful batch lock should be cleaned up for object-a");
+        let guard_b = slow_lock
+            .get_write_lock(ObjectKey::new("bucket", "object-b"), "owner-b", Duration::from_millis(100))
+            .await
+            .expect("late successful batch lock should be cleaned up for object-b");
+
+        drop(guard_a);
+        drop(guard_b);
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    #[serial]
+    async fn test_acquire_dist_delete_object_locks_batch_fails_early_and_cleans_up_late_successes() {
+        let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
+
+        let manager_fast = Arc::new(rustfs_lock::GlobalLockManager::new());
+        let manager_slow = Arc::new(rustfs_lock::GlobalLockManager::new());
+
+        let client_fast: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(manager_fast));
+        let client_fail_1: Arc<dyn LockClient> = Arc::new(FailingClient);
+        let client_fail_2: Arc<dyn LockClient> = Arc::new(FailingClient);
+        let client_slow: Arc<dyn LockClient> = Arc::new(DelayedBatchClient {
+            inner: Arc::new(LocalClient::with_manager(manager_slow.clone())),
+            delay: Duration::from_millis(250),
+        });
+
+        let set_disks = make_test_set_disks(vec![client_fast, client_fail_1, client_fail_2, client_slow]).await;
+        let batch = rustfs_lock::BatchLockRequest::new(set_disks.locker_owner.as_str())
+            .with_all_or_nothing(false)
+            .add_write_lock(ObjectKey::new("bucket", "object-a"))
+            .add_write_lock(ObjectKey::new("bucket", "object-b"));
+
+        let started = Instant::now();
+        let (failed_map, locked_objects, held_lock_ids_by_client) =
+            set_disks.acquire_dist_delete_object_locks_batch(&batch).await;
+
+        assert!(
+            started.elapsed() < Duration::from_millis(150),
+            "batch distributed delete locks should fail as soon as quorum becomes impossible"
+        );
+        assert!(locked_objects.is_empty());
+        assert!(failed_map.contains_key(&("bucket".to_string(), "object-a".to_string())));
+        assert!(failed_map.contains_key(&("bucket".to_string(), "object-b".to_string())));
+        assert_eq!(held_lock_ids_by_client.iter().map(Vec::len).sum::<usize>(), 0);
+
+        tokio::time::sleep(Duration::from_millis(350)).await;
+
+        let slow_lock = NamespaceLock::with_local_manager("slow-node".to_string(), manager_slow);
+        let guard_a = slow_lock
+            .get_write_lock(ObjectKey::new("bucket", "object-a"), "owner-b", Duration::from_millis(100))
+            .await
+            .expect("late successful batch failure cleanup should release object-a");
+        let guard_b = slow_lock
+            .get_write_lock(ObjectKey::new("bucket", "object-b"), "owner-b", Duration::from_millis(100))
+            .await
+            .expect("late successful batch failure cleanup should release object-b");
+
+        drop(guard_a);
+        drop(guard_b);
+    }
+
     #[test]
     fn test_common_parity() {
         // Test common parity calculation
@@ -4761,6 +5095,45 @@ mod tests {
         assert!(should_heal);
     }
 
+    #[tokio::test]
+    async fn test_get_disks_info_uses_runtime_snapshot_for_suspect_and_offline_disks() {
+        let format = FormatV3::new(1, 3);
+        let mut temp_dirs = Vec::new();
+        let mut endpoints = Vec::new();
+        let mut disks = Vec::new();
+
+        for disk_idx in 0..3 {
+            let (dir, endpoint, disk) = make_formatted_local_disk_for_info_test(disk_idx, &format).await;
+            temp_dirs.push(dir);
+            endpoints.push(endpoint);
+            disks.push(Some(disk));
+        }
+
+        disks[1]
+            .as_ref()
+            .expect("disk 1 should exist")
+            .force_runtime_state_for_test(RuntimeDriveHealthState::Suspect);
+        disks[2]
+            .as_ref()
+            .expect("disk 2 should exist")
+            .force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
+
+        let info = get_disks_info(&disks, &endpoints).await;
+        assert_eq!(info.len(), 3);
+
+        assert_eq!(info[0].state, "ok");
+        assert_eq!(info[0].runtime_state.as_deref(), Some("online"));
+        assert!(!info[0].drive_path.is_empty(), "online disk should keep immediate disk_info probe");
+
+        assert_eq!(info[1].state, "suspect");
+        assert_eq!(info[1].runtime_state.as_deref(), Some("suspect"));
+        assert!(info[1].drive_path.is_empty(), "suspect disk should use runtime snapshot fallback");
+
+        assert_eq!(info[2].state, "offline");
+        assert_eq!(info[2].runtime_state.as_deref(), Some("offline"));
+        assert!(info[2].drive_path.is_empty(), "offline disk should use runtime snapshot fallback");
+    }
+
     #[test]
     fn test_dangling_meta_errs_count() {
         // Test counting dangling metadata errors

crates/ecstore/src/set_disk/lock.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use super::*;
+use crate::disk::health_state::DriveMembershipSnapshot;
 
 impl SetDisks {
     pub(super) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
@@ -72,41 +73,28 @@ impl SetDisks {
     }
 
     pub(super) async fn get_online_disks(&self) -> Vec<Option<DiskStore>> {
-        let mut disks = self.get_disks_internal().await;
-
-        // TODO: diskinfo filter online
-
-        let mut new_disk = Vec::with_capacity(disks.len());
-
-        for disk in disks.iter() {
-            if let Some(d) = disk
-                && d.is_online().await
-            {
-                new_disk.push(disk.clone());
-            }
-        }
+        let snapshot = self.drive_membership_snapshot().await;
+        let mut disks = snapshot.strict_online_candidates().into_iter().map(Some).collect::<Vec<_>>();
 
         let mut rng = rand::rng();
-
         disks.shuffle(&mut rng);
 
-        new_disk
-        // let disks = self.get_disks_internal().await;
-        // let (filtered, _) = self.filter_online_disks(disks).await;
-        // filtered.into_iter().filter(|disk| disk.is_some()).collect()
+        disks
     }
 
     pub(super) async fn get_online_local_disks(&self) -> Vec<Option<DiskStore>> {
-        let mut disks = self.get_online_disks().await;
+        let snapshot = self.drive_membership_snapshot().await;
+        let mut disks = snapshot
+            .strict_online_local_candidates()
+            .into_iter()
+            .map(Some)
+            .collect::<Vec<_>>();
 
         let mut rng = rand::rng();
 
         disks.shuffle(&mut rng);
 
         disks
-            .into_iter()
-            .filter(|v| v.as_ref().is_some_and(|d| d.is_local()))
-            .collect()
     }
 
     pub async fn get_online_disks_with_healing(&self, incl_healing: bool) -> (Vec<DiskStore>, bool) {
@@ -114,28 +102,41 @@ impl SetDisks {
         (disks, healing > 0)
     }
 
-    pub async fn get_online_disks_with_healing_and_info(&self, incl_healing: bool) -> (Vec<DiskStore>, Vec<DiskInfo>, usize) {
-        let mut disks = self.get_disks_internal().await;
+    pub async fn drive_membership_snapshot(&self) -> DriveMembershipSnapshot {
+        let disks = self.get_disks_internal().await;
+        DriveMembershipSnapshot::from_optional_disks(&disks)
+    }
 
-        let mut infos = Vec::with_capacity(disks.len());
+    pub async fn get_online_disks_with_healing_and_info(&self, incl_healing: bool) -> (Vec<DiskStore>, Vec<DiskInfo>, usize) {
+        let snapshot = self.drive_membership_snapshot().await;
+        let mut disks = snapshot.scanner_heal_candidates().into_iter().map(Some).collect::<Vec<_>>();
+
+        let mut infos: Vec<Option<DiskInfo>> = vec![None; disks.len()];
 
         let mut futures = Vec::with_capacity(disks.len());
-        let mut numbers: Vec<usize> = (0..disks.len()).collect();
         {
             let mut rng = rand::rng();
             disks.shuffle(&mut rng);
-
-            numbers.shuffle(&mut rng);
         }
 
-        for &i in numbers.iter() {
-            let disk = disks[i].clone();
+        for (i, disk) in disks.iter().cloned().enumerate() {
             futures.push(async move {
-                if let Some(disk) = disk {
-                    disk.disk_info(&DiskInfoOptions::default()).await
+                let info = if let Some(disk) = disk {
+                    match disk.disk_info(&DiskInfoOptions::default()).await {
+                        Ok(info) => info,
+                        Err(err) => DiskInfo {
+                            error: err.to_string(),
+                            ..Default::default()
+                        },
+                    }
                 } else {
-                    Err(DiskError::DiskNotFound)
-                }
+                    DiskInfo {
+                        error: DiskError::DiskNotFound.to_string(),
+                        ..Default::default()
+                    }
+                };
+
+                Ok((i, info))
             });
         }
 
@@ -143,13 +144,13 @@ impl SetDisks {
         let processor = get_global_processors().metadata_processor();
         let results = processor.execute_batch(futures).await;
 
-        for result in results {
+        for (submitted_idx, result) in results.into_iter().enumerate() {
             match result {
-                Ok(res) => {
-                    infos.push(res);
+                Ok((disk_idx, info)) => {
+                    infos[disk_idx] = Some(info);
                 }
                 Err(err) => {
-                    infos.push(DiskInfo {
+                    infos[submitted_idx] = Some(DiskInfo {
                         error: err.to_string(),
                         ..Default::default()
                     });
@@ -167,8 +168,11 @@ impl SetDisks {
         let mut new_disks = Vec::new();
         let mut new_infos = Vec::new();
 
-        for &i in numbers.iter() {
-            let (info, disk) = (infos[i].clone(), disks[i].clone());
+        for (disk, info) in disks.into_iter().zip(infos.into_iter()) {
+            let Some(info) = info else {
+                continue;
+            };
+
             if !info.error.is_empty() || disk.is_none() {
                 continue;
             }
@@ -367,3 +371,160 @@ impl SetDisks {
         Ok((new_disks, new_infos, healing))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::store_init::save_format_file;
+    use tempfile::TempDir;
+    use tokio::sync::RwLock;
+
+    async fn make_formatted_local_disk(disk_idx: usize, format: &FormatV3) -> (TempDir, Endpoint, DiskStore) {
+        let dir = tempfile::tempdir().expect("tempdir should be created");
+        let mut endpoint =
+            Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
+        endpoint.set_pool_index(0);
+        endpoint.set_set_index(0);
+        endpoint.set_disk_index(disk_idx);
+
+        let disk = new_disk(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .expect("local disk should be created");
+
+        let mut disk_format = format.clone();
+        disk_format.erasure.this = format.erasure.sets[0][disk_idx];
+        save_format_file(&Some(disk.clone()), &Some(disk_format))
+            .await
+            .expect("format should be saved");
+
+        (dir, endpoint, disk)
+    }
+
+    #[tokio::test]
+    async fn get_online_disks_with_healing_and_info_keeps_disk_and_info_aligned() {
+        let disk_count = 8;
+        let format = FormatV3::new(1, disk_count);
+
+        let mut temp_dirs = Vec::with_capacity(disk_count);
+        let mut endpoints = Vec::with_capacity(disk_count);
+        let mut disks = Vec::with_capacity(disk_count);
+
+        for disk_idx in 0..disk_count {
+            let (temp_dir, endpoint, disk) = make_formatted_local_disk(disk_idx, &format).await;
+            temp_dirs.push(temp_dir);
+            endpoints.push(endpoint);
+            disks.push(Some(disk));
+        }
+
+        let set_disks = SetDisks::new(
+            "test-owner".to_string(),
+            Arc::new(RwLock::new(disks)),
+            disk_count,
+            disk_count / 2,
+            0,
+            0,
+            endpoints,
+            format,
+            Vec::new(),
+        )
+        .await;
+
+        for _ in 0..32 {
+            let (online_disks, infos, healing) = set_disks.get_online_disks_with_healing_and_info(false).await;
+            assert_eq!(healing, 0);
+            assert_eq!(online_disks.len(), disk_count);
+            assert_eq!(infos.len(), disk_count);
+
+            for (disk, info) in online_disks.iter().zip(infos.iter()) {
+                assert!(
+                    info.error.is_empty(),
+                    "unexpected disk_info error for {}: {}",
+                    disk.endpoint(),
+                    info.error
+                );
+                assert_eq!(info.endpoint, disk.endpoint().to_string());
+                assert_eq!(
+                    info.id,
+                    disk.get_disk_id().await.expect("disk id lookup should succeed"),
+                    "disk info should stay aligned with disk {}",
+                    disk.endpoint()
+                );
+            }
+        }
+
+        drop(temp_dirs);
+    }
+
+    #[tokio::test]
+    async fn drive_membership_snapshot_filters_offline_disks_from_candidates() {
+        let disk_count = 4;
+        let format = FormatV3::new(1, disk_count);
+
+        let mut temp_dirs = Vec::with_capacity(disk_count);
+        let mut endpoints = Vec::with_capacity(disk_count);
+        let mut disks = Vec::with_capacity(disk_count);
+
+        for disk_idx in 0..disk_count {
+            let (temp_dir, endpoint, disk) = make_formatted_local_disk(disk_idx, &format).await;
+            temp_dirs.push(temp_dir);
+            endpoints.push(endpoint);
+            disks.push(Some(disk));
+        }
+
+        let set_disks = SetDisks::new(
+            "test-owner".to_string(),
+            Arc::new(RwLock::new(disks)),
+            disk_count,
+            disk_count / 2,
+            0,
+            0,
+            endpoints,
+            format,
+            Vec::new(),
+        )
+        .await;
+
+        let all_disks = set_disks.get_disks_internal().await;
+        all_disks[1]
+            .as_ref()
+            .expect("disk 1 should exist")
+            .force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Suspect);
+        all_disks[2]
+            .as_ref()
+            .expect("disk 2 should exist")
+            .force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Returning);
+        all_disks[3]
+            .as_ref()
+            .expect("disk 3 should exist")
+            .force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Offline);
+
+        let snapshot = set_disks.drive_membership_snapshot().await;
+        assert_eq!(snapshot.online.len(), 1);
+        assert_eq!(snapshot.suspect.len(), 1);
+        assert_eq!(snapshot.returning.len(), 1);
+        assert_eq!(snapshot.offline.len(), 1);
+        assert_eq!(snapshot.scanner_heal_candidates().len(), 3);
+
+        let strict_online = set_disks.get_online_disks().await;
+        assert_eq!(strict_online.len(), 1, "strict online selection should exclude suspect/returning/offline");
+
+        let (online_disks, infos, healing) = set_disks.get_online_disks_with_healing_and_info(false).await;
+        assert_eq!(healing, 0);
+        assert_eq!(online_disks.len(), 3);
+        assert_eq!(infos.len(), 3);
+        assert!(
+            online_disks
+                .iter()
+                .all(|disk| { disk.runtime_state() != crate::disk::health_state::RuntimeDriveHealthState::Offline }),
+            "offline disks should be filtered by membership snapshot"
+        );
+
+        drop(temp_dirs);
+    }
+}