fix(ecstore): harden runtime read-path quorum handling (#2872)

crates/ecstore/src/cache_value/metacache_set.rs

@@ -85,7 +85,7 @@ impl Clone for ListPathRawOptions {
 
 pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
     if opts.disks.is_empty() {
-        return Err(DiskError::other("list_path_raw: 0 drives provided"));
+        return Err(DiskError::ErasureReadQuorum);
     }
 
     let mut jobs: Vec<tokio::task::JoinHandle<std::result::Result<(), DiskError>>> = Vec::new();
@@ -415,6 +415,15 @@ mod tests {
     use super::*;
     use rustfs_filemeta::MetacacheWriter;
 
+    #[tokio::test]
+    async fn list_path_raw_empty_disks_returns_read_quorum() {
+        let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())
+            .await
+            .expect_err("empty drive list should fail");
+
+        assert_eq!(err, DiskError::ErasureReadQuorum);
+    }
+
     #[tokio::test]
     async fn peek_with_timeout_times_out_on_silent_reader() {
         let (_writer, reader) = tokio::io::duplex(64);

crates/ecstore/src/data_usage.rs

@@ -15,7 +15,11 @@
 pub mod local_snapshot;
 
 use crate::{
-    bucket::metadata_sys::get_replication_config, config::com::read_config, disk::DiskAPI, error::Error, store::ECStore,
+    bucket::metadata_sys::get_replication_config,
+    config::com::read_config,
+    disk::DiskAPI,
+    error::{Error, classify_system_path_failure_reason},
+    store::ECStore,
     store_api::ListOperations,
 };
 pub use local_snapshot::{
@@ -26,6 +30,7 @@ pub use local_snapshot::{
 use rustfs_common::data_usage::{
     BucketTargetUsageInfo, BucketUsageInfo, DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, SizeSummary,
 };
+use rustfs_io_metrics::record_system_path_failure;
 use rustfs_utils::path::SLASH_SEPARATOR;
 use std::{
     collections::{HashMap, HashSet, hash_map::Entry},
@@ -109,7 +114,16 @@ pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsa
     let buf: Vec<u8> = match read_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH).await {
         Ok(data) => data,
         Err(e) => {
-            error!("Failed to read data usage info from backend: {}", e);
+            let reason = classify_system_path_failure_reason(&e);
+            record_system_path_failure("data_usage", "read_primary", reason);
+            error!(
+                path_kind = "data_usage",
+                operation = "read_primary",
+                reason,
+                object = %DATA_USAGE_OBJ_NAME_PATH.as_str(),
+                error = %e,
+                "system path read failed"
+            );
 
             match read_config(store.clone(), format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()).as_str()).await {
                 Ok(data) => data,
@@ -117,7 +131,16 @@ pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsa
                     if e == Error::ConfigNotFound {
                         return Ok(DataUsageInfo::default());
                     }
-                    error!("Failed to read data usage info from backend: {}", e);
+                    let reason = classify_system_path_failure_reason(&e);
+                    record_system_path_failure("data_usage", "read_backup", reason);
+                    error!(
+                        path_kind = "data_usage",
+                        operation = "read_backup",
+                        reason,
+                        object = %format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()),
+                        error = %e,
+                        "system path read failed"
+                    );
                     return Err(Error::other(e));
                 }
             }

crates/ecstore/src/disk/disk_store.rs

@@ -31,7 +31,7 @@ use std::{
     path::PathBuf,
     sync::{
         Arc,
-        atomic::{AtomicI64, AtomicU32, Ordering},
+        atomic::{AtomicI64, AtomicU32, AtomicU64, Ordering},
     },
     time::Duration,
 };
@@ -138,6 +138,14 @@ pub struct DiskHealthTracker {
     pub offline_since_unix_secs: AtomicI64,
     /// Last runtime state transition timestamp
     pub last_transition_unix_secs: AtomicI64,
+    /// Last successfully probed total space in bytes
+    pub last_capacity_total: AtomicU64,
+    /// Last successfully probed used space in bytes
+    pub last_capacity_used: AtomicU64,
+    /// Last successfully probed free space in bytes
+    pub last_capacity_free: AtomicU64,
+    /// Last successful capacity probe timestamp
+    pub last_capacity_probe_unix_secs: AtomicI64,
 }
 
 impl DiskHealthTracker {
@@ -158,6 +166,10 @@ impl DiskHealthTracker {
             consecutive_successes: AtomicU32::new(0),
             offline_since_unix_secs: AtomicI64::new(0),
             last_transition_unix_secs: AtomicI64::new(now / 1_000_000_000),
+            last_capacity_total: AtomicU64::new(0),
+            last_capacity_used: AtomicU64::new(0),
+            last_capacity_free: AtomicU64::new(0),
+            last_capacity_probe_unix_secs: AtomicI64::new(0),
         }
     }
 
@@ -170,6 +182,28 @@ impl DiskHealthTracker {
         self.last_success.store(now, Ordering::Relaxed);
     }
 
+    pub fn record_capacity_probe(&self, total: u64, used: u64, free: u64) {
+        self.last_capacity_total.store(total, Ordering::Release);
+        self.last_capacity_used.store(used, Ordering::Release);
+        self.last_capacity_free.store(free, Ordering::Release);
+        self.last_capacity_probe_unix_secs
+            .store(current_unix_secs() as i64, Ordering::Release);
+    }
+
+    pub fn last_capacity_snapshot(&self) -> Option<(u64, u64, u64, u64)> {
+        let ts = self.last_capacity_probe_unix_secs.load(Ordering::Acquire);
+        if ts <= 0 {
+            return None;
+        }
+
+        Some((
+            self.last_capacity_total.load(Ordering::Acquire),
+            self.last_capacity_used.load(Ordering::Acquire),
+            self.last_capacity_free.load(Ordering::Acquire),
+            ts as u64,
+        ))
+    }
+
     /// Check if disk is faulty
     pub fn is_faulty(&self) -> bool {
         self.status.load(Ordering::Acquire) == DISK_HEALTH_FAULTY
@@ -463,6 +497,14 @@ impl LocalDiskWrapper {
         self.health.offline_duration().map(|duration| duration.as_secs())
     }
 
+    pub fn last_capacity_snapshot(&self) -> Option<(u64, u64, u64, u64)> {
+        self.health.last_capacity_snapshot()
+    }
+
+    pub fn record_capacity_probe(&self, total: u64, used: u64, free: u64) {
+        self.health.record_capacity_probe(total, used, free);
+    }
+
     #[cfg(test)]
     pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
         self.health.force_runtime_state_for_test(state);

crates/ecstore/src/disk/mod.rs

@@ -427,6 +427,20 @@ impl Disk {
         }
     }
 
+    pub fn last_capacity_snapshot(&self) -> Option<(u64, u64, u64, u64)> {
+        match self {
+            Disk::Local(local_disk) => local_disk.last_capacity_snapshot(),
+            Disk::Remote(remote_disk) => remote_disk.last_capacity_snapshot(),
+        }
+    }
+
+    pub fn record_capacity_probe(&self, total: u64, used: u64, free: u64) {
+        match self {
+            Disk::Local(local_disk) => local_disk.record_capacity_probe(total, used, free),
+            Disk::Remote(remote_disk) => remote_disk.record_capacity_probe(total, used, free),
+        }
+    }
+
     #[cfg(test)]
     pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
         match self {

crates/ecstore/src/error.rs

@@ -678,6 +678,18 @@ pub fn is_err_read_quorum(err: &Error) -> bool {
     matches!(err, &StorageError::ErasureReadQuorum)
 }
 
+pub fn classify_system_path_failure_reason(err: &Error) -> &'static str {
+    match err {
+        StorageError::ConfigNotFound => "config_not_found",
+        StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _) => "read_quorum",
+        StorageError::Io(io_err) => match io_err.kind() {
+            std::io::ErrorKind::TimedOut => "timeout",
+            _ => "io",
+        },
+        _ => "other",
+    }
+}
+
 pub fn is_err_invalid_upload_id(err: &Error) -> bool {
     matches!(err, &StorageError::InvalidUploadID(_, _, _))
 }

crates/ecstore/src/rpc/remote_disk.rs

@@ -135,6 +135,14 @@ impl RemoteDisk {
         self.health.offline_duration().map(|duration| duration.as_secs())
     }
 
+    pub fn last_capacity_snapshot(&self) -> Option<(u64, u64, u64, u64)> {
+        self.health.last_capacity_snapshot()
+    }
+
+    pub fn record_capacity_probe(&self, total: u64, used: u64, free: u64) {
+        self.health.record_capacity_probe(total, used, free);
+    }
+
     #[cfg(test)]
     pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
         self.health.force_runtime_state_for_test(state);

crates/ecstore/src/set_disk.rs

@@ -101,7 +101,7 @@ use s3s::header::{X_AMZ_OBJECT_LOCK_LEGAL_HOLD, X_AMZ_OBJECT_LOCK_MODE, X_AMZ_OB
 use sha2::{Digest, Sha256};
 use std::hash::Hash;
 use std::mem::{self};
-use std::time::{Instant, SystemTime};
+use std::time::{Instant, SystemTime, UNIX_EPOCH};
 use std::{
     collections::{HashMap, HashSet},
     io::{Cursor, Write},
@@ -4146,56 +4146,78 @@ async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<ru
         if let Some(disk) = pool {
             let runtime_state = disk.runtime_state();
             let offline_duration_seconds = disk.offline_duration_secs();
-            if runtime_state.should_probe_for_admin() {
+            let capacity_snapshot = disk.last_capacity_snapshot();
+            if runtime_state.should_probe_for_admin()
+                || runtime_state == crate::disk::health_state::RuntimeDriveHealthState::Suspect
+            {
                 match disk.disk_info(&DiskInfoOptions::default()).await {
-                    Ok(res) => ret.push(rustfs_madmin::Disk {
-                        endpoint: eps[i].to_string(),
-                        local: eps[i].is_local,
-                        pool_index: eps[i].pool_idx,
-                        set_index: eps[i].set_idx,
-                        disk_index: eps[i].disk_idx,
-                        state: "ok".to_owned(),
+                    Ok(res) => {
+                        disk.record_capacity_probe(res.total, res.used, res.free);
+                        ret.push(rustfs_madmin::Disk {
+                            endpoint: eps[i].to_string(),
+                            local: eps[i].is_local,
+                            pool_index: eps[i].pool_idx,
+                            set_index: eps[i].set_idx,
+                            disk_index: eps[i].disk_idx,
+                            state: "ok".to_owned(),
 
-                        root_disk: res.root_disk,
-                        drive_path: res.mount_path.clone(),
-                        healing: res.healing,
-                        scanning: res.scanning,
-                        runtime_state: Some(runtime_state.as_str().to_string()),
-                        offline_duration_seconds,
+                            root_disk: res.root_disk,
+                            drive_path: res.mount_path.clone(),
+                            healing: res.healing,
+                            scanning: res.scanning,
+                            runtime_state: Some(runtime_state.as_str().to_string()),
+                            offline_duration_seconds,
+                            capacity_observation_source: Some("live_probe".to_owned()),
+                            capacity_observation_age_seconds: Some(0),
 
-                        uuid: res.id.map_or_else(|| "".to_string(), |id| id.to_string()),
-                        major: res.major as u32,
-                        minor: res.minor as u32,
-                        model: None,
-                        total_space: res.total,
-                        used_space: res.used,
-                        available_space: res.free,
-                        physical_device_ids: (!res.physical_device_ids.is_empty()).then_some(res.physical_device_ids.clone()),
-                        utilization: {
-                            if res.total > 0 {
-                                res.used as f64 / res.total as f64 * 100_f64
-                            } else {
-                                0_f64
-                            }
-                        },
-                        used_inodes: res.used_inodes,
-                        free_inodes: res.free_inodes,
-                        ..Default::default()
-                    }),
-                    Err(err) => ret.push(rustfs_madmin::Disk {
-                        state: err.to_string(),
-                        endpoint: eps[i].to_string(),
-                        local: eps[i].is_local,
-                        pool_index: eps[i].pool_idx,
-                        set_index: eps[i].set_idx,
-                        disk_index: eps[i].disk_idx,
-                        runtime_state: Some(runtime_state.as_str().to_string()),
-                        offline_duration_seconds,
-                        ..Default::default()
-                    }),
+                            uuid: res.id.map_or_else(|| "".to_string(), |id| id.to_string()),
+                            major: res.major as u32,
+                            minor: res.minor as u32,
+                            model: None,
+                            total_space: res.total,
+                            used_space: res.used,
+                            available_space: res.free,
+                            physical_device_ids: (!res.physical_device_ids.is_empty()).then_some(res.physical_device_ids.clone()),
+                            utilization: utilization_percent(res.total, res.used),
+                            used_inodes: res.used_inodes,
+                            free_inodes: res.free_inodes,
+                            ..Default::default()
+                        });
+                    }
+                    Err(err) => {
+                        let mut disk_info = rustfs_madmin::Disk {
+                            state: err.to_string(),
+                            endpoint: eps[i].to_string(),
+                            local: eps[i].is_local,
+                            pool_index: eps[i].pool_idx,
+                            set_index: eps[i].set_idx,
+                            disk_index: eps[i].disk_idx,
+                            runtime_state: Some(runtime_state.as_str().to_string()),
+                            offline_duration_seconds,
+                            ..Default::default()
+                        };
+                        if let Some((total, used, free, _)) = capacity_snapshot {
+                            disk_info.total_space = total;
+                            disk_info.used_space = used;
+                            disk_info.available_space = free;
+                            disk_info.utilization = utilization_percent(total, used);
+                            disk_info.capacity_observation_source = Some("snapshot".to_owned());
+                            disk_info.capacity_observation_age_seconds = capacity_snapshot
+                                .map(|(_, _, _, probe_unix_secs)| capacity_snapshot_age_seconds(probe_unix_secs));
+                        } else {
+                            disk_info.capacity_observation_source = Some("missing".to_owned());
+                            disk_info.capacity_observation_age_seconds = Some(0);
+                        }
+                        ret.push(disk_info);
+                    }
                 }
             } else {
-                ret.push(build_runtime_snapshot_disk(&eps[i], runtime_state, offline_duration_seconds));
+                ret.push(build_runtime_snapshot_disk(
+                    &eps[i],
+                    runtime_state,
+                    offline_duration_seconds,
+                    capacity_snapshot,
+                ));
             }
         } else {
             ret.push(rustfs_madmin::Disk {
@@ -4207,6 +4229,8 @@ async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<ru
                 runtime_state: None,
                 offline_duration_seconds: None,
                 state: DiskError::DiskNotFound.to_string(),
+                capacity_observation_source: Some("missing".to_owned()),
+                capacity_observation_age_seconds: Some(0),
                 ..Default::default()
             })
         }
@@ -4219,8 +4243,9 @@ fn build_runtime_snapshot_disk(
     endpoint: &Endpoint,
     runtime_state: crate::disk::health_state::RuntimeDriveHealthState,
     offline_duration_seconds: Option<u64>,
+    capacity_snapshot: Option<(u64, u64, u64, u64)>,
 ) -> rustfs_madmin::Disk {
-    rustfs_madmin::Disk {
+    let mut disk = rustfs_madmin::Disk {
         endpoint: endpoint.to_string(),
         local: endpoint.is_local,
         pool_index: endpoint.pool_idx,
@@ -4230,7 +4255,38 @@ fn build_runtime_snapshot_disk(
         runtime_state: Some(runtime_state.as_str().to_string()),
         offline_duration_seconds,
         ..Default::default()
+    };
+
+    if let Some((total, used, free, _)) = capacity_snapshot {
+        disk.total_space = total;
+        disk.used_space = used;
+        disk.available_space = free;
+        disk.utilization = utilization_percent(total, used);
+        disk.capacity_observation_source = Some("snapshot".to_owned());
+        disk.capacity_observation_age_seconds =
+            capacity_snapshot.map(|(_, _, _, probe_unix_secs)| capacity_snapshot_age_seconds(probe_unix_secs));
+    } else {
+        disk.capacity_observation_source = Some("missing".to_owned());
+        disk.capacity_observation_age_seconds = Some(0);
     }
+
+    disk
+}
+
+fn utilization_percent(total: u64, used: u64) -> f64 {
+    if total > 0 {
+        used as f64 / total as f64 * 100_f64
+    } else {
+        0_f64
+    }
+}
+
+fn capacity_snapshot_age_seconds(probe_unix_secs: u64) -> u64 {
+    let now_unix_secs = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|dur| dur.as_secs())
+        .unwrap_or(probe_unix_secs);
+    now_unix_secs.saturating_sub(probe_unix_secs)
 }
 async fn get_storage_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> rustfs_madmin::StorageInfo {
     // let mut disks = get_disks_info(disks, eps).await;
@@ -5313,7 +5369,7 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_get_disks_info_uses_runtime_snapshot_for_suspect_and_offline_disks() {
+    async fn test_get_disks_info_preserves_runtime_state_for_suspect_and_offline_disks() {
         let format = FormatV3::new(1, 3);
         let mut temp_dirs = Vec::new();
         let mut endpoints = Vec::new();
@@ -5342,9 +5398,9 @@ mod tests {
         assert_eq!(info[0].runtime_state.as_deref(), Some("online"));
         assert!(!info[0].drive_path.is_empty(), "online disk should keep immediate disk_info probe");
 
-        assert_eq!(info[1].state, "suspect");
+        assert_eq!(info[1].state, "ok");
         assert_eq!(info[1].runtime_state.as_deref(), Some("suspect"));
-        assert!(info[1].drive_path.is_empty(), "suspect disk should use runtime snapshot fallback");
+        assert!(!info[1].drive_path.is_empty(), "suspect disk should still probe for fresher disk info");
 
         assert_eq!(info[2].state, "offline");
         assert_eq!(info[2].runtime_state.as_deref(), Some("offline"));

crates/ecstore/src/set_disk/lock.rs

@@ -107,101 +107,111 @@ impl SetDisks {
         DriveMembershipSnapshot::from_optional_disks(&disks)
     }
 
+    fn reprobe_runtime_candidates_once(&self, disks: &[DiskStore]) {
+        for disk in disks {
+            if disk.runtime_state() != crate::disk::health_state::RuntimeDriveHealthState::Online {
+                disk.reset_health_for_store_init_retry();
+            }
+        }
+    }
+
     pub async fn get_online_disks_with_healing_and_info(&self, incl_healing: bool) -> (Vec<DiskStore>, Vec<DiskInfo>, usize) {
         let snapshot = self.drive_membership_snapshot().await;
-        let mut disks = snapshot.scanner_heal_candidates().into_iter().map(Some).collect::<Vec<_>>();
+        let mut membership_candidates = snapshot.scanner_heal_candidates();
+        let mut reprobed = false;
 
-        let mut infos: Vec<Option<DiskInfo>> = vec![None; disks.len()];
+        loop {
+            let mut disks = membership_candidates.clone();
+            let mut infos: Vec<Option<DiskInfo>> = vec![None; disks.len()];
 
-        let mut futures = Vec::with_capacity(disks.len());
-        {
-            let mut rng = rand::rng();
-            disks.shuffle(&mut rng);
-        }
+            let mut futures = Vec::with_capacity(disks.len());
+            {
+                let mut rng = rand::rng();
+                disks.shuffle(&mut rng);
+            }
 
-        for (i, disk) in disks.iter().cloned().enumerate() {
-            futures.push(async move {
-                let info = if let Some(disk) = disk {
-                    match disk.disk_info(&DiskInfoOptions::default()).await {
+            for (i, disk) in disks.iter().cloned().enumerate() {
+                futures.push(async move {
+                    let info = match disk.disk_info(&DiskInfoOptions::default()).await {
                         Ok(info) => info,
                         Err(err) => DiskInfo {
                             error: err.to_string(),
                             ..Default::default()
                         },
+                    };
+
+                    Ok((i, info))
+                });
+            }
+
+            let processor = get_global_processors().metadata_processor();
+            let results = processor.execute_batch(futures).await;
+
+            for (submitted_idx, result) in results.into_iter().enumerate() {
+                match result {
+                    Ok((disk_idx, info)) => {
+                        infos[disk_idx] = Some(info);
                     }
-                } else {
-                    DiskInfo {
-                        error: DiskError::DiskNotFound.to_string(),
-                        ..Default::default()
+                    Err(err) => {
+                        infos[submitted_idx] = Some(DiskInfo {
+                            error: err.to_string(),
+                            ..Default::default()
+                        });
                     }
+                }
+            }
+
+            let mut healing: usize = 0;
+
+            let mut scanning_disks = Vec::new();
+            let mut healing_disks = Vec::new();
+            let mut scanning_infos = Vec::new();
+            let mut healing_infos = Vec::new();
+
+            let mut new_disks = Vec::new();
+            let mut new_infos = Vec::new();
+
+            for (disk, info) in disks.into_iter().zip(infos) {
+                let Some(info) = info else {
+                    continue;
                 };
 
-                Ok((i, info))
-            });
-        }
-
-        // Use optimized batch processor for disk info retrieval
-        let processor = get_global_processors().metadata_processor();
-        let results = processor.execute_batch(futures).await;
-
-        for (submitted_idx, result) in results.into_iter().enumerate() {
-            match result {
-                Ok((disk_idx, info)) => {
-                    infos[disk_idx] = Some(info);
-                }
-                Err(err) => {
-                    infos[submitted_idx] = Some(DiskInfo {
-                        error: err.to_string(),
-                        ..Default::default()
-                    });
-                }
-            }
-        }
-
-        let mut healing: usize = 0;
-
-        let mut scanning_disks = Vec::new();
-        let mut healing_disks = Vec::new();
-        let mut scanning_infos = Vec::new();
-        let mut healing_infos = Vec::new();
-
-        let mut new_disks = Vec::new();
-        let mut new_infos = Vec::new();
-
-        for (disk, info) in disks.into_iter().zip(infos) {
-            let Some(info) = info else {
-                continue;
-            };
-
-            if !info.error.is_empty() || disk.is_none() {
-                continue;
-            }
-
-            if info.healing {
-                healing += 1;
-                if incl_healing {
-                    healing_disks.push(disk.unwrap());
-                    healing_infos.push(info);
+                if !info.error.is_empty() {
+                    continue;
                 }
 
-                continue;
+                if info.healing {
+                    healing += 1;
+                    if incl_healing {
+                        healing_disks.push(disk);
+                        healing_infos.push(info);
+                    }
+
+                    continue;
+                }
+
+                if !info.scanning {
+                    new_disks.push(disk);
+                    new_infos.push(info);
+                } else {
+                    scanning_disks.push(disk);
+                    scanning_infos.push(info);
+                }
             }
 
-            if !info.healing {
-                new_disks.push(disk.unwrap());
-                new_infos.push(info);
-            } else {
-                scanning_disks.push(disk.unwrap());
-                scanning_infos.push(info);
+            new_disks.extend(scanning_disks);
+            new_infos.extend(scanning_infos);
+            new_disks.extend(healing_disks);
+            new_infos.extend(healing_infos);
+
+            if !new_disks.is_empty() || membership_candidates.is_empty() || reprobed {
+                return (new_disks, new_infos, healing);
             }
+
+            reprobed = true;
+            self.reprobe_runtime_candidates_once(&membership_candidates);
+            membership_candidates = self.drive_membership_snapshot().await.scanner_heal_candidates();
         }
-
-        new_disks.extend(scanning_disks);
-        new_infos.extend(scanning_infos);
-        new_disks.extend(healing_disks);
-        new_infos.extend(healing_infos);
-
-        (new_disks, new_infos, healing)
     }
 
     pub(super) async fn _get_local_disks(&self) -> Vec<Option<DiskStore>> {
@@ -527,4 +537,50 @@ mod tests {
 
         drop(temp_dirs);
     }
+
+    #[tokio::test]
+    async fn get_online_disks_with_healing_and_info_reprobes_runtime_candidates_once() {
+        let disk_count = 4;
+        let format = FormatV3::new(1, disk_count);
+
+        let mut temp_dirs = Vec::with_capacity(disk_count);
+        let mut endpoints = Vec::with_capacity(disk_count);
+        let mut disks = Vec::with_capacity(disk_count);
+
+        for disk_idx in 0..disk_count {
+            let (temp_dir, endpoint, disk) = make_formatted_local_disk(disk_idx, &format).await;
+            temp_dirs.push(temp_dir);
+            endpoints.push(endpoint);
+            disks.push(Some(disk));
+        }
+
+        let set_disks = SetDisks::new(
+            "test-owner".to_string(),
+            Arc::new(RwLock::new(disks)),
+            disk_count,
+            disk_count / 2,
+            0,
+            0,
+            endpoints,
+            format,
+            Vec::new(),
+        )
+        .await;
+
+        let all_disks = set_disks.get_disks_internal().await;
+        for disk in all_disks.iter().flatten() {
+            disk.force_runtime_state_for_test(crate::disk::health_state::RuntimeDriveHealthState::Returning);
+        }
+
+        let (online_disks, infos, healing) = set_disks.get_online_disks_with_healing_and_info(false).await;
+        assert_eq!(healing, 0);
+        assert_eq!(online_disks.len(), disk_count);
+        assert_eq!(infos.len(), disk_count);
+        assert!(
+            infos.iter().all(|info| info.error.is_empty()),
+            "runtime reprobe should recover a usable candidate set without probe errors"
+        );
+
+        drop(temp_dirs);
+    }
 }

crates/ecstore/src/store_list_objects.rs

@@ -40,7 +40,7 @@ use std::sync::Arc;
 use tokio::sync::broadcast::{self};
 use tokio::sync::mpsc::{self, Receiver, Sender};
 use tokio_util::sync::CancellationToken;
-use tracing::{error, info};
+use tracing::{error, info, warn};
 use uuid::Uuid;
 
 const MAX_OBJECT_LIST: i32 = 1000;
@@ -50,6 +50,19 @@ const MAX_OBJECT_LIST: i32 = 1000;
 
 const METACACHE_SHARE_PREFIX: bool = false;
 
+fn ensure_non_empty_listing_disks(bucket: &str, path: &str, disks: &[DiskStore]) -> Result<()> {
+    if disks.is_empty() {
+        warn!(
+            bucket = %bucket,
+            path = %path,
+            "listing candidate disks collapsed to empty set"
+        );
+        return Err(StorageError::ErasureReadQuorum);
+    }
+
+    Ok(())
+}
+
 pub fn max_keys_plus_one(max_keys: i32, add_one: bool) -> i32 {
     let mut max_keys = max_keys;
     if !(0..=MAX_OBJECT_LIST).contains(&max_keys) {
@@ -761,6 +774,7 @@ impl ECStore {
                     };
 
                     let path = base_dir_from_prefix(prefix);
+                    ensure_non_empty_listing_disks(bucket, &path, &disks)?;
 
                     let mut filter_prefix = {
                         prefix
@@ -1253,6 +1267,7 @@ impl SetDisks {
         }
 
         let listing_quorum = ((ask_disks + 1) / 2) as usize;
+        ensure_non_empty_listing_disks(&opts.bucket, &opts.base_dir, &disks)?;
 
         let mut fallback_disks = Vec::new();