fix(lifecycle): prevent eager date-expiry deletion on config update (#2708)

2026-08-31 01:09:23 +00:00 · 2026-04-28 18:26:14 +08:00
parent e0b8c4fd42
commit 2953558f41
28 changed files with 3069 additions and 593 deletions
@@ -87,25 +87,70 @@ impl AsyncBatchProcessor {
        T: Send + 'static,
        F: Future<Output = Result<T>> + Send + 'static,
    {
-        let results = self.execute_batch(tasks).await;
-        let mut successes = Vec::new();
+        if required_successes == 0 {
+            return Ok(Vec::new());
+        }

-        for value in results.into_iter().flatten() {
-            successes.push(value);
-            if successes.len() >= required_successes {
-                return Ok(successes);
+        if tasks.is_empty() {
+            return Err(Error::other(format!(
+                "Insufficient successful results: got 0, needed {required_successes}"
+            )));
+        }
+
+        let semaphore = Arc::new(tokio::sync::Semaphore::new(self.max_concurrent));
+        let mut join_set = JoinSet::new();
+        let mut successes = Vec::new();
+        let mut pending_tasks = tasks.len();
+        let mut first_error = None;
+
+        for task in tasks {
+            let sem = semaphore.clone();
+            join_set.spawn(async move {
+                let _permit = sem.acquire().await.map_err(|_| Error::other("Semaphore error"))?;
+                task.await
+            });
+        }
+
+        while let Some(join_result) = join_set.join_next().await {
+            pending_tasks = pending_tasks.saturating_sub(1);
+
+            match join_result {
+                Ok(Ok(value)) => {
+                    successes.push(value);
+                    if successes.len() >= required_successes {
+                        return Ok(successes);
+                    }
+                }
+                Ok(Err(err)) => {
+                    if first_error.is_none() {
+                        first_error = Some(err);
+                    }
+                }
+                Err(join_error) => {
+                    if first_error.is_none() {
+                        first_error = Some(Error::other(format!("Task panicked in quorum batch processor: {join_error}")));
+                    }
+                }
+            }
+
+            if successes.len() + pending_tasks < required_successes {
+                return Err(first_error.unwrap_or_else(|| {
+                    Error::other(format!(
+                        "Insufficient successful results: got {}, needed {}",
+                        successes.len(),
+                        required_successes
+                    ))
+                }));
            }
        }

-        if successes.len() >= required_successes {
-            Ok(successes)
-        } else {
-            Err(Error::other(format!(
+        Err(first_error.unwrap_or_else(|| {
+            Error::other(format!(
                "Insufficient successful results: got {}, needed {}",
                successes.len(),
                required_successes
-            )))
-        }
+            ))
+        }))
    }
 }

@@ -228,4 +273,52 @@ mod tests {
        let successes = results.unwrap();
        assert!(successes.len() >= 2);
    }
+
+    #[tokio::test]
+    async fn test_batch_processor_quorum_returns_before_slow_tail() {
+        let processor = AsyncBatchProcessor::new(4);
+        let started = std::time::Instant::now();
+
+        let tasks: Vec<_> = [(10_u64, Ok(1_i32)), (15, Ok(2)), (250, Ok(3))]
+            .into_iter()
+            .map(|(delay_ms, outcome)| async move {
+                tokio::time::sleep(Duration::from_millis(delay_ms)).await;
+                outcome
+            })
+            .collect();
+
+        let results = processor
+            .execute_batch_with_quorum(tasks, 2)
+            .await
+            .expect("quorum should succeed");
+        assert_eq!(results.len(), 2);
+        assert!(started.elapsed() < Duration::from_millis(100));
+    }
+
+    #[tokio::test]
+    async fn test_batch_processor_quorum_fails_once_quorum_becomes_impossible() {
+        let processor = AsyncBatchProcessor::new(4);
+        let started = std::time::Instant::now();
+
+        let tasks: Vec<_> = vec![
+            (10_u64, Ok(1_i32)),
+            (15, Err(Error::other("first failure"))),
+            (20, Err(Error::other("second failure"))),
+            (250, Ok(4)),
+        ]
+        .into_iter()
+        .map(|(delay_ms, outcome)| async move {
+            tokio::time::sleep(Duration::from_millis(delay_ms)).await;
+            outcome
+        })
+        .collect();
+
+        let err = processor
+            .execute_batch_with_quorum(tasks, 3)
+            .await
+            .expect_err("quorum should fail once it becomes impossible");
+
+        assert!(err.to_string().contains("first failure"));
+        assert!(started.elapsed() < Duration::from_millis(120));
+    }
 }
@@ -54,8 +54,8 @@ use rustfs_filemeta::{
 use rustfs_s3_common::EventName;
 use rustfs_utils::{get_env_i64, get_env_usize, path::encode_dir_object, string::strings_has_prefix_fold};
 use s3s::dto::{
-    BucketLifecycleConfiguration, DefaultRetention, ReplicationConfiguration, RestoreRequest, RestoreRequestType, RestoreStatus,
-    Timestamp,
+    BucketLifecycleConfiguration, DefaultRetention, ExpirationStatus, ReplicationConfiguration, RestoreRequest,
+    RestoreRequestType, RestoreStatus, Timestamp,
 };
 use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION};
 use sha2::{Digest, Sha256};
@@ -92,6 +92,7 @@ const ENV_STALE_UPLOADS_EXPIRY: &str = "RUSTFS_API_STALE_UPLOADS_EXPIRY";
 const ENV_STALE_UPLOADS_CLEANUP_INTERVAL: &str = "RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL";
 const DEFAULT_STALE_UPLOADS_EXPIRY: StdDuration = StdDuration::from_secs(24 * 60 * 60);
 const DEFAULT_STALE_UPLOADS_CLEANUP_INTERVAL: StdDuration = StdDuration::from_secs(6 * 60 * 60);
+const DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS: i64 = 5;

 lazy_static! {
    pub static ref GLOBAL_ExpiryState: Arc<RwLock<ExpiryState>> = ExpiryState::new();
@@ -1241,6 +1242,29 @@ pub async fn enqueue_transition_for_existing_objects(api: Arc<ECStore>, bucket:
    }
 }

+fn lifecycle_rule_has_date_expiration(lc: &BucketLifecycleConfiguration, rule_id: &str) -> bool {
+    lc.rules.iter().any(|rule| {
+        rule.status == ExpirationStatus::from_static(ExpirationStatus::ENABLED)
+            && rule.id.as_deref() == Some(rule_id)
+            && rule.expiration.as_ref().is_some_and(|expiration| expiration.date.is_some())
+    })
+}
+
+fn should_defer_date_expiry_for_recent_config_update(lc: &BucketLifecycleConfiguration, now: OffsetDateTime) -> bool {
+    lc.expiry_updated_at.as_ref().is_some_and(|updated_at| {
+        let updated_at = OffsetDateTime::from(updated_at.clone());
+        now.unix_timestamp().saturating_sub(updated_at.unix_timestamp()) < DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS
+    })
+}
+
+async fn apply_existing_object_expiry(api: Arc<ECStore>, object: &ObjectInfo, event: &lifecycle::Event, src: &LcEventSrc) {
+    if object.is_remote() {
+        apply_expiry_on_transitioned_object(api, object, event, src).await;
+    } else {
+        apply_expiry_on_non_transitioned_objects(api, object, event, src).await;
+    }
+}
+
 pub async fn enqueue_expiry_for_existing_objects(api: Arc<ECStore>, bucket: &str) -> Result<(), Error> {
    let Ok((lc, _)) = metadata_sys::get_lifecycle_config(bucket).await else {
        return Ok(());
@@ -1253,6 +1277,8 @@ pub async fn enqueue_expiry_for_existing_objects(api: Arc<ECStore>, bucket: &str
    let mut marker = None;
    let mut version_marker = None;
    let src = LcEventSrc::Scanner;
+    let defer_date_expiry_once = should_defer_date_expiry_for_recent_config_update(&lc, OffsetDateTime::now_utc());
+    let mut date_expiry_deferred_once = false;

    loop {
        let page = api
@@ -1269,15 +1295,16 @@ pub async fn enqueue_expiry_for_existing_objects(api: Arc<ECStore>, bucket: &str
                | IlmAction::DeleteRestoredVersionAction
                | IlmAction::DeleteAllVersionsAction
                | IlmAction::DelMarkerDeleteAllVersionsAction => {
-                    if event
-                        .due
-                        .is_some_and(|due| due.unix_timestamp() <= OffsetDateTime::now_utc().unix_timestamp())
-                    {
-                        if object.is_remote() {
-                            apply_expiry_on_transitioned_object(api.clone(), object, &event, &src).await;
-                        } else {
-                            apply_expiry_on_non_transitioned_objects(api.clone(), object, &event, &src).await;
+                    let now = OffsetDateTime::now_utc();
+                    if event.due.is_some_and(|due| due.unix_timestamp() <= now.unix_timestamp()) {
+                        if defer_date_expiry_once
+                            && !date_expiry_deferred_once
+                            && lifecycle_rule_has_date_expiration(&lc, &event.rule_id)
+                        {
+                            tokio::time::sleep(StdDuration::from_secs(DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS as u64)).await;
+                            date_expiry_deferred_once = true;
                        }
+                        apply_existing_object_expiry(api.clone(), object, &event, &src).await;
                    } else {
                        apply_expiry_rule(&event, &src, object).await;
                    }
@@ -1977,9 +2004,10 @@ pub async fn apply_lifecycle_action(event: &lifecycle::Event, src: &LcEventSrc,
 #[cfg(test)]
 mod tests {
    use super::{
-        StaleMultipartUploadCandidate, cleanup_empty_multipart_sha_dirs_on_local_disks, cleanup_stale_multipart_uploads_once_at,
-        lifecycle_deleted_object, lifecycle_version_purge_state_from_completed_targets,
-        mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate, replication_state_for_delete,
+        DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS, StaleMultipartUploadCandidate, cleanup_empty_multipart_sha_dirs_on_local_disks,
+        cleanup_stale_multipart_uploads_once_at, lifecycle_deleted_object, lifecycle_rule_has_date_expiration,
+        lifecycle_version_purge_state_from_completed_targets, mark_delete_opts_skip_decommissioned_on_remote_success,
+        merge_stale_multipart_candidate, replication_state_for_delete, should_defer_date_expiry_for_recent_config_update,
        should_reuse_lifecycle_delete_replication_state,
    };
    use crate::bucket::metadata::BUCKET_LIFECYCLE_CONFIG;
@@ -1994,6 +2022,7 @@ mod tests {
        BucketOperations, BucketOptions, MakeBucketOptions, MultipartOperations, ObjectInfo, ObjectOptions, PutObjReader,
    };
    use rustfs_filemeta::{ReplicateDecision, VersionPurgeStatusType};
+    use s3s::dto::{BucketLifecycleConfiguration, ExpirationStatus, LifecycleExpiration, LifecycleRule, Timestamp};
    use serial_test::serial;
    use sha2::{Digest, Sha256};
    use std::collections::HashMap;
@@ -2014,6 +2043,49 @@ mod tests {
        assert!(opts.skip_decommissioned);
    }

+    #[test]
+    fn lifecycle_rule_has_date_expiration_detects_enabled_date_rule() {
+        let lc = BucketLifecycleConfiguration {
+            expiry_updated_at: None,
+            rules: vec![LifecycleRule {
+                status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
+                expiration: Some(LifecycleExpiration {
+                    date: Some(Timestamp::from(OffsetDateTime::now_utc())),
+                    ..Default::default()
+                }),
+                id: Some("rule-date".to_string()),
+                abort_incomplete_multipart_upload: None,
+                del_marker_expiration: None,
+                filter: None,
+                noncurrent_version_expiration: None,
+                noncurrent_version_transitions: None,
+                prefix: None,
+                transitions: None,
+            }],
+        };
+
+        assert!(lifecycle_rule_has_date_expiration(&lc, "rule-date"));
+        assert!(!lifecycle_rule_has_date_expiration(&lc, "missing-rule"));
+    }
+
+    #[test]
+    fn should_defer_date_expiry_for_recent_config_update_respects_grace_window() {
+        let now = OffsetDateTime::now_utc();
+        let recent = BucketLifecycleConfiguration {
+            expiry_updated_at: Some(Timestamp::from(now - time::Duration::seconds(1))),
+            rules: Vec::new(),
+        };
+        let stale = BucketLifecycleConfiguration {
+            expiry_updated_at: Some(Timestamp::from(
+                now - time::Duration::seconds(DATE_EXPIRY_EXISTING_OBJECTS_GRACE_SECS + 1),
+            )),
+            rules: Vec::new(),
+        };
+
+        assert!(should_defer_date_expiry_for_recent_config_update(&recent, now));
+        assert!(!should_defer_date_expiry_for_recent_config_update(&stale, now));
+    }
+
    #[test]
    fn mark_delete_opts_skip_decommissioned_on_remote_success_preserves_false_on_failure() {
        let mut opts = ObjectOptions::default();
@@ -46,9 +46,7 @@ const DISK_HEALTH_FAULTY: u32 = 1;

 pub const ENV_RUSTFS_DRIVE_ACTIVE_MONITORING: &str = "RUSTFS_DRIVE_ACTIVE_MONITORING";
 pub const DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING: bool = true;
-pub const CHECK_EVERY: Duration = Duration::from_secs(15);
 pub const SKIP_IF_SUCCESS_BEFORE: Duration = Duration::from_secs(5);
-pub const CHECK_TIMEOUT_DURATION: Duration = Duration::from_secs(5);

 lazy_static::lazy_static! {
    static ref TEST_DATA: Bytes = Bytes::from(vec![42u8; 2048]);
@@ -104,6 +102,20 @@ pub fn get_drive_walkdir_stall_timeout() -> Duration {
    )
 }

+pub fn get_drive_active_check_interval() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS,
+        rustfs_config::DEFAULT_DRIVE_ACTIVE_CHECK_INTERVAL_SECS,
+    ))
+}
+
+pub fn get_drive_active_check_timeout() -> Duration {
+    Duration::from_secs(rustfs_utils::get_env_u64(
+        rustfs_config::ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS,
+        rustfs_config::DEFAULT_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS,
+    ))
+}
+
 /// DiskHealthTracker tracks the health status of a disk.
 /// Similar to Go's diskHealthTracker.
 #[derive(Debug)]
@@ -474,7 +486,7 @@ impl LocalDiskWrapper {
    async fn monitor_disk_writable(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
        // TODO: config interval

-        let mut interval = time::interval(CHECK_EVERY);
+        let mut interval = time::interval(get_drive_active_check_interval());

        loop {
            tokio::select! {
@@ -507,7 +519,16 @@ impl LocalDiskWrapper {


                    let test_obj = format!("health-check-{}", Uuid::new_v4());
-                    if Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, true, CHECK_TIMEOUT_DURATION).await.is_err()
+                    if Self::perform_health_check(
+                        disk.clone(),
+                        &TEST_BUCKET,
+                        &test_obj,
+                        &TEST_DATA,
+                        true,
+                        get_drive_active_check_timeout(),
+                    )
+                    .await
+                    .is_err()
                        && health.mark_failure(&disk.endpoint(), "active_health_check_failed")
                    {
                        // Health check failed, disk is considered faulty
@@ -613,7 +634,16 @@ impl LocalDiskWrapper {
                    }

                    let test_obj = format!("health-check-{}", Uuid::new_v4());
-                    match Self::perform_health_check(disk.clone(), &TEST_BUCKET, &test_obj, &TEST_DATA, false, CHECK_TIMEOUT_DURATION).await {
+                    match Self::perform_health_check(
+                        disk.clone(),
+                        &TEST_BUCKET,
+                        &test_obj,
+                        &TEST_DATA,
+                        false,
+                        get_drive_active_check_timeout(),
+                    )
+                    .await
+                    {
                        Ok(_) => {
                            let state_before = health.runtime_state();
                            let is_online = health.mark_recovery_success(&disk.endpoint(), "recovery_probe_success");
@@ -1137,6 +1167,40 @@ mod tests {
        });
    }

+    #[test]
+    fn drive_active_check_interval_uses_default_when_unset() {
+        temp_env::with_var_unset(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, || {
+            assert_eq!(
+                get_drive_active_check_interval(),
+                Duration::from_secs(rustfs_config::DEFAULT_DRIVE_ACTIVE_CHECK_INTERVAL_SECS)
+            );
+        });
+    }
+
+    #[test]
+    fn drive_active_check_interval_reads_env_override() {
+        temp_env::with_var(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, Some("3"), || {
+            assert_eq!(get_drive_active_check_interval(), Duration::from_secs(3));
+        });
+    }
+
+    #[test]
+    fn drive_active_check_timeout_uses_default_when_unset() {
+        temp_env::with_var_unset(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS, || {
+            assert_eq!(
+                get_drive_active_check_timeout(),
+                Duration::from_secs(rustfs_config::DEFAULT_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS)
+            );
+        });
+    }
+
+    #[test]
+    fn drive_active_check_timeout_reads_env_override() {
+        temp_env::with_var(rustfs_config::ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS, Some("1"), || {
+            assert_eq!(get_drive_active_check_timeout(), Duration::from_secs(1));
+        });
+    }
+
    #[test]
    fn runtime_state_transitions_from_online_to_suspect_then_offline() {
        temp_env::with_var(rustfs_config::ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD, Some("2"), || {
@@ -12,11 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

+use crate::disk::error::{DiskError, Error as DiskErrorType};
 use crate::rpc::{TONIC_RPC_PREFIX, gen_signature_headers};
 use http::Method;
 use rustfs_common::GLOBAL_CONN_MAP;
 use rustfs_protos::{create_new_channel, proto_gen::node_service::node_service_client::NodeServiceClient};
-use std::error::Error;
+use std::{error::Error, io::ErrorKind};
 use tonic::{service::interceptor::InterceptedService, transport::Channel};
 use tracing::debug;

@@ -51,6 +52,46 @@ pub async fn node_service_time_out_client_no_auth(
    node_service_time_out_client(addr, TonicInterceptor::NoOp(NoOpInterceptor)).await
 }

+pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool {
+    match err {
+        DiskError::Timeout => true,
+        DiskError::Io(io_err) => {
+            if matches!(
+                io_err.kind(),
+                ErrorKind::TimedOut
+                    | ErrorKind::ConnectionRefused
+                    | ErrorKind::ConnectionReset
+                    | ErrorKind::BrokenPipe
+                    | ErrorKind::NotConnected
+                    | ErrorKind::ConnectionAborted
+                    | ErrorKind::UnexpectedEof
+            ) {
+                return true;
+            }
+
+            let message = io_err.to_string().to_ascii_lowercase();
+            [
+                "transport error",
+                "unavailable",
+                "error trying to connect",
+                "connection refused",
+                "connection reset",
+                "broken pipe",
+                "not connected",
+                "unexpected eof",
+                "timed out",
+                "deadline has elapsed",
+                "connection closed",
+                "connection aborted",
+                "tcp connect error",
+            ]
+            .iter()
+            .any(|needle| message.contains(needle))
+        }
+        _ => false,
+    }
+}
+
 pub struct TonicSignatureInterceptor;

 impl tonic::service::Interceptor for TonicSignatureInterceptor {
@@ -18,13 +18,15 @@ use crate::disk::error::{Error, Result};
 use crate::disk::error_reduce::{BUCKET_OP_IGNORED_ERRS, is_all_buckets_not_found, reduce_write_quorum_errs};
 use crate::disk::{DiskAPI, DiskStore, disk_store::get_max_timeout_duration};
 use crate::global::GLOBAL_LOCAL_DISK_MAP;
-use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
+use crate::rpc::client::{
+    TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
+};
 use crate::store::all_local_disk;
 use crate::store_utils::is_reserved_or_invalid_bucket;
 use crate::{
    disk::{
        self, VolumeInfo,
-        disk_store::{CHECK_EVERY, CHECK_TIMEOUT_DURATION, DiskHealthTracker},
+        disk_store::{DiskHealthTracker, get_drive_active_check_interval, get_drive_active_check_timeout},
    },
    endpoints::{EndpointServerPools, Node},
    store_api::{BucketInfo, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
@@ -613,7 +615,7 @@ impl RemotePeerS3Client {

    /// Monitor remote peer health periodically
    async fn monitor_remote_peer_health(addr: String, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
-        let mut interval = time::interval(CHECK_EVERY);
+        let mut interval = time::interval(get_drive_active_check_interval());

        loop {
            tokio::select! {
@@ -682,7 +684,7 @@ impl RemotePeerS3Client {
        let port = url.port_or_known_default().unwrap_or(80);

        // Try to establish TCP connection
-        match timeout(CHECK_TIMEOUT_DURATION, TcpStream::connect((host, port))).await {
+        match timeout(get_drive_active_check_timeout(), TcpStream::connect((host, port))).await {
            Ok(Ok(_)) => Ok(()),
            _ => Err(Error::other(format!("Cannot connect to {host}:{port}"))),
        }
@@ -717,16 +719,39 @@ impl RemotePeerS3Client {
                    self.health.log_success();
                }
                self.health.decrement_waiting();
+                if let Err(err) = &operation_result
+                    && is_network_like_disk_error(err)
+                {
+                    self.mark_faulty_and_start_recovery("operation_network_error").await;
+                }
                operation_result
            }
            Err(_) => {
                // Timeout occurred, mark peer as potentially faulty
                self.health.decrement_waiting();
+                self.mark_faulty_and_start_recovery("operation_timeout").await;
                warn!("Remote peer operation timeout after {:?}", timeout_duration);
                Err(Error::other(format!("Remote peer operation timeout after {timeout_duration:?}")))
            }
        }
    }
+
+    async fn mark_faulty_and_start_recovery(&self, reason: &'static str) {
+        if self.health.swap_ok_to_faulty() {
+            warn!(
+                addr = %self.addr,
+                reason,
+                "Remote peer marked faulty after network failure"
+            );
+
+            let health = Arc::clone(&self.health);
+            let cancel_token = self.cancel_token.clone();
+            let addr = self.addr.clone();
+            tokio::spawn(async move {
+                Self::monitor_remote_peer_recovery(addr, health, cancel_token).await;
+            });
+        }
+    }
 }

 #[async_trait]
@@ -1001,3 +1026,69 @@ pub async fn heal_bucket_local(bucket: &str, opts: &HealOpts) -> Result<HealResu
 async fn clone_drives() -> Vec<Option<DiskStore>> {
    GLOBAL_LOCAL_DISK_MAP.read().await.values().cloned().collect::<Vec<_>>()
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn test_remote_peer(addr: &str) -> RemotePeerS3Client {
+        let node = Node {
+            url: url::Url::parse(addr).expect("test peer URL should parse"),
+            pools: vec![0],
+            is_local: false,
+            grid_host: addr.to_string(),
+        };
+
+        RemotePeerS3Client {
+            node: Some(node),
+            pools: Some(vec![0]),
+            addr: addr.to_string(),
+            health: Arc::new(DiskHealthTracker::new()),
+            cancel_token: CancellationToken::new(),
+        }
+    }
+
+    #[tokio::test]
+    async fn test_execute_with_timeout_marks_remote_peer_faulty_on_network_like_error() {
+        let client = test_remote_peer("http://peer-network-error:9000");
+
+        let err = client
+            .execute_with_timeout(
+                || async {
+                    Err::<(), Error>(DiskError::Io(std::io::Error::new(
+                        std::io::ErrorKind::ConnectionRefused,
+                        "connection refused",
+                    )))
+                },
+                Duration::from_secs(1),
+            )
+            .await
+            .expect_err("network-like error should fail");
+
+        assert_eq!(
+            match &err {
+                DiskError::Io(io_err) => io_err.kind(),
+                other => panic!("expected io network error, got {other:?}"),
+            },
+            std::io::ErrorKind::ConnectionRefused
+        );
+        assert!(client.health.is_faulty(), "network-like errors should mark remote peer faulty");
+
+        client.cancel_token.cancel();
+    }
+
+    #[tokio::test]
+    async fn test_execute_with_timeout_keeps_remote_peer_online_for_business_error() {
+        let client = test_remote_peer("http://peer-business-error:9000");
+
+        let err = client
+            .execute_with_timeout(|| async { Err::<(), Error>(DiskError::FileNotFound) }, Duration::from_secs(1))
+            .await
+            .expect_err("business error should fail");
+
+        assert_eq!(err, DiskError::FileNotFound);
+        assert!(!client.health.is_faulty(), "business errors should not mark remote peer faulty");
+
+        client.cancel_token.cancel();
+    }
+}
@@ -16,15 +16,17 @@ use crate::disk::{
    CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, FileInfoVersions, FileReader,
    FileWriter, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
    disk_store::{
-        CHECK_EVERY, CHECK_TIMEOUT_DURATION, DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING,
-        SKIP_IF_SUCCESS_BEFORE, get_drive_disk_info_timeout, get_drive_list_dir_timeout, get_drive_metadata_timeout,
-        get_drive_walkdir_stall_timeout, get_drive_walkdir_timeout, get_max_timeout_duration,
+        DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
+        get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
+        get_drive_metadata_timeout, get_drive_walkdir_stall_timeout, get_drive_walkdir_timeout, get_max_timeout_duration,
    },
    endpoint::Endpoint,
    health_state::{RuntimeDriveHealthState, get_drive_returning_probe_interval, record_drive_runtime_state},
 };
 use crate::disk::{disk_store::DiskHealthTracker, error::DiskError, local::ScanGuard};
-use crate::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
+use crate::rpc::client::{
+    TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
+};
 use crate::set_disk::DEFAULT_READ_BUFFER_SIZE;
 use crate::{
    disk::error::{Error, Result},
@@ -47,7 +49,7 @@ use rustfs_protos::proto_gen::node_service::{
 use rustfs_rio::{HttpReader, HttpWriter};
 use serde::{Serialize, de::DeserializeOwned};
 use std::{
-    io::{Cursor, ErrorKind},
+    io::Cursor,
    path::PathBuf,
    sync::{
        Arc,
@@ -176,7 +178,7 @@ impl RemoteDisk {
        health: Arc<DiskHealthTracker>,
        cancel_token: CancellationToken,
    ) {
-        let mut interval = time::interval(CHECK_EVERY);
+        let mut interval = time::interval(get_drive_active_check_interval());

        // Perform basic connectivity check
        if Self::perform_connectivity_check(&addr).await.is_err() && health.mark_offline(&endpoint, "connectivity_probe_failed") {
@@ -281,7 +283,7 @@ impl RemoteDisk {
        let port = url.port_or_known_default().unwrap_or(80);

        // Try to establish TCP connection
-        match timeout(CHECK_TIMEOUT_DURATION, TcpStream::connect((host, port))).await {
+        match timeout(get_drive_active_check_timeout(), TcpStream::connect((host, port))).await {
            Ok(Ok(stream)) => {
                drop(stream);
                Ok(())
@@ -334,10 +336,10 @@ impl RemoteDisk {
                }
                self.health.decrement_waiting();
                if let Err(err) = &operation_result
-                    && Self::is_timeout_like_error(err)
+                    && is_network_like_disk_error(err)
                {
                    counter!(
-                        "rustfs_drive_op_timeout_total",
+                        "rustfs_drive_op_network_error_total",
                        "endpoint" => self.endpoint.to_string(),
                        "op" => op.to_string()
                    )
@@ -347,9 +349,9 @@ impl RemoteDisk {
                        addr = %self.addr,
                        op,
                        timeout_ms = timeout_duration.as_millis(),
-                        "Remote disk operation returned a timeout-like error"
+                        "Remote disk operation returned a network-like error"
                    );
-                    self.mark_faulty_and_evict("operation_timeout_error").await;
+                    self.mark_faulty_and_evict("operation_network_error").await;
                }
                operation_result
            }
@@ -375,10 +377,6 @@ impl RemoteDisk {
        }
    }

-    fn is_timeout_like_error(err: &Error) -> bool {
-        matches!(err, DiskError::Timeout) || matches!(err, DiskError::Io(io_err) if io_err.kind() == ErrorKind::TimedOut)
-    }
-
    async fn mark_faulty_and_evict(&self, reason: &'static str) {
        if self.health.mark_offline(&self.endpoint, reason) {
            self.spawn_recovery_monitor_if_needed();
@@ -2073,6 +2071,96 @@ mod tests {
        );
    }

+    #[tokio::test]
+    async fn test_execute_with_timeout_marks_faulty_on_network_like_error() {
+        let addr = "http://127.0.0.1:59993".to_string();
+        let url = url::Url::parse(&format!("{addr}/data")).unwrap();
+        let endpoint = Endpoint {
+            url,
+            is_local: false,
+            pool_idx: 0,
+            set_idx: 0,
+            disk_idx: 0,
+        };
+
+        let remote_disk = RemoteDisk::new(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .unwrap();
+
+        let channel = TonicEndpoint::from_shared(addr.clone()).unwrap().connect_lazy();
+        GLOBAL_CONN_MAP.write().await.insert(addr.clone(), channel);
+
+        let err = remote_disk
+            .execute_with_timeout(
+                || async {
+                    Err::<(), Error>(DiskError::Io(std::io::Error::new(
+                        std::io::ErrorKind::ConnectionRefused,
+                        "connection refused",
+                    )))
+                },
+                Duration::from_secs(1),
+            )
+            .await
+            .expect_err("network-like operation error should fail");
+
+        assert_eq!(
+            match &err {
+                DiskError::Io(io_err) => io_err.kind(),
+                other => panic!("expected io network error, got {other:?}"),
+            },
+            std::io::ErrorKind::ConnectionRefused
+        );
+        assert!(!remote_disk.is_online().await, "network-like errors should mark remote disk faulty");
+        assert!(
+            !GLOBAL_CONN_MAP.read().await.contains_key(&addr),
+            "network-like errors should evict cached connection"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_execute_with_timeout_keeps_remote_disk_online_for_business_error() {
+        let addr = "http://127.0.0.1:59994".to_string();
+        let url = url::Url::parse(&format!("{addr}/data")).unwrap();
+        let endpoint = Endpoint {
+            url,
+            is_local: false,
+            pool_idx: 0,
+            set_idx: 0,
+            disk_idx: 0,
+        };
+
+        let remote_disk = RemoteDisk::new(
+            &endpoint,
+            &DiskOption {
+                cleanup: false,
+                health_check: false,
+            },
+        )
+        .await
+        .unwrap();
+
+        let channel = TonicEndpoint::from_shared(addr.clone()).unwrap().connect_lazy();
+        GLOBAL_CONN_MAP.write().await.insert(addr.clone(), channel);
+
+        let err = remote_disk
+            .execute_with_timeout(|| async { Err::<(), Error>(DiskError::FileNotFound) }, Duration::from_secs(1))
+            .await
+            .expect_err("business error should still fail the operation");
+
+        assert_eq!(err, DiskError::FileNotFound);
+        assert!(remote_disk.is_online().await, "business errors should not mark remote disk faulty");
+        assert!(
+            GLOBAL_CONN_MAP.read().await.contains_key(&addr),
+            "business errors should not evict cached connection"
+        );
+    }
+
    #[test]
    fn test_remote_disk_sync_properties() {
        let url = url::Url::parse("https://secure-remote:9000/data").unwrap();
@@ -13,6 +13,87 @@
 // limitations under the License.

 use super::*;
+use std::future::Future;
+use std::time::Duration;
+use tokio::task::JoinSet;
+
+async fn collect_list_parts_results<F>(
+    tasks: Vec<F>,
+    read_quorum: usize,
+) -> disk::error::Result<(Vec<Option<DiskError>>, Vec<Vec<String>>)>
+where
+    F: Future<Output = disk::error::Result<Vec<String>>> + Send + 'static,
+{
+    let mut errs = vec![Some(DiskError::DiskNotFound); tasks.len()];
+    let mut object_parts = vec![Vec::new(); tasks.len()];
+    let mut successful_responses = 0usize;
+    let mut pending = tasks.len();
+    let mut join_set = JoinSet::new();
+
+    for (index, task) in tasks.into_iter().enumerate() {
+        join_set.spawn(async move { (index, task.await) });
+    }
+
+    while let Some(join_result) = join_set.join_next().await {
+        pending = pending.saturating_sub(1);
+
+        match join_result {
+            Ok((index, Ok(parts))) => {
+                errs[index] = None;
+                object_parts[index] = parts;
+                successful_responses += 1;
+            }
+            Ok((index, Err(err))) => {
+                errs[index] = Some(err);
+            }
+            Err(_) => {}
+        }
+
+        if successful_responses + pending < read_quorum {
+            return Err(DiskError::ErasureReadQuorum);
+        }
+    }
+
+    Ok((errs, object_parts))
+}
+
+fn reduce_quorum_part_numbers(object_parts: Vec<Vec<String>>, read_quorum: usize) -> Vec<usize> {
+    let mut part_quorum_map: HashMap<usize, usize> = HashMap::new();
+
+    for drive_parts in object_parts {
+        let mut parts_with_meta_count: HashMap<usize, usize> = HashMap::new();
+
+        // part files can be either part.N or part.N.meta
+        for part_path in drive_parts {
+            if let Some(num_str) = part_path.strip_prefix("part.") {
+                if let Some(meta_idx) = num_str.find(".meta") {
+                    if let Ok(part_num) = num_str[..meta_idx].parse::<usize>() {
+                        *parts_with_meta_count.entry(part_num).or_insert(0) += 1;
+                    }
+                } else if let Ok(part_num) = num_str.parse::<usize>() {
+                    *parts_with_meta_count.entry(part_num).or_insert(0) += 1;
+                }
+            }
+        }
+
+        // Include only part.N.meta files with corresponding part.N
+        for (&part_num, &cnt) in &parts_with_meta_count {
+            if cnt >= 2 {
+                *part_quorum_map.entry(part_num).or_insert(0) += 1;
+            }
+        }
+    }
+
+    let mut part_numbers = Vec::with_capacity(part_quorum_map.len());
+    for (part_num, count) in part_quorum_map {
+        if count >= read_quorum {
+            part_numbers.push(part_num);
+        }
+    }
+
+    part_numbers.sort();
+    part_numbers
+}

 impl SetDisks {
    pub(super) async fn list_parts(
@@ -21,10 +102,13 @@ impl SetDisks {
        read_quorum: usize,
    ) -> disk::error::Result<Vec<usize>> {
        let mut futures = Vec::with_capacity(disks.len());
-        for (i, disk) in disks.iter().enumerate() {
+        let part_path = part_path.to_string();
+        for disk in disks.iter() {
+            let disk = disk.clone();
+            let part_path = part_path.clone();
            futures.push(async move {
                if let Some(disk) = disk {
-                    disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path, -1)
+                    disk.list_dir(RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_MULTIPART_BUCKET, part_path.as_str(), -1)
                        .await
                } else {
                    Err(DiskError::DiskNotFound)
@@ -35,60 +119,15 @@ impl SetDisks {
        let mut errs = Vec::with_capacity(disks.len());
        let mut object_parts = Vec::with_capacity(disks.len());

-        let results = join_all(futures).await;
-        for result in results {
-            match result {
-                Ok(res) => {
-                    errs.push(None);
-                    object_parts.push(res);
-                }
-                Err(e) => {
-                    errs.push(Some(e));
-                    object_parts.push(vec![]);
-                }
-            }
-        }
+        let (collected_errs, collected_parts) = collect_list_parts_results(futures, read_quorum).await?;
+        errs.extend(collected_errs);
+        object_parts.extend(collected_parts);

        if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
            return Err(err);
        }

-        let mut part_quorum_map: HashMap<usize, usize> = HashMap::new();
-
-        for drive_parts in object_parts {
-            let mut parts_with_meta_count: HashMap<usize, usize> = HashMap::new();
-
-            // part files can be either part.N or part.N.meta
-            for part_path in drive_parts {
-                if let Some(num_str) = part_path.strip_prefix("part.") {
-                    if let Some(meta_idx) = num_str.find(".meta") {
-                        if let Ok(part_num) = num_str[..meta_idx].parse::<usize>() {
-                            *parts_with_meta_count.entry(part_num).or_insert(0) += 1;
-                        }
-                    } else if let Ok(part_num) = num_str.parse::<usize>() {
-                        *parts_with_meta_count.entry(part_num).or_insert(0) += 1;
-                    }
-                }
-            }
-
-            // Include only part.N.meta files with corresponding part.N
-            for (&part_num, &cnt) in &parts_with_meta_count {
-                if cnt >= 2 {
-                    *part_quorum_map.entry(part_num).or_insert(0) += 1;
-                }
-            }
-        }
-
-        let mut part_numbers = Vec::with_capacity(part_quorum_map.len());
-        for (part_num, count) in part_quorum_map {
-            if count >= read_quorum {
-                part_numbers.push(part_num);
-            }
-        }
-
-        part_numbers.sort();
-
-        Ok(part_numbers)
+        Ok(reduce_quorum_part_numbers(object_parts, read_quorum))
    }

    #[tracing::instrument(level = "debug", skip(self))]
@@ -145,3 +184,78 @@ impl SetDisks {
        Ok((fi, parts_metadata))
    }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn collect_list_parts_results_fails_early_when_quorum_is_impossible() {
+        let started = std::time::Instant::now();
+        let tasks: Vec<_> = vec![
+            (10_u64, Err(DiskError::DiskNotFound)),
+            (15, Err(DiskError::DiskNotFound)),
+            (250, Ok::<Vec<String>, DiskError>(vec!["part.1".to_string(), "part.1.meta".to_string()])),
+        ]
+        .into_iter()
+        .map(|(delay_ms, outcome)| async move {
+            tokio::time::sleep(Duration::from_millis(delay_ms)).await;
+            outcome
+        })
+        .collect();
+
+        let err = collect_list_parts_results(tasks, 2)
+            .await
+            .expect_err("quorum should become impossible before slow tail completes");
+
+        assert_eq!(err, DiskError::ErasureReadQuorum);
+        assert!(started.elapsed() < Duration::from_millis(120));
+    }
+
+    #[tokio::test]
+    async fn collect_list_parts_results_tolerates_single_panicked_task_when_quorum_is_met() {
+        let tasks: Vec<_> = vec![(5_u64, true), (10, false), (12, false)]
+            .into_iter()
+            .map(|(delay_ms, should_panic)| async move {
+                tokio::time::sleep(Duration::from_millis(delay_ms)).await;
+                if should_panic {
+                    panic!("simulated task panic");
+                }
+                Ok::<Vec<String>, DiskError>(vec!["part.1".to_string(), "part.1.meta".to_string()])
+            })
+            .collect();
+
+        let (errs, object_parts) = collect_list_parts_results(tasks, 2)
+            .await
+            .expect("quorum should still succeed");
+        assert_eq!(errs.iter().filter(|err| err.is_none()).count(), 2);
+        assert_eq!(object_parts.iter().filter(|parts| !parts.is_empty()).count(), 2);
+    }
+
+    #[test]
+    fn reduce_quorum_part_numbers_only_keeps_parts_present_on_quorum_of_drives() {
+        let object_parts = vec![
+            vec![
+                "part.1".to_string(),
+                "part.1.meta".to_string(),
+                "part.2".to_string(),
+                "part.2.meta".to_string(),
+            ],
+            vec![
+                "part.1".to_string(),
+                "part.1.meta".to_string(),
+                "part.3".to_string(),
+                "part.3.meta".to_string(),
+            ],
+            vec![
+                "part.1".to_string(),
+                "part.1.meta".to_string(),
+                "part.2".to_string(),
+                "part.2.meta".to_string(),
+            ],
+        ];
+
+        let parts = reduce_quorum_part_numbers(object_parts, 2);
+        assert_eq!(parts, vec![1, 2]);
+    }
+}
@@ -14,6 +14,88 @@

 use super::*;
 use rustfs_config::{DEFAULT_OBJECT_ZERO_COPY_ENABLE, ENV_OBJECT_ZERO_COPY_ENABLE};
+use std::future::Future;
+use tokio::task::JoinSet;
+
+async fn collect_read_multiple_results<F>(
+    tasks: Vec<F>,
+    read_quorum: usize,
+) -> std::result::Result<(Vec<Option<Vec<ReadMultipleResp>>>, Vec<Option<DiskError>>), ()>
+where
+    F: Future<Output = disk::error::Result<Vec<ReadMultipleResp>>> + Send + 'static,
+{
+    let mut responses = vec![None; tasks.len()];
+    let mut errors = vec![Some(DiskError::DiskNotFound); tasks.len()];
+    let mut successful_responses = 0usize;
+    let mut pending = tasks.len();
+    let mut join_set = JoinSet::new();
+
+    for (index, task) in tasks.into_iter().enumerate() {
+        join_set.spawn(async move { (index, task.await) });
+    }
+
+    while let Some(join_result) = join_set.join_next().await {
+        pending = pending.saturating_sub(1);
+
+        match join_result {
+            Ok((index, Ok(resp))) => {
+                responses[index] = Some(resp);
+                errors[index] = None;
+                successful_responses += 1;
+            }
+            Ok((index, Err(err))) => {
+                errors[index] = Some(err);
+            }
+            Err(_) => {}
+        }
+
+        if successful_responses + pending < read_quorum {
+            return Err(());
+        }
+    }
+
+    Ok((responses, errors))
+}
+
+async fn collect_read_parts_results<F>(
+    tasks: Vec<F>,
+    read_quorum: usize,
+) -> std::result::Result<(Vec<Option<Vec<ObjectPartInfo>>>, Vec<Option<DiskError>>), ()>
+where
+    F: Future<Output = disk::error::Result<Vec<ObjectPartInfo>>> + Send + 'static,
+{
+    let mut responses = vec![None; tasks.len()];
+    let mut errors = vec![Some(DiskError::DiskNotFound); tasks.len()];
+    let mut successful_responses = 0usize;
+    let mut pending = tasks.len();
+    let mut join_set = JoinSet::new();
+
+    for (index, task) in tasks.into_iter().enumerate() {
+        join_set.spawn(async move { (index, task.await) });
+    }
+
+    while let Some(join_result) = join_set.join_next().await {
+        pending = pending.saturating_sub(1);
+
+        match join_result {
+            Ok((index, Ok(resp))) => {
+                responses[index] = Some(resp);
+                errors[index] = None;
+                successful_responses += 1;
+            }
+            Ok((index, Err(err))) => {
+                errors[index] = Some(err);
+            }
+            Err(_) => {}
+        }
+
+        if successful_responses + pending < read_quorum {
+            return Err(());
+        }
+    }
+
+    Ok((responses, errors))
+}

 impl SetDisks {
    pub(super) async fn read_parts(
@@ -25,9 +107,6 @@ impl SetDisks {
    ) -> disk::error::Result<Vec<ObjectPartInfo>> {
        let mut errs = Vec::with_capacity(disks.len());
        let mut object_parts = Vec::with_capacity(disks.len());
-
-        // Use batch processor for better performance
-        let processor = get_global_processors().read_processor();
        let bucket = bucket.to_string();
        let part_meta_paths = part_meta_paths.to_vec();

@@ -48,19 +127,13 @@ impl SetDisks {
            })
            .collect();

-        let results = processor.execute_batch(tasks).await;
-        for result in results {
-            match result {
-                Ok(res) => {
-                    errs.push(None);
-                    object_parts.push(res);
-                }
-                Err(e) => {
-                    errs.push(Some(e));
-                    object_parts.push(vec![]);
-                }
-            }
-        }
+        let (responses, collected_errors) = match collect_read_parts_results(tasks, read_quorum).await {
+            Ok(collected) => collected,
+            Err(()) => return Err(DiskError::ErasureReadQuorum),
+        };
+
+        errs.extend(collected_errors);
+        object_parts.extend(responses.into_iter().map(|resp| resp.unwrap_or_default()));

        if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
            return Err(err);
@@ -384,10 +457,23 @@ impl SetDisks {
        read_quorum: usize,
    ) -> Vec<ReadMultipleResp> {
        let mut futures = Vec::with_capacity(disks.len());
-        let mut ress = Vec::with_capacity(disks.len());
-        let mut errors = Vec::with_capacity(disks.len());
+        let empty_quorum_result = || {
+            req.files
+                .iter()
+                .map(|want| ReadMultipleResp {
+                    bucket: req.bucket.clone(),
+                    prefix: req.prefix.clone(),
+                    file: want.clone(),
+                    exists: false,
+                    error: Error::ErasureReadQuorum.to_string(),
+                    data: Vec::new(),
+                    mod_time: None,
+                })
+                .collect::<Vec<_>>()
+        };

        for disk in disks.iter() {
+            let disk = disk.clone();
            let req = req.clone();
            futures.push(async move {
                if let Some(disk) = disk {
@@ -398,19 +484,10 @@ impl SetDisks {
            });
        }

-        let results = join_all(futures).await;
-        for result in results {
-            match result {
-                Ok(res) => {
-                    ress.push(Some(res));
-                    errors.push(None);
-                }
-                Err(e) => {
-                    ress.push(None);
-                    errors.push(Some(e));
-                }
-            }
-        }
+        let (ress, errors) = match collect_read_multiple_results(futures, read_quorum).await {
+            Ok(collected) => collected,
+            Err(()) => return empty_quorum_result(),
+        };

        // debug!("ReadMultipleResp ress {:?}", ress);
        // debug!("ReadMultipleResp errors {:?}", errors);
@@ -839,3 +916,181 @@ impl SetDisks {
        Ok(())
    }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn collect_read_multiple_results_fails_early_when_quorum_is_impossible() {
+        let started = std::time::Instant::now();
+        let resp = ReadMultipleResp {
+            bucket: "bucket".to_string(),
+            prefix: "prefix".to_string(),
+            file: "file".to_string(),
+            exists: true,
+            error: String::new(),
+            data: vec![1],
+            mod_time: None,
+        };
+
+        let tasks: Vec<_> = vec![
+            (10_u64, Err(DiskError::DiskNotFound)),
+            (15, Err(DiskError::DiskNotFound)),
+            (250, Ok::<Vec<ReadMultipleResp>, DiskError>(vec![resp])),
+        ]
+        .into_iter()
+        .map(|(delay_ms, outcome)| async move {
+            tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
+            outcome
+        })
+        .collect();
+
+        let result = collect_read_multiple_results(tasks, 2).await;
+        assert!(result.is_err(), "quorum should become impossible before slow tail completes");
+        assert!(started.elapsed() < std::time::Duration::from_millis(120));
+    }
+
+    #[tokio::test]
+    async fn collect_read_multiple_results_returns_collected_responses_on_quorum() {
+        let resp = ReadMultipleResp {
+            bucket: "bucket".to_string(),
+            prefix: "prefix".to_string(),
+            file: "file".to_string(),
+            exists: true,
+            error: String::new(),
+            data: vec![1, 2, 3],
+            mod_time: None,
+        };
+
+        let tasks: Vec<_> = vec![
+            (10_u64, Ok::<Vec<ReadMultipleResp>, DiskError>(vec![resp.clone()])),
+            (15, Ok::<Vec<ReadMultipleResp>, DiskError>(vec![resp.clone()])),
+            (250, Err(DiskError::DiskNotFound)),
+        ]
+        .into_iter()
+        .map(|(delay_ms, outcome)| async move {
+            tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
+            outcome
+        })
+        .collect();
+
+        let (responses, errors) = collect_read_multiple_results(tasks, 2).await.expect("quorum should succeed");
+
+        assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2);
+        assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
+    }
+
+    #[tokio::test]
+    async fn collect_read_multiple_results_tolerates_single_panicked_task_when_quorum_is_met() {
+        let resp = ReadMultipleResp {
+            bucket: "bucket".to_string(),
+            prefix: "prefix".to_string(),
+            file: "file".to_string(),
+            exists: true,
+            error: String::new(),
+            data: vec![1, 2, 3],
+            mod_time: None,
+        };
+
+        let tasks: Vec<_> = vec![(5_u64, true), (10, false), (12, false)]
+            .into_iter()
+            .map(|(delay_ms, should_panic)| {
+                let resp = resp.clone();
+                async move {
+                    tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
+                    if should_panic {
+                        panic!("simulated task panic");
+                    }
+                    Ok::<Vec<ReadMultipleResp>, DiskError>(vec![resp])
+                }
+            })
+            .collect();
+
+        let (responses, errors) = collect_read_multiple_results(tasks, 2)
+            .await
+            .expect("quorum should still succeed");
+        assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2);
+        assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
+    }
+
+    #[tokio::test]
+    async fn collect_read_parts_results_fails_early_when_quorum_is_impossible() {
+        let started = std::time::Instant::now();
+        let part = ObjectPartInfo {
+            number: 1,
+            etag: "etag".to_string(),
+            ..Default::default()
+        };
+
+        let tasks: Vec<_> = vec![
+            (10_u64, Err(DiskError::DiskNotFound)),
+            (15, Err(DiskError::DiskNotFound)),
+            (250, Ok::<Vec<ObjectPartInfo>, DiskError>(vec![part])),
+        ]
+        .into_iter()
+        .map(|(delay_ms, outcome)| async move {
+            tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
+            outcome
+        })
+        .collect();
+
+        let result = collect_read_parts_results(tasks, 2).await;
+        assert!(result.is_err(), "quorum should become impossible before slow tail completes");
+        assert!(started.elapsed() < std::time::Duration::from_millis(120));
+    }
+
+    #[tokio::test]
+    async fn collect_read_parts_results_returns_collected_responses_on_quorum() {
+        let part = ObjectPartInfo {
+            number: 1,
+            etag: "etag".to_string(),
+            ..Default::default()
+        };
+
+        let tasks: Vec<_> = vec![
+            (10_u64, Ok::<Vec<ObjectPartInfo>, DiskError>(vec![part.clone()])),
+            (15, Ok::<Vec<ObjectPartInfo>, DiskError>(vec![part.clone()])),
+            (250, Err(DiskError::DiskNotFound)),
+        ]
+        .into_iter()
+        .map(|(delay_ms, outcome)| async move {
+            tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
+            outcome
+        })
+        .collect();
+
+        let (responses, errors) = collect_read_parts_results(tasks, 2).await.expect("quorum should succeed");
+        assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2);
+        assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
+    }
+
+    #[tokio::test]
+    async fn collect_read_parts_results_tolerates_single_panicked_task_when_quorum_is_met() {
+        let part = ObjectPartInfo {
+            number: 1,
+            etag: "etag".to_string(),
+            ..Default::default()
+        };
+
+        let tasks: Vec<_> = vec![(5_u64, true), (10, false), (12, false)]
+            .into_iter()
+            .map(|(delay_ms, should_panic)| {
+                let part = part.clone();
+                async move {
+                    tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
+                    if should_panic {
+                        panic!("simulated task panic");
+                    }
+                    Ok::<Vec<ObjectPartInfo>, DiskError>(vec![part])
+                }
+            })
+            .collect();
+
+        let (responses, errors) = collect_read_parts_results(tasks, 2)
+            .await
+            .expect("quorum should still succeed");
+        assert_eq!(responses.iter().filter(|item| item.is_some()).count(), 2);
+        assert_eq!(errors.iter().filter(|item| item.is_none()).count(), 2);
+    }
+}