fix(ecstore): harden rebalance data movement (#3234)

* fix(ecstore): harden rebalance data movement

* fix(ecstore): preserve failed rebalance status

* fix(ecstore): avoid rebalance walkdir total timeout

* fix(ecstore): retry rebalance listing timeouts

* fix(ecstore): restrict data movement resume target

* refactor(ecstore): simplify multipart movement target

* fix(ecstore): restore resume target checks

* perf(ecstore): speed up rebalance bucket merges

* fix: keep rebalance listing alive on transient failures

---------

Co-authored-by: houseme <housemecn@gmail.com>

crates/ecstore/src/store/multipart.rs

@@ -115,10 +115,24 @@ impl ECStore {
         object: &str,
         opts: &ObjectOptions,
     ) -> Result<MultipartUploadResult> {
+        self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts)
+            .await
+            .map(|(res, _)| res)
+    }
+
+    pub(crate) async fn handle_new_multipart_upload_with_pool_idx(
+        &self,
+        bucket: &str,
+        object: &str,
+        opts: &ObjectOptions,
+    ) -> Result<(MultipartUploadResult, usize)> {
         check_new_multipart_args(bucket, object)?;
 
         if self.single_pool() {
-            return self.pools[0].new_multipart_upload(bucket, object, opts).await;
+            return self.pools[0]
+                .new_multipart_upload(bucket, object, opts)
+                .await
+                .map(|res| (res, 0));
         }
 
         for (idx, pool) in self.pools.iter().enumerate() {
@@ -130,7 +144,8 @@ impl ECStore {
                 .await?;
 
             if !res.uploads.is_empty() {
-                return self.pools[idx].new_multipart_upload(bucket, object, opts).await;
+                let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
+                return Ok((res, idx));
             }
         }
         let idx = self.get_pool_idx(bucket, object, -1).await?;
@@ -142,7 +157,8 @@ impl ECStore {
             ));
         }
 
-        self.pools[idx].new_multipart_upload(bucket, object, opts).await
+        let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
+        Ok((res, idx))
     }
 
     #[instrument(skip(self))]

crates/ecstore/src/store/object.rs

@@ -233,6 +233,89 @@ fn data_movement_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> Object
     lookup_opts
 }
 
+fn effective_object_actual_size(info: &ObjectInfo) -> Option<i64> {
+    info.get_actual_size().ok()
+}
+
+fn is_equivalent_data_movement_delete_marker(source: &ObjectInfo, target: &ObjectInfo) -> bool {
+    is_data_movement_delete_marker(source)
+        && is_data_movement_delete_marker(target)
+        && source.version_id == target.version_id
+        && source.mod_time == target.mod_time
+}
+
+fn is_data_movement_delete_marker(info: &ObjectInfo) -> bool {
+    info.delete_marker
+}
+
+fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo, target: &ObjectInfo) -> bool {
+    source.version_id == target.version_id
+        && !target.delete_marker
+        && source.size == target.size
+        && source.get_etag() == target.etag
+        && source.checksum == target.checksum
+        && source.mod_time == target.mod_time
+        && source.transition_status == target.transitioned_object.status
+        && source.transitioned_objname == target.transitioned_object.name
+        && source.transition_tier == target.transitioned_object.tier
+        && source
+            .transition_version_id
+            .map(|version_id| version_id.to_string())
+            .unwrap_or_default()
+            == target.transitioned_object.version_id
+        && effective_object_actual_size(target) == Some(source.size)
+}
+
+fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
+    target_pool_idx != src_pool_idx
+}
+
+fn resolve_data_movement_resume_target_pool(
+    selected_target_pool_idx: usize,
+    resume_target_pool_idx: Option<usize>,
+    src_pool_idx: usize,
+) -> usize {
+    if should_check_data_movement_resume_target(src_pool_idx, selected_target_pool_idx) {
+        selected_target_pool_idx
+    } else {
+        resume_target_pool_idx.unwrap_or(selected_target_pool_idx)
+    }
+}
+
+fn resolve_data_movement_delete_marker_resume_result(
+    target_result: Result<Option<ObjectInfo>>,
+    source: &ObjectInfo,
+    src_pool_idx: usize,
+    target_pool_idx: usize,
+) -> Result<bool> {
+    if !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) {
+        return Ok(false);
+    }
+
+    let Some(target) = target_result? else {
+        return Ok(false);
+    };
+
+    Ok(is_equivalent_data_movement_delete_marker(source, &target))
+}
+
+fn resolve_data_movement_tiered_resume_result(
+    target_result: Result<Option<ObjectInfo>>,
+    source: &rustfs_filemeta::FileInfo,
+    src_pool_idx: usize,
+    target_pool_idx: usize,
+) -> Result<bool> {
+    if !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) {
+        return Ok(false);
+    }
+
+    let Some(target) = target_result? else {
+        return Ok(false);
+    };
+
+    Ok(is_equivalent_data_movement_tiered_object(source, &target))
+}
+
 impl ECStore {
     fn map_namespace_lock_error(bucket: &str, object: &str, mode: &'static str, err: rustfs_lock::LockError) -> StorageError {
         match err {
@@ -377,6 +460,62 @@ impl ECStore {
         }
     }
 
+    async fn find_data_movement_target_info(
+        &self,
+        bucket: &str,
+        object: &str,
+        target_pool_idx: usize,
+        opts: &ObjectOptions,
+    ) -> Result<Option<ObjectInfo>> {
+        let lookup_opts = version_aware_lookup_opts(opts, true);
+
+        let Some(pool) = self.pools.get(target_pool_idx) else {
+            return Err(Error::other(format!(
+                "data movement resume target pool {target_pool_idx} is out of range for {bucket}/{object}"
+            )));
+        };
+
+        match pool.get_object_info(bucket, object, &lookup_opts).await {
+            Ok(info) => Ok(Some(info)),
+            Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(None),
+            Err(err) => Err(err),
+        }
+    }
+
+    async fn has_equivalent_data_movement_delete_marker(
+        &self,
+        bucket: &str,
+        object: &str,
+        source: &ObjectInfo,
+        opts: &ObjectOptions,
+        target_pool_idx: usize,
+    ) -> Result<bool> {
+        resolve_data_movement_delete_marker_resume_result(
+            self.find_data_movement_target_info(bucket, object, target_pool_idx, opts)
+                .await,
+            source,
+            opts.src_pool_idx,
+            target_pool_idx,
+        )
+    }
+
+    async fn has_equivalent_data_movement_tiered_object(
+        &self,
+        bucket: &str,
+        object: &str,
+        source: &rustfs_filemeta::FileInfo,
+        opts: &ObjectOptions,
+        target_pool_idx: usize,
+    ) -> Result<bool> {
+        resolve_data_movement_tiered_resume_result(
+            self.find_data_movement_target_info(bucket, object, target_pool_idx, opts)
+                .await,
+            source,
+            opts.src_pool_idx,
+            target_pool_idx,
+        )
+    }
+
     fn resolve_decommission_target_pool_idx_result(result: Result<usize>, bucket: &str, object: &str) -> Result<usize> {
         result.map_err(|err| Error::other(format!("failed to select decommission target pool for {bucket}/{object}: {err}")))
     }
@@ -419,6 +558,17 @@ impl ECStore {
             )?
         };
         if opts.data_movement && idx == opts.src_pool_idx {
+            let resume_target_pool_idx = self
+                .get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx)
+                .await;
+            let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx);
+            if self
+                .has_equivalent_data_movement_tiered_object(bucket, &object, fi, opts, target_pool_idx)
+                .await?
+            {
+                return Ok(());
+            }
+
             return Err(StorageError::DataMovementOverwriteErr(
                 bucket.to_owned(),
                 object.to_owned(),
@@ -666,17 +816,44 @@ impl ECStore {
         let gopts = version_aware_lookup_opts(&opts, true);
 
         if opts.data_movement {
-            let existing_pool_idx = self
-                .get_pool_info_existing_with_opts(bucket, object, &gopts)
-                .await
-                .map(|(pinfo, _)| pinfo.index);
-            let target_pool_idx =
+            let existing_pool_info = self.get_pool_info_existing_with_opts(bucket, object, &gopts).await;
+            let existing_pool_idx = existing_pool_info
+                .as_ref()
+                .map(|(pinfo, _)| pinfo.index)
+                .map_err(Clone::clone);
+            let selected_target_pool_idx =
                 match select_data_movement_target_pool(existing_pool_idx, opts.src_pool_idx, opts.delete_marker)? {
                     Some(pool_idx) => pool_idx,
                     None => self.get_pool_idx_no_lock(bucket, object, 0).await?,
                 };
+            let resume_target_pool_idx = if selected_target_pool_idx == opts.src_pool_idx {
+                self.get_available_pool_idx_excluding(bucket, object, 0, opts.src_pool_idx)
+                    .await
+            } else {
+                None
+            };
+            let target_pool_idx =
+                resolve_data_movement_resume_target_pool(selected_target_pool_idx, resume_target_pool_idx, opts.src_pool_idx);
+
+            if opts.src_pool_idx == selected_target_pool_idx {
+                if let Ok((source_pool_info, _)) = existing_pool_info
+                    && opts.delete_marker
+                    && is_data_movement_delete_marker(&source_pool_info.object_info)
+                    && self
+                        .has_equivalent_data_movement_delete_marker(
+                            bucket,
+                            object,
+                            &source_pool_info.object_info,
+                            &opts,
+                            target_pool_idx,
+                        )
+                        .await?
+                {
+                    let mut obj = source_pool_info.object_info;
+                    obj.name = decode_dir_object(object);
+                    return Ok(obj);
+                }
 
-            if opts.src_pool_idx == target_pool_idx {
                 return Err(StorageError::DataMovementOverwriteErr(
                     bucket.to_owned(),
                     object.to_owned(),
@@ -684,7 +861,9 @@ impl ECStore {
                 ));
             }
 
-            let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?;
+            let mut obj = self.pools[selected_target_pool_idx]
+                .delete_object(bucket, object, opts)
+                .await?;
             obj.name = decode_dir_object(obj.name.as_str());
             return Ok(obj);
         }
@@ -1086,6 +1265,9 @@ impl ECStore {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::bucket::lifecycle::bucket_lifecycle_ops::TransitionedObject;
+    use crate::bucket::lifecycle::core::TRANSITION_COMPLETE;
+    use bytes::Bytes;
     use std::io::Cursor;
     use tokio::io::AsyncReadExt;
 
@@ -1108,6 +1290,240 @@ mod tests {
         assert_eq!(target, Some(0));
     }
 
+    #[test]
+    fn equivalent_data_movement_delete_marker_requires_same_version_and_mod_time() {
+        let version_id = Uuid::nil();
+        let mod_time = OffsetDateTime::UNIX_EPOCH;
+        let source = ObjectInfo {
+            version_id: Some(version_id),
+            delete_marker: true,
+            mod_time: Some(mod_time),
+            ..Default::default()
+        };
+        let target = source.clone();
+
+        assert!(is_equivalent_data_movement_delete_marker(&source, &target));
+
+        let mismatched = ObjectInfo {
+            mod_time: Some(mod_time + Duration::from_secs(1)),
+            ..target
+        };
+        assert!(!is_equivalent_data_movement_delete_marker(&source, &mismatched));
+    }
+
+    #[test]
+    fn equivalent_data_movement_delete_marker_rejects_live_object() {
+        let source = ObjectInfo {
+            delete_marker: true,
+            ..Default::default()
+        };
+        let target = ObjectInfo {
+            delete_marker: false,
+            ..source.clone()
+        };
+
+        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
+    }
+
+    #[test]
+    fn data_movement_delete_marker_resume_accepts_equivalent_target() {
+        let source = ObjectInfo {
+            version_id: Some(Uuid::nil()),
+            delete_marker: true,
+            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
+            ..Default::default()
+        };
+
+        let should_resume = resolve_data_movement_delete_marker_resume_result(Ok(Some(source.clone())), &source, 0, 1)
+            .expect("equivalent delete marker target should be evaluated");
+
+        assert!(should_resume);
+    }
+
+    #[test]
+    fn data_movement_delete_marker_resume_rejects_source_pool_target() {
+        let source = ObjectInfo {
+            version_id: Some(Uuid::nil()),
+            delete_marker: true,
+            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
+            ..Default::default()
+        };
+
+        let should_resume = resolve_data_movement_delete_marker_resume_result(Ok(Some(source.clone())), &source, 0, 0)
+            .expect("source-pool target should be rejected before target lookup");
+
+        assert!(!should_resume);
+    }
+
+    #[test]
+    fn data_movement_resume_target_prefers_selected_non_source_pool() {
+        let target_pool_idx = resolve_data_movement_resume_target_pool(2, Some(3), 1);
+        assert_eq!(target_pool_idx, 2);
+    }
+
+    #[test]
+    fn data_movement_resume_target_uses_resolved_non_source_pool_when_selected_is_source() {
+        let target_pool_idx = resolve_data_movement_resume_target_pool(1, Some(3), 1);
+        assert_eq!(target_pool_idx, 3);
+    }
+
+    #[test]
+    fn data_movement_resume_target_keeps_source_when_no_other_pool_is_available() {
+        let target_pool_idx = resolve_data_movement_resume_target_pool(1, None, 1);
+        assert_eq!(target_pool_idx, 1);
+    }
+
+    #[test]
+    fn data_movement_delete_marker_resume_propagates_target_lookup_error() {
+        let source = ObjectInfo {
+            delete_marker: true,
+            ..Default::default()
+        };
+        let result = resolve_data_movement_delete_marker_resume_result(Err(Error::SlowDown), &source, 0, 1);
+
+        assert!(matches!(result, Err(Error::SlowDown)));
+    }
+
+    #[test]
+    fn equivalent_data_movement_tiered_object_accepts_matching_transition_metadata() {
+        let version_id = Uuid::nil();
+        let transition_version_id = Uuid::new_v4();
+        let mod_time = OffsetDateTime::UNIX_EPOCH;
+        let source = FileInfo {
+            version_id: Some(version_id),
+            size: 1024,
+            mod_time: Some(mod_time),
+            checksum: Some(Bytes::from_static(b"checksum")),
+            transition_status: TRANSITION_COMPLETE.to_string(),
+            transitioned_objname: "remote/object".to_string(),
+            transition_tier: "WARM".to_string(),
+            transition_version_id: Some(transition_version_id),
+            metadata: HashMap::from([("etag".to_string(), "etag-value".to_string())]),
+            ..Default::default()
+        };
+        let target = ObjectInfo {
+            version_id: Some(version_id),
+            size: 1024,
+            mod_time: Some(mod_time),
+            checksum: Some(Bytes::from_static(b"checksum")),
+            etag: Some("etag-value".to_string()),
+            transitioned_object: TransitionedObject {
+                name: "remote/object".to_string(),
+                version_id: transition_version_id.to_string(),
+                tier: "WARM".to_string(),
+                status: TRANSITION_COMPLETE.to_string(),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+
+        assert!(is_equivalent_data_movement_tiered_object(&source, &target));
+    }
+
+    #[test]
+    fn equivalent_data_movement_tiered_object_rejects_transition_mismatch() {
+        let source = FileInfo {
+            version_id: Some(Uuid::nil()),
+            size: 1024,
+            transition_status: TRANSITION_COMPLETE.to_string(),
+            transitioned_objname: "remote/source".to_string(),
+            transition_tier: "WARM".to_string(),
+            ..Default::default()
+        };
+        let target = ObjectInfo {
+            version_id: source.version_id,
+            size: 1024,
+            transitioned_object: TransitionedObject {
+                name: "remote/target".to_string(),
+                tier: "WARM".to_string(),
+                status: TRANSITION_COMPLETE.to_string(),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+
+        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
+    }
+
+    #[test]
+    fn data_movement_tiered_resume_accepts_equivalent_target() {
+        let version_id = Uuid::nil();
+        let transition_version_id = Uuid::new_v4();
+        let source = FileInfo {
+            version_id: Some(version_id),
+            size: 1024,
+            transition_status: TRANSITION_COMPLETE.to_string(),
+            transitioned_objname: "remote/object".to_string(),
+            transition_tier: "WARM".to_string(),
+            transition_version_id: Some(transition_version_id),
+            metadata: HashMap::from([("etag".to_string(), "etag-value".to_string())]),
+            ..Default::default()
+        };
+        let target = ObjectInfo {
+            version_id: Some(version_id),
+            size: 1024,
+            etag: Some("etag-value".to_string()),
+            transitioned_object: TransitionedObject {
+                name: "remote/object".to_string(),
+                version_id: transition_version_id.to_string(),
+                tier: "WARM".to_string(),
+                status: TRANSITION_COMPLETE.to_string(),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+
+        let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 1)
+            .expect("equivalent tiered target should be evaluated");
+
+        assert!(should_resume);
+    }
+
+    #[test]
+    fn data_movement_tiered_resume_rejects_source_pool_target() {
+        let version_id = Uuid::nil();
+        let source = FileInfo {
+            version_id: Some(version_id),
+            size: 1024,
+            transition_status: TRANSITION_COMPLETE.to_string(),
+            transitioned_objname: "remote/object".to_string(),
+            transition_tier: "WARM".to_string(),
+            metadata: HashMap::from([("etag".to_string(), "etag-value".to_string())]),
+            ..Default::default()
+        };
+        let target = ObjectInfo {
+            version_id: Some(version_id),
+            size: 1024,
+            etag: Some("etag-value".to_string()),
+            transitioned_object: TransitionedObject {
+                name: "remote/object".to_string(),
+                tier: "WARM".to_string(),
+                status: TRANSITION_COMPLETE.to_string(),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+
+        let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 0)
+            .expect("source-pool target should be rejected before target lookup");
+
+        assert!(!should_resume);
+    }
+
+    #[test]
+    fn data_movement_tiered_resume_rejects_missing_target() {
+        let source = FileInfo {
+            version_id: Some(Uuid::nil()),
+            size: 1024,
+            ..Default::default()
+        };
+
+        let should_resume = resolve_data_movement_tiered_resume_result(Ok(None), &source, 0, 1)
+            .expect("missing tiered target should be evaluated");
+
+        assert!(!should_resume);
+    }
+
     #[test]
     fn latest_object_access_delete_marker_error_returns_none_for_live_object() {
         let info = ObjectInfo::default();

crates/ecstore/src/store/rebalance.rs

@@ -21,6 +21,11 @@ struct LatestObjectInfoCandidate {
     err: Option<Error>,
 }
 
+struct RebalanceDeletePoolResult {
+    pool_idx: usize,
+    result: Result<ObjectInfo>,
+}
+
 fn pool_lookup_not_found_error(bucket: &str, object: &str, opts: &ObjectOptions) -> Error {
     let object = decode_dir_object(object);
 
@@ -39,6 +44,52 @@ fn resolve_rebalance_delete_from_all_pools_result(result: Result<ObjectInfo>, bu
     result.map_err(|err| Error::other(format!("failed to delete rebalance source object {bucket}/{object}: {err}")))
 }
 
+fn is_ignorable_rebalance_delete_error(err: &Error) -> bool {
+    is_err_object_not_found(err) || is_err_version_not_found(err)
+}
+
+fn rebalance_delete_pool_error(pool_idx: usize, bucket: &str, object: &str, err: Error) -> Error {
+    Error::other(format!("pool {pool_idx} delete failed for {bucket}/{object}: {err}"))
+}
+
+fn resolve_rebalance_delete_from_all_pools_results(
+    results: Vec<RebalanceDeletePoolResult>,
+    bucket: &str,
+    object: &str,
+) -> Result<ObjectInfo> {
+    let mut deleted = None;
+    let mut ignored_error = None;
+
+    for pool_result in results {
+        let pool_idx = pool_result.pool_idx;
+        match pool_result.result {
+            Ok(info) => {
+                if deleted.is_none() {
+                    deleted = Some(info);
+                }
+            }
+            Err(err) if is_ignorable_rebalance_delete_error(&err) => {
+                ignored_error = Some((pool_idx, err));
+            }
+            Err(err) => {
+                return Err(rebalance_delete_pool_error(pool_idx, bucket, object, err));
+            }
+        }
+    }
+
+    if let Some(info) = deleted {
+        return Ok(info);
+    }
+
+    if let Some((pool_idx, err)) = ignored_error {
+        return Err(rebalance_delete_pool_error(pool_idx, bucket, object, err));
+    }
+
+    Err(Error::other(format!(
+        "failed to delete rebalance source object {bucket}/{object}: no pools were attempted"
+    )))
+}
+
 fn rebalance_disk_set_lookup_error(pool_idx: usize, set_idx: usize, pool_count: usize) -> Error {
     Error::other(format!(
         "failed to resolve rebalance disk set: pool index {pool_idx}, set index {set_idx}, pool count {pool_count}",
@@ -220,6 +271,41 @@ impl ECStore {
         None
     }
 
+    pub(super) async fn get_available_pool_idx_excluding(
+        &self,
+        bucket: &str,
+        object: &str,
+        size: i64,
+        excluded_pool_idx: usize,
+    ) -> Option<usize> {
+        let mut server_pools = self.get_server_pools_available_space(bucket, object, size).await;
+        server_pools.filter_max_used(100 - (100_f64 * DISK_RESERVE_FRACTION) as u64);
+
+        if let Some(pool) = server_pools.0.get_mut(excluded_pool_idx) {
+            pool.available = 0;
+        }
+
+        let total = server_pools.total_available();
+        if total == 0 {
+            return None;
+        }
+
+        let mut rng = rand::rng();
+        let random_u64: u64 = rng.random_range(0..total);
+
+        let choose = random_u64 % total;
+        let mut at_total = 0;
+
+        for pool in server_pools.iter() {
+            at_total += pool.available;
+            if at_total > choose && pool.available > 0 {
+                return Some(pool.index);
+            }
+        }
+
+        None
+    }
+
     async fn get_server_pools_available_space(&self, bucket: &str, object: &str, size: i64) -> ServerPoolsAvailableSpace {
         let mut n_sets = vec![0; self.pools.len()];
         let mut infos = vec![Vec::new(); self.pools.len()];
@@ -507,38 +593,34 @@ impl ECStore {
         opts: &ObjectOptions,
         errs: Vec<PoolErr>,
     ) -> Result<ObjectInfo> {
-        let mut objs = Vec::new();
-        let mut derrs = Vec::new();
+        let mut results = Vec::with_capacity(errs.len());
 
         for pe in errs.iter() {
             if let Some(err) = &pe.err
                 && err == &StorageError::ErasureWriteQuorum
             {
-                objs.push(None);
-                derrs.push(Some(StorageError::ErasureWriteQuorum));
+                if let Some(idx) = pe.index {
+                    results.push(RebalanceDeletePoolResult {
+                        pool_idx: idx,
+                        result: Err(StorageError::ErasureWriteQuorum),
+                    });
+                }
                 continue;
             }
 
             if let Some(idx) = pe.index {
-                match self.pools[idx].delete_object(bucket, object, opts.clone()).await {
-                    Ok(res) => {
-                        objs.push(Some(res));
-
-                        derrs.push(None);
-                    }
-                    Err(err) => {
-                        objs.push(None);
-                        derrs.push(Some(err));
-                    }
-                }
+                results.push(RebalanceDeletePoolResult {
+                    pool_idx: idx,
+                    result: self.pools[idx].delete_object(bucket, object, opts.clone()).await,
+                });
             }
         }
 
-        if let Some(e) = &derrs[0] {
-            return resolve_rebalance_delete_from_all_pools_result(Err(e.clone()), bucket, object);
-        }
-
-        resolve_rebalance_delete_from_all_pools_result(Ok(objs[0].as_ref().unwrap().clone()), bucket, object)
+        resolve_rebalance_delete_from_all_pools_result(
+            resolve_rebalance_delete_from_all_pools_results(results, bucket, object),
+            bucket,
+            object,
+        )
     }
 
     pub async fn reload_pool_meta(&self) -> Result<()> {
@@ -902,6 +984,139 @@ mod tests {
         assert!(rendered.contains(&Error::SlowDown.to_string()), "{rendered}");
     }
 
+    #[test]
+    fn resolve_rebalance_delete_from_all_pools_results_fails_on_later_pool_error() {
+        let err = resolve_rebalance_delete_from_all_pools_results(
+            vec![
+                RebalanceDeletePoolResult {
+                    pool_idx: 0,
+                    result: Ok(ObjectInfo {
+                        bucket: "bucket".to_string(),
+                        name: "object".to_string(),
+                        ..Default::default()
+                    }),
+                },
+                RebalanceDeletePoolResult {
+                    pool_idx: 1,
+                    result: Err(Error::SlowDown),
+                },
+            ],
+            "bucket",
+            "object",
+        )
+        .expect_err("non-ignorable errors from later pools must not be hidden");
+        let rendered = err.to_string();
+
+        assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
+        assert!(rendered.contains(&Error::SlowDown.to_string()), "{rendered}");
+    }
+
+    #[test]
+    fn resolve_rebalance_delete_from_all_pools_results_ignores_later_not_found_after_success() {
+        let info = ObjectInfo {
+            bucket: "bucket".to_string(),
+            name: "object".to_string(),
+            ..Default::default()
+        };
+
+        let resolved = resolve_rebalance_delete_from_all_pools_results(
+            vec![
+                RebalanceDeletePoolResult {
+                    pool_idx: 0,
+                    result: Ok(info.clone()),
+                },
+                RebalanceDeletePoolResult {
+                    pool_idx: 1,
+                    result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
+                },
+            ],
+            "bucket",
+            "object",
+        )
+        .expect("not-found errors from other pools should be ignored when a delete succeeds");
+
+        assert_eq!(resolved.bucket, info.bucket);
+        assert_eq!(resolved.name, info.name);
+    }
+
+    #[test]
+    fn resolve_rebalance_delete_from_all_pools_results_accepts_success_after_not_found() {
+        let info = ObjectInfo {
+            bucket: "bucket".to_string(),
+            name: "object".to_string(),
+            ..Default::default()
+        };
+
+        let resolved = resolve_rebalance_delete_from_all_pools_results(
+            vec![
+                RebalanceDeletePoolResult {
+                    pool_idx: 0,
+                    result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
+                },
+                RebalanceDeletePoolResult {
+                    pool_idx: 1,
+                    result: Ok(info.clone()),
+                },
+            ],
+            "bucket",
+            "object",
+        )
+        .expect("a successful delete should pass even when an earlier pool reports not-found");
+
+        assert_eq!(resolved.bucket, info.bucket);
+        assert_eq!(resolved.name, info.name);
+    }
+
+    #[test]
+    fn resolve_rebalance_delete_from_all_pools_results_fails_when_all_results_are_ignored_errors() {
+        let err = resolve_rebalance_delete_from_all_pools_results(
+            vec![
+                RebalanceDeletePoolResult {
+                    pool_idx: 0,
+                    result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
+                },
+                RebalanceDeletePoolResult {
+                    pool_idx: 1,
+                    result: Err(Error::VersionNotFound("bucket".to_string(), "object".to_string(), "vid-1".to_string())),
+                },
+            ],
+            "bucket",
+            "object",
+        )
+        .expect_err("all ignored errors without any successful delete should still fail");
+        let rendered = err.to_string();
+
+        assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
+        assert!(rendered.contains("Version not found"), "{rendered}");
+    }
+
+    #[test]
+    fn resolve_rebalance_delete_from_all_pools_results_fails_on_write_quorum_even_with_success() {
+        let err = resolve_rebalance_delete_from_all_pools_results(
+            vec![
+                RebalanceDeletePoolResult {
+                    pool_idx: 0,
+                    result: Ok(ObjectInfo {
+                        bucket: "bucket".to_string(),
+                        name: "object".to_string(),
+                        ..Default::default()
+                    }),
+                },
+                RebalanceDeletePoolResult {
+                    pool_idx: 1,
+                    result: Err(Error::ErasureWriteQuorum),
+                },
+            ],
+            "bucket",
+            "object",
+        )
+        .expect_err("write quorum failures must fail the aggregate delete");
+        let rendered = err.to_string();
+
+        assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
+        assert!(rendered.contains(&Error::ErasureWriteQuorum.to_string()), "{rendered}");
+    }
+
     #[test]
     fn rebalance_disk_set_lookup_error_formats_pool_and_set_context() {
         let err = rebalance_disk_set_lookup_error(2, 7, 3);