fix(ecstore): harden rebalance data movement (#3234)

* fix(ecstore): harden rebalance data movement * fix(ecstore): preserve failed rebalance status * fix(ecstore): avoid rebalance walkdir total timeout * fix(ecstore): retry rebalance listing timeouts * fix(ecstore): restrict data movement resume target * refactor(ecstore): simplify multipart movement target * fix(ecstore): restore resume target checks * perf(ecstore): speed up rebalance bucket merges * fix: keep rebalance listing alive on transient failures --------- Co-authored-by: houseme <housemecn@gmail.com>
2026-08-25 21:46:50 +00:00 · 2026-06-08 18:40:26 +08:00
parent 2fc1cf90e0
commit 9504dff595
11 changed files with 2673 additions and 139 deletions
@@ -21,6 +21,11 @@ struct LatestObjectInfoCandidate {
    err: Option<Error>,
 }

+struct RebalanceDeletePoolResult {
+    pool_idx: usize,
+    result: Result<ObjectInfo>,
+}
+
 fn pool_lookup_not_found_error(bucket: &str, object: &str, opts: &ObjectOptions) -> Error {
    let object = decode_dir_object(object);

@@ -39,6 +44,52 @@ fn resolve_rebalance_delete_from_all_pools_result(result: Result<ObjectInfo>, bu
    result.map_err(|err| Error::other(format!("failed to delete rebalance source object {bucket}/{object}: {err}")))
 }

+fn is_ignorable_rebalance_delete_error(err: &Error) -> bool {
+    is_err_object_not_found(err) || is_err_version_not_found(err)
+}
+
+fn rebalance_delete_pool_error(pool_idx: usize, bucket: &str, object: &str, err: Error) -> Error {
+    Error::other(format!("pool {pool_idx} delete failed for {bucket}/{object}: {err}"))
+}
+
+fn resolve_rebalance_delete_from_all_pools_results(
+    results: Vec<RebalanceDeletePoolResult>,
+    bucket: &str,
+    object: &str,
+) -> Result<ObjectInfo> {
+    let mut deleted = None;
+    let mut ignored_error = None;
+
+    for pool_result in results {
+        let pool_idx = pool_result.pool_idx;
+        match pool_result.result {
+            Ok(info) => {
+                if deleted.is_none() {
+                    deleted = Some(info);
+                }
+            }
+            Err(err) if is_ignorable_rebalance_delete_error(&err) => {
+                ignored_error = Some((pool_idx, err));
+            }
+            Err(err) => {
+                return Err(rebalance_delete_pool_error(pool_idx, bucket, object, err));
+            }
+        }
+    }
+
+    if let Some(info) = deleted {
+        return Ok(info);
+    }
+
+    if let Some((pool_idx, err)) = ignored_error {
+        return Err(rebalance_delete_pool_error(pool_idx, bucket, object, err));
+    }
+
+    Err(Error::other(format!(
+        "failed to delete rebalance source object {bucket}/{object}: no pools were attempted"
+    )))
+}
+
 fn rebalance_disk_set_lookup_error(pool_idx: usize, set_idx: usize, pool_count: usize) -> Error {
    Error::other(format!(
        "failed to resolve rebalance disk set: pool index {pool_idx}, set index {set_idx}, pool count {pool_count}",
@@ -220,6 +271,41 @@ impl ECStore {
        None
    }

+    pub(super) async fn get_available_pool_idx_excluding(
+        &self,
+        bucket: &str,
+        object: &str,
+        size: i64,
+        excluded_pool_idx: usize,
+    ) -> Option<usize> {
+        let mut server_pools = self.get_server_pools_available_space(bucket, object, size).await;
+        server_pools.filter_max_used(100 - (100_f64 * DISK_RESERVE_FRACTION) as u64);
+
+        if let Some(pool) = server_pools.0.get_mut(excluded_pool_idx) {
+            pool.available = 0;
+        }
+
+        let total = server_pools.total_available();
+        if total == 0 {
+            return None;
+        }
+
+        let mut rng = rand::rng();
+        let random_u64: u64 = rng.random_range(0..total);
+
+        let choose = random_u64 % total;
+        let mut at_total = 0;
+
+        for pool in server_pools.iter() {
+            at_total += pool.available;
+            if at_total > choose && pool.available > 0 {
+                return Some(pool.index);
+            }
+        }
+
+        None
+    }
+
    async fn get_server_pools_available_space(&self, bucket: &str, object: &str, size: i64) -> ServerPoolsAvailableSpace {
        let mut n_sets = vec![0; self.pools.len()];
        let mut infos = vec![Vec::new(); self.pools.len()];
@@ -507,38 +593,34 @@ impl ECStore {
        opts: &ObjectOptions,
        errs: Vec<PoolErr>,
    ) -> Result<ObjectInfo> {
-        let mut objs = Vec::new();
-        let mut derrs = Vec::new();
+        let mut results = Vec::with_capacity(errs.len());

        for pe in errs.iter() {
            if let Some(err) = &pe.err
                && err == &StorageError::ErasureWriteQuorum
            {
-                objs.push(None);
-                derrs.push(Some(StorageError::ErasureWriteQuorum));
+                if let Some(idx) = pe.index {
+                    results.push(RebalanceDeletePoolResult {
+                        pool_idx: idx,
+                        result: Err(StorageError::ErasureWriteQuorum),
+                    });
+                }
                continue;
            }

            if let Some(idx) = pe.index {
-                match self.pools[idx].delete_object(bucket, object, opts.clone()).await {
-                    Ok(res) => {
-                        objs.push(Some(res));
-
-                        derrs.push(None);
-                    }
-                    Err(err) => {
-                        objs.push(None);
-                        derrs.push(Some(err));
-                    }
-                }
+                results.push(RebalanceDeletePoolResult {
+                    pool_idx: idx,
+                    result: self.pools[idx].delete_object(bucket, object, opts.clone()).await,
+                });
            }
        }

-        if let Some(e) = &derrs[0] {
-            return resolve_rebalance_delete_from_all_pools_result(Err(e.clone()), bucket, object);
-        }
-
-        resolve_rebalance_delete_from_all_pools_result(Ok(objs[0].as_ref().unwrap().clone()), bucket, object)
+        resolve_rebalance_delete_from_all_pools_result(
+            resolve_rebalance_delete_from_all_pools_results(results, bucket, object),
+            bucket,
+            object,
+        )
    }

    pub async fn reload_pool_meta(&self) -> Result<()> {
@@ -902,6 +984,139 @@ mod tests {
        assert!(rendered.contains(&Error::SlowDown.to_string()), "{rendered}");
    }

+    #[test]
+    fn resolve_rebalance_delete_from_all_pools_results_fails_on_later_pool_error() {
+        let err = resolve_rebalance_delete_from_all_pools_results(
+            vec![
+                RebalanceDeletePoolResult {
+                    pool_idx: 0,
+                    result: Ok(ObjectInfo {
+                        bucket: "bucket".to_string(),
+                        name: "object".to_string(),
+                        ..Default::default()
+                    }),
+                },
+                RebalanceDeletePoolResult {
+                    pool_idx: 1,
+                    result: Err(Error::SlowDown),
+                },
+            ],
+            "bucket",
+            "object",
+        )
+        .expect_err("non-ignorable errors from later pools must not be hidden");
+        let rendered = err.to_string();
+
+        assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
+        assert!(rendered.contains(&Error::SlowDown.to_string()), "{rendered}");
+    }
+
+    #[test]
+    fn resolve_rebalance_delete_from_all_pools_results_ignores_later_not_found_after_success() {
+        let info = ObjectInfo {
+            bucket: "bucket".to_string(),
+            name: "object".to_string(),
+            ..Default::default()
+        };
+
+        let resolved = resolve_rebalance_delete_from_all_pools_results(
+            vec![
+                RebalanceDeletePoolResult {
+                    pool_idx: 0,
+                    result: Ok(info.clone()),
+                },
+                RebalanceDeletePoolResult {
+                    pool_idx: 1,
+                    result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
+                },
+            ],
+            "bucket",
+            "object",
+        )
+        .expect("not-found errors from other pools should be ignored when a delete succeeds");
+
+        assert_eq!(resolved.bucket, info.bucket);
+        assert_eq!(resolved.name, info.name);
+    }
+
+    #[test]
+    fn resolve_rebalance_delete_from_all_pools_results_accepts_success_after_not_found() {
+        let info = ObjectInfo {
+            bucket: "bucket".to_string(),
+            name: "object".to_string(),
+            ..Default::default()
+        };
+
+        let resolved = resolve_rebalance_delete_from_all_pools_results(
+            vec![
+                RebalanceDeletePoolResult {
+                    pool_idx: 0,
+                    result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
+                },
+                RebalanceDeletePoolResult {
+                    pool_idx: 1,
+                    result: Ok(info.clone()),
+                },
+            ],
+            "bucket",
+            "object",
+        )
+        .expect("a successful delete should pass even when an earlier pool reports not-found");
+
+        assert_eq!(resolved.bucket, info.bucket);
+        assert_eq!(resolved.name, info.name);
+    }
+
+    #[test]
+    fn resolve_rebalance_delete_from_all_pools_results_fails_when_all_results_are_ignored_errors() {
+        let err = resolve_rebalance_delete_from_all_pools_results(
+            vec![
+                RebalanceDeletePoolResult {
+                    pool_idx: 0,
+                    result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
+                },
+                RebalanceDeletePoolResult {
+                    pool_idx: 1,
+                    result: Err(Error::VersionNotFound("bucket".to_string(), "object".to_string(), "vid-1".to_string())),
+                },
+            ],
+            "bucket",
+            "object",
+        )
+        .expect_err("all ignored errors without any successful delete should still fail");
+        let rendered = err.to_string();
+
+        assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
+        assert!(rendered.contains("Version not found"), "{rendered}");
+    }
+
+    #[test]
+    fn resolve_rebalance_delete_from_all_pools_results_fails_on_write_quorum_even_with_success() {
+        let err = resolve_rebalance_delete_from_all_pools_results(
+            vec![
+                RebalanceDeletePoolResult {
+                    pool_idx: 0,
+                    result: Ok(ObjectInfo {
+                        bucket: "bucket".to_string(),
+                        name: "object".to_string(),
+                        ..Default::default()
+                    }),
+                },
+                RebalanceDeletePoolResult {
+                    pool_idx: 1,
+                    result: Err(Error::ErasureWriteQuorum),
+                },
+            ],
+            "bucket",
+            "object",
+        )
+        .expect_err("write quorum failures must fail the aggregate delete");
+        let rendered = err.to_string();
+
+        assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
+        assert!(rendered.contains(&Error::ErasureWriteQuorum.to_string()), "{rendered}");
+    }
+
    #[test]
    fn rebalance_disk_set_lookup_error_formats_pool_and_set_context() {
        let err = rebalance_disk_set_lookup_error(2, 7, 3);