fix(ecstore): harden rebalance data movement (#3234)

* fix(ecstore): harden rebalance data movement * fix(ecstore): preserve failed rebalance status * fix(ecstore): avoid rebalance walkdir total timeout * fix(ecstore): retry rebalance listing timeouts * fix(ecstore): restrict data movement resume target * refactor(ecstore): simplify multipart movement target * fix(ecstore): restore resume target checks * perf(ecstore): speed up rebalance bucket merges * fix: keep rebalance listing alive on transient failures --------- Co-authored-by: houseme <housemecn@gmail.com>
2026-08-12 08:06:54 +00:00 · 2026-06-08 18:40:26 +08:00
parent 2fc1cf90e0
commit 9504dff595
11 changed files with 2673 additions and 139 deletions
@@ -233,6 +233,89 @@ fn data_movement_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> Object
    lookup_opts
 }

+fn effective_object_actual_size(info: &ObjectInfo) -> Option<i64> {
+    info.get_actual_size().ok()
+}
+
+fn is_equivalent_data_movement_delete_marker(source: &ObjectInfo, target: &ObjectInfo) -> bool {
+    is_data_movement_delete_marker(source)
+        && is_data_movement_delete_marker(target)
+        && source.version_id == target.version_id
+        && source.mod_time == target.mod_time
+}
+
+fn is_data_movement_delete_marker(info: &ObjectInfo) -> bool {
+    info.delete_marker
+}
+
+fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo, target: &ObjectInfo) -> bool {
+    source.version_id == target.version_id
+        && !target.delete_marker
+        && source.size == target.size
+        && source.get_etag() == target.etag
+        && source.checksum == target.checksum
+        && source.mod_time == target.mod_time
+        && source.transition_status == target.transitioned_object.status
+        && source.transitioned_objname == target.transitioned_object.name
+        && source.transition_tier == target.transitioned_object.tier
+        && source
+            .transition_version_id
+            .map(|version_id| version_id.to_string())
+            .unwrap_or_default()
+            == target.transitioned_object.version_id
+        && effective_object_actual_size(target) == Some(source.size)
+}
+
+fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
+    target_pool_idx != src_pool_idx
+}
+
+fn resolve_data_movement_resume_target_pool(
+    selected_target_pool_idx: usize,
+    resume_target_pool_idx: Option<usize>,
+    src_pool_idx: usize,
+) -> usize {
+    if should_check_data_movement_resume_target(src_pool_idx, selected_target_pool_idx) {
+        selected_target_pool_idx
+    } else {
+        resume_target_pool_idx.unwrap_or(selected_target_pool_idx)
+    }
+}
+
+fn resolve_data_movement_delete_marker_resume_result(
+    target_result: Result<Option<ObjectInfo>>,
+    source: &ObjectInfo,
+    src_pool_idx: usize,
+    target_pool_idx: usize,
+) -> Result<bool> {
+    if !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) {
+        return Ok(false);
+    }
+
+    let Some(target) = target_result? else {
+        return Ok(false);
+    };
+
+    Ok(is_equivalent_data_movement_delete_marker(source, &target))
+}
+
+fn resolve_data_movement_tiered_resume_result(
+    target_result: Result<Option<ObjectInfo>>,
+    source: &rustfs_filemeta::FileInfo,
+    src_pool_idx: usize,
+    target_pool_idx: usize,
+) -> Result<bool> {
+    if !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) {
+        return Ok(false);
+    }
+
+    let Some(target) = target_result? else {
+        return Ok(false);
+    };
+
+    Ok(is_equivalent_data_movement_tiered_object(source, &target))
+}
+
 impl ECStore {
    fn map_namespace_lock_error(bucket: &str, object: &str, mode: &'static str, err: rustfs_lock::LockError) -> StorageError {
        match err {
@@ -377,6 +460,62 @@ impl ECStore {
        }
    }

+    async fn find_data_movement_target_info(
+        &self,
+        bucket: &str,
+        object: &str,
+        target_pool_idx: usize,
+        opts: &ObjectOptions,
+    ) -> Result<Option<ObjectInfo>> {
+        let lookup_opts = version_aware_lookup_opts(opts, true);
+
+        let Some(pool) = self.pools.get(target_pool_idx) else {
+            return Err(Error::other(format!(
+                "data movement resume target pool {target_pool_idx} is out of range for {bucket}/{object}"
+            )));
+        };
+
+        match pool.get_object_info(bucket, object, &lookup_opts).await {
+            Ok(info) => Ok(Some(info)),
+            Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(None),
+            Err(err) => Err(err),
+        }
+    }
+
+    async fn has_equivalent_data_movement_delete_marker(
+        &self,
+        bucket: &str,
+        object: &str,
+        source: &ObjectInfo,
+        opts: &ObjectOptions,
+        target_pool_idx: usize,
+    ) -> Result<bool> {
+        resolve_data_movement_delete_marker_resume_result(
+            self.find_data_movement_target_info(bucket, object, target_pool_idx, opts)
+                .await,
+            source,
+            opts.src_pool_idx,
+            target_pool_idx,
+        )
+    }
+
+    async fn has_equivalent_data_movement_tiered_object(
+        &self,
+        bucket: &str,
+        object: &str,
+        source: &rustfs_filemeta::FileInfo,
+        opts: &ObjectOptions,
+        target_pool_idx: usize,
+    ) -> Result<bool> {
+        resolve_data_movement_tiered_resume_result(
+            self.find_data_movement_target_info(bucket, object, target_pool_idx, opts)
+                .await,
+            source,
+            opts.src_pool_idx,
+            target_pool_idx,
+        )
+    }
+
    fn resolve_decommission_target_pool_idx_result(result: Result<usize>, bucket: &str, object: &str) -> Result<usize> {
        result.map_err(|err| Error::other(format!("failed to select decommission target pool for {bucket}/{object}: {err}")))
    }
@@ -419,6 +558,17 @@ impl ECStore {
            )?
        };
        if opts.data_movement && idx == opts.src_pool_idx {
+            let resume_target_pool_idx = self
+                .get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx)
+                .await;
+            let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx);
+            if self
+                .has_equivalent_data_movement_tiered_object(bucket, &object, fi, opts, target_pool_idx)
+                .await?
+            {
+                return Ok(());
+            }
+
            return Err(StorageError::DataMovementOverwriteErr(
                bucket.to_owned(),
                object.to_owned(),
@@ -666,17 +816,44 @@ impl ECStore {
        let gopts = version_aware_lookup_opts(&opts, true);

        if opts.data_movement {
-            let existing_pool_idx = self
-                .get_pool_info_existing_with_opts(bucket, object, &gopts)
-                .await
-                .map(|(pinfo, _)| pinfo.index);
-            let target_pool_idx =
+            let existing_pool_info = self.get_pool_info_existing_with_opts(bucket, object, &gopts).await;
+            let existing_pool_idx = existing_pool_info
+                .as_ref()
+                .map(|(pinfo, _)| pinfo.index)
+                .map_err(Clone::clone);
+            let selected_target_pool_idx =
                match select_data_movement_target_pool(existing_pool_idx, opts.src_pool_idx, opts.delete_marker)? {
                    Some(pool_idx) => pool_idx,
                    None => self.get_pool_idx_no_lock(bucket, object, 0).await?,
                };
+            let resume_target_pool_idx = if selected_target_pool_idx == opts.src_pool_idx {
+                self.get_available_pool_idx_excluding(bucket, object, 0, opts.src_pool_idx)
+                    .await
+            } else {
+                None
+            };
+            let target_pool_idx =
+                resolve_data_movement_resume_target_pool(selected_target_pool_idx, resume_target_pool_idx, opts.src_pool_idx);
+
+            if opts.src_pool_idx == selected_target_pool_idx {
+                if let Ok((source_pool_info, _)) = existing_pool_info
+                    && opts.delete_marker
+                    && is_data_movement_delete_marker(&source_pool_info.object_info)
+                    && self
+                        .has_equivalent_data_movement_delete_marker(
+                            bucket,
+                            object,
+                            &source_pool_info.object_info,
+                            &opts,
+                            target_pool_idx,
+                        )
+                        .await?
+                {
+                    let mut obj = source_pool_info.object_info;
+                    obj.name = decode_dir_object(object);
+                    return Ok(obj);
+                }

-            if opts.src_pool_idx == target_pool_idx {
                return Err(StorageError::DataMovementOverwriteErr(
                    bucket.to_owned(),
                    object.to_owned(),
@@ -684,7 +861,9 @@ impl ECStore {
                ));
            }

-            let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?;
+            let mut obj = self.pools[selected_target_pool_idx]
+                .delete_object(bucket, object, opts)
+                .await?;
            obj.name = decode_dir_object(obj.name.as_str());
            return Ok(obj);
        }
@@ -1086,6 +1265,9 @@ impl ECStore {
 #[cfg(test)]
 mod tests {
    use super::*;
+    use crate::bucket::lifecycle::bucket_lifecycle_ops::TransitionedObject;
+    use crate::bucket::lifecycle::core::TRANSITION_COMPLETE;
+    use bytes::Bytes;
    use std::io::Cursor;
    use tokio::io::AsyncReadExt;

@@ -1108,6 +1290,240 @@ mod tests {
        assert_eq!(target, Some(0));
    }

+    #[test]
+    fn equivalent_data_movement_delete_marker_requires_same_version_and_mod_time() {
+        let version_id = Uuid::nil();
+        let mod_time = OffsetDateTime::UNIX_EPOCH;
+        let source = ObjectInfo {
+            version_id: Some(version_id),
+            delete_marker: true,
+            mod_time: Some(mod_time),
+            ..Default::default()
+        };
+        let target = source.clone();
+
+        assert!(is_equivalent_data_movement_delete_marker(&source, &target));
+
+        let mismatched = ObjectInfo {
+            mod_time: Some(mod_time + Duration::from_secs(1)),
+            ..target
+        };
+        assert!(!is_equivalent_data_movement_delete_marker(&source, &mismatched));
+    }
+
+    #[test]
+    fn equivalent_data_movement_delete_marker_rejects_live_object() {
+        let source = ObjectInfo {
+            delete_marker: true,
+            ..Default::default()
+        };
+        let target = ObjectInfo {
+            delete_marker: false,
+            ..source.clone()
+        };
+
+        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
+    }
+
+    #[test]
+    fn data_movement_delete_marker_resume_accepts_equivalent_target() {
+        let source = ObjectInfo {
+            version_id: Some(Uuid::nil()),
+            delete_marker: true,
+            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
+            ..Default::default()
+        };
+
+        let should_resume = resolve_data_movement_delete_marker_resume_result(Ok(Some(source.clone())), &source, 0, 1)
+            .expect("equivalent delete marker target should be evaluated");
+
+        assert!(should_resume);
+    }
+
+    #[test]
+    fn data_movement_delete_marker_resume_rejects_source_pool_target() {
+        let source = ObjectInfo {
+            version_id: Some(Uuid::nil()),
+            delete_marker: true,
+            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
+            ..Default::default()
+        };
+
+        let should_resume = resolve_data_movement_delete_marker_resume_result(Ok(Some(source.clone())), &source, 0, 0)
+            .expect("source-pool target should be rejected before target lookup");
+
+        assert!(!should_resume);
+    }
+
+    #[test]
+    fn data_movement_resume_target_prefers_selected_non_source_pool() {
+        let target_pool_idx = resolve_data_movement_resume_target_pool(2, Some(3), 1);
+        assert_eq!(target_pool_idx, 2);
+    }
+
+    #[test]
+    fn data_movement_resume_target_uses_resolved_non_source_pool_when_selected_is_source() {
+        let target_pool_idx = resolve_data_movement_resume_target_pool(1, Some(3), 1);
+        assert_eq!(target_pool_idx, 3);
+    }
+
+    #[test]
+    fn data_movement_resume_target_keeps_source_when_no_other_pool_is_available() {
+        let target_pool_idx = resolve_data_movement_resume_target_pool(1, None, 1);
+        assert_eq!(target_pool_idx, 1);
+    }
+
+    #[test]
+    fn data_movement_delete_marker_resume_propagates_target_lookup_error() {
+        let source = ObjectInfo {
+            delete_marker: true,
+            ..Default::default()
+        };
+        let result = resolve_data_movement_delete_marker_resume_result(Err(Error::SlowDown), &source, 0, 1);
+
+        assert!(matches!(result, Err(Error::SlowDown)));
+    }
+
+    #[test]
+    fn equivalent_data_movement_tiered_object_accepts_matching_transition_metadata() {
+        let version_id = Uuid::nil();
+        let transition_version_id = Uuid::new_v4();
+        let mod_time = OffsetDateTime::UNIX_EPOCH;
+        let source = FileInfo {
+            version_id: Some(version_id),
+            size: 1024,
+            mod_time: Some(mod_time),
+            checksum: Some(Bytes::from_static(b"checksum")),
+            transition_status: TRANSITION_COMPLETE.to_string(),
+            transitioned_objname: "remote/object".to_string(),
+            transition_tier: "WARM".to_string(),
+            transition_version_id: Some(transition_version_id),
+            metadata: HashMap::from([("etag".to_string(), "etag-value".to_string())]),
+            ..Default::default()
+        };
+        let target = ObjectInfo {
+            version_id: Some(version_id),
+            size: 1024,
+            mod_time: Some(mod_time),
+            checksum: Some(Bytes::from_static(b"checksum")),
+            etag: Some("etag-value".to_string()),
+            transitioned_object: TransitionedObject {
+                name: "remote/object".to_string(),
+                version_id: transition_version_id.to_string(),
+                tier: "WARM".to_string(),
+                status: TRANSITION_COMPLETE.to_string(),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+
+        assert!(is_equivalent_data_movement_tiered_object(&source, &target));
+    }
+
+    #[test]
+    fn equivalent_data_movement_tiered_object_rejects_transition_mismatch() {
+        let source = FileInfo {
+            version_id: Some(Uuid::nil()),
+            size: 1024,
+            transition_status: TRANSITION_COMPLETE.to_string(),
+            transitioned_objname: "remote/source".to_string(),
+            transition_tier: "WARM".to_string(),
+            ..Default::default()
+        };
+        let target = ObjectInfo {
+            version_id: source.version_id,
+            size: 1024,
+            transitioned_object: TransitionedObject {
+                name: "remote/target".to_string(),
+                tier: "WARM".to_string(),
+                status: TRANSITION_COMPLETE.to_string(),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+
+        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
+    }
+
+    #[test]
+    fn data_movement_tiered_resume_accepts_equivalent_target() {
+        let version_id = Uuid::nil();
+        let transition_version_id = Uuid::new_v4();
+        let source = FileInfo {
+            version_id: Some(version_id),
+            size: 1024,
+            transition_status: TRANSITION_COMPLETE.to_string(),
+            transitioned_objname: "remote/object".to_string(),
+            transition_tier: "WARM".to_string(),
+            transition_version_id: Some(transition_version_id),
+            metadata: HashMap::from([("etag".to_string(), "etag-value".to_string())]),
+            ..Default::default()
+        };
+        let target = ObjectInfo {
+            version_id: Some(version_id),
+            size: 1024,
+            etag: Some("etag-value".to_string()),
+            transitioned_object: TransitionedObject {
+                name: "remote/object".to_string(),
+                version_id: transition_version_id.to_string(),
+                tier: "WARM".to_string(),
+                status: TRANSITION_COMPLETE.to_string(),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+
+        let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 1)
+            .expect("equivalent tiered target should be evaluated");
+
+        assert!(should_resume);
+    }
+
+    #[test]
+    fn data_movement_tiered_resume_rejects_source_pool_target() {
+        let version_id = Uuid::nil();
+        let source = FileInfo {
+            version_id: Some(version_id),
+            size: 1024,
+            transition_status: TRANSITION_COMPLETE.to_string(),
+            transitioned_objname: "remote/object".to_string(),
+            transition_tier: "WARM".to_string(),
+            metadata: HashMap::from([("etag".to_string(), "etag-value".to_string())]),
+            ..Default::default()
+        };
+        let target = ObjectInfo {
+            version_id: Some(version_id),
+            size: 1024,
+            etag: Some("etag-value".to_string()),
+            transitioned_object: TransitionedObject {
+                name: "remote/object".to_string(),
+                tier: "WARM".to_string(),
+                status: TRANSITION_COMPLETE.to_string(),
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+
+        let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 0)
+            .expect("source-pool target should be rejected before target lookup");
+
+        assert!(!should_resume);
+    }
+
+    #[test]
+    fn data_movement_tiered_resume_rejects_missing_target() {
+        let source = FileInfo {
+            version_id: Some(Uuid::nil()),
+            size: 1024,
+            ..Default::default()
+        };
+
+        let should_resume = resolve_data_movement_tiered_resume_result(Ok(None), &source, 0, 1)
+            .expect("missing tiered target should be evaluated");
+
+        assert!(!should_resume);
+    }
+
    #[test]
    fn latest_object_access_delete_marker_error_returns_none_for_live_object() {
        let info = ObjectInfo::default();