fix(rebalance): converge multipart data movement retries (#6057)

* fix(rebalance): converge multipart data movement retries * fix(rebalance): harden multipart retry replacement * fix(rebalance): isolate internal multipart uploads * test(ecstore): adapt metadata mutation fixtures * fix(rebalance): preserve transition metadata semantics * refactor(ecstore): reuse internal metadata matcher * Revert "refactor(ecstore): reuse internal metadata matcher" This reverts commit c87ca0328f. * refactor(rebalance): reuse data movement log constants * fix(rebalance): isolate migration-owned state * fix(rebalance): preserve pre-gate retry compatibility
2026-08-18 18:46:17 +00:00 · 2026-08-13 14:12:26 +08:00
parent 11eecdc888
commit e11fcfbd08
36 changed files with 6725 additions and 845 deletions
@@ -3292,7 +3292,11 @@ impl SetDisks {
        // Force the full quorum fanout (allow_early_stop=false): `disks` is the
        // write target below, and an early-stop subset would only carry read
        // quorum, failing write quorum on update_object_meta (backlog#872).
-        let (fi, _, disks) = self.get_object_fileinfo_gated(bucket, object, opts, false, false).await?;
+        let mut read_opts = opts.clone();
+        read_opts.include_part_checksums = true;
+        let (fi, _, disks) = self
+            .get_object_fileinfo_gated(bucket, object, &read_opts, false, false)
+            .await?;
        let mut fi = fi.into_owned();

        fi.metadata.insert(AMZ_OBJECT_TAGGING.to_owned(), tags.to_owned());
@@ -3301,6 +3305,7 @@ impl SetDisks {
                fi.metadata.insert(key.clone(), value.clone());
            }
        }
+        fi.acknowledge_data_movement();

        #[cfg(test)]
        pause_object_tagging_commit(bucket, object).await;
@@ -3503,11 +3508,26 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
            }
        };

-        fi.metadata = (*src_info.user_defined).clone();
-
-        if let Some(etag) = &src_info.etag {
-            fi.metadata.insert("etag".to_owned(), etag.clone());
+        let preserved_part_checksums = if (src_info.metadata_only || src_info.version_only)
+            && rustfs_utils::http::contains_key_str(&fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS)
+        {
+            Self::hydrate_selected_fileinfo_part_checksums(&mut fi).map_err(|_| Error::FileCorrupt)?;
+            Some(
+                rustfs_utils::http::get_consistent_str(&fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS)
+                    .ok_or(Error::FileCorrupt)?
+                    .to_string(),
+            )
+        } else {
+            None
+        };
+        let mut replacement_metadata = (*src_info.user_defined).clone();
+        if let Some(part_checksums) = preserved_part_checksums {
+            rustfs_utils::http::insert_str(&mut replacement_metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, part_checksums);
        }
+        if let Some(etag) = &src_info.etag {
+            replacement_metadata.insert("etag".to_owned(), etag.clone());
+        }
+        fi.metadata = replacement_metadata.clone();

        let mod_time = OffsetDateTime::now_utc();
        fi.mod_time = Some(mod_time);
@@ -3538,10 +3558,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {

            for fi in metas.iter_mut() {
                if fi.has_valid_erasure_geometry() {
-                    fi.metadata = (*src_info.user_defined).clone();
-                    if let Some(etag) = &src_info.etag {
-                        fi.metadata.insert("etag".to_owned(), etag.clone());
-                    }
+                    fi.metadata.clone_from(&replacement_metadata);
                    fi.mod_time = Some(mod_time);
                    fi.version_id = version_id;
                    fi.versioned = src_opts.versioned || src_opts.version_suspended;
@@ -4482,7 +4499,10 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
            }

            fi.version_id = if let Some(vid) = opts.version_id.as_ref() {
-                Some(Uuid::parse_str(vid.as_str())?)
+                let vid = Uuid::parse_str(vid.as_str())?;
+                (!opts.version_suspended || !vid.is_nil()).then_some(vid)
+            } else if opts.version_suspended {
+                None
            } else if opts.versioned {
                Some(Uuid::new_v4())
            } else {
@@ -4507,7 +4527,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
        // Create a single object deletion request
        let mut dfi = FileInfo {
            name: object.to_string(),
-            version_id: opts.version_id.as_ref().and_then(|v| Uuid::parse_str(v).ok()),
+            version_id: opts
+                .version_id
+                .as_ref()
+                .and_then(|v| Uuid::parse_str(v).ok())
+                .filter(|vid| !opts.version_suspended || !vid.is_nil()),
            mark_deleted: mark_delete,
            deleted: delete_marker,
            mod_time: Some(mod_time),
@@ -4654,6 +4678,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
            }
        }

+        fi.acknowledge_data_movement();
+
        if opts.mod_time.is_some() {
            fi.mod_time = opts.mod_time;
        }
@@ -4720,7 +4746,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
        //     _lock_guard = guard_opt;
        // }

-        let (fi, meta_arr, online_disks) = self.get_object_fileinfo(bucket, object, opts, true, false).await?;
+        let mut transition_read_opts = opts.clone();
+        transition_read_opts.include_part_checksums = true;
+        let (fi, meta_arr, online_disks) = self
+            .get_object_fileinfo(bucket, object, &transition_read_opts, true, false)
+            .await?;
        let mut fi = fi.into_owned();
        /*if err != nil {
            return Err(to_object_err(err, vec![bucket, object]));
@@ -4778,6 +4808,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
        let transaction_id = transaction.transaction_id;
        let dest_obj = transaction.remote_object.clone();
        let mut transition_meta = (*oi.user_defined).clone();
+        rustfs_utils::http::remove_str(&mut transition_meta, rustfs_utils::http::SUFFIX_PART_CHECKSUMS);
        transition_meta.insert("name".to_string(), object.to_string());
        rustfs_utils::http::metadata_compat::insert_str(
            &mut transition_meta,
@@ -4938,6 +4969,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
        let mut commit_opts = opts.clone();
        commit_opts.no_lock = true;
        commit_opts.metadata_cache_safe = false;
+        commit_opts.include_part_checksums = true;
        let transition_lock_guard = if opts.no_lock {
            None
        } else {
@@ -5191,7 +5223,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                achieved: 0,
            });
        }
-        let fi = self.clone().get_object_fileinfo(bucket, object, opts, true, false).await;
+        let mut restore_read_opts = opts.clone();
+        restore_read_opts.include_part_checksums = true;
+        let fi = self
+            .clone()
+            .get_object_fileinfo(bucket, object, &restore_read_opts, true, false)
+            .await;
        drop(bucket_lifecycle_guard);
        if let Err(err) = fi {
            return set_restore_header_fn(&mut oi, Some(to_object_err(err, vec![bucket, object]))).await;
@@ -5211,7 +5248,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
                expected_operation_id.to_string(),
            );
        }
-        let restore_commit_metadata = if let Some(expected_operation_id) = expected_operation_id {
+        let mut restore_commit_metadata = if let Some(expected_operation_id) = expected_operation_id {
            let mut metadata = HashMap::new();
            metadata.insert(X_AMZ_RESTORE.as_str().to_string(), "ongoing-request=\"false\"".to_string());
            rustfs_utils::http::metadata_compat::insert_str(
@@ -5223,6 +5260,15 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
        } else {
            HashMap::new()
        };
+        if let Some(part_checksums) =
+            rustfs_utils::http::get_consistent_str(&actual_fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS)
+        {
+            rustfs_utils::http::insert_str(
+                &mut restore_commit_metadata,
+                rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
+                part_checksums.to_string(),
+            );
+        }
        // The restore copy-back re-writes this same object via put_object /
        // new_multipart_upload / complete_multipart_upload, each of which takes
        // the object write lock in its commit phase. The caller
@@ -6103,7 +6149,7 @@ mod get_object_downstream_close_accounting_tests {
 mod metadata_mutation_generation_tests {
    use super::hermetic_set_disks_support::hermetic_set_disks_isolated as hermetic_set_disks;
    use super::*;
-    use crate::disk::DiskAPI as _;
+    use crate::disk::{DiskAPI as _, ReadOptions};
    use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};

    async fn put_and_prime(
@@ -6140,6 +6186,31 @@ mod metadata_mutation_generation_tests {
        );
    }

+    async fn persist_part_checksum_sidecar(set_disks: &Arc<SetDisks>, bucket: &str, object: &str, value: &str) {
+        let (fi, _, disks) = set_disks
+            .get_object_fileinfo(
+                bucket,
+                object,
+                &ObjectOptions {
+                    no_lock: true,
+                    metadata_cache_safe: false,
+                    ..Default::default()
+                },
+                true,
+                false,
+            )
+            .await
+            .expect("object metadata should be readable before adding the checksum sidecar");
+        let mut fi = fi.into_owned();
+        let disks = disks.into_owned();
+        rustfs_utils::http::insert_str(&mut fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS, value.to_string());
+        set_disks
+            .update_object_meta(bucket, object, fi, &disks)
+            .await
+            .expect("checksum sidecar should be persisted");
+        set_disks.invalidate_get_object_metadata_cache(bucket, object).await;
+    }
+
    #[tokio::test]
    #[serial_test::serial(metadata_cache_invalidation_probe)]
    async fn metadata_semantic_mutation_generation_matrix_retires_cached_snapshot() {
@@ -6226,6 +6297,112 @@ mod metadata_mutation_generation_tests {
        );
        assert_retired(&set_disks, &metadata_key).await;
    }
+
+    #[tokio::test]
+    async fn metadata_only_copy_preserves_valid_part_checksums_and_rejects_conflicting_aliases() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "metadata-copy-part-checksums-bucket";
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+
+        let valid_object = "valid-sidecar";
+        let (mut valid_source, _) = put_and_prime(&set_disks, bucket, valid_object, b"valid sidecar body").await;
+        persist_part_checksum_sidecar(&set_disks, bucket, valid_object, r#"[[1,[["CRC32C","AAAAAA=="]]]]"#).await;
+        valid_source.metadata_only = true;
+        Arc::make_mut(&mut valid_source.user_defined).insert("x-amz-meta-copy".to_string(), "updated".to_string());
+        set_disks
+            .copy_object(
+                bucket,
+                valid_object,
+                bucket,
+                valid_object,
+                &mut valid_source,
+                &ObjectOptions::default(),
+                &ObjectOptions::default(),
+            )
+            .await
+            .expect("metadata-only copy should preserve a valid checksum sidecar");
+        let copied = set_disks
+            .get_object_info(
+                bucket,
+                valid_object,
+                &ObjectOptions {
+                    include_part_checksums: true,
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("copied object should retain readable part checksums");
+        assert_eq!(
+            copied.parts[0]
+                .checksums
+                .as_ref()
+                .and_then(|checksums| checksums.get("CRC32C"))
+                .map(String::as_str),
+            Some("AAAAAA==")
+        );
+
+        let conflicting_object = "conflicting-sidecar";
+        let (mut conflicting_source, _) =
+            put_and_prime(&set_disks, bucket, conflicting_object, b"conflicting sidecar body").await;
+        let (fi, _, disks) = set_disks
+            .get_object_fileinfo(
+                bucket,
+                conflicting_object,
+                &ObjectOptions {
+                    no_lock: true,
+                    metadata_cache_safe: false,
+                    ..Default::default()
+                },
+                true,
+                false,
+            )
+            .await
+            .expect("conflicting object metadata should be readable before corruption is injected");
+        let mut fi = fi.into_owned();
+        let disks = disks.into_owned();
+        let rustfs_key = format!(
+            "{}{}",
+            rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
+            rustfs_utils::http::SUFFIX_PART_CHECKSUMS
+        );
+        let minio_key = format!(
+            "{}{}",
+            rustfs_utils::http::MINIO_INTERNAL_PREFIX,
+            rustfs_utils::http::SUFFIX_PART_CHECKSUMS
+        );
+        fi.metadata
+            .insert(rustfs_key.clone(), r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string());
+        fi.metadata
+            .insert(minio_key.clone(), r#"[[1,[["CRC32C","AQAAAA=="]]]]"#.to_string());
+        set_disks
+            .update_object_meta(bucket, conflicting_object, fi, &disks)
+            .await
+            .expect("conflicting aliases should be persisted for the fail-closed regression");
+        set_disks
+            .invalidate_get_object_metadata_cache(bucket, conflicting_object)
+            .await;
+        conflicting_source.metadata_only = true;
+        let err = set_disks
+            .copy_object(
+                bucket,
+                conflicting_object,
+                bucket,
+                conflicting_object,
+                &mut conflicting_source,
+                &ObjectOptions::default(),
+                &ObjectOptions::default(),
+            )
+            .await
+            .expect_err("metadata-only copy must reject conflicting checksum aliases");
+        assert!(matches!(err, Error::FileCorrupt));
+        let raw_err = disk_stores[0]
+            .read_version("", bucket, conflicting_object, "", &ReadOptions::default())
+            .await
+            .expect_err("the rejected copy must leave the conflicting persisted aliases fail-closed");
+        assert!(matches!(raw_err, crate::disk::error::DiskError::FileCorrupt));
+    }
 }

 #[cfg(all(test, feature = "test-util"))]
@@ -6287,6 +6464,7 @@ mod transition_commit_failure_tests {
        }
    }

+    #[tokio::test]
    #[serial_test::serial(restore_multipart_failure_point)]
    async fn multipart_restore_aborts_every_post_create_failure() {
        let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
@@ -6333,6 +6511,32 @@ mod transition_commit_failure_tests {
            )
            .await
            .expect("source multipart upload should complete");
+        let (source_fi, _, online_disks) = set_disks
+            .get_object_fileinfo(
+                bucket,
+                object,
+                &ObjectOptions {
+                    no_lock: true,
+                    metadata_cache_safe: false,
+                    ..Default::default()
+                },
+                true,
+                false,
+            )
+            .await
+            .expect("source metadata should be readable before adding the checksum sidecar");
+        let mut source_fi = source_fi.into_owned();
+        let online_disks = online_disks.into_owned();
+        rustfs_utils::http::insert_str(
+            &mut source_fi.metadata,
+            rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
+            r#"[[1,[["CRC32C","AAAAAA=="]]],[2,[["CRC32C","AQAAAA=="]]]]"#.to_string(),
+        );
+        set_disks
+            .update_object_meta(bucket, object, source_fi, &online_disks)
+            .await
+            .expect("source checksum sidecar should be persisted before transition");
+        set_disks.invalidate_get_object_metadata_cache(bucket, object).await;
        let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
        register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
        set_disks
@@ -6416,9 +6620,32 @@ mod transition_commit_failure_tests {
            "successful multipart completion must disarm cleanup without aborting"
        );
        let restored = set_disks
-            .get_object_info(bucket, object, &ObjectOptions::default())
+            .get_object_info(
+                bucket,
+                object,
+                &ObjectOptions {
+                    include_part_checksums: true,
+                    ..Default::default()
+                },
+            )
            .await
            .expect("successful multipart restore must leave the committed object intact");
+        assert_eq!(
+            restored.parts[0]
+                .checksums
+                .as_ref()
+                .and_then(|checksums| checksums.get("CRC32C"))
+                .map(String::as_str),
+            Some("AAAAAA==")
+        );
+        assert_eq!(
+            restored.parts[1]
+                .checksums
+                .as_ref()
+                .and_then(|checksums| checksums.get("CRC32C"))
+                .map(String::as_str),
+            Some("AQAAAA==")
+        );
        let restore_header = restored
            .user_defined
            .get(s3s::header::X_AMZ_RESTORE.as_str())
@@ -7900,6 +8127,88 @@ mod transition_upload_integrity_tests {
            .expect("source object should be written")
    }

+    #[tokio::test]
+    #[serial_test::serial]
+    async fn data_movement_tiered_metadata_is_create_only_under_object_lock() {
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
+        let bucket = "tiered-data-movement-create-only";
+        let object = "object.bin";
+        let version_id = Uuid::new_v4();
+        for disk in &disk_stores {
+            disk.make_volume(bucket).await.expect("bucket volume should be created");
+        }
+        let mut reader = PutObjReader::from_vec(b"existing target".to_vec());
+        set_disks
+            .put_object(
+                bucket,
+                object,
+                &mut reader,
+                &ObjectOptions {
+                    versioned: true,
+                    version_id: Some(version_id.to_string()),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect("existing target should be written");
+        let conflicting = FileInfo {
+            volume: bucket.to_string(),
+            name: object.to_string(),
+            version_id: Some(version_id),
+            mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND),
+            size: 12,
+            parts: vec![ObjectPartInfo {
+                number: 1,
+                size: 12,
+                actual_size: 12,
+                etag: "part-etag".to_string(),
+                ..Default::default()
+            }],
+            transition_status: TRANSITION_COMPLETE.to_string(),
+            transition_tier: "WARM".to_string(),
+            transitioned_objname: "remote/object-a".to_string(),
+            transition_version: Some("remote-version-a".to_string()),
+            transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
+            fresh: true,
+            ..Default::default()
+        };
+        let err = set_disks
+            .decommission_tiered_object(
+                bucket,
+                object,
+                &conflicting,
+                &ObjectOptions {
+                    versioned: true,
+                    version_id: Some(version_id.to_string()),
+                    mod_time: conflicting.mod_time,
+                    data_movement: true,
+                    http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
+                    ..Default::default()
+                },
+            )
+            .await
+            .expect_err("data movement must not overwrite an existing tiered version");
+        assert!(matches!(err, StorageError::PreconditionFailed));
+
+        let (stored, _, _) = set_disks
+            .get_object_fileinfo(
+                bucket,
+                object,
+                &ObjectOptions {
+                    version_id: Some(version_id.to_string()),
+                    no_lock: true,
+                    metadata_cache_safe: false,
+                    ..Default::default()
+                },
+                true,
+                false,
+            )
+            .await
+            .expect("the existing target should remain readable");
+        assert_ne!(stored.transition_status, TRANSITION_COMPLETE);
+        assert!(stored.transition_version.is_none());
+    }
+
    fn transition_options(original: &ObjectInfo, tier_name: String) -> ObjectOptions {
        ObjectOptions {
            no_lock: true,
@@ -7954,6 +8263,31 @@ mod transition_upload_integrity_tests {
        let object = "object.bin";
        let payload = b"transition remote object must be bound to its transaction id".repeat(1024);
        let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
+        let (source_fi, _, online_disks) = set_disks
+            .get_object_fileinfo(
+                bucket,
+                object,
+                &ObjectOptions {
+                    no_lock: true,
+                    metadata_cache_safe: false,
+                    ..Default::default()
+                },
+                true,
+                false,
+            )
+            .await
+            .expect("source metadata should be readable");
+        let mut source_fi = source_fi.into_owned();
+        let online_disks = online_disks.into_owned();
+        rustfs_utils::http::insert_str(
+            &mut source_fi.metadata,
+            rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
+            r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(),
+        );
+        set_disks
+            .update_object_meta(bucket, object, source_fi, &online_disks)
+            .await
+            .expect("source checksum sidecar should be persisted");
        let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
        let remote_version = Uuid::new_v4().to_string();
        let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
@@ -7978,6 +8312,7 @@ mod transition_upload_integrity_tests {
                &ObjectOptions {
                    no_lock: true,
                    metadata_cache_safe: false,
+                    include_part_checksums: true,
                    ..Default::default()
                },
                true,
@@ -7991,6 +8326,22 @@ mod transition_upload_integrity_tests {
            fi.transition_version_id,
            Some(Uuid::parse_str(&remote_version).expect("test version id should parse"))
        );
+        assert_eq!(
+            fi.parts[0]
+                .checksums
+                .as_ref()
+                .and_then(|checksums| checksums.get("CRC32C"))
+                .map(String::as_str),
+            Some("AAAAAA==")
+        );
+        let remote_metadata = backend
+            .metadata(remote_object)
+            .await
+            .expect("remote metadata should be stored");
+        assert!(
+            !rustfs_utils::http::contains_key_str(&remote_metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS),
+            "the internal checksum sidecar must not be uploaded as remote user metadata"
+        );
        assert!(backend.contains(remote_object).await, "committed remote object should remain available");
    }

@@ -8067,6 +8418,7 @@ mod transition_upload_integrity_tests {
                object,
                &expected,
                &[],
+                crate::data_movement::SourceCleanupBucketFence::default(),
                "test_data_movement",
            )
            .await
@@ -8091,6 +8443,72 @@ mod transition_upload_integrity_tests {
        assert_local_source_intact(&set_disks, bucket, object, &payload).await;
    }

+    #[tokio::test(flavor = "current_thread", start_paused = true)]
+    #[serial_test::serial]
+    async fn data_movement_cleanup_aborts_after_bucket_fence_loss() {
+        let refresh_calls = Arc::new(AtomicUsize::new(0));
+        let lockers: Vec<Arc<dyn LockClient>> = (0..4)
+            .map(|_| Arc::new(LockLostRefreshClient::new(Arc::clone(&refresh_calls))) as Arc<dyn LockClient>)
+            .collect();
+        let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
+        let bucket = "data-movement-cleanup-bucket-fence-lost";
+        let object = "object.bin";
+        let payload = b"lost bucket fence must preserve the source".repeat(1024);
+        write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
+        let expected = set_disks
+            .load_file_info_versions_exact(bucket, object)
+            .await
+            .expect("source versions should be readable")
+            .expect("source versions should exist");
+
+        let distributed_setup = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
+        let bucket_guard = set_disks
+            .new_ns_lock(bucket, crate::storage_api_contracts::bucket::BUCKET_LIFECYCLE_LOCK_OBJECT)
+            .await
+            .expect("create bucket lifecycle lock")
+            .get_read_lock(get_lock_acquire_timeout())
+            .await
+            .expect("acquire bucket lifecycle read lock");
+        let local_cleanup_setup = SetupTypeGuard::switch_to(SetupType::Erasure).await;
+        let barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(bucket, object);
+
+        let cleanup_set = Arc::clone(&set_disks);
+        let cleanup = tokio::spawn(async move {
+            let result = crate::data_movement::cleanup_source_entry_if_unchanged(
+                cleanup_set,
+                bucket,
+                object,
+                &expected,
+                &[],
+                crate::data_movement::SourceCleanupBucketFence {
+                    expected_incarnation_id: None,
+                    lifecycle_guard: Some(&bucket_guard),
+                },
+                "test_data_movement",
+            )
+            .await;
+            drop(bucket_guard);
+            result
+        });
+        barrier.wait_until_paused().await;
+        tokio::time::advance(Duration::from_secs(11)).await;
+        tokio::task::yield_now().await;
+        assert!(refresh_calls.load(Ordering::SeqCst) > 0, "bucket guard refresh must run before commit");
+        barrier.release();
+
+        let error = cleanup
+            .await
+            .expect("cleanup task should not panic")
+            .expect_err("cleanup must fail after its bucket lifecycle fence loses refresh quorum");
+        assert!(matches!(
+            error,
+            crate::data_movement::SourceCleanupError::Storage(StorageError::NamespaceLockQuorumUnavailable { .. })
+        ));
+        assert_local_source_intact(&set_disks, bucket, object, &payload).await;
+        drop(local_cleanup_setup);
+        drop(distributed_setup);
+    }
+
    #[tokio::test]
    #[serial_test::serial]
    async fn partial_remote_acceptance_cleans_exact_candidate_and_preserves_source() {
@@ -9925,6 +10343,33 @@ mod put_object_tags_early_stop_regression_tests {
                    .await
                    .expect("put_object should succeed");

+                let (fi, _, disks) = set_disks
+                    .get_object_fileinfo(
+                        bucket,
+                        object,
+                        &ObjectOptions {
+                            no_lock: true,
+                            metadata_cache_safe: false,
+                            ..Default::default()
+                        },
+                        true,
+                        false,
+                    )
+                    .await
+                    .expect("object metadata should be readable before adding the checksum sidecar");
+                let mut fi = fi.into_owned();
+                let disks = disks.into_owned();
+                rustfs_utils::http::insert_str(
+                    &mut fi.metadata,
+                    rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
+                    r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(),
+                );
+                set_disks
+                    .update_object_meta(bucket, object, fi, &disks)
+                    .await
+                    .expect("checksum sidecar should be persisted before tagging");
+                set_disks.invalidate_get_object_metadata_cache(bucket, object).await;
+
                let tags = "unit=backlog881&stage=regression";
                set_disks
                    .put_object_tags(bucket, object, tags, &ObjectOptions::default())
@@ -9944,6 +10389,11 @@ mod put_object_tags_early_stop_regression_tests {
                        Some(tags),
                        "disk {idx} must carry the tag written under early-stop (write set not shrunk to a read-quorum subset)"
                    );
+                    assert_eq!(
+                        rustfs_utils::http::get_consistent_str(&fi.metadata, rustfs_utils::http::SUFFIX_PART_CHECKSUMS),
+                        Some(r#"[[1,[["CRC32C","AAAAAA=="]]]]"#),
+                        "disk {idx} must retain the checksum sidecar across the tag metadata update"
+                    );
                }
            },
        )