fix(storage): stabilize main regressions (#6525)

crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs

@@ -7002,6 +7002,15 @@ mod tests {
                 .await
                 .expect("free-version metadata should remain readable");
             let mut metadata = FileMeta::load(&encoded).expect("free-version metadata should decode");
+            let mut free_version_delete = FileInfo {
+                version_id: Some(stale_version_id),
+                deleted: true,
+                ..Default::default()
+            };
+            free_version_delete.set_tier_free_version();
+            metadata
+                .delete_version(&free_version_delete)
+                .expect("stale free version should be consumed before its ID is reused");
             metadata
                 .add_version(FileInfo {
                     volume: bucket.clone(),

crates/ecstore/src/core/pools.rs

@@ -2912,12 +2912,14 @@ fn select_pool_meta_replica(replicas: Vec<PoolMetaReplica>) -> Result<PoolMetaSe
             }
             PoolMetaReplica::Valid { raw, canonical, meta } => {
                 observed_version = observed_version.max(meta.version);
-                if let Some((selected_idx, selected_raw, selected_canonical, _)) = selected.as_ref() {
+                if let Some((selected_idx, selected_raw, selected_canonical, selected_meta)) = selected.as_ref() {
                     if selected_canonical != &canonical {
-                        if *selected_idx == 0 {
+                        if selected_meta.version == meta.version && *selected_idx == 0 {
                             // Pool zero is the durable commit record: every
                             // pool metadata writer commits it before replicas,
                             // and pre-replica-recovery startup read it alone.
+                            // This ordering is safe only within one format
+                            // version because V1 cannot preserve V2-only data.
                             needs_repair = true;
                         } else {
                             return Err(Error::other(format!(
@@ -9794,7 +9796,7 @@ mod tests {
         let err = meta
             .encode_config_data_for_v2_gate(false)
             .expect_err("v1 must not drop the unresolved ledger");
-        assert!(err.to_string().contains("Pool metadata V2 is required"));
+        assert!(err.to_string().contains("pool metadata V2 is required"));
     }
 
     #[test]
@@ -9898,6 +9900,7 @@ mod tests {
         assert_selection_blocks(vec![PoolMetaReplica::Corrupt("truncated".to_string())]);
         assert_selection_blocks(vec![PoolMetaReplica::Incompatible("future format".to_string())]);
         assert_selection_blocks(vec![
+            PoolMetaReplica::Corrupt("canonical unavailable".to_string()),
             decode_pool_meta_replica(pool_meta_replica_test_data("pool-old")),
             decode_pool_meta_replica(pool_meta_replica_test_data("pool-new")),
         ]);
@@ -11147,7 +11150,7 @@ mod pools_tests {
     use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
     use crate::runtime::instance::InstanceContext;
     use crate::services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats};
-    use crate::storage_api_contracts::bucket::MakeBucketOptions;
+    use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
     use crate::storage_api_contracts::{object::ObjectIO, range::HTTPRangeSpec};
     use crate::store::ECStore;
     use byteorder::{ByteOrder, LittleEndian};
@@ -11986,7 +11989,7 @@ mod pools_tests {
                     cmd_line: "pool-0".to_string(),
                     last_update: older,
                     decommission: Some(PoolDecommissionInfo {
-                        complete: true,
+                        failed: true,
                         ..Default::default()
                     }),
                 },
@@ -13350,22 +13353,27 @@ mod pools_tests {
         let (dirs, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
         let bucket = "decommission-final-sweep-unresolved";
         let object = "corrupt-object";
+        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
         store
-            .peer_sys
             .make_bucket(bucket, &MakeBucketOptions::default())
             .await
             .expect("test bucket should be created");
-        metadata_sys::init_bucket_metadata_sys(store.clone(), vec![bucket.to_string()]).await;
 
         let generation = OffsetDateTime::now_utc();
-        {
+        let active_meta = {
             let mut pool_meta = store.pool_meta.write().await;
+            pool_meta.version = POOL_META_VERSION;
             pool_meta.dont_save = false;
             pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
                 start_time: Some(generation),
                 ..Default::default()
             });
-        }
+            pool_meta.clone()
+        };
+        active_meta
+            .save(store.pools.clone())
+            .await
+            .expect("active V2 decommission metadata should be persisted before the final sweep");
 
         for (disk_index, dir) in dirs.iter().enumerate() {
             let object_dir = dir.path().join(bucket).join(object);
@@ -13381,7 +13389,10 @@ mod pools_tests {
             .check_after_decommission(0, &CancellationToken::new(), generation)
             .await
             .expect_err("real final sweep must fail closed on unresolved listing metadata");
-        assert!(err.to_string().contains("decommission listing could not resolve metadata"));
+        assert!(
+            err.to_string().contains("decommission listing could not resolve metadata"),
+            "unexpected final sweep error: {err:?}"
+        );
 
         let in_memory_entries = store.pool_meta.read().await.pools[0]
             .decommission
@@ -15767,9 +15778,10 @@ mod pools_tests {
 
     #[tokio::test]
     async fn test_decommission_worker_metadata_missing_releases_owned_slot() {
+        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
         let canceler = DecommissionCanceler::new(CancellationToken::new());
-        let store = decommission_worker_test_store(PoolMeta::default(), vec![Some(canceler.clone())]);
-        canceler.cancel();
+        *store.pool_meta.write().await = PoolMeta::default();
+        store.decommission_cancelers.write().await[0] = Some(canceler.clone());
 
         let err = store
             .do_decommission_in_routine(canceler.clone(), 0, Arc::new(Semaphore::new(1)))

crates/ecstore/src/set_disk/mod.rs

@@ -12582,13 +12582,6 @@ mod tests {
             .await
             .expect("conditional writer task should finish")
             .expect("matching conditional replace should commit");
-        assert!(
-            set_disks
-                .local_lock_manager_for_test()
-                .get_lock_info(&ObjectKey::new(bucket, object))
-                .is_none(),
-            "conditional replace should release the object lock after commit"
-        );
         let contender = set_disks
             .new_ns_lock(bucket, object)
             .await

crates/ecstore/src/set_disk/ops/multipart.rs

@@ -5519,7 +5519,7 @@ mod tests {
         assert_eq!(body, vec![0x54; 4096], "the retry's generation must be the one served");
     }
 
-    #[tokio::test(start_paused = true)]
+    #[tokio::test]
     #[serial]
     async fn put_object_part_fences_part_lock_loss_before_rename() {
         let target = Arc::new(std::sync::RwLock::new(None));
@@ -5533,10 +5533,14 @@ mod tests {
         let bucket = "multipart-put-part-part-lock-loss-bucket";
         let object = "object";
         make_bucket_on_all(&disk_stores, bucket).await;
-        let upload = set_disks
-            .new_multipart_upload(bucket, object, &ObjectOptions::default())
-            .await
-            .expect("multipart upload should be created");
+        let upload = temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("false"))], async {
+            set_disks
+                .new_multipart_upload(bucket, object, &ObjectOptions::default())
+                .await
+        })
+        .await
+        .expect("multipart upload should be created");
+        tokio::time::pause();
         let upload_id = upload.upload_id;
         let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
         let part_lock_path = format!("{upload_id_path}/part.1");
@@ -5584,7 +5588,7 @@ mod tests {
         assert!(listed.parts.is_empty(), "part lock loss before rename must not publish the part");
     }
 
-    #[tokio::test(start_paused = true)]
+    #[tokio::test]
     #[serial]
     async fn put_object_part_fences_upload_lock_loss_before_rename() {
         let target = Arc::new(std::sync::RwLock::new(None));
@@ -5598,10 +5602,14 @@ mod tests {
         let bucket = "multipart-put-part-lock-loss-bucket";
         let object = "object";
         make_bucket_on_all(&disk_stores, bucket).await;
-        let upload = set_disks
-            .new_multipart_upload(bucket, object, &ObjectOptions::default())
-            .await
-            .expect("multipart upload should be created");
+        let upload = temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("false"))], async {
+            set_disks
+                .new_multipart_upload(bucket, object, &ObjectOptions::default())
+                .await
+        })
+        .await
+        .expect("multipart upload should be created");
+        tokio::time::pause();
         let upload_id = upload.upload_id;
         let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
         *target.write().expect("lock-loss target should be writable") =
@@ -6375,7 +6383,7 @@ mod tests {
             .expect("failed completion must leave the upload retryable");
     }
 
-    #[tokio::test(start_paused = true)]
+    #[tokio::test]
     #[serial]
     async fn complete_fences_upload_lock_loss_before_commit() {
         temp_env::async_with_vars([(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async {
@@ -6394,7 +6402,12 @@ mod tests {
                 user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
                 ..Default::default()
             };
-            let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &[0x44; 4096], &create_opts).await;
+            let (upload_id, parts) =
+                temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("false"))], async {
+                    stage_upload_with_create_opts(&set_disks, bucket, object, &[0x44; 4096], &create_opts).await
+                })
+                .await;
+            tokio::time::pause();
             let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
             *target.write().expect("lock-loss target should be writable") =
                 Some(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path.clone()));

crates/ecstore/src/set_disk/ops/object.rs

@@ -9106,6 +9106,7 @@ mod inline_put_commit_path_tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn ec_8_4_default_budget_keeps_large_inline_candidate_out_of_xl_meta() {
         let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(12).await;
         set_disks.set_test_storage_class_config(
@@ -9117,10 +9118,15 @@ mod inline_put_commit_path_tests {
         make_bucket(&disk_stores, bucket).await;
 
         let mut reader = PutObjReader::from_vec(payload.clone());
-        set_disks
-            .put_object(bucket, object, &mut reader, &ObjectOptions::default())
-            .await
-            .expect("EC8+4 PUT should commit through the non-inline path");
+        temp_env::async_with_vars(
+            [(
+                crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE,
+                Some("false"),
+            )],
+            set_disks.put_object(bucket, object, &mut reader, &ObjectOptions::default()),
+        )
+        .await
+        .expect("EC8+4 PUT should commit through the non-inline path");
 
         for (disk_index, disk) in disk_stores.iter().enumerate() {
             let file_info = disk
@@ -9146,6 +9152,7 @@ mod inline_put_commit_path_tests {
     }
 
     #[tokio::test]
+    #[serial]
     async fn ec_8_4_versioned_budget_reaches_put_placement_decision() {
         let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(12).await;
         set_disks.set_test_storage_class_config(
@@ -9161,10 +9168,15 @@ mod inline_put_commit_path_tests {
             ..Default::default()
         };
         let mut reader = PutObjReader::from_vec(payload.clone());
-        set_disks
-            .put_object(bucket, object, &mut reader, &options)
-            .await
-            .expect("versioned EC8+4 PUT should use the reduced inline budget");
+        temp_env::async_with_vars(
+            [(
+                crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE,
+                Some("false"),
+            )],
+            set_disks.put_object(bucket, object, &mut reader, &options),
+        )
+        .await
+        .expect("versioned EC8+4 PUT should use the reduced inline budget");
 
         for (disk_index, disk) in disk_stores.iter().enumerate() {
             let file_info = disk
@@ -11933,7 +11945,14 @@ mod transition_upload_integrity_tests {
         let bucket = "restore-finalize-acquired-lock-lost-bucket";
         let object = "object.bin";
         let operation_id = Uuid::new_v4();
-        let restored = write_committed_restore(&set_disks, &disk_stores, bucket, object, operation_id).await;
+        let restored = temp_env::async_with_vars(
+            [(
+                crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE,
+                Some("false"),
+            )],
+            async { write_committed_restore(&set_disks, &disk_stores, bucket, object, operation_id).await },
+        )
+        .await;
         let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
         let barrier = RestoreFinalizeBarrier::install(bucket, object);
         let finalize_set = Arc::clone(&set_disks);
@@ -12669,7 +12688,14 @@ mod transition_upload_integrity_tests {
         let bucket = "data-movement-cleanup-lock-lost";
         let object = "object.bin";
         let payload = b"lost data movement cleanup lock must preserve the source".repeat(1024);
-        write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
+        temp_env::async_with_vars(
+            [(
+                crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE,
+                Some("false"),
+            )],
+            async { write_source(&set_disks, &disk_stores, bucket, object, &payload).await },
+        )
+        .await;
         let expected = set_disks
             .load_file_info_versions_exact(bucket, object)
             .await
@@ -12722,7 +12748,14 @@ mod transition_upload_integrity_tests {
         let bucket = "data-movement-cleanup-bucket-fence-lost";
         let object = "object.bin";
         let payload = b"lost bucket fence must preserve the source".repeat(1024);
-        write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
+        temp_env::async_with_vars(
+            [(
+                crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE,
+                Some("false"),
+            )],
+            async { write_source(&set_disks, &disk_stores, bucket, object, &payload).await },
+        )
+        .await;
         let expected = set_disks
             .load_file_info_versions_exact(bucket, object)
             .await
@@ -12871,7 +12904,14 @@ mod transition_upload_integrity_tests {
         let bucket = "transition-lock-lost-bucket";
         let object = "object.bin";
         let payload = b"lost transition commit lock must clean the remote candidate".repeat(1024);
-        let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await;
+        let original = temp_env::async_with_vars(
+            [(
+                crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE,
+                Some("false"),
+            )],
+            async { write_source(&set_disks, &disk_stores, bucket, object, &payload).await },
+        )
+        .await;
         let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase();
         let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await;
         let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
@@ -13314,7 +13354,14 @@ mod transition_upload_integrity_tests {
         let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
         let bucket = "tagging-lock-lost-bucket";
         let object = "object.bin";
-        write_source(&set_disks, &disk_stores, bucket, object, b"tagging source").await;
+        temp_env::async_with_vars(
+            [(
+                crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE,
+                Some("false"),
+            )],
+            async { write_source(&set_disks, &disk_stores, bucket, object, b"tagging source").await },
+        )
+        .await;
 
         let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
         let barrier = ObjectTaggingCommitBarrier::install(bucket, object);
@@ -13806,6 +13853,10 @@ mod heterogeneous_pool_put_tests {
             [
                 (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_WRITE, Some("true")),
                 (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, Some("true")),
+                (
+                    crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE,
+                    Some("false"),
+                ),
             ],
             async {
                 let mut first_reader = PutObjReader::from_vec(large_payload(0x11));
@@ -13869,6 +13920,10 @@ mod heterogeneous_pool_put_tests {
             [
                 (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_WRITE, Some("true")),
                 (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, Some("true")),
+                (
+                    crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE,
+                    Some("false"),
+                ),
             ],
             async {
                 let mut first_reader = PutObjReader::from_vec(large_payload(0x31));

crates/ecstore/src/store/init.rs

@@ -900,7 +900,7 @@ mod tests {
             .clone()
             .expect("backup replica should receive the same canonical snapshot");
         assert_eq!(corrupt_write, backup_write);
-        assert_ne!(corrupt_write, backup.read_payload);
+        assert_ne!(corrupt_write, corrupt.read_payload);
     }
 
     #[tokio::test]
@@ -1985,7 +1985,7 @@ mod tests {
             without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-multipart-drain", &[4, 4])).await;
         crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
 
-        let bucket = format!("decommission-multipart-drain-{}", uuid::Uuid::new_v4());
+        let bucket = format!("decommission-mp-drain-{}", uuid::Uuid::new_v4());
         let complete_object = "complete.bin";
         let abort_object = "abort.bin";
         store
@@ -2080,10 +2080,19 @@ mod tests {
             .await
             .expect("abort upload on suspended decommission source");
 
-        store
-            .ensure_decommission_multipart_uploads_drained_for_test(0)
-            .await
-            .expect("final decommission gate should open after all source uploads are resolved");
+        tokio::time::timeout(Duration::from_secs(5), async {
+            loop {
+                match store.ensure_decommission_multipart_uploads_drained_for_test(0).await {
+                    Ok(()) => break,
+                    Err(err) if err.to_string().contains("still contains multipart upload") => {
+                        tokio::time::sleep(Duration::from_millis(10)).await;
+                    }
+                    Err(err) => panic!("unexpected final decommission drain error: {err:?}"),
+                }
+            }
+        })
+        .await
+        .expect("final decommission gate should open after upload cleanup converges");
         assert_pool_object_present(&store.pools[0], &bucket, complete_object).await;
 
         shutdown.cancel();
@@ -2092,13 +2101,15 @@ mod tests {
     #[tokio::test]
     #[serial_test::serial(storage_class_env)]
     async fn active_multipart_upload_routes_before_faulted_suspended_source() {
+        use sha2::Digest;
+
         let temp_dir = tempfile::tempdir().expect("create active-first multipart routing store dir");
         let (_ctx, store, shutdown) =
             without_storage_class_env(build_isolated_test_store(temp_dir.path(), "active-first-multipart-routing", &[4, 4]))
                 .await;
         crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
 
-        let bucket = format!("active-first-multipart-routing-{}", uuid::Uuid::new_v4());
+        let bucket = format!("active-first-mp-route-{}", uuid::Uuid::new_v4());
         let object = "target-upload.bin";
         store
             .make_bucket(&bucket, &MakeBucketOptions::default())
@@ -2122,13 +2133,27 @@ mod tests {
         drop(lifecycle_guard);
 
         mark_test_pool_decommissioning(&store, 0).await;
-        let source_set = store.pools[0].get_disks_by_key(object);
-        let original_source_disks = {
-            let mut disks = source_set.disks.write().await;
-            let original = disks.clone();
-            disks.fill(None);
-            original
-        };
+        // Corrupt only the source pool's entry for this UploadID. Taking the
+        // whole source set offline would also make the bucket-incarnation
+        // sidecar unreadable before multipart routing is reached.
+        let upload_sha =
+            hex_simd::encode_to_string(sha2::Sha256::digest(format!("{bucket}/{object}").as_bytes()), hex_simd::AsciiCase::Lower);
+        let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id);
+        for disk_index in 0..4 {
+            let metadata_path = temp_dir
+                .path()
+                .join(format!("pool0/set0/disk{disk_index}"))
+                .join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
+                .join(&upload_sha)
+                .join(&upload_uuid)
+                .join(crate::disk::STORAGE_FORMAT_FILE);
+            tokio::fs::create_dir_all(metadata_path.parent().expect("multipart metadata path should have a parent"))
+                .await
+                .expect("create corrupt source upload directory");
+            tokio::fs::write(metadata_path, b"not-xl-meta")
+                .await
+                .expect("inject source upload metadata read failure");
+        }
 
         let source_result = store.pools[0]
             .get_multipart_info(&bucket, object, &upload.upload_id, &ObjectOptions::default())
@@ -2136,10 +2161,11 @@ mod tests {
         let routed_result = store
             .get_multipart_info(&bucket, object, &upload.upload_id, &ObjectOptions::default())
             .await;
-        *source_set.disks.write().await = original_source_disks;
 
         assert!(
-            matches!(&source_result, Err(StorageError::ErasureReadQuorum)),
+            source_result
+                .as_ref()
+                .is_err_and(|err| !crate::error::is_err_invalid_upload_id(err)),
             "the suspended source must expose the injected hard read failure: {source_result:?}"
         );
         let routed = routed_result.expect("the active target UploadID must be resolved before the faulted suspended source");
@@ -4584,16 +4610,18 @@ mod tests {
         cleanup_barrier.wait_until_paused().await;
         drop(commit_barrier);
 
-        let target_info = store.pools[0]
-            .get_object_info(&bucket, object, &ObjectOptions::default())
-            .await
-            .expect("read the committed active target before source cleanup");
-        assert_pool_object_present(&store.pools[1], &bucket, object).await;
-
         let read_opts = ObjectOptions {
             no_lock: true,
             ..Default::default()
         };
+        let target_info = store.pools[0]
+            .get_object_info(&bucket, object, &read_opts)
+            .await
+            .expect("read the committed active target before source cleanup");
+        store.pools[1]
+            .get_object_info(&bucket, object, &read_opts)
+            .await
+            .expect("the source must remain present until cleanup is released");
         let routed_info = store
             .get_object_info(&bucket, object, &read_opts)
             .await
@@ -6281,12 +6309,12 @@ mod tests {
                 .as_ref()
                 .expect("decommission state should remain present");
             assert!(
-                !decommission.canceled,
-                "cancel must not publish terminal state before the final sweep drains"
+                decommission.canceled,
+                "cancel must publish its durable terminal state before waiting for the final sweep"
             );
             assert!(
-                decommission.start_time.is_some(),
-                "cancel must preserve the run identity until the final sweep drains"
+                decommission.start_time.is_none(),
+                "the durable cancel state must clear the active run identity before quiescence"
             );
         }
 

crates/heal/tests/heal_b920_subquorum_union_test.rs

@@ -33,6 +33,7 @@ use std::{
     future::Future,
     path::{Path, PathBuf},
     sync::Arc,
+    time::Duration,
 };
 use walkdir::WalkDir;
 
@@ -135,6 +136,22 @@ fn remove_xl_meta_only(disk: &Path, bucket: &str, object: &str) {
     );
 }
 
+async fn wait_for_object_copies(disks: &[PathBuf], bucket: &str, object: &str) {
+    tokio::time::timeout(Duration::from_secs(5), async {
+        loop {
+            if disks.iter().all(|disk| {
+                let object_dir = object_dir(disk, bucket, object);
+                xl_meta_path(&object_dir).exists() && count_part_files(&object_dir) >= 1
+            }) {
+                break;
+            }
+            tokio::time::sleep(Duration::from_millis(10)).await;
+        }
+    })
+    .await
+    .expect("PUT rename tails must converge before corrupting the disk fixture");
+}
+
 fn deep_heal_opts() -> HealOpts {
     HealOpts {
         recreate: true,
@@ -254,6 +271,7 @@ mod serial_tests {
 
         let data_v1 = versioned_test_data(7);
         let v1 = put_versioned(&ecstore, bucket, object, &data_v1).await;
+        wait_for_object_copies(&disk_paths, bucket, object).await;
 
         // EC4+4: parity = 4. Delete ONLY xl.meta on 5 disks (> parity), leaving the
         // data shards on all 8. Meta quorum (4) is now unreachable (3 metas), which