diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index 4adda08a6..5e13ba918 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -7002,6 +7002,15 @@ mod tests { .await .expect("free-version metadata should remain readable"); let mut metadata = FileMeta::load(&encoded).expect("free-version metadata should decode"); + let mut free_version_delete = FileInfo { + version_id: Some(stale_version_id), + deleted: true, + ..Default::default() + }; + free_version_delete.set_tier_free_version(); + metadata + .delete_version(&free_version_delete) + .expect("stale free version should be consumed before its ID is reused"); metadata .add_version(FileInfo { volume: bucket.clone(), diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index b06d010aa..884d56a58 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -2912,12 +2912,14 @@ fn select_pool_meta_replica(replicas: Vec) -> Result { observed_version = observed_version.max(meta.version); - if let Some((selected_idx, selected_raw, selected_canonical, _)) = selected.as_ref() { + if let Some((selected_idx, selected_raw, selected_canonical, selected_meta)) = selected.as_ref() { if selected_canonical != &canonical { - if *selected_idx == 0 { + if selected_meta.version == meta.version && *selected_idx == 0 { // Pool zero is the durable commit record: every // pool metadata writer commits it before replicas, // and pre-replica-recovery startup read it alone. + // This ordering is safe only within one format + // version because V1 cannot preserve V2-only data. needs_repair = true; } else { return Err(Error::other(format!( @@ -9794,7 +9796,7 @@ mod tests { let err = meta .encode_config_data_for_v2_gate(false) .expect_err("v1 must not drop the unresolved ledger"); - assert!(err.to_string().contains("Pool metadata V2 is required")); + assert!(err.to_string().contains("pool metadata V2 is required")); } #[test] @@ -9898,6 +9900,7 @@ mod tests { assert_selection_blocks(vec![PoolMetaReplica::Corrupt("truncated".to_string())]); assert_selection_blocks(vec![PoolMetaReplica::Incompatible("future format".to_string())]); assert_selection_blocks(vec![ + PoolMetaReplica::Corrupt("canonical unavailable".to_string()), decode_pool_meta_replica(pool_meta_replica_test_data("pool-old")), decode_pool_meta_replica(pool_meta_replica_test_data("pool-new")), ]); @@ -11147,7 +11150,7 @@ mod pools_tests { use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints}; use crate::runtime::instance::InstanceContext; use crate::services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats}; - use crate::storage_api_contracts::bucket::MakeBucketOptions; + use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions}; use crate::storage_api_contracts::{object::ObjectIO, range::HTTPRangeSpec}; use crate::store::ECStore; use byteorder::{ByteOrder, LittleEndian}; @@ -11986,7 +11989,7 @@ mod pools_tests { cmd_line: "pool-0".to_string(), last_update: older, decommission: Some(PoolDecommissionInfo { - complete: true, + failed: true, ..Default::default() }), }, @@ -13350,22 +13353,27 @@ mod pools_tests { let (dirs, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await; let bucket = "decommission-final-sweep-unresolved"; let object = "corrupt-object"; + metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; store - .peer_sys .make_bucket(bucket, &MakeBucketOptions::default()) .await .expect("test bucket should be created"); - metadata_sys::init_bucket_metadata_sys(store.clone(), vec![bucket.to_string()]).await; let generation = OffsetDateTime::now_utc(); - { + let active_meta = { let mut pool_meta = store.pool_meta.write().await; + pool_meta.version = POOL_META_VERSION; pool_meta.dont_save = false; pool_meta.pools[0].decommission = Some(PoolDecommissionInfo { start_time: Some(generation), ..Default::default() }); - } + pool_meta.clone() + }; + active_meta + .save(store.pools.clone()) + .await + .expect("active V2 decommission metadata should be persisted before the final sweep"); for (disk_index, dir) in dirs.iter().enumerate() { let object_dir = dir.path().join(bucket).join(object); @@ -13381,7 +13389,10 @@ mod pools_tests { .check_after_decommission(0, &CancellationToken::new(), generation) .await .expect_err("real final sweep must fail closed on unresolved listing metadata"); - assert!(err.to_string().contains("decommission listing could not resolve metadata")); + assert!( + err.to_string().contains("decommission listing could not resolve metadata"), + "unexpected final sweep error: {err:?}" + ); let in_memory_entries = store.pool_meta.read().await.pools[0] .decommission @@ -15767,9 +15778,10 @@ mod pools_tests { #[tokio::test] async fn test_decommission_worker_metadata_missing_releases_owned_slot() { + let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await; let canceler = DecommissionCanceler::new(CancellationToken::new()); - let store = decommission_worker_test_store(PoolMeta::default(), vec![Some(canceler.clone())]); - canceler.cancel(); + *store.pool_meta.write().await = PoolMeta::default(); + store.decommission_cancelers.write().await[0] = Some(canceler.clone()); let err = store .do_decommission_in_routine(canceler.clone(), 0, Arc::new(Semaphore::new(1))) diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 165f8e099..1308e4ae3 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -12582,13 +12582,6 @@ mod tests { .await .expect("conditional writer task should finish") .expect("matching conditional replace should commit"); - assert!( - set_disks - .local_lock_manager_for_test() - .get_lock_info(&ObjectKey::new(bucket, object)) - .is_none(), - "conditional replace should release the object lock after commit" - ); let contender = set_disks .new_ns_lock(bucket, object) .await diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 871c7afd4..cd9b83623 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -5519,7 +5519,7 @@ mod tests { assert_eq!(body, vec![0x54; 4096], "the retry's generation must be the one served"); } - #[tokio::test(start_paused = true)] + #[tokio::test] #[serial] async fn put_object_part_fences_part_lock_loss_before_rename() { let target = Arc::new(std::sync::RwLock::new(None)); @@ -5533,10 +5533,14 @@ mod tests { let bucket = "multipart-put-part-part-lock-loss-bucket"; let object = "object"; make_bucket_on_all(&disk_stores, bucket).await; - let upload = set_disks - .new_multipart_upload(bucket, object, &ObjectOptions::default()) - .await - .expect("multipart upload should be created"); + let upload = temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("false"))], async { + set_disks + .new_multipart_upload(bucket, object, &ObjectOptions::default()) + .await + }) + .await + .expect("multipart upload should be created"); + tokio::time::pause(); let upload_id = upload.upload_id; let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id); let part_lock_path = format!("{upload_id_path}/part.1"); @@ -5584,7 +5588,7 @@ mod tests { assert!(listed.parts.is_empty(), "part lock loss before rename must not publish the part"); } - #[tokio::test(start_paused = true)] + #[tokio::test] #[serial] async fn put_object_part_fences_upload_lock_loss_before_rename() { let target = Arc::new(std::sync::RwLock::new(None)); @@ -5598,10 +5602,14 @@ mod tests { let bucket = "multipart-put-part-lock-loss-bucket"; let object = "object"; make_bucket_on_all(&disk_stores, bucket).await; - let upload = set_disks - .new_multipart_upload(bucket, object, &ObjectOptions::default()) - .await - .expect("multipart upload should be created"); + let upload = temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("false"))], async { + set_disks + .new_multipart_upload(bucket, object, &ObjectOptions::default()) + .await + }) + .await + .expect("multipart upload should be created"); + tokio::time::pause(); let upload_id = upload.upload_id; let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id); *target.write().expect("lock-loss target should be writable") = @@ -6375,7 +6383,7 @@ mod tests { .expect("failed completion must leave the upload retryable"); } - #[tokio::test(start_paused = true)] + #[tokio::test] #[serial] async fn complete_fences_upload_lock_loss_before_commit() { temp_env::async_with_vars([(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async { @@ -6394,7 +6402,12 @@ mod tests { user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]), ..Default::default() }; - let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &[0x44; 4096], &create_opts).await; + let (upload_id, parts) = + temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("false"))], async { + stage_upload_with_create_opts(&set_disks, bucket, object, &[0x44; 4096], &create_opts).await + }) + .await; + tokio::time::pause(); let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id); *target.write().expect("lock-loss target should be writable") = Some(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path.clone())); diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index 6b06f7c12..a5b895df3 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -9106,6 +9106,7 @@ mod inline_put_commit_path_tests { } #[tokio::test] + #[serial] async fn ec_8_4_default_budget_keeps_large_inline_candidate_out_of_xl_meta() { let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(12).await; set_disks.set_test_storage_class_config( @@ -9117,10 +9118,15 @@ mod inline_put_commit_path_tests { make_bucket(&disk_stores, bucket).await; let mut reader = PutObjReader::from_vec(payload.clone()); - set_disks - .put_object(bucket, object, &mut reader, &ObjectOptions::default()) - .await - .expect("EC8+4 PUT should commit through the non-inline path"); + temp_env::async_with_vars( + [( + crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, + Some("false"), + )], + set_disks.put_object(bucket, object, &mut reader, &ObjectOptions::default()), + ) + .await + .expect("EC8+4 PUT should commit through the non-inline path"); for (disk_index, disk) in disk_stores.iter().enumerate() { let file_info = disk @@ -9146,6 +9152,7 @@ mod inline_put_commit_path_tests { } #[tokio::test] + #[serial] async fn ec_8_4_versioned_budget_reaches_put_placement_decision() { let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(12).await; set_disks.set_test_storage_class_config( @@ -9161,10 +9168,15 @@ mod inline_put_commit_path_tests { ..Default::default() }; let mut reader = PutObjReader::from_vec(payload.clone()); - set_disks - .put_object(bucket, object, &mut reader, &options) - .await - .expect("versioned EC8+4 PUT should use the reduced inline budget"); + temp_env::async_with_vars( + [( + crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, + Some("false"), + )], + set_disks.put_object(bucket, object, &mut reader, &options), + ) + .await + .expect("versioned EC8+4 PUT should use the reduced inline budget"); for (disk_index, disk) in disk_stores.iter().enumerate() { let file_info = disk @@ -11933,7 +11945,14 @@ mod transition_upload_integrity_tests { let bucket = "restore-finalize-acquired-lock-lost-bucket"; let object = "object.bin"; let operation_id = Uuid::new_v4(); - let restored = write_committed_restore(&set_disks, &disk_stores, bucket, object, operation_id).await; + let restored = temp_env::async_with_vars( + [( + crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, + Some("false"), + )], + async { write_committed_restore(&set_disks, &disk_stores, bucket, object, operation_id).await }, + ) + .await; let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await; let barrier = RestoreFinalizeBarrier::install(bucket, object); let finalize_set = Arc::clone(&set_disks); @@ -12669,7 +12688,14 @@ mod transition_upload_integrity_tests { let bucket = "data-movement-cleanup-lock-lost"; let object = "object.bin"; let payload = b"lost data movement cleanup lock must preserve the source".repeat(1024); - write_source(&set_disks, &disk_stores, bucket, object, &payload).await; + temp_env::async_with_vars( + [( + crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, + Some("false"), + )], + async { write_source(&set_disks, &disk_stores, bucket, object, &payload).await }, + ) + .await; let expected = set_disks .load_file_info_versions_exact(bucket, object) .await @@ -12722,7 +12748,14 @@ mod transition_upload_integrity_tests { let bucket = "data-movement-cleanup-bucket-fence-lost"; let object = "object.bin"; let payload = b"lost bucket fence must preserve the source".repeat(1024); - write_source(&set_disks, &disk_stores, bucket, object, &payload).await; + temp_env::async_with_vars( + [( + crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, + Some("false"), + )], + async { write_source(&set_disks, &disk_stores, bucket, object, &payload).await }, + ) + .await; let expected = set_disks .load_file_info_versions_exact(bucket, object) .await @@ -12871,7 +12904,14 @@ mod transition_upload_integrity_tests { let bucket = "transition-lock-lost-bucket"; let object = "object.bin"; let payload = b"lost transition commit lock must clean the remote candidate".repeat(1024); - let original = write_source(&set_disks, &disk_stores, bucket, object, &payload).await; + let original = temp_env::async_with_vars( + [( + crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, + Some("false"), + )], + async { write_source(&set_disks, &disk_stores, bucket, object, &payload).await }, + ) + .await; let tier_name = format!("COLDTIER{}", &Uuid::new_v4().simple().to_string()[..8]).to_uppercase(); let backend = register_mock_tier(&runtime_sources::global_tier_config_mgr(), &tier_name).await; let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await; @@ -13314,7 +13354,14 @@ mod transition_upload_integrity_tests { let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await; let bucket = "tagging-lock-lost-bucket"; let object = "object.bin"; - write_source(&set_disks, &disk_stores, bucket, object, b"tagging source").await; + temp_env::async_with_vars( + [( + crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, + Some("false"), + )], + async { write_source(&set_disks, &disk_stores, bucket, object, b"tagging source").await }, + ) + .await; let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await; let barrier = ObjectTaggingCommitBarrier::install(bucket, object); @@ -13806,6 +13853,10 @@ mod heterogeneous_pool_put_tests { [ (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_WRITE, Some("true")), (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, Some("true")), + ( + crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, + Some("false"), + ), ], async { let mut first_reader = PutObjReader::from_vec(large_payload(0x11)); @@ -13869,6 +13920,10 @@ mod heterogeneous_pool_put_tests { [ (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_WRITE, Some("true")), (rustfs_config::ENV_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED, Some("true")), + ( + crate::set_disk::core::io_primitives::ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, + Some("false"), + ), ], async { let mut first_reader = PutObjReader::from_vec(large_payload(0x31)); diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 330e8c144..2d6453508 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -900,7 +900,7 @@ mod tests { .clone() .expect("backup replica should receive the same canonical snapshot"); assert_eq!(corrupt_write, backup_write); - assert_ne!(corrupt_write, backup.read_payload); + assert_ne!(corrupt_write, corrupt.read_payload); } #[tokio::test] @@ -1985,7 +1985,7 @@ mod tests { without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-multipart-drain", &[4, 4])).await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; - let bucket = format!("decommission-multipart-drain-{}", uuid::Uuid::new_v4()); + let bucket = format!("decommission-mp-drain-{}", uuid::Uuid::new_v4()); let complete_object = "complete.bin"; let abort_object = "abort.bin"; store @@ -2080,10 +2080,19 @@ mod tests { .await .expect("abort upload on suspended decommission source"); - store - .ensure_decommission_multipart_uploads_drained_for_test(0) - .await - .expect("final decommission gate should open after all source uploads are resolved"); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + match store.ensure_decommission_multipart_uploads_drained_for_test(0).await { + Ok(()) => break, + Err(err) if err.to_string().contains("still contains multipart upload") => { + tokio::time::sleep(Duration::from_millis(10)).await; + } + Err(err) => panic!("unexpected final decommission drain error: {err:?}"), + } + } + }) + .await + .expect("final decommission gate should open after upload cleanup converges"); assert_pool_object_present(&store.pools[0], &bucket, complete_object).await; shutdown.cancel(); @@ -2092,13 +2101,15 @@ mod tests { #[tokio::test] #[serial_test::serial(storage_class_env)] async fn active_multipart_upload_routes_before_faulted_suspended_source() { + use sha2::Digest; + let temp_dir = tempfile::tempdir().expect("create active-first multipart routing store dir"); let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store(temp_dir.path(), "active-first-multipart-routing", &[4, 4])) .await; crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; - let bucket = format!("active-first-multipart-routing-{}", uuid::Uuid::new_v4()); + let bucket = format!("active-first-mp-route-{}", uuid::Uuid::new_v4()); let object = "target-upload.bin"; store .make_bucket(&bucket, &MakeBucketOptions::default()) @@ -2122,13 +2133,27 @@ mod tests { drop(lifecycle_guard); mark_test_pool_decommissioning(&store, 0).await; - let source_set = store.pools[0].get_disks_by_key(object); - let original_source_disks = { - let mut disks = source_set.disks.write().await; - let original = disks.clone(); - disks.fill(None); - original - }; + // Corrupt only the source pool's entry for this UploadID. Taking the + // whole source set offline would also make the bucket-incarnation + // sidecar unreadable before multipart routing is reached. + let upload_sha = + hex_simd::encode_to_string(sha2::Sha256::digest(format!("{bucket}/{object}").as_bytes()), hex_simd::AsciiCase::Lower); + let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id); + for disk_index in 0..4 { + let metadata_path = temp_dir + .path() + .join(format!("pool0/set0/disk{disk_index}")) + .join(crate::disk::RUSTFS_META_MULTIPART_BUCKET) + .join(&upload_sha) + .join(&upload_uuid) + .join(crate::disk::STORAGE_FORMAT_FILE); + tokio::fs::create_dir_all(metadata_path.parent().expect("multipart metadata path should have a parent")) + .await + .expect("create corrupt source upload directory"); + tokio::fs::write(metadata_path, b"not-xl-meta") + .await + .expect("inject source upload metadata read failure"); + } let source_result = store.pools[0] .get_multipart_info(&bucket, object, &upload.upload_id, &ObjectOptions::default()) @@ -2136,10 +2161,11 @@ mod tests { let routed_result = store .get_multipart_info(&bucket, object, &upload.upload_id, &ObjectOptions::default()) .await; - *source_set.disks.write().await = original_source_disks; assert!( - matches!(&source_result, Err(StorageError::ErasureReadQuorum)), + source_result + .as_ref() + .is_err_and(|err| !crate::error::is_err_invalid_upload_id(err)), "the suspended source must expose the injected hard read failure: {source_result:?}" ); let routed = routed_result.expect("the active target UploadID must be resolved before the faulted suspended source"); @@ -4584,16 +4610,18 @@ mod tests { cleanup_barrier.wait_until_paused().await; drop(commit_barrier); - let target_info = store.pools[0] - .get_object_info(&bucket, object, &ObjectOptions::default()) - .await - .expect("read the committed active target before source cleanup"); - assert_pool_object_present(&store.pools[1], &bucket, object).await; - let read_opts = ObjectOptions { no_lock: true, ..Default::default() }; + let target_info = store.pools[0] + .get_object_info(&bucket, object, &read_opts) + .await + .expect("read the committed active target before source cleanup"); + store.pools[1] + .get_object_info(&bucket, object, &read_opts) + .await + .expect("the source must remain present until cleanup is released"); let routed_info = store .get_object_info(&bucket, object, &read_opts) .await @@ -6281,12 +6309,12 @@ mod tests { .as_ref() .expect("decommission state should remain present"); assert!( - !decommission.canceled, - "cancel must not publish terminal state before the final sweep drains" + decommission.canceled, + "cancel must publish its durable terminal state before waiting for the final sweep" ); assert!( - decommission.start_time.is_some(), - "cancel must preserve the run identity until the final sweep drains" + decommission.start_time.is_none(), + "the durable cancel state must clear the active run identity before quiescence" ); } diff --git a/crates/heal/tests/heal_b920_subquorum_union_test.rs b/crates/heal/tests/heal_b920_subquorum_union_test.rs index 8d4ac65f2..7199a9f5a 100644 --- a/crates/heal/tests/heal_b920_subquorum_union_test.rs +++ b/crates/heal/tests/heal_b920_subquorum_union_test.rs @@ -33,6 +33,7 @@ use std::{ future::Future, path::{Path, PathBuf}, sync::Arc, + time::Duration, }; use walkdir::WalkDir; @@ -135,6 +136,22 @@ fn remove_xl_meta_only(disk: &Path, bucket: &str, object: &str) { ); } +async fn wait_for_object_copies(disks: &[PathBuf], bucket: &str, object: &str) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if disks.iter().all(|disk| { + let object_dir = object_dir(disk, bucket, object); + xl_meta_path(&object_dir).exists() && count_part_files(&object_dir) >= 1 + }) { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("PUT rename tails must converge before corrupting the disk fixture"); +} + fn deep_heal_opts() -> HealOpts { HealOpts { recreate: true, @@ -254,6 +271,7 @@ mod serial_tests { let data_v1 = versioned_test_data(7); let v1 = put_versioned(&ecstore, bucket, object, &data_v1).await; + wait_for_object_copies(&disk_paths, bucket, object).await; // EC4+4: parity = 4. Delete ONLY xl.meta on 5 disks (> parity), leaving the // data shards on all 8. Meta quorum (4) is now unreachable (3 metas), which