From 0e27f57c401ee3eb93d71a5041d4f019eff95b0f Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Thu, 27 Aug 2026 11:26:20 +0800 Subject: [PATCH] fix(test): wait for EC materialization in relocated-pool resume fixture (#6718) fix(test): wait for EC write materialization before staging relocated-pool fixture An erasure-coded write returns once write-quorum disks commit, so a lagging disk can legally still be missing its xl.meta when the relocated-pool resume test starts staging its fixture by iterating every disk of the owning pool. Under CI load this raced into a NotFound panic in the staging loop. Add a bounded readiness poll that waits for xl.meta on every pool disk before the normalization and staging steps. Fixes #6703 --- rustfs/src/app/object/get.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/rustfs/src/app/object/get.rs b/rustfs/src/app/object/get.rs index 6a3ecdcbd..ca84b6345 100644 --- a/rustfs/src/app/object/get.rs +++ b/rustfs/src/app/object/get.rs @@ -7023,6 +7023,29 @@ mod tests { body } + // An erasure-coded write returns once write-quorum disks commit, so a + // lagging disk can legally still be missing its xl.meta when the write + // call returns. Fixtures that iterate every disk of the owning pool must + // wait for full materialization first, or they race the trailing disk + // writes under CI load (issue #6703). Bounded so a genuinely failed disk + // write still surfaces as a test failure instead of a hang. + async fn wait_for_object_on_every_disk(disk_paths: &[std::path::PathBuf], bucket: &str, object: &str) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + loop { + if disk_paths + .iter() + .all(|path| path.join(bucket).join(object).join("xl.meta").is_file()) + { + return; + } + assert!( + std::time::Instant::now() < deadline, + "object {bucket}/{object} must materialize xl.meta on every pool disk within the readiness window" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + } + // Deletes the given part files from every version data dir present on the // disks, simulating rebalance removing the object data while xl.meta stays // readable. Returns the number of version dirs visited and files removed. @@ -7172,6 +7195,7 @@ mod tests { .any(|path| path.join(&bucket).join(object).join("xl.meta").is_file()) }) .expect("multipart object must be placed in one source pool"); + wait_for_object_on_every_disk(&pool_disk_paths[upload_pool], &bucket, object).await; if upload_pool != 0 { let mut normalized_disks = 0; for (source_disk, target_disk) in pool_disk_paths[upload_pool].iter().zip(&pool_disk_paths[0]) {