From 9ae4ca5f9911c120c814d70dfe002602c6d15b7c Mon Sep 17 00:00:00 2001 From: houseme Date: Wed, 8 Jul 2026 04:01:27 +0800 Subject: [PATCH] test(ecstore): fit concurrent-resend guard inside lock acquire ceiling (#4384) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under a full nextest run on loaded machines the legitimately serialized cross-disk commits in concurrent_resend_same_part_commits_one_generation exceed the acquire deadline by themselves: observed at the 5s default, at the 30s production default (#4370), and on CI even at 60s — which is a hard ceiling, because fast_lock clamps every requested timeout to MAX_ACQUIRE_TIMEOUT (60s) while the Timeout error still reports the requested value, so raising the env override higher is a no-op. Request the full 60s ceiling explicitly and cap the queue depth at three concurrent resends, bounding the last waiter to two serialized commits (~12s each on the slowest observed CI runner). Three resends still race the streaming phase and contend on the commit lock, which is all the generation-mixing regression (backlog#853) needs; every correctness assertion is unchanged. Co-authored-by: heihutu --- .../bucket/lifecycle/bucket_lifecycle_ops.rs | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index cfeccddb4..c6982a2ca 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -4710,7 +4710,7 @@ mod tests { // Distinct payloads with distinct sizes: a mixed-generation reassembly // would produce bytes matching none of them (or fail the read outright). - let candidates: Vec> = (0..6) + let candidates: Vec> = (0..3) .map(|g| { let len = 4096 + g * 512; vec![b'a' + g as u8; len] @@ -4722,14 +4722,22 @@ mod tests { // 1. A lost/stolen fast-lock wakeup could strand a waiter until the // deadline — fixed for real in fast_lock::shard by bounding each // notification wait (NOTIFY_WAIT_CAP re-polling). - // 2. Under the full nextest suite on loaded CI disks, the six - // *legitimately serialized* cross-disk commits can exceed the small - // default acquire timeout (5s) all by themselves — observed on CI - // even with fix 1 in place. Raise the timeout to the production - // default (30s) for the concurrent section so the guard asserts the - // correctness property (exactly one intact generation), not disk - // latency. `#[serial]` keeps the process-wide env override isolated. - let results = temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("30"))], async { + // 2. Under the full nextest suite on loaded CI disks, the + // *legitimately serialized* cross-disk commits can exceed the + // acquire deadline all by themselves — observed on CI at the 5s + // default and the 30s production default with six resends, and + // again at 60s, which is a hard ceiling: fast_lock clamps every + // requested timeout to MAX_ACQUIRE_TIMEOUT (60s), so raising the + // env override higher is a no-op (the Timeout error still reports + // the requested value). Keep the guard about the correctness + // property, not disk latency: request the full 60s ceiling and cap + // the queue depth at three resends, so the last waiter sits behind + // at most two serialized commits (~12s each on the slowest observed + // CI runner, comfortably inside the deadline). Three concurrent + // resends still race the streaming phase and contend on the commit + // lock, which is all the generation-mixing regression needs. + // `#[serial]` keeps the process-wide env override isolated. + let results = temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("60"))], async { let mut tasks = tokio::task::JoinSet::new(); for payload in candidates.iter().cloned() { let store = ecstore.clone();