From 2dfa3d3c3641cc04e2fd982a8f384bd9b074d2da Mon Sep 17 00:00:00 2001
From: houseme <housemecn@gmail.com>
Date: Tue, 7 Jul 2026 22:07:05 +0800
Subject: [PATCH] test(ecstore): restore 30s lock-timeout guard for concurrent
 resend test (#4370)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

concurrent_resend_same_part_commits_one_generation keeps failing on CI
with Lock(Timeout ... 5s) even after the fast-lock lost-wakeup fix
(NOTIFY_WAIT_CAP re-polling) landed — observed on rustfs#4365's Test and
Lint runs. Two independent causes exist and both are real:

1. the lost/stolen wakeup stall — fixed in fast_lock::shard; and
2. the six legitimately serialized cross-disk commits exceeding the
   small 5s default acquire timeout under full-suite CI disk load.

The lost-wakeup fix reverted the earlier 30s test override on the
assumption that cause 1 was the whole story; CI shows cause 2 stands on
its own. Restore the temp_env override (production-default 30s) around
the concurrent section so the regression guard asserts the correctness
property (exactly one intact generation), not CI disk latency. The
NOTIFY_WAIT_CAP fix stays untouched.

Verification:
- cargo test -p rustfs-ecstore --lib concurrent_resend_same_part_commits_one_generation

Ref: rustfs/backlog#882

Co-authored-by: heihutu <heihutu@gmail.com>
---
 .../bucket/lifecycle/bucket_lifecycle_ops.rs  | 60 +++++++++++--------
 1 file changed, 36 insertions(+), 24 deletions(-)

diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
index f9d3658d7..cfeccddb4 100644
--- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
+++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs
@@ -4717,31 +4717,43 @@ mod tests {
             })
             .collect();
 
-        let mut tasks = tokio::task::JoinSet::new();
-        for payload in candidates.iter().cloned() {
-            let store = ecstore.clone();
-            let bucket = bucket.clone();
-            let upload_id = upload.upload_id.clone();
-            tasks.spawn(async move {
-                let mut data = PutObjReader::from_vec(payload.clone());
-                store
-                    .put_object_part(&bucket, object, &upload_id, 1, &mut data, &ObjectOptions::default())
-                    .await
-                    .map(|info| (info, payload))
-            });
-        }
+        // Two independent causes can produce a spurious lock-acquire timeout
+        // here, and both must stay covered:
+        // 1. A lost/stolen fast-lock wakeup could strand a waiter until the
+        //    deadline — fixed for real in fast_lock::shard by bounding each
+        //    notification wait (NOTIFY_WAIT_CAP re-polling).
+        // 2. Under the full nextest suite on loaded CI disks, the six
+        //    *legitimately serialized* cross-disk commits can exceed the small
+        //    default acquire timeout (5s) all by themselves — observed on CI
+        //    even with fix 1 in place. Raise the timeout to the production
+        //    default (30s) for the concurrent section so the guard asserts the
+        //    correctness property (exactly one intact generation), not disk
+        //    latency. `#[serial]` keeps the process-wide env override isolated.
+        let results = temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("30"))], async {
+            let mut tasks = tokio::task::JoinSet::new();
+            for payload in candidates.iter().cloned() {
+                let store = ecstore.clone();
+                let bucket = bucket.clone();
+                let upload_id = upload.upload_id.clone();
+                tasks.spawn(async move {
+                    let mut data = PutObjReader::from_vec(payload.clone());
+                    store
+                        .put_object_part(&bucket, object, &upload_id, 1, &mut data, &ObjectOptions::default())
+                        .await
+                        .map(|info| (info, payload))
+                });
+            }
 
-        // Every concurrent resend must succeed. The per-uploadId commit lock must
-        // not starve a waiter into a lock-acquire timeout: the shared fast-lock
-        // notify pool could otherwise route this lock's wakeup to a waiter of an
-        // unrelated lock and strand this one until the deadline (fixed in
-        // fast_lock::shard by bounding each notification wait, so a lost/stolen
-        // wakeup degrades to bounded re-polling instead of a hard timeout).
-        let mut results = Vec::new();
-        while let Some(joined) = tasks.join_next().await {
-            let outcome = joined.expect("put_object_part task should not panic");
-            results.push(outcome.expect("every concurrent same-part resend must succeed without lock timeout"));
-        }
+            // Every concurrent resend must succeed; the commit lock must never
+            // starve a waiter into a timeout.
+            let mut results = Vec::new();
+            while let Some(joined) = tasks.join_next().await {
+                let outcome = joined.expect("put_object_part task should not panic");
+                results.push(outcome.expect("every concurrent same-part resend must succeed without lock timeout"));
+            }
+            results
+        })
+        .await;
         assert_eq!(results.len(), candidates.len());
 
         // Exactly one generation is visible after the serialized commits, and its