From 2039c05c170caa531600fd5faf4d7d800ebf89b6 Mon Sep 17 00:00:00 2001
From: Ben Duncan <5178243+benduncan@users.noreply.github.com>
Date: Thu, 17 Sep 2026 10:59:39 -0500
Subject: [PATCH] fix(ci): take the host benchmark lock before killing
 predastore nodes (#1065)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

cell-24 clears leftover predastore nodes with `pkill -KILL -f '/bin/s3d
-config'`, which matches every predastore node on the runner and not only the
ones this run left behind. The predastore repository runs the same harnesses on
the same machine, so "still running" does not mean "stale" — on 2026-09-16 this
step killed a predastore benchmark 32 seconds into its workload, and the run
failed with 61 connection-refused errors that look nothing like their cause.

Both kills now run under the host benchmark lock the harnesses themselves take,
so anything alive while it is held really is a leftover. The second one needs it
as much as the first: by the time cleanup runs the harness has released the
lock, so another repository's benchmark can already be up.
---
 .github/workflows/e2e-nightly.yml | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e-nightly.yml b/.github/workflows/e2e-nightly.yml
index 11a773b41..9085e79d6 100644
--- a/.github/workflows/e2e-nightly.yml
+++ b/.github/workflows/e2e-nightly.yml
@@ -1493,8 +1493,16 @@ jobs:
 
       # Leftovers from a run killed before its cleanup trap: a surviving node
       # holds its ports and fails this run somewhere unrelated.
+      #
+      # Under the host benchmark lock, because this pattern matches every
+      # predastore node on the runner and the predastore repository runs the
+      # same harnesses on the same machine. Holding the lock is what makes
+      # "still running" mean "stale" rather than "somebody else's benchmark":
+      # on 2026-09-16 this step killed a predastore run 32s into its workload.
       - name: Clear stale nodes
-        run: pkill -KILL -f '/bin/s3d -config' || true
+        run: |
+          flock -w 3600 "${PREDA_BENCH_LOCK:-/var/lock/predastore-bench.lock}" \
+            pkill -KILL -f '/bin/s3d -config' || true
 
       - name: Run fault injection
         working-directory: predastore
@@ -1514,10 +1522,14 @@ jobs:
           if-no-files-found: warn
           retention-days: 90
 
+      # Under the lock for the same reason as "Clear stale nodes": by the time
+      # this runs the harness has released it, so another repository's benchmark
+      # can already have a cluster of its own up.
       - name: Clean up
         if: always()
         run: |
-          pkill -KILL -f '/bin/s3d -config' || true
+          flock -w 3600 "${PREDA_BENCH_LOCK:-/var/lock/predastore-bench.lock}" \
+            pkill -KILL -f '/bin/s3d -config' || true
           rm -rf "$STRESS_RESULTS_ROOT"
 
   # Reading a nightly meant opening every cell job, and the Actions UI does not