From 5c269a3970eefb7137cee6a9836f4903af12d460 Mon Sep 17 00:00:00 2001
From: joshsiv-mulga <mrk@mulgadc.com>
Date: Mon, 18 May 2026 19:47:57 +1000
Subject: [PATCH] ci(e2e-go): auto-cancel + reap orphans to prevent runner
 wedge

A cancelled run on the single-runner ci-single host has been leaving
orphan tofu/terraform procs (and stale tfstate locks) behind, which
caused the next push's tofu destroy to hang for 45m+ and saturate the
runner host until sshd refused new connections.

- Add `concurrency: e2e-go-${ref}, cancel-in-progress: true` so a new
  push to feat/e2e-go-harness cancels the in-flight run instead of
  stacking on the same self-hosted runner.
- New `Reap Orphan Procs` step at job start: SIGTERM then SIGKILL any
  lingering `tofu`/`terraform` from the runner user, and unlink any
  stale `.terraform.tfstate.lock.info`.
- Harden `Teardown`: drop `set -e` (scrub-env must always run as the
  libvirt-level fallback), reap orphans before destroy, and wrap
  `tofu destroy` in `timeout 600` so a wedged provider can't pin the
  job to its 45m timeout.
---
 .github/workflows/e2e-go.yml | 44 +++++++++++++++++++++++++++++++++---
 1 file changed, 41 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/e2e-go.yml b/.github/workflows/e2e-go.yml
index 65c418787..84eacb46c 100644
--- a/.github/workflows/e2e-go.yml
+++ b/.github/workflows/e2e-go.yml
@@ -31,6 +31,13 @@ permissions:
   contents: read
   statuses: write
 
+# Pushes to the same ref stack on the single `ci-single` self-hosted runner,
+# and a cancelled run can leave orphan `tofu`/`go test -c` procs behind that
+# wedge the runner host. Auto-cancel keeps at most one in-flight run per ref.
+concurrency:
+  group: e2e-go-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   GIT_BRANCH: ${{ github.ref_name }}
   SUITES: ${{ inputs.suites || 'e2e-cert e2e-lb' }}
@@ -44,6 +51,24 @@ jobs:
       TOFU_LOG: /tmp/e2e-go-tofu-single-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
     steps:
+      - name: Reap Orphan Procs
+        # A cancelled prior run can leave `tofu`/`terraform` mid-apply or a
+        # libvirt `qemu+ssh` connection orphaned. Kill anything from previous
+        # invocations belonging to this runner user before we touch state.
+        # SIGTERM first to let tofu release its state lock, then SIGKILL.
+        run: |
+          set +e
+          UID_SELF=$(id -u)
+          pgrep -u "$UID_SELF" -af '^(tofu|terraform)\b|/tofu |/terraform ' || true
+          pkill -u "$UID_SELF" -TERM -x tofu 2>/dev/null
+          pkill -u "$UID_SELF" -TERM -x terraform 2>/dev/null
+          sleep 3
+          pkill -u "$UID_SELF" -KILL -x tofu 2>/dev/null
+          pkill -u "$UID_SELF" -KILL -x terraform 2>/dev/null
+          # Drop any stale tofu state lock from a SIGKILL'd apply.
+          find "$HOME" -maxdepth 6 -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          true
+
       - name: Clean Workspace
         run: sudo rm -rf "$GITHUB_WORKSPACE"/*
 
@@ -240,13 +265,26 @@ jobs:
       - name: Teardown
         if: always()
         working-directory: mulga/scripts/tofu-cluster
+        # `set -e` off — teardown must run scrub-env even if tofu destroy
+        # fails; scrub is the libvirt-level fallback when tofu state is
+        # inconsistent with the remote Proxmox node.
         run: |
-          set -euo pipefail
+          set +e
           # shellcheck source=/dev/null
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          # Cancelled jobs from this point on can orphan an in-flight
+          # tofu/libvirt SSH; clear them before destroy so the next run's
+          # pre-clean isn't doing it.
+          UID_SELF=$(id -u)
+          pkill -u "$UID_SELF" -TERM -x tofu 2>/dev/null
+          pkill -u "$UID_SELF" -TERM -x terraform 2>/dev/null
+          sleep 2
+          pkill -u "$UID_SELF" -KILL -x tofu 2>/dev/null
+          pkill -u "$UID_SELF" -KILL -x terraform 2>/dev/null
+          find "$HOME" -maxdepth 6 -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
-          quiet_run "tofu destroy"          "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
-          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
+          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 
       - name: Fail job if any suite failed