name: E2E (Go)

# Standalone workflow for the Go E2E harness (tests/e2e/{cert,lb,...}).
# Mirrors e2e.yml's single-node provisioning chain but runs `go test` via
# gotestsum so we get JUnit XML + per-test artifact directories surfaced to
# GitHub Actions. Manual-dispatch only while the conversion lands — push
# trigger lifts once the single-node Go port is green and replaces
# run-e2e.sh; matches e2e-ddil.yml's Phase 1 trigger model.

on:
  workflow_dispatch:
    inputs:
      sha:
        description: "PR head SHA to report status on"
        required: false
      suites:
        description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb')"
        required: false
        default: "e2e-single e2e-multinode e2e-cert e2e-lb"
      test_run:
        description: "Optional regex for -test.run (e.g. '^TestMultinodeIPSec$'). Empty = run all tests in suite."
        required: false
        default: ""

permissions:
  contents: read
  statuses: write

# Pushes to the same ref stack on the single `ci-single` self-hosted runner,
# and a cancelled run can leave orphan `tofu`/`go test -c` procs behind that
# wedge the runner host. Auto-cancel keeps at most one in-flight run per ref.
concurrency:
  group: e2e-go-${{ github.ref }}
  cancel-in-progress: true

env:
  GIT_BRANCH: ${{ github.ref_name }}
  SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb' }}
  TEST_RUN: ${{ inputs.test_run || '' }}

jobs:
  e2e_go_single:
    name: Single-Node E2E (Go)
    # Skip when the dispatch input requests no in-VM suites (e.g.
    # suites='e2e-reboot' only). Push triggers and the default SUITES list
    # both include single/cert/lb so the inputs.suites null-check keeps the
    # job running on plain PR pushes.
    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-lb')
    runs-on: [ self-hosted, ci-single ]
    timeout-minutes: 45
    env:
      TOFU_LOG: /tmp/e2e-go-tofu-single-${{ github.run_id }}.log
      ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
    steps:
      - name: Reap Orphan Procs
        # A cancelled prior run can leave `tofu`/`terraform` mid-apply or a
        # libvirt `qemu+ssh` connection orphaned. Kill only procs whose cwd is
        # under THIS runner's workspace — every self-hosted runner on this host
        # shares the `tf-user` UID, so a UID-wide pkill would SIGKILL sibling
        # workflows (e2e.yml/e2e-ddil.yml/...) mid-apply. SIGTERM first to let
        # tofu release its state lock, then SIGKILL.
        run: |
          set +e
          UID_SELF=$(id -u)
          WS="${GITHUB_WORKSPACE}"
          reap() {
            local sig="$1"
            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
              case "$cwd" in
                "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
              esac
            done
          }
          reap TERM
          sleep 3
          reap KILL
          # Drop only this workspace's stale tofu state locks from a SIGKILL'd apply.
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          true

      - name: Clean Workspace
        run: sudo rm -rf "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        id: app-token
        with:
          client-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

      - uses: actions/checkout@v6
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

      - uses: actions/checkout@v6
        with:
          path: spinifex

      - uses: actions/setup-go@v6
        with:
          go-version-file: spinifex/go.mod
          # Self-hosted runner; the cache upload to GitHub blob storage
          # dominates total job time and the local Go module cache already
          # warms across runs.
          cache: false

      - name: Re-enable Go toolchain auto-download
        # setup-go@v6 exports GOTOOLCHAIN=local to $GITHUB_ENV (v5 left it
        # "auto"). build-install-artifacts.sh prepends /usr/local/go/bin to
        # PATH on the self-hosted runner, and that Go is older than
        # spinifex/go.mod's `go` directive; under "local" the build fails
        # with `requires go >= X.Y.Z`. Restoring "auto" lets the resolved
        # Go self-heal by fetching the named minor on demand.
        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"

      - name: Prune rootless docker cache
        # Self-hosted runner uses rootless docker; build-microvm-image.sh
        # leaks ~100 MiB overlay2 layer per failed build. Prune before each
        # run keeps /home/<ciuser>/.local/share/docker bounded.
        run: |
          docker container prune -f || true
          docker image prune -af || true
          docker buildx prune -af --filter "until=24h" || true
          docker system df || true

      - name: Build Install Artifacts
        working-directory: mulga/scripts/tofu-cluster
        run: |
          ./build-install-artifacts.sh "${GIT_BRANCH}" \
            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
            "${RUNNER_TEMP}/setup.sh" \
            "${RUNNER_TEMP}/spinifex-tests.tar"

      - name: Provision VM
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set -euo pipefail
          # shellcheck source=/dev/null
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          export TF_VAR_git_branch="${GIT_BRANCH}"
          quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
          quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
          quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
          quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          ok "VM provisioned"

      - name: Bootstrap via Binary Install
        working-directory: mulga/scripts/tofu-cluster
        run: |
          SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
            "${RUNNER_TEMP}/setup.sh" \
            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"

      - name: Resolve VM SSH details
        id: vm
        working-directory: mulga/scripts/tofu-cluster
        run: |
          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
          echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT"
          echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"

      - name: Build E2E test binaries on runner
        working-directory: spinifex/tests/e2e
        run: |
          set -euo pipefail
          # shellcheck source=/dev/null
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          # SUITES is in make-target form (e.g. "e2e-cert e2e-lb"); strip the
          # "e2e-" prefix to get package directory names for `go test -c`.
          # 'reboot' is filtered out here — that suite runs runner-resident
          # via the e2e_go_reboot job, not in-VM, so this build step doesn't
          # need its binary.
          # Filter out reboot (runs on e2e_go_reboot) and multinode (runs on
          # e2e_go_multinode). Single VM has no 3-node cluster — building +
          # shipping multinode.test here would just produce a binary that
          # skips on its own SPINIFEX_MODE check.
          SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|multinode)$' | tr '\n' ' ')
          if [ -z "$(echo "${SUITES_BUILD}" | tr -d ' ')" ]; then
            echo "no in-VM suites requested (SUITES=${SUITES}); nothing to build"
            exit 0
          fi
          banner "build ${SUITES_BUILD// /, }"
          group_open "go test -c"
          make e2e-build SUITES_BUILD="${SUITES_BUILD}"
          group_close
          for f in _bin/*.test; do
            ok "$(basename "$f") $(stat -c '%s' "$f" | numfmt --to=iec)"
          done

      - name: Install go-junit-report on runner
        run: |
          # Converts `go test -v` text output (from the pre-built binaries on
          # the VM) into the junit-*.xml that "Publish test summary" expects.
          go install github.com/jstemmer/go-junit-report/v2@latest
          echo "$HOME/go/bin" >> "$GITHUB_PATH"

      - name: Ship test binaries to VM
        run: |
          set -euo pipefail
          # shellcheck source=/dev/null
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
          ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin"
          tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin"
          ok "shipped binaries to $VM"

      - name: Run Go E2E suites
        id: e2e
        run: |
          set +e
          # SUITES are make-target names ("e2e-cert" → cert.test). Each
          # pre-built binary is self-contained (testdata //go:embed'd) so the
          # VM needs no go toolchain, no source tree, no gotestsum. Each
          # suite gets its own ::group:: + cyan banner inside so the Actions
          # UI collapses suite output by default; expand only what failed.
          # 'reboot' is filtered: runs via e2e_go_reboot on the runner, not here.
          SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|multinode)$' | tr '\n' ' ')
          if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
            echo "no in-VM suites requested (SUITES=${SUITES}); skipping"
            echo "rc=0" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          # Optional regex passed through to `go test -test.run`. Empty input
          # means run all tests in the suite (default behaviour).
          if [ -n "${TEST_RUN}" ]; then
            TEST_RUN_FLAG="-test.run=${TEST_RUN}"
          else
            TEST_RUN_FLAG=""
          fi
          VM_HOST="tf-user@${{ steps.vm.outputs.wan_ip }}"
          SSH_KEY="${{ steps.vm.outputs.ssh_key }}"
          # On cancel / runner SIGTERM / step timeout, force-kill any test
          # binaries left running on the VM. Without this, the remote `.test`
          # process keeps holding the suite's NATS connections + tmp dirs and
          # the subsequent Teardown's tofu-destroy races with its file
          # handles. -o ConnectTimeout=5 so a wedged VM doesn't block cleanup.
          cleanup_remote() {
            ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \
              'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
              2>/dev/null || true
          }
          trap cleanup_remote EXIT INT TERM
          # `ssh -tt` forces PTY allocation. When the runner cancels this
          # step, the local ssh dies and the remote sshd sends SIGHUP through
          # the PTY to its children — without -tt the remote `.test` would
          # orphan and keep running until its own 30m timeout.
          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" \
            "$VM_HOST" \
            "set -u; mkdir -p '${ARTIFACT_DIR}'; \
             cd \$HOME/spinifex-e2e-bin; \
             R=\$'\033[0m' B=\$'\033[1m' C=\$'\033[36m' G=\$'\033[32m' RED=\$'\033[31m'; \
             FAIL=0; \
             for SUITE in ${SUITES_RUN}; do \
               echo \"::group::suite \$SUITE\"; \
               printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
               # Record suite wall-clock start so the analyzer can locate
               # the right journal window. junit-report's timestamp attr
               # marks conversion time, not test start, so we cannot use it.
               date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
               GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                 ARTIFACT_DIR='${ARTIFACT_DIR}' \
                 ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
                 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \
                      | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
               rc=\${PIPESTATUS[0]}; \
               echo \"::endgroup::\"; \
               if [ \$rc -ne 0 ]; then \
                 printf '%s✗%s suite %s failed (exit %d)\n' \"\$RED\" \"\$R\" \"\$SUITE\" \"\$rc\"; \
                 echo \"::error::suite \$SUITE failed (exit \$rc)\"; \
                 echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; \
                 FAIL=1; \
               else \
                 printf '%s✓%s suite %s passed\n' \"\$G\" \"\$R\" \"\$SUITE\"; \
               fi; \
             done; \
             exit \$FAIL"
          E2E_RC=$?
          echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
          exit 0  # don't fail the step — artifact upload + final gate happen below

      - name: Capture network/IPsec diagnostics on VM
        if: always()
        run: |
          set +e
          # shellcheck source=/dev/null
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
          # Ship the diag script, run it into the VM's ARTIFACT_DIR, leave the
          # output for the next step's tar to pull down. Captures OVN/OVS/IPsec
          # state at failure time — before teardown destroys the VM.
          scp "${SSH_OPTS[@]}" \
            "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
            "$VM":/tmp/capture-net-diag.sh
          ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag'"
          ok "net diagnostics captured"

      - name: Collect artifacts from VM
        if: always()
        run: |
          set +e
          # shellcheck source=/dev/null
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          mkdir -p "${ARTIFACT_DIR}"
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
          # `|| true` so a missing dir on an early failure doesn't mask the
          # original error.
          ssh "${SSH_OPTS[@]}" "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
          ok "artifacts pulled to ${ARTIFACT_DIR}"
          # `-o short-iso` writes RFC3339-ish timestamps (e.g.
          # `2026-05-19T12:32:34+1000 host …`) so the analyzer can window-slice
          # the journal around each failure's wall-clock span.
          ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
            > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null
          ok "journal: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || echo 0) lines"

      - name: Convert test logs to JUnit XML
        if: always()
        run: |
          # VM runs the pre-built test binaries with `-test.v` and tees plain
          # text logs (no Go toolchain on the VM to emit test2json). Convert
          # each log to the junit-<suite>.xml shape "Publish test summary"
          # already parses.
          shopt -s nullglob
          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
            BASE=$(basename "$LOG" .log)
            SUITE=${BASE#test-}
            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
          done

      - name: Publish test summary
        if: always()
        uses: ./spinifex/.github/actions/junit-report
        with:
          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
          title: "Go E2E results"

      - name: Analyze e2e failures
        if: always()
        uses: ./spinifex/.github/actions/e2e-analyze
        with:
          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
          log-dir: ${{ env.ARTIFACT_DIR }}
          title: "E2E failure analysis"

      - uses: actions/upload-artifact@v7
        if: always()
        with:
          name: e2e-go-artifacts-${{ github.run_id }}
          path: ${{ env.ARTIFACT_DIR }}
          if-no-files-found: warn
          retention-days: 14

      - name: Teardown
        if: always()
        working-directory: mulga/scripts/tofu-cluster
        # `set -e` off — teardown must run scrub-env even if tofu destroy
        # fails; scrub is the libvirt-level fallback when tofu state is
        # inconsistent with the remote Proxmox node.
        run: |
          set +e
          # shellcheck source=/dev/null
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          # Kill any test binaries still running on the VM (cancelled mid-suite
          # paths) BEFORE tofu destroy yanks the qcow2 from underneath them.
          # Scoped to this run's VM by wan_ip — safe with sibling agents on
          # the same host. ConnectTimeout=5 so an already-wedged VM doesn't
          # block teardown.
          if [ -n "${{ steps.vm.outputs.wan_ip }}" ] && [ -n "${{ steps.vm.outputs.ssh_key }}" ]; then
            ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
              -i "${{ steps.vm.outputs.ssh_key }}" \
              tf-user@${{ steps.vm.outputs.wan_ip }} \
              'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
              2>/dev/null || true
          fi
          # Cancelled jobs from this point on can orphan an in-flight
          # tofu/libvirt SSH; clear them before destroy so the next run's
          # pre-clean isn't doing it. Scoped to this runner's workspace via
          # /proc/$pid/cwd — sibling runners share the tf-user UID, so a
          # UID-wide pkill would nuke their tofu mid-apply.
          UID_SELF=$(id -u)
          WS="${GITHUB_WORKSPACE}"
          reap() {
            local sig="$1"
            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
              case "$cwd" in
                "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;;
              esac
            done
          }
          reap TERM
          sleep 2
          reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
          ok "teardown complete"

      - name: Fail job if any suite failed
        if: always()
        run: |
          if [ "${{ steps.e2e.outputs.rc }}" != "0" ] || [ -s "${ARTIFACT_DIR}/suite-status.txt" ]; then
            echo "::error::Go E2E suites failed (rc=${{ steps.e2e.outputs.rc }})"
            cat "${ARTIFACT_DIR}/suite-status.txt" 2>/dev/null || true
            exit 1
          fi

  e2e_go_multinode:
    name: Multi-Node E2E (Go)
    # Opt-in via e2e-multinode in the SUITES list. Runs the Go multinode
    # suite (port of run-multinode-e2e.sh) plus cert + lb against a 3-node
    # tofu cluster. Mirrors e2e_go_single's binary-ship + per-suite flow,
    # adapted for the ci-multi runner pool and SPINIFEX_NODES env wiring.
    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-multinode')
    runs-on: [ self-hosted, ci-multi ]
    timeout-minutes: 60
    env:
      TOFU_LOG: /tmp/e2e-go-tofu-multi-${{ github.run_id }}.log
      ARTIFACT_DIR: /tmp/spinifex-e2e-multi-artifacts-${{ github.run_id }}
    steps:
      - name: Reap Orphan Procs
        run: |
          set +e
          UID_SELF=$(id -u)
          WS="${GITHUB_WORKSPACE}"
          reap() {
            local sig="$1"
            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
              case "$cwd" in
                "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
              esac
            done
          }
          reap TERM
          sleep 3
          reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          true

      - name: Clean Workspace
        run: sudo rm -rf "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        id: app-token
        with:
          client-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

      - uses: actions/checkout@v6
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

      - uses: actions/checkout@v6
        with:
          path: spinifex

      - uses: actions/setup-go@v6
        with:
          go-version-file: spinifex/go.mod
          cache: false

      - name: Re-enable Go toolchain auto-download
        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"

      - name: Prune rootless docker cache
        # Self-hosted runner uses rootless docker; build-microvm-image.sh
        # leaks ~100 MiB overlay2 layer per failed build. Prune before each
        # run keeps /home/<ciuser>/.local/share/docker bounded.
        run: |
          docker container prune -f || true
          docker image prune -af || true
          docker buildx prune -af --filter "until=24h" || true
          docker system df || true

      - name: Build Install Artifacts
        working-directory: mulga/scripts/tofu-cluster
        run: |
          ./build-install-artifacts.sh "${GIT_BRANCH}" \
            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
            "${RUNNER_TEMP}/setup.sh" \
            "${RUNNER_TEMP}/spinifex-tests.tar"

      - name: Provision VMs
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set -euo pipefail
          # shellcheck source=/dev/null
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          export TF_VAR_git_branch="${GIT_BRANCH}"
          quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
          quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
          quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
          quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          ok "3-node cluster provisioned"

      - name: Bootstrap via Binary Install
        working-directory: mulga/scripts/tofu-cluster
        run: |
          SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
            "${RUNNER_TEMP}/setup.sh" \
            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"

      - name: Resolve cluster SSH details
        id: cluster
        working-directory: mulga/scripts/tofu-cluster
        run: |
          INVENTORY=$(tofu output -json inventory)
          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
          # Comma-separated node list for SPINIFEX_NODES (ClusterFromEnv)
          # and SPINIFEX_NODE_IPS (Env.NodeIPs). Both consume the same csv.
          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
          NODES_CSV=""
          for i in $(seq 0 $((NODE_COUNT - 1))); do
            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
            if [ -z "$NODES_CSV" ]; then NODES_CSV="$ip"; else NODES_CSV="$NODES_CSV,$ip"; fi
          done
          echo "node1_ip=${NODE1_IP}" >> "$GITHUB_OUTPUT"
          echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
          echo "nodes_csv=${NODES_CSV}" >> "$GITHUB_OUTPUT"

      - name: Build E2E test binaries on runner
        working-directory: spinifex/tests/e2e
        run: |
          set -euo pipefail
          # shellcheck source=/dev/null
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          # Strip e2e- prefix, drop reboot (runner-resident) and single
          # (single-VM job only) so the multinode job builds only what it
          # ships: multinode, cert, lb.
          SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|single)$' | tr '\n' ' ')
          if [ -z "$(echo "${SUITES_BUILD}" | tr -d ' ')" ]; then
            echo "no multinode-eligible suites requested (SUITES=${SUITES}); nothing to build"
            exit 0
          fi
          banner "build ${SUITES_BUILD// /, }"
          group_open "go test -c"
          make e2e-build SUITES_BUILD="${SUITES_BUILD}"
          group_close
          for f in _bin/*.test; do
            ok "$(basename "$f") $(stat -c '%s' "$f" | numfmt --to=iec)"
          done

      - name: Install go-junit-report on runner
        run: |
          go install github.com/jstemmer/go-junit-report/v2@latest
          echo "$HOME/go/bin" >> "$GITHUB_PATH"

      - name: Ship test binaries to node1
        run: |
          set -euo pipefail
          # shellcheck source=/dev/null
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
          VM=tf-user@${{ steps.cluster.outputs.node1_ip }}
          ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin"
          tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin"
          ok "shipped binaries to $VM"

      - name: Pre-flight diagnostics on node1
        # Captures cluster + tf-user state directly via SSH so the post-mortem
        # in run 26163994815-style spx-NATS failures has structured data
        # before the test binary runs. Best-effort (|| true on everything) —
        # this step never fails the job, only feeds the artifact bundle.
        run: |
          set +e
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" 2>/dev/null || true
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
          VM=tf-user@${{ steps.cluster.outputs.node1_ip }}
          mkdir -p "${ARTIFACT_DIR}"
          DIAG="${ARTIFACT_DIR}/preflight-diag.log"
          {
            echo "=== id (tf-user groups) ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'id; echo; getent group spinifex || true'
            echo
            echo "=== ls -la /etc/spinifex/ ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'ls -la /etc/spinifex/ 2>&1; echo; ls -la /etc/spinifex/nats/ 2>&1'
            echo
            echo "=== spinifex.toml NATS section ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'grep -A4 "\[nodes\\..*\\.nats" /etc/spinifex/spinifex.toml 2>&1 | head -40'
            echo
            echo "=== systemctl is-active spinifex-* ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'for s in spinifex-nats spinifex-predastore spinifex-awsgw spinifex-vpcd spinifex-daemon; do printf "%-22s %s\n" "$s" "$(systemctl is-active $s 2>&1)"; done'
            echo
            echo "=== which spx + version ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'which spx; spx --version 2>&1 || true'
            echo
            echo "=== spx get nodes (direct ssh, tf-user) ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'spx get nodes --timeout 5s 2>&1; echo "exit=$?"'
            echo
            echo "=== spx top nodes (direct ssh, tf-user) ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'spx top nodes 2>&1 | head -30; echo "exit=$?"'
            echo
            echo "=== NATS routez (curl 127.0.0.1:8222) ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'curl -fsS http://127.0.0.1:8222/routez 2>&1 | head -40'
            echo
            echo "=== aws describe-instance-types via gateway (tf-user) ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'AWS_PROFILE=spinifex AWS_CA_BUNDLE=/etc/spinifex/ca.pem aws --endpoint-url https://${{ steps.cluster.outputs.node1_ip }}:9999 ec2 describe-instance-types --query "InstanceTypes[*].InstanceType" --output text 2>&1 | head -5'
          } 2>&1 | tee "$DIAG"
          true

      - name: Run Go E2E suites
        id: e2e
        run: |
          set +e
          # multinode + cert + lb only — see SUITES_BUILD comment above.
          SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|single)$' | tr '\n' ' ')
          if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
            echo "no multinode-eligible suites requested (SUITES=${SUITES}); skipping"
            echo "rc=0" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          if [ -n "${TEST_RUN}" ]; then
            TEST_RUN_FLAG="-test.run=${TEST_RUN}"
          else
            TEST_RUN_FLAG=""
          fi
          VM_HOST="tf-user@${{ steps.cluster.outputs.node1_ip }}"
          SSH_KEY="${{ steps.cluster.outputs.ssh_key }}"
          NODES_CSV="${{ steps.cluster.outputs.nodes_csv }}"
          cleanup_remote() {
            ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \
              'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
              2>/dev/null || true
          }
          trap cleanup_remote EXIT INT TERM
          # SPINIFEX_NODES + SPINIFEX_NODE_IPS share the csv; ClusterFromEnv
          # uses the first; cert/lb use Env.NodeIPs for SAN + endpoint maps.
          # SPINIFEX_SSH_KEY default ~/.ssh/tf-user-ap-southeast-2 lines up
          # with the bash peer_ssh path — bootstrap-install.sh writes it.
          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" \
            "$VM_HOST" \
            "set -u; mkdir -p '${ARTIFACT_DIR}'; \
             cd \$HOME/spinifex-e2e-bin; \
             R=\$'\033[0m' B=\$'\033[1m' C=\$'\033[36m' G=\$'\033[32m' RED=\$'\033[31m'; \
             FAIL=0; \
             for SUITE in ${SUITES_RUN}; do \
               echo \"::group::suite \$SUITE\"; \
               printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
               date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
               GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
                 SPINIFEX_NODES='${NODES_CSV}' \
                 SPINIFEX_NODE_IPS='${NODES_CSV}' \
                 SPINIFEX_SSH_USER=tf-user \
                 SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
                 ARTIFACT_DIR='${ARTIFACT_DIR}' \
                 ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
                 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \
                      | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
               rc=\${PIPESTATUS[0]}; \
               echo \"::endgroup::\"; \
               if [ \$rc -ne 0 ]; then \
                 printf '%s✗%s suite %s failed (exit %d)\n' \"\$RED\" \"\$R\" \"\$SUITE\" \"\$rc\"; \
                 echo \"::error::suite \$SUITE failed (exit \$rc)\"; \
                 echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; \
                 FAIL=1; \
               else \
                 printf '%s✓%s suite %s passed\n' \"\$G\" \"\$R\" \"\$SUITE\"; \
               fi; \
             done; \
             exit \$FAIL"
          E2E_RC=$?
          echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
          exit 0

      - name: Capture network/IPsec diagnostics on all nodes
        if: always()
        run: |
          set +e
          # shellcheck source=/dev/null
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
          # Capture per-node into the node1 ARTIFACT_DIR (where Collect pulls
          # from). One subdir per node so xfrm SA state is comparable across
          # peers — critical for diagnosing the IPsec wiring.
          IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
          NODE1_IP="${NODE_LIST[0]}"
          scp "${SSH_OPTS[@]}" \
            "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
            tf-user@"$NODE1_IP":/tmp/capture-net-diag.sh
          for i in "${!NODE_LIST[@]}"; do
            idx=$((i + 1))
            ip="${NODE_LIST[$i]}"
            if [ "$idx" -ne 1 ]; then
              scp "${SSH_OPTS[@]}" \
                "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
                tf-user@"$ip":/tmp/capture-net-diag.sh
            fi
            ssh "${SSH_OPTS[@]}" tf-user@"$ip" \
              "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag-node${idx}'"
            # node1 keeps its diag locally; pull node2..N back into node1's
            # ARTIFACT_DIR so the existing tar-pull below catches everything.
            if [ "$idx" -ne 1 ]; then
              ssh "${SSH_OPTS[@]}" tf-user@"$ip" \
                "tar -C '${ARTIFACT_DIR}' -cf - 'net-diag-node${idx}' 2>/dev/null || true" \
                | ssh "${SSH_OPTS[@]}" tf-user@"$NODE1_IP" \
                  "tar -C '${ARTIFACT_DIR}' -xf - 2>/dev/null"
            fi
            ok "net diagnostics node${idx} captured"
          done

      - name: Collect artifacts from node1
        if: always()
        run: |
          set +e
          # shellcheck source=/dev/null
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          mkdir -p "${ARTIFACT_DIR}"
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
          VM=tf-user@${{ steps.cluster.outputs.node1_ip }}
          ssh "${SSH_OPTS[@]}" "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
          ok "artifacts pulled to ${ARTIFACT_DIR}"
          # Fan out journal collection across all 3 nodes so a failure
          # signal on node2/3 isn't blind. Naming matches DumpAllNodeLogs.
          IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
          for i in "${!NODE_LIST[@]}"; do
            idx=$((i + 1))
            ip="${NODE_LIST[$i]}"
            ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
              > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
            ok "journal node${idx}: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null || echo 0) lines"
          done
          # openvswitch-ipsec journal per node (separate file).
          # Debian unit name; the binary inside is /usr/share/openvswitch/scripts/ovs-ctl
          # which spawns ovs-monitor-ipsec as a child.
          for i in "${!NODE_LIST[@]}"; do
            idx=$((i + 1))
            ip="${NODE_LIST[$i]}"
            ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \
              > "${ARTIFACT_DIR}/openvswitch-ipsec-node${idx}.log" 2>/dev/null
          done

      - name: Convert test logs to JUnit XML
        if: always()
        run: |
          shopt -s nullglob
          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
            BASE=$(basename "$LOG" .log)
            SUITE=${BASE#test-}
            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
          done

      - name: Publish test summary
        if: always()
        uses: ./spinifex/.github/actions/junit-report
        with:
          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
          title: "Go Multi-Node E2E results"

      - name: Analyze e2e failures
        if: always()
        uses: ./spinifex/.github/actions/e2e-analyze
        with:
          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
          log-dir: ${{ env.ARTIFACT_DIR }}
          title: "Multi-Node E2E failure analysis"

      - uses: actions/upload-artifact@v7
        if: always()
        with:
          name: e2e-go-multi-artifacts-${{ github.run_id }}
          path: ${{ env.ARTIFACT_DIR }}
          if-no-files-found: warn
          retention-days: 14

      - name: Teardown
        if: always()
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set +e
          # shellcheck source=/dev/null
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          if [ -n "${{ steps.cluster.outputs.node1_ip }}" ] && [ -n "${{ steps.cluster.outputs.ssh_key }}" ]; then
            IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
            for ip in "${NODE_LIST[@]}"; do
              ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
                -i "${{ steps.cluster.outputs.ssh_key }}" \
                tf-user@"$ip" \
                'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
                2>/dev/null || true
            done
          fi
          UID_SELF=$(id -u)
          WS="${GITHUB_WORKSPACE}"
          reap() {
            local sig="$1"
            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
              case "$cwd" in
                "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;;
              esac
            done
          }
          reap TERM
          sleep 2
          reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
          ok "teardown complete"

      - name: Fail job if any suite failed
        if: always()
        run: |
          if [ "${{ steps.e2e.outputs.rc }}" != "0" ] || [ -s "${ARTIFACT_DIR}/suite-status.txt" ]; then
            echo "::error::Go multi-node E2E suites failed (rc=${{ steps.e2e.outputs.rc }})"
            cat "${ARTIFACT_DIR}/suite-status.txt" 2>/dev/null || true
            exit 1
          fi

  e2e_go_reboot:
    name: Reboot Resilience E2E (Go)
    # Opt-in: only fires when the SUITES dispatch input lists e2e-reboot.
    # The reboot test reboots the spinifex VM, so it must run on the runner
    # (not shipped to the VM like the cert/lb/single suites). Runs in
    # parallel with e2e_go_single — both target [self-hosted, ci-single]
    # and GitHub Actions serialises on the underlying runner allocation,
    # so no explicit needs: ordering is required.
    if: contains(github.event.inputs.suites, 'e2e-reboot')
    runs-on: [ self-hosted, ci-single ]
    timeout-minutes: 60
    env:
      TOFU_LOG: /tmp/e2e-go-tofu-reboot-${{ github.run_id }}.log
      ARTIFACT_DIR: /tmp/spinifex-e2e-reboot-artifacts-${{ github.run_id }}
    steps:
      - name: Reap Orphan Procs
        run: |
          set +e
          UID_SELF=$(id -u)
          WS="${GITHUB_WORKSPACE}"
          reap() {
            local sig="$1"
            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
              case "$cwd" in
                "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
              esac
            done
          }
          reap TERM
          sleep 3
          reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          true

      - name: Clean Workspace
        run: sudo rm -rf "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        id: app-token
        with:
          client-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

      - uses: actions/checkout@v6
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

      - uses: actions/checkout@v6
        with:
          path: spinifex

      - uses: actions/setup-go@v6
        with:
          go-version-file: spinifex/go.mod
          cache: false

      - name: Re-enable Go toolchain auto-download
        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"

      - name: Prune rootless docker cache
        # Self-hosted runner uses rootless docker; build-microvm-image.sh
        # leaks ~100 MiB overlay2 layer per failed build. Prune before each
        # run keeps /home/<ciuser>/.local/share/docker bounded.
        run: |
          docker container prune -f || true
          docker image prune -af || true
          docker buildx prune -af --filter "until=24h" || true
          docker system df || true

      - name: Build Install Artifacts
        working-directory: mulga/scripts/tofu-cluster
        run: |
          ./build-install-artifacts.sh "${GIT_BRANCH}" \
            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
            "${RUNNER_TEMP}/setup.sh" \
            "${RUNNER_TEMP}/spinifex-tests.tar"

      - name: Provision VM
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set -euo pipefail
          # shellcheck source=/dev/null
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          export TF_VAR_git_branch="${GIT_BRANCH}"
          quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
          quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
          quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
          quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          ok "VM provisioned"

      - name: Bootstrap via Binary Install
        working-directory: mulga/scripts/tofu-cluster
        run: |
          SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
            "${RUNNER_TEMP}/setup.sh" \
            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"

      - name: Resolve VM SSH details
        id: vm
        working-directory: mulga/scripts/tofu-cluster
        run: |
          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
          echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT"
          echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"

      - name: Build reboot.test on runner
        working-directory: spinifex/tests/e2e
        run: |
          set -euo pipefail
          # shellcheck source=/dev/null
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          banner "build reboot"
          group_open "go test -c"
          make e2e-build SUITES_BUILD="reboot"
          group_close
          ok "$(basename _bin/reboot.test) $(stat -c '%s' _bin/reboot.test | numfmt --to=iec)"

      - name: Fetch AWS credentials from VM
        id: creds
        run: |
          set -euo pipefail
          # The spinifex CLI on the VM (via `spx admin init` during bootstrap)
          # writes the [spinifex] profile to tf-user's ~/.aws/credentials.
          # Pull it over SSH and expose as SPINIFEX_AWS_* env vars (the same
          # static-cred path used by harness.NewAWSClient). Mirrors the cred
          # hop already proven by e2e-ddil.yml.
          SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
                    -i "${{ steps.vm.outputs.ssh_key }}")
          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
          CREDS=$(ssh "${SSH_OPTS[@]}" "$VM" 'cat ~/.aws/credentials')
          AWS_AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
          AWS_SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
          if [ -z "$AWS_AK" ] || [ -z "$AWS_SK" ]; then
            echo "::error::Failed to extract spinifex AWS credentials from ${{ steps.vm.outputs.wan_ip }}"
            exit 1
          fi
          echo "::add-mask::${AWS_AK}"
          echo "::add-mask::${AWS_SK}"
          echo "ak=${AWS_AK}" >> "$GITHUB_OUTPUT"
          echo "sk=${AWS_SK}" >> "$GITHUB_OUTPUT"

      - name: Run reboot suite on runner
        id: e2e
        run: |
          set +e
          mkdir -p "${ARTIFACT_DIR}"
          # Banner + capture wall-clock start so the e2e-analyze action can
          # window-slice the journal around the test span (matches the
          # convention in e2e_go_single).
          date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-reboot.start"
          R=$'\033[0m'; B=$'\033[1m'; C=$'\033[36m'
          printf '\n%s%s━━ suite reboot ━━%s\n' "$B" "$C" "$R"

          if [ -n "${TEST_RUN}" ]; then
            TEST_RUN_FLAG="-test.run=${TEST_RUN}"
          else
            TEST_RUN_FLAG=""
          fi
          GITHUB_ACTIONS=true FORCE_COLOR=1 \
            SPINIFEX_E2E=1 SPINIFEX_MODE=single \
            SPINIFEX_WAN_IP="${{ steps.vm.outputs.wan_ip }}" \
            SPINIFEX_SSH_KEY="${{ steps.vm.outputs.ssh_key }}" \
            SPINIFEX_AWS_ENDPOINT="https://${{ steps.vm.outputs.wan_ip }}:9999" \
            SPINIFEX_AWS_ACCESS_KEY_ID="${{ steps.creds.outputs.ak }}" \
            SPINIFEX_AWS_SECRET_ACCESS_KEY="${{ steps.creds.outputs.sk }}" \
            ARTIFACT_DIR="${ARTIFACT_DIR}" \
            ./spinifex/tests/e2e/_bin/reboot.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
            2>&1 | tee "${ARTIFACT_DIR}/test-reboot.log" \
                 | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'
          rc=${PIPESTATUS[0]}
          echo "rc=${rc}" >> "$GITHUB_OUTPUT"
          exit 0

      - name: Collect VM journals
        if: always()
        run: |
          set +e
          # shellcheck source=/dev/null
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          mkdir -p "${ARTIFACT_DIR}"
          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 \
                    -i "${{ steps.vm.outputs.ssh_key }}")
          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
          ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
            > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || true
          ok "journal: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || echo 0) lines"

      - name: Install go-junit-report
        if: always()
        run: |
          go install github.com/jstemmer/go-junit-report/v2@latest
          echo "$HOME/go/bin" >> "$GITHUB_PATH"

      - name: Convert test log to JUnit XML
        if: always()
        run: |
          shopt -s nullglob
          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
            BASE=$(basename "$LOG" .log)
            SUITE=${BASE#test-}
            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
          done

      - name: Publish test summary
        if: always()
        uses: ./spinifex/.github/actions/junit-report
        with:
          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
          title: "Go E2E reboot results"

      - name: Analyze e2e failures
        if: always()
        uses: ./spinifex/.github/actions/e2e-analyze
        with:
          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
          log-dir: ${{ env.ARTIFACT_DIR }}
          title: "Reboot E2E failure analysis"

      - uses: actions/upload-artifact@v7
        if: always()
        with:
          name: e2e-go-reboot-artifacts-${{ github.run_id }}
          path: ${{ env.ARTIFACT_DIR }}
          if-no-files-found: warn
          retention-days: 14

      - name: Teardown
        if: always()
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set +e
          # shellcheck source=/dev/null
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          UID_SELF=$(id -u)
          WS="${GITHUB_WORKSPACE}"
          reap() {
            local sig="$1"
            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
              case "$cwd" in
                "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;;
              esac
            done
          }
          reap TERM
          sleep 2
          reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
          ok "teardown complete"

      - name: Fail job if reboot suite failed
        if: always()
        run: |
          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
            echo "::error::Reboot suite failed (rc=${{ steps.e2e.outputs.rc }})"
            exit 1
          fi

  e2e_go_status:
    name: Report E2E (Go) Status
    # Always post status. When inputs.sha is unset (manual dispatch without
    # a PR head), fall back to github.sha — the resolved commit SHA of the
    # dispatched ref, always present. The previous `&& inputs.sha` gate
    # plus `statuses/${{ inputs.sha }}` produced an 'invalid hash' error
    # whenever the workflow was dispatched without the optional input
    # (tests passed, status step silently failed).
    if: always()
    needs: [ e2e_go_single, e2e_go_multinode, e2e_go_reboot ]
    runs-on: [ self-hosted, ci-single ]
    timeout-minutes: 5
    steps:
      - name: Report Status to PR
        env:
          GH_TOKEN: ${{ github.token }}
          REPORT_SHA: ${{ inputs.sha || github.sha }}
        run: |
          if [[ -z "${REPORT_SHA}" ]]; then
            echo "::error::no SHA available (inputs.sha and github.sha both empty)"
            exit 1
          fi
          # Treat skipped as success — e2e_go_multinode and e2e_go_reboot
          # only run when opted in via the SUITES input, so a single-only
          # dispatch should still post a green status when the single job
          # passes and the others skipped.
          single_ok=true; multi_ok=true; reboot_ok=true
          case "${{ needs.e2e_go_single.result }}" in success|skipped) ;; *) single_ok=false ;; esac
          case "${{ needs.e2e_go_multinode.result }}" in success|skipped) ;; *) multi_ok=false ;; esac
          case "${{ needs.e2e_go_reboot.result }}" in success|skipped) ;; *) reboot_ok=false ;; esac
          if $single_ok && $multi_ok && $reboot_ok; then
            STATE="success"
            DESC="Go E2E suites passed"
          else
            STATE="failure"
            DESC="Go E2E failed: single=${{ needs.e2e_go_single.result }} multi=${{ needs.e2e_go_multinode.result }} reboot=${{ needs.e2e_go_reboot.result }}"
          fi
          gh api "repos/${{ github.repository }}/statuses/${REPORT_SHA}" \
            -f state="$STATE" \
            -f context=e2e-go \
            -f description="$DESC" \
            -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"