spinifex/.github/workflows/e2e.yml

name: E2E

on:
  push:
    branches:
      - main
      - dev
    paths:
      - "**/*.go"
      - go.mod
      - go.sum
      - Makefile
      - tests/e2e/**
      - .github/workflows/e2e.yml
  workflow_dispatch:
    inputs:
      suites:
        description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-eks e2e-iam')"
        required: false
        default: "e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam"
      test_run:
        description: "Optional regex for -test.run (e.g. '^TestMultinodeIPSec$'). Empty = run all tests in suite."
        required: false
        default: ""

permissions:
  contents: read
  statuses: write

env:
  GIT_BRANCH: ${{ github.ref_name }}
  SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam' }}
  TEST_RUN: ${{ inputs.test_run || '' }}
  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
  # empty string disables telemetry entirely.
  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}

jobs:
  # In-process integration front gate. Provisions nothing -- no tofu, no
  # docker, no Spinifex daemons, no fixed ports -- so it runs directly on the
  # runner host rather than inside a provisioned VM, and completes in seconds.
  #
  # It runs on ci-single rather than a GitHub-hosted runner so it shares the
  # toolchain and sibling-repo build of the live suites, and so the run stays
  # entirely on the LAN that hosts the Elastic sink. Holding a ci-single slot
  # for seconds is negligible against the 50-minute e2e_single that follows.
  #
  # e2e_single and e2e_multi both require this job to succeed, so a red
  # integration run is caught before either provisions a tofu env or pays the
  # Spinifex install -- which is the expensive half, not the tofu apply.
  integration:
    name: Integration (front gate)
    runs-on: [self-hosted, ci-single]
    timeout-minutes: 10
    steps:
      # This runner is shared with e2e_single, which leaves root-owned tofu and
      # mount artifacts behind; a plain checkout would fail on them. Unmount
      # anything still bound under the workspace before wiping it.
      - name: Clean Workspace
        run: |
          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*

      - uses: actions/checkout@v7
      - uses: actions/setup-go@v7
        with:
          go-version-file: go.mod
          cache: false
      - name: Run in-process integration tests
        run: make test-integration

  e2e_single:
    name: Single-Node E2E
    needs: [integration]
    # The integration front gate must be green: a red integration run stops
    # here, before this job provisions a tofu env or pays the Spinifex install.
    #
    # The dispatch check gates on exactly the suites this job's Run step
    # allow-lists (single|iam|cert|eks|ecs|storagegrowth). Gating on a suite the
    # job won't run — lb, which only executes in the multinode job — would
    # provision a tofu VM and pay the Spinifex install only to run nothing and
    # tear down. Push triggers keep running via the null-check on inputs.suites.
    if: needs.integration.result == 'success' && (github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-iam') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-ecs') || contains(github.event.inputs.suites, 'e2e-storagegrowth'))
    runs-on: [self-hosted, ci-single]
    timeout-minutes: 50
    env:
      TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
      ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
    steps:
      - name: Reap Orphan Procs
        run: |
          set +e
          UID_SELF=$(id -u)
          WS="${GITHUB_WORKSPACE}"
          reap() {
            local sig="$1"
            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
              case "$cwd" in
                "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
              esac
            done
          }
          reap TERM
          sleep 3
          reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          true

      - name: Clean Workspace
        run: |
          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
          # ARTIFACT_DIR lives in /tmp, outside the workspace, so a re-run on
          # the same self-hosted runner reuses it. Wipe it so a prior attempt's
          # stale suite-status.txt / analysis dirs can't leak into this attempt.
          rm -rf "${ARTIFACT_DIR}"

      - uses: actions/create-github-app-token@v3
        id: app-token
        with:
          client-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

      - uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

      - uses: actions/checkout@v7
        with:
          path: spinifex

      - uses: actions/setup-go@v7
        with:
          go-version-file: spinifex/go.mod
          cache: false

      - name: Re-enable Go toolchain auto-download
        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"

      - name: Reclaim docker build cache
        run: |
          docker container prune -f || true
          docker image prune -f || true
          docker buildx prune -af --filter "until=24h" || true
          docker system df || true

      - name: Build Install Artifacts
        working-directory: mulga/scripts/tofu-cluster
        run: |
          ./build-install-artifacts.sh "${GIT_BRANCH}" \
            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
            "${RUNNER_TEMP}/setup.sh" \
            "${RUNNER_TEMP}/spinifex-tests.tar"

      - name: Provision VM
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set -euo pipefail
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          export TF_VAR_git_branch="${GIT_BRANCH}"
          quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
          quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
          quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
          quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          ok "VM provisioned"

      - name: Bootstrap via Binary Install
        working-directory: mulga/scripts/tofu-cluster
        run: |
          SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
            "${RUNNER_TEMP}/setup.sh" \
            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"

      - name: Resolve VM SSH details
        id: vm
        working-directory: mulga/scripts/tofu-cluster
        run: |
          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
          echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT"
          echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"

      # The eks suite's CreateCluster boots a control-plane VM from the
      # eks-node AMI, so that AMI must exist in the VM's image store before the
      # suite runs. build-system-image.sh customizes the image with libguestfs
      # (an isolated userspace appliance) — it touches no host block device, so
      # it is safe on the shared runner. The build needs the spinifex source +
      # network (runner only); the import needs predastore/NATS (VM only), so
      # build on the runner then ship the raw and import on the VM.
      - name: Build eks-node AMI (libguestfs, runner)
        if: contains(env.SUITES, 'e2e-eks')
        working-directory: spinifex
        run: |
          set -euo pipefail
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          banner "build eks-node AMI (libguestfs — no host devices)"
          sudo apt-get update -qq
          sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
          # Debian/Ubuntu ship /boot/vmlinuz-* mode 0600; the libguestfs
          # appliance needs to read a host kernel. direct backend = no libvirt.
          export LIBGUESTFS_BACKEND=direct
          sudo chmod 0644 /boot/vmlinuz-* || true
          group_open "make build-eks-node-image"
          make build-eks-node-image
          group_close
          ok "raw: $(du -h /tmp/eks-node-image-build/eks-node-alpine.raw | cut -f1)"

      - name: Import eks-node AMI on VM
        if: contains(env.SUITES, 'e2e-eks')
        run: |
          set -euo pipefail
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
            spinifex/scripts/images/eks-node/manifest.conf)"
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
          banner "import eks-node AMI to VM image store"
          scp "${SSH_OPTS[@]}" \
            /tmp/eks-node-image-build/eks-node-alpine.raw \
            "$VM":/tmp/eks-node-alpine.raw
          ssh "${SSH_OPTS[@]}" "$VM" \
            "spx admin images import --file /tmp/eks-node-alpine.raw \
               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1
          ok "eks-node AMI imported"

      # The eks data-plane subtest (TestEKS/KubectlGetNodes) drives the full
      # `aws eks get-token` → kubectl → apiserver → token-webhook → gateway
      # broker → STS verify path. The e2e binaries run ON the VM, so kubectl
      # must be on the VM's PATH or harness.NewKubectl t.Skips the subtest.
      # Pin to the k3s server version (setup.sh K3S_VERSION) — kubectl is
      # ±1-minor skew tolerant; matching is simplest. Checksum-verified.
      - name: Install kubectl on VM
        if: contains(env.SUITES, 'e2e-eks')
        run: |
          set -euo pipefail
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
          banner "install kubectl on VM (eks data-plane subtest)"
          ssh "${SSH_OPTS[@]}" "$VM" 'bash -s' <<'EOF'
          set -euo pipefail
          KVER=v1.32.5
          case "$(uname -m)" in
            x86_64)  ARCH=amd64 ;;
            aarch64) ARCH=arm64 ;;
            *) echo "unsupported arch $(uname -m)" >&2; exit 1 ;;
          esac
          cd /tmp
          curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl"
          curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl.sha256"
          echo "$(cat kubectl.sha256)  kubectl" | sha256sum -c -
          sudo install -m 0755 kubectl /usr/local/bin/kubectl
          kubectl version --client
          EOF
          ok "kubectl installed on VM"

      - name: Build E2E test binaries on runner
        working-directory: spinifex/tests/e2e
        run: |
          set -euo pipefail
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(multinode)$' | tr '\n' ' ')
          if [ -z "$(echo "${SUITES_BUILD}" | tr -d ' ')" ]; then
            echo "no in-VM suites requested (SUITES=${SUITES}); nothing to build"
            exit 0
          fi
          banner "build ${SUITES_BUILD// /, }"
          group_open "go test -c"
          make e2e-build SUITES_BUILD="${SUITES_BUILD}"
          group_close
          for f in _bin/*.test; do
            ok "$(basename "$f") $(stat -c '%s' "$f" | numfmt --to=iec)"
          done

      - name: Install go-junit-report on runner
        run: |
          go install github.com/jstemmer/go-junit-report/v2@latest
          echo "$HOME/go/bin" >> "$GITHUB_PATH"

      - name: Ship test binaries to VM
        run: |
          set -euo pipefail
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
          ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin"
          tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin"
          ok "shipped binaries to $VM"

      - name: Start packet capture on VM br-wan
        run: |
          set +e
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
          scp "${SSH_OPTS[@]}" \
            "${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \
            "$VM":/tmp/start-net-capture.sh
          ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/start-net-capture.sh" || true

      - name: Run E2E suites
        id: e2e
        run: |
          set +e
          # Allow-list, not a deny-list: a suite runs here only if named. Deny-lists
          # silently fan a newly added suite out to every job, which is how lb ended up
          # running twice per e2e run.
          SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE 'single|iam|cert|eks|ecs|storagegrowth' | tr '\n' ' ')
          if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
            echo "no in-VM suites requested (SUITES=${SUITES}); skipping"
            echo "rc=0" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          if [ -n "${TEST_RUN}" ]; then
            # The value is expanded into a command that a remote shell re-parses over
            # ssh, so single-quote it — otherwise regex metacharacters like ( | ) $ are
            # taken as shell syntax on the VM and bash fails before the test binary runs.
            TEST_RUN_FLAG="-test.run='${TEST_RUN}'"
          else
            TEST_RUN_FLAG=""
          fi
          VM_HOST="tf-user@${{ steps.vm.outputs.wan_ip }}"
          SSH_KEY="${{ steps.vm.outputs.ssh_key }}"
          cleanup_remote() {
            ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \
              'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
              2>/dev/null || true
          }
          trap cleanup_remote EXIT INT TERM
          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" \
            "$VM_HOST" \
            "set -u; mkdir -p '${ARTIFACT_DIR}'; \
             cd \$HOME/spinifex-e2e-bin; \
             R=\$'\033[0m' B=\$'\033[1m' C=\$'\033[36m' G=\$'\033[32m' RED=\$'\033[31m'; \
             FAIL=0; \
             for SUITE in ${SUITES_RUN}; do \
               echo \"::group::suite \$SUITE\"; \
               printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
               date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
               GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                 ARTIFACT_DIR='${ARTIFACT_DIR}' \
                 ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
                 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \
                      | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
               rc=\${PIPESTATUS[0]}; \
               echo \"::endgroup::\"; \
               if [ \$rc -ne 0 ]; then \
                 printf '%s✗%s suite %s failed (exit %d)\n' \"\$RED\" \"\$R\" \"\$SUITE\" \"\$rc\"; \
                 echo \"::error::suite \$SUITE failed (exit \$rc)\"; \
                 echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; \
                 FAIL=1; \
               else \
                 printf '%s✓%s suite %s passed\n' \"\$G\" \"\$R\" \"\$SUITE\"; \
               fi; \
             done; \
             exit \$FAIL"
          E2E_RC=$?
          echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
          exit 0

      - name: Capture network/IPsec diagnostics on VM
        if: always()
        run: |
          set +e
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
          scp "${SSH_OPTS[@]}" \
            "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
            "$VM":/tmp/capture-net-diag.sh
          ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag'"
          ok "net diagnostics captured"

      - name: Collect artifacts from VM
        if: always()
        run: |
          set +e
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          mkdir -p "${ARTIFACT_DIR}"
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
          ssh "${SSH_OPTS[@]}" "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
          ok "artifacts pulled to ${ARTIFACT_DIR}"
          ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
            > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null
          ok "journal: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || echo 0) lines"
          # LB/system microVM serial consoles. The lb-agent logs its nginx (NLB)
          # engine activation and any `nginx -t`/reload error + active-probe
          # results to the guest serial console, which the host journal never
          # captures. These files persist in /run/spinifex past VM teardown, so
          # collecting them here makes an NLB "0/N healthy" timeout diagnosable.
          mkdir -p "${ARTIFACT_DIR}/lb-console"
          ssh "${SSH_OPTS[@]}" "$VM" 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
            | tar -C "${ARTIFACT_DIR}/lb-console" -xf - 2>/dev/null
          ok "lb-console: $(ls "${ARTIFACT_DIR}/lb-console" 2>/dev/null | wc -l) files"

      - name: Convert test logs to JUnit XML
        if: always()
        run: |
          shopt -s nullglob
          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
            BASE=$(basename "$LOG" .log)
            SUITE=${BASE#test-}
            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
          done

      - name: Publish test summary
        if: always()
        uses: ./spinifex/.github/actions/junit-report
        with:
          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
          title: "E2E results"

      - name: Analyze e2e failures
        if: always()
        uses: ./spinifex/.github/actions/e2e-analyze
        with:
          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
          log-dir: ${{ env.ARTIFACT_DIR }}
          title: "E2E failure analysis"

      - uses: actions/upload-artifact@v7
        if: always()
        with:
          name: e2e-artifacts-${{ github.run_id }}
          path: ${{ env.ARTIFACT_DIR }}
          if-no-files-found: warn
          retention-days: 14

      - name: Teardown
        if: always()
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set +e
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          if [ -n "${{ steps.vm.outputs.wan_ip }}" ] && [ -n "${{ steps.vm.outputs.ssh_key }}" ]; then
            ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
              -i "${{ steps.vm.outputs.ssh_key }}" \
              tf-user@${{ steps.vm.outputs.wan_ip }} \
              'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
              2>/dev/null || true
          fi
          UID_SELF=$(id -u)
          WS="${GITHUB_WORKSPACE}"
          reap() {
            local sig="$1"
            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
              case "$cwd" in
                "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;;
              esac
            done
          }
          reap TERM
          sleep 2
          reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
          ok "teardown complete"

      - name: Upload tofu log
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: tofu-log-single-${{ github.run_id }}
          path: ${{ env.TOFU_LOG }}
          if-no-files-found: ignore
          retention-days: 14

      - name: Fail job if any suite failed
        if: always()
        run: |
          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
            echo "::error::E2E suites failed (rc=${{ steps.e2e.outputs.rc }})"
            cat "${ARTIFACT_DIR}/suite-status.txt" 2>/dev/null || true
            exit 1
          fi

  e2e_multi:
    name: Multi-Node E2E
    needs: [integration]
    # Gated on the integration front gate for the same reason as e2e_single:
    # the 3-node env is never stood up if the control plane is already broken.
    # The dispatch check gates on exactly the suites this job's Run step
    # allow-lists (multinode|lb|cert) so lb — which only runs here — still
    # provisions the 3-node env when requested on its own, and the env stays
    # unprovisioned when none of them are asked for.
    if: needs.integration.result == 'success' && (github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-multinode') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-cert'))
    runs-on: [self-hosted, ci-multi]
    timeout-minutes: 60
    env:
      TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
      ARTIFACT_DIR: /tmp/spinifex-e2e-multi-artifacts-${{ github.run_id }}
    steps:
      - name: Reap Orphan Procs
        run: |
          set +e
          UID_SELF=$(id -u)
          WS="${GITHUB_WORKSPACE}"
          reap() {
            local sig="$1"
            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
              case "$cwd" in
                "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
              esac
            done
          }
          reap TERM
          sleep 3
          reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          true

      - name: Clean Workspace
        run: |
          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
          # ARTIFACT_DIR lives in /tmp, outside the workspace, so a re-run on
          # the same self-hosted runner reuses it. Wipe it so a prior attempt's
          # stale suite-status.txt / analysis dirs can't leak into this attempt.
          rm -rf "${ARTIFACT_DIR}"

      - uses: actions/create-github-app-token@v3
        id: app-token
        with:
          client-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

      - uses: actions/checkout@v7
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

      - uses: actions/checkout@v7
        with:
          path: spinifex

      - uses: actions/setup-go@v7
        with:
          go-version-file: spinifex/go.mod
          cache: false

      - name: Re-enable Go toolchain auto-download
        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"

      - name: Reclaim docker build cache
        run: |
          docker container prune -f || true
          docker image prune -f || true
          docker buildx prune -af --filter "until=24h" || true
          docker system df || true

      - name: Build Install Artifacts
        working-directory: mulga/scripts/tofu-cluster
        run: |
          ./build-install-artifacts.sh "${GIT_BRANCH}" \
            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
            "${RUNNER_TEMP}/setup.sh" \
            "${RUNNER_TEMP}/spinifex-tests.tar"

      - name: Provision VMs
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set -euo pipefail
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          export TF_VAR_git_branch="${GIT_BRANCH}"
          quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
          quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
          quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
          quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          ok "3-node cluster provisioned"

      - name: Bootstrap via Binary Install
        working-directory: mulga/scripts/tofu-cluster
        run: |
          SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
            "${RUNNER_TEMP}/setup.sh" \
            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"

      - name: Resolve cluster SSH details
        id: cluster
        working-directory: mulga/scripts/tofu-cluster
        run: |
          INVENTORY=$(tofu output -json inventory)
          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
          NODES_CSV=""
          for i in $(seq 0 $((NODE_COUNT - 1))); do
            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
            if [ -z "$NODES_CSV" ]; then NODES_CSV="$ip"; else NODES_CSV="$NODES_CSV,$ip"; fi
          done
          echo "node1_ip=${NODE1_IP}" >> "$GITHUB_OUTPUT"
          echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
          echo "nodes_csv=${NODES_CSV}" >> "$GITHUB_OUTPUT"

      - name: Build E2E test binaries on runner
        working-directory: spinifex/tests/e2e
        run: |
          set -euo pipefail
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          # single + iam are single-node-only suites — their fixtures skip
          # unless SPINIFEX_MODE=single, so building/running them on multinode
          # is wasted work.
          # eks is excluded on multinode: EKS KV is a clustered (Raft) JetStream
          # stream and create-cluster needs an elected leader, but multinode
          # JetStream clustering isn't solid yet (mulga-omq/z2z/5yg). EKS is
          # validated single-node only for now; re-enable here once clustered
          # JetStream + multinode EKS are a deliberate target.
          SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(single|eks|iam)$' | tr '\n' ' ')
          if [ -z "$(echo "${SUITES_BUILD}" | tr -d ' ')" ]; then
            echo "no multinode-eligible suites requested (SUITES=${SUITES}); nothing to build"
            exit 0
          fi
          banner "build ${SUITES_BUILD// /, }"
          group_open "go test -c"
          make e2e-build SUITES_BUILD="${SUITES_BUILD}"
          group_close
          for f in _bin/*.test; do
            ok "$(basename "$f") $(stat -c '%s' "$f" | numfmt --to=iec)"
          done

      - name: Install go-junit-report on runner
        run: |
          go install github.com/jstemmer/go-junit-report/v2@latest
          echo "$HOME/go/bin" >> "$GITHUB_PATH"

      - name: Ship test binaries to node1
        run: |
          set -euo pipefail
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
          VM=tf-user@${{ steps.cluster.outputs.node1_ip }}
          ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin"
          tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin"
          ok "shipped binaries to $VM"

      - name: Start packet capture on all nodes br-wan
        run: |
          set +e
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
          IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
          for ip in "${NODE_LIST[@]}"; do
              [ -z "$ip" ] && continue
              scp "${SSH_OPTS[@]}" \
                  "${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \
                  tf-user@"$ip":/tmp/start-net-capture.sh
              ssh "${SSH_OPTS[@]}" tf-user@"$ip" "bash /tmp/start-net-capture.sh" || true
          done

      - name: Pre-flight diagnostics on node1
        run: |
          set +e
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" 2>/dev/null || true
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
          VM=tf-user@${{ steps.cluster.outputs.node1_ip }}
          mkdir -p "${ARTIFACT_DIR}"
          DIAG="${ARTIFACT_DIR}/preflight-diag.log"
          {
            echo "=== id (tf-user groups) ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'id; echo; getent group spinifex || true'
            echo
            echo "=== ls -la /etc/spinifex/ ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'ls -la /etc/spinifex/ 2>&1; echo; ls -la /etc/spinifex/nats/ 2>&1'
            echo
            echo "=== spinifex.toml NATS section ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'grep -A4 "\[nodes\\..*\\.nats" /etc/spinifex/spinifex.toml 2>&1 | head -40'
            echo
            echo "=== systemctl is-active spinifex-* ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'for s in spinifex-nats spinifex-predastore spinifex-awsgw spinifex-vpcd spinifex-daemon; do printf "%-22s %s\n" "$s" "$(systemctl is-active $s 2>&1)"; done'
            echo
            echo "=== spx get nodes (direct ssh, tf-user) ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'spx get nodes --timeout 5s 2>&1; echo "exit=$?"'
            echo
            echo "=== spx top nodes (direct ssh, tf-user) ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'spx top nodes 2>&1 | head -30; echo "exit=$?"'
            echo
            echo "=== NATS routez (curl 127.0.0.1:8222) ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'curl -fsS http://127.0.0.1:8222/routez 2>&1 | head -40'
            echo
            echo "=== aws describe-instance-types via gateway (tf-user) ==="
            ssh "${SSH_OPTS[@]}" "$VM" 'AWS_PROFILE=spinifex AWS_CA_BUNDLE=/etc/spinifex/ca.pem aws --endpoint-url https://${{ steps.cluster.outputs.node1_ip }}:9999 ec2 describe-instance-types --query "InstanceTypes[*].InstanceType" --output text 2>&1 | head -5'
          } 2>&1 | tee "$DIAG"
          true

      - name: Run E2E suites
        id: e2e
        run: |
          set +e
          # Allow-list, not a deny-list — see the single-node job for why.
          SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE 'multinode|lb|cert' | tr '\n' ' ')
          if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
            echo "no multinode-eligible suites requested (SUITES=${SUITES}); skipping"
            echo "rc=0" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          if [ -n "${TEST_RUN}" ]; then
            # Single-quote for the remote shell — see the single-node job for the
            # metacharacter-injection reason.
            TEST_RUN_FLAG="-test.run='${TEST_RUN}'"
          else
            TEST_RUN_FLAG=""
          fi
          VM_HOST="tf-user@${{ steps.cluster.outputs.node1_ip }}"
          SSH_KEY="${{ steps.cluster.outputs.ssh_key }}"
          NODES_CSV="${{ steps.cluster.outputs.nodes_csv }}"
          cleanup_remote() {
            ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \
              'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
              2>/dev/null || true
          }
          trap cleanup_remote EXIT INT TERM
          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" \
            "$VM_HOST" \
            "set -u; mkdir -p '${ARTIFACT_DIR}'; \
             cd \$HOME/spinifex-e2e-bin; \
             R=\$'\033[0m' B=\$'\033[1m' C=\$'\033[36m' G=\$'\033[32m' RED=\$'\033[31m'; \
             FAIL=0; \
             for SUITE in ${SUITES_RUN}; do \
               echo \"::group::suite \$SUITE\"; \
               printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
               date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
               GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
                 SPINIFEX_NODES='${NODES_CSV}' \
                 SPINIFEX_NODE_IPS='${NODES_CSV}' \
                 SPINIFEX_SSH_USER=tf-user \
                 SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
                 ARTIFACT_DIR='${ARTIFACT_DIR}' \
                 ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
                 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \
                      | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
               rc=\${PIPESTATUS[0]}; \
               echo \"::endgroup::\"; \
               if [ \$rc -ne 0 ]; then \
                 printf '%s✗%s suite %s failed (exit %d)\n' \"\$RED\" \"\$R\" \"\$SUITE\" \"\$rc\"; \
                 echo \"::error::suite \$SUITE failed (exit \$rc)\"; \
                 echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; \
                 FAIL=1; \
               else \
                 printf '%s✓%s suite %s passed\n' \"\$G\" \"\$R\" \"\$SUITE\"; \
               fi; \
             done; \
             exit \$FAIL"
          E2E_RC=$?
          echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
          exit 0

      - name: Capture network/IPsec diagnostics on all nodes
        if: always()
        run: |
          set +e
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
          IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
          NODE1_IP="${NODE_LIST[0]}"
          scp "${SSH_OPTS[@]}" \
            "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
            tf-user@"$NODE1_IP":/tmp/capture-net-diag.sh
          for i in "${!NODE_LIST[@]}"; do
            idx=$((i + 1))
            ip="${NODE_LIST[$i]}"
            if [ "$idx" -ne 1 ]; then
              scp "${SSH_OPTS[@]}" \
                "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
                tf-user@"$ip":/tmp/capture-net-diag.sh
            fi
            ssh "${SSH_OPTS[@]}" tf-user@"$ip" \
              "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag-node${idx}'"
            if [ "$idx" -ne 1 ]; then
              ssh "${SSH_OPTS[@]}" tf-user@"$ip" \
                "tar -C '${ARTIFACT_DIR}' -cf - 'net-diag-node${idx}' 2>/dev/null || true" \
                | ssh "${SSH_OPTS[@]}" tf-user@"$NODE1_IP" \
                  "tar -C '${ARTIFACT_DIR}' -xf - 2>/dev/null"
            fi
            ok "net diagnostics node${idx} captured"
          done

      - name: Collect artifacts from node1
        if: always()
        run: |
          set +e
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          mkdir -p "${ARTIFACT_DIR}"
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
          VM=tf-user@${{ steps.cluster.outputs.node1_ip }}
          ssh "${SSH_OPTS[@]}" "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
          ok "artifacts pulled to ${ARTIFACT_DIR}"
          IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
          for i in "${!NODE_LIST[@]}"; do
            idx=$((i + 1))
            ip="${NODE_LIST[$i]}"
            ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
              > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
            ok "journal node${idx}: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null || echo 0) lines"
          done
          for i in "${!NODE_LIST[@]}"; do
            idx=$((i + 1))
            ip="${NODE_LIST[$i]}"
            ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \
              > "${ARTIFACT_DIR}/openvswitch-ipsec-node${idx}.log" 2>/dev/null
          done
          # LB/system microVM serial consoles, per node. The lb-agent logs its
          # nginx (NLB) engine activation and any `nginx -t`/reload error +
          # active-probe results to the guest serial console (never in the host
          # journal). Files persist in /run/spinifex past VM teardown.
          for i in "${!NODE_LIST[@]}"; do
            idx=$((i + 1))
            ip="${NODE_LIST[$i]}"
            mkdir -p "${ARTIFACT_DIR}/lb-console/node${idx}"
            ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
              | tar -C "${ARTIFACT_DIR}/lb-console/node${idx}" -xf - 2>/dev/null
            ok "lb-console node${idx}: $(ls "${ARTIFACT_DIR}/lb-console/node${idx}" 2>/dev/null | wc -l) files"
          done

      - name: Convert test logs to JUnit XML
        if: always()
        run: |
          shopt -s nullglob
          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
            BASE=$(basename "$LOG" .log)
            SUITE=${BASE#test-}
            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
          done

      - name: Publish test summary
        if: always()
        uses: ./spinifex/.github/actions/junit-report
        with:
          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
          title: "Multi-Node E2E results"

      - name: Analyze e2e failures
        if: always()
        uses: ./spinifex/.github/actions/e2e-analyze
        with:
          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
          log-dir: ${{ env.ARTIFACT_DIR }}
          title: "Multi-Node E2E failure analysis"

      - uses: actions/upload-artifact@v7
        if: always()
        with:
          name: e2e-multi-artifacts-${{ github.run_id }}
          path: ${{ env.ARTIFACT_DIR }}
          if-no-files-found: warn
          retention-days: 14

      - name: Teardown
        if: always()
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set +e
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          if [ -n "${{ steps.cluster.outputs.node1_ip }}" ] && [ -n "${{ steps.cluster.outputs.ssh_key }}" ]; then
            IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
            for ip in "${NODE_LIST[@]}"; do
              ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
                -i "${{ steps.cluster.outputs.ssh_key }}" \
                tf-user@"$ip" \
                'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
                2>/dev/null || true
            done
          fi
          UID_SELF=$(id -u)
          WS="${GITHUB_WORKSPACE}"
          reap() {
            local sig="$1"
            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
              case "$cwd" in
                "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;;
              esac
            done
          }
          reap TERM
          sleep 2
          reap KILL
          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
          ok "teardown complete"

      - name: Upload tofu log
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: tofu-log-multi-${{ github.run_id }}
          path: ${{ env.TOFU_LOG }}
          if-no-files-found: ignore
          retention-days: 14

      - name: Fail job if any suite failed
        if: always()
        run: |
          if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
            echo "::error::Multi-node E2E suites failed (rc=${{ steps.e2e.outputs.rc }})"
            cat "${ARTIFACT_DIR}/suite-status.txt" 2>/dev/null || true
            exit 1
          fi

  e2e_status:
    name: Report E2E Status
    if: always()
    needs: [integration, e2e_single, e2e_multi]
    runs-on: ubuntu-26.04
    timeout-minutes: 5
    steps:
      - name: Report Status to PR
        env:
          GH_TOKEN: ${{ github.token }}
          REPORT_SHA: ${{ github.sha }}
        run: |
          # integration must have actually succeeded -- unlike e2e_single/multi,
          # "skipped" is never an acceptable result for it (it has no `if:` of
          # its own), so a red or skipped integration run fails the gate here
          # even though e2e_single/e2e_multi themselves report "skipped" (their
          # `if:` never let them start).
          integration_ok=true
          case "${{ needs.integration.result }}" in success) ;; *) integration_ok=false ;; esac
          single_ok=true; multi_ok=true
          case "${{ needs.e2e_single.result }}" in success|skipped) ;; *) single_ok=false ;; esac
          case "${{ needs.e2e_multi.result }}" in success|skipped) ;; *) multi_ok=false ;; esac
          if $integration_ok && $single_ok && $multi_ok; then
            STATE="success"
            DESC="E2E suites passed"
          else
            STATE="failure"
            DESC="E2E failed: integration=${{ needs.integration.result }} single=${{ needs.e2e_single.result }} multi=${{ needs.e2e_multi.result }}"
          fi
          gh api "repos/${{ github.repository }}/statuses/${REPORT_SHA}" \
            -f state="$STATE" \
            -f context=e2e \
            -f description="$DESC" \
            -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  # Emits one summary doc per run to the central Elastic sink, keyed by
  # labels.ci_run_id, so Kibana can list "did run X pass, and where are its
  # artifacts" without hand-aggregating traces-apm*/logs-apm.error*. This is
  # a separate self-hosted job (not a step on e2e_status) because e2e_status
  # runs on GitHub-hosted ubuntu-26.04, which has no route to the private
  # 192.168.1.13 sink, and moving e2e_status off GH-hosted would risk the PR
  # status-report check.
  e2e_es_summary:
    name: ES CI Run Summary
    needs: [integration, e2e_single, e2e_multi]
    if: always()
    runs-on: [self-hosted, ci-single]
    timeout-minutes: 5
    permissions:
      actions: read
    env:
      # Repo var overrides; defaults to the LAN Elastic sink reachable from
      # ci-single/ci-multi runners. Anonymous ES requests are superuser on
      # this host, so no credentials are needed for the POST.
      MULGA_ES_SINK: ${{ vars.MULGA_ES_SINK || '192.168.1.13:9200' }}
    steps:
      # Either leg may not have run (e.g. a workflow_dispatch that only
      # requested single-node suites), so a missing artifact is tolerated —
      # the aggregation script below treats an absent suite-status.txt as
      # "no failures from that leg".
      - name: Download single-node artifacts
        uses: actions/download-artifact@v8
        continue-on-error: true
        with:
          name: e2e-artifacts-${{ github.run_id }}
          path: ${{ github.workspace }}/es-summary-single

      - name: Download multi-node artifacts
        uses: actions/download-artifact@v8
        continue-on-error: true
        with:
          name: e2e-multi-artifacts-${{ github.run_id }}
          path: ${{ github.workspace }}/es-summary-multi

      - name: Build and post CI run summary
        env:
          GH_TOKEN: ${{ github.token }}
          REPO: ${{ github.repository }}
          RUN_ID: ${{ github.run_id }}
          RUN_ATTEMPT: ${{ github.run_attempt }}
          WORKFLOW: ${{ github.workflow }}
          BRANCH: ${{ github.head_ref || github.ref_name }}
          SHA: ${{ github.sha }}
          SERVER_URL: ${{ github.server_url }}
          INTEGRATION_RESULT: ${{ needs.integration.result }}
          SINGLE_RESULT: ${{ needs.e2e_single.result }}
          MULTI_RESULT: ${{ needs.e2e_multi.result }}
          SINGLE_DIR: ${{ github.workspace }}/es-summary-single
          MULTI_DIR: ${{ github.workspace }}/es-summary-multi
        run: |
          set -euo pipefail

          SINGLE_STATUS="${SINGLE_DIR}/suite-status.txt"
          MULTI_STATUS="${MULTI_DIR}/suite-status.txt"

          # Collect "<suite>: FAILED" lines from both legs' suite-status.txt
          # (either may be absent — a leg that didn't run, or a passing leg
          # that never wrote the file). Accumulate as newline-separated text
          # rather than a bash array so this stays correct under `set -u`
          # even when nothing failed.
          FAILED_SUITES=""
          for f in "$SINGLE_STATUS" "$MULTI_STATUS"; do
            [ -f "$f" ] || continue
            while IFS= read -r line; do
              case "$line" in
                *": FAILED") FAILED_SUITES="${FAILED_SUITES}${line%: FAILED}"$'\n' ;;
              esac
            done < "$f"
          done

          FAILED_JSON=$(printf '%s' "$FAILED_SUITES" | jq -R -s 'split("\n") | map(select(length > 0))')
          FAILED_COUNT=$(printf '%s' "$FAILED_JSON" | jq 'length')

          # Mirror e2e_status's integration_ok/single_ok/multi_ok logic exactly
          # so the PR status and this ES doc can never disagree on overall
          # pass/fail. Like e2e_status, "skipped" is only acceptable for
          # single/multi (their `if:` may legitimately not select them); a
          # skipped or failed integration always fails the gate.
          integration_ok=true
          case "$INTEGRATION_RESULT" in success) ;; *) integration_ok=false ;; esac
          single_ok=true
          case "$SINGLE_RESULT" in success | skipped) ;; *) single_ok=false ;; esac
          multi_ok=true
          case "$MULTI_RESULT" in success | skipped) ;; *) multi_ok=false ;; esac

          STATE=success
          if [ "$integration_ok" != true ] || [ "$single_ok" != true ] || [ "$multi_ok" != true ] || [ "$FAILED_COUNT" -gt 0 ]; then
            STATE=failure
          fi

          RUN_STARTED_AT=$(gh api "repos/${REPO}/actions/runs/${RUN_ID}" --jq .run_started_at)
          START_EPOCH=$(date -u -d "${RUN_STARTED_AT}" +%s)
          NOW_EPOCH=$(date -u +%s)
          DURATION=$((NOW_EPOCH - START_EPOCH))

          DOC=$(jq -n \
            --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
            --arg run_id "${RUN_ID}" \
            --arg workflow "${WORKFLOW}" \
            --arg branch "${BRANCH}" \
            --arg sha "${SHA}" \
            --arg run_attempt "${RUN_ATTEMPT}" \
            --arg state "${STATE}" \
            --arg integration_result "${INTEGRATION_RESULT}" \
            --arg single_result "${SINGLE_RESULT}" \
            --arg multi_result "${MULTI_RESULT}" \
            --argjson duration "${DURATION}" \
            --arg artifact_url "${SERVER_URL}/${REPO}/actions/runs/${RUN_ID}" \
            --argjson failed_suites "${FAILED_JSON}" \
            '{
              "@timestamp": $ts,
              labels: { ci_run_id: $run_id },
              ci: { workflow: $workflow, branch: $branch, sha: $sha, run_attempt: $run_attempt },
              state: $state,
              integration_result: $integration_result,
              single_result: $single_result,
              multi_result: $multi_result,
              failed_suites: $failed_suites,
              duration_seconds: $duration,
              artifact_url: $artifact_url
            }')

          echo "$DOC" | jq .

          HTTP_CODE=$(curl -sS -o /tmp/es-summary-resp.json -w '%{http_code}' \
            -X POST "http://${MULGA_ES_SINK}/mulga-ci-runs/_doc" \
            -H 'Content-Type: application/json' \
            -d "${DOC}")

          if [ "$HTTP_CODE" != "200" ] && [ "$HTTP_CODE" != "201" ]; then
            echo "::error::ES POST to ${MULGA_ES_SINK}/mulga-ci-runs failed (HTTP ${HTTP_CODE})"
            cat /tmp/es-summary-resp.json
            exit 1
          fi
          echo "CI run summary doc posted to ${MULGA_ES_SINK}/mulga-ci-runs (HTTP ${HTTP_CODE})"