name: E2E Nightly

# Triggered nightly by a Cloudflare Worker that dispatches this workflow once on
# `dev` and once on `main`, rather than by a `schedule:` here. GitHub registers a
# schedule only from the default branch, so a cron in this file can never run
# `main` and `dev` as separate nightlies, and it would always execute `main`'s
# copy of this file — leaving a fix to the workflow itself untestable until it
# had already shipped. A dispatch takes both the file and the source from its ref.
on:
  workflow_dispatch:
    inputs:
      cells:
        description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection, 30 routed NAT on three nodes, 31 instance recovery on three nodes, 32 load balancer recovery on three nodes, 33 KV bucket quorum under node loss on three nodes"
        required: false

permissions:
  contents: read
  actions: read

# Every cell builds its cluster on the same five hypervisors, so two runs in
# flight at once compete for their disks and their libvirt guests. Queue the
# second rather than cancelling it: a dispatch is usually a verification run
# and discarding it loses the result someone is waiting on.
#
# The group is shared with e2e-suites.yml and e2e-ddil.yml, which draw on the
# same runner pools. e2e.yml is deliberately outside it: a single-cluster push
# run is cheap enough to overlap, and grouping it stalled every push behind a
# multi-hour nightly.
concurrency:
  group: e2e-hypervisors
  cancel-in-progress: false

env:
  GIT_BRANCH: ${{ github.ref_name }}
  UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-26.04.qcow2
  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}

jobs:
  select:
    runs-on: ubuntu-26.04
    outputs:
      cells: ${{ steps.pick.outputs.cells }}
      any: ${{ steps.pick.outputs.any }}
    steps:
      - uses: actions/checkout@v7.0.1
        with:
          sparse-checkout: |
            .github/e2e-cells.json
            .github/scripts/suite-sets.sh
          sparse-checkout-cone-mode: false
      - id: pick
        env:
          CELLS: ${{ github.event_name == 'workflow_dispatch' && inputs.cells || '' }}
          SETUP_ALLOWANCE_MIN: 10
        run: |
          if [ -z "$CELLS" ]; then
            sel=$(jq -c . .github/e2e-cells.json)
          else
            sel=$(jq -c --arg c "$CELLS" \
              '[.[] | . as $e | select(($c | split(",")) | index($e.cell|tostring))]' \
              .github/e2e-cells.json)
          fi
          echo "cells=$sel" >> "$GITHUB_OUTPUT"
          [ "$(jq 'length' <<<"$sel")" -gt 0 ] && any=true || any=false
          echo "any=$any" >> "$GITHUB_OUTPUT"
          echo "selected: $(jq -c '[.[].cell]' <<<"$sel")"

          # The inner timeout must always fire before the outer one. go test
          # reports where it stopped and the suite's own failure; a job timeout
          # reports only "The operation was canceled". cell-27 ran a 50m rds
          # suite inside a 50m job and could never finish either way.
          . .github/scripts/suite-sets.sh
          # Cells with no suite are filtered in jq, not with an empty-field test:
          # tab is IFS whitespace, so read collapses a run of them and an empty
          # middle column silently shifts every later one left.
          bad=0
          while IFS=$'\t' read -r cell suite budget; do
            want="$(e2e_suite_timeout "$suite")"; want="${want%m}"
            if [ "$budget" -lt $((want + SETUP_ALLOWANCE_MIN)) ]; then
              echo "::error::cell-$cell gives suite '$suite' ${want}m inside a ${budget}m job. Raise timeout_minutes in .github/e2e-cells.json to at least $((want + SETUP_ALLOWANCE_MIN)), or lower the suite budget in suite-sets.sh." >&2
              bad=1
            fi
          done < <(jq -r '.[]|select((.suite//"")!="")|[.cell,.suite,(.timeout_minutes//50)]|@tsv' <<<"$sel")

          # A permutation cell names no suite and runs the nightly subset for its
          # topology, so its bound is the longest of those. max() and not sum():
          # whichever suite hangs must panic on its own timeout first, and a sum
          # would demand a budget the set has never come close to using.
          while IFS=$'\t' read -r cell topo budget; do
            case "$topo" in
              *multi) set -- ${E2E_SUITES_NIGHTLY_MULTI} ;;
              *)      set -- ${E2E_SUITES_NIGHTLY_SINGLE} ;;
            esac
            want=0
            for suite in "$@"; do
              t="$(e2e_suite_timeout "$suite")"; t="${t%m}"
              [ "$t" -gt "$want" ] && want="$t"
            done
            if [ "$budget" -lt $((want + SETUP_ALLOWANCE_MIN)) ]; then
              echo "::error::cell-$cell runs the $topo nightly subset, whose longest suite asks ${want}m, inside a ${budget}m job. Raise timeout_minutes in .github/e2e-cells.json to at least $((want + SETUP_ALLOWANCE_MIN))." >&2
              bad=1
            fi
          done < <(jq -r '.[]|select((.suite//"")=="")|[.cell,.topo,(.timeout_minutes//50)]|@tsv' <<<"$sel")
          [ "$bad" -eq 0 ]

  matrix:
    name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{
      matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
    needs: select
    if: needs.select.outputs.any == 'true'
    strategy:
      fail-fast: false
      max-parallel: 3
      matrix:
        include: ${{ fromJSON(needs.select.outputs.cells) }}
    runs-on: [self-hosted, "${{ matrix.runner }}"]
    timeout-minutes: ${{ matrix.timeout_minutes || 50 }}
    env:
      TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
      ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}
      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=${{ matrix.cell }},ci.matrix=${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}

    steps:
      - name: Clean Workspace
        run: |
          sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
          if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3.3.0
        id: app-token
        with:
          client-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

      - uses: actions/checkout@v7.0.1
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

      - uses: actions/checkout@v7.0.1
        with:
          path: spinifex

      # Twelve cells share one run id and every cell names its hosts
      # node1/node2/node3, so ci.cell above separates them and ci.job_id here
      # links each cell's telemetry to the job log that produced it. Runs
      # before Bootstrap bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env.
      - name: Resolve CI job id
        uses: $/.github/actions/otel-job-id
        with:
          job-name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"

      - name: Build Install Artifacts
        working-directory: mulga/scripts/tofu-cluster
        env:
          MATRIX_SUITE: ${{ matrix.suite }}
        run: |
          # The script's default set is the nightly baseline. A service cell
          # runs one suite outside it, and asks for that binary by name.
          ./build-install-artifacts.sh "${GIT_BRANCH}" \
            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
            "${RUNNER_TEMP}/setup.sh" \
            "${RUNNER_TEMP}/spinifex-tests.tar" \
            --extra-suites "${MATRIX_SUITE}"

      # Every runner agent runs on one host, while their nodes sit on several, so
      # this samples the runner's host and not the node's hypervisor. Kept for
      # runner-side contention; the node's own capacity is sampled after.
      - name: Sample runner-host load (before)
        if: always()
        run: |
          mkdir -p "${ARTIFACT_DIR}"
          {
            echo "=== before: $(date -u +%Y-%m-%dT%H:%M:%SZ) runner=${RUNNER_NAME} host=$(hostname) ==="
            echo "loadavg: $(cat /proc/loadavg)"
            grep -E '^(MemTotal|MemAvailable|SwapFree):' /proc/meminfo || true
            echo "cpu(steal is field 8): $(grep '^cpu ' /proc/stat || true)"
            echo "running qemu: $(pgrep -c qemu-system-x86 || echo 0)"
          } >> "${ARTIFACT_DIR}/hypervisor-load.txt"

      - name: Provision VMs
        working-directory: mulga/scripts/tofu-cluster
        env:
          MATRIX_HOST: ${{ matrix.host }}
        run: |
          export TF_VAR_git_branch="${GIT_BRANCH}"

          # Match the family, not a pinned version: an exact match silently stops
          # overriding the image when the matrix bumps release, dropping the cell
          # back onto the debian gold.
          TOFU_VAR_ARGS=""
          case "$MATRIX_HOST" in
            ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
          esac

          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          tofu init -input=false >> "$TOFU_LOG" 2>&1
          tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
          tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
          ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
          # shellcheck disable=SC2086
          retry_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false

      - name: Bootstrap via Binary Install
        working-directory: mulga/scripts/tofu-cluster
        env:
          MATRIX_NET: ${{ matrix.net }}
        run: |
          if [ "$MATRIX_NET" = "nat" ]; then
            # Routed-NAT mode with a static public pool: exercises both the
            # Tier-1 transit datapath and the Tier-2 EIP ingress lane.
            export SPX_EXTERNAL_MODE=nat
            export SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}"
          elif [ "$MATRIX_NET" = "dhcp" ]; then
            unset SPX_EXTERNAL_POOL
            export SPX_EXTERNAL_SOURCE=dhcp
          else
            export SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}"
          fi

          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
            "${RUNNER_TEMP}/setup.sh" \
            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"

      - name: Run Single Node E2E
        if: matrix.topo == 'single' && matrix.extra == '' && matrix.suite == ''
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set -euo pipefail
          # Nightly runs a documented subset of the single-node set; the assert
          # fails the cell if that subset ever names a suite this topology
          # cannot fully exercise.
          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
          e2e_suites_assert_subset "e2e-nightly single" "${E2E_SUITES_NIGHTLY_SINGLE}" "${E2E_SUITES_SINGLE}"

          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
             for SUITE in ${E2E_SUITES_NIGHTLY_SINGLE}; do \
               echo \"::group::suite \$SUITE\"; \
               date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
               GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                 ARTIFACT_DIR='${ARTIFACT_DIR}' \
                 ./\$SUITE.test -test.v -test.timeout=30m \
                 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
               rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
               if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
             done; exit \$FAIL"

      - name: Run NAT Uplink E2E
        if: matrix.extra == 'nat-single'
        working-directory: mulga/scripts/tofu-cluster
        # Skips cleanly (exit 0) when the tested branch predates the natuplink
        # suite so scheduled runs against dev stay green until the merge lands.
        run: |
          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
             if [ ! -f natuplink.test ]; then echo 'natuplink.test not in tests tar (branch predates suite) — skipping'; exit 0; fi; \
             echo '::group::suite natuplink'; \
             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-natuplink.start'; \
             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
               ARTIFACT_DIR='${ARTIFACT_DIR}' \
               ./natuplink.test -test.v -test.timeout=30m \
               2>&1 | tee '${ARTIFACT_DIR}/test-natuplink.log'; \
             rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
             if [ \$rc -ne 0 ]; then echo 'natuplink: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"

      # natuplink proves a guest with a public IP reaches the internet. The two
      # surfaces it never touches are a private subnet's isolation and its NAT
      # gateway, and TestVPCEgressPaths is the authority on both — it skipped in
      # nat mode until the fixture learned that routed NAT carries EIPs too.
      - name: Run VPC Egress Paths in NAT mode
        if: matrix.extra == 'nat-single'
        working-directory: mulga/scripts/tofu-cluster
        run: |
          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
             echo '::group::suite single/TestVPCEgressPaths'; \
             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-vpcegress-nat.start'; \
             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
               ARTIFACT_DIR='${ARTIFACT_DIR}' \
               ./single.test -test.v -test.timeout=30m -test.run '^TestVPCEgressPaths\$' \
               2>&1 | tee '${ARTIFACT_DIR}/test-vpcegress-nat.log'; \
             rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
             if [ \$rc -ne 0 ]; then echo 'vpcegress-nat: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"

      - name: Run Multi Node E2E
        if: matrix.topo == 'real-multi' && matrix.suite == '' && matrix.extra == ''
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set -euo pipefail
          # See the single-node step: same subset guard, multi-node set.
          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
          e2e_suites_assert_subset "e2e-nightly multi" "${E2E_SUITES_NIGHTLY_MULTI}" "${E2E_SUITES_MULTI}"

          INVENTORY=$(tofu output -json inventory)
          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"

          NODES_CSV=""
          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
          for i in $(seq 0 $((NODE_COUNT - 1))); do
            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
            NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip"
          done

          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
             for SUITE in ${E2E_SUITES_NIGHTLY_MULTI}; do \
               echo \"::group::suite \$SUITE\"; \
               date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
               GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
                 SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
                 SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
                 ARTIFACT_DIR='${ARTIFACT_DIR}' \
                 ./\$SUITE.test -test.v -test.timeout=30m \
                 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
               rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
               if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
             done; exit \$FAIL"

      # The same suite as the nat-single cell, run against three nodes: there it
      # proves routed NAT works at all, here that each node delivers its own
      # guests' public IPs and no others — which one node cannot show.
      - name: Run NAT Uplink Multi E2E
        if: matrix.extra == 'nat-multi'
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set -euo pipefail
          INVENTORY=$(tofu output -json inventory)
          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"

          NODES_CSV=""
          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
          for i in $(seq 0 $((NODE_COUNT - 1))); do
            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
            NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip"
          done

          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
             if [ ! -f natuplink.test ]; then echo 'natuplink.test not in tests tar (branch predates suite) — skipping'; exit 0; fi; \
             echo '::group::suite natuplink'; \
             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-natuplink.start'; \
             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
               SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
               SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
               ARTIFACT_DIR='${ARTIFACT_DIR}' \
               ./natuplink.test -test.v -test.timeout=30m \
               2>&1 | tee '${ARTIFACT_DIR}/test-natuplink.log'; \
             rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
             if [ \$rc -ne 0 ]; then echo 'natuplink: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"

      # The dedicated suite cells only. Resolved once here rather than in each
      # step below, which would repeat the same three lines five times.
      - name: Resolve VM SSH details
        id: vm
        if: matrix.suite != '' && matrix.topo == 'single'
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set -euo pipefail
          INVENTORY=$(tofu output -json inventory)
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          echo "wan_ip=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')" >> "$GITHUB_OUTPUT"
          echo "ssh_key=${SSH_KEY/#\~/$HOME}" >> "$GITHUB_OUTPUT"

      # The eks suite's CreateCluster boots a control-plane VM from the eks-node
      # AMI, so that AMI must be in the VM's image store before the suite runs.
      # build-system-image.sh customizes with libguestfs (an isolated userspace
      # appliance touching no host block device), so it is safe on the shared
      # runner. The build needs source + network (runner only) and the import
      # needs predastore/NATS (VM only), hence the split.
      - name: Build eks-node AMI (libguestfs, runner)
        if: matrix.suite == 'eks'
        working-directory: spinifex
        run: |
          set -euo pipefail
          # direct backend = no libvirt. Set here rather than in the script:
          # it has to be in the environment the build itself runs in.
          export LIBGUESTFS_BACKEND=direct
          "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ensure-libguestfs.sh"
          make build-eks-node-image

      - name: Import eks-node AMI on VM
        if: matrix.suite == 'eks'
        run: |
          set -euo pipefail
          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
            spinifex/scripts/images/eks-node/manifest.conf)"
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
          scp "${SSH_OPTS[@]}" \
            /tmp/eks-node-image-build/eks-node-alpine.raw \
            "$VM":/tmp/eks-node-alpine.raw
          ssh "${SSH_OPTS[@]}" "$VM" \
            "spx admin images import --file /tmp/eks-node-alpine.raw \
               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1

      # The eks data-plane subtest (TestEKS/KubectlGetNodes) drives the full
      # `aws eks get-token` -> kubectl -> apiserver -> token-webhook -> gateway
      # broker -> STS verify path. The e2e binaries run ON the VM, so kubectl
      # must be on the VM's PATH or harness.NewKubectl skips the subtest and the
      # cell reports green having never exercised the data plane. Pinned to the
      # k3s server version (setup.sh K3S_VERSION) and checksum-verified.
      - name: Install kubectl on VM
        if: matrix.suite == 'eks'
        run: |
          set -euo pipefail
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
          ssh "${SSH_OPTS[@]}" tf-user@${{ steps.vm.outputs.wan_ip }} 'bash -s' <<'EOF'
          set -euo pipefail
          KVER=v1.32.5
          case "$(uname -m)" in
            x86_64)  ARCH=amd64 ;;
            aarch64) ARCH=arm64 ;;
            *) echo "unsupported arch $(uname -m)" >&2; exit 1 ;;
          esac
          cd /tmp
          curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl"
          curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl.sha256"
          echo "$(cat kubectl.sha256)  kubectl" | sha256sum -c -
          sudo install -m 0755 kubectl /usr/local/bin/kubectl
          kubectl version --client
          EOF

      # RegisterContainerInstance boots the container instance from the ecs-node
      # AMI, and resolveECSNodeAMI matches on the managed-by tag rather than the
      # name, so the tag is not optional. The image directory is named for
      # IMAGE_NAME in the manifest (ecs-agent), not for the make target.
      - name: Build ecs-node AMI (libguestfs, runner)
        if: matrix.suite == 'ecs'
        working-directory: spinifex
        run: |
          set -euo pipefail
          # direct backend = no libvirt. Set here rather than in the script:
          # it has to be in the environment the build itself runs in.
          export LIBGUESTFS_BACKEND=direct
          "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ensure-libguestfs.sh"
          make build-ecs-node-image

      - name: Import ecs-node AMI on VM
        if: matrix.suite == 'ecs'
        run: |
          set -euo pipefail
          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
            spinifex/scripts/images/ecs-agent/manifest.conf)"
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
          scp "${SSH_OPTS[@]}" \
            /tmp/ecs-agent-image-build/ecs-agent-alpine.raw \
            "$VM":/tmp/ecs-agent-alpine.raw
          ssh "${SSH_OPTS[@]}" "$VM" \
            "spx admin images import --file /tmp/ecs-agent-alpine.raw \
               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1

      # CreateDBInstance boots the DB VM from an RDS AMI, so both engines' images
      # must be in the store before the suite runs. The suite's psql and mariadb
      # clients need no step here: they run inside a client VM that installs them
      # from its own user-data, because a private-only endpoint is unreachable
      # from the runner and from the cluster host alike.
      - name: Build RDS AMIs (libguestfs, runner)
        if: matrix.suite == 'rds'
        working-directory: spinifex
        run: |
          set -euo pipefail
          # direct backend = no libvirt. Set here rather than in the script:
          # it has to be in the environment the build itself runs in.
          export LIBGUESTFS_BACKEND=direct
          "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ensure-libguestfs.sh"
          for IMAGE in rds-postgres rds-mariadb; do
            make "build-${IMAGE}-image"
          done

      - name: Import RDS AMIs on VM
        if: matrix.suite == 'rds'
        run: |
          set -euo pipefail
          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
          for IMAGE in rds-postgres rds-mariadb; do
            eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
              "spinifex/scripts/images/${IMAGE}/manifest.conf")"
            # SYSTEM_TAG carries four tags, not one: managed-by marks it a system
            # image, engine/engine-version are what an EngineVersion request
            # resolves against, and the contract tag gates the data-volume format
            # grant. --tag is repeatable, so expand rather than pass the whole
            # string as a single value.
            TAG_ARGS=""
            for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
            scp "${SSH_OPTS[@]}" \
              "/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" \
              "$VM":"/tmp/${IMAGE}-alpine.raw"
            ssh "${SSH_OPTS[@]}" "$VM" \
              "spx admin images import --file /tmp/${IMAGE}-alpine.raw \
                 --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
                 --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
          done

      - name: Run Service Suite E2E
        if: matrix.suite != '' && matrix.topo == 'single'
        env:
          SUITE: ${{ matrix.suite }}
        run: |
          set -euo pipefail
          # Same guard as the permutation steps, against the full single-node
          # set: a cell naming a suite this topology cannot fully exercise fails
          # here rather than skipping half its assertions.
          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
          e2e_suites_assert_subset "e2e-nightly cell-${{ matrix.cell }}" "${SUITE}" "${E2E_SUITES_SINGLE}"
          TMO="$(e2e_suite_timeout "${SUITE}")"

          ssh -tt -o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}" \
            tf-user@${{ steps.vm.outputs.wan_ip }} \
            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
             echo \"::group::suite ${SUITE}\"; \
             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
               ARTIFACT_DIR='${ARTIFACT_DIR}' \
               ./${SUITE}.test -test.v -test.timeout=${TMO} \
               2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log'; \
             rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
             if [ \$rc -ne 0 ]; then echo \"${SUITE}: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; fi; \
             exit \$rc"

      # The multi-node twin of the step above. Separate rather than merged
      # because the suite needs every node's address to reach the whole cluster,
      # which the single-node path has no way to supply.
      - name: Run Multi Suite E2E
        if: matrix.suite != '' && matrix.topo == 'real-multi'
        working-directory: mulga/scripts/tofu-cluster
        env:
          SUITE: ${{ matrix.suite }}
        run: |
          set -euo pipefail
          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
          e2e_suites_assert_subset "e2e-nightly cell-${{ matrix.cell }}" "${SUITE}" "${E2E_SUITES_MULTI}"
          TMO="$(e2e_suite_timeout "${SUITE}")"

          INVENTORY=$(tofu output -json inventory)
          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"

          NODES_CSV=""
          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
          for i in $(seq 0 $((NODE_COUNT - 1))); do
            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
            NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip"
          done

          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
             echo \"::group::suite ${SUITE}\"; \
             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
               SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
               SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
               ARTIFACT_DIR='${ARTIFACT_DIR}' \
               ./${SUITE}.test -test.v -test.timeout=${TMO} \
               2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log'; \
             rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
             if [ \$rc -ne 0 ]; then echo \"${SUITE}: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; fi; \
             exit \$rc"

      - name: Import RDS PostgreSQL AMI
        if: matrix.extra == 'tofu-examples'
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set -euo pipefail
          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
            "spx admin images import --name spinifex-rds-postgres"

      - name: Run Tofu Workbooks
        if: matrix.extra == 'tofu-examples'
        working-directory: mulga/scripts/tofu-cluster
        # The driver emits go test -v markers per workbook; teeing to
        # test-tofu.log feeds the same go-junit-report -> e2e-analyze -> upload
        # chain the Go suites use, so cell 17 produces an RCA bundle too.
        run: |
          set +e
          mkdir -p "${ARTIFACT_DIR}"
          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
          date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-tofu.start"
          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
            "cd ~/e2e-tests && ./run-tofu-examples-e2e.sh" \
            2>&1 | tee "${ARTIFACT_DIR}/test-tofu.log"
          rc=${PIPESTATUS[0]}
          if [ $rc -ne 0 ]; then echo "tofu: FAILED" >> "${ARTIFACT_DIR}/suite-status.txt"; fi
          exit $rc

      - name: Run Reboot Resilience E2E
        if: matrix.extra == 'reboot-single'
        working-directory: mulga/scripts/tofu-cluster
        # reboot.test runs from the runner (NOT via ssh-and-run) so it survives the
        # systemctl-reboot-induced ssh disconnect mid-run.
        run: |
          set +e
          mkdir -p "${ARTIFACT_DIR}"
          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"

          REBOOT_DIR="${RUNNER_TEMP}/reboot-e2e"
          mkdir -p "$REBOOT_DIR"
          tar xf "${RUNNER_TEMP}/spinifex-tests.tar" -C "$REBOOT_DIR"
          chmod +x "$REBOOT_DIR/_bin/reboot.test"

          SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$SSH_KEY")
          CREDS=$(ssh "${SSH_OPTS[@]}" tf-user@${WAN_IP} 'cat ~/.aws/credentials')
          AWS_AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
          AWS_SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
          echo "::add-mask::${AWS_AK}"; echo "::add-mask::${AWS_SK}"

          date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-reboot.start"
          GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
            SPINIFEX_WAN_IP="${WAN_IP}" SPINIFEX_SSH_KEY="${SSH_KEY}" \
            SPINIFEX_AWS_ENDPOINT="https://${WAN_IP}:9999" \
            SPINIFEX_AWS_ACCESS_KEY_ID="${AWS_AK}" SPINIFEX_AWS_SECRET_ACCESS_KEY="${AWS_SK}" \
            ARTIFACT_DIR="${ARTIFACT_DIR}" \
            "$REBOOT_DIR/_bin/reboot.test" -test.v -test.timeout=30m \
            2>&1 | tee "${ARTIFACT_DIR}/test-reboot.log"
          rc=${PIPESTATUS[0]}
          if [ $rc -ne 0 ]; then echo "reboot: FAILED" >> "${ARTIFACT_DIR}/suite-status.txt"; fi
          exit $rc

      - name: Sample runner-host load (after)
        if: always()
        run: |
          mkdir -p "${ARTIFACT_DIR}"
          {
            echo "=== after: $(date -u +%Y-%m-%dT%H:%M:%SZ) runner=${RUNNER_NAME} host=$(hostname) ==="
            echo "loadavg: $(cat /proc/loadavg)"
            grep -E '^(MemTotal|MemAvailable|SwapFree):' /proc/meminfo || true
            echo "cpu(steal is field 8): $(grep '^cpu ' /proc/stat || true)"
            echo "running qemu: $(pgrep -c qemu-system-x86 || echo 0)"
          } >> "${ARTIFACT_DIR}/hypervisor-load.txt"

      # The node, not the runner's host, is what the guests are scheduled on, so
      # its load is the figure that explains a starved guest. Never fails the
      # cell: a node already gone is the case this is trying to describe.
      - name: Sample node load (after)
        if: always()
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set +e
          mkdir -p "${ARTIFACT_DIR}"
          INVENTORY=$(tofu output -json inventory 2>/dev/null)
          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip' 2>/dev/null)
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path' 2>/dev/null)
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
          [ -n "$NODE1_IP" ] && [ "$NODE1_IP" != "null" ] || exit 0
          [ -n "$SSH_KEY" ] && [ "$SSH_KEY" != "null" ] || exit 0
          {
            echo "=== node after: $(date -u +%Y-%m-%dT%H:%M:%SZ) ip=${NODE1_IP} ==="
            ssh -o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=10 \
              -i "$SSH_KEY" "tf-user@${NODE1_IP}" \
              'echo "host: $(hostname)"; echo "loadavg: $(cat /proc/loadavg)";
               grep -E "^(MemTotal|MemAvailable|SwapFree):" /proc/meminfo;
               echo "cpu(steal is field 8): $(grep "^cpu " /proc/stat)";
               echo "running qemu: $(pgrep -c qemu-system-x86 || echo 0)"' 2>&1
          } >> "${ARTIFACT_DIR}/node-load.txt"
          exit 0

      - name: Collect e2e artifacts
        if: always()
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set +e
          mkdir -p "${ARTIFACT_DIR}"
          INVENTORY=$(tofu output -json inventory 2>/dev/null) || { echo "no inventory; skipping"; exit 0; }
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path'); SSH_KEY="${SSH_KEY/#\~/$HOME}"
          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY")

          ssh "${SSH_OPTS[@]}" tf-user@${NODE1_IP} "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null

          for i in $(seq 0 $((NODE_COUNT - 1))); do
            IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip"); idx=$((i + 1))
            mkdir -p "${ARTIFACT_DIR}/vm-console-node${idx}"
            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
              | tar -C "${ARTIFACT_DIR}/vm-console-node${idx}" -xf - 2>/dev/null
            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
              > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \
              > "${ARTIFACT_DIR}/openvswitch-ipsec-node${idx}.log" 2>/dev/null
            # The unit filter above drops kernel and OOM messages, which are
            # exactly what explains one node behaving unlike its siblings.
            # Gzipped on the far side; the whole journal is far larger.
            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl --no-pager --output=short-iso --since "90 min ago" | gzip -6' \
              > "${ARTIFACT_DIR}/full-journal-node${idx}.log.gz" 2>/dev/null
            # ovn-controller and ovs-vswitchd log to files, never to journald:
            # without these a cross-chassis black-hole cannot be attributed to
            # IPsec versus SB port binding.
            mkdir -p "${ARTIFACT_DIR}/ovs-logs-node${idx}"
            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo tar -C /var/log -cf - ovn openvswitch 2>/dev/null || true' \
              | tar -C "${ARTIFACT_DIR}/ovs-logs-node${idx}" -xf - 2>/dev/null
          done

          cat "${ARTIFACT_DIR}"/spinifex-journal-node*.log \
            > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || true
          echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"

      # Whether br-mgmt is a per-host island or one L2 segment spanning the
      # cluster decides whether a system instance can reach the gateway address
      # it was baked with. Nothing in a failing run records which one was built,
      # and the environment is destroyed before it can be asked.
      - name: Collect management-fabric topology
        if: always()
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set +e
          mkdir -p "${ARTIFACT_DIR}"
          OUT="${ARTIFACT_DIR}/mgmt-fabric.txt"
          INVENTORY=$(tofu output -json inventory 2>/dev/null) || { echo "no inventory; skipping"; exit 0; }
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path'); SSH_KEY="${SSH_KEY/#\~/$HOME}"
          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY")

          declare -a WAN=() MGMT=()
          for i in $(seq 0 $((NODE_COUNT - 1))); do
            WAN[$i]=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
          done

          for i in $(seq 0 $((NODE_COUNT - 1))); do
            idx=$((i + 1))
            {
              echo "=== node${idx} (${WAN[$i]}) ==="
              ssh "${SSH_OPTS[@]}" tf-user@"${WAN[$i]}" \
                'sudo ovs-vsctl list-ports br-mgmt 2>&1 | sed "s/^/ports: /"; \
                 ip -4 addr show br-mgmt 2>&1 | sed "s/^/addr:  /"; \
                 sudo ovs-vsctl get Bridge br-mgmt fail_mode 2>&1 | sed "s/^/fail:  /"'
              echo
            } >> "$OUT" 2>&1
            MGMT[$i]=$(ssh "${SSH_OPTS[@]}" tf-user@"${WAN[$i]}" \
              "ip -4 -o addr show br-mgmt 2>/dev/null | awk '{print \$4}' | cut -d/ -f1" 2>/dev/null)
          done

          # An island cluster reaches only its own address and that is correct;
          # a spanned one must reach every peer. Either result is a finding, and
          # a mixed result is the failure this matrix exists to catch.
          {
            echo "=== br-mgmt reachability (row = prober, column = target) ==="
            printf '%-10s' "from\\to"
            for i in $(seq 0 $((NODE_COUNT - 1))); do printf '%-18s' "node$((i+1)):${MGMT[$i]:-none}"; done
            echo
            for i in $(seq 0 $((NODE_COUNT - 1))); do
              printf '%-10s' "node$((i+1))"
              for j in $(seq 0 $((NODE_COUNT - 1))); do
                if [ -z "${MGMT[$j]}" ]; then printf '%-18s' "-"; continue; fi
                if ssh "${SSH_OPTS[@]}" tf-user@"${WAN[$i]}" \
                     "ping -c1 -W1 ${MGMT[$j]} >/dev/null 2>&1"; then
                  printf '%-18s' ok
                else
                  printf '%-18s' FAIL
                fi
              done
              echo
            done
          } >> "$OUT" 2>&1

          cat "$OUT"

      - name: Install go-junit-report
        if: always()
        run: |
          # Version from go.mod's tool directive, checked against go.sum.
          GOWORK=off go -C spinifex install github.com/jstemmer/go-junit-report/v2
          echo "$HOME/go/bin" >> "$GITHUB_PATH"

      - name: Convert test logs to JUnit XML
        if: always()
        run: |
          shopt -s nullglob
          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
            BASE=$(basename "$LOG" .log); SUITE=${BASE#test-}
            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
          done

      # Tolerated on failure: the sink being unreachable must not turn a
      # green test run red.
      - name: Ship test results to Elasticsearch
        if: always()
        continue-on-error: true
        uses: $/.github/actions/junit-to-es
        with:
          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
          cell: ${{ matrix.cell }}

      - name: Analyze e2e failures
        if: always()
        uses: $/.github/actions/e2e-analyze
        with:
          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
          log-dir: ${{ env.ARTIFACT_DIR }}
          title: "E2E failure analysis (cell ${{ matrix.cell }})"

      - uses: actions/upload-artifact@v7.0.2
        if: always()
        with:
          name: e2e-artifacts-cell-${{ matrix.cell }}-${{ github.run_id }}
          path: ${{ env.ARTIFACT_DIR }}
          if-no-files-found: warn
          retention-days: 14

      # The XML on its own, so the roll-up can read every cell's counts without
      # pulling nineteen journals it has no use for.
      - name: Upload junit only
        if: always()
        uses: actions/upload-artifact@v7.0.2
        with:
          name: junit-cell-${{ matrix.cell }}-${{ github.run_id }}
          path: ${{ env.ARTIFACT_DIR }}/junit-*.xml
          if-no-files-found: ignore
          retention-days: 14

      # Gated like every other step, not just always(): a cell the dispatch
      # deselected never checked out or ran tofu init, so destroying would fail
      # on missing plugins and report the cell red for correctly doing nothing.
      - name: Teardown
        if: always()
        working-directory: mulga/scripts/tofu-cluster
        env:
          MATRIX_HOST: ${{ matrix.host }}
        run: |
          # Must mirror the Provision step's override, or destroy runs against a
          # different image path than apply did.
          TOFU_VAR_ARGS=""
          case "$MATRIX_HOST" in
            ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
          esac
          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
          tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
          timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
          # shellcheck disable=SC2086
          retry_run "tofu destroy" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false
          ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true

      - name: Upload tofu log
        if: always()
        uses: actions/upload-artifact@v7.0.2
        with:
          name: tofu-log-cell-${{ matrix.cell }}-${{ github.run_id }}
          path: ${{ env.TOFU_LOG }}
          if-no-files-found: ignore
          retention-days: 14

  # Bare-metal single-node suite. Doesn't fit the tofu matrix shape, so it
  # runs as a sibling job via the reusable e2e-baremetal workflow. Gated like a
  # matrix cell ("cell 20"): blank cells is the whole nightly, or name 20 to run
  # it alone. A reusable call needs its own ref, since it cannot read GIT_BRANCH.
  baremetal:
    name: cell-20 (baremetal/pxe/real-hw)
    if: >-
      inputs.cells == '' ||
      contains(format(',{0},', inputs.cells), ',20,')
    permissions:
      contents: read
      statuses: write
    uses: ./.github/workflows/e2e-baremetal.yml
    with:
      ref: ${{ github.ref_name }}
      # A reusable-workflow call gets none of this workflow's context (env:
      # above, matrix.cell) for free — cell-20 needs its own attrs string.
      # There's no matrix leg here, just the one fixed baremetal cell.
      otel-attrs: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=20,ci.matrix=baremetal/pxe/real-hw
    secrets: inherit

  # The guest disk-performance gate. Same bare-metal node as cell 20 and
  # therefore queued behind it by the shared concurrency group, which is the
  # point: perf numbers off a cell still running other suites are noise, and
  # the availability assertions measure host responsiveness under a single
  # tenant's load, which a co-tenant correctness suite would confound.
  #
  # It reimages the cluster first, like every other caller of that workflow, so
  # it is a fresh node rather than whatever cell-20 left behind.
  diskperf:
    name: cell-28 (baremetal/diskperf)
    needs: baremetal
    if: >-
      !cancelled() && (
        inputs.cells == '' ||
        contains(format(',{0},', inputs.cells), ',28,')
      )
    permissions:
      contents: read
      statuses: write
    uses: ./.github/workflows/e2e-baremetal.yml
    with:
      ref: ${{ github.ref_name }}
      cell: "28"
      suites: diskperf
      otel-attrs: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=28,ci.matrix=baremetal/pxe/diskperf
    secrets: inherit

  # ISO cells. Unlike 1-19 these need no tofu: the harnesses run QEMU directly
  # on the runner. The nodes are therefore L1, which keeps the instances they
  # launch at L2 and their benchmark numbers comparable between the two cells.
  #
  # Serialised against each other because the guest MACs are fixed
  # (52:54:00:12:...): stable MACs are what make DHCP discovery work, and two
  # concurrent runs would put duplicates on the rack LAN, which is a confusing
  # failure for everyone on it, not just CI.
  iso-single:
    name: cell-21 (iso/dhcp/single)
    if: >-
      inputs.cells == '' ||
      contains(format(',{0},', inputs.cells), ',21,')
    runs-on: [self-hosted, ci-single]
    # ISO build, install, boot and pool re-form is ~35 of these on its own,
    # before the workload step imports an AMI and launches an instance.
    timeout-minutes: 90
    concurrency:
      group: iso-qemu-rack
      cancel-in-progress: false
    env:
      CI_WAN_MODE: dhcp
      CI_PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
      # What a real node gets. The harness default of 4G is a dev-box size, and
      # importing an 820 MiB AMI on top of the running stack does not fit in it.
      MEM: 8G
      SMP: 8
      # .230 is consumed by the OVN router's SNAT, leaving .231-.234 for
      # instances. Disjoint from cell-22's pool so a run that dies holding its
      # SNAT address cannot collide with the next cell.
      EXT_POOL_START: 192.168.0.230
      EXT_POOL_END: 192.168.0.234
      EXT_GATEWAY: 192.168.1.1
      EXT_PREFIX_LEN: 23
      # Pins where the benchmark writes, so collecting and removing it later are
      # both exact. Left to itself the harness picks a timestamped name.
      BENCH_OUT: /tmp/cell21-bench-${{ github.run_id }}
    steps:
      - name: Clean Workspace
        run: |
          sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*

      - name: Configure work directories
        run: |
          E2E_ROOT=/mnt/nvme/e2e
          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"

      - uses: actions/create-github-app-token@v3.3.0
        id: app-token
        with:
          client-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

      - uses: actions/checkout@v7.0.1
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          path: mulga

      - uses: actions/checkout@v7.0.1
        with:
          ref: ${{ env.GIT_BRANCH }}
          path: mulga/spinifex

      - name: Resolve sibling refs
        id: siblings
        env:
          BRANCH: ${{ env.GIT_BRANCH }}
        run: bash .github/resolve-sibling-refs.sh
        working-directory: mulga/spinifex

      - uses: actions/checkout@v7.0.1
        with:
          repository: mulgadc/viperblock
          ref: ${{ steps.siblings.outputs.viperblock }}
          path: mulga/viperblock

      - uses: actions/checkout@v7.0.1
        with:
          repository: mulgadc/predastore
          ref: ${{ steps.siblings.outputs.predastore }}
          path: mulga/predastore

      - uses: actions/checkout@v7.0.1
        with:
          repository: mulgadc/northstar
          ref: ${{ steps.siblings.outputs.northstar }}
          path: mulga/northstar

      - uses: actions/checkout@v7.0.1
        with:
          repository: mulgadc/bluebottle
          ref: ${{ steps.siblings.outputs.bluebottle }}
          path: mulga/bluebottle

      # Assert the host can do what the cell assumes, before spending 40
      # minutes finding out it cannot. A checklist in a plan doc goes stale;
      # this does not.
      - name: Preflight
        run: |
          set -euo pipefail
          mkdir -p "$TMPDIR"
          ip link show "${BRIDGE:-br-wan}" >/dev/null ||
            { echo "::error::bridge br-wan not found on this runner"; exit 1; }
          grep -qE "^allow +${BRIDGE:-br-wan}$" /etc/qemu/bridge.conf ||
            { echo "::error::br-wan not permitted in /etc/qemu/bridge.conf"; exit 1; }
          command -v sshpass >/dev/null ||
            { echo "::error::sshpass not installed"; exit 1; }
          free_mb=$(free -m | awk '/^Mem:/ {print $7}')
          [ "$free_mb" -ge 8192 ] ||
            { echo "::error::need 8 GiB available, have ${free_mb} MiB"; exit 1; }
          # A 40G image on tmpfs competes with the 8 GiB the VM itself needs.
          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
          [ "$fstype" != tmpfs ] ||
            { echo "::error::$TMPDIR is tmpfs — the node disk would be in RAM"; exit 1; }
          df -h "$TMPDIR"
          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
          [ "$avail_gb" -ge 60 ] ||
            { echo "::error::need 60 GiB on $TMPDIR, have ${avail_gb} GiB"; exit 1; }

      - name: Build artefacts
        working-directory: mulga/scripts/iso-builder
        run: make build-rootfs build-squashfs build-initramfs

      # One key for both halves of the cell. test-workload.sh SSHes to the node
      # AND registers the same key as the EC2 key pair it uses to reach the
      # instances, so the key test-smoke.sh installs has to be that one — its
      # default ~/.ssh/id_*.pub would leave the workload step locked out.
      - name: Prepare SSH key
        run: |
          set -euo pipefail
          mkdir -p "$HOME/.ssh" && chmod 700 "$HOME/.ssh"
          # Two keys, because the guest key must not share a name with the one
          # smoke-test.sh registers from the node: the node holds that private
          # half, and IMDS caches material by key name for five minutes, so
          # re-importing over it is invisible to the guests.
          for k in spinifex-key spinifex-workload-key; do
            [ -f "$HOME/.ssh/$k" ] ||
              ssh-keygen -t ed25519 -N "" -f "$HOME/.ssh/$k" -C "$k-ci"
          done

      # Boots the ISO, installs unattended, reads the lease off the serial
      # console, runs the in-guest smoke test, and installs the key for the
      # next step.
      - name: Install and smoke test
        id: smoke
        working-directory: mulga/scripts/iso-builder
        run: |
          set -euo pipefail
          # KEEP_NODE: the harness owns the only VM there is and tears it down
          # on exit, so without this the workload step below has no node left.
          KEEP_NODE=1 SSH_IDENTITY="$HOME/.ssh/spinifex-key.pub" \
            ./test/test-smoke.sh | tee /tmp/smoke-$GITHUB_RUN_ID.log
          ip=$(sed -n 's/^\[smoke\] node address: //p' /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
          [ -n "$ip" ] || { echo "::error::smoke test did not report a node address"; exit 1; }
          echo "node_ip=$ip" >> "$GITHUB_OUTPUT"

      # The customer path: AMI, key, VPC, instance, SSH in. --benchmark records
      # host and guest disk numbers so a regression shows up as a number rather
      # than a story.
      - name: Workload and benchmark
        working-directory: mulga/scripts/iso-builder
        run: |
          set -euo pipefail
          cat > /tmp/cell21.env <<EOF
          NODES=(${{ steps.smoke.outputs.node_ip }})
          NODE_NAMES=(spinifex-ci)
          SSH_USER=spinifex
          SSH_KEY_PATH=$HOME/.ssh/spinifex-workload-key
          # SSH_KEY_PATH reaches the guests, SSH_IDENTITY reaches the node, and
          # they are deliberately different keys. Without the second, node SSH
          # falls back to the runner's default identity, which the node has
          # never seen.
          SSH_IDENTITY=$HOME/.ssh/spinifex-key
          EXT_POOL_START=$EXT_POOL_START
          EXT_POOL_END=$EXT_POOL_END
          EXT_GATEWAY=$EXT_GATEWAY
          EXT_PREFIX_LEN=$EXT_PREFIX_LEN
          EOF
          ./test/test-workload.sh --env /tmp/cell21.env --benchmark

      # The node's own journal, which nothing else preserves: the serial log
      # stops at login and the harness owns the only node there is. Pulled
      # before teardown and never fatal, so a passing night keeps it too.
      - name: Collect node journal
        if: always() && steps.smoke.outputs.node_ip != ''
        env:
          NODE_IP: ${{ steps.smoke.outputs.node_ip }}
        run: |
          mkdir -p /tmp/cell21-artifacts-$GITHUB_RUN_ID
          timeout 180 ssh -o BatchMode=yes -o StrictHostKeyChecking=no \
            -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 \
            -i "$HOME/.ssh/spinifex-key" "spinifex@${NODE_IP}" \
            'sudo journalctl -b --no-pager --output=short-iso | gzip -6' \
            > "/tmp/cell21-artifacts-$GITHUB_RUN_ID/node-journal.log.gz" ||
            echo "::warning::could not pull the node journal from ${NODE_IP}"

      - name: Collect logs
        if: always()
        run: |
          mkdir -p /tmp/cell21-artifacts-$GITHUB_RUN_ID
          cp /tmp/smoke-$GITHUB_RUN_ID.log /tmp/cell21-artifacts-$GITHUB_RUN_ID/ ||
            echo "::warning::no smoke log to collect"
          # fio JSON and the summary table. Warned about rather than swallowed:
          # a benchmark that ran and produced nothing is worth knowing about.
          if [ -d "$BENCH_OUT" ]; then
            cp -r "$BENCH_OUT" /tmp/cell21-artifacts-$GITHUB_RUN_ID/benchmark
          else
            echo "::warning::no benchmark results at $BENCH_OUT"
          fi
          # The guest's own view. Without it a node that dies mid-test is only
          # visible as the harness command that stopped producing output.
          serial=$(sed -n 's/^\[smoke\] serial log preserved: //p' \
                   /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
          [ -n "$serial" ] && cp "$serial" /tmp/cell21-artifacts-$GITHUB_RUN_ID/serial.log || true
          node=$(sed -n 's/^\[smoke\] node logs preserved: //p' \
                 /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
          [ -n "$node" ] && cp "$node" /tmp/cell21-artifacts-$GITHUB_RUN_ID/node.log || true
          # QEMU's stderr no longer reaches the step log: it is redirected so a
          # kept-alive VM cannot hold the step's stdout open.
          qemu=$(sed -n 's/^\[smoke\] qemu log preserved: //p' \
                 /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
          [ -n "$qemu" ] && cp "$qemu" /tmp/cell21-artifacts-$GITHUB_RUN_ID/qemu.log || true

      - uses: actions/upload-artifact@v7.0.2
        if: always()
        with:
          name: e2e-artifacts-cell-21-${{ github.run_id }}
          path: /tmp/cell21-artifacts-${{ github.run_id }}
          if-no-files-found: warn
          # Matches the predastore benchmark in cell-23: a night's numbers are
          # only useful against the nights before them.
          retention-days: 90

      - name: Teardown
        if: always()
        working-directory: mulga/scripts/iso-builder
        run: |
          # No env file means the workload step never ran, so there is nothing
          # of its to clean up.
          if [ -f /tmp/cell21.env ]; then
            ./test/test-workload.sh --env /tmp/cell21.env --cleanup || true
          fi
          sudo pkill -f spinifex-smoke-test || true
          rm -rf /tmp/cell21-artifacts-$GITHUB_RUN_ID /tmp/cell21.env \
                 /tmp/smoke-$GITHUB_RUN_ID.log "$BENCH_OUT"
          # Runs before BENCH_OUT existed left a timestamped directory behind
          # every night. Sweep anything older than a day.
          find /tmp -maxdepth 1 -name 'spinifex-benchmark-*' -type d -mmin +1440 \
               -exec rm -rf {} + 2>/dev/null || true
          # KEEP_NODE leaves the node's disk and serial log behind for us.
          sleep 2
          rm -f "$TMPDIR"/tmp.*.qcow2 "$TMPDIR"/tmp.*.log "$TMPDIR"/tmp.*-node.log

  iso-multi:
    name: cell-22 (iso/dhcp/three-node)
    # After cell-21 rather than beside it: same concurrency group, and running
    # them together would need 32 GiB on one box.
    needs: [iso-single]
    if: >-
      !cancelled() && (
      inputs.cells == '' ||
      contains(format(',{0},', inputs.cells), ',22,') )
    runs-on: [self-hosted, ci-multi]
    timeout-minutes: 90
    concurrency:
      group: iso-qemu-rack
      cancel-in-progress: false
    env:
      MEM: 8G
      SMP: 8
      PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
      EXT_POOL_START: 192.168.0.235
      EXT_POOL_END: 192.168.0.239
      EXT_GATEWAY: 192.168.1.1
      EXT_PREFIX_LEN: 23
    steps:
      - name: Clean Workspace
        run: |
          sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*

      - name: Configure work directories
        run: |
          E2E_ROOT=/mnt/nvme/e2e
          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
          # Pinned so the per-node serial logs can be collected afterwards, and
          # kept beside TMPDIR rather than in /tmp: it also holds nine 40G node
          # disks, which is what the tmpfs assertion below is about.
          echo "WORK_DIR=$E2E_ROOT/work/cell22-$GITHUB_RUN_ID" >> "$GITHUB_ENV"

      - uses: actions/create-github-app-token@v3.3.0
        id: app-token
        with:
          client-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

      - uses: actions/checkout@v7.0.1
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          path: mulga

      - uses: actions/checkout@v7.0.1
        with:
          ref: ${{ env.GIT_BRANCH }}
          path: mulga/spinifex

      - name: Resolve sibling refs
        id: siblings
        env:
          BRANCH: ${{ env.GIT_BRANCH }}
        run: bash .github/resolve-sibling-refs.sh
        working-directory: mulga/spinifex

      - uses: actions/checkout@v7.0.1
        with:
          repository: mulgadc/viperblock
          ref: ${{ steps.siblings.outputs.viperblock }}
          path: mulga/viperblock

      - uses: actions/checkout@v7.0.1
        with:
          repository: mulgadc/predastore
          ref: ${{ steps.siblings.outputs.predastore }}
          path: mulga/predastore

      - uses: actions/checkout@v7.0.1
        with:
          repository: mulgadc/northstar
          ref: ${{ steps.siblings.outputs.northstar }}
          path: mulga/northstar

      - uses: actions/checkout@v7.0.1
        with:
          repository: mulgadc/bluebottle
          ref: ${{ steps.siblings.outputs.bluebottle }}
          path: mulga/bluebottle

      # 3 x 8 GiB plus 9 sparse 40G disks, before the instances the cluster
      # launches on top.
      - name: Preflight
        run: |
          set -euo pipefail
          mkdir -p "$TMPDIR"
          free_mb=$(free -m | awk '/^Mem:/ {print $7}')
          [ "$free_mb" -ge 26624 ] ||
            { echo "::error::need 26 GiB available, have ${free_mb} MiB"; exit 1; }
          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
          [ "$fstype" != tmpfs ] ||
            { echo "::error::$TMPDIR is tmpfs — nine node disks would be in RAM"; exit 1; }
          df -h "$TMPDIR"
          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
          [ "$avail_gb" -ge 120 ] ||
            { echo "::error::need 120 GiB on $TMPDIR ($fstype), have ${avail_gb} GiB"; exit 1; }

      - name: Build artefacts
        working-directory: mulga/scripts/iso-builder
        run: make build-rootfs build-squashfs build-initramfs

      # Four phases: install three nodes, convert to one cluster, assert it
      # formed (including the external pool), then pull node 1's power and
      # assert the survivors hold quorum.
      #
      # Phase 5 (test-workload.sh, as cell-21 runs it) is not wired yet: the
      # nodes are still on user-mode NAT and reachable only through the
      # harness's own port forwards, and test-cluster.sh does not open the
      # formation port. Both land with mulga-yxtda and the bridged WAN mode;
      # until then this cell proves the cluster forms and survives, not that a
      # customer can use it.
      - name: Install, form and verify the cluster
        working-directory: mulga/scripts/iso-builder
        run: |
          set -euo pipefail
          ./test/test-cluster.sh 2>&1 | tee /tmp/cell22-$GITHUB_RUN_ID.log

      - name: Collect logs
        if: always()
        run: |
          mkdir -p /tmp/cell22-artifacts-$GITHUB_RUN_ID
          cp /tmp/cell22-$GITHUB_RUN_ID.log /tmp/cell22-artifacts-$GITHUB_RUN_ID/ ||
            echo "::warning::no cluster log to collect"
          # Serial only. The same directory holds the nodes' disk images, which
          # are gigabytes and must not go into an artifact.
          cp "$WORK_DIR"/*-serial.log /tmp/cell22-artifacts-$GITHUB_RUN_ID/ 2>/dev/null ||
            echo "no per-node serial logs — the run passed and cleaned up after itself"

      - uses: actions/upload-artifact@v7.0.2
        if: always()
        with:
          name: e2e-artifacts-cell-22-${{ github.run_id }}
          path: /tmp/cell22-artifacts-${{ github.run_id }}
          if-no-files-found: warn
          retention-days: 90

      - name: Teardown
        if: always()
        run: |
          sudo pkill -f spinifex-cluster-test || true
          rm -rf /tmp/cell22-artifacts-$GITHUB_RUN_ID /tmp/cell22-$GITHUB_RUN_ID.log \
                 "$WORK_DIR"

  # Predastore throughput ("cell 23"). Predastore has no runners of its own and
  # spinifex depends on it, so the benchmark lives here and checks predastore
  # out as a sibling.
  #
  # needs: is the whole point. A throughput benchmark sharing a host with a cell
  # measures the neighbour, and ci-single and ci-multi both resolve to
  # bottlebrush, so "a different label" would not have separated them. needs: on
  # a matrix job waits for every leg, so this starts on a quiet machine.
  #
  # !cancelled() rather than always(): a failed cell must not silently skip the
  # benchmark and leave a green-looking night with no number in it, but always()
  # also overrides cancellation and keeps the run starting jobs after a cancel.
  performance:
    name: cell-23 (predastore/throughput)
    needs: [matrix, baremetal, iso-single, iso-multi]
    if: >-
      !cancelled() && (
      inputs.cells == '' ||
      contains(format(',{0},', inputs.cells), ',23,') )
    runs-on: [self-hosted, ci-single]
    timeout-minutes: 60
    env:
      # Two-minute samples. The 30s smoke preset is too short to compare nights.
      PERF_PRESET: compare
    steps:
      - name: Clean Workspace
        run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*

      - name: Configure work directories
        run: |
          E2E_ROOT=/mnt/nvme/e2e
          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
          echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-$GITHUB_RUN_ID" >> "$GITHUB_ENV"

      - name: Checkout Spinifex
        uses: actions/checkout@v7.0.1
        with:
          path: spinifex

      # Which predastore branch to measure: the same-named branch when one
      # exists, dev otherwise. Matches how the e2e path already picks siblings,
      # so a cross-repo change is benchmarked as a set.
      - name: Resolve sibling refs
        id: siblings
        env:
          BRANCH: ${{ github.ref_name }}
        run: bash .github/resolve-sibling-refs.sh
        working-directory: spinifex

      - name: Checkout Predastore
        uses: actions/checkout@v7.0.1
        with:
          repository: mulgadc/predastore
          ref: ${{ steps.siblings.outputs.predastore }}
          path: predastore

      # The benchmark works in $TMPDIR and predastore refuses writes below a 5%
      # free-space watermark. Without this the symptom is several hundred
      # "storage pool is at capacity" lines from warp and no statement of how
      # much space there actually was.
      #
      # TMPDIR is pointed at the runner's disk above, and the tmpfs assertion is
      # the one that matters: a storage benchmark on a RAM-backed filesystem
      # measures memcpy. It would report a number that looks excellent, sit in
      # the baseline, and never notice a disk regression.
      - name: Preflight
        run: |
          set -euo pipefail
          mkdir -p "$TMPDIR"
          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
          case "$fstype" in
            tmpfs|ramfs)
              echo "::error::$TMPDIR is $fstype — a storage benchmark there measures RAM"
              exit 1 ;;
          esac
          # Work dirs from runs that were killed before their cleanup trap ran.
          # -mtime +0 so a concurrent run's directory is never in scope.
          find "$TMPDIR" -maxdepth 1 -name 'predastore-e2e-performance.*' \
               -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
          df -h "$TMPDIR"
          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
          pct_used=$(df --output=pcent "$TMPDIR" | tail -1 | tr -dc '0-9')
          if [ "$pct_used" -ge 90 ] || [ "$avail_gb" -lt 20 ]; then
            echo "::error::$TMPDIR ($fstype) is ${pct_used}% full with ${avail_gb} GiB free."
            echo "::error::The benchmark needs headroom above predastore's 5% watermark."
            exit 1
          fi

      # PERF_PORT_OFFSET defaults to 10000, so the benchmark's own nodes do not
      # collide with a cluster left on the default ports.
      - name: Run throughput benchmark
        working-directory: predastore
        run: |
          set -euo pipefail
          # Into the results directory, so the artifact reads on its own rather
          # than needing the job log it was printed into. The benchmark only
          # ever creates this directory, so a log placed here first survives.
          mkdir -p "$PERF_RESULTS_ROOT"
          make e2e-performance 2>&1 | tee "$PERF_RESULTS_ROOT/e2e-performance.log"

      # Report-only. The baseline comparison and the -10% gate are mulga-wpjsx;
      # until the run-to-run spread is known from real nights, a threshold would
      # be a guess, and a gate that cries wolf gets muted.
      - uses: actions/upload-artifact@v7.0.2
        if: always()
        with:
          name: predastore-perf-${{ github.run_id }}
          path: ${{ env.PERF_RESULTS_ROOT }}
          if-no-files-found: warn
          retention-days: 90

      - name: Clean up results
        if: always()
        run: rm -rf "$PERF_RESULTS_ROOT"

  # Fault injection, not throughput. Cell-23 asks how fast predastore is when
  # every node is healthy; this asks whether it survives one that is not.
  #
  # After cell-23 rather than beside it: both want a quiet single runner, and a
  # benchmark sharing a machine with a fault injection run measures the wrong
  # thing. The port offset keeps them apart even so.
  stress:
    name: cell-24 (predastore/fault-injection)
    needs: [matrix, baremetal, iso-single, iso-multi, performance]
    if: >-
      !cancelled() && (
      inputs.cells == '' ||
      contains(format(',{0},', inputs.cells), ',24,') )
    runs-on: [self-hosted, ci-single]
    # The thirteen scenarios run serially and are ~35 minutes of work on their
    # own, measured from the per-scenario predastore matrix, before checkout
    # and build. 30 could not have passed whatever the disk did.
    timeout-minutes: 75
    env:
      # Freezes a follower. STRESS_HOST=leader is a known failure: a gate keeps
      # the frozen node first in its meta read order and pays the full client
      # timeout per key, so listing slows in proportion to object count.
      STRESS_HOST: follower
      # Clear of cell-23's default 10000, so the two never collide on a runner
      # that lets them overlap.
      STRESS_PORT_OFFSET: "20000"
    steps:
      - name: Clean Workspace
        run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*

      # STRESS_WORK_ROOT is set as well as TMPDIR because the harness ignores
      # TMPDIR for the scenarios that need the most disk: large-object and
      # multipart-upload default to $HOME/.cache/predastore-e2e, on the theory
      # that TMPDIR is commonly tmpfs. Here it is a 938G nvme, so pointing them
      # at it is both correct and the only path with room.
      - name: Configure work directories
        run: |
          E2E_ROOT=/mnt/nvme/e2e
          for d in work stress-work; do
            mkdir -p "$E2E_ROOT/$d" 2>/dev/null ||
              { sudo mkdir -p "$E2E_ROOT/$d" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/$d"; }
          done
          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
          echo "STRESS_WORK_ROOT=$E2E_ROOT/stress-work" >> "$GITHUB_ENV"
          echo "STRESS_RESULTS_ROOT=$E2E_ROOT/predastore-stress-$GITHUB_RUN_ID" >> "$GITHUB_ENV"

      - name: Checkout Spinifex
        uses: actions/checkout@v7.0.1
        with:
          path: spinifex

      - name: Resolve sibling refs
        id: siblings
        env:
          BRANCH: ${{ github.ref_name }}
        run: bash .github/resolve-sibling-refs.sh
        working-directory: spinifex

      - name: Checkout Predastore
        uses: actions/checkout@v7.0.1
        with:
          repository: mulgadc/predastore
          ref: ${{ steps.siblings.outputs.predastore }}
          path: predastore

      # A frozen node is indistinguishable from a slow one, so a run on a
      # loaded or nearly full disk fails as a timeout and reads as a real
      # fault. The checks are the cheap way to tell those apart afterwards.
      - name: Preflight
        run: |
          set -euo pipefail
          mkdir -p "$TMPDIR" "$STRESS_WORK_ROOT"
          for d in "$TMPDIR" "$STRESS_WORK_ROOT" "$HOME/.cache/predastore-e2e"; do
            [ -d "$d" ] || continue
            find "$d" -maxdepth 1 -name 'predastore-e2e-stress.*' \
                 -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
          done
          # Both paths, because they can be different filesystems and the one
          # holding the large objects is not the one TMPDIR names. large-object
          # needs 2GiB and 4GiB sharded with parity, multipart 1GiBx8.
          rc=0
          for d in "$TMPDIR" "$STRESS_WORK_ROOT"; do
            df -h "$d"
            avail_gb=$(df -BG --output=avail "$d" | tail -1 | tr -dc '0-9')
            if [ "$avail_gb" -lt 40 ]; then
              echo "::error::$d has ${avail_gb} GiB free; the run needs headroom above predastore's 5% watermark and room for the 1GiBx8 multipart case."
              rc=1
            fi
          done
          exit "$rc"

      # Leftovers from a run killed before its cleanup trap: a surviving node
      # holds its ports and fails this run somewhere unrelated.
      #
      # Under the host benchmark lock, because this pattern matches every
      # predastore node on the runner and the predastore repository runs the
      # same harnesses on the same machine. Holding the lock is what makes
      # "still running" mean "stale" rather than "somebody else's benchmark":
      # on 2026-09-16 this step killed a predastore run 32s into its workload.
      - name: Clear stale nodes
        run: |
          flock -w 3600 "${PREDA_BENCH_LOCK:-/var/lock/predastore-bench.lock}" \
            pkill -KILL -f '/bin/s3d -config' || true

      - name: Run fault injection
        working-directory: predastore
        run: |
          set -euo pipefail
          # events.txt carries the timeline and the FAIL line, but not the build
          # output or anything Warp wrote before the harness took over. The
          # harness only creates this directory, so a log placed here survives.
          mkdir -p "$STRESS_RESULTS_ROOT"
          make e2e-stress 2>&1 | tee "$STRESS_RESULTS_ROOT/e2e-stress.log"

      - uses: actions/upload-artifact@v7.0.2
        if: always()
        with:
          name: predastore-stress-${{ github.run_id }}
          path: ${{ env.STRESS_RESULTS_ROOT }}
          if-no-files-found: warn
          retention-days: 90

      # Under the lock for the same reason as "Clear stale nodes": by the time
      # this runs the harness has released it, so another repository's benchmark
      # can already have a cluster of its own up.
      - name: Clean up
        if: always()
        run: |
          flock -w 3600 "${PREDA_BENCH_LOCK:-/var/lock/predastore-bench.lock}" \
            pkill -KILL -f '/bin/s3d -config' || true
          rm -rf "$STRESS_RESULTS_ROOT"

  # Reading a nightly meant opening every cell job, and the Actions UI does not
  # separate a cell that failed from one cancelled out from under it. needs:
  # every cell so this runs last; always() so a red run is the one it describes.
  summary:
    name: run summary
    needs:
      [
        select,
        matrix,
        baremetal,
        diskperf,
        iso-single,
        iso-multi,
        performance,
        stress,
      ]
    if: always()
    runs-on: ubuntu-26.04
    steps:
      - uses: actions/checkout@v7.0.1
        with:
          sparse-checkout: .github/scripts/e2e-run-summary.py
          sparse-checkout-cone-mode: false

      - name: Collect junit
        uses: actions/download-artifact@v8.0.2
        with:
          pattern: junit-cell-*-${{ github.run_id }}
          path: junit
        continue-on-error: true

      - name: Write run summary
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          JUNIT_ROOT: junit
        run: python3 .github/scripts/e2e-run-summary.py