spinifex/.github/workflows/e2e-nightly.yml

name: E2E Nightly

on:
  schedule:
    - cron: "0 14 * * 0-4" # 00:00 Mon-Fri AEST = 14:00 Sun-Thu UTC
  workflow_dispatch:
    inputs:
      cells:
        description: "Comma-separated cell numbers to run (blank = all)"
        required: false

permissions:
  contents: read

# Scheduled runs are loaded from the default branch (main), so the schedule
# trigger must stay in sync on main for cron to fire.
env:
  GIT_BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
  UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-24.04.qcow2
  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
  # empty string disables telemetry entirely.
  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
  SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}

jobs:
  matrix:
    name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{
      matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
    strategy:
      fail-fast: false
      max-parallel: 3
      matrix:
        include:
          - { cell: 1,  install: binary, topo: single,     net: dhcp,   host: debian-13,    arch: x86_64, runner: ci-single, extra: "" }
          - { cell: 2,  install: binary, topo: single,     net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-single, extra: "" }
          - { cell: 3,  install: binary, topo: single,     net: dhcp,   host: ubuntu-24.04, arch: x86_64, runner: ci-single, extra: "" }
          - { cell: 8,  install: binary, topo: real-multi, net: dhcp,   host: debian-13,    arch: x86_64, runner: ci-multi,  extra: "" }
          - { cell: 9,  install: binary, topo: real-multi, net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-multi,  extra: "" }
          - { cell: 10, install: binary, topo: real-multi, net: dhcp,   host: ubuntu-24.04, arch: x86_64, runner: ci-multi,  extra: "" }
          - { cell: 17, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-pseudo, extra: "tofu-examples" }
          - { cell: 18, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-single, extra: "reboot-single" }
          - { cell: 19, install: binary, topo: single,     net: nat,    host: debian-13,    arch: x86_64, runner: ci-single, extra: "nat-single" }

    runs-on: [ self-hosted, "${{ matrix.runner }}" ]
    timeout-minutes: 35
    env:
      TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
      ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}

    steps:
      # blank or scheduled run = full matrix.
      # Input must be comma-separated with no whitespace (e.g. '1,2,17', not '1, 2').
      - name: Gate on Dispatch Input
        id: gate
        env:
          CELL: ${{ matrix.cell }}
          CELLS: ${{ github.event_name == 'workflow_dispatch' && inputs.cells || '' }}
        run: |
          if [ -z "$CELLS" ] || [[ ",${CELLS}," == *",${CELL},"* ]]; then
            echo "selected=true" >> "$GITHUB_OUTPUT"
            echo "cell ${CELL} selected"
          else
            echo "selected=false" >> "$GITHUB_OUTPUT"
            echo "cell ${CELL} not in '${CELLS}' — subsequent steps will skip"
          fi

      - name: Clean Workspace
        if: steps.gate.outputs.selected == 'true'
        run: |
          sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
          if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*

      - uses: actions/create-github-app-token@v3
        if: steps.gate.outputs.selected == 'true'
        id: app-token
        with:
          client-id: ${{ secrets.APP_ID }}
          private-key: ${{ secrets.APP_PRIVATE_KEY }}
          repositories: mulga

      - uses: actions/checkout@v7
        if: steps.gate.outputs.selected == 'true'
        with:
          repository: mulgadc/mulga
          token: ${{ steps.app-token.outputs.token }}
          sparse-checkout: scripts/tofu-cluster
          path: mulga

      - uses: actions/checkout@v7
        if: steps.gate.outputs.selected == 'true'
        with:
          path: spinifex

      - name: Build Install Artifacts
        if: steps.gate.outputs.selected == 'true'
        working-directory: mulga/scripts/tofu-cluster
        run: |
          ./build-install-artifacts.sh "${GIT_BRANCH}" \
            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
            "${RUNNER_TEMP}/setup.sh" \
            "${RUNNER_TEMP}/spinifex-tests.tar"

      - name: Provision VMs
        if: steps.gate.outputs.selected == 'true'
        working-directory: mulga/scripts/tofu-cluster
        env:
          MATRIX_HOST: ${{ matrix.host }}
        run: |
          export TF_VAR_git_branch="${GIT_BRANCH}"

          TOFU_VAR_ARGS=""
          if [ "$MATRIX_HOST" = "ubuntu-24.04" ]; then
            TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
          fi

          tofu init -input=false >> "$TOFU_LOG" 2>&1
          tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
          tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
          ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
          tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1

      - name: Bootstrap via Binary Install
        if: steps.gate.outputs.selected == 'true'
        working-directory: mulga/scripts/tofu-cluster
        env:
          MATRIX_NET: ${{ matrix.net }}
        run: |
          if [ "$MATRIX_NET" = "nat" ]; then
            # Routed-NAT mode with a static public pool: exercises both the
            # Tier-1 transit datapath and the Tier-2 EIP ingress lane.
            export SPX_EXTERNAL_MODE=nat
            export SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}"
          elif [ "$MATRIX_NET" = "dhcp" ]; then
            unset SPX_EXTERNAL_POOL
            export SPX_EXTERNAL_SOURCE=dhcp
          else
            export SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}"
          fi

          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
            "${RUNNER_TEMP}/setup.sh" \
            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"

      - name: Run Single Node E2E
        if: steps.gate.outputs.selected == 'true' && matrix.topo == 'single' && matrix.extra == ''
        working-directory: mulga/scripts/tofu-cluster
        run: |
          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
             for SUITE in single cert lb; do \
               echo \"::group::suite \$SUITE\"; \
               date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
               GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                 ARTIFACT_DIR='${ARTIFACT_DIR}' \
                 ./\$SUITE.test -test.v -test.timeout=30m \
                 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
               rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
               if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
             done; exit \$FAIL"

      - name: Run NAT Uplink E2E
        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'nat-single'
        working-directory: mulga/scripts/tofu-cluster
        # Skips cleanly (exit 0) when the tested branch predates the natuplink
        # suite so scheduled runs against dev stay green until the merge lands.
        run: |
          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
             if [ ! -f natuplink.test ]; then echo 'natuplink.test not in tests tar (branch predates suite) — skipping'; exit 0; fi; \
             echo '::group::suite natuplink'; \
             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-natuplink.start'; \
             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
               ARTIFACT_DIR='${ARTIFACT_DIR}' \
               ./natuplink.test -test.v -test.timeout=30m \
               2>&1 | tee '${ARTIFACT_DIR}/test-natuplink.log'; \
             rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
             if [ \$rc -ne 0 ]; then echo 'natuplink: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"

      - name: Run Multi Node E2E
        if: steps.gate.outputs.selected == 'true' && matrix.topo == 'real-multi'
        working-directory: mulga/scripts/tofu-cluster
        run: |
          INVENTORY=$(tofu output -json inventory)
          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"

          NODES_CSV=""
          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
          for i in $(seq 0 $((NODE_COUNT - 1))); do
            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
            NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip"
          done

          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
             for SUITE in multinode cert lb; do \
               echo \"::group::suite \$SUITE\"; \
               date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
               GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
                 SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
                 SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
                 ARTIFACT_DIR='${ARTIFACT_DIR}' \
                 ./\$SUITE.test -test.v -test.timeout=30m \
                 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
               rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
               if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
             done; exit \$FAIL"

      - name: Run Tofu Workbooks
        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples'
        working-directory: mulga/scripts/tofu-cluster
        # The driver emits go test -v markers per workbook; teeing to
        # test-tofu.log feeds the same go-junit-report -> e2e-analyze -> upload
        # chain the Go suites use, so cell 17 produces an RCA bundle too.
        run: |
          set +e
          mkdir -p "${ARTIFACT_DIR}"
          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
          date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-tofu.start"
          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
            "cd ~/e2e-tests && ./run-tofu-examples-e2e.sh" \
            2>&1 | tee "${ARTIFACT_DIR}/test-tofu.log"
          rc=${PIPESTATUS[0]}
          if [ $rc -ne 0 ]; then echo "tofu: FAILED" >> "${ARTIFACT_DIR}/suite-status.txt"; fi
          exit $rc

      - name: Run Reboot Resilience E2E
        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'reboot-single'
        working-directory: mulga/scripts/tofu-cluster
        # reboot.test runs from the runner (NOT via ssh-and-run) so it survives the
        # systemctl-reboot-induced ssh disconnect mid-run.
        run: |
          set +e
          mkdir -p "${ARTIFACT_DIR}"
          INVENTORY=$(tofu output -json inventory)
          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"

          REBOOT_DIR="${RUNNER_TEMP}/reboot-e2e"
          mkdir -p "$REBOOT_DIR"
          tar xf "${RUNNER_TEMP}/spinifex-tests.tar" -C "$REBOOT_DIR"
          chmod +x "$REBOOT_DIR/_bin/reboot.test"

          SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$SSH_KEY")
          CREDS=$(ssh "${SSH_OPTS[@]}" tf-user@${WAN_IP} 'cat ~/.aws/credentials')
          AWS_AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
          AWS_SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
          echo "::add-mask::${AWS_AK}"; echo "::add-mask::${AWS_SK}"

          date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-reboot.start"
          GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
            SPINIFEX_WAN_IP="${WAN_IP}" SPINIFEX_SSH_KEY="${SSH_KEY}" \
            SPINIFEX_AWS_ENDPOINT="https://${WAN_IP}:9999" \
            SPINIFEX_AWS_ACCESS_KEY_ID="${AWS_AK}" SPINIFEX_AWS_SECRET_ACCESS_KEY="${AWS_SK}" \
            ARTIFACT_DIR="${ARTIFACT_DIR}" \
            "$REBOOT_DIR/_bin/reboot.test" -test.v -test.timeout=30m \
            2>&1 | tee "${ARTIFACT_DIR}/test-reboot.log"
          rc=${PIPESTATUS[0]}
          if [ $rc -ne 0 ]; then echo "reboot: FAILED" >> "${ARTIFACT_DIR}/suite-status.txt"; fi
          exit $rc

      - name: Collect e2e artifacts
        if: always() && steps.gate.outputs.selected == 'true'
        working-directory: mulga/scripts/tofu-cluster
        run: |
          set +e
          mkdir -p "${ARTIFACT_DIR}"
          INVENTORY=$(tofu output -json inventory 2>/dev/null) || { echo "no inventory; skipping"; exit 0; }
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path'); SSH_KEY="${SSH_KEY/#\~/$HOME}"
          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY")

          ssh "${SSH_OPTS[@]}" tf-user@${NODE1_IP} "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
            | tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null

          for i in $(seq 0 $((NODE_COUNT - 1))); do
            IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip"); idx=$((i + 1))
            mkdir -p "${ARTIFACT_DIR}/vm-console-node${idx}"
            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
              | tar -C "${ARTIFACT_DIR}/vm-console-node${idx}" -xf - 2>/dev/null
            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
              > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
          done

          cat "${ARTIFACT_DIR}"/spinifex-journal-node*.log \
            > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || true
          echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"

      - name: Install go-junit-report
        if: always() && steps.gate.outputs.selected == 'true'
        run: |
          go install github.com/jstemmer/go-junit-report/v2@latest
          echo "$HOME/go/bin" >> "$GITHUB_PATH"

      - name: Convert test logs to JUnit XML
        if: always() && steps.gate.outputs.selected == 'true'
        run: |
          shopt -s nullglob
          for LOG in "${ARTIFACT_DIR}"/test-*.log; do
            BASE=$(basename "$LOG" .log); SUITE=${BASE#test-}
            go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
          done

      - name: Analyze e2e failures
        if: always() && steps.gate.outputs.selected == 'true'
        uses: ./spinifex/.github/actions/e2e-analyze
        with:
          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
          log-dir: ${{ env.ARTIFACT_DIR }}
          title: "E2E failure analysis (cell ${{ matrix.cell }})"

      - uses: actions/upload-artifact@v7
        if: always() && steps.gate.outputs.selected == 'true'
        with:
          name: e2e-artifacts-cell-${{ matrix.cell }}-${{ github.run_id }}
          path: ${{ env.ARTIFACT_DIR }}
          if-no-files-found: warn
          retention-days: 14

      - name: Teardown
        if: always()
        working-directory: mulga/scripts/tofu-cluster
        env:
          MATRIX_HOST: ${{ matrix.host }}
        run: |
          TOFU_VAR_ARGS=""
          if [ "$MATRIX_HOST" = "ubuntu-24.04" ]; then
            TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
          fi
          tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
          timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
          tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
          ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true

  # Bare-metal single-node suite. Doesn't fit the tofu matrix shape, so it
  # runs as a sibling job via the reusable e2e-baremetal workflow. Gated like a
  # matrix cell ("cell 20"): every weeknight cron, or dispatch with 20 selected
  # (blank cells = all). Tests `dev` on schedule to match GIT_BRANCH above.
  baremetal:
    name: cell-20 (baremetal/pxe/real-hw)
    if: >-
      github.event_name == 'schedule' ||
      inputs.cells == '' ||
      contains(format(',{0},', inputs.cells), ',20,')
    permissions:
      contents: read
      statuses: write
    uses: ./.github/workflows/e2e-baremetal.yml
    with:
      ref: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
    secrets: inherit