mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 19:01:42 +00:00
Josh Siviour
0cdf591c7c
Cells 2, 3, 9 and 10 ran on spinifex-image-builder-ubuntu-24.04.qcow2, built 2026-04-20 — five weeks before openvswitch-ipsec and strongswan-charon were added to the image builder. Those cells therefore had no OVN IPsec support and ran intra-AZ Geneve in plaintext. Every other workflow is already on 26.04. The host_image_override guards now match ubuntu-* rather than a pinned version. An exact match is what let this drift go unnoticed: bumping the matrix release silently stops overriding the image and drops the cell onto the debian gold.
417 lines
21 KiB
YAML
417 lines
21 KiB
YAML
name: E2E Nightly
|
|
|
|
on:
|
|
schedule:
|
|
- cron: "0 14 * * 0-4" # 00:00 Mon-Fri AEST = 14:00 Sun-Thu UTC
|
|
workflow_dispatch:
|
|
inputs:
|
|
cells:
|
|
description: "Comma-separated cell numbers to run (blank = all)"
|
|
required: false
|
|
|
|
permissions:
|
|
contents: read
|
|
# actions: read lets the "Resolve CI job id" step below look up this job's
|
|
# numeric id via the Actions API — github.job is only the YAML key, no
|
|
# context exposes the id itself.
|
|
actions: read
|
|
|
|
# Scheduled runs are loaded from the default branch (main), so the schedule
|
|
# trigger must stay in sync on main for cron to fire.
|
|
env:
|
|
GIT_BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
|
|
UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-26.04.qcow2
|
|
# OTLP export to the central Elastic sink (wattle). Repo variable overrides;
|
|
# empty string disables telemetry entirely.
|
|
SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
|
|
SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
|
|
|
|
jobs:
|
|
matrix:
|
|
name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{
|
|
matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
|
|
strategy:
|
|
fail-fast: false
|
|
max-parallel: 3
|
|
matrix:
|
|
include:
|
|
- { cell: 1, install: binary, topo: single, net: dhcp, host: debian-13, arch: x86_64, runner: ci-single, extra: "" }
|
|
- { cell: 2, install: binary, topo: single, net: static, host: ubuntu-26.04, arch: x86_64, runner: ci-single, extra: "" }
|
|
- { cell: 3, install: binary, topo: single, net: dhcp, host: ubuntu-26.04, arch: x86_64, runner: ci-single, extra: "" }
|
|
- { cell: 8, install: binary, topo: real-multi, net: dhcp, host: debian-13, arch: x86_64, runner: ci-multi, extra: "" }
|
|
- { cell: 9, install: binary, topo: real-multi, net: static, host: ubuntu-26.04, arch: x86_64, runner: ci-multi, extra: "" }
|
|
- { cell: 10, install: binary, topo: real-multi, net: dhcp, host: ubuntu-26.04, arch: x86_64, runner: ci-multi, extra: "" }
|
|
- { cell: 17, install: binary, topo: single, net: static, host: debian-13, arch: x86_64, runner: ci-pseudo, extra: "tofu-examples" }
|
|
- { cell: 18, install: binary, topo: single, net: static, host: debian-13, arch: x86_64, runner: ci-single, extra: "reboot-single" }
|
|
- { cell: 19, install: binary, topo: single, net: nat, host: debian-13, arch: x86_64, runner: ci-single, extra: "nat-single" }
|
|
|
|
runs-on: [ self-hosted, "${{ matrix.runner }}" ]
|
|
timeout-minutes: 35
|
|
env:
|
|
TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
|
|
ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}
|
|
# Job-level env wins over the workflow-level default above, so telemetry
|
|
# from this cell carries ci.cell/ci.matrix. Nine cells run under one
|
|
# run id and every cell names its hosts node1/node2/node3, so without
|
|
# this every cell's telemetry collapses into one unattributable bucket.
|
|
SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=${{ matrix.cell }},ci.matrix=${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}
|
|
|
|
steps:
|
|
# blank or scheduled run = full matrix.
|
|
# Input must be comma-separated with no whitespace (e.g. '1,2,17', not '1, 2').
|
|
- name: Gate on Dispatch Input
|
|
id: gate
|
|
env:
|
|
CELL: ${{ matrix.cell }}
|
|
CELLS: ${{ github.event_name == 'workflow_dispatch' && inputs.cells || '' }}
|
|
run: |
|
|
if [ -z "$CELLS" ] || [[ ",${CELLS}," == *",${CELL},"* ]]; then
|
|
echo "selected=true" >> "$GITHUB_OUTPUT"
|
|
echo "cell ${CELL} selected"
|
|
else
|
|
echo "selected=false" >> "$GITHUB_OUTPUT"
|
|
echo "cell ${CELL} not in '${CELLS}' — subsequent steps will skip"
|
|
fi
|
|
|
|
- name: Clean Workspace
|
|
if: steps.gate.outputs.selected == 'true'
|
|
run: |
|
|
sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
|
|
if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
|
|
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
|
|
|
- uses: actions/create-github-app-token@v3
|
|
if: steps.gate.outputs.selected == 'true'
|
|
id: app-token
|
|
with:
|
|
client-id: ${{ secrets.APP_ID }}
|
|
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
|
repositories: mulga
|
|
|
|
- uses: actions/checkout@v7
|
|
if: steps.gate.outputs.selected == 'true'
|
|
with:
|
|
repository: mulgadc/mulga
|
|
token: ${{ steps.app-token.outputs.token }}
|
|
sparse-checkout: scripts/tofu-cluster
|
|
path: mulga
|
|
|
|
- uses: actions/checkout@v7
|
|
if: steps.gate.outputs.selected == 'true'
|
|
with:
|
|
path: spinifex
|
|
|
|
# Nine cells share one run id and every cell names its hosts
|
|
# node1/node2/node3, so ci.cell above separates them and ci.job_id here
|
|
# links each cell's telemetry to the job log that produced it. Runs
|
|
# before Bootstrap bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env.
|
|
- name: Resolve CI job id
|
|
if: steps.gate.outputs.selected == 'true'
|
|
uses: ./spinifex/.github/actions/otel-job-id
|
|
with:
|
|
job-name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
|
|
|
|
- name: Build Install Artifacts
|
|
if: steps.gate.outputs.selected == 'true'
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
./build-install-artifacts.sh "${GIT_BRANCH}" \
|
|
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
|
|
"${RUNNER_TEMP}/setup.sh" \
|
|
"${RUNNER_TEMP}/spinifex-tests.tar"
|
|
|
|
- name: Provision VMs
|
|
if: steps.gate.outputs.selected == 'true'
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
env:
|
|
MATRIX_HOST: ${{ matrix.host }}
|
|
run: |
|
|
export TF_VAR_git_branch="${GIT_BRANCH}"
|
|
|
|
# Match the family, not a pinned version: an exact match silently stops
|
|
# overriding the image when the matrix bumps release, dropping the cell
|
|
# back onto the debian gold.
|
|
TOFU_VAR_ARGS=""
|
|
case "$MATRIX_HOST" in
|
|
ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
|
|
esac
|
|
|
|
tofu init -input=false >> "$TOFU_LOG" 2>&1
|
|
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
|
|
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
|
|
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
|
|
tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
|
|
|
|
- name: Bootstrap via Binary Install
|
|
if: steps.gate.outputs.selected == 'true'
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
env:
|
|
MATRIX_NET: ${{ matrix.net }}
|
|
run: |
|
|
if [ "$MATRIX_NET" = "nat" ]; then
|
|
# Routed-NAT mode with a static public pool: exercises both the
|
|
# Tier-1 transit datapath and the Tier-2 EIP ingress lane.
|
|
export SPX_EXTERNAL_MODE=nat
|
|
export SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}"
|
|
elif [ "$MATRIX_NET" = "dhcp" ]; then
|
|
unset SPX_EXTERNAL_POOL
|
|
export SPX_EXTERNAL_SOURCE=dhcp
|
|
else
|
|
export SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}"
|
|
fi
|
|
|
|
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
|
|
"${RUNNER_TEMP}/setup.sh" \
|
|
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
|
|
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
|
|
|
|
- name: Run Single Node E2E
|
|
if: steps.gate.outputs.selected == 'true' && matrix.topo == 'single' && matrix.extra == ''
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
set -euo pipefail
|
|
# Nightly runs a documented subset of the single-node set; the assert
|
|
# fails the cell if that subset ever names a suite this topology
|
|
# cannot fully exercise.
|
|
. "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
|
|
e2e_suites_assert_subset "e2e-nightly single" "${E2E_SUITES_NIGHTLY_SINGLE}" "${E2E_SUITES_SINGLE}"
|
|
|
|
INVENTORY=$(tofu output -json inventory)
|
|
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
|
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
|
|
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
|
|
for SUITE in ${E2E_SUITES_NIGHTLY_SINGLE}; do \
|
|
echo \"::group::suite \$SUITE\"; \
|
|
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
|
|
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
|
|
ARTIFACT_DIR='${ARTIFACT_DIR}' \
|
|
./\$SUITE.test -test.v -test.timeout=30m \
|
|
2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
|
|
rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
|
|
if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
|
|
done; exit \$FAIL"
|
|
|
|
- name: Run NAT Uplink E2E
|
|
if: steps.gate.outputs.selected == 'true' && matrix.extra == 'nat-single'
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
# Skips cleanly (exit 0) when the tested branch predates the natuplink
|
|
# suite so scheduled runs against dev stay green until the merge lands.
|
|
run: |
|
|
INVENTORY=$(tofu output -json inventory)
|
|
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
|
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
|
|
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
|
|
if [ ! -f natuplink.test ]; then echo 'natuplink.test not in tests tar (branch predates suite) — skipping'; exit 0; fi; \
|
|
echo '::group::suite natuplink'; \
|
|
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-natuplink.start'; \
|
|
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
|
|
ARTIFACT_DIR='${ARTIFACT_DIR}' \
|
|
./natuplink.test -test.v -test.timeout=30m \
|
|
2>&1 | tee '${ARTIFACT_DIR}/test-natuplink.log'; \
|
|
rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
|
|
if [ \$rc -ne 0 ]; then echo 'natuplink: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"
|
|
|
|
- name: Run Multi Node E2E
|
|
if: steps.gate.outputs.selected == 'true' && matrix.topo == 'real-multi'
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
set -euo pipefail
|
|
# See the single-node step: same subset guard, multi-node set.
|
|
. "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
|
|
e2e_suites_assert_subset "e2e-nightly multi" "${E2E_SUITES_NIGHTLY_MULTI}" "${E2E_SUITES_MULTI}"
|
|
|
|
INVENTORY=$(tofu output -json inventory)
|
|
NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
|
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
|
|
NODES_CSV=""
|
|
NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
|
|
for i in $(seq 0 $((NODE_COUNT - 1))); do
|
|
ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
|
|
NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip"
|
|
done
|
|
|
|
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
|
|
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
|
|
for SUITE in ${E2E_SUITES_NIGHTLY_MULTI}; do \
|
|
echo \"::group::suite \$SUITE\"; \
|
|
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
|
|
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
|
|
SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
|
|
SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
|
|
ARTIFACT_DIR='${ARTIFACT_DIR}' \
|
|
./\$SUITE.test -test.v -test.timeout=30m \
|
|
2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
|
|
rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
|
|
if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
|
|
done; exit \$FAIL"
|
|
|
|
- name: Run Tofu Workbooks
|
|
if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples'
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
# The driver emits go test -v markers per workbook; teeing to
|
|
# test-tofu.log feeds the same go-junit-report -> e2e-analyze -> upload
|
|
# chain the Go suites use, so cell 17 produces an RCA bundle too.
|
|
run: |
|
|
set +e
|
|
mkdir -p "${ARTIFACT_DIR}"
|
|
INVENTORY=$(tofu output -json inventory)
|
|
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
|
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-tofu.start"
|
|
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
|
|
"cd ~/e2e-tests && ./run-tofu-examples-e2e.sh" \
|
|
2>&1 | tee "${ARTIFACT_DIR}/test-tofu.log"
|
|
rc=${PIPESTATUS[0]}
|
|
if [ $rc -ne 0 ]; then echo "tofu: FAILED" >> "${ARTIFACT_DIR}/suite-status.txt"; fi
|
|
exit $rc
|
|
|
|
- name: Run Reboot Resilience E2E
|
|
if: steps.gate.outputs.selected == 'true' && matrix.extra == 'reboot-single'
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
# reboot.test runs from the runner (NOT via ssh-and-run) so it survives the
|
|
# systemctl-reboot-induced ssh disconnect mid-run.
|
|
run: |
|
|
set +e
|
|
mkdir -p "${ARTIFACT_DIR}"
|
|
INVENTORY=$(tofu output -json inventory)
|
|
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
|
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
|
|
REBOOT_DIR="${RUNNER_TEMP}/reboot-e2e"
|
|
mkdir -p "$REBOOT_DIR"
|
|
tar xf "${RUNNER_TEMP}/spinifex-tests.tar" -C "$REBOOT_DIR"
|
|
chmod +x "$REBOOT_DIR/_bin/reboot.test"
|
|
|
|
SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$SSH_KEY")
|
|
CREDS=$(ssh "${SSH_OPTS[@]}" tf-user@${WAN_IP} 'cat ~/.aws/credentials')
|
|
AWS_AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
|
|
AWS_SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
|
|
echo "::add-mask::${AWS_AK}"; echo "::add-mask::${AWS_SK}"
|
|
|
|
date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-reboot.start"
|
|
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
|
|
SPINIFEX_WAN_IP="${WAN_IP}" SPINIFEX_SSH_KEY="${SSH_KEY}" \
|
|
SPINIFEX_AWS_ENDPOINT="https://${WAN_IP}:9999" \
|
|
SPINIFEX_AWS_ACCESS_KEY_ID="${AWS_AK}" SPINIFEX_AWS_SECRET_ACCESS_KEY="${AWS_SK}" \
|
|
ARTIFACT_DIR="${ARTIFACT_DIR}" \
|
|
"$REBOOT_DIR/_bin/reboot.test" -test.v -test.timeout=30m \
|
|
2>&1 | tee "${ARTIFACT_DIR}/test-reboot.log"
|
|
rc=${PIPESTATUS[0]}
|
|
if [ $rc -ne 0 ]; then echo "reboot: FAILED" >> "${ARTIFACT_DIR}/suite-status.txt"; fi
|
|
exit $rc
|
|
|
|
- name: Collect e2e artifacts
|
|
if: always() && steps.gate.outputs.selected == 'true'
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
set +e
|
|
mkdir -p "${ARTIFACT_DIR}"
|
|
INVENTORY=$(tofu output -json inventory 2>/dev/null) || { echo "no inventory; skipping"; exit 0; }
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path'); SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
|
|
NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
|
SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY")
|
|
|
|
ssh "${SSH_OPTS[@]}" tf-user@${NODE1_IP} "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
|
|
| tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
|
|
|
|
for i in $(seq 0 $((NODE_COUNT - 1))); do
|
|
IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip"); idx=$((i + 1))
|
|
mkdir -p "${ARTIFACT_DIR}/vm-console-node${idx}"
|
|
ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
|
|
| tar -C "${ARTIFACT_DIR}/vm-console-node${idx}" -xf - 2>/dev/null
|
|
ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
|
|
> "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
|
|
done
|
|
|
|
cat "${ARTIFACT_DIR}"/spinifex-journal-node*.log \
|
|
> "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || true
|
|
echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"
|
|
|
|
- name: Install go-junit-report
|
|
if: always() && steps.gate.outputs.selected == 'true'
|
|
run: |
|
|
go install github.com/jstemmer/go-junit-report/v2@latest
|
|
echo "$HOME/go/bin" >> "$GITHUB_PATH"
|
|
|
|
- name: Convert test logs to JUnit XML
|
|
if: always() && steps.gate.outputs.selected == 'true'
|
|
run: |
|
|
shopt -s nullglob
|
|
for LOG in "${ARTIFACT_DIR}"/test-*.log; do
|
|
BASE=$(basename "$LOG" .log); SUITE=${BASE#test-}
|
|
go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
|
|
done
|
|
|
|
- name: Analyze e2e failures
|
|
if: always() && steps.gate.outputs.selected == 'true'
|
|
uses: ./spinifex/.github/actions/e2e-analyze
|
|
with:
|
|
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
|
log-dir: ${{ env.ARTIFACT_DIR }}
|
|
title: "E2E failure analysis (cell ${{ matrix.cell }})"
|
|
|
|
- uses: actions/upload-artifact@v7
|
|
if: always() && steps.gate.outputs.selected == 'true'
|
|
with:
|
|
name: e2e-artifacts-cell-${{ matrix.cell }}-${{ github.run_id }}
|
|
path: ${{ env.ARTIFACT_DIR }}
|
|
if-no-files-found: warn
|
|
retention-days: 14
|
|
|
|
- name: Teardown
|
|
if: always()
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
env:
|
|
MATRIX_HOST: ${{ matrix.host }}
|
|
run: |
|
|
# Must mirror the Provision step's override, or destroy runs against a
|
|
# different image path than apply did.
|
|
TOFU_VAR_ARGS=""
|
|
case "$MATRIX_HOST" in
|
|
ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
|
|
esac
|
|
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
|
|
timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
|
|
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
|
|
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
|
|
|
|
- name: Upload tofu log
|
|
if: always()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: tofu-log-cell-${{ matrix.cell }}-${{ github.run_id }}
|
|
path: ${{ env.TOFU_LOG }}
|
|
if-no-files-found: ignore
|
|
retention-days: 14
|
|
|
|
# Bare-metal single-node suite. Doesn't fit the tofu matrix shape, so it
|
|
# runs as a sibling job via the reusable e2e-baremetal workflow. Gated like a
|
|
# matrix cell ("cell 20"): every weeknight cron, or dispatch with 20 selected
|
|
# (blank cells = all). Tests `dev` on schedule to match GIT_BRANCH above.
|
|
baremetal:
|
|
name: cell-20 (baremetal/pxe/real-hw)
|
|
if: >-
|
|
github.event_name == 'schedule' ||
|
|
inputs.cells == '' ||
|
|
contains(format(',{0},', inputs.cells), ',20,')
|
|
permissions:
|
|
contents: read
|
|
statuses: write
|
|
uses: ./.github/workflows/e2e-baremetal.yml
|
|
with:
|
|
ref: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
|
|
# A reusable-workflow call gets none of this workflow's context (env:
|
|
# above, matrix.cell) for free — cell-20 needs its own attrs string.
|
|
# There's no matrix leg here, just the one fixed baremetal cell.
|
|
otel-attrs: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=20,ci.matrix=baremetal/pxe/real-hw
|
|
secrets: inherit
|