gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-20 11:21:49 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
412eb618b01b1b844b09da7a9c01b2e02c03216a
spinifex/.github/workflows/e2e.yml
T
joshsiv-mulga af690b3299 test(elbv2): collect LB microVM guest console in CI + surface NLB probe failures 
NLB target health stalls at 0/N in CI while the data plane works locally.
The lb-agent's nginx engine activation, any nginx -t/reload error, and its
active-probe results are logged to the LB microVM serial console — which the
host journal never captures and the GetConsoleOutput API path reports as
NotFound, so the failure is undiagnosable from CI artifacts.

- e2e.yml: pull /run/spinifex/console-*.log off each node in the single- and
  multi-node collect steps (files persist past VM teardown) into lb-console/.
- lbagent healthcheck: log probe failures at Warn (was Debug) so unreachable
  targets surface at the agent's default level on the captured console.
2026-06-05 09:30:24 +10:00

1049 lines
45 KiB
YAML
Raw Blame History

name: E2E
on:
push:
branches:
- main
- dev
paths:
- "**/*.go"
- go.mod
- go.sum
- Makefile
- tests/e2e/**
- .github/workflows/e2e.yml
workflow_dispatch:
inputs:
sha:
description: "PR head SHA to report status on"
required: false
suites:
description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb')"
required: false
default: "e2e-single e2e-multinode e2e-cert e2e-lb"
test_run:
description: "Optional regex for -test.run (e.g. '^TestMultinodeIPSec$'). Empty = run all tests in suite."
required: false
default: ""
permissions:
contents: read
statuses: write
env:
GIT_BRANCH: ${{ github.ref_name }}
SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb' }}
TEST_RUN: ${{ inputs.test_run || '' }}
jobs:
e2e_single:
name: Single-Node E2E
# Skip when the dispatch input requests no in-VM suites (e.g.
# suites='e2e-reboot' only). Push triggers and the default SUITES list
# both include single/cert/lb so the inputs.suites null-check keeps the
# job running on plain PR pushes.
if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-lb')
runs-on: [ self-hosted, ci-single ]
timeout-minutes: 45
env:
TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
steps:
- name: Reap Orphan Procs
run: |
set +e
UID_SELF=$(id -u)
WS="${GITHUB_WORKSPACE}"
reap() {
local sig="$1"
for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
case "$cwd" in
"$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
esac
done
}
reap TERM
sleep 3
reap KILL
find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
true
- name: Clean Workspace
run: sudo rm -rf "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v6
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- uses: actions/checkout@v6
with:
path: spinifex
- uses: actions/setup-go@v6
with:
go-version-file: spinifex/go.mod
cache: false
- name: Re-enable Go toolchain auto-download
run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
- name: Prune rootless docker cache
run: |
docker container prune -f || true
docker image prune -af || true
docker buildx prune -af --filter "until=24h" || true
docker system df || true
- name: Build Install Artifacts
working-directory: mulga/scripts/tofu-cluster
run: |
./build-install-artifacts.sh "${GIT_BRANCH}" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-tests.tar"
- name: Provision VM
working-directory: mulga/scripts/tofu-cluster
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
export TF_VAR_git_branch="${GIT_BRANCH}"
quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false
quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
quiet_run "scrub-env (pre-clean)" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
quiet_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
ok "VM provisioned"
- name: Bootstrap via Binary Install
working-directory: mulga/scripts/tofu-cluster
run: |
SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
- name: Resolve VM SSH details
id: vm
working-directory: mulga/scripts/tofu-cluster
run: |
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT"
echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
- name: Build E2E test binaries on runner
working-directory: spinifex/tests/e2e
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|multinode)$' | tr '\n' ' ')
if [ -z "$(echo "${SUITES_BUILD}" | tr -d ' ')" ]; then
echo "no in-VM suites requested (SUITES=${SUITES}); nothing to build"
exit 0
fi
banner "build ${SUITES_BUILD// /, }"
group_open "go test -c"
make e2e-build SUITES_BUILD="${SUITES_BUILD}"
group_close
for f in _bin/*.test; do
ok "$(basename "$f") $(stat -c '%s' "$f" | numfmt --to=iec)"
done
- name: Install go-junit-report on runner
run: |
go install github.com/jstemmer/go-junit-report/v2@latest
echo "$HOME/go/bin" >> "$GITHUB_PATH"
- name: Ship test binaries to VM
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin"
tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin"
ok "shipped binaries to $VM"
- name: Start packet capture on VM br-wan
run: |
set +e
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
scp "${SSH_OPTS[@]}" \
"${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \
"$VM":/tmp/start-net-capture.sh
ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/start-net-capture.sh" || true
- name: Run E2E suites
id: e2e
run: |
set +e
SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|multinode)$' | tr '\n' ' ')
if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
echo "no in-VM suites requested (SUITES=${SUITES}); skipping"
echo "rc=0" >> "$GITHUB_OUTPUT"
exit 0
fi
if [ -n "${TEST_RUN}" ]; then
TEST_RUN_FLAG="-test.run=${TEST_RUN}"
else
TEST_RUN_FLAG=""
fi
VM_HOST="tf-user@${{ steps.vm.outputs.wan_ip }}"
SSH_KEY="${{ steps.vm.outputs.ssh_key }}"
cleanup_remote() {
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \
'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
2>/dev/null || true
}
trap cleanup_remote EXIT INT TERM
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" \
"$VM_HOST" \
"set -u; mkdir -p '${ARTIFACT_DIR}'; \
cd \$HOME/spinifex-e2e-bin; \
R=\$'\033[0m' B=\$'\033[1m' C=\$'\033[36m' G=\$'\033[32m' RED=\$'\033[31m'; \
FAIL=0; \
for SUITE in ${SUITES_RUN}; do \
echo \"::group::suite \$SUITE\"; \
printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
ARTIFACT_DIR='${ARTIFACT_DIR}' \
./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \
| sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
rc=\${PIPESTATUS[0]}; \
echo \"::endgroup::\"; \
if [ \$rc -ne 0 ]; then \
printf '%s✗%s suite %s failed (exit %d)\n' \"\$RED\" \"\$R\" \"\$SUITE\" \"\$rc\"; \
echo \"::error::suite \$SUITE failed (exit \$rc)\"; \
echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; \
FAIL=1; \
else \
printf '%s✓%s suite %s passed\n' \"\$G\" \"\$R\" \"\$SUITE\"; \
fi; \
done; \
exit \$FAIL"
E2E_RC=$?
echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
exit 0
- name: Capture network/IPsec diagnostics on VM
if: always()
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
scp "${SSH_OPTS[@]}" \
"${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
"$VM":/tmp/capture-net-diag.sh
ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag'"
ok "net diagnostics captured"
- name: Collect artifacts from VM
if: always()
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
mkdir -p "${ARTIFACT_DIR}"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
ssh "${SSH_OPTS[@]}" "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
| tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
ok "artifacts pulled to ${ARTIFACT_DIR}"
ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
> "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null
ok "journal: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || echo 0) lines"
# LB/system microVM serial consoles. The lb-agent logs its nginx (NLB)
# engine activation and any `nginx -t`/reload error + active-probe
# results to the guest serial console, which the host journal never
# captures. These files persist in /run/spinifex past VM teardown, so
# collecting them here makes an NLB "0/N healthy" timeout diagnosable.
mkdir -p "${ARTIFACT_DIR}/lb-console"
ssh "${SSH_OPTS[@]}" "$VM" 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
| tar -C "${ARTIFACT_DIR}/lb-console" -xf - 2>/dev/null
ok "lb-console: $(ls "${ARTIFACT_DIR}/lb-console" 2>/dev/null | wc -l) files"
- name: Convert test logs to JUnit XML
if: always()
run: |
shopt -s nullglob
for LOG in "${ARTIFACT_DIR}"/test-*.log; do
BASE=$(basename "$LOG" .log)
SUITE=${BASE#test-}
go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
done
- name: Publish test summary
if: always()
uses: ./spinifex/.github/actions/junit-report
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
title: "E2E results"
- name: Analyze e2e failures
if: always()
uses: ./spinifex/.github/actions/e2e-analyze
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
log-dir: ${{ env.ARTIFACT_DIR }}
title: "E2E failure analysis"
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-artifacts-${{ github.run_id }}
path: ${{ env.ARTIFACT_DIR }}
if-no-files-found: warn
retention-days: 14
- name: Teardown
if: always()
working-directory: mulga/scripts/tofu-cluster
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
if [ -n "${{ steps.vm.outputs.wan_ip }}" ] && [ -n "${{ steps.vm.outputs.ssh_key }}" ]; then
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
-i "${{ steps.vm.outputs.ssh_key }}" \
tf-user@${{ steps.vm.outputs.wan_ip }} \
'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
2>/dev/null || true
fi
UID_SELF=$(id -u)
WS="${GITHUB_WORKSPACE}"
reap() {
local sig="$1"
for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
case "$cwd" in
"$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;;
esac
done
}
reap TERM
sleep 2
reap KILL
find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
quiet_run "tofu destroy" "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
ok "teardown complete"
- name: Fail job if any suite failed
if: always()
run: |
if [ "${{ steps.e2e.outputs.rc }}" != "0" ] || [ -s "${ARTIFACT_DIR}/suite-status.txt" ]; then
echo "::error::E2E suites failed (rc=${{ steps.e2e.outputs.rc }})"
cat "${ARTIFACT_DIR}/suite-status.txt" 2>/dev/null || true
exit 1
fi
e2e_real_multi:
name: Real Multi-Node E2E
if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-multinode')
runs-on: [ self-hosted, ci-multi ]
timeout-minutes: 60
env:
TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
ARTIFACT_DIR: /tmp/spinifex-e2e-multi-artifacts-${{ github.run_id }}
steps:
- name: Reap Orphan Procs
run: |
set +e
UID_SELF=$(id -u)
WS="${GITHUB_WORKSPACE}"
reap() {
local sig="$1"
for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
case "$cwd" in
"$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
esac
done
}
reap TERM
sleep 3
reap KILL
find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
true
- name: Clean Workspace
run: sudo rm -rf "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v6
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- uses: actions/checkout@v6
with:
path: spinifex
- uses: actions/setup-go@v6
with:
go-version-file: spinifex/go.mod
cache: false
- name: Re-enable Go toolchain auto-download
run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
- name: Prune rootless docker cache
run: |
docker container prune -f || true
docker image prune -af || true
docker buildx prune -af --filter "until=24h" || true
docker system df || true
- name: Build Install Artifacts
working-directory: mulga/scripts/tofu-cluster
run: |
./build-install-artifacts.sh "${GIT_BRANCH}" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-tests.tar"
- name: Provision VMs
working-directory: mulga/scripts/tofu-cluster
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
export TF_VAR_git_branch="${GIT_BRANCH}"
quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false
quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
quiet_run "scrub-env (pre-clean)" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
quiet_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
ok "3-node cluster provisioned"
- name: Bootstrap via Binary Install
working-directory: mulga/scripts/tofu-cluster
run: |
SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
- name: Resolve cluster SSH details
id: cluster
working-directory: mulga/scripts/tofu-cluster
run: |
INVENTORY=$(tofu output -json inventory)
NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
NODES_CSV=""
for i in $(seq 0 $((NODE_COUNT - 1))); do
ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
if [ -z "$NODES_CSV" ]; then NODES_CSV="$ip"; else NODES_CSV="$NODES_CSV,$ip"; fi
done
echo "node1_ip=${NODE1_IP}" >> "$GITHUB_OUTPUT"
echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
echo "nodes_csv=${NODES_CSV}" >> "$GITHUB_OUTPUT"
- name: Build E2E test binaries on runner
working-directory: spinifex/tests/e2e
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|single)$' | tr '\n' ' ')
if [ -z "$(echo "${SUITES_BUILD}" | tr -d ' ')" ]; then
echo "no multinode-eligible suites requested (SUITES=${SUITES}); nothing to build"
exit 0
fi
banner "build ${SUITES_BUILD// /, }"
group_open "go test -c"
make e2e-build SUITES_BUILD="${SUITES_BUILD}"
group_close
for f in _bin/*.test; do
ok "$(basename "$f") $(stat -c '%s' "$f" | numfmt --to=iec)"
done
- name: Install go-junit-report on runner
run: |
go install github.com/jstemmer/go-junit-report/v2@latest
echo "$HOME/go/bin" >> "$GITHUB_PATH"
- name: Ship test binaries to node1
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
VM=tf-user@${{ steps.cluster.outputs.node1_ip }}
ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin"
tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin"
ok "shipped binaries to $VM"
- name: Start packet capture on all nodes br-wan
run: |
set +e
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
for ip in "${NODE_LIST[@]}"; do
[ -z "$ip" ] && continue
scp "${SSH_OPTS[@]}" \
"${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \
tf-user@"$ip":/tmp/start-net-capture.sh
ssh "${SSH_OPTS[@]}" tf-user@"$ip" "bash /tmp/start-net-capture.sh" || true
done
- name: Pre-flight diagnostics on node1
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" 2>/dev/null || true
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
VM=tf-user@${{ steps.cluster.outputs.node1_ip }}
mkdir -p "${ARTIFACT_DIR}"
DIAG="${ARTIFACT_DIR}/preflight-diag.log"
{
echo "=== id (tf-user groups) ==="
ssh "${SSH_OPTS[@]}" "$VM" 'id; echo; getent group spinifex || true'
echo
echo "=== ls -la /etc/spinifex/ ==="
ssh "${SSH_OPTS[@]}" "$VM" 'ls -la /etc/spinifex/ 2>&1; echo; ls -la /etc/spinifex/nats/ 2>&1'
echo
echo "=== spinifex.toml NATS section ==="
ssh "${SSH_OPTS[@]}" "$VM" 'grep -A4 "\[nodes\\..*\\.nats" /etc/spinifex/spinifex.toml 2>&1 | head -40'
echo
echo "=== systemctl is-active spinifex-* ==="
ssh "${SSH_OPTS[@]}" "$VM" 'for s in spinifex-nats spinifex-predastore spinifex-awsgw spinifex-vpcd spinifex-daemon; do printf "%-22s %s\n" "$s" "$(systemctl is-active $s 2>&1)"; done'
echo
echo "=== which spx + version ==="
ssh "${SSH_OPTS[@]}" "$VM" 'which spx; spx --version 2>&1 || true'
echo
echo "=== spx get nodes (direct ssh, tf-user) ==="
ssh "${SSH_OPTS[@]}" "$VM" 'spx get nodes --timeout 5s 2>&1; echo "exit=$?"'
echo
echo "=== spx top nodes (direct ssh, tf-user) ==="
ssh "${SSH_OPTS[@]}" "$VM" 'spx top nodes 2>&1 | head -30; echo "exit=$?"'
echo
echo "=== NATS routez (curl 127.0.0.1:8222) ==="
ssh "${SSH_OPTS[@]}" "$VM" 'curl -fsS http://127.0.0.1:8222/routez 2>&1 | head -40'
echo
echo "=== aws describe-instance-types via gateway (tf-user) ==="
ssh "${SSH_OPTS[@]}" "$VM" 'AWS_PROFILE=spinifex AWS_CA_BUNDLE=/etc/spinifex/ca.pem aws --endpoint-url https://${{ steps.cluster.outputs.node1_ip }}:9999 ec2 describe-instance-types --query "InstanceTypes[*].InstanceType" --output text 2>&1 | head -5'
} 2>&1 | tee "$DIAG"
true
- name: Run E2E suites
id: e2e
run: |
set +e
SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|single)$' | tr '\n' ' ')
if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
echo "no multinode-eligible suites requested (SUITES=${SUITES}); skipping"
echo "rc=0" >> "$GITHUB_OUTPUT"
exit 0
fi
if [ -n "${TEST_RUN}" ]; then
TEST_RUN_FLAG="-test.run=${TEST_RUN}"
else
TEST_RUN_FLAG=""
fi
VM_HOST="tf-user@${{ steps.cluster.outputs.node1_ip }}"
SSH_KEY="${{ steps.cluster.outputs.ssh_key }}"
NODES_CSV="${{ steps.cluster.outputs.nodes_csv }}"
cleanup_remote() {
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \
'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
2>/dev/null || true
}
trap cleanup_remote EXIT INT TERM
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" \
"$VM_HOST" \
"set -u; mkdir -p '${ARTIFACT_DIR}'; \
cd \$HOME/spinifex-e2e-bin; \
R=\$'\033[0m' B=\$'\033[1m' C=\$'\033[36m' G=\$'\033[32m' RED=\$'\033[31m'; \
FAIL=0; \
for SUITE in ${SUITES_RUN}; do \
echo \"::group::suite \$SUITE\"; \
printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
SPINIFEX_NODES='${NODES_CSV}' \
SPINIFEX_NODE_IPS='${NODES_CSV}' \
SPINIFEX_SSH_USER=tf-user \
SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
ARTIFACT_DIR='${ARTIFACT_DIR}' \
./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \
| sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
rc=\${PIPESTATUS[0]}; \
echo \"::endgroup::\"; \
if [ \$rc -ne 0 ]; then \
printf '%s✗%s suite %s failed (exit %d)\n' \"\$RED\" \"\$R\" \"\$SUITE\" \"\$rc\"; \
echo \"::error::suite \$SUITE failed (exit \$rc)\"; \
echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; \
FAIL=1; \
else \
printf '%s✓%s suite %s passed\n' \"\$G\" \"\$R\" \"\$SUITE\"; \
fi; \
done; \
exit \$FAIL"
E2E_RC=$?
echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
exit 0
- name: Capture network/IPsec diagnostics on all nodes
if: always()
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
NODE1_IP="${NODE_LIST[0]}"
scp "${SSH_OPTS[@]}" \
"${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
tf-user@"$NODE1_IP":/tmp/capture-net-diag.sh
for i in "${!NODE_LIST[@]}"; do
idx=$((i + 1))
ip="${NODE_LIST[$i]}"
if [ "$idx" -ne 1 ]; then
scp "${SSH_OPTS[@]}" \
"${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
tf-user@"$ip":/tmp/capture-net-diag.sh
fi
ssh "${SSH_OPTS[@]}" tf-user@"$ip" \
"bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag-node${idx}'"
if [ "$idx" -ne 1 ]; then
ssh "${SSH_OPTS[@]}" tf-user@"$ip" \
"tar -C '${ARTIFACT_DIR}' -cf - 'net-diag-node${idx}' 2>/dev/null || true" \
| ssh "${SSH_OPTS[@]}" tf-user@"$NODE1_IP" \
"tar -C '${ARTIFACT_DIR}' -xf - 2>/dev/null"
fi
ok "net diagnostics node${idx} captured"
done
- name: Collect artifacts from node1
if: always()
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
mkdir -p "${ARTIFACT_DIR}"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
VM=tf-user@${{ steps.cluster.outputs.node1_ip }}
ssh "${SSH_OPTS[@]}" "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
| tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
ok "artifacts pulled to ${ARTIFACT_DIR}"
IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
for i in "${!NODE_LIST[@]}"; do
idx=$((i + 1))
ip="${NODE_LIST[$i]}"
ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
> "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
ok "journal node${idx}: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null || echo 0) lines"
done
for i in "${!NODE_LIST[@]}"; do
idx=$((i + 1))
ip="${NODE_LIST[$i]}"
ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \
> "${ARTIFACT_DIR}/openvswitch-ipsec-node${idx}.log" 2>/dev/null
done
# LB/system microVM serial consoles, per node. The lb-agent logs its
# nginx (NLB) engine activation and any `nginx -t`/reload error +
# active-probe results to the guest serial console (never in the host
# journal). Files persist in /run/spinifex past VM teardown.
for i in "${!NODE_LIST[@]}"; do
idx=$((i + 1))
ip="${NODE_LIST[$i]}"
mkdir -p "${ARTIFACT_DIR}/lb-console/node${idx}"
ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
| tar -C "${ARTIFACT_DIR}/lb-console/node${idx}" -xf - 2>/dev/null
ok "lb-console node${idx}: $(ls "${ARTIFACT_DIR}/lb-console/node${idx}" 2>/dev/null | wc -l) files"
done
- name: Convert test logs to JUnit XML
if: always()
run: |
shopt -s nullglob
for LOG in "${ARTIFACT_DIR}"/test-*.log; do
BASE=$(basename "$LOG" .log)
SUITE=${BASE#test-}
go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
done
- name: Publish test summary
if: always()
uses: ./spinifex/.github/actions/junit-report
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
title: "Multi-Node E2E results"
- name: Analyze e2e failures
if: always()
uses: ./spinifex/.github/actions/e2e-analyze
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
log-dir: ${{ env.ARTIFACT_DIR }}
title: "Multi-Node E2E failure analysis"
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-multi-artifacts-${{ github.run_id }}
path: ${{ env.ARTIFACT_DIR }}
if-no-files-found: warn
retention-days: 14
- name: Teardown
if: always()
working-directory: mulga/scripts/tofu-cluster
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
if [ -n "${{ steps.cluster.outputs.node1_ip }}" ] && [ -n "${{ steps.cluster.outputs.ssh_key }}" ]; then
IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
for ip in "${NODE_LIST[@]}"; do
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
-i "${{ steps.cluster.outputs.ssh_key }}" \
tf-user@"$ip" \
'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
2>/dev/null || true
done
fi
UID_SELF=$(id -u)
WS="${GITHUB_WORKSPACE}"
reap() {
local sig="$1"
for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
case "$cwd" in
"$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;;
esac
done
}
reap TERM
sleep 2
reap KILL
find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
quiet_run "tofu destroy" "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
ok "teardown complete"
- name: Fail job if any suite failed
if: always()
run: |
if [ "${{ steps.e2e.outputs.rc }}" != "0" ] || [ -s "${ARTIFACT_DIR}/suite-status.txt" ]; then
echo "::error::Multi-node E2E suites failed (rc=${{ steps.e2e.outputs.rc }})"
cat "${ARTIFACT_DIR}/suite-status.txt" 2>/dev/null || true
exit 1
fi
e2e_reboot:
name: Reboot Resilience E2E
# Opt-in: only fires when the SUITES dispatch input lists e2e-reboot.
# The reboot test reboots the spinifex VM, so it must run on the runner
# (not shipped to the VM like the cert/lb/single suites). Runs in
# parallel with e2e_single — both target [self-hosted, ci-single]
# and GitHub Actions serialises on the underlying runner allocation,
# so no explicit needs: ordering is required.
if: contains(github.event.inputs.suites, 'e2e-reboot')
runs-on: [ self-hosted, ci-single ]
timeout-minutes: 60
env:
TOFU_LOG: /tmp/e2e-tofu-reboot-${{ github.run_id }}.log
ARTIFACT_DIR: /tmp/spinifex-e2e-reboot-artifacts-${{ github.run_id }}
steps:
- name: Reap Orphan Procs
run: |
set +e
UID_SELF=$(id -u)
WS="${GITHUB_WORKSPACE}"
reap() {
local sig="$1"
for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
case "$cwd" in
"$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
esac
done
}
reap TERM
sleep 3
reap KILL
find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
true
- name: Clean Workspace
run: sudo rm -rf "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v6
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- uses: actions/checkout@v6
with:
path: spinifex
- uses: actions/setup-go@v6
with:
go-version-file: spinifex/go.mod
cache: false
- name: Re-enable Go toolchain auto-download
run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
- name: Prune rootless docker cache
run: |
docker container prune -f || true
docker image prune -af || true
docker buildx prune -af --filter "until=24h" || true
docker system df || true
- name: Build Install Artifacts
working-directory: mulga/scripts/tofu-cluster
run: |
./build-install-artifacts.sh "${GIT_BRANCH}" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-tests.tar"
- name: Provision VM
working-directory: mulga/scripts/tofu-cluster
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
export TF_VAR_git_branch="${GIT_BRANCH}"
quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false
quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
quiet_run "scrub-env (pre-clean)" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
quiet_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
ok "VM provisioned"
- name: Bootstrap via Binary Install
working-directory: mulga/scripts/tofu-cluster
run: |
SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
- name: Resolve VM SSH details
id: vm
working-directory: mulga/scripts/tofu-cluster
run: |
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT"
echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
- name: Build reboot.test on runner
working-directory: spinifex/tests/e2e
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
banner "build reboot"
group_open "go test -c"
make e2e-build SUITES_BUILD="reboot"
group_close
ok "$(basename _bin/reboot.test) $(stat -c '%s' _bin/reboot.test | numfmt --to=iec)"
- name: Fetch AWS credentials from VM
id: creds
run: |
set -euo pipefail
SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
CREDS=$(ssh "${SSH_OPTS[@]}" "$VM" 'cat ~/.aws/credentials')
AWS_AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
AWS_SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
if [ -z "$AWS_AK" ] || [ -z "$AWS_SK" ]; then
echo "::error::Failed to extract spinifex AWS credentials from ${{ steps.vm.outputs.wan_ip }}"
exit 1
fi
echo "::add-mask::${AWS_AK}"
echo "::add-mask::${AWS_SK}"
echo "ak=${AWS_AK}" >> "$GITHUB_OUTPUT"
echo "sk=${AWS_SK}" >> "$GITHUB_OUTPUT"
- name: Run reboot suite on runner
id: e2e
run: |
set +e
mkdir -p "${ARTIFACT_DIR}"
date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-reboot.start"
R=$'\033[0m'; B=$'\033[1m'; C=$'\033[36m'
printf '\n%s%s━━ suite reboot ━━%s\n' "$B" "$C" "$R"
if [ -n "${TEST_RUN}" ]; then
TEST_RUN_FLAG="-test.run=${TEST_RUN}"
else
TEST_RUN_FLAG=""
fi
GITHUB_ACTIONS=true FORCE_COLOR=1 \
SPINIFEX_E2E=1 SPINIFEX_MODE=single \
SPINIFEX_WAN_IP="${{ steps.vm.outputs.wan_ip }}" \
SPINIFEX_SSH_KEY="${{ steps.vm.outputs.ssh_key }}" \
SPINIFEX_AWS_ENDPOINT="https://${{ steps.vm.outputs.wan_ip }}:9999" \
SPINIFEX_AWS_ACCESS_KEY_ID="${{ steps.creds.outputs.ak }}" \
SPINIFEX_AWS_SECRET_ACCESS_KEY="${{ steps.creds.outputs.sk }}" \
ARTIFACT_DIR="${ARTIFACT_DIR}" \
./spinifex/tests/e2e/_bin/reboot.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
2>&1 | tee "${ARTIFACT_DIR}/test-reboot.log" \
| sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'
rc=${PIPESTATUS[0]}
echo "rc=${rc}" >> "$GITHUB_OUTPUT"
exit 0
- name: Collect VM journals
if: always()
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
mkdir -p "${ARTIFACT_DIR}"
SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 \
-i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
> "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || true
ok "journal: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || echo 0) lines"
- name: Install go-junit-report
if: always()
run: |
go install github.com/jstemmer/go-junit-report/v2@latest
echo "$HOME/go/bin" >> "$GITHUB_PATH"
- name: Convert test log to JUnit XML
if: always()
run: |
shopt -s nullglob
for LOG in "${ARTIFACT_DIR}"/test-*.log; do
BASE=$(basename "$LOG" .log)
SUITE=${BASE#test-}
go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
done
- name: Publish test summary
if: always()
uses: ./spinifex/.github/actions/junit-report
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
title: "E2E reboot results"
- name: Analyze e2e failures
if: always()
uses: ./spinifex/.github/actions/e2e-analyze
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
log-dir: ${{ env.ARTIFACT_DIR }}
title: "Reboot E2E failure analysis"
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-reboot-artifacts-${{ github.run_id }}
path: ${{ env.ARTIFACT_DIR }}
if-no-files-found: warn
retention-days: 14
- name: Teardown
if: always()
working-directory: mulga/scripts/tofu-cluster
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
UID_SELF=$(id -u)
WS="${GITHUB_WORKSPACE}"
reap() {
local sig="$1"
for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
case "$cwd" in
"$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;;
esac
done
}
reap TERM
sleep 2
reap KILL
find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
quiet_run "tofu destroy" "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
ok "teardown complete"
- name: Fail job if reboot suite failed
if: always()
run: |
if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
echo "::error::Reboot suite failed (rc=${{ steps.e2e.outputs.rc }})"
exit 1
fi
e2e_status:
name: Report E2E Status
if: always()
needs: [ e2e_single, e2e_real_multi, e2e_reboot ]
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Report Status to PR
env:
GH_TOKEN: ${{ github.token }}
REPORT_SHA: ${{ inputs.sha || github.sha }}
run: |
if [[ -z "${REPORT_SHA}" ]]; then
echo "::error::no SHA available (inputs.sha and github.sha both empty)"
exit 1
fi
single_ok=true; multi_ok=true; reboot_ok=true
case "${{ needs.e2e_single.result }}" in success|skipped) ;; *) single_ok=false ;; esac
case "${{ needs.e2e_real_multi.result }}" in success|skipped) ;; *) multi_ok=false ;; esac
case "${{ needs.e2e_reboot.result }}" in success|skipped) ;; *) reboot_ok=false ;; esac
if $single_ok && $multi_ok && $reboot_ok; then
STATE="success"
DESC="E2E suites passed"
else
STATE="failure"
DESC="E2E failed: single=${{ needs.e2e_single.result }} multi=${{ needs.e2e_real_multi.result }} reboot=${{ needs.e2e_reboot.result }}"
fi
gh api "repos/${{ github.repository }}/statuses/${REPORT_SHA}" \
-f state="$STATE" \
-f context=e2e \
-f description="$DESC" \
-f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
Reference in New Issue View Git Blame Copy Permalink