gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-15 17:11:30 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
07dce613eb5ac760bf5316b606eef5664e595eb3
spinifex/.github/workflows/e2e-suites.yml
T
Josh Siviour 0e08b9d256 feat: ship per-test JUnit results to the Elasticsearch sink (#829) 
* feat: ship per-test JUnit results to the Elasticsearch sink

Every e2e workflow already converts its test log to JUnit XML and renders
it into the job summary, then discards it with the artifact. mulga-ci-runs
records which suite failed; nothing records which test failed, so
root-causing a failure means reading raw Actions logs across historical
runs.

The junit-to-es action bulk-indexes one doc per leaf test case into
mulga-ci-test-results: name, subtest, status, duration, failure text, run
id, cell, suite, branch and sha. Parent t.Run wrappers are dropped so a
failing subtest counts once, matching the job summary renderer. The doc id
is deterministic, so a replayed post overwrites rather than
double-counting.

Called from every workflow that produces JUnit XML, and tolerated on
failure: an unreachable sink must not turn a green test run red.

* feat: classify unparsed JUnit cases as unknown, not error

go-junit-report emits <error message="No test result found"> when a
=== RUN line has no matching result line, which means the log it was fed
was truncated or filtered. Recording that as a failure would put 78
phantom failures per green bare-metal run into the pass rate.
2026-08-18 22:54:54 +10:00

929 lines
39 KiB
YAML
Raw Blame History

# E2E per-suite blocks. Each suite appears as its own node in the Actions
# graph:
#
# single_provision ─┬─ single_suite (matrix: the single-node-eligible suites)
# └─ single_teardown (if: always)
# multi_provision ─┬─ multi_suite (matrix: the multi-node-eligible suites)
# └─ multi_teardown (if: always)
#
# Both matrices come from .github/scripts/suite-sets.sh, the one definition of
# which suites a topology can run, shared with e2e.yml and e2e-nightly.yml.
#
# Constraint: tofu state is runner-filesystem-local, so *_provision and
# *_teardown must execute on the same self-hosted runner (one runner per label
# today). Teardown reuses the persisted mulga/ tofu-cluster workspace and does
# NOT re-checkout mulga (that would wipe local state). Suite-matrix legs reach
# the already-provisioned VM over its WAN IP using the key path the provision
# job emits; they share the runner home dir, so the key file persists.
#
# This file does NOT post the gating `e2e` commit status — it runs alongside
# e2e.yml for validation. Once green across a few manual runs, fold the
# triggers from e2e.yml here and retire the loop.
name: E2E (suite blocks)
on:
workflow_dispatch:
inputs:
single_suites:
description: "Single-node suites to run as matrix legs (empty = every single-node-eligible suite, per .github/scripts/suite-sets.sh)"
required: false
default: ""
multi_suites:
description: "Multi-node suites to run as matrix legs (empty = every multi-node-eligible suite, per .github/scripts/suite-sets.sh)"
required: false
default: ""
run_multinode:
description: "Also run the multinode env block"
type: boolean
required: false
default: true
test_run:
description: "Optional -test.run regex (empty = all tests in suite)"
required: false
default: ""
permissions:
contents: read
# actions: read lets the otel-job-id action look up a provision job's numeric
# id via the Actions API — github.job is only the YAML key, no context
# exposes it.
actions: read
env:
GIT_BRANCH: ${{ github.ref_name }}
TEST_RUN: ${{ inputs.test_run || '' }}
# OTLP export to the central Elastic sink (wattle). Repo variable overrides;
# empty string disables telemetry entirely.
SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
jobs:
# ---------------------------------------------------------------------------
# Single-node env
# ---------------------------------------------------------------------------
single_provision:
name: Single env · provision
runs-on: [self-hosted, ci-single]
timeout-minutes: 40
outputs:
wan_ip: ${{ steps.vm.outputs.wan_ip }}
ssh_key: ${{ steps.vm.outputs.ssh_key }}
suites: ${{ steps.suites.outputs.list }}
matrix: ${{ steps.suites.outputs.matrix }}
env:
TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
# This job (not single_suite) is where "Bootstrap via Binary Install"
# bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env, before the
# suite matrix even exists — single_suite's legs share this one
# already-booted VM, so daemon telemetry can't be scoped any finer
# than "single env" per suite leg. ci.cell=single distinguishes it
# from multi_provision's VMs within the same run id.
SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=single
steps:
- name: Reap Orphan Procs
run: |
set +e
UID_SELF=$(id -u)
WS="${GITHUB_WORKSPACE}"
reap() {
local sig="$1"
for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
case "$cwd" in
"$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
esac
done
}
reap TERM; sleep 3; reap KILL
find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
true
- name: Clean Workspace
run: |
awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v7
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- uses: actions/checkout@v7
with:
path: spinifex
- uses: actions/setup-go@v7
with:
go-version-file: spinifex/go.mod
cache: false
- name: Re-enable Go toolchain auto-download
run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
# Default and allow-list both come from the shared definition, so this
# workflow cannot drift from e2e.yml about what a single node can run.
- name: Compute suite matrix
id: suites
env:
REQUESTED: ${{ inputs.single_suites }}
run: |
set -euo pipefail
. "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
LIST="${REQUESTED:-${E2E_SUITES_SINGLE}}"
e2e_suites_assert_subset "single_suites" "${LIST}" "${E2E_SUITES_SINGLE}"
echo "list=${LIST}" >> "$GITHUB_OUTPUT"
# Unquoted on purpose: the space-separated list must word-split into
# one JSON element per suite.
# shellcheck disable=SC2086
JSON=$(printf '%s\n' ${LIST} | jq -R . | jq -cs .)
echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
- name: Reclaim docker build cache
run: |
docker container prune -f || true
docker image prune -f || true
docker buildx prune -af --filter "until=24h" || true
docker system df || true
# ci.job_id names the provision job, not the suite job that later drives
# the env: the suite legs share this env and its baked telemetry.env, so
# the provisioner is the only job the daemon telemetry can belong to.
- name: Resolve CI job id
uses: ./spinifex/.github/actions/otel-job-id
with:
job-name: Single env · provision
- name: Build Install Artifacts
working-directory: mulga/scripts/tofu-cluster
run: |
./build-install-artifacts.sh "${GIT_BRANCH}" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-tests.tar"
- name: Provision VM
working-directory: mulga/scripts/tofu-cluster
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
export TF_VAR_git_branch="${GIT_BRANCH}"
quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false
quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
quiet_run "scrub-env (pre-clean)" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
quiet_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
ok "VM provisioned"
- name: Bootstrap via Binary Install
working-directory: mulga/scripts/tofu-cluster
run: |
SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
- name: Resolve VM SSH details
id: vm
working-directory: mulga/scripts/tofu-cluster
run: |
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT"
echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
- name: Build eks-node AMI (libguestfs, runner)
if: contains(steps.suites.outputs.list, 'eks')
working-directory: spinifex
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
banner "build eks-node AMI (libguestfs — no host devices)"
sudo apt-get update -qq
sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
export LIBGUESTFS_BACKEND=direct
sudo chmod 0644 /boot/vmlinuz-* || true
group_open "make build-eks-node-image"
make build-eks-node-image
group_close
ok "raw: $(du -h /tmp/eks-node-image-build/eks-node-alpine.raw | cut -f1)"
- name: Import eks-node AMI on VM
if: contains(steps.suites.outputs.list, 'eks')
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
spinifex/scripts/images/eks-node/manifest.conf)"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
banner "import eks-node AMI to VM image store"
scp "${SSH_OPTS[@]}" \
/tmp/eks-node-image-build/eks-node-alpine.raw \
"$VM":/tmp/eks-node-alpine.raw
ssh "${SSH_OPTS[@]}" "$VM" \
"spx admin images import --file /tmp/eks-node-alpine.raw \
--distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
--boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1
ok "eks-node AMI imported"
- name: Install kubectl on VM
if: contains(steps.suites.outputs.list, 'eks')
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
banner "install kubectl on VM (eks data-plane subtest)"
ssh "${SSH_OPTS[@]}" "$VM" 'bash -s' <<'EOF'
set -euo pipefail
KVER=v1.32.5
case "$(uname -m)" in
x86_64) ARCH=amd64 ;;
aarch64) ARCH=arm64 ;;
*) echo "unsupported arch $(uname -m)" >&2; exit 1 ;;
esac
cd /tmp
curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl"
curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl.sha256"
echo "$(cat kubectl.sha256) kubectl" | sha256sum -c -
sudo install -m 0755 kubectl /usr/local/bin/kubectl
kubectl version --client
EOF
ok "kubectl installed on VM"
- name: Build E2E test binaries on runner
working-directory: spinifex/tests/e2e
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
# teardown always builds: the run-scoped sweep binary runs in the
# teardown job, it is never a selectable suite leg.
SUITES_BUILD="${{ steps.suites.outputs.list }} teardown"
banner "build ${SUITES_BUILD// /, }"
group_open "go test -c"
make e2e-build SUITES_BUILD="${SUITES_BUILD}"
group_close
for f in _bin/*.test; do
ok "$(basename "$f") $(stat -c '%s' "$f" | numfmt --to=iec)"
done
- name: Ship test binaries to VM
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin"
tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin"
ok "shipped binaries to $VM"
- name: Start packet capture on VM br-wan
run: |
set +e
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
scp "${SSH_OPTS[@]}" \
"${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \
"$VM":/tmp/start-net-capture.sh
ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/start-net-capture.sh" || true
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7
with:
name: tofu-log-single-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
if-no-files-found: ignore
retention-days: 14
single_suite:
name: Single env · suite
needs: single_provision
runs-on: [self-hosted, ci-single]
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
suite: ${{ fromJSON(needs.single_provision.outputs.matrix) }}
env:
WAN_IP: ${{ needs.single_provision.outputs.wan_ip }}
SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }}
ARTIFACT_DIR: /tmp/spinifex-e2e-single-${{ github.run_id }}-${{ matrix.suite }}
steps:
- uses: actions/checkout@v7
with:
path: spinifex
- uses: actions/setup-go@v7
with:
go-version-file: spinifex/go.mod
cache: false
- name: Install go-junit-report
run: |
go install github.com/jstemmer/go-junit-report/v2@latest
echo "$HOME/go/bin" >> "$GITHUB_PATH"
- name: Run ${{ matrix.suite }}
id: e2e
run: |
set +e
SUITE="${{ matrix.suite }}"
VM_HOST="tf-user@${WAN_IP}"
TEST_RUN_FLAG=""
[ -n "${TEST_RUN}" ] && TEST_RUN_FLAG="-test.run=${TEST_RUN}"
cleanup_remote() {
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \
"pkill -u tf-user -TERM -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; true" \
2>/dev/null || true
}
trap cleanup_remote EXIT INT TERM
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" "$VM_HOST" \
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/spinifex-e2e-bin; \
R=\$'\033[0m' B=\$'\033[1m' C=\$'\033[36m' G=\$'\033[32m' RED=\$'\033[31m'; \
printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"${SUITE}\" \"\$R\"; \
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
ARTIFACT_DIR='${ARTIFACT_DIR}' \
./${SUITE}.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log' \
| sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
exit \${PIPESTATUS[0]}"
echo "rc=$?" >> "$GITHUB_OUTPUT"
exit 0
- name: Collect suite artifacts from VM
if: always()
run: |
set +e
mkdir -p "${ARTIFACT_DIR}"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${SSH_KEY}")
VM=tf-user@${WAN_IP}
ssh "${SSH_OPTS[@]}" "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
| tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
true
- name: Convert test log to JUnit XML
if: always()
run: |
shopt -s nullglob
for LOG in "${ARTIFACT_DIR}"/test-*.log; do
BASE=$(basename "$LOG" .log); SUITE=${BASE#test-}
go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
done
- name: Publish test summary
if: always()
uses: ./spinifex/.github/actions/junit-report
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
title: "E2E ${{ matrix.suite }} results"
# Tolerated on failure: the sink being unreachable must not turn a
# green test run red.
- name: Ship test results to Elasticsearch
if: always()
continue-on-error: true
uses: ./spinifex/.github/actions/junit-to-es
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
cell: single
- name: Analyze e2e failures
if: always()
uses: ./spinifex/.github/actions/e2e-analyze
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
log-dir: ${{ env.ARTIFACT_DIR }}
title: "E2E ${{ matrix.suite }} failure analysis"
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-single-${{ matrix.suite }}-${{ github.run_id }}
path: ${{ env.ARTIFACT_DIR }}
if-no-files-found: warn
retention-days: 14
- name: Fail job if suite failed
if: always()
run: |
if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
echo "::error::suite ${{ matrix.suite }} failed (rc=${{ steps.e2e.outputs.rc }})"
exit 1
fi
single_teardown:
name: Single env · teardown
needs: [single_provision, single_suite]
if: always()
runs-on: [self-hosted, ci-single]
timeout-minutes: 20
env:
TOFU_LOG: /tmp/e2e-tofu-single-td-${{ github.run_id }}.log
ARTIFACT_DIR: /tmp/spinifex-e2e-single-diag-${{ github.run_id }}
WAN_IP: ${{ needs.single_provision.outputs.wan_ip }}
SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }}
steps:
# Reuse the persisted mulga/ tofu-cluster workspace from single_provision.
# Re-checkout only spinifex (for ci-fmt + diag scripts); checking out mulga
# again would wipe the local tofu state and orphan the VM.
- uses: actions/checkout@v7
with:
path: spinifex
- name: Capture network/IPsec diagnostics on VM
if: ${{ needs.single_provision.result == 'success' && env.WAN_IP != '' }}
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
mkdir -p "${ARTIFACT_DIR}"
SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}")
VM=tf-user@${WAN_IP}
scp "${SSH_OPTS[@]}" \
"${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
"$VM":/tmp/capture-net-diag.sh 2>/dev/null
ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag'" 2>/dev/null
ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "90 min ago"' \
> "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null
mkdir -p "${ARTIFACT_DIR}/lb-console"
ssh "${SSH_OPTS[@]}" "$VM" 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
| tar -C "${ARTIFACT_DIR}/lb-console" -xf - 2>/dev/null
true
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-single-diag-${{ github.run_id }}
path: ${{ env.ARTIFACT_DIR }}
if-no-files-found: ignore
retention-days: 14
# Reclaim any resource leaked by a crashed/interrupted suite before the VM
# is destroyed. Scoped to this run's tag, best-effort, never fails the job.
- name: Sweep leaked run resources on VM
if: ${{ needs.single_provision.result == 'success' && env.WAN_IP != '' }}
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}")
VM=tf-user@${WAN_IP}
banner "sweep run-tagged resources (run ${{ github.run_id }})"
ssh "${SSH_OPTS[@]}" "$VM" \
"cd \$HOME/spinifex-e2e-bin 2>/dev/null && \
SPINIFEX_E2E=1 SPINIFEX_MODE=single \
SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
./teardown.test -test.v -test.run TestSweepRunResources -test.timeout=10m" \
2>&1 || true
true
- name: Teardown VM
if: always()
working-directory: mulga/scripts/tofu-cluster
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
if [ -n "${WAN_IP}" ] && [ -n "${SSH_KEY}" ]; then
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}" \
tf-user@${WAN_IP} \
'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
2>/dev/null || true
fi
UID_SELF=$(id -u); WS="${GITHUB_WORKSPACE}"
reap() {
local sig="$1"
for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
case "$cwd" in "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;; esac
done
}
reap TERM; sleep 2; reap KILL
find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
quiet_run "telemetry flush" "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
quiet_run "tofu destroy" "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
ok "teardown complete"
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7
with:
name: tofu-log-single-td-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
if-no-files-found: ignore
retention-days: 14
# ---------------------------------------------------------------------------
# Multinode env
# ---------------------------------------------------------------------------
multi_provision:
name: Multi env · provision
if: ${{ inputs.run_multinode }}
runs-on: [self-hosted, ci-multi]
timeout-minutes: 45
outputs:
node1_ip: ${{ steps.cluster.outputs.node1_ip }}
ssh_key: ${{ steps.cluster.outputs.ssh_key }}
nodes_csv: ${{ steps.cluster.outputs.nodes_csv }}
suites: ${{ steps.suites.outputs.list }}
matrix: ${{ steps.suites.outputs.matrix }}
env:
TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
# See single_provision above — daemon telemetry is baked in here, at
# provision time, before the multi_suite matrix exists.
SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=multi
steps:
- name: Reap Orphan Procs
run: |
set +e
UID_SELF=$(id -u); WS="${GITHUB_WORKSPACE}"
reap() {
local sig="$1"
for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
case "$cwd" in "$WS"|"$WS"/*) echo "reap $sig pid=$pid"; kill "-$sig" "$pid" 2>/dev/null ;; esac
done
}
reap TERM; sleep 3; reap KILL
find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
true
- name: Clean Workspace
run: |
awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v7
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- uses: actions/checkout@v7
with:
path: spinifex
- uses: actions/setup-go@v7
with:
go-version-file: spinifex/go.mod
cache: false
- name: Re-enable Go toolchain auto-download
run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
- name: Reclaim docker build cache
run: |
docker container prune -f || true
docker image prune -f || true
docker buildx prune -af --filter "until=24h" || true
docker system df || true
# See the matching note on single_provision.
- name: Resolve CI job id
uses: ./spinifex/.github/actions/otel-job-id
with:
job-name: Multi env · provision
- name: Build Install Artifacts
working-directory: mulga/scripts/tofu-cluster
run: |
./build-install-artifacts.sh "${GIT_BRANCH}" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-tests.tar"
- name: Provision VMs
working-directory: mulga/scripts/tofu-cluster
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
export TF_VAR_git_branch="${GIT_BRANCH}"
quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false
quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
quiet_run "scrub-env (pre-clean)" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
quiet_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
ok "3-node cluster provisioned"
- name: Bootstrap via Binary Install
working-directory: mulga/scripts/tofu-cluster
run: |
SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
- name: Resolve cluster SSH details
id: cluster
working-directory: mulga/scripts/tofu-cluster
run: |
INVENTORY=$(tofu output -json inventory)
NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path'); SSH_KEY="${SSH_KEY/#\~/$HOME}"
NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
NODES_CSV=""
for i in $(seq 0 $((NODE_COUNT - 1))); do
ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
if [ -z "$NODES_CSV" ]; then NODES_CSV="$ip"; else NODES_CSV="$NODES_CSV,$ip"; fi
done
echo "node1_ip=${NODE1_IP}" >> "$GITHUB_OUTPUT"
echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
echo "nodes_csv=${NODES_CSV}" >> "$GITHUB_OUTPUT"
# Mirrors single_provision's step: the multi leg previously ran multinode
# alone, silently dropping the lb and cert coverage that only a
# multi-node env can complete.
- name: Compute suite matrix
id: suites
env:
REQUESTED: ${{ inputs.multi_suites }}
run: |
set -euo pipefail
. "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
LIST="${REQUESTED:-${E2E_SUITES_MULTI}}"
e2e_suites_assert_subset "multi_suites" "${LIST}" "${E2E_SUITES_MULTI}"
echo "list=${LIST}" >> "$GITHUB_OUTPUT"
# Unquoted on purpose: the space-separated list must word-split into
# one JSON element per suite.
# shellcheck disable=SC2086
JSON=$(printf '%s\n' ${LIST} | jq -R . | jq -cs .)
echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
- name: Build multinode binaries on runner
working-directory: spinifex/tests/e2e
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
# teardown always builds: the run-scoped sweep binary runs in the
# teardown job, it is never a selectable suite leg.
SUITES_BUILD="${{ steps.suites.outputs.list }} teardown"
banner "build ${SUITES_BUILD// /, }"
group_open "go test -c"
make e2e-build SUITES_BUILD="${SUITES_BUILD}"
group_close
ok "$(basename _bin/multinode.test) $(stat -c '%s' _bin/multinode.test | numfmt --to=iec)"
- name: Ship test binaries to node1
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
VM=tf-user@${{ steps.cluster.outputs.node1_ip }}
ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin"
tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin"
ok "shipped binaries to $VM"
- name: Start packet capture on all nodes
run: |
set +e
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
for ip in "${NODE_LIST[@]}"; do
[ -z "$ip" ] && continue
scp "${SSH_OPTS[@]}" \
"${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \
tf-user@"$ip":/tmp/start-net-capture.sh
ssh "${SSH_OPTS[@]}" tf-user@"$ip" "bash /tmp/start-net-capture.sh" || true
done
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7
with:
name: tofu-log-multi-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
if-no-files-found: ignore
retention-days: 14
multi_suite:
name: Multi env · suite
needs: multi_provision
runs-on: [self-hosted, ci-multi]
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
suite: ${{ fromJSON(needs.multi_provision.outputs.matrix) }}
env:
NODE1_IP: ${{ needs.multi_provision.outputs.node1_ip }}
SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
ARTIFACT_DIR: /tmp/spinifex-e2e-multi-${{ github.run_id }}-${{ matrix.suite }}
steps:
- uses: actions/checkout@v7
with:
path: spinifex
- uses: actions/setup-go@v7
with:
go-version-file: spinifex/go.mod
cache: false
- name: Install go-junit-report
run: |
go install github.com/jstemmer/go-junit-report/v2@latest
echo "$HOME/go/bin" >> "$GITHUB_PATH"
- name: Run ${{ matrix.suite }}
id: e2e
run: |
set +e
SUITE="${{ matrix.suite }}"
VM_HOST="tf-user@${NODE1_IP}"
TEST_RUN_FLAG=""
[ -n "${TEST_RUN}" ] && TEST_RUN_FLAG="-test.run=${TEST_RUN}"
cleanup_remote() {
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \
"pkill -u tf-user -TERM -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; true" \
2>/dev/null || true
}
trap cleanup_remote EXIT INT TERM
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" "$VM_HOST" \
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/spinifex-e2e-bin; \
printf '\n━━ suite %s ━━\n' \"${SUITE}\"; \
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
ARTIFACT_DIR='${ARTIFACT_DIR}' \
./${SUITE}.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log' \
| sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
exit \${PIPESTATUS[0]}"
echo "rc=$?" >> "$GITHUB_OUTPUT"
exit 0
- name: Collect suite artifacts from node1
if: always()
run: |
set +e
mkdir -p "${ARTIFACT_DIR}"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${SSH_KEY}")
ssh "${SSH_OPTS[@]}" tf-user@${NODE1_IP} "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
| tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
true
- name: Convert test log to JUnit XML
if: always()
run: |
shopt -s nullglob
for LOG in "${ARTIFACT_DIR}"/test-*.log; do
BASE=$(basename "$LOG" .log); SUITE=${BASE#test-}
go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
done
- name: Publish test summary
if: always()
uses: ./spinifex/.github/actions/junit-report
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
title: "Multi-Node E2E ${{ matrix.suite }} results"
# Tolerated on failure: the sink being unreachable must not turn a
# green test run red.
- name: Ship test results to Elasticsearch
if: always()
continue-on-error: true
uses: ./spinifex/.github/actions/junit-to-es
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
cell: multi
- name: Analyze e2e failures
if: always()
uses: ./spinifex/.github/actions/e2e-analyze
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
log-dir: ${{ env.ARTIFACT_DIR }}
title: "Multi-Node E2E ${{ matrix.suite }} failure analysis"
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-multi-${{ matrix.suite }}-${{ github.run_id }}
path: ${{ env.ARTIFACT_DIR }}
if-no-files-found: warn
retention-days: 14
- name: Fail job if suite failed
if: always()
run: |
if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
echo "::error::suite ${{ matrix.suite }} failed (rc=${{ steps.e2e.outputs.rc }})"
exit 1
fi
multi_teardown:
name: Multi env · teardown
needs: [multi_provision, multi_suite]
if: always() && inputs.run_multinode
runs-on: [self-hosted, ci-multi]
timeout-minutes: 20
env:
TOFU_LOG: /tmp/e2e-tofu-multi-td-${{ github.run_id }}.log
SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
steps:
- uses: actions/checkout@v7
with:
path: spinifex
# Reclaim leaked run-tagged resources before the cluster is destroyed.
- name: Sweep leaked run resources on node1
if: ${{ needs.multi_provision.result == 'success' && needs.multi_provision.outputs.node1_ip != '' }}
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}")
VM=tf-user@${{ needs.multi_provision.outputs.node1_ip }}
banner "sweep run-tagged resources (run ${{ github.run_id }})"
ssh "${SSH_OPTS[@]}" "$VM" \
"cd \$HOME/spinifex-e2e-bin 2>/dev/null && \
SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
./teardown.test -test.v -test.run TestSweepRunResources -test.timeout=10m" \
2>&1 || true
true
- name: Teardown cluster
if: always()
working-directory: mulga/scripts/tofu-cluster
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
if [ -n "${NODES_CSV}" ] && [ -n "${SSH_KEY}" ]; then
IFS=',' read -ra NODE_LIST <<< "${NODES_CSV}"
for ip in "${NODE_LIST[@]}"; do
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}" \
tf-user@"$ip" \
'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
2>/dev/null || true
done
fi
UID_SELF=$(id -u); WS="${GITHUB_WORKSPACE}"
reap() {
local sig="$1"
for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
case "$cwd" in "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;; esac
done
}
reap TERM; sleep 2; reap KILL
find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
quiet_run "telemetry flush" "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
quiet_run "tofu destroy" "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
ok "teardown complete"
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7
with:
name: tofu-log-multi-td-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
if-no-files-found: ignore
retention-days: 14
e2e_suites_status:
name: E2E suite blocks · summary
if: always()
needs: [single_suite, multi_suite]
runs-on: ubuntu-26.04
timeout-minutes: 5
steps:
- name: Summarize
run: |
echo "## E2E suite-block results" >> "$GITHUB_STEP_SUMMARY"
echo "single_suite: ${{ needs.single_suite.result }}" >> "$GITHUB_STEP_SUMMARY"
echo "multi_suite: ${{ needs.multi_suite.result }}" >> "$GITHUB_STEP_SUMMARY"
case "${{ needs.single_suite.result }}" in success|skipped) ;; *) exit 1 ;; esac
case "${{ needs.multi_suite.result }}" in success|skipped) ;; *) exit 1 ;; esac
Reference in New Issue View Git Blame Copy Permalink