gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-15 17:11:30 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
d178ef5992eb0bbd4f96b00ffdc587c8956fe518
spinifex/.github/workflows/e2e-suites.yml
T
Julian Sommer 2e09461080 chore: cleanup code comments
2026-08-04 14:20:19 +10:00

909 lines
38 KiB
YAML
Raw Blame History

# E2E per-suite blocks. Each suite appears as its own node in the Actions
# graph:
#
# single_provision ─┬─ single_suite (matrix: the single-node-eligible suites)
# └─ single_teardown (if: always)
# multi_provision ─┬─ multi_suite (matrix: the multi-node-eligible suites)
# └─ multi_teardown (if: always)
#
# Both matrices come from .github/scripts/suite-sets.sh, the one definition of
# which suites a topology can run, shared with e2e.yml and e2e-nightly.yml.
#
# Constraint: tofu state is runner-filesystem-local, so *_provision and
# *_teardown must execute on the same self-hosted runner (one runner per label
# today). Teardown reuses the persisted mulga/ tofu-cluster workspace and does
# NOT re-checkout mulga (that would wipe local state). Suite-matrix legs reach
# the already-provisioned VM over its WAN IP using the key path the provision
# job emits; they share the runner home dir, so the key file persists.
#
# This file does NOT post the gating `e2e` commit status — it runs alongside
# e2e.yml for validation. Once green across a few manual runs, fold the
# triggers from e2e.yml here and retire the loop.
name: E2E (suite blocks)
on:
workflow_dispatch:
inputs:
single_suites:
description: "Single-node suites to run as matrix legs (empty = every single-node-eligible suite, per .github/scripts/suite-sets.sh)"
required: false
default: ""
multi_suites:
description: "Multi-node suites to run as matrix legs (empty = every multi-node-eligible suite, per .github/scripts/suite-sets.sh)"
required: false
default: ""
run_multinode:
description: "Also run the multinode env block"
type: boolean
required: false
default: true
test_run:
description: "Optional -test.run regex (empty = all tests in suite)"
required: false
default: ""
permissions:
contents: read
# actions: read lets the otel-job-id action look up a provision job's numeric
# id via the Actions API — github.job is only the YAML key, no context
# exposes it.
actions: read
env:
GIT_BRANCH: ${{ github.ref_name }}
TEST_RUN: ${{ inputs.test_run || '' }}
# OTLP export to the central Elastic sink (wattle). Repo variable overrides;
# empty string disables telemetry entirely.
SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
jobs:
# ---------------------------------------------------------------------------
# Single-node env
# ---------------------------------------------------------------------------
single_provision:
name: Single env · provision
runs-on: [self-hosted, ci-single]
timeout-minutes: 40
outputs:
wan_ip: ${{ steps.vm.outputs.wan_ip }}
ssh_key: ${{ steps.vm.outputs.ssh_key }}
suites: ${{ steps.suites.outputs.list }}
matrix: ${{ steps.suites.outputs.matrix }}
env:
TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
# This job (not single_suite) is where "Bootstrap via Binary Install"
# bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env, before the
# suite matrix even exists — single_suite's legs share this one
# already-booted VM, so daemon telemetry can't be scoped any finer
# than "single env" per suite leg. ci.cell=single distinguishes it
# from multi_provision's VMs within the same run id.
SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=single
steps:
- name: Reap Orphan Procs
run: |
set +e
UID_SELF=$(id -u)
WS="${GITHUB_WORKSPACE}"
reap() {
local sig="$1"
for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
case "$cwd" in
"$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
esac
done
}
reap TERM; sleep 3; reap KILL
find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
true
- name: Clean Workspace
run: |
awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v7
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- uses: actions/checkout@v7
with:
path: spinifex
- uses: actions/setup-go@v7
with:
go-version-file: spinifex/go.mod
cache: false
- name: Re-enable Go toolchain auto-download
run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
# Default and allow-list both come from the shared definition, so this
# workflow cannot drift from e2e.yml about what a single node can run.
- name: Compute suite matrix
id: suites
env:
REQUESTED: ${{ inputs.single_suites }}
run: |
set -euo pipefail
. "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
LIST="${REQUESTED:-${E2E_SUITES_SINGLE}}"
e2e_suites_assert_subset "single_suites" "${LIST}" "${E2E_SUITES_SINGLE}"
echo "list=${LIST}" >> "$GITHUB_OUTPUT"
# Unquoted on purpose: the space-separated list must word-split into
# one JSON element per suite.
# shellcheck disable=SC2086
JSON=$(printf '%s\n' ${LIST} | jq -R . | jq -cs .)
echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
- name: Reclaim docker build cache
run: |
docker container prune -f || true
docker image prune -f || true
docker buildx prune -af --filter "until=24h" || true
docker system df || true
# ci.job_id names the provision job, not the suite job that later drives
# the env: the suite legs share this env and its baked telemetry.env, so
# the provisioner is the only job the daemon telemetry can belong to.
- name: Resolve CI job id
uses: ./spinifex/.github/actions/otel-job-id
with:
job-name: Single env · provision
- name: Build Install Artifacts
working-directory: mulga/scripts/tofu-cluster
run: |
./build-install-artifacts.sh "${GIT_BRANCH}" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-tests.tar"
- name: Provision VM
working-directory: mulga/scripts/tofu-cluster
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
export TF_VAR_git_branch="${GIT_BRANCH}"
quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false
quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
quiet_run "scrub-env (pre-clean)" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
quiet_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
ok "VM provisioned"
- name: Bootstrap via Binary Install
working-directory: mulga/scripts/tofu-cluster
run: |
SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
- name: Resolve VM SSH details
id: vm
working-directory: mulga/scripts/tofu-cluster
run: |
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT"
echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
- name: Build eks-node AMI (libguestfs, runner)
if: contains(steps.suites.outputs.list, 'eks')
working-directory: spinifex
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
banner "build eks-node AMI (libguestfs — no host devices)"
sudo apt-get update -qq
sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
export LIBGUESTFS_BACKEND=direct
sudo chmod 0644 /boot/vmlinuz-* || true
group_open "make build-eks-node-image"
make build-eks-node-image
group_close
ok "raw: $(du -h /tmp/eks-node-image-build/eks-node-alpine.raw | cut -f1)"
- name: Import eks-node AMI on VM
if: contains(steps.suites.outputs.list, 'eks')
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
spinifex/scripts/images/eks-node/manifest.conf)"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
banner "import eks-node AMI to VM image store"
scp "${SSH_OPTS[@]}" \
/tmp/eks-node-image-build/eks-node-alpine.raw \
"$VM":/tmp/eks-node-alpine.raw
ssh "${SSH_OPTS[@]}" "$VM" \
"spx admin images import --file /tmp/eks-node-alpine.raw \
--distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
--boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1
ok "eks-node AMI imported"
- name: Install kubectl on VM
if: contains(steps.suites.outputs.list, 'eks')
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
banner "install kubectl on VM (eks data-plane subtest)"
ssh "${SSH_OPTS[@]}" "$VM" 'bash -s' <<'EOF'
set -euo pipefail
KVER=v1.32.5
case "$(uname -m)" in
x86_64) ARCH=amd64 ;;
aarch64) ARCH=arm64 ;;
*) echo "unsupported arch $(uname -m)" >&2; exit 1 ;;
esac
cd /tmp
curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl"
curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl.sha256"
echo "$(cat kubectl.sha256) kubectl" | sha256sum -c -
sudo install -m 0755 kubectl /usr/local/bin/kubectl
kubectl version --client
EOF
ok "kubectl installed on VM"
- name: Build E2E test binaries on runner
working-directory: spinifex/tests/e2e
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
# teardown always builds: the run-scoped sweep binary runs in the
# teardown job, it is never a selectable suite leg.
SUITES_BUILD="${{ steps.suites.outputs.list }} teardown"
banner "build ${SUITES_BUILD// /, }"
group_open "go test -c"
make e2e-build SUITES_BUILD="${SUITES_BUILD}"
group_close
for f in _bin/*.test; do
ok "$(basename "$f") $(stat -c '%s' "$f" | numfmt --to=iec)"
done
- name: Ship test binaries to VM
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin"
tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin"
ok "shipped binaries to $VM"
- name: Start packet capture on VM br-wan
run: |
set +e
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
scp "${SSH_OPTS[@]}" \
"${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \
"$VM":/tmp/start-net-capture.sh
ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/start-net-capture.sh" || true
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7
with:
name: tofu-log-single-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
if-no-files-found: ignore
retention-days: 14
single_suite:
name: Single env · suite
needs: single_provision
runs-on: [self-hosted, ci-single]
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
suite: ${{ fromJSON(needs.single_provision.outputs.matrix) }}
env:
WAN_IP: ${{ needs.single_provision.outputs.wan_ip }}
SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }}
ARTIFACT_DIR: /tmp/spinifex-e2e-single-${{ github.run_id }}-${{ matrix.suite }}
steps:
- uses: actions/checkout@v7
with:
path: spinifex
- uses: actions/setup-go@v7
with:
go-version-file: spinifex/go.mod
cache: false
- name: Install go-junit-report
run: |
go install github.com/jstemmer/go-junit-report/v2@latest
echo "$HOME/go/bin" >> "$GITHUB_PATH"
- name: Run ${{ matrix.suite }}
id: e2e
run: |
set +e
SUITE="${{ matrix.suite }}"
VM_HOST="tf-user@${WAN_IP}"
TEST_RUN_FLAG=""
[ -n "${TEST_RUN}" ] && TEST_RUN_FLAG="-test.run=${TEST_RUN}"
cleanup_remote() {
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \
"pkill -u tf-user -TERM -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; true" \
2>/dev/null || true
}
trap cleanup_remote EXIT INT TERM
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" "$VM_HOST" \
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/spinifex-e2e-bin; \
R=\$'\033[0m' B=\$'\033[1m' C=\$'\033[36m' G=\$'\033[32m' RED=\$'\033[31m'; \
printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"${SUITE}\" \"\$R\"; \
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
ARTIFACT_DIR='${ARTIFACT_DIR}' \
./${SUITE}.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log' \
| sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
exit \${PIPESTATUS[0]}"
echo "rc=$?" >> "$GITHUB_OUTPUT"
exit 0
- name: Collect suite artifacts from VM
if: always()
run: |
set +e
mkdir -p "${ARTIFACT_DIR}"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${SSH_KEY}")
VM=tf-user@${WAN_IP}
ssh "${SSH_OPTS[@]}" "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
| tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
true
- name: Convert test log to JUnit XML
if: always()
run: |
shopt -s nullglob
for LOG in "${ARTIFACT_DIR}"/test-*.log; do
BASE=$(basename "$LOG" .log); SUITE=${BASE#test-}
go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
done
- name: Publish test summary
if: always()
uses: ./spinifex/.github/actions/junit-report
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
title: "E2E ${{ matrix.suite }} results"
- name: Analyze e2e failures
if: always()
uses: ./spinifex/.github/actions/e2e-analyze
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
log-dir: ${{ env.ARTIFACT_DIR }}
title: "E2E ${{ matrix.suite }} failure analysis"
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-single-${{ matrix.suite }}-${{ github.run_id }}
path: ${{ env.ARTIFACT_DIR }}
if-no-files-found: warn
retention-days: 14
- name: Fail job if suite failed
if: always()
run: |
if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
echo "::error::suite ${{ matrix.suite }} failed (rc=${{ steps.e2e.outputs.rc }})"
exit 1
fi
single_teardown:
name: Single env · teardown
needs: [single_provision, single_suite]
if: always()
runs-on: [self-hosted, ci-single]
timeout-minutes: 20
env:
TOFU_LOG: /tmp/e2e-tofu-single-td-${{ github.run_id }}.log
ARTIFACT_DIR: /tmp/spinifex-e2e-single-diag-${{ github.run_id }}
WAN_IP: ${{ needs.single_provision.outputs.wan_ip }}
SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }}
steps:
# Reuse the persisted mulga/ tofu-cluster workspace from single_provision.
# Re-checkout only spinifex (for ci-fmt + diag scripts); checking out mulga
# again would wipe the local tofu state and orphan the VM.
- uses: actions/checkout@v7
with:
path: spinifex
- name: Capture network/IPsec diagnostics on VM
if: ${{ needs.single_provision.result == 'success' && env.WAN_IP != '' }}
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
mkdir -p "${ARTIFACT_DIR}"
SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}")
VM=tf-user@${WAN_IP}
scp "${SSH_OPTS[@]}" \
"${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
"$VM":/tmp/capture-net-diag.sh 2>/dev/null
ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag'" 2>/dev/null
ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "90 min ago"' \
> "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null
mkdir -p "${ARTIFACT_DIR}/lb-console"
ssh "${SSH_OPTS[@]}" "$VM" 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
| tar -C "${ARTIFACT_DIR}/lb-console" -xf - 2>/dev/null
true
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-single-diag-${{ github.run_id }}
path: ${{ env.ARTIFACT_DIR }}
if-no-files-found: ignore
retention-days: 14
# Reclaim any resource leaked by a crashed/interrupted suite before the VM
# is destroyed. Scoped to this run's tag, best-effort, never fails the job.
- name: Sweep leaked run resources on VM
if: ${{ needs.single_provision.result == 'success' && env.WAN_IP != '' }}
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}")
VM=tf-user@${WAN_IP}
banner "sweep run-tagged resources (run ${{ github.run_id }})"
ssh "${SSH_OPTS[@]}" "$VM" \
"cd \$HOME/spinifex-e2e-bin 2>/dev/null && \
SPINIFEX_E2E=1 SPINIFEX_MODE=single \
SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
./teardown.test -test.v -test.run TestSweepRunResources -test.timeout=10m" \
2>&1 || true
true
- name: Teardown VM
if: always()
working-directory: mulga/scripts/tofu-cluster
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
if [ -n "${WAN_IP}" ] && [ -n "${SSH_KEY}" ]; then
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}" \
tf-user@${WAN_IP} \
'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
2>/dev/null || true
fi
UID_SELF=$(id -u); WS="${GITHUB_WORKSPACE}"
reap() {
local sig="$1"
for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
case "$cwd" in "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;; esac
done
}
reap TERM; sleep 2; reap KILL
find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
quiet_run "telemetry flush" "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
quiet_run "tofu destroy" "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
ok "teardown complete"
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7
with:
name: tofu-log-single-td-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
if-no-files-found: ignore
retention-days: 14
# ---------------------------------------------------------------------------
# Multinode env
# ---------------------------------------------------------------------------
multi_provision:
name: Multi env · provision
if: ${{ inputs.run_multinode }}
runs-on: [self-hosted, ci-multi]
timeout-minutes: 45
outputs:
node1_ip: ${{ steps.cluster.outputs.node1_ip }}
ssh_key: ${{ steps.cluster.outputs.ssh_key }}
nodes_csv: ${{ steps.cluster.outputs.nodes_csv }}
suites: ${{ steps.suites.outputs.list }}
matrix: ${{ steps.suites.outputs.matrix }}
env:
TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
# See single_provision above — daemon telemetry is baked in here, at
# provision time, before the multi_suite matrix exists.
SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=multi
steps:
- name: Reap Orphan Procs
run: |
set +e
UID_SELF=$(id -u); WS="${GITHUB_WORKSPACE}"
reap() {
local sig="$1"
for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
case "$cwd" in "$WS"|"$WS"/*) echo "reap $sig pid=$pid"; kill "-$sig" "$pid" 2>/dev/null ;; esac
done
}
reap TERM; sleep 3; reap KILL
find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
true
- name: Clean Workspace
run: |
awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v7
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- uses: actions/checkout@v7
with:
path: spinifex
- uses: actions/setup-go@v7
with:
go-version-file: spinifex/go.mod
cache: false
- name: Re-enable Go toolchain auto-download
run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
- name: Reclaim docker build cache
run: |
docker container prune -f || true
docker image prune -f || true
docker buildx prune -af --filter "until=24h" || true
docker system df || true
# See the matching note on single_provision.
- name: Resolve CI job id
uses: ./spinifex/.github/actions/otel-job-id
with:
job-name: Multi env · provision
- name: Build Install Artifacts
working-directory: mulga/scripts/tofu-cluster
run: |
./build-install-artifacts.sh "${GIT_BRANCH}" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-tests.tar"
- name: Provision VMs
working-directory: mulga/scripts/tofu-cluster
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
export TF_VAR_git_branch="${GIT_BRANCH}"
quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false
quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
quiet_run "scrub-env (pre-clean)" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
quiet_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
ok "3-node cluster provisioned"
- name: Bootstrap via Binary Install
working-directory: mulga/scripts/tofu-cluster
run: |
SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
- name: Resolve cluster SSH details
id: cluster
working-directory: mulga/scripts/tofu-cluster
run: |
INVENTORY=$(tofu output -json inventory)
NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path'); SSH_KEY="${SSH_KEY/#\~/$HOME}"
NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
NODES_CSV=""
for i in $(seq 0 $((NODE_COUNT - 1))); do
ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
if [ -z "$NODES_CSV" ]; then NODES_CSV="$ip"; else NODES_CSV="$NODES_CSV,$ip"; fi
done
echo "node1_ip=${NODE1_IP}" >> "$GITHUB_OUTPUT"
echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
echo "nodes_csv=${NODES_CSV}" >> "$GITHUB_OUTPUT"
# Mirrors single_provision's step: the multi leg previously ran multinode
# alone, silently dropping the lb and cert coverage that only a
# multi-node env can complete.
- name: Compute suite matrix
id: suites
env:
REQUESTED: ${{ inputs.multi_suites }}
run: |
set -euo pipefail
. "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
LIST="${REQUESTED:-${E2E_SUITES_MULTI}}"
e2e_suites_assert_subset "multi_suites" "${LIST}" "${E2E_SUITES_MULTI}"
echo "list=${LIST}" >> "$GITHUB_OUTPUT"
# Unquoted on purpose: the space-separated list must word-split into
# one JSON element per suite.
# shellcheck disable=SC2086
JSON=$(printf '%s\n' ${LIST} | jq -R . | jq -cs .)
echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
- name: Build multinode binaries on runner
working-directory: spinifex/tests/e2e
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
# teardown always builds: the run-scoped sweep binary runs in the
# teardown job, it is never a selectable suite leg.
SUITES_BUILD="${{ steps.suites.outputs.list }} teardown"
banner "build ${SUITES_BUILD// /, }"
group_open "go test -c"
make e2e-build SUITES_BUILD="${SUITES_BUILD}"
group_close
ok "$(basename _bin/multinode.test) $(stat -c '%s' _bin/multinode.test | numfmt --to=iec)"
- name: Ship test binaries to node1
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
VM=tf-user@${{ steps.cluster.outputs.node1_ip }}
ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin"
tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin"
ok "shipped binaries to $VM"
- name: Start packet capture on all nodes
run: |
set +e
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
for ip in "${NODE_LIST[@]}"; do
[ -z "$ip" ] && continue
scp "${SSH_OPTS[@]}" \
"${GITHUB_WORKSPACE}/spinifex/.github/scripts/start-net-capture.sh" \
tf-user@"$ip":/tmp/start-net-capture.sh
ssh "${SSH_OPTS[@]}" tf-user@"$ip" "bash /tmp/start-net-capture.sh" || true
done
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7
with:
name: tofu-log-multi-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
if-no-files-found: ignore
retention-days: 14
multi_suite:
name: Multi env · suite
needs: multi_provision
runs-on: [self-hosted, ci-multi]
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
suite: ${{ fromJSON(needs.multi_provision.outputs.matrix) }}
env:
NODE1_IP: ${{ needs.multi_provision.outputs.node1_ip }}
SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
ARTIFACT_DIR: /tmp/spinifex-e2e-multi-${{ github.run_id }}-${{ matrix.suite }}
steps:
- uses: actions/checkout@v7
with:
path: spinifex
- uses: actions/setup-go@v7
with:
go-version-file: spinifex/go.mod
cache: false
- name: Install go-junit-report
run: |
go install github.com/jstemmer/go-junit-report/v2@latest
echo "$HOME/go/bin" >> "$GITHUB_PATH"
- name: Run ${{ matrix.suite }}
id: e2e
run: |
set +e
SUITE="${{ matrix.suite }}"
VM_HOST="tf-user@${NODE1_IP}"
TEST_RUN_FLAG=""
[ -n "${TEST_RUN}" ] && TEST_RUN_FLAG="-test.run=${TEST_RUN}"
cleanup_remote() {
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \
"pkill -u tf-user -TERM -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f \"/spinifex-e2e-bin/${SUITE}\.test( |\$)\" 2>/dev/null; true" \
2>/dev/null || true
}
trap cleanup_remote EXIT INT TERM
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" "$VM_HOST" \
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/spinifex-e2e-bin; \
printf '\n━━ suite %s ━━\n' \"${SUITE}\"; \
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
ARTIFACT_DIR='${ARTIFACT_DIR}' \
./${SUITE}.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log' \
| sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
exit \${PIPESTATUS[0]}"
echo "rc=$?" >> "$GITHUB_OUTPUT"
exit 0
- name: Collect suite artifacts from node1
if: always()
run: |
set +e
mkdir -p "${ARTIFACT_DIR}"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${SSH_KEY}")
ssh "${SSH_OPTS[@]}" tf-user@${NODE1_IP} "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
| tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
true
- name: Convert test log to JUnit XML
if: always()
run: |
shopt -s nullglob
for LOG in "${ARTIFACT_DIR}"/test-*.log; do
BASE=$(basename "$LOG" .log); SUITE=${BASE#test-}
go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
done
- name: Publish test summary
if: always()
uses: ./spinifex/.github/actions/junit-report
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
title: "Multi-Node E2E ${{ matrix.suite }} results"
- name: Analyze e2e failures
if: always()
uses: ./spinifex/.github/actions/e2e-analyze
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
log-dir: ${{ env.ARTIFACT_DIR }}
title: "Multi-Node E2E ${{ matrix.suite }} failure analysis"
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-multi-${{ matrix.suite }}-${{ github.run_id }}
path: ${{ env.ARTIFACT_DIR }}
if-no-files-found: warn
retention-days: 14
- name: Fail job if suite failed
if: always()
run: |
if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
echo "::error::suite ${{ matrix.suite }} failed (rc=${{ steps.e2e.outputs.rc }})"
exit 1
fi
multi_teardown:
name: Multi env · teardown
needs: [multi_provision, multi_suite]
if: always() && inputs.run_multinode
runs-on: [self-hosted, ci-multi]
timeout-minutes: 20
env:
TOFU_LOG: /tmp/e2e-tofu-multi-td-${{ github.run_id }}.log
SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
NODES_CSV: ${{ needs.multi_provision.outputs.nodes_csv }}
steps:
- uses: actions/checkout@v7
with:
path: spinifex
# Reclaim leaked run-tagged resources before the cluster is destroyed.
- name: Sweep leaked run resources on node1
if: ${{ needs.multi_provision.result == 'success' && needs.multi_provision.outputs.node1_ip != '' }}
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}")
VM=tf-user@${{ needs.multi_provision.outputs.node1_ip }}
banner "sweep run-tagged resources (run ${{ github.run_id }})"
ssh "${SSH_OPTS[@]}" "$VM" \
"cd \$HOME/spinifex-e2e-bin 2>/dev/null && \
SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
./teardown.test -test.v -test.run TestSweepRunResources -test.timeout=10m" \
2>&1 || true
true
- name: Teardown cluster
if: always()
working-directory: mulga/scripts/tofu-cluster
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
if [ -n "${NODES_CSV}" ] && [ -n "${SSH_KEY}" ]; then
IFS=',' read -ra NODE_LIST <<< "${NODES_CSV}"
for ip in "${NODE_LIST[@]}"; do
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}" \
tf-user@"$ip" \
'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
2>/dev/null || true
done
fi
UID_SELF=$(id -u); WS="${GITHUB_WORKSPACE}"
reap() {
local sig="$1"
for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
case "$cwd" in "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;; esac
done
}
reap TERM; sleep 2; reap KILL
find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
quiet_run "telemetry flush" "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
quiet_run "tofu destroy" "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
ok "teardown complete"
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7
with:
name: tofu-log-multi-td-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
if-no-files-found: ignore
retention-days: 14
e2e_suites_status:
name: E2E suite blocks · summary
if: always()
needs: [single_suite, multi_suite]
runs-on: ubuntu-26.04
timeout-minutes: 5
steps:
- name: Summarize
run: |
echo "## E2E suite-block results" >> "$GITHUB_STEP_SUMMARY"
echo "single_suite: ${{ needs.single_suite.result }}" >> "$GITHUB_STEP_SUMMARY"
echo "multi_suite: ${{ needs.multi_suite.result }}" >> "$GITHUB_STEP_SUMMARY"
case "${{ needs.single_suite.result }}" in success|skipped) ;; *) exit 1 ;; esac
case "${{ needs.multi_suite.result }}" in success|skipped) ;; *) exit 1 ;; esac
Reference in New Issue View Git Blame Copy Permalink