mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-23 04:36:40 +00:00
joshsiv-mulga
09636987ab
GitHub now warns Node 20 actions are deprecated. e2e-go.yml lagged the rest of the repo; bump to match versions already in use: - actions/setup-go@v5 -> @v6 (matches spinifex.yml) - actions/upload-artifact@v4 -> @v7 (matches release.yml, e2e-ddil.yml) checkout@v6 and create-github-app-token@v3 are already current across all workflows. The local junit-report action is `using: composite` (shell-only), so the Node runtime change does not apply.
361 lines
16 KiB
YAML
361 lines
16 KiB
YAML
name: E2E (Go)
|
|
|
|
# Standalone workflow for the Go E2E harness (tests/e2e/{cert,lb,...}).
|
|
# Mirrors e2e.yml's single-node provisioning chain but runs `go test` via
|
|
# gotestsum so we get JUnit XML + per-test artifact directories surfaced to
|
|
# GitHub Actions. Manual-dispatch only while the conversion lands — push
|
|
# trigger lifts once the single-node Go port (mulga-siv-89) is green and
|
|
# replaces run-e2e.sh; matches e2e-ddil.yml's Phase 1 trigger model.
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
sha:
|
|
description: "PR head SHA to report status on"
|
|
required: false
|
|
suites:
|
|
description: "Space-separated Make targets to run (e.g. 'e2e-cert e2e-lb')"
|
|
required: false
|
|
default: "e2e-cert e2e-lb"
|
|
|
|
permissions:
|
|
contents: read
|
|
statuses: write
|
|
|
|
# Pushes to the same ref stack on the single `ci-single` self-hosted runner,
|
|
# and a cancelled run can leave orphan `tofu`/`go test -c` procs behind that
|
|
# wedge the runner host. Auto-cancel keeps at most one in-flight run per ref.
|
|
concurrency:
|
|
group: e2e-go-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
GIT_BRANCH: ${{ github.ref_name }}
|
|
SUITES: ${{ inputs.suites || 'e2e-cert e2e-lb' }}
|
|
|
|
jobs:
|
|
e2e_go_single:
|
|
name: Single-Node E2E (Go)
|
|
runs-on: [ self-hosted, ci-single ]
|
|
timeout-minutes: 45
|
|
env:
|
|
TOFU_LOG: /tmp/e2e-go-tofu-single-${{ github.run_id }}.log
|
|
ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
|
|
steps:
|
|
- name: Reap Orphan Procs
|
|
# A cancelled prior run can leave `tofu`/`terraform` mid-apply or a
|
|
# libvirt `qemu+ssh` connection orphaned. Kill only procs whose cwd is
|
|
# under THIS runner's workspace — every self-hosted runner on this host
|
|
# shares the `tf-user` UID, so a UID-wide pkill would SIGKILL sibling
|
|
# workflows (e2e.yml/e2e-ddil.yml/...) mid-apply. SIGTERM first to let
|
|
# tofu release its state lock, then SIGKILL.
|
|
run: |
|
|
set +e
|
|
UID_SELF=$(id -u)
|
|
WS="${GITHUB_WORKSPACE}"
|
|
reap() {
|
|
local sig="$1"
|
|
for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
|
|
cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
|
|
case "$cwd" in
|
|
"$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
|
|
esac
|
|
done
|
|
}
|
|
reap TERM
|
|
sleep 3
|
|
reap KILL
|
|
# Drop only this workspace's stale tofu state locks from a SIGKILL'd apply.
|
|
find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
|
|
true
|
|
|
|
- name: Clean Workspace
|
|
run: sudo rm -rf "$GITHUB_WORKSPACE"/*
|
|
|
|
- uses: actions/create-github-app-token@v3
|
|
id: app-token
|
|
with:
|
|
client-id: ${{ secrets.APP_ID }}
|
|
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
|
repositories: mulga
|
|
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
repository: mulgadc/mulga
|
|
token: ${{ steps.app-token.outputs.token }}
|
|
sparse-checkout: scripts/tofu-cluster
|
|
path: mulga
|
|
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
path: spinifex
|
|
|
|
- uses: actions/setup-go@v6
|
|
with:
|
|
go-version-file: spinifex/go.mod
|
|
# Self-hosted runner; the cache upload to GitHub blob storage
|
|
# dominates total job time and the local Go module cache already
|
|
# warms across runs.
|
|
cache: false
|
|
|
|
- name: Build Install Artifacts
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
./build-install-artifacts.sh "${GIT_BRANCH}" \
|
|
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
|
|
"${RUNNER_TEMP}/setup.sh" \
|
|
"${RUNNER_TEMP}/spinifex-tests.tar"
|
|
|
|
- name: Provision VM
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
set -euo pipefail
|
|
# shellcheck source=/dev/null
|
|
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
|
|
export TF_VAR_git_branch="${GIT_BRANCH}"
|
|
quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false
|
|
quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
|
|
quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
|
|
quiet_run "scrub-env (pre-clean)" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
|
|
quiet_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
|
|
ok "VM provisioned"
|
|
|
|
- name: Bootstrap via Binary Install
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
|
|
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
|
|
"${RUNNER_TEMP}/setup.sh" \
|
|
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
|
|
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
|
|
|
|
- name: Resolve VM SSH details
|
|
id: vm
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
INVENTORY=$(tofu output -json inventory)
|
|
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
|
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT"
|
|
echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Build E2E test binaries on runner
|
|
working-directory: spinifex/tests/e2e
|
|
run: |
|
|
set -euo pipefail
|
|
# shellcheck source=/dev/null
|
|
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
|
|
# SUITES is in make-target form (e.g. "e2e-cert e2e-lb"); strip the
|
|
# "e2e-" prefix to get package directory names for `go test -c`.
|
|
SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g')
|
|
banner "build ${SUITES_BUILD// /, }"
|
|
group_open "go test -c"
|
|
make e2e-build SUITES_BUILD="${SUITES_BUILD}"
|
|
group_close
|
|
for f in _bin/*.test; do
|
|
ok "$(basename "$f") $(stat -c '%s' "$f" | numfmt --to=iec)"
|
|
done
|
|
|
|
- name: Install go-junit-report on runner
|
|
run: |
|
|
# Converts `go test -v` text output (from the pre-built binaries on
|
|
# the VM) into the junit-*.xml that "Publish test summary" expects.
|
|
go install github.com/jstemmer/go-junit-report/v2@latest
|
|
echo "$HOME/go/bin" >> "$GITHUB_PATH"
|
|
|
|
- name: Ship test binaries to VM
|
|
run: |
|
|
set -euo pipefail
|
|
# shellcheck source=/dev/null
|
|
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
|
|
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
|
|
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
|
|
ssh "${SSH_OPTS[@]}" "$VM" "rm -rf \$HOME/spinifex-e2e-bin && mkdir -p \$HOME/spinifex-e2e-bin"
|
|
tar -C spinifex/tests/e2e/_bin -cf - . | ssh "${SSH_OPTS[@]}" "$VM" "tar -xf - -C \$HOME/spinifex-e2e-bin"
|
|
ok "shipped binaries to $VM"
|
|
|
|
- name: Run Go E2E suites
|
|
id: e2e
|
|
run: |
|
|
set +e
|
|
# SUITES are make-target names ("e2e-cert" → cert.test). Each
|
|
# pre-built binary is self-contained (testdata //go:embed'd) so the
|
|
# VM needs no go toolchain, no source tree, no gotestsum. Each
|
|
# suite gets its own ::group:: + cyan banner inside so the Actions
|
|
# UI collapses suite output by default; expand only what failed.
|
|
SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g')
|
|
VM_HOST="tf-user@${{ steps.vm.outputs.wan_ip }}"
|
|
SSH_KEY="${{ steps.vm.outputs.ssh_key }}"
|
|
# On cancel / runner SIGTERM / step timeout, force-kill any test
|
|
# binaries left running on the VM. Without this, the remote `.test`
|
|
# process keeps holding the suite's NATS connections + tmp dirs and
|
|
# the subsequent Teardown's tofu-destroy races with its file
|
|
# handles. -o ConnectTimeout=5 so a wedged VM doesn't block cleanup.
|
|
cleanup_remote() {
|
|
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" "$VM_HOST" \
|
|
'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
|
|
2>/dev/null || true
|
|
}
|
|
trap cleanup_remote EXIT INT TERM
|
|
# `ssh -tt` forces PTY allocation. When the runner cancels this
|
|
# step, the local ssh dies and the remote sshd sends SIGHUP through
|
|
# the PTY to its children — without -tt the remote `.test` would
|
|
# orphan and keep running until its own 30m timeout.
|
|
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" \
|
|
"$VM_HOST" \
|
|
"set -u; mkdir -p '${ARTIFACT_DIR}'; \
|
|
cd \$HOME/spinifex-e2e-bin; \
|
|
R=\$'\033[0m' B=\$'\033[1m' C=\$'\033[36m' G=\$'\033[32m' RED=\$'\033[31m'; \
|
|
FAIL=0; \
|
|
for SUITE in ${SUITES_RUN}; do \
|
|
echo \"::group::suite \$SUITE\"; \
|
|
printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
|
|
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
|
|
ARTIFACT_DIR='${ARTIFACT_DIR}' \
|
|
./\$SUITE.test -test.v -test.timeout=30m \
|
|
2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \
|
|
| sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
|
|
rc=\${PIPESTATUS[0]}; \
|
|
echo \"::endgroup::\"; \
|
|
if [ \$rc -ne 0 ]; then \
|
|
printf '%s✗%s suite %s failed (exit %d)\n' \"\$RED\" \"\$R\" \"\$SUITE\" \"\$rc\"; \
|
|
echo \"::error::suite \$SUITE failed (exit \$rc)\"; \
|
|
echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; \
|
|
FAIL=1; \
|
|
else \
|
|
printf '%s✓%s suite %s passed\n' \"\$G\" \"\$R\" \"\$SUITE\"; \
|
|
fi; \
|
|
done; \
|
|
exit \$FAIL"
|
|
E2E_RC=$?
|
|
echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
|
|
exit 0 # don't fail the step — artifact upload + final gate happen below
|
|
|
|
- name: Collect artifacts from VM
|
|
if: always()
|
|
run: |
|
|
set +e
|
|
# shellcheck source=/dev/null
|
|
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
|
|
mkdir -p "${ARTIFACT_DIR}"
|
|
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
|
|
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
|
|
# `|| true` so a missing dir on an early failure doesn't mask the
|
|
# original error.
|
|
ssh "${SSH_OPTS[@]}" "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
|
|
| tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
|
|
ok "artifacts pulled to ${ARTIFACT_DIR}"
|
|
ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --since "60 min ago"' \
|
|
> "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null
|
|
ok "journal: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || echo 0) lines"
|
|
|
|
- name: Convert test logs to JUnit XML
|
|
if: always()
|
|
run: |
|
|
# VM runs the pre-built test binaries with `-test.v` and tees plain
|
|
# text logs (no Go toolchain on the VM to emit test2json). Convert
|
|
# each log to the junit-<suite>.xml shape "Publish test summary"
|
|
# already parses.
|
|
shopt -s nullglob
|
|
for LOG in "${ARTIFACT_DIR}"/test-*.log; do
|
|
BASE=$(basename "$LOG" .log)
|
|
SUITE=${BASE#test-}
|
|
go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
|
|
done
|
|
|
|
- name: Publish test summary
|
|
if: always()
|
|
uses: ./spinifex/.github/actions/junit-report
|
|
with:
|
|
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
|
title: "Go E2E results"
|
|
|
|
- uses: actions/upload-artifact@v7
|
|
if: always()
|
|
with:
|
|
name: e2e-go-artifacts-${{ github.run_id }}
|
|
path: ${{ env.ARTIFACT_DIR }}
|
|
if-no-files-found: warn
|
|
retention-days: 14
|
|
|
|
- name: Teardown
|
|
if: always()
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
# `set -e` off — teardown must run scrub-env even if tofu destroy
|
|
# fails; scrub is the libvirt-level fallback when tofu state is
|
|
# inconsistent with the remote Proxmox node.
|
|
run: |
|
|
set +e
|
|
# shellcheck source=/dev/null
|
|
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
|
|
# Kill any test binaries still running on the VM (cancelled mid-suite
|
|
# paths) BEFORE tofu destroy yanks the qcow2 from underneath them.
|
|
# Scoped to this run's VM by wan_ip — safe with sibling agents on
|
|
# the same host. ConnectTimeout=5 so an already-wedged VM doesn't
|
|
# block teardown.
|
|
if [ -n "${{ steps.vm.outputs.wan_ip }}" ] && [ -n "${{ steps.vm.outputs.ssh_key }}" ]; then
|
|
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
|
|
-i "${{ steps.vm.outputs.ssh_key }}" \
|
|
tf-user@${{ steps.vm.outputs.wan_ip }} \
|
|
'pkill -u tf-user -TERM -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; sleep 1; pkill -u tf-user -KILL -f "/spinifex-e2e-bin/.*\.test( |$)" 2>/dev/null; true' \
|
|
2>/dev/null || true
|
|
fi
|
|
# Cancelled jobs from this point on can orphan an in-flight
|
|
# tofu/libvirt SSH; clear them before destroy so the next run's
|
|
# pre-clean isn't doing it. Scoped to this runner's workspace via
|
|
# /proc/$pid/cwd — sibling runners share the tf-user UID, so a
|
|
# UID-wide pkill would nuke their tofu mid-apply.
|
|
UID_SELF=$(id -u)
|
|
WS="${GITHUB_WORKSPACE}"
|
|
reap() {
|
|
local sig="$1"
|
|
for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
|
|
cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
|
|
case "$cwd" in
|
|
"$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;;
|
|
esac
|
|
done
|
|
}
|
|
reap TERM
|
|
sleep 2
|
|
reap KILL
|
|
find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
|
|
quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
|
|
quiet_run "tofu destroy" "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
|
|
quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
|
|
ok "teardown complete"
|
|
|
|
- name: Fail job if any suite failed
|
|
if: always()
|
|
run: |
|
|
if [ "${{ steps.e2e.outputs.rc }}" != "0" ] || [ -s "${ARTIFACT_DIR}/suite-status.txt" ]; then
|
|
echo "::error::Go E2E suites failed (rc=${{ steps.e2e.outputs.rc }})"
|
|
cat "${ARTIFACT_DIR}/suite-status.txt" 2>/dev/null || true
|
|
exit 1
|
|
fi
|
|
|
|
e2e_go_status:
|
|
name: Report E2E (Go) Status
|
|
if: always() && inputs.sha
|
|
needs: [ e2e_go_single ]
|
|
runs-on: [ self-hosted, ci-single ]
|
|
timeout-minutes: 5
|
|
steps:
|
|
- name: Report Status to PR
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: |
|
|
if [[ "${{ needs.e2e_go_single.result }}" == "success" ]]; then
|
|
STATE="success"
|
|
DESC="Go E2E suites passed"
|
|
else
|
|
STATE="failure"
|
|
DESC="Go E2E failed: single=${{ needs.e2e_go_single.result }}"
|
|
fi
|
|
gh api "repos/${{ github.repository }}/statuses/${{ inputs.sha }}" \
|
|
-f state="$STATE" \
|
|
-f context=e2e-go \
|
|
-f description="$DESC" \
|
|
-f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|