mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 17:11:30 +00:00
joshsiv-mulga
348 lines
16 KiB
YAML
348 lines
16 KiB
YAML
# GPU E2E — drives wattle's persistent, shared-host Spinifex stack over SSH to
|
|
# exercise GPU passthrough (raw EC2, ECS task GPU exposure, EKS pod GPU
|
|
# exposure). See docs/development/feature/wattle-deploy-and-ci.md (Part B).
|
|
#
|
|
# ---------------------------------------------------------------------------
|
|
# ONE-TIME HOST-SIDE SETUP — NOT part of this repo, a human must do this:
|
|
#
|
|
# 1. Register ONE self-hosted GitHub Actions runner on a banksia host with
|
|
# the dedicated label `ci-gpu` (in addition to `self-hosted`). Do
|
|
# NOT add it to the `ci-single` / `ci-multi` pool and do NOT give any
|
|
# other runner the `ci-gpu` label — a second labelled runner would
|
|
# let two GPU jobs race the single RTX A1000. With exactly one runner
|
|
# carrying the label, `runs-on: [self-hosted, ci-gpu]` alone forces
|
|
# serialization; the `concurrency:` block below queues same-group runs
|
|
# when that runner is already busy with a live job.
|
|
# 2. Drop the wattle SSH private key on that runner's filesystem and wire
|
|
# it into this repo as `secrets.WATTLE_SSH_KEY` (raw private key
|
|
# contents). Optionally override `vars.WATTLE_SSH_HOST` /
|
|
# `vars.WATTLE_SSH_USER` if they differ from the defaults below.
|
|
# 3. Confirm the runner user can drive wattle's control plane over SSH. The
|
|
# job calls `spx` (read-only checks) and the `aws` CLI with the already-
|
|
# configured `spinifex` profile directly — no wrapper scripts, no install
|
|
# on wattle (tf-user is already in the `spinifex` group there):
|
|
# ssh tf-user@192.168.1.13 'AWS_PROFILE=spinifex aws ec2 describe-regions'
|
|
# 4. Manual-work mutex: an agent doing hands-on GPU testing on wattle can
|
|
# block CI by touching a sentinel file in the RUNNER's home directory
|
|
# (NOT on wattle):
|
|
# touch ~/.ci-gpu-paused # blocks CI
|
|
# rm ~/.ci-gpu-paused # resumes CI
|
|
# or by setting the repo/environment variable `CI_GPU_PAUSED=true`
|
|
# (no runner filesystem access needed). Either one causes this workflow
|
|
# to skip cleanly (job succeeds, all work steps skipped) instead of
|
|
# queuing behind manual work indefinitely.
|
|
#
|
|
# Runbook: docs/development/feature/wattle-deploy-and-ci.md (Part B).
|
|
# ---------------------------------------------------------------------------
|
|
|
|
name: GPU E2E
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
test_run:
|
|
description: "Optional -test.run regex scoped to tests/e2e/gpu (empty = run all GPU tests)"
|
|
required: false
|
|
default: ""
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
# Single dedicated runner (see setup note 1 above) + this group together
|
|
# enforce single-flight on the one RTX A1000: a second dispatch has nowhere
|
|
# else to land (unique label) and, if it somehow overlaps in scheduling,
|
|
# queues here rather than cancelling/stomping the in-flight run.
|
|
concurrency:
|
|
group: ci-gpu
|
|
cancel-in-progress: false
|
|
|
|
env:
|
|
# OTLP export to the central Elastic sink (wattle is the sink itself — see
|
|
# Part A.2). Repo variable overrides; carried for correlation even though
|
|
# the GPU suite's own daemon-side spans are emitted by wattle's already-
|
|
# configured obs-agent, not by this job.
|
|
SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
|
|
# ci.cell=gpu keeps this workflow's telemetry separable from the tofu-env
|
|
# workflows'. Nothing in this job bakes these attrs into a daemon (see the
|
|
# note above), so the label only takes effect once wattle's obs-agent is
|
|
# given them; there is deliberately no ci.job_id lookup for the same reason.
|
|
SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=gpu
|
|
WATTLE_SSH_HOST: ${{ vars.WATTLE_SSH_HOST || '192.168.1.13' }}
|
|
WATTLE_SSH_USER: ${{ vars.WATTLE_SSH_USER || 'tf-user' }}
|
|
TEST_RUN: ${{ inputs.test_run || '' }}
|
|
|
|
jobs:
|
|
gpu_e2e:
|
|
name: GPU E2E (wattle)
|
|
runs-on: [self-hosted, ci-gpu]
|
|
timeout-minutes: 75
|
|
env:
|
|
ARTIFACT_DIR: /tmp/spinifex-gpu-e2e-artifacts-${{ github.run_id }}
|
|
steps:
|
|
# Manual-work mutex gate — see setup note 4 above. Checked BEFORE any
|
|
# SSH to wattle so a paused run touches the box zero times.
|
|
- name: Manual-work mutex gate
|
|
id: gate
|
|
env:
|
|
REPO_VAR_PAUSED: ${{ vars.CI_GPU_PAUSED }}
|
|
run: |
|
|
set -euo pipefail
|
|
PAUSED=false
|
|
if [ -f "$HOME/.ci-gpu-paused" ]; then
|
|
PAUSED=true
|
|
echo "::warning::ci-gpu runner paused via ~/.ci-gpu-paused sentinel on the runner — skipping this run (manual GPU work in progress on wattle). Remove the file to resume CI."
|
|
fi
|
|
if [ "$(echo "${REPO_VAR_PAUSED:-}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
|
|
PAUSED=true
|
|
echo "::warning::ci-gpu paused via repo/environment variable CI_GPU_PAUSED=true — skipping this run."
|
|
fi
|
|
echo "paused=${PAUSED}" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Paused — nothing to do
|
|
if: steps.gate.outputs.paused == 'true'
|
|
run: echo "ci-gpu is paused; job exits cleanly without touching wattle. See gate step above for the reason."
|
|
|
|
- name: Clean Workspace
|
|
if: steps.gate.outputs.paused != 'true'
|
|
run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
|
|
|
- uses: actions/checkout@v7
|
|
if: steps.gate.outputs.paused != 'true'
|
|
|
|
- uses: actions/setup-go@v7
|
|
if: steps.gate.outputs.paused != 'true'
|
|
with:
|
|
go-version-file: go.mod
|
|
cache: false
|
|
|
|
- name: Install wattle SSH key
|
|
if: steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
set -euo pipefail
|
|
install -m 0600 /dev/null "${RUNNER_TEMP}/wattle-ci"
|
|
echo "${{ secrets.WATTLE_SSH_KEY }}" > "${RUNNER_TEMP}/wattle-ci"
|
|
echo "ssh_opts=-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${RUNNER_TEMP}/wattle-ci" >> "$GITHUB_ENV"
|
|
|
|
# Healthcheck: confirm wattle's persistent AWS-compat control plane is
|
|
# actually up before spending time building/shipping the test binary.
|
|
- name: Healthcheck — describe-regions
|
|
if: steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
set -euo pipefail
|
|
ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" \
|
|
'AWS_PROFILE=spinifex aws ec2 describe-regions --output text'
|
|
|
|
# Ensure the base EC2 GPU AMI is present — the raw-EC2 GPU passthrough
|
|
# test is the always-on baseline, so CI self-provisions it rather than
|
|
# depending on a manual import. `--name` selects the catalog entry (pulls
|
|
# the prebuilt image from iso.mulgadc.com); `--ami-name` is REQUIRED to
|
|
# name the AMI ubuntu-26.04-nvidia-gpu-x86_64 — without it the import
|
|
# falls back to the generic ami-<distro>-<version>-<arch>, which the
|
|
# tests/e2e/gpu filter would never match. Idempotent — skipped once
|
|
# wattle's persistent loop-fs already has the named AMI (the 4.4G source
|
|
# qcow2 is cached in the images dir, so a re-import wouldn't re-download).
|
|
# `spx admin images import` writes local spinifex storage, so it needs
|
|
# the `spinifex` group active (sg); aws-CLI calls hit the gateway over
|
|
# HTTPS and don't.
|
|
- name: Ensure base GPU AMI present (import-if-absent)
|
|
if: steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
set -euo pipefail
|
|
REMOTE='
|
|
set -e
|
|
BASE=ubuntu-26.04-nvidia-gpu-x86_64
|
|
present() {
|
|
AWS_PROFILE=spinifex aws ec2 describe-images \
|
|
--filters "Name=name,Values=$BASE" "Name=state,Values=available" \
|
|
--query "Images[0].ImageId" --output text 2>/dev/null
|
|
}
|
|
ID=$(present)
|
|
if [ -n "$ID" ] && [ "$ID" != "None" ]; then
|
|
echo "base GPU AMI already present: $BASE = $ID"; exit 0
|
|
fi
|
|
echo "base GPU AMI $BASE absent — importing prebuilt from catalog..."
|
|
sg spinifex -c "AWS_PROFILE=spinifex spx admin images import --name $BASE --ami-name $BASE"
|
|
for i in 1 2 3 4 5 6; do
|
|
ID=$(present)
|
|
[ -n "$ID" ] && [ "$ID" != "None" ] && break
|
|
sleep 5
|
|
done
|
|
[ -n "$ID" ] && [ "$ID" != "None" ] || { echo "import ran but $BASE still not available" >&2; exit 1; }
|
|
echo "imported base GPU AMI: $BASE = $ID"
|
|
'
|
|
ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
|
|
|
|
# Preflight — the base GPU AMI is REQUIRED: the setup step above just
|
|
# ensured it, so a miss here means the import silently failed. Hard-fail
|
|
# rather than run a false-green all-skipped suite. ECS/EKS node AMIs are
|
|
# opt-in tiers (heavier — node AMIs + cluster spin-up); their tests
|
|
# self-skip when absent, so only warn.
|
|
- name: Preflight — verify GPU AMIs
|
|
if: steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
set -euo pipefail
|
|
REMOTE='
|
|
set -e
|
|
present() {
|
|
AWS_PROFILE=spinifex aws ec2 describe-images \
|
|
--filters "Name=name,Values=$1" "Name=state,Values=available" \
|
|
--query "Images[0].ImageId" --output text 2>/dev/null
|
|
}
|
|
BASE=ubuntu-26.04-nvidia-gpu-x86_64
|
|
ID=$(present "$BASE")
|
|
if [ -z "$ID" ] || [ "$ID" = "None" ]; then
|
|
echo "required base GPU AMI $BASE missing — cannot run the baseline EC2 GPU test" >&2
|
|
exit 1
|
|
fi
|
|
echo "required: $BASE = $ID"
|
|
for OPT in spinifex-ecs-node-gpu spinifex-eks-node spinifex-eks-node-gpu; do
|
|
OID=$(present "$OPT")
|
|
if [ -z "$OID" ] || [ "$OID" = "None" ]; then
|
|
echo "::warning::$OPT not imported — its GPU test(s) will skip (opt-in tier)"
|
|
else
|
|
echo "opt-in: $OPT = $OID"
|
|
fi
|
|
done
|
|
'
|
|
ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
|
|
|
|
# Same build command CI's other e2e workflows use to cross-compile a
|
|
# suite into a shippable binary (tests/e2e/Makefile e2e-build target,
|
|
# SUITES_BUILD scoped to just gpu). Building on the runner keeps go
|
|
# toolchain / module cache off wattle entirely.
|
|
- name: Build GPU e2e test binary on runner
|
|
if: steps.gate.outputs.paused != 'true'
|
|
working-directory: tests/e2e
|
|
run: |
|
|
set -euo pipefail
|
|
make e2e-build SUITES_BUILD=gpu
|
|
ls -la _bin/gpu.test
|
|
|
|
- name: Install go-junit-report on runner
|
|
if: steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
go install github.com/jstemmer/go-junit-report/v2@latest
|
|
echo "$HOME/go/bin" >> "$GITHUB_PATH"
|
|
|
|
- name: Ship GPU e2e test binary to wattle
|
|
if: steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
set -euo pipefail
|
|
VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
|
|
ssh ${{ env.ssh_opts }} "$VM" "mkdir -p \$HOME/spinifex-gpu-e2e-bin"
|
|
scp ${{ env.ssh_opts }} tests/e2e/_bin/gpu.test "$VM":spinifex-gpu-e2e-bin/gpu.test
|
|
|
|
# Same invocation shape e2e.yml uses to run a pre-built suite binary
|
|
# (SPINIFEX_E2E=1 SPINIFEX_MODE=single, ./<suite>.test -test.v
|
|
# -test.timeout=NNm), plus SPINIFEX_WAN_IP: wattle's ~24 bridges make
|
|
# net.InterfaceAddrs() enumeration pick a co-tenant bridge instead of
|
|
# the advertised gateway (mulga-7suds), so the override is required
|
|
# here (proper fix tracked separately: tests/e2e/harness/env.go should
|
|
# prefer spinifex.toml's advertised bridge).
|
|
- name: Run GPU e2e suite on wattle
|
|
id: e2e
|
|
if: steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
set +e
|
|
VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
|
|
if [ -n "${TEST_RUN}" ]; then
|
|
TEST_RUN_FLAG="-test.run=${TEST_RUN}"
|
|
else
|
|
TEST_RUN_FLAG=""
|
|
fi
|
|
ssh -tt ${{ env.ssh_opts }} "$VM" \
|
|
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/spinifex-gpu-e2e-bin; \
|
|
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
|
|
SPINIFEX_WAN_IP='${WATTLE_SSH_HOST}' \
|
|
ARTIFACT_DIR='${ARTIFACT_DIR}' \
|
|
./gpu.test -test.v -test.timeout=60m ${TEST_RUN_FLAG} \
|
|
2>&1 | tee '${ARTIFACT_DIR}/test-gpu.log'; \
|
|
exit \${PIPESTATUS[0]}"
|
|
E2E_RC=$?
|
|
echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
|
|
exit 0
|
|
|
|
- name: Collect artifacts from wattle
|
|
if: always() && steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
set +e
|
|
mkdir -p "${ARTIFACT_DIR}"
|
|
VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
|
|
ssh ${{ env.ssh_opts }} "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
|
|
| tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
|
|
echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"
|
|
|
|
- name: Convert test log to JUnit XML
|
|
if: always() && steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
shopt -s nullglob
|
|
for LOG in "${ARTIFACT_DIR}"/test-*.log; do
|
|
BASE=$(basename "$LOG" .log)
|
|
SUITE=${BASE#test-}
|
|
go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
|
|
done
|
|
|
|
- name: Publish test summary
|
|
if: always() && steps.gate.outputs.paused != 'true'
|
|
uses: ./.github/actions/junit-report
|
|
with:
|
|
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
|
title: "GPU E2E results"
|
|
|
|
- name: Analyze e2e failures
|
|
if: always() && steps.gate.outputs.paused != 'true'
|
|
uses: ./.github/actions/e2e-analyze
|
|
with:
|
|
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
|
log-dir: ${{ env.ARTIFACT_DIR }}
|
|
title: "GPU E2E failure analysis"
|
|
|
|
- uses: actions/upload-artifact@v7
|
|
if: always() && steps.gate.outputs.paused != 'true'
|
|
with:
|
|
name: gpu-e2e-artifacts-${{ github.run_id }}
|
|
path: ${{ env.ARTIFACT_DIR }}
|
|
if-no-files-found: warn
|
|
retention-days: 14
|
|
|
|
# Cleanup is NEVER a host teardown: it never runs setup-ovn.sh,
|
|
# dev-reset, or cluster-reset. It only reconciles EC2-API-level state —
|
|
# terminate any GPU-typed instance still pending/running (the Go
|
|
# suite's own t.Cleanup already does this on normal pass/fail exit;
|
|
# this is the backstop for a hard `-test.timeout` kill or a workflow
|
|
# cancellation, which skip deferred Go cleanups). Termination triggers
|
|
# gpuManager.Release() on wattle, returning the A1000 to the pool for
|
|
# the next queued run or manual test.
|
|
- name: Cleanup — terminate leftover GPU instances, release GPU
|
|
if: always() && steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
set +e
|
|
VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
|
|
REMOTE='
|
|
set +e
|
|
GPU_TYPES=$(AWS_PROFILE=spinifex aws ec2 describe-instance-types \
|
|
--query "InstanceTypes[?GpuInfo!=null].InstanceType" --output text | tr "\t" ",")
|
|
if [ -z "$GPU_TYPES" ]; then
|
|
echo "no GPU instance types advertised; nothing to reconcile"
|
|
exit 0
|
|
fi
|
|
IDS=$(AWS_PROFILE=spinifex aws ec2 describe-instances \
|
|
--filters "Name=instance-type,Values=${GPU_TYPES}" "Name=instance-state-name,Values=pending,running" \
|
|
--query "Reservations[].Instances[].InstanceId" --output text)
|
|
if [ -z "$IDS" ] || [ "$IDS" = "None" ]; then
|
|
echo "no leftover GPU instances — GPU already released"
|
|
exit 0
|
|
fi
|
|
echo "terminating leftover GPU instance(s): $IDS"
|
|
AWS_PROFILE=spinifex aws ec2 terminate-instances --instance-ids $IDS
|
|
'
|
|
ssh ${{ env.ssh_opts }} "$VM" "$REMOTE"
|
|
|
|
- name: Fail job if GPU suite failed
|
|
if: always() && steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
|
|
echo "::error::GPU e2e suite failed (rc=${{ steps.e2e.outputs.rc }}) — see analysis above; raw ES/APM traces at ${SPINIFEX_OTLP_ENDPOINT} (ci.run_id=${{ github.run_id }})"
|
|
exit 1
|
|
fi
|