mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 17:11:30 +00:00
Julian Sommer
307 lines
13 KiB
YAML
307 lines
13 KiB
YAML
# GPU E2E — drives wattle's persistent, shared-host Spinifex stack over SSH to
|
|
# exercise GPU passthrough (raw EC2, ECS task GPU exposure, EKS pod GPU
|
|
# exposure).
|
|
#
|
|
# ---------------------------------------------------------------------------
|
|
# ONE-TIME HOST-SIDE SETUP — NOT part of this repo, a human must do this:
|
|
#
|
|
# 1. Register ONE self-hosted GitHub Actions runner on a banksia host with
|
|
# the dedicated label `ci-gpu` (in addition to `self-hosted`). Do
|
|
# NOT add it to the `ci-single` / `ci-multi` pool and do NOT give any
|
|
# other runner the `ci-gpu` label — a second labelled runner would
|
|
# let two GPU jobs race the single RTX A1000. With exactly one runner
|
|
# carrying the label, `runs-on: [self-hosted, ci-gpu]` alone forces
|
|
# serialization; the `concurrency:` block below queues same-group runs
|
|
# when that runner is already busy with a live job.
|
|
# 2. Drop the wattle SSH private key on that runner's filesystem and wire
|
|
# it into this repo as `secrets.WATTLE_SSH_KEY` (raw private key
|
|
# contents). Optionally override `vars.WATTLE_SSH_HOST` /
|
|
# `vars.WATTLE_SSH_USER` if they differ from the defaults below.
|
|
# 3. Confirm the runner user can drive wattle's control plane over SSH. The
|
|
# job calls `spx` (read-only checks) and the `aws` CLI with the already-
|
|
# configured `spinifex` profile directly — no wrapper scripts, no install
|
|
# on wattle (tf-user is already in the `spinifex` group there):
|
|
# ssh tf-user@192.168.1.13 'AWS_PROFILE=spinifex aws ec2 describe-regions'
|
|
# 4. Manual-work mutex: an agent doing hands-on GPU testing on wattle can
|
|
# block CI by touching a sentinel file in the RUNNER's home directory
|
|
# (NOT on wattle):
|
|
# touch ~/.ci-gpu-paused # blocks CI
|
|
# rm ~/.ci-gpu-paused # resumes CI
|
|
# or by setting the repo/environment variable `CI_GPU_PAUSED=true`
|
|
# (no runner filesystem access needed). Either one causes this workflow
|
|
# to skip cleanly (job succeeds, all work steps skipped) instead of
|
|
# queuing behind manual work indefinitely.
|
|
#
|
|
# ---------------------------------------------------------------------------
|
|
|
|
name: GPU E2E
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
test_run:
|
|
description: "Optional -test.run regex scoped to tests/e2e/gpu (empty = run all GPU tests)"
|
|
required: false
|
|
default: ""
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
# Single dedicated runner (see setup note 1 above) + this group together
|
|
# enforce single-flight on the one RTX A1000: a second dispatch has nowhere
|
|
# else to land (unique label) and, if it somehow overlaps in scheduling,
|
|
# queues here rather than cancelling/stomping the in-flight run.
|
|
concurrency:
|
|
group: ci-gpu
|
|
cancel-in-progress: false
|
|
|
|
env:
|
|
SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
|
|
SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=gpu
|
|
WATTLE_SSH_HOST: ${{ vars.WATTLE_SSH_HOST || '192.168.1.13' }}
|
|
WATTLE_SSH_USER: ${{ vars.WATTLE_SSH_USER || 'tf-user' }}
|
|
TEST_RUN: ${{ inputs.test_run || '' }}
|
|
|
|
jobs:
|
|
gpu_e2e:
|
|
name: GPU E2E (wattle)
|
|
runs-on: [self-hosted, ci-gpu]
|
|
timeout-minutes: 75
|
|
env:
|
|
ARTIFACT_DIR: /tmp/spinifex-gpu-e2e-artifacts-${{ github.run_id }}
|
|
steps:
|
|
- name: Manual-work mutex gate
|
|
id: gate
|
|
env:
|
|
REPO_VAR_PAUSED: ${{ vars.CI_GPU_PAUSED }}
|
|
run: |
|
|
set -euo pipefail
|
|
PAUSED=false
|
|
if [ -f "$HOME/.ci-gpu-paused" ]; then
|
|
PAUSED=true
|
|
echo "::warning::ci-gpu runner paused via ~/.ci-gpu-paused sentinel on the runner — skipping this run (manual GPU work in progress on wattle). Remove the file to resume CI."
|
|
fi
|
|
if [ "$(echo "${REPO_VAR_PAUSED:-}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then
|
|
PAUSED=true
|
|
echo "::warning::ci-gpu paused via repo/environment variable CI_GPU_PAUSED=true — skipping this run."
|
|
fi
|
|
echo "paused=${PAUSED}" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Paused — nothing to do
|
|
if: steps.gate.outputs.paused == 'true'
|
|
run: echo "ci-gpu is paused; job exits cleanly without touching wattle. See gate step above for the reason."
|
|
|
|
- name: Clean Workspace
|
|
if: steps.gate.outputs.paused != 'true'
|
|
run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
|
|
|
- uses: actions/checkout@v7
|
|
if: steps.gate.outputs.paused != 'true'
|
|
|
|
- uses: actions/setup-go@v7
|
|
if: steps.gate.outputs.paused != 'true'
|
|
with:
|
|
go-version-file: go.mod
|
|
cache: false
|
|
|
|
- name: Install wattle SSH key
|
|
if: steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
set -euo pipefail
|
|
install -m 0600 /dev/null "${RUNNER_TEMP}/wattle-ci"
|
|
echo "${{ secrets.WATTLE_SSH_KEY }}" > "${RUNNER_TEMP}/wattle-ci"
|
|
echo "ssh_opts=-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${RUNNER_TEMP}/wattle-ci" >> "$GITHUB_ENV"
|
|
|
|
- name: Healthcheck — describe-regions
|
|
if: steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
set -euo pipefail
|
|
ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" \
|
|
'AWS_PROFILE=spinifex aws ec2 describe-regions --output text'
|
|
|
|
- name: Ensure base GPU AMI present (import-if-absent)
|
|
if: steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
set -euo pipefail
|
|
REMOTE='
|
|
set -e
|
|
BASE=ubuntu-26.04-nvidia-gpu-x86_64
|
|
present() {
|
|
AWS_PROFILE=spinifex aws ec2 describe-images \
|
|
--filters "Name=name,Values=$BASE" "Name=state,Values=available" \
|
|
--query "Images[0].ImageId" --output text 2>/dev/null
|
|
}
|
|
ID=$(present)
|
|
if [ -n "$ID" ] && [ "$ID" != "None" ]; then
|
|
echo "base GPU AMI already present: $BASE = $ID"; exit 0
|
|
fi
|
|
echo "base GPU AMI $BASE absent — importing prebuilt from catalog..."
|
|
sg spinifex -c "AWS_PROFILE=spinifex spx admin images import --name $BASE --ami-name $BASE"
|
|
for i in 1 2 3 4 5 6; do
|
|
ID=$(present)
|
|
[ -n "$ID" ] && [ "$ID" != "None" ] && break
|
|
sleep 5
|
|
done
|
|
[ -n "$ID" ] && [ "$ID" != "None" ] || { echo "import ran but $BASE still not available" >&2; exit 1; }
|
|
echo "imported base GPU AMI: $BASE = $ID"
|
|
'
|
|
ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
|
|
|
|
- name: Preflight — verify GPU AMIs
|
|
if: steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
set -euo pipefail
|
|
REMOTE='
|
|
set -e
|
|
present() {
|
|
AWS_PROFILE=spinifex aws ec2 describe-images \
|
|
--filters "Name=name,Values=$1" "Name=state,Values=available" \
|
|
--query "Images[0].ImageId" --output text 2>/dev/null
|
|
}
|
|
BASE=ubuntu-26.04-nvidia-gpu-x86_64
|
|
ID=$(present "$BASE")
|
|
if [ -z "$ID" ] || [ "$ID" = "None" ]; then
|
|
echo "required base GPU AMI $BASE missing — cannot run the baseline EC2 GPU test" >&2
|
|
exit 1
|
|
fi
|
|
echo "required: $BASE = $ID"
|
|
for OPT in spinifex-ecs-node-gpu spinifex-eks-node spinifex-eks-node-gpu; do
|
|
OID=$(present "$OPT")
|
|
if [ -z "$OID" ] || [ "$OID" = "None" ]; then
|
|
echo "::warning::$OPT not imported — its GPU test(s) will skip (opt-in tier)"
|
|
else
|
|
echo "opt-in: $OPT = $OID"
|
|
fi
|
|
done
|
|
'
|
|
ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
|
|
|
|
- name: Build GPU e2e test binary on runner
|
|
if: steps.gate.outputs.paused != 'true'
|
|
working-directory: tests/e2e
|
|
run: |
|
|
set -euo pipefail
|
|
make e2e-build SUITES_BUILD=gpu
|
|
ls -la _bin/gpu.test
|
|
|
|
- name: Install go-junit-report on runner
|
|
if: steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
go install github.com/jstemmer/go-junit-report/v2@latest
|
|
echo "$HOME/go/bin" >> "$GITHUB_PATH"
|
|
|
|
- name: Ship GPU e2e test binary to wattle
|
|
if: steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
set -euo pipefail
|
|
VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
|
|
ssh ${{ env.ssh_opts }} "$VM" "mkdir -p \$HOME/spinifex-gpu-e2e-bin"
|
|
scp ${{ env.ssh_opts }} tests/e2e/_bin/gpu.test "$VM":spinifex-gpu-e2e-bin/gpu.test
|
|
|
|
- name: Run GPU e2e suite on wattle
|
|
id: e2e
|
|
if: steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
set +e
|
|
VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
|
|
if [ -n "${TEST_RUN}" ]; then
|
|
TEST_RUN_FLAG="-test.run=${TEST_RUN}"
|
|
else
|
|
TEST_RUN_FLAG=""
|
|
fi
|
|
ssh -tt ${{ env.ssh_opts }} "$VM" \
|
|
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/spinifex-gpu-e2e-bin; \
|
|
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
|
|
SPINIFEX_WAN_IP='${WATTLE_SSH_HOST}' \
|
|
ARTIFACT_DIR='${ARTIFACT_DIR}' \
|
|
./gpu.test -test.v -test.timeout=60m ${TEST_RUN_FLAG} \
|
|
2>&1 | tee '${ARTIFACT_DIR}/test-gpu.log'; \
|
|
exit \${PIPESTATUS[0]}"
|
|
E2E_RC=$?
|
|
echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
|
|
exit 0
|
|
|
|
- name: Collect artifacts from wattle
|
|
if: always() && steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
set +e
|
|
mkdir -p "${ARTIFACT_DIR}"
|
|
VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
|
|
ssh ${{ env.ssh_opts }} "$VM" "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
|
|
| tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
|
|
echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"
|
|
|
|
- name: Convert test log to JUnit XML
|
|
if: always() && steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
shopt -s nullglob
|
|
for LOG in "${ARTIFACT_DIR}"/test-*.log; do
|
|
BASE=$(basename "$LOG" .log)
|
|
SUITE=${BASE#test-}
|
|
go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
|
|
done
|
|
|
|
- name: Publish test summary
|
|
if: always() && steps.gate.outputs.paused != 'true'
|
|
uses: ./.github/actions/junit-report
|
|
with:
|
|
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
|
title: "GPU E2E results"
|
|
|
|
- name: Ship test results to Elasticsearch
|
|
if: always() && steps.gate.outputs.paused != 'true'
|
|
continue-on-error: true
|
|
uses: ./.github/actions/junit-to-es
|
|
with:
|
|
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
|
cell: gpu
|
|
|
|
- name: Analyze e2e failures
|
|
if: always() && steps.gate.outputs.paused != 'true'
|
|
uses: ./.github/actions/e2e-analyze
|
|
with:
|
|
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
|
log-dir: ${{ env.ARTIFACT_DIR }}
|
|
title: "GPU E2E failure analysis"
|
|
|
|
- uses: actions/upload-artifact@v7
|
|
if: always() && steps.gate.outputs.paused != 'true'
|
|
with:
|
|
name: gpu-e2e-artifacts-${{ github.run_id }}
|
|
path: ${{ env.ARTIFACT_DIR }}
|
|
if-no-files-found: warn
|
|
retention-days: 14
|
|
|
|
- name: Cleanup — terminate leftover GPU instances, release GPU
|
|
if: always() && steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
set +e
|
|
VM="${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}"
|
|
REMOTE='
|
|
set +e
|
|
GPU_TYPES=$(AWS_PROFILE=spinifex aws ec2 describe-instance-types \
|
|
--query "InstanceTypes[?GpuInfo!=null].InstanceType" --output text | tr "\t" ",")
|
|
if [ -z "$GPU_TYPES" ]; then
|
|
echo "no GPU instance types advertised; nothing to reconcile"
|
|
exit 0
|
|
fi
|
|
IDS=$(AWS_PROFILE=spinifex aws ec2 describe-instances \
|
|
--filters "Name=instance-type,Values=${GPU_TYPES}" "Name=instance-state-name,Values=pending,running" \
|
|
--query "Reservations[].Instances[].InstanceId" --output text)
|
|
if [ -z "$IDS" ] || [ "$IDS" = "None" ]; then
|
|
echo "no leftover GPU instances — GPU already released"
|
|
exit 0
|
|
fi
|
|
echo "terminating leftover GPU instance(s): $IDS"
|
|
AWS_PROFILE=spinifex aws ec2 terminate-instances --instance-ids $IDS
|
|
'
|
|
ssh ${{ env.ssh_opts }} "$VM" "$REMOTE"
|
|
|
|
- name: Fail job if GPU suite failed
|
|
if: always() && steps.gate.outputs.paused != 'true'
|
|
run: |
|
|
if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
|
|
echo "::error::GPU e2e suite failed (rc=${{ steps.e2e.outputs.rc }}) — see analysis above; raw ES/APM traces at ${SPINIFEX_OTLP_ENDPOINT} (ci.run_id=${{ github.run_id }})"
|
|
exit 1
|
|
fi
|