gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-21 19:56:33 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
29faaa78fdbc3bcb0bbf69a669a0da291a458f5d
spinifex/.github/workflows/e2e-nightly.yml
T

1493 lines
68 KiB
YAML
Raw Blame History

name: E2E Nightly
on:
schedule:
- cron: "0 14 * * 0-4" # 00:00 Mon-Fri AEST = 14:00 Sun-Thu UTC
workflow_dispatch:
inputs:
cells:
description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection"
required: false
permissions:
contents: read
actions: read
# Every cell builds its cluster on the same five hypervisors, so two runs in
# flight at once compete for their disks and their libvirt guests. Queue the
# second rather than cancelling it: a dispatch is usually a verification run
# and discarding it loses the result someone is waiting on.
#
# The group is shared with e2e-suites.yml and e2e-ddil.yml, which draw on the
# same runner pools. e2e.yml is deliberately outside it: a single-cluster push
# run is cheap enough to overlap, and grouping it stalled every push behind a
# multi-hour nightly.
concurrency:
group: e2e-hypervisors
cancel-in-progress: false
env:
GIT_BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-26.04.qcow2
SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
jobs:
select:
runs-on: ubuntu-26.04
outputs:
cells: ${{ steps.pick.outputs.cells }}
any: ${{ steps.pick.outputs.any }}
steps:
- uses: actions/checkout@v7
with:
sparse-checkout: |
.github/e2e-cells.json
.github/scripts/suite-sets.sh
sparse-checkout-cone-mode: false
- id: pick
env:
CELLS: ${{ github.event_name == 'workflow_dispatch' && inputs.cells || '' }}
SETUP_ALLOWANCE_MIN: 10
run: |
if [ -z "$CELLS" ]; then
sel=$(jq -c . .github/e2e-cells.json)
else
sel=$(jq -c --arg c "$CELLS" \
'[.[] | . as $e | select(($c | split(",")) | index($e.cell|tostring))]' \
.github/e2e-cells.json)
fi
echo "cells=$sel" >> "$GITHUB_OUTPUT"
[ "$(jq 'length' <<<"$sel")" -gt 0 ] && any=true || any=false
echo "any=$any" >> "$GITHUB_OUTPUT"
echo "selected: $(jq -c '[.[].cell]' <<<"$sel")"
# The inner timeout must always fire before the outer one. go test
# reports where it stopped and the suite's own failure; a job timeout
# reports only "The operation was canceled". cell-27 ran a 50m rds
# suite inside a 50m job and could never finish either way.
. .github/scripts/suite-sets.sh
# Cells with no suite are filtered in jq, not with an empty-field test:
# tab is IFS whitespace, so read collapses a run of them and an empty
# middle column silently shifts every later one left.
bad=0
while IFS=$'\t' read -r cell suite budget; do
want="$(e2e_suite_timeout "$suite")"; want="${want%m}"
if [ "$budget" -lt $((want + SETUP_ALLOWANCE_MIN)) ]; then
echo "::error::cell-$cell gives suite '$suite' ${want}m inside a ${budget}m job. Raise timeout_minutes in .github/e2e-cells.json to at least $((want + SETUP_ALLOWANCE_MIN)), or lower the suite budget in suite-sets.sh." >&2
bad=1
fi
done < <(jq -r '.[]|select((.suite//"")!="")|[.cell,.suite,(.timeout_minutes//50)]|@tsv' <<<"$sel")
# A permutation cell names no suite and runs the nightly subset for its
# topology, so its bound is the longest of those. max() and not sum():
# whichever suite hangs must panic on its own timeout first, and a sum
# would demand a budget the set has never come close to using.
while IFS=$'\t' read -r cell topo budget; do
case "$topo" in
*multi) set -- ${E2E_SUITES_NIGHTLY_MULTI} ;;
*) set -- ${E2E_SUITES_NIGHTLY_SINGLE} ;;
esac
want=0
for suite in "$@"; do
t="$(e2e_suite_timeout "$suite")"; t="${t%m}"
[ "$t" -gt "$want" ] && want="$t"
done
if [ "$budget" -lt $((want + SETUP_ALLOWANCE_MIN)) ]; then
echo "::error::cell-$cell runs the $topo nightly subset, whose longest suite asks ${want}m, inside a ${budget}m job. Raise timeout_minutes in .github/e2e-cells.json to at least $((want + SETUP_ALLOWANCE_MIN))." >&2
bad=1
fi
done < <(jq -r '.[]|select((.suite//"")=="")|[.cell,.topo,(.timeout_minutes//50)]|@tsv' <<<"$sel")
[ "$bad" -eq 0 ]
matrix:
name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{
matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
needs: select
if: needs.select.outputs.any == 'true'
strategy:
fail-fast: false
max-parallel: 3
matrix:
include: ${{ fromJSON(needs.select.outputs.cells) }}
runs-on: [self-hosted, "${{ matrix.runner }}"]
timeout-minutes: ${{ matrix.timeout_minutes || 50 }}
env:
TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}
SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=${{ matrix.cell }},ci.matrix=${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}
steps:
- name: Clean Workspace
run: |
sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v7
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- uses: actions/checkout@v7
with:
path: spinifex
# Twelve cells share one run id and every cell names its hosts
# node1/node2/node3, so ci.cell above separates them and ci.job_id here
# links each cell's telemetry to the job log that produced it. Runs
# before Bootstrap bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env.
- name: Resolve CI job id
uses: ./spinifex/.github/actions/otel-job-id
with:
job-name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
- name: Build Install Artifacts
working-directory: mulga/scripts/tofu-cluster
env:
MATRIX_SUITE: ${{ matrix.suite }}
run: |
# The script's default set is the nightly baseline. A service cell
# runs one suite outside it, and asks for that binary by name.
./build-install-artifacts.sh "${GIT_BRANCH}" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-tests.tar" \
--extra-suites "${MATRIX_SUITE}"
# Every runner agent runs on one host, while their nodes sit on several, so
# this samples the runner's host and not the node's hypervisor. Kept for
# runner-side contention; the node's own capacity is sampled after.
- name: Sample runner-host load (before)
if: always()
run: |
mkdir -p "${ARTIFACT_DIR}"
{
echo "=== before: $(date -u +%Y-%m-%dT%H:%M:%SZ) runner=${RUNNER_NAME} host=$(hostname) ==="
echo "loadavg: $(cat /proc/loadavg)"
grep -E '^(MemTotal|MemAvailable|SwapFree):' /proc/meminfo || true
echo "cpu(steal is field 8): $(grep '^cpu ' /proc/stat || true)"
echo "running qemu: $(pgrep -c qemu-system-x86 || echo 0)"
} >> "${ARTIFACT_DIR}/hypervisor-load.txt"
- name: Provision VMs
working-directory: mulga/scripts/tofu-cluster
env:
MATRIX_HOST: ${{ matrix.host }}
run: |
export TF_VAR_git_branch="${GIT_BRANCH}"
# Match the family, not a pinned version: an exact match silently stops
# overriding the image when the matrix bumps release, dropping the cell
# back onto the debian gold.
TOFU_VAR_ARGS=""
case "$MATRIX_HOST" in
ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
esac
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
tofu init -input=false >> "$TOFU_LOG" 2>&1
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
# shellcheck disable=SC2086
retry_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false
- name: Bootstrap via Binary Install
working-directory: mulga/scripts/tofu-cluster
env:
MATRIX_NET: ${{ matrix.net }}
run: |
if [ "$MATRIX_NET" = "nat" ]; then
# Routed-NAT mode with a static public pool: exercises both the
# Tier-1 transit datapath and the Tier-2 EIP ingress lane.
export SPX_EXTERNAL_MODE=nat
export SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}"
elif [ "$MATRIX_NET" = "dhcp" ]; then
unset SPX_EXTERNAL_POOL
export SPX_EXTERNAL_SOURCE=dhcp
else
export SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}"
fi
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
- name: Run Single Node E2E
if: matrix.topo == 'single' && matrix.extra == '' && matrix.suite == ''
working-directory: mulga/scripts/tofu-cluster
run: |
set -euo pipefail
# Nightly runs a documented subset of the single-node set; the assert
# fails the cell if that subset ever names a suite this topology
# cannot fully exercise.
. "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
e2e_suites_assert_subset "e2e-nightly single" "${E2E_SUITES_NIGHTLY_SINGLE}" "${E2E_SUITES_SINGLE}"
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
for SUITE in ${E2E_SUITES_NIGHTLY_SINGLE}; do \
echo \"::group::suite \$SUITE\"; \
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
ARTIFACT_DIR='${ARTIFACT_DIR}' \
./\$SUITE.test -test.v -test.timeout=30m \
2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
done; exit \$FAIL"
- name: Run NAT Uplink E2E
if: matrix.extra == 'nat-single'
working-directory: mulga/scripts/tofu-cluster
# Skips cleanly (exit 0) when the tested branch predates the natuplink
# suite so scheduled runs against dev stay green until the merge lands.
run: |
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
if [ ! -f natuplink.test ]; then echo 'natuplink.test not in tests tar (branch predates suite) — skipping'; exit 0; fi; \
echo '::group::suite natuplink'; \
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-natuplink.start'; \
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
ARTIFACT_DIR='${ARTIFACT_DIR}' \
./natuplink.test -test.v -test.timeout=30m \
2>&1 | tee '${ARTIFACT_DIR}/test-natuplink.log'; \
rc=\${PIPESTATUS[0]}; echo '::endgroup::'; \
if [ \$rc -ne 0 ]; then echo 'natuplink: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"
- name: Run Multi Node E2E
if: matrix.topo == 'real-multi' && matrix.suite == ''
working-directory: mulga/scripts/tofu-cluster
run: |
set -euo pipefail
# See the single-node step: same subset guard, multi-node set.
. "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
e2e_suites_assert_subset "e2e-nightly multi" "${E2E_SUITES_NIGHTLY_MULTI}" "${E2E_SUITES_MULTI}"
INVENTORY=$(tofu output -json inventory)
NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
NODES_CSV=""
NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
for i in $(seq 0 $((NODE_COUNT - 1))); do
ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip"
done
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
for SUITE in ${E2E_SUITES_NIGHTLY_MULTI}; do \
echo \"::group::suite \$SUITE\"; \
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
ARTIFACT_DIR='${ARTIFACT_DIR}' \
./\$SUITE.test -test.v -test.timeout=30m \
2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
done; exit \$FAIL"
# The dedicated suite cells only. Resolved once here rather than in each
# step below, which would repeat the same three lines five times.
- name: Resolve VM SSH details
id: vm
if: matrix.suite != '' && matrix.topo == 'single'
working-directory: mulga/scripts/tofu-cluster
run: |
set -euo pipefail
INVENTORY=$(tofu output -json inventory)
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
echo "wan_ip=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')" >> "$GITHUB_OUTPUT"
echo "ssh_key=${SSH_KEY/#\~/$HOME}" >> "$GITHUB_OUTPUT"
# The eks suite's CreateCluster boots a control-plane VM from the eks-node
# AMI, so that AMI must be in the VM's image store before the suite runs.
# build-system-image.sh customizes with libguestfs (an isolated userspace
# appliance touching no host block device), so it is safe on the shared
# runner. The build needs source + network (runner only) and the import
# needs predastore/NATS (VM only), hence the split.
- name: Build eks-node AMI (libguestfs, runner)
if: matrix.suite == 'eks'
working-directory: spinifex
run: |
set -euo pipefail
# direct backend = no libvirt. Set here rather than in the script:
# it has to be in the environment the build itself runs in.
export LIBGUESTFS_BACKEND=direct
"${GITHUB_WORKSPACE}/spinifex/.github/scripts/ensure-libguestfs.sh"
make build-eks-node-image
- name: Import eks-node AMI on VM
if: matrix.suite == 'eks'
run: |
set -euo pipefail
eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
spinifex/scripts/images/eks-node/manifest.conf)"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
scp "${SSH_OPTS[@]}" \
/tmp/eks-node-image-build/eks-node-alpine.raw \
"$VM":/tmp/eks-node-alpine.raw
ssh "${SSH_OPTS[@]}" "$VM" \
"spx admin images import --file /tmp/eks-node-alpine.raw \
--distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
--boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1
# The eks data-plane subtest (TestEKS/KubectlGetNodes) drives the full
# `aws eks get-token` -> kubectl -> apiserver -> token-webhook -> gateway
# broker -> STS verify path. The e2e binaries run ON the VM, so kubectl
# must be on the VM's PATH or harness.NewKubectl skips the subtest and the
# cell reports green having never exercised the data plane. Pinned to the
# k3s server version (setup.sh K3S_VERSION) and checksum-verified.
- name: Install kubectl on VM
if: matrix.suite == 'eks'
run: |
set -euo pipefail
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
ssh "${SSH_OPTS[@]}" tf-user@${{ steps.vm.outputs.wan_ip }} 'bash -s' <<'EOF'
set -euo pipefail
KVER=v1.32.5
case "$(uname -m)" in
x86_64) ARCH=amd64 ;;
aarch64) ARCH=arm64 ;;
*) echo "unsupported arch $(uname -m)" >&2; exit 1 ;;
esac
cd /tmp
curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl"
curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl.sha256"
echo "$(cat kubectl.sha256) kubectl" | sha256sum -c -
sudo install -m 0755 kubectl /usr/local/bin/kubectl
kubectl version --client
EOF
# RegisterContainerInstance boots the container instance from the ecs-node
# AMI, and resolveECSNodeAMI matches on the managed-by tag rather than the
# name, so the tag is not optional. The image directory is named for
# IMAGE_NAME in the manifest (ecs-agent), not for the make target.
- name: Build ecs-node AMI (libguestfs, runner)
if: matrix.suite == 'ecs'
working-directory: spinifex
run: |
set -euo pipefail
# direct backend = no libvirt. Set here rather than in the script:
# it has to be in the environment the build itself runs in.
export LIBGUESTFS_BACKEND=direct
"${GITHUB_WORKSPACE}/spinifex/.github/scripts/ensure-libguestfs.sh"
make build-ecs-node-image
- name: Import ecs-node AMI on VM
if: matrix.suite == 'ecs'
run: |
set -euo pipefail
eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
spinifex/scripts/images/ecs-agent/manifest.conf)"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
scp "${SSH_OPTS[@]}" \
/tmp/ecs-agent-image-build/ecs-agent-alpine.raw \
"$VM":/tmp/ecs-agent-alpine.raw
ssh "${SSH_OPTS[@]}" "$VM" \
"spx admin images import --file /tmp/ecs-agent-alpine.raw \
--distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
--boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1
# CreateDBInstance boots the DB VM from an RDS AMI, so both engines' images
# must be in the store before the suite runs. The suite's psql and mariadb
# clients need no step here: they run inside a client VM that installs them
# from its own user-data, because a private-only endpoint is unreachable
# from the runner and from the cluster host alike.
- name: Build RDS AMIs (libguestfs, runner)
if: matrix.suite == 'rds'
working-directory: spinifex
run: |
set -euo pipefail
# direct backend = no libvirt. Set here rather than in the script:
# it has to be in the environment the build itself runs in.
export LIBGUESTFS_BACKEND=direct
"${GITHUB_WORKSPACE}/spinifex/.github/scripts/ensure-libguestfs.sh"
for IMAGE in rds-postgres rds-mariadb; do
make "build-${IMAGE}-image"
done
- name: Import RDS AMIs on VM
if: matrix.suite == 'rds'
run: |
set -euo pipefail
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
for IMAGE in rds-postgres rds-mariadb; do
eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
"spinifex/scripts/images/${IMAGE}/manifest.conf")"
# SYSTEM_TAG carries four tags, not one: managed-by marks it a system
# image, engine/engine-version are what an EngineVersion request
# resolves against, and the contract tag gates the data-volume format
# grant. --tag is repeatable, so expand rather than pass the whole
# string as a single value.
TAG_ARGS=""
for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
scp "${SSH_OPTS[@]}" \
"/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" \
"$VM":"/tmp/${IMAGE}-alpine.raw"
ssh "${SSH_OPTS[@]}" "$VM" \
"spx admin images import --file /tmp/${IMAGE}-alpine.raw \
--distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
--boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
done
- name: Run Service Suite E2E
if: matrix.suite != '' && matrix.topo == 'single'
env:
SUITE: ${{ matrix.suite }}
run: |
set -euo pipefail
# Same guard as the permutation steps, against the full single-node
# set: a cell naming a suite this topology cannot fully exercise fails
# here rather than skipping half its assertions.
. "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
e2e_suites_assert_subset "e2e-nightly cell-${{ matrix.cell }}" "${SUITE}" "${E2E_SUITES_SINGLE}"
TMO="$(e2e_suite_timeout "${SUITE}")"
ssh -tt -o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}" \
tf-user@${{ steps.vm.outputs.wan_ip }} \
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
echo \"::group::suite ${SUITE}\"; \
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
ARTIFACT_DIR='${ARTIFACT_DIR}' \
./${SUITE}.test -test.v -test.timeout=${TMO} \
2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log'; \
rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
if [ \$rc -ne 0 ]; then echo \"${SUITE}: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; fi; \
exit \$rc"
# The multi-node twin of the step above. Separate rather than merged
# because the suite needs every node's address to reach the whole cluster,
# which the single-node path has no way to supply.
- name: Run Multi Suite E2E
if: matrix.suite != '' && matrix.topo == 'real-multi'
working-directory: mulga/scripts/tofu-cluster
env:
SUITE: ${{ matrix.suite }}
run: |
set -euo pipefail
. "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
e2e_suites_assert_subset "e2e-nightly cell-${{ matrix.cell }}" "${SUITE}" "${E2E_SUITES_MULTI}"
TMO="$(e2e_suite_timeout "${SUITE}")"
INVENTORY=$(tofu output -json inventory)
NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
NODES_CSV=""
NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
for i in $(seq 0 $((NODE_COUNT - 1))); do
ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip"
done
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
echo \"::group::suite ${SUITE}\"; \
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
ARTIFACT_DIR='${ARTIFACT_DIR}' \
./${SUITE}.test -test.v -test.timeout=${TMO} \
2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log'; \
rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
if [ \$rc -ne 0 ]; then echo \"${SUITE}: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; fi; \
exit \$rc"
- name: Import RDS PostgreSQL AMI
if: matrix.extra == 'tofu-examples'
working-directory: mulga/scripts/tofu-cluster
run: |
set -euo pipefail
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
"spx admin images import --name spinifex-rds-postgres"
- name: Run Tofu Workbooks
if: matrix.extra == 'tofu-examples'
working-directory: mulga/scripts/tofu-cluster
# The driver emits go test -v markers per workbook; teeing to
# test-tofu.log feeds the same go-junit-report -> e2e-analyze -> upload
# chain the Go suites use, so cell 17 produces an RCA bundle too.
run: |
set +e
mkdir -p "${ARTIFACT_DIR}"
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-tofu.start"
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
"cd ~/e2e-tests && ./run-tofu-examples-e2e.sh" \
2>&1 | tee "${ARTIFACT_DIR}/test-tofu.log"
rc=${PIPESTATUS[0]}
if [ $rc -ne 0 ]; then echo "tofu: FAILED" >> "${ARTIFACT_DIR}/suite-status.txt"; fi
exit $rc
- name: Run Reboot Resilience E2E
if: matrix.extra == 'reboot-single'
working-directory: mulga/scripts/tofu-cluster
# reboot.test runs from the runner (NOT via ssh-and-run) so it survives the
# systemctl-reboot-induced ssh disconnect mid-run.
run: |
set +e
mkdir -p "${ARTIFACT_DIR}"
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
REBOOT_DIR="${RUNNER_TEMP}/reboot-e2e"
mkdir -p "$REBOOT_DIR"
tar xf "${RUNNER_TEMP}/spinifex-tests.tar" -C "$REBOOT_DIR"
chmod +x "$REBOOT_DIR/_bin/reboot.test"
SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$SSH_KEY")
CREDS=$(ssh "${SSH_OPTS[@]}" tf-user@${WAN_IP} 'cat ~/.aws/credentials')
AWS_AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
AWS_SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
echo "::add-mask::${AWS_AK}"; echo "::add-mask::${AWS_SK}"
date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-reboot.start"
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
SPINIFEX_WAN_IP="${WAN_IP}" SPINIFEX_SSH_KEY="${SSH_KEY}" \
SPINIFEX_AWS_ENDPOINT="https://${WAN_IP}:9999" \
SPINIFEX_AWS_ACCESS_KEY_ID="${AWS_AK}" SPINIFEX_AWS_SECRET_ACCESS_KEY="${AWS_SK}" \
ARTIFACT_DIR="${ARTIFACT_DIR}" \
"$REBOOT_DIR/_bin/reboot.test" -test.v -test.timeout=30m \
2>&1 | tee "${ARTIFACT_DIR}/test-reboot.log"
rc=${PIPESTATUS[0]}
if [ $rc -ne 0 ]; then echo "reboot: FAILED" >> "${ARTIFACT_DIR}/suite-status.txt"; fi
exit $rc
- name: Sample runner-host load (after)
if: always()
run: |
mkdir -p "${ARTIFACT_DIR}"
{
echo "=== after: $(date -u +%Y-%m-%dT%H:%M:%SZ) runner=${RUNNER_NAME} host=$(hostname) ==="
echo "loadavg: $(cat /proc/loadavg)"
grep -E '^(MemTotal|MemAvailable|SwapFree):' /proc/meminfo || true
echo "cpu(steal is field 8): $(grep '^cpu ' /proc/stat || true)"
echo "running qemu: $(pgrep -c qemu-system-x86 || echo 0)"
} >> "${ARTIFACT_DIR}/hypervisor-load.txt"
# The node, not the runner's host, is what the guests are scheduled on, so
# its load is the figure that explains a starved guest. Never fails the
# cell: a node already gone is the case this is trying to describe.
- name: Sample node load (after)
if: always()
working-directory: mulga/scripts/tofu-cluster
run: |
set +e
mkdir -p "${ARTIFACT_DIR}"
INVENTORY=$(tofu output -json inventory 2>/dev/null)
NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip' 2>/dev/null)
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path' 2>/dev/null)
SSH_KEY="${SSH_KEY/#\~/$HOME}"
[ -n "$NODE1_IP" ] && [ "$NODE1_IP" != "null" ] || exit 0
[ -n "$SSH_KEY" ] && [ "$SSH_KEY" != "null" ] || exit 0
{
echo "=== node after: $(date -u +%Y-%m-%dT%H:%M:%SZ) ip=${NODE1_IP} ==="
ssh -o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=10 \
-i "$SSH_KEY" "tf-user@${NODE1_IP}" \
'echo "host: $(hostname)"; echo "loadavg: $(cat /proc/loadavg)";
grep -E "^(MemTotal|MemAvailable|SwapFree):" /proc/meminfo;
echo "cpu(steal is field 8): $(grep "^cpu " /proc/stat)";
echo "running qemu: $(pgrep -c qemu-system-x86 || echo 0)"' 2>&1
} >> "${ARTIFACT_DIR}/node-load.txt"
exit 0
- name: Collect e2e artifacts
if: always()
working-directory: mulga/scripts/tofu-cluster
run: |
set +e
mkdir -p "${ARTIFACT_DIR}"
INVENTORY=$(tofu output -json inventory 2>/dev/null) || { echo "no inventory; skipping"; exit 0; }
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path'); SSH_KEY="${SSH_KEY/#\~/$HOME}"
NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY")
ssh "${SSH_OPTS[@]}" tf-user@${NODE1_IP} "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
| tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
for i in $(seq 0 $((NODE_COUNT - 1))); do
IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip"); idx=$((i + 1))
mkdir -p "${ARTIFACT_DIR}/vm-console-node${idx}"
ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
| tar -C "${ARTIFACT_DIR}/vm-console-node${idx}" -xf - 2>/dev/null
ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
> "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \
> "${ARTIFACT_DIR}/openvswitch-ipsec-node${idx}.log" 2>/dev/null
# The unit filter above drops kernel and OOM messages, which are
# exactly what explains one node behaving unlike its siblings.
# Gzipped on the far side; the whole journal is far larger.
ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl --no-pager --output=short-iso --since "90 min ago" | gzip -6' \
> "${ARTIFACT_DIR}/full-journal-node${idx}.log.gz" 2>/dev/null
# ovn-controller and ovs-vswitchd log to files, never to journald:
# without these a cross-chassis black-hole cannot be attributed to
# IPsec versus SB port binding.
mkdir -p "${ARTIFACT_DIR}/ovs-logs-node${idx}"
ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo tar -C /var/log -cf - ovn openvswitch 2>/dev/null || true' \
| tar -C "${ARTIFACT_DIR}/ovs-logs-node${idx}" -xf - 2>/dev/null
done
cat "${ARTIFACT_DIR}"/spinifex-journal-node*.log \
> "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || true
echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"
- name: Install go-junit-report
if: always()
run: |
go install github.com/jstemmer/go-junit-report/v2@latest
echo "$HOME/go/bin" >> "$GITHUB_PATH"
- name: Convert test logs to JUnit XML
if: always()
run: |
shopt -s nullglob
for LOG in "${ARTIFACT_DIR}"/test-*.log; do
BASE=$(basename "$LOG" .log); SUITE=${BASE#test-}
go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
done
# Tolerated on failure: the sink being unreachable must not turn a
# green test run red.
- name: Ship test results to Elasticsearch
if: always()
continue-on-error: true
uses: ./spinifex/.github/actions/junit-to-es
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
cell: ${{ matrix.cell }}
- name: Analyze e2e failures
if: always()
uses: ./spinifex/.github/actions/e2e-analyze
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
log-dir: ${{ env.ARTIFACT_DIR }}
title: "E2E failure analysis (cell ${{ matrix.cell }})"
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-artifacts-cell-${{ matrix.cell }}-${{ github.run_id }}
path: ${{ env.ARTIFACT_DIR }}
if-no-files-found: warn
retention-days: 14
# The XML on its own, so the roll-up can read every cell's counts without
# pulling nineteen journals it has no use for.
- name: Upload junit only
if: always()
uses: actions/upload-artifact@v7
with:
name: junit-cell-${{ matrix.cell }}-${{ github.run_id }}
path: ${{ env.ARTIFACT_DIR }}/junit-*.xml
if-no-files-found: ignore
retention-days: 14
# Gated like every other step, not just always(): a cell the dispatch
# deselected never checked out or ran tofu init, so destroying would fail
# on missing plugins and report the cell red for correctly doing nothing.
- name: Teardown
if: always()
working-directory: mulga/scripts/tofu-cluster
env:
MATRIX_HOST: ${{ matrix.host }}
run: |
# Must mirror the Provision step's override, or destroy runs against a
# different image path than apply did.
TOFU_VAR_ARGS=""
case "$MATRIX_HOST" in
ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
esac
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
# shellcheck disable=SC2086
retry_run "tofu destroy" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
- name: Upload tofu log
if: always()
uses: actions/upload-artifact@v7
with:
name: tofu-log-cell-${{ matrix.cell }}-${{ github.run_id }}
path: ${{ env.TOFU_LOG }}
if-no-files-found: ignore
retention-days: 14
# Bare-metal single-node suite. Doesn't fit the tofu matrix shape, so it
# runs as a sibling job via the reusable e2e-baremetal workflow. Gated like a
# matrix cell ("cell 20"): every weeknight cron, or dispatch with 20 selected
# (blank cells = all). Tests `dev` on schedule to match GIT_BRANCH above.
baremetal:
name: cell-20 (baremetal/pxe/real-hw)
if: >-
github.event_name == 'schedule' ||
inputs.cells == '' ||
contains(format(',{0},', inputs.cells), ',20,')
permissions:
contents: read
statuses: write
uses: ./.github/workflows/e2e-baremetal.yml
with:
ref: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
# A reusable-workflow call gets none of this workflow's context (env:
# above, matrix.cell) for free — cell-20 needs its own attrs string.
# There's no matrix leg here, just the one fixed baremetal cell.
otel-attrs: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=20,ci.matrix=baremetal/pxe/real-hw
secrets: inherit
# The guest disk-performance gate. Same bare-metal node as cell 20 and
# therefore queued behind it by the shared concurrency group, which is the
# point: perf numbers off a cell still running other suites are noise, and
# the availability assertions measure host responsiveness under a single
# tenant's load, which a co-tenant correctness suite would confound.
#
# It reimages the cluster first, like every other caller of that workflow, so
# it is a fresh node rather than whatever cell-20 left behind.
diskperf:
name: cell-28 (baremetal/diskperf)
needs: baremetal
if: >-
!cancelled() && (
github.event_name == 'schedule' ||
inputs.cells == '' ||
contains(format(',{0},', inputs.cells), ',28,')
)
permissions:
contents: read
statuses: write
uses: ./.github/workflows/e2e-baremetal.yml
with:
ref: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
cell: "28"
suites: diskperf
otel-attrs: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=28,ci.matrix=baremetal/pxe/diskperf
secrets: inherit
# ISO cells. Unlike 1-19 these need no tofu: the harnesses run QEMU directly
# on the runner. The nodes are therefore L1, which keeps the instances they
# launch at L2 and their benchmark numbers comparable between the two cells.
#
# Serialised against each other because the guest MACs are fixed
# (52:54:00:12:...): stable MACs are what make DHCP discovery work, and two
# concurrent runs would put duplicates on the rack LAN, which is a confusing
# failure for everyone on it, not just CI.
iso-single:
name: cell-21 (iso/dhcp/single)
if: >-
github.event_name == 'schedule' ||
inputs.cells == '' ||
contains(format(',{0},', inputs.cells), ',21,')
runs-on: [self-hosted, ci-single]
# ISO build, install, boot and pool re-form is ~35 of these on its own,
# before the workload step imports an AMI and launches an instance.
timeout-minutes: 90
concurrency:
group: iso-qemu-rack
cancel-in-progress: false
env:
CI_WAN_MODE: dhcp
CI_PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
# What a real node gets. The harness default of 4G is a dev-box size, and
# importing an 820 MiB AMI on top of the running stack does not fit in it.
MEM: 8G
SMP: 8
# .230 is consumed by the OVN router's SNAT, leaving .231-.234 for
# instances. Disjoint from cell-22's pool so a run that dies holding its
# SNAT address cannot collide with the next cell.
EXT_POOL_START: 192.168.0.230
EXT_POOL_END: 192.168.0.234
EXT_GATEWAY: 192.168.1.1
EXT_PREFIX_LEN: 23
# Pins where the benchmark writes, so collecting and removing it later are
# both exact. Left to itself the harness picks a timestamped name.
BENCH_OUT: /tmp/cell21-bench-${{ github.run_id }}
steps:
- name: Clean Workspace
run: |
sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- name: Configure work directories
run: |
E2E_ROOT=/mnt/nvme/e2e
mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
{ sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v7
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
path: mulga
- uses: actions/checkout@v7
with:
ref: ${{ env.GIT_BRANCH }}
path: mulga/spinifex
- name: Resolve sibling refs
id: siblings
env:
BRANCH: ${{ env.GIT_BRANCH }}
run: bash .github/resolve-sibling-refs.sh
working-directory: mulga/spinifex
- uses: actions/checkout@v7
with:
repository: mulgadc/viperblock
ref: ${{ steps.siblings.outputs.viperblock }}
path: mulga/viperblock
- uses: actions/checkout@v7
with:
repository: mulgadc/predastore
ref: ${{ steps.siblings.outputs.predastore }}
path: mulga/predastore
- uses: actions/checkout@v7
with:
repository: mulgadc/northstar
ref: ${{ steps.siblings.outputs.northstar }}
path: mulga/northstar
- uses: actions/checkout@v7
with:
repository: mulgadc/bluebottle
ref: ${{ steps.siblings.outputs.bluebottle }}
path: mulga/bluebottle
# Assert the host can do what the cell assumes, before spending 40
# minutes finding out it cannot. A checklist in a plan doc goes stale;
# this does not.
- name: Preflight
run: |
set -euo pipefail
mkdir -p "$TMPDIR"
ip link show "${BRIDGE:-br-wan}" >/dev/null ||
{ echo "::error::bridge br-wan not found on this runner"; exit 1; }
grep -qE "^allow +${BRIDGE:-br-wan}$" /etc/qemu/bridge.conf ||
{ echo "::error::br-wan not permitted in /etc/qemu/bridge.conf"; exit 1; }
command -v sshpass >/dev/null ||
{ echo "::error::sshpass not installed"; exit 1; }
free_mb=$(free -m | awk '/^Mem:/ {print $7}')
[ "$free_mb" -ge 8192 ] ||
{ echo "::error::need 8 GiB available, have ${free_mb} MiB"; exit 1; }
# A 40G image on tmpfs competes with the 8 GiB the VM itself needs.
fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
[ "$fstype" != tmpfs ] ||
{ echo "::error::$TMPDIR is tmpfs — the node disk would be in RAM"; exit 1; }
df -h "$TMPDIR"
avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
[ "$avail_gb" -ge 60 ] ||
{ echo "::error::need 60 GiB on $TMPDIR, have ${avail_gb} GiB"; exit 1; }
- name: Build artefacts
working-directory: mulga/scripts/iso-builder
run: make build-rootfs build-squashfs build-initramfs
# One key for both halves of the cell. test-workload.sh SSHes to the node
# AND registers the same key as the EC2 key pair it uses to reach the
# instances, so the key test-smoke.sh installs has to be that one — its
# default ~/.ssh/id_*.pub would leave the workload step locked out.
- name: Prepare SSH key
run: |
set -euo pipefail
mkdir -p "$HOME/.ssh" && chmod 700 "$HOME/.ssh"
# Two keys, because the guest key must not share a name with the one
# smoke-test.sh registers from the node: the node holds that private
# half, and IMDS caches material by key name for five minutes, so
# re-importing over it is invisible to the guests.
for k in spinifex-key spinifex-workload-key; do
[ -f "$HOME/.ssh/$k" ] ||
ssh-keygen -t ed25519 -N "" -f "$HOME/.ssh/$k" -C "$k-ci"
done
# Boots the ISO, installs unattended, reads the lease off the serial
# console, runs the in-guest smoke test, and installs the key for the
# next step.
- name: Install and smoke test
id: smoke
working-directory: mulga/scripts/iso-builder
run: |
set -euo pipefail
# KEEP_NODE: the harness owns the only VM there is and tears it down
# on exit, so without this the workload step below has no node left.
KEEP_NODE=1 SSH_IDENTITY="$HOME/.ssh/spinifex-key.pub" \
./test/test-smoke.sh | tee /tmp/smoke-$GITHUB_RUN_ID.log
ip=$(sed -n 's/^\[smoke\] node address: //p' /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
[ -n "$ip" ] || { echo "::error::smoke test did not report a node address"; exit 1; }
echo "node_ip=$ip" >> "$GITHUB_OUTPUT"
# The customer path: AMI, key, VPC, instance, SSH in. --benchmark records
# host and guest disk numbers so a regression shows up as a number rather
# than a story.
- name: Workload and benchmark
working-directory: mulga/scripts/iso-builder
run: |
set -euo pipefail
cat > /tmp/cell21.env <<EOF
NODES=(${{ steps.smoke.outputs.node_ip }})
NODE_NAMES=(spinifex-ci)
SSH_USER=spinifex
SSH_KEY_PATH=$HOME/.ssh/spinifex-workload-key
# SSH_KEY_PATH reaches the guests, SSH_IDENTITY reaches the node, and
# they are deliberately different keys. Without the second, node SSH
# falls back to the runner's default identity, which the node has
# never seen.
SSH_IDENTITY=$HOME/.ssh/spinifex-key
EXT_POOL_START=$EXT_POOL_START
EXT_POOL_END=$EXT_POOL_END
EXT_GATEWAY=$EXT_GATEWAY
EXT_PREFIX_LEN=$EXT_PREFIX_LEN
EOF
./test/test-workload.sh --env /tmp/cell21.env --benchmark
# The node's own journal, which nothing else preserves: the serial log
# stops at login and the harness owns the only node there is. Pulled
# before teardown and never fatal, so a passing night keeps it too.
- name: Collect node journal
if: always() && steps.smoke.outputs.node_ip != ''
env:
NODE_IP: ${{ steps.smoke.outputs.node_ip }}
run: |
mkdir -p /tmp/cell21-artifacts-$GITHUB_RUN_ID
timeout 180 ssh -o BatchMode=yes -o StrictHostKeyChecking=no \
-o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 \
-i "$HOME/.ssh/spinifex-key" "spinifex@${NODE_IP}" \
'sudo journalctl -b --no-pager --output=short-iso | gzip -6' \
> "/tmp/cell21-artifacts-$GITHUB_RUN_ID/node-journal.log.gz" ||
echo "::warning::could not pull the node journal from ${NODE_IP}"
- name: Collect logs
if: always()
run: |
mkdir -p /tmp/cell21-artifacts-$GITHUB_RUN_ID
cp /tmp/smoke-$GITHUB_RUN_ID.log /tmp/cell21-artifacts-$GITHUB_RUN_ID/ ||
echo "::warning::no smoke log to collect"
# fio JSON and the summary table. Warned about rather than swallowed:
# a benchmark that ran and produced nothing is worth knowing about.
if [ -d "$BENCH_OUT" ]; then
cp -r "$BENCH_OUT" /tmp/cell21-artifacts-$GITHUB_RUN_ID/benchmark
else
echo "::warning::no benchmark results at $BENCH_OUT"
fi
# The guest's own view. Without it a node that dies mid-test is only
# visible as the harness command that stopped producing output.
serial=$(sed -n 's/^\[smoke\] serial log preserved: //p' \
/tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
[ -n "$serial" ] && cp "$serial" /tmp/cell21-artifacts-$GITHUB_RUN_ID/serial.log || true
node=$(sed -n 's/^\[smoke\] node logs preserved: //p' \
/tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
[ -n "$node" ] && cp "$node" /tmp/cell21-artifacts-$GITHUB_RUN_ID/node.log || true
# QEMU's stderr no longer reaches the step log: it is redirected so a
# kept-alive VM cannot hold the step's stdout open.
qemu=$(sed -n 's/^\[smoke\] qemu log preserved: //p' \
/tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
[ -n "$qemu" ] && cp "$qemu" /tmp/cell21-artifacts-$GITHUB_RUN_ID/qemu.log || true
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-artifacts-cell-21-${{ github.run_id }}
path: /tmp/cell21-artifacts-${{ github.run_id }}
if-no-files-found: warn
# Matches the predastore benchmark in cell-23: a night's numbers are
# only useful against the nights before them.
retention-days: 90
- name: Teardown
if: always()
working-directory: mulga/scripts/iso-builder
run: |
# No env file means the workload step never ran, so there is nothing
# of its to clean up.
if [ -f /tmp/cell21.env ]; then
./test/test-workload.sh --env /tmp/cell21.env --cleanup || true
fi
sudo pkill -f spinifex-smoke-test || true
rm -rf /tmp/cell21-artifacts-$GITHUB_RUN_ID /tmp/cell21.env \
/tmp/smoke-$GITHUB_RUN_ID.log "$BENCH_OUT"
# Runs before BENCH_OUT existed left a timestamped directory behind
# every night. Sweep anything older than a day.
find /tmp -maxdepth 1 -name 'spinifex-benchmark-*' -type d -mmin +1440 \
-exec rm -rf {} + 2>/dev/null || true
# KEEP_NODE leaves the node's disk and serial log behind for us.
sleep 2
rm -f "$TMPDIR"/tmp.*.qcow2 "$TMPDIR"/tmp.*.log "$TMPDIR"/tmp.*-node.log
iso-multi:
name: cell-22 (iso/dhcp/three-node)
# After cell-21 rather than beside it: same concurrency group, and running
# them together would need 32 GiB on one box.
needs: [iso-single]
if: >-
!cancelled() && (
github.event_name == 'schedule' ||
inputs.cells == '' ||
contains(format(',{0},', inputs.cells), ',22,') )
runs-on: [self-hosted, ci-multi]
timeout-minutes: 90
concurrency:
group: iso-qemu-rack
cancel-in-progress: false
env:
MEM: 8G
SMP: 8
PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
EXT_POOL_START: 192.168.0.235
EXT_POOL_END: 192.168.0.239
EXT_GATEWAY: 192.168.1.1
EXT_PREFIX_LEN: 23
steps:
- name: Clean Workspace
run: |
sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- name: Configure work directories
run: |
E2E_ROOT=/mnt/nvme/e2e
mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
{ sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
# Pinned so the per-node serial logs can be collected afterwards, and
# kept beside TMPDIR rather than in /tmp: it also holds nine 40G node
# disks, which is what the tmpfs assertion below is about.
echo "WORK_DIR=$E2E_ROOT/work/cell22-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v7
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
path: mulga
- uses: actions/checkout@v7
with:
ref: ${{ env.GIT_BRANCH }}
path: mulga/spinifex
- name: Resolve sibling refs
id: siblings
env:
BRANCH: ${{ env.GIT_BRANCH }}
run: bash .github/resolve-sibling-refs.sh
working-directory: mulga/spinifex
- uses: actions/checkout@v7
with:
repository: mulgadc/viperblock
ref: ${{ steps.siblings.outputs.viperblock }}
path: mulga/viperblock
- uses: actions/checkout@v7
with:
repository: mulgadc/predastore
ref: ${{ steps.siblings.outputs.predastore }}
path: mulga/predastore
- uses: actions/checkout@v7
with:
repository: mulgadc/northstar
ref: ${{ steps.siblings.outputs.northstar }}
path: mulga/northstar
- uses: actions/checkout@v7
with:
repository: mulgadc/bluebottle
ref: ${{ steps.siblings.outputs.bluebottle }}
path: mulga/bluebottle
# 3 x 8 GiB plus 9 sparse 40G disks, before the instances the cluster
# launches on top.
- name: Preflight
run: |
set -euo pipefail
mkdir -p "$TMPDIR"
free_mb=$(free -m | awk '/^Mem:/ {print $7}')
[ "$free_mb" -ge 26624 ] ||
{ echo "::error::need 26 GiB available, have ${free_mb} MiB"; exit 1; }
fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
[ "$fstype" != tmpfs ] ||
{ echo "::error::$TMPDIR is tmpfs — nine node disks would be in RAM"; exit 1; }
df -h "$TMPDIR"
avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
[ "$avail_gb" -ge 120 ] ||
{ echo "::error::need 120 GiB on $TMPDIR ($fstype), have ${avail_gb} GiB"; exit 1; }
- name: Build artefacts
working-directory: mulga/scripts/iso-builder
run: make build-rootfs build-squashfs build-initramfs
# Four phases: install three nodes, convert to one cluster, assert it
# formed (including the external pool), then pull node 1's power and
# assert the survivors hold quorum.
#
# Phase 5 (test-workload.sh, as cell-21 runs it) is not wired yet: the
# nodes are still on user-mode NAT and reachable only through the
# harness's own port forwards, and test-cluster.sh does not open the
# formation port. Both land with mulga-yxtda and the bridged WAN mode;
# until then this cell proves the cluster forms and survives, not that a
# customer can use it.
- name: Install, form and verify the cluster
working-directory: mulga/scripts/iso-builder
run: |
set -euo pipefail
./test/test-cluster.sh 2>&1 | tee /tmp/cell22-$GITHUB_RUN_ID.log
- name: Collect logs
if: always()
run: |
mkdir -p /tmp/cell22-artifacts-$GITHUB_RUN_ID
cp /tmp/cell22-$GITHUB_RUN_ID.log /tmp/cell22-artifacts-$GITHUB_RUN_ID/ ||
echo "::warning::no cluster log to collect"
# Serial only. The same directory holds the nodes' disk images, which
# are gigabytes and must not go into an artifact.
cp "$WORK_DIR"/*-serial.log /tmp/cell22-artifacts-$GITHUB_RUN_ID/ 2>/dev/null ||
echo "no per-node serial logs — the run passed and cleaned up after itself"
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-artifacts-cell-22-${{ github.run_id }}
path: /tmp/cell22-artifacts-${{ github.run_id }}
if-no-files-found: warn
retention-days: 90
- name: Teardown
if: always()
run: |
sudo pkill -f spinifex-cluster-test || true
rm -rf /tmp/cell22-artifacts-$GITHUB_RUN_ID /tmp/cell22-$GITHUB_RUN_ID.log \
"$WORK_DIR"
# Predastore throughput ("cell 23"). Predastore has no runners of its own and
# spinifex depends on it, so the benchmark lives here and checks predastore
# out as a sibling.
#
# needs: is the whole point. A throughput benchmark sharing a host with a cell
# measures the neighbour, and ci-single and ci-multi both resolve to
# bottlebrush, so "a different label" would not have separated them. needs: on
# a matrix job waits for every leg, so this starts on a quiet machine.
#
# !cancelled() rather than always(): a failed cell must not silently skip the
# benchmark and leave a green-looking night with no number in it, but always()
# also overrides cancellation and keeps the run starting jobs after a cancel.
performance:
name: cell-23 (predastore/throughput)
needs: [matrix, baremetal, iso-single, iso-multi]
if: >-
!cancelled() && (
github.event_name == 'schedule' ||
inputs.cells == '' ||
contains(format(',{0},', inputs.cells), ',23,') )
runs-on: [self-hosted, ci-single]
timeout-minutes: 60
env:
# Two-minute samples. The 30s smoke preset is too short to compare nights.
PERF_PRESET: compare
steps:
- name: Clean Workspace
run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- name: Configure work directories
run: |
E2E_ROOT=/mnt/nvme/e2e
mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
{ sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
- name: Checkout Spinifex
uses: actions/checkout@v7
with:
path: spinifex
# Which predastore branch to measure: the same-named branch when one
# exists, dev otherwise. Matches how the e2e path already picks siblings,
# so a cross-repo change is benchmarked as a set.
- name: Resolve sibling refs
id: siblings
env:
BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
run: bash .github/resolve-sibling-refs.sh
working-directory: spinifex
- name: Checkout Predastore
uses: actions/checkout@v7
with:
repository: mulgadc/predastore
ref: ${{ steps.siblings.outputs.predastore }}
path: predastore
# The benchmark works in $TMPDIR and predastore refuses writes below a 5%
# free-space watermark. Without this the symptom is several hundred
# "storage pool is at capacity" lines from warp and no statement of how
# much space there actually was.
#
# TMPDIR is pointed at the runner's disk above, and the tmpfs assertion is
# the one that matters: a storage benchmark on a RAM-backed filesystem
# measures memcpy. It would report a number that looks excellent, sit in
# the baseline, and never notice a disk regression.
- name: Preflight
run: |
set -euo pipefail
mkdir -p "$TMPDIR"
fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
case "$fstype" in
tmpfs|ramfs)
echo "::error::$TMPDIR is $fstype — a storage benchmark there measures RAM"
exit 1 ;;
esac
# Work dirs from runs that were killed before their cleanup trap ran.
# -mtime +0 so a concurrent run's directory is never in scope.
find "$TMPDIR" -maxdepth 1 -name 'predastore-e2e-performance.*' \
-type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
df -h "$TMPDIR"
avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
pct_used=$(df --output=pcent "$TMPDIR" | tail -1 | tr -dc '0-9')
if [ "$pct_used" -ge 90 ] || [ "$avail_gb" -lt 20 ]; then
echo "::error::$TMPDIR ($fstype) is ${pct_used}% full with ${avail_gb} GiB free."
echo "::error::The benchmark needs headroom above predastore's 5% watermark."
exit 1
fi
# PERF_PORT_OFFSET defaults to 10000, so the benchmark's own nodes do not
# collide with a cluster left on the default ports.
- name: Run throughput benchmark
working-directory: predastore
run: |
set -euo pipefail
# Into the results directory, so the artifact reads on its own rather
# than needing the job log it was printed into. The benchmark only
# ever creates this directory, so a log placed here first survives.
mkdir -p "$PERF_RESULTS_ROOT"
make e2e-performance 2>&1 | tee "$PERF_RESULTS_ROOT/e2e-performance.log"
# Report-only. The baseline comparison and the -10% gate are mulga-wpjsx;
# until the run-to-run spread is known from real nights, a threshold would
# be a guess, and a gate that cries wolf gets muted.
- uses: actions/upload-artifact@v7
if: always()
with:
name: predastore-perf-${{ github.run_id }}
path: ${{ env.PERF_RESULTS_ROOT }}
if-no-files-found: warn
retention-days: 90
- name: Clean up results
if: always()
run: rm -rf "$PERF_RESULTS_ROOT"
# Fault injection, not throughput. Cell-23 asks how fast predastore is when
# every node is healthy; this asks whether it survives one that is not.
#
# After cell-23 rather than beside it: both want a quiet single runner, and a
# benchmark sharing a machine with a fault injection run measures the wrong
# thing. The port offset keeps them apart even so.
stress:
name: cell-24 (predastore/fault-injection)
needs: [matrix, baremetal, iso-single, iso-multi, performance]
if: >-
!cancelled() && (
github.event_name == 'schedule' ||
inputs.cells == '' ||
contains(format(',{0},', inputs.cells), ',24,') )
runs-on: [self-hosted, ci-single]
# The thirteen scenarios run serially and are ~35 minutes of work on their
# own, measured from the per-scenario predastore matrix, before checkout
# and build. 30 could not have passed whatever the disk did.
timeout-minutes: 75
env:
# Freezes a follower. STRESS_HOST=leader is a known failure: a gate keeps
# the frozen node first in its meta read order and pays the full client
# timeout per key, so listing slows in proportion to object count.
STRESS_HOST: follower
# Clear of cell-23's default 10000, so the two never collide on a runner
# that lets them overlap.
STRESS_PORT_OFFSET: "20000"
steps:
- name: Clean Workspace
run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
# STRESS_WORK_ROOT is set as well as TMPDIR because the harness ignores
# TMPDIR for the scenarios that need the most disk: large-object and
# multipart-upload default to $HOME/.cache/predastore-e2e, on the theory
# that TMPDIR is commonly tmpfs. Here it is a 938G nvme, so pointing them
# at it is both correct and the only path with room.
- name: Configure work directories
run: |
E2E_ROOT=/mnt/nvme/e2e
for d in work stress-work; do
mkdir -p "$E2E_ROOT/$d" 2>/dev/null ||
{ sudo mkdir -p "$E2E_ROOT/$d" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/$d"; }
done
echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
echo "STRESS_WORK_ROOT=$E2E_ROOT/stress-work" >> "$GITHUB_ENV"
echo "STRESS_RESULTS_ROOT=$E2E_ROOT/predastore-stress-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
- name: Checkout Spinifex
uses: actions/checkout@v7
with:
path: spinifex
- name: Resolve sibling refs
id: siblings
env:
BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
run: bash .github/resolve-sibling-refs.sh
working-directory: spinifex
- name: Checkout Predastore
uses: actions/checkout@v7
with:
repository: mulgadc/predastore
ref: ${{ steps.siblings.outputs.predastore }}
path: predastore
# A frozen node is indistinguishable from a slow one, so a run on a
# loaded or nearly full disk fails as a timeout and reads as a real
# fault. The checks are the cheap way to tell those apart afterwards.
- name: Preflight
run: |
set -euo pipefail
mkdir -p "$TMPDIR" "$STRESS_WORK_ROOT"
for d in "$TMPDIR" "$STRESS_WORK_ROOT" "$HOME/.cache/predastore-e2e"; do
[ -d "$d" ] || continue
find "$d" -maxdepth 1 -name 'predastore-e2e-stress.*' \
-type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
done
# Both paths, because they can be different filesystems and the one
# holding the large objects is not the one TMPDIR names. large-object
# needs 2GiB and 4GiB sharded with parity, multipart 1GiBx8.
rc=0
for d in "$TMPDIR" "$STRESS_WORK_ROOT"; do
df -h "$d"
avail_gb=$(df -BG --output=avail "$d" | tail -1 | tr -dc '0-9')
if [ "$avail_gb" -lt 40 ]; then
echo "::error::$d has ${avail_gb} GiB free; the run needs headroom above predastore's 5% watermark and room for the 1GiBx8 multipart case."
rc=1
fi
done
exit "$rc"
# Leftovers from a run killed before its cleanup trap: a surviving node
# holds its ports and fails this run somewhere unrelated.
- name: Clear stale nodes
run: pkill -KILL -f '/bin/s3d -config' || true
- name: Run fault injection
working-directory: predastore
run: |
set -euo pipefail
# events.txt carries the timeline and the FAIL line, but not the build
# output or anything Warp wrote before the harness took over. The
# harness only creates this directory, so a log placed here survives.
mkdir -p "$STRESS_RESULTS_ROOT"
make e2e-stress 2>&1 | tee "$STRESS_RESULTS_ROOT/e2e-stress.log"
- uses: actions/upload-artifact@v7
if: always()
with:
name: predastore-stress-${{ github.run_id }}
path: ${{ env.STRESS_RESULTS_ROOT }}
if-no-files-found: warn
retention-days: 90
- name: Clean up
if: always()
run: |
pkill -KILL -f '/bin/s3d -config' || true
rm -rf "$STRESS_RESULTS_ROOT"
# Reading a nightly meant opening every cell job, and the Actions UI does not
# separate a cell that failed from one cancelled out from under it. needs:
# every cell so this runs last; always() so a red run is the one it describes.
summary:
name: run summary
needs:
[
select,
matrix,
baremetal,
diskperf,
iso-single,
iso-multi,
performance,
stress,
]
if: always()
runs-on: ubuntu-26.04
steps:
- uses: actions/checkout@v7
with:
sparse-checkout: .github/scripts/e2e-run-summary.py
sparse-checkout-cone-mode: false
- name: Collect junit
uses: actions/download-artifact@v8
with:
pattern: junit-cell-*-${{ github.run_id }}
path: junit
continue-on-error: true
- name: Write run summary
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
JUNIT_ROOT: junit
run: python3 .github/scripts/e2e-run-summary.py
Reference in New Issue View Git Blame Copy Permalink