gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-20 03:11:46 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
a8591380cb3ed757ca55152bb347bb404b0d683e
spinifex/.github/workflows/e2e-nightly.yml
T
Julian Sommer a216d6f596 chore(ci): point nightly e2e at dev branch 
Scheduled runs were using github.ref_name, which equals the default branch
(main) — so the nightly was testing main. Hardcode GIT_BRANCH=dev so the
build-install-artifacts.sh + tofu provisioning target dev's HEAD instead.

GitHub always loads scheduled-workflow files from the default branch, so
this change must stay in sync on main for the schedule to use it.
2026-05-05 09:54:53 +10:00

220 lines
10 KiB
YAML
Raw Blame History

name: E2E Nightly
on:
schedule:
- cron: "0 14 * * 0-4" # 00:00 Mon-Fri AEST = 14:00 Sun-Thu UTC
workflow_dispatch:
inputs:
cells:
description: "Comma-separated cell numbers to run (blank = all)"
required: false
permissions:
contents: read
# Scheduled runs target dev (active development trunk). GitHub loads scheduled
# workflow files from the repo's default branch, so this file must be kept in
# sync on main for the schedule to use it.
env:
GIT_BRANCH: dev
UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-24.04.qcow2
jobs:
matrix:
name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{
matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
strategy:
fail-fast: false
max-parallel: 3
matrix:
include:
- { cell: 1, install: binary, topo: single, net: dhcp, host: debian-13, arch: x86_64, runner: ci-single, extra: "" }
- { cell: 2, install: binary, topo: single, net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-single, extra: "" }
- { cell: 3, install: binary, topo: single, net: dhcp, host: ubuntu-24.04, arch: x86_64, runner: ci-single, extra: "" }
- { cell: 8, install: binary, topo: real-multi, net: dhcp, host: debian-13, arch: x86_64, runner: ci-multi, extra: "" }
- { cell: 9, install: binary, topo: real-multi, net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-multi, extra: "" }
- { cell: 10, install: binary, topo: real-multi, net: dhcp, host: ubuntu-24.04, arch: x86_64, runner: ci-multi, extra: "" }
- { cell: 17, install: binary, topo: single, net: static, host: debian-13, arch: x86_64, runner: ci-pseudo, extra: "tofu-examples" }
- { cell: 18, install: binary, topo: single, net: static, host: debian-13, arch: x86_64, runner: ci-single, extra: "reboot-single" }
runs-on: [ self-hosted, "${{ matrix.runner }}" ]
timeout-minutes: 35
env:
TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
steps:
# blank or scheduled run = full matrix.
# Input must be comma-separated with no whitespace (e.g. '1,2,17', not '1, 2').
- name: Gate on Dispatch Input
id: gate
env:
CELL: ${{ matrix.cell }}
CELLS: ${{ github.event_name == 'workflow_dispatch' && inputs.cells || '' }}
run: |
if [ -z "$CELLS" ] || [[ ",${CELLS}," == *",${CELL},"* ]]; then
echo "selected=true" >> "$GITHUB_OUTPUT"
echo "cell ${CELL} selected"
else
echo "selected=false" >> "$GITHUB_OUTPUT"
echo "cell ${CELL} not in '${CELLS}' — subsequent steps will skip"
fi
- name: Clean Workspace
if: steps.gate.outputs.selected == 'true'
run: rm -rf "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3
if: steps.gate.outputs.selected == 'true'
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v6
if: steps.gate.outputs.selected == 'true'
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- name: Build Install Artifacts
if: steps.gate.outputs.selected == 'true'
working-directory: mulga/scripts/tofu-cluster
run: |
./build-install-artifacts.sh "${GIT_BRANCH}" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-tests.tar"
- name: Provision VMs
if: steps.gate.outputs.selected == 'true'
working-directory: mulga/scripts/tofu-cluster
env:
MATRIX_HOST: ${{ matrix.host }}
run: |
export TF_VAR_git_branch="${GIT_BRANCH}"
TOFU_VAR_ARGS=""
if [ "$MATRIX_HOST" = "ubuntu-24.04" ]; then
TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
fi
tofu init -input=false >> "$TOFU_LOG" 2>&1
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
- name: Bootstrap via Binary Install
if: steps.gate.outputs.selected == 'true'
working-directory: mulga/scripts/tofu-cluster
env:
MATRIX_NET: ${{ matrix.net }}
run: |
if [ "$MATRIX_NET" = "dhcp" ]; then
unset SPX_EXTERNAL_POOL
export SPX_EXTERNAL_SOURCE=dhcp
else
export SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}"
fi
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
- name: Run Single Node E2E
if: steps.gate.outputs.selected == 'true' && matrix.topo == 'single' && matrix.extra == ''
working-directory: mulga/scripts/tofu-cluster
run: |
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
"cd ~/e2e-tests && ./run-e2e.sh && ./run-cert-e2e.sh && ENDPOINT=https://${WAN_IP}:9999 ./run-lb-e2e.sh"
- name: Run Multi Node E2E
if: steps.gate.outputs.selected == 'true' && matrix.topo == 'real-multi'
working-directory: mulga/scripts/tofu-cluster
run: |
INVENTORY=$(tofu output -json inventory)
NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
NODE_IPS=""
NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
for i in $(seq 0 $((NODE_COUNT - 1))); do
NODE_IPS="${NODE_IPS} $(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")"
done
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
"cd ~/e2e-tests && ./run-multinode-e2e.sh ${NODE_IPS} && ./run-cert-e2e.sh --multinode ${NODE_IPS} && ENDPOINT=https://${NODE1_IP}:9999 ./run-lb-e2e.sh --nodes ${NODE_IPS}"
- name: Run Tofu Workbooks
if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples'
working-directory: mulga/scripts/tofu-cluster
run: |
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
"cd ~/e2e-tests && ./run-tofu-examples-e2e.sh"
- name: Run Reboot Resilience E2E
if: steps.gate.outputs.selected == 'true' && matrix.extra == 'reboot-single'
working-directory: mulga/scripts/tofu-cluster
# Driver runs from the runner (NOT via ssh-and-run) so it survives the
# systemctl-reboot-induced ssh disconnect mid-run.
run: |
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
REBOOT_DIR="${RUNNER_TEMP}/reboot-e2e"
mkdir -p "$REBOOT_DIR"
tar xf "${RUNNER_TEMP}/spinifex-tests.tar" -C "$REBOOT_DIR"
chmod +x "$REBOOT_DIR/run-reboot-e2e.sh"
WAN_IP="$WAN_IP" SSH_KEY="$SSH_KEY" "$REBOOT_DIR/run-reboot-e2e.sh"
- name: Dump Cluster Journals on Failure
if: failure() && steps.gate.outputs.selected == 'true'
working-directory: mulga/scripts/tofu-cluster
run: |
INVENTORY=$(tofu output -json inventory 2>/dev/null) || { echo "no inventory; skipping"; exit 0; }
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
for i in $(seq 0 $((NODE_COUNT - 1))); do
IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
echo "===== node$((i+1)) ($IP) ====="
for svc in spinifex-nats spinifex-predastore spinifex-viperblock spinifex-daemon spinifex-awsgw spinifex-vpcd dhcpcd; do
echo "--- $svc ---"
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" tf-user@${IP} \
"sudo journalctl -u $svc --no-pager -n 200 2>/dev/null || echo '(not found)'" 2>&1 || echo "(node unreachable)"
done
echo "--- DHCP client state ---"
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY" tf-user@${IP} \
"systemctl is-enabled dhcpcd.service 2>/dev/null; systemctl is-active dhcpcd.service 2>/dev/null; sudo ss -lupn 'sport = 68' 2>/dev/null; pgrep -laf dhcpcd 2>/dev/null" 2>&1 || echo "(node unreachable)"
done
- name: Teardown
if: always()
working-directory: mulga/scripts/tofu-cluster
env:
MATRIX_HOST: ${{ matrix.host }}
run: |
TOFU_VAR_ARGS=""
if [ "$MATRIX_HOST" = "ubuntu-24.04" ]; then
TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
fi
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
Reference in New Issue View Git Blame Copy Permalink