mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 19:01:42 +00:00
Julian Sommer
335 lines
16 KiB
YAML
335 lines
16 KiB
YAML
name: E2E Nightly
|
|
|
|
on:
|
|
schedule:
|
|
- cron: "0 14 * * 0-4" # 00:00 Mon-Fri AEST = 14:00 Sun-Thu UTC
|
|
workflow_dispatch:
|
|
inputs:
|
|
cells:
|
|
description: "Comma-separated cell numbers to run (blank = all)"
|
|
required: false
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
# Scheduled runs are loaded from the default branch (main), so the schedule
|
|
# trigger must stay in sync on main for cron to fire.
|
|
env:
|
|
GIT_BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
|
|
UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-24.04.qcow2
|
|
|
|
jobs:
|
|
matrix:
|
|
name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{
|
|
matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
|
|
strategy:
|
|
fail-fast: false
|
|
max-parallel: 3
|
|
matrix:
|
|
include:
|
|
- { cell: 1, install: binary, topo: single, net: dhcp, host: debian-13, arch: x86_64, runner: ci-single, extra: "" }
|
|
- { cell: 2, install: binary, topo: single, net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-single, extra: "" }
|
|
- { cell: 3, install: binary, topo: single, net: dhcp, host: ubuntu-24.04, arch: x86_64, runner: ci-single, extra: "" }
|
|
- { cell: 8, install: binary, topo: real-multi, net: dhcp, host: debian-13, arch: x86_64, runner: ci-multi, extra: "" }
|
|
- { cell: 9, install: binary, topo: real-multi, net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-multi, extra: "" }
|
|
- { cell: 10, install: binary, topo: real-multi, net: dhcp, host: ubuntu-24.04, arch: x86_64, runner: ci-multi, extra: "" }
|
|
- { cell: 17, install: binary, topo: single, net: static, host: debian-13, arch: x86_64, runner: ci-pseudo, extra: "tofu-examples" }
|
|
- { cell: 18, install: binary, topo: single, net: static, host: debian-13, arch: x86_64, runner: ci-single, extra: "reboot-single" }
|
|
|
|
runs-on: [ self-hosted, "${{ matrix.runner }}" ]
|
|
timeout-minutes: 35
|
|
env:
|
|
TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
|
|
ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}
|
|
|
|
steps:
|
|
# blank or scheduled run = full matrix.
|
|
# Input must be comma-separated with no whitespace (e.g. '1,2,17', not '1, 2').
|
|
- name: Gate on Dispatch Input
|
|
id: gate
|
|
env:
|
|
CELL: ${{ matrix.cell }}
|
|
CELLS: ${{ github.event_name == 'workflow_dispatch' && inputs.cells || '' }}
|
|
run: |
|
|
if [ -z "$CELLS" ] || [[ ",${CELLS}," == *",${CELL},"* ]]; then
|
|
echo "selected=true" >> "$GITHUB_OUTPUT"
|
|
echo "cell ${CELL} selected"
|
|
else
|
|
echo "selected=false" >> "$GITHUB_OUTPUT"
|
|
echo "cell ${CELL} not in '${CELLS}' — subsequent steps will skip"
|
|
fi
|
|
|
|
- name: Clean Workspace
|
|
if: steps.gate.outputs.selected == 'true'
|
|
run: |
|
|
sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
|
|
if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
|
|
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
|
|
|
|
- uses: actions/create-github-app-token@v3
|
|
if: steps.gate.outputs.selected == 'true'
|
|
id: app-token
|
|
with:
|
|
client-id: ${{ secrets.APP_ID }}
|
|
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
|
repositories: mulga
|
|
|
|
- uses: actions/checkout@v6
|
|
if: steps.gate.outputs.selected == 'true'
|
|
with:
|
|
repository: mulgadc/mulga
|
|
token: ${{ steps.app-token.outputs.token }}
|
|
sparse-checkout: scripts/tofu-cluster
|
|
path: mulga
|
|
|
|
- uses: actions/checkout@v6
|
|
if: steps.gate.outputs.selected == 'true'
|
|
with:
|
|
path: spinifex
|
|
|
|
- name: Build Install Artifacts
|
|
if: steps.gate.outputs.selected == 'true'
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
./build-install-artifacts.sh "${GIT_BRANCH}" \
|
|
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
|
|
"${RUNNER_TEMP}/setup.sh" \
|
|
"${RUNNER_TEMP}/spinifex-tests.tar"
|
|
|
|
- name: Provision VMs
|
|
if: steps.gate.outputs.selected == 'true'
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
env:
|
|
MATRIX_HOST: ${{ matrix.host }}
|
|
run: |
|
|
export TF_VAR_git_branch="${GIT_BRANCH}"
|
|
|
|
TOFU_VAR_ARGS=""
|
|
if [ "$MATRIX_HOST" = "ubuntu-24.04" ]; then
|
|
TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
|
|
fi
|
|
|
|
tofu init -input=false >> "$TOFU_LOG" 2>&1
|
|
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
|
|
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
|
|
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
|
|
tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
|
|
|
|
- name: Bootstrap via Binary Install
|
|
if: steps.gate.outputs.selected == 'true'
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
env:
|
|
MATRIX_NET: ${{ matrix.net }}
|
|
run: |
|
|
if [ "$MATRIX_NET" = "dhcp" ]; then
|
|
unset SPX_EXTERNAL_POOL
|
|
export SPX_EXTERNAL_SOURCE=dhcp
|
|
else
|
|
export SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}"
|
|
fi
|
|
|
|
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
|
|
"${RUNNER_TEMP}/setup.sh" \
|
|
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
|
|
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
|
|
|
|
- name: Run Single Node E2E
|
|
if: steps.gate.outputs.selected == 'true' && matrix.topo == 'single' && matrix.extra == ''
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
INVENTORY=$(tofu output -json inventory)
|
|
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
|
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
|
|
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
|
|
for SUITE in single cert lb; do \
|
|
echo \"::group::suite \$SUITE\"; \
|
|
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
|
|
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
|
|
ARTIFACT_DIR='${ARTIFACT_DIR}' \
|
|
./\$SUITE.test -test.v -test.timeout=30m \
|
|
2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
|
|
rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
|
|
if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
|
|
done; exit \$FAIL"
|
|
|
|
- name: Run Multi Node E2E
|
|
if: steps.gate.outputs.selected == 'true' && matrix.topo == 'real-multi'
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
INVENTORY=$(tofu output -json inventory)
|
|
NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
|
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
|
|
NODES_CSV=""
|
|
NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
|
|
for i in $(seq 0 $((NODE_COUNT - 1))); do
|
|
ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
|
|
NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip"
|
|
done
|
|
|
|
ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
|
|
"set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; FAIL=0; \
|
|
for SUITE in multinode cert lb; do \
|
|
echo \"::group::suite \$SUITE\"; \
|
|
date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
|
|
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
|
|
SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
|
|
SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
|
|
ARTIFACT_DIR='${ARTIFACT_DIR}' \
|
|
./\$SUITE.test -test.v -test.timeout=30m \
|
|
2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log'; \
|
|
rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
|
|
if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
|
|
done; exit \$FAIL"
|
|
|
|
- name: Run Tofu Workbooks
|
|
if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples'
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
# The driver emits go test -v markers per workbook; teeing to
|
|
# test-tofu.log feeds the same go-junit-report -> e2e-analyze -> upload
|
|
# chain the Go suites use, so cell 17 produces an RCA bundle too.
|
|
run: |
|
|
set +e
|
|
mkdir -p "${ARTIFACT_DIR}"
|
|
INVENTORY=$(tofu output -json inventory)
|
|
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
|
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-tofu.start"
|
|
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
|
|
"cd ~/e2e-tests && ./run-tofu-examples-e2e.sh" \
|
|
2>&1 | tee "${ARTIFACT_DIR}/test-tofu.log"
|
|
rc=${PIPESTATUS[0]}
|
|
if [ $rc -ne 0 ]; then echo "tofu: FAILED" >> "${ARTIFACT_DIR}/suite-status.txt"; fi
|
|
exit $rc
|
|
|
|
- name: Run Reboot Resilience E2E
|
|
if: steps.gate.outputs.selected == 'true' && matrix.extra == 'reboot-single'
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
# reboot.test runs from the runner (NOT via ssh-and-run) so it survives the
|
|
# systemctl-reboot-induced ssh disconnect mid-run.
|
|
run: |
|
|
set +e
|
|
mkdir -p "${ARTIFACT_DIR}"
|
|
INVENTORY=$(tofu output -json inventory)
|
|
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
|
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
|
|
REBOOT_DIR="${RUNNER_TEMP}/reboot-e2e"
|
|
mkdir -p "$REBOOT_DIR"
|
|
tar xf "${RUNNER_TEMP}/spinifex-tests.tar" -C "$REBOOT_DIR"
|
|
chmod +x "$REBOOT_DIR/_bin/reboot.test"
|
|
|
|
SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$SSH_KEY")
|
|
CREDS=$(ssh "${SSH_OPTS[@]}" tf-user@${WAN_IP} 'cat ~/.aws/credentials')
|
|
AWS_AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
|
|
AWS_SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
|
|
echo "::add-mask::${AWS_AK}"; echo "::add-mask::${AWS_SK}"
|
|
|
|
date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-reboot.start"
|
|
GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
|
|
SPINIFEX_WAN_IP="${WAN_IP}" SPINIFEX_SSH_KEY="${SSH_KEY}" \
|
|
SPINIFEX_AWS_ENDPOINT="https://${WAN_IP}:9999" \
|
|
SPINIFEX_AWS_ACCESS_KEY_ID="${AWS_AK}" SPINIFEX_AWS_SECRET_ACCESS_KEY="${AWS_SK}" \
|
|
ARTIFACT_DIR="${ARTIFACT_DIR}" \
|
|
"$REBOOT_DIR/_bin/reboot.test" -test.v -test.timeout=30m \
|
|
2>&1 | tee "${ARTIFACT_DIR}/test-reboot.log"
|
|
rc=${PIPESTATUS[0]}
|
|
if [ $rc -ne 0 ]; then echo "reboot: FAILED" >> "${ARTIFACT_DIR}/suite-status.txt"; fi
|
|
exit $rc
|
|
|
|
- name: Collect e2e artifacts
|
|
if: always() && steps.gate.outputs.selected == 'true'
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
set +e
|
|
mkdir -p "${ARTIFACT_DIR}"
|
|
INVENTORY=$(tofu output -json inventory 2>/dev/null) || { echo "no inventory; skipping"; exit 0; }
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path'); SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
|
|
NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
|
SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY")
|
|
|
|
ssh "${SSH_OPTS[@]}" tf-user@${NODE1_IP} "tar -C '${ARTIFACT_DIR}' -cf - . 2>/dev/null || true" \
|
|
| tar -C "${ARTIFACT_DIR}" -xf - 2>/dev/null
|
|
|
|
for i in $(seq 0 $((NODE_COUNT - 1))); do
|
|
IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip"); idx=$((i + 1))
|
|
mkdir -p "${ARTIFACT_DIR}/vm-console-node${idx}"
|
|
ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
|
|
| tar -C "${ARTIFACT_DIR}/vm-console-node${idx}" -xf - 2>/dev/null
|
|
ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
|
|
> "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
|
|
done
|
|
|
|
cat "${ARTIFACT_DIR}"/spinifex-journal-node*.log \
|
|
> "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || true
|
|
echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"
|
|
|
|
- name: Install go-junit-report
|
|
if: always() && steps.gate.outputs.selected == 'true'
|
|
run: |
|
|
go install github.com/jstemmer/go-junit-report/v2@latest
|
|
echo "$HOME/go/bin" >> "$GITHUB_PATH"
|
|
|
|
- name: Convert test logs to JUnit XML
|
|
if: always() && steps.gate.outputs.selected == 'true'
|
|
run: |
|
|
shopt -s nullglob
|
|
for LOG in "${ARTIFACT_DIR}"/test-*.log; do
|
|
BASE=$(basename "$LOG" .log); SUITE=${BASE#test-}
|
|
go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
|
|
done
|
|
|
|
- name: Analyze e2e failures
|
|
if: always() && steps.gate.outputs.selected == 'true'
|
|
uses: ./spinifex/.github/actions/e2e-analyze
|
|
with:
|
|
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
|
|
log-dir: ${{ env.ARTIFACT_DIR }}
|
|
title: "E2E failure analysis (cell ${{ matrix.cell }})"
|
|
|
|
- uses: actions/upload-artifact@v7
|
|
if: always() && steps.gate.outputs.selected == 'true'
|
|
with:
|
|
name: e2e-artifacts-cell-${{ matrix.cell }}-${{ github.run_id }}
|
|
path: ${{ env.ARTIFACT_DIR }}
|
|
if-no-files-found: warn
|
|
retention-days: 14
|
|
|
|
- name: Teardown
|
|
if: always()
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
env:
|
|
MATRIX_HOST: ${{ matrix.host }}
|
|
run: |
|
|
TOFU_VAR_ARGS=""
|
|
if [ "$MATRIX_HOST" = "ubuntu-24.04" ]; then
|
|
TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}"
|
|
fi
|
|
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
|
|
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
|
|
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
|
|
|
|
# Bare-metal single-node suite. Doesn't fit the tofu matrix shape, so it
|
|
# runs as a sibling job via the reusable e2e-baremetal workflow. Gated like a
|
|
# matrix cell ("cell 20"): every weeknight cron, or dispatch with 20 selected
|
|
# (blank cells = all). Tests `dev` on schedule to match GIT_BRANCH above.
|
|
baremetal:
|
|
name: cell-20 (baremetal/pxe/real-hw)
|
|
if: >-
|
|
github.event_name == 'schedule' ||
|
|
inputs.cells == '' ||
|
|
contains(format(',{0},', inputs.cells), ',20,')
|
|
permissions:
|
|
contents: read
|
|
statuses: write
|
|
uses: ./.github/workflows/e2e-baremetal.yml
|
|
with:
|
|
ref: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
|
|
secrets: inherit
|