gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-15 17:11:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: remove reboot e2e

Browse Source
This commit is contained in:
Julian Sommer
2026-06-12 10:46:14 +10:00
parent d076a434d4
commit 2df35d3822
1 changed files with 14 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/e2e.yml
+14 -292
View File
@@ -36,10 +36,9 @@ env:
TEST_RUN: ${{ inputs.test_run || '' }}
jobs:
# Shadow-mode suite selector (Bead 4, mulga-siv-278). Computes which suites
# the diff *would* run and logs it to the job summary. Does NOT gate or
# override the real SUITES env yet — it runs alongside the full matrix so we
# can compare predicted vs actual before flipping it on (Bead 6).
# Shadow-mode suite selector. Computes which suites the diff *would* run and logs it to the job summary.
# Does NOT gate or override the real SUITES env yet — it runs alongside the full matrix so we
# can compare predicted vs actual before flipping it on.
select_suites:
name: Select Suites (shadow)
runs-on: ubuntu-latest
@@ -76,11 +75,11 @@ jobs:
e2e_single:
name: Single-Node E2E
# Skip when the dispatch input requests no in-VM suites (e.g.
# suites='e2e-reboot' only). Push triggers and the default SUITES list
# suites='e2e-multinode' only). Push triggers and the default SUITES list
# both include single/cert/lb so the inputs.suites null-check keeps the
# job running on plain PR pushes.
if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-eks')
runs-on: [ self-hosted, ci-single ]
runs-on: [self-hosted, ci-single]
timeout-minutes: 45
env:
TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
@@ -108,11 +107,6 @@ jobs:
- name: Clean Workspace
run: |
# Detach any leaked mounts under the workspace before deleting. A
# killed image build can leave a `mount --bind /dev` under its build
# dir; if `rm -rf` traverses it as root it wipes the host's /dev
# (deletes /dev/null and wedges the box). umount deepest-first, then
# forbid rm from crossing any mount boundary as a hard backstop.
awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
@@ -221,7 +215,6 @@ jobs:
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
# Pull import args straight from the manifest so they never drift.
eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
spinifex/scripts/images/eks-node/manifest.conf)"
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
@@ -275,7 +268,7 @@ jobs:
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|multinode)$' | tr '\n' ' ')
SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(multinode)$' | tr '\n' ' ')
if [ -z "$(echo "${SUITES_BUILD}" | tr -d ' ')" ]; then
echo "no in-VM suites requested (SUITES=${SUITES}); nothing to build"
exit 0
@@ -317,7 +310,7 @@ jobs:
id: e2e
run: |
set +e
SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|multinode)$' | tr '\n' ' ')
SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(multinode)$' | tr '\n' ' ')
if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
echo "no in-VM suites requested (SUITES=${SUITES}); skipping"
echo "rc=0" >> "$GITHUB_OUTPUT"
@@ -482,7 +475,7 @@ jobs:
e2e_real_multi:
name: Real Multi-Node E2E
if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-multinode')
runs-on: [ self-hosted, ci-multi ]
runs-on: [self-hosted, ci-multi]
timeout-minutes: 60
env:
TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
@@ -510,11 +503,6 @@ jobs:
- name: Clean Workspace
run: |
# Detach any leaked mounts under the workspace before deleting. A
# killed image build can leave a `mount --bind /dev` under its build
# dir; if `rm -rf` traverses it as root it wipes the host's /dev
# (deletes /dev/null and wedges the box). umount deepest-first, then
# forbid rm from crossing any mount boundary as a hard backstop.
awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
@@ -610,7 +598,7 @@ jobs:
# JetStream clustering isn't solid yet (mulga-omq/z2z/5yg). EKS is
# validated single-node only for now; re-enable here once clustered
# JetStream + multinode EKS are a deliberate target.
SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|single|eks)$' | tr '\n' ' ')
SUITES_BUILD=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(single|eks)$' | tr '\n' ' ')
if [ -z "$(echo "${SUITES_BUILD}" | tr -d ' ')" ]; then
echo "no multinode-eligible suites requested (SUITES=${SUITES}); nothing to build"
exit 0
@@ -693,7 +681,7 @@ jobs:
id: e2e
run: |
set +e
SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(reboot|single|eks)$' | tr '\n' ' ')
SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -vE '^(single|eks)$' | tr '\n' ' ')
if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
echo "no multinode-eligible suites requested (SUITES=${SUITES}); skipping"
echo "rc=0" >> "$GITHUB_OUTPUT"
@@ -894,275 +882,10 @@ jobs:
exit 1
fi
e2e_reboot:
name: Reboot Resilience E2E
# Opt-in: only fires when the SUITES dispatch input lists e2e-reboot.
# The reboot test reboots the spinifex VM, so it must run on the runner
# (not shipped to the VM like the cert/lb/single suites). Runs in
# parallel with e2e_single — both target [self-hosted, ci-single]
# and GitHub Actions serialises on the underlying runner allocation,
# so no explicit needs: ordering is required.
if: contains(github.event.inputs.suites, 'e2e-reboot')
runs-on: [ self-hosted, ci-single ]
timeout-minutes: 60
env:
TOFU_LOG: /tmp/e2e-tofu-reboot-${{ github.run_id }}.log
ARTIFACT_DIR: /tmp/spinifex-e2e-reboot-artifacts-${{ github.run_id }}
steps:
- name: Reap Orphan Procs
run: |
set +e
UID_SELF=$(id -u)
WS="${GITHUB_WORKSPACE}"
reap() {
local sig="$1"
for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
case "$cwd" in
"$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
esac
done
}
reap TERM
sleep 3
reap KILL
find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
true
- name: Clean Workspace
run: |
# Detach any leaked mounts under the workspace before deleting. A
# killed image build can leave a `mount --bind /dev` under its build
# dir; if `rm -rf` traverses it as root it wipes the host's /dev
# (deletes /dev/null and wedges the box). umount deepest-first, then
# forbid rm from crossing any mount boundary as a hard backstop.
awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
| sort -r | xargs -r -I{} sudo umount -l -R {} || true
sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
repositories: mulga
- uses: actions/checkout@v6
with:
repository: mulgadc/mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
path: mulga
- uses: actions/checkout@v6
with:
path: spinifex
- uses: actions/setup-go@v6
with:
go-version-file: spinifex/go.mod
cache: false
- name: Re-enable Go toolchain auto-download
run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
- name: Prune rootless docker cache
run: |
docker container prune -f || true
docker image prune -af || true
docker buildx prune -af --filter "until=24h" || true
docker system df || true
- name: Build Install Artifacts
working-directory: mulga/scripts/tofu-cluster
run: |
./build-install-artifacts.sh "${GIT_BRANCH}" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-tests.tar"
- name: Provision VM
working-directory: mulga/scripts/tofu-cluster
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
export TF_VAR_git_branch="${GIT_BRANCH}"
quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false
quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
quiet_run "scrub-env (pre-clean)" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
quiet_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
ok "VM provisioned"
- name: Bootstrap via Binary Install
working-directory: mulga/scripts/tofu-cluster
run: |
SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
"${RUNNER_TEMP}/setup.sh" \
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
- name: Resolve VM SSH details
id: vm
working-directory: mulga/scripts/tofu-cluster
run: |
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT"
echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
- name: Build reboot.test on runner
working-directory: spinifex/tests/e2e
run: |
set -euo pipefail
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
banner "build reboot"
group_open "go test -c"
make e2e-build SUITES_BUILD="reboot"
group_close
ok "$(basename _bin/reboot.test) $(stat -c '%s' _bin/reboot.test | numfmt --to=iec)"
- name: Fetch AWS credentials from VM
id: creds
run: |
set -euo pipefail
SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
CREDS=$(ssh "${SSH_OPTS[@]}" "$VM" 'cat ~/.aws/credentials')
AWS_AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
AWS_SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
if [ -z "$AWS_AK" ] || [ -z "$AWS_SK" ]; then
echo "::error::Failed to extract spinifex AWS credentials from ${{ steps.vm.outputs.wan_ip }}"
exit 1
fi
echo "::add-mask::${AWS_AK}"
echo "::add-mask::${AWS_SK}"
echo "ak=${AWS_AK}" >> "$GITHUB_OUTPUT"
echo "sk=${AWS_SK}" >> "$GITHUB_OUTPUT"
- name: Run reboot suite on runner
id: e2e
run: |
set +e
mkdir -p "${ARTIFACT_DIR}"
date -u +%Y-%m-%dT%H:%M:%SZ > "${ARTIFACT_DIR}/test-reboot.start"
R=$'\033[0m'; B=$'\033[1m'; C=$'\033[36m'
printf '\n%s%s━━ suite reboot ━━%s\n' "$B" "$C" "$R"
if [ -n "${TEST_RUN}" ]; then
TEST_RUN_FLAG="-test.run=${TEST_RUN}"
else
TEST_RUN_FLAG=""
fi
GITHUB_ACTIONS=true FORCE_COLOR=1 \
SPINIFEX_E2E=1 SPINIFEX_MODE=single \
SPINIFEX_WAN_IP="${{ steps.vm.outputs.wan_ip }}" \
SPINIFEX_SSH_KEY="${{ steps.vm.outputs.ssh_key }}" \
SPINIFEX_AWS_ENDPOINT="https://${{ steps.vm.outputs.wan_ip }}:9999" \
SPINIFEX_AWS_ACCESS_KEY_ID="${{ steps.creds.outputs.ak }}" \
SPINIFEX_AWS_SECRET_ACCESS_KEY="${{ steps.creds.outputs.sk }}" \
ARTIFACT_DIR="${ARTIFACT_DIR}" \
./spinifex/tests/e2e/_bin/reboot.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
2>&1 | tee "${ARTIFACT_DIR}/test-reboot.log" \
| sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'
rc=${PIPESTATUS[0]}
echo "rc=${rc}" >> "$GITHUB_OUTPUT"
exit 0
- name: Collect VM journals
if: always()
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
mkdir -p "${ARTIFACT_DIR}"
SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 \
-i "${{ steps.vm.outputs.ssh_key }}")
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
> "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || true
ok "journal: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || echo 0) lines"
- name: Install go-junit-report
if: always()
run: |
go install github.com/jstemmer/go-junit-report/v2@latest
echo "$HOME/go/bin" >> "$GITHUB_PATH"
- name: Convert test log to JUnit XML
if: always()
run: |
shopt -s nullglob
for LOG in "${ARTIFACT_DIR}"/test-*.log; do
BASE=$(basename "$LOG" .log)
SUITE=${BASE#test-}
go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
done
- name: Publish test summary
if: always()
uses: ./spinifex/.github/actions/junit-report
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
title: "E2E reboot results"
- name: Analyze e2e failures
if: always()
uses: ./spinifex/.github/actions/e2e-analyze
with:
junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
log-dir: ${{ env.ARTIFACT_DIR }}
title: "Reboot E2E failure analysis"
- uses: actions/upload-artifact@v7
if: always()
with:
name: e2e-reboot-artifacts-${{ github.run_id }}
path: ${{ env.ARTIFACT_DIR }}
if-no-files-found: warn
retention-days: 14
- name: Teardown
if: always()
working-directory: mulga/scripts/tofu-cluster
run: |
set +e
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
UID_SELF=$(id -u)
WS="${GITHUB_WORKSPACE}"
reap() {
local sig="$1"
for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
case "$cwd" in
"$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;;
esac
done
}
reap TERM
sleep 2
reap KILL
find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
quiet_run "tofu destroy" "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
ok "teardown complete"
- name: Fail job if reboot suite failed
if: always()
run: |
if [ "${{ steps.e2e.outputs.rc }}" != "0" ]; then
echo "::error::Reboot suite failed (rc=${{ steps.e2e.outputs.rc }})"
exit 1
fi
e2e_status:
name: Report E2E Status
if: always()
needs: [ e2e_single, e2e_real_multi, e2e_reboot ]
needs: [e2e_single, e2e_real_multi]
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
@@ -1175,16 +898,15 @@ jobs:
echo "::error::no SHA available (inputs.sha and github.sha both empty)"
exit 1
fi
single_ok=true; multi_ok=true; reboot_ok=true
single_ok=true; multi_ok=true
case "${{ needs.e2e_single.result }}" in success|skipped) ;; *) single_ok=false ;; esac
case "${{ needs.e2e_real_multi.result }}" in success|skipped) ;; *) multi_ok=false ;; esac
case "${{ needs.e2e_reboot.result }}" in success|skipped) ;; *) reboot_ok=false ;; esac
if $single_ok && $multi_ok && $reboot_ok; then
if $single_ok && $multi_ok; then
STATE="success"
DESC="E2E suites passed"
else
STATE="failure"
DESC="E2E failed: single=${{ needs.e2e_single.result }} multi=${{ needs.e2e_real_multi.result }} reboot=${{ needs.e2e_reboot.result }}"
DESC="E2E failed: single=${{ needs.e2e_single.result }} multi=${{ needs.e2e_real_multi.result }}"
fi
gh api "repos/${{ github.repository }}/statuses/${REPORT_SHA}" \
-f state="$STATE" \