mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 17:11:30 +00:00
Josh Siviour
0e08b9d256
* feat: ship per-test JUnit results to the Elasticsearch sink Every e2e workflow already converts its test log to JUnit XML and renders it into the job summary, then discards it with the artifact. mulga-ci-runs records which suite failed; nothing records which test failed, so root-causing a failure means reading raw Actions logs across historical runs. The junit-to-es action bulk-indexes one doc per leaf test case into mulga-ci-test-results: name, subtest, status, duration, failure text, run id, cell, suite, branch and sha. Parent t.Run wrappers are dropped so a failing subtest counts once, matching the job summary renderer. The doc id is deterministic, so a replayed post overwrites rather than double-counting. Called from every workflow that produces JUnit XML, and tolerated on failure: an unreachable sink must not turn a green test run red. * feat: classify unparsed JUnit cases as unknown, not error go-junit-report emits <error message="No test result found"> when a === RUN line has no matching result line, which means the log it was fed was truncated or filtered. Recording that as a failure would put 78 phantom failures per green bare-metal run into the pass rate.
243 lines
11 KiB
YAML
243 lines
11 KiB
YAML
name: E2E DDIL
|
|
|
|
# This workflow is manually triggered while the DDIL scenarios are being
|
|
# validated.
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
quarantined:
|
|
description: "Comma-separated scenario letters to quarantine (A..F)"
|
|
required: false
|
|
default: ""
|
|
|
|
permissions:
|
|
contents: read
|
|
# actions: read lets the otel-job-id action look up this job's numeric id via
|
|
# the Actions API — github.job is only the YAML key, no context exposes it.
|
|
actions: read
|
|
|
|
env:
|
|
GIT_BRANCH: ${{ github.ref_name }}
|
|
# OTLP export to the central Elastic sink (wattle). Repo variable overrides;
|
|
# empty string disables telemetry entirely.
|
|
SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
|
|
SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
|
|
|
|
jobs:
|
|
ddil:
|
|
name: DDIL Real Multi-Node
|
|
runs-on: [ self-hosted, ci-multi ]
|
|
timeout-minutes: 60
|
|
env:
|
|
TOFU_LOG: /tmp/e2e-tofu-ddil-${{ github.run_id }}.log
|
|
# Job-level env wins over the workflow-level default. ci.cell=ddil marks
|
|
# this workflow's telemetry so a DDIL run's partition-induced errors are
|
|
# never mistaken for a healthy multi-node run's in the sink.
|
|
SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=ddil
|
|
steps:
|
|
- name: Clean Workspace
|
|
run: sudo rm -rf "$GITHUB_WORKSPACE"/*
|
|
|
|
- uses: actions/create-github-app-token@v3
|
|
id: app-token
|
|
with:
|
|
client-id: ${{ secrets.APP_ID }}
|
|
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
|
repositories: mulga
|
|
|
|
- uses: actions/checkout@v7
|
|
name: Checkout Spinifex
|
|
with:
|
|
path: spinifex
|
|
|
|
- uses: actions/checkout@v7
|
|
name: Checkout Mulga (tofu-cluster)
|
|
with:
|
|
repository: mulgadc/mulga
|
|
token: ${{ steps.app-token.outputs.token }}
|
|
sparse-checkout: scripts/tofu-cluster
|
|
path: mulga
|
|
|
|
- name: Install gotestsum
|
|
env:
|
|
GOTESTSUM_VERSION: "1.12.0"
|
|
run: |
|
|
# Prebuilt tarball, not `go install`: gotestsum v1.12.0 pulls
|
|
# x/tools v0.19.0 whose tokeninternal static-assert pegs a
|
|
# narrow Go window (~1.22). The self-hosted runner's Go falls
|
|
# outside that window, so source-install fails with
|
|
# `invalid array length -delta * delta`. A prebuilt binary
|
|
# sidesteps toolchain coupling entirely.
|
|
BIN_DIR="${RUNNER_TEMP}/gotestsum-bin"
|
|
mkdir -p "${BIN_DIR}"
|
|
if [ ! -x "${BIN_DIR}/gotestsum" ]; then
|
|
curl -fsSL \
|
|
"https://github.com/gotestyourself/gotestsum/releases/download/v${GOTESTSUM_VERSION}/gotestsum_${GOTESTSUM_VERSION}_linux_amd64.tar.gz" \
|
|
| tar -xz -C "${BIN_DIR}" gotestsum
|
|
fi
|
|
echo "${BIN_DIR}" >> "${GITHUB_PATH}"
|
|
"${BIN_DIR}/gotestsum" --version
|
|
|
|
# Adds ci.job_id alongside ci.cell, before Bootstrap bakes the attrs into
|
|
# the VMs, so a telemetry doc points back at this job's log directly.
|
|
- name: Resolve CI job id
|
|
uses: ./spinifex/.github/actions/otel-job-id
|
|
with:
|
|
job-name: DDIL Real Multi-Node
|
|
|
|
- name: Build Install Artifacts
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
./build-install-artifacts.sh "${GIT_BRANCH}" \
|
|
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
|
|
"${RUNNER_TEMP}/setup.sh" \
|
|
"${RUNNER_TEMP}/spinifex-tests.tar"
|
|
|
|
- name: Provision VMs
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
set -euo pipefail
|
|
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
|
|
export TF_VAR_git_branch="${GIT_BRANCH}"
|
|
quiet_run "tofu init" "$TOFU_LOG" tofu init -input=false
|
|
quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
|
|
quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
|
|
quiet_run "scrub-env (pre-clean)" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
|
|
quiet_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
|
|
ok "VMs provisioned"
|
|
|
|
- name: Bootstrap via Binary Install
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
|
|
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
|
|
"${RUNNER_TEMP}/setup.sh" \
|
|
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
|
|
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
|
|
|
|
- name: Run DDIL Scenarios
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
set -o pipefail
|
|
INVENTORY=$(tofu output -json inventory)
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
|
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
SSH_USER=$(echo "$INVENTORY" | jq -r '.ssh_user')
|
|
REGION=$(echo "$INVENTORY" | jq -r '.region')
|
|
# DDIL_NODES must be the cluster-traffic IPs: PartitionNode
|
|
# installs iptables DROP rules for each peer, so the runner's
|
|
# SSH source IP must differ from these. On single-NIC
|
|
# tofu-cluster, wan_ip == cluster IP and the runner's outbound
|
|
# IP is distinct — see e2e-ddil-test-harness.md §3 Scenario C.
|
|
NODE_IPS=$(echo "$INVENTORY" | jq -r '[.nodes[].wan_ip] | join(",")')
|
|
PRIMARY_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
|
echo "DDIL_NODES=${NODE_IPS}"
|
|
|
|
# Witness VMs talk to the cluster's AWS gateway via aws-sdk-go,
|
|
# which resolves credentials from the standard chain. spx admin
|
|
# init writes [spinifex] to ${SSH_USER}'s ~/.aws/credentials on
|
|
# node1 — pull them over SSH and export as the SDK's env-var
|
|
# entries so the harness can launch instances without depending
|
|
# on a cluster-bootstrapped profile being present on the runner.
|
|
CREDS=$(ssh -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
|
"${SSH_USER}@${PRIMARY_IP}" 'cat ~/.aws/credentials')
|
|
AWS_ACCESS_KEY_ID=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
|
|
AWS_SECRET_ACCESS_KEY=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
|
|
if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ]; then
|
|
echo "::error::Failed to extract spinifex AWS credentials from ${PRIMARY_IP}"
|
|
exit 1
|
|
fi
|
|
|
|
GUEST_KEY="${RUNNER_TEMP}/spinifex-key"
|
|
scp -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
|
"${SSH_USER}@${PRIMARY_IP}:~/.ssh/spinifex-key" "${GUEST_KEY}"
|
|
chmod 600 "${GUEST_KEY}"
|
|
|
|
mkdir -p "${RUNNER_TEMP}/ddil-results"
|
|
cd "${GITHUB_WORKSPACE}/spinifex"
|
|
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
|
|
banner "DDIL scenarios"
|
|
DDIL_NODES="${NODE_IPS}" \
|
|
DDIL_SSH_USER="${SSH_USER}" \
|
|
DDIL_SSH_KEY="${SSH_KEY}" \
|
|
DDIL_GUEST_SSH_KEY="${GUEST_KEY}" \
|
|
DDIL_QUARANTINED="${{ inputs.quarantined }}" \
|
|
AWS_REGION="${REGION}" \
|
|
AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \
|
|
AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \
|
|
gotestsum --jsonfile "${RUNNER_TEMP}/ddil-results/results.json" \
|
|
--junitfile "${RUNNER_TEMP}/ddil-results/junit-ddil.xml" \
|
|
--format testname \
|
|
-- -tags=e2e -timeout 45m ./tests/e2e/ddil/scenarios/...
|
|
rc=$?
|
|
if [[ $rc -eq 0 ]]; then ok "all scenarios passed"; else bad "scenarios failed (rc=$rc)"; fi
|
|
exit $rc
|
|
|
|
- name: Publish test summary
|
|
if: always()
|
|
uses: ./spinifex/.github/actions/junit-report
|
|
with:
|
|
junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
|
|
title: "DDIL scenario results"
|
|
|
|
# Tolerated on failure: the sink being unreachable must not turn a
|
|
# green test run red.
|
|
- name: Ship test results to Elasticsearch
|
|
if: always()
|
|
continue-on-error: true
|
|
uses: ./spinifex/.github/actions/junit-to-es
|
|
with:
|
|
junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
|
|
cell: ddil
|
|
|
|
- name: Collect Cluster Logs
|
|
if: failure()
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
set +e
|
|
INVENTORY=$(tofu output -json inventory)
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
|
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
SSH_USER=$(echo "$INVENTORY" | jq -r '.ssh_user')
|
|
LOG_DIR="${RUNNER_TEMP}/ddil-results/cluster-logs"
|
|
mkdir -p "${LOG_DIR}"
|
|
|
|
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
|
|
mapfile -t NODE_IPS < <(echo "$INVENTORY" | jq -r '.nodes[].wan_ip')
|
|
for ip in "${NODE_IPS[@]}"; do
|
|
SSH="ssh -i ${SSH_KEY} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${SSH_USER}@${ip}"
|
|
${SSH} 'sudo journalctl -u "spinifex-*" --no-pager -n 5000' > "${LOG_DIR}/${ip}-spinifex.log" 2>&1
|
|
${SSH} 'ps auxw | grep -E "qemu-system|spinifex" | grep -v grep' > "${LOG_DIR}/${ip}-procs.txt" 2>&1
|
|
${SSH} 'ls -la /var/lib/spinifex 2>/dev/null; ls -la /var/log/spinifex 2>/dev/null' > "${LOG_DIR}/${ip}-state.txt" 2>&1
|
|
ok "$ip: $(wc -l < "${LOG_DIR}/${ip}-spinifex.log") journal lines, $(wc -l < "${LOG_DIR}/${ip}-procs.txt") procs"
|
|
done
|
|
|
|
- name: Upload DDIL Results
|
|
if: always()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: ddil-results-${{ github.run_id }}
|
|
path: ${{ runner.temp }}/ddil-results
|
|
retention-days: 30
|
|
|
|
- name: Teardown
|
|
if: always()
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
set -euo pipefail
|
|
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
|
|
quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
|
|
quiet_run "telemetry flush" "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
|
|
quiet_run "tofu destroy" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
|
|
quiet_run "scrub-env" "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
|
|
ok "teardown complete"
|
|
|
|
- name: Upload tofu log
|
|
if: always()
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: tofu-log-ddil-${{ github.run_id }}
|
|
path: ${{ env.TOFU_LOG }}
|
|
if-no-files-found: ignore
|
|
retention-days: 30
|