mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 10:52:04 +00:00
Julian Sommer
288 lines
15 KiB
YAML
288 lines
15 KiB
YAML
name: E2E Cloud Vendor Nightly
|
|
|
|
# Dispatched by the same Cloudflare Worker that drives e2e-nightly.yml, once per
|
|
# ref, rather than by a `schedule:` here. GitHub registers a schedule only from
|
|
# the default branch, so a cron in this file could never run `dev` and a feature
|
|
# branch as separate nightlies, and it would always execute the default branch's
|
|
# copy of this file -- leaving a fix to the workflow itself untestable until it
|
|
# had already shipped. A dispatch takes both the file and the source from its ref.
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
topologies:
|
|
description: "Space-separated topologies (blank = vm-single vm-multi bm vm-single-principal). One name runs one topology."
|
|
required: false
|
|
workbooks:
|
|
description: "Space-separated workbooks (blank = the driver's default list)"
|
|
required: false
|
|
source:
|
|
description: "tree (build this ref) or release (the published installer)"
|
|
required: false
|
|
default: tree
|
|
channel:
|
|
description: "With source=release: latest, or dev for the newest prerelease"
|
|
required: false
|
|
default: latest
|
|
keep_on_fail:
|
|
description: "Leave a failed topology up to inspect. It keeps billing."
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
auth:
|
|
description: "How nodes reach the OCI API: instance_principal (needs the tenancy dynamic group) or api_key (needs the credential hook). The vm-single-principal topology always uses instance_principal."
|
|
required: false
|
|
default: instance_principal
|
|
type: choice
|
|
options: [instance_principal, api_key]
|
|
no_external_pool:
|
|
description: "Form without the OCI allocator. Skips every workbook needing a public address. Not applied to vm-single-principal."
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
runner:
|
|
description: "Runner label (blank = the OCI_E2E_RUNNER variable, else ubuntu-26.04)"
|
|
required: false
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
# Every topology is built in one OCI compartment under one set of resource names,
|
|
# so two runs at once would each see the other's instances and the sweep in
|
|
# e2e-oci.sh could destroy a cluster the other run was still testing. Queue the
|
|
# second rather than cancelling it: a dispatch is usually a verification run.
|
|
concurrency:
|
|
group: e2e-oci
|
|
cancel-in-progress: false
|
|
|
|
jobs:
|
|
oci:
|
|
# Hosted by default: this job only drives Terraform and SSH, and the work
|
|
# happens on OCI, so nothing here needs a hypervisor. Overridable because the
|
|
# argument for a per-provider self-hosted runner is control, not capacity --
|
|
# a dedicated label keeps a two-hour cloud wait off the hypervisor labels and
|
|
# lifts the six-hour cap a hosted job carries.
|
|
runs-on: ${{ inputs.runner || vars.OCI_E2E_RUNNER || 'ubuntu-26.04' }}
|
|
# Three topologies including a bare-metal build, each with its own install,
|
|
# formation and workbook pass. The sweep still runs if this fires.
|
|
timeout-minutes: 300
|
|
steps:
|
|
# Set here rather than in a job-level env block: the runner context is not
|
|
# available at job level, only from a step onwards, and a workflow that
|
|
# references it there is rejected as invalid before any job starts.
|
|
#
|
|
# Outside github.workspace on purpose. The checkout wipes untracked files, so
|
|
# on a persistent runner it would delete the Terraform state of a killed run
|
|
# before the next sweep could read it, leaving instances nothing can find.
|
|
- name: Put the Terraform state outside the checkout
|
|
run: echo "OCI_STATE_ROOT=$RUNNER_WORKSPACE/oci-state" >> "$GITHUB_ENV"
|
|
|
|
- uses: actions/checkout@v7.0.1
|
|
with:
|
|
path: spinifex
|
|
|
|
- uses: actions/create-github-app-token@v3.3.0
|
|
id: app-token
|
|
with:
|
|
client-id: ${{ secrets.APP_ID }}
|
|
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
|
# mulga alone. It is the only private repository of the six, and the
|
|
# five that build-install-artifacts.sh clones are public, so the builder
|
|
# needs no credential and this token is not widened to reach them.
|
|
repositories: mulga
|
|
|
|
# build-install-artifacts.sh lives in mulga. The workbooks and their driver
|
|
# are in spinifex, so only the one directory is needed here.
|
|
- uses: actions/checkout@v7.0.1
|
|
with:
|
|
repository: mulgadc/mulga
|
|
path: mulga
|
|
token: ${{ steps.app-token.outputs.token }}
|
|
sparse-checkout: scripts/tofu-cluster
|
|
|
|
- uses: hashicorp/setup-terraform@v4.0.1
|
|
with:
|
|
terraform_wrapper: false
|
|
|
|
# Written to the runner's own filesystem and nowhere else. The OCI API
|
|
# credential is not here at all: it reaches Terraform as OCI_PRIVATE_KEY in
|
|
# the environment, so no key file is ever created.
|
|
- name: Install the throwaway SSH key
|
|
env:
|
|
OCI_E2E_SSH_PRIVATE_KEY: ${{ secrets.OCI_E2E_SSH_PRIVATE_KEY }}
|
|
OCI_E2E_SSH_PUBLIC_KEY: ${{ secrets.OCI_E2E_SSH_PUBLIC_KEY }}
|
|
run: |
|
|
set -euo pipefail
|
|
mkdir -p ~/.ssh
|
|
printf '%s\n' "$OCI_E2E_SSH_PRIVATE_KEY" > ~/.ssh/oci-spx
|
|
printf '%s\n' "$OCI_E2E_SSH_PUBLIC_KEY" > ~/.ssh/oci-spx.pub
|
|
chmod 600 ~/.ssh/oci-spx
|
|
ssh-keygen -y -f ~/.ssh/oci-spx > /dev/null
|
|
|
|
- name: Run the OCI topologies
|
|
working-directory: spinifex/scripts/terraform/oci-spx
|
|
env:
|
|
OCI_TENANCY_OCID: ${{ secrets.OCI_TENANCY_OCID }}
|
|
OCI_USER_OCID: ${{ secrets.OCI_USER_OCID }}
|
|
OCI_FINGERPRINT: ${{ secrets.OCI_FINGERPRINT }}
|
|
OCI_PRIVATE_KEY: ${{ secrets.OCI_PRIVATE_KEY }}
|
|
OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
|
|
TF_VAR_compartment_ocid: ${{ secrets.OCI_COMPARTMENT_OCID }}
|
|
MULGA_ROOT: ${{ github.workspace }}/mulga
|
|
SPX_REF: ${{ github.ref_name }}
|
|
OCI_TOPOLOGIES: ${{ inputs.topologies }}
|
|
OCI_SOURCE: ${{ inputs.source }}
|
|
OCI_CHANNEL: ${{ inputs.channel }}
|
|
# Independent of the pool, deliberately. These were one switch, which made
|
|
# the combination we actually need -- an API key with the allocator running
|
|
# -- impossible to ask for, so public-IP allocation went untested.
|
|
OCI_INSTANCE_PRINCIPAL: ${{ inputs.auth == 'api_key' && '0' || '1' }}
|
|
# Skipping the pool records the allocator and every public-address workbook
|
|
# as SKIPPED, so a green run with this set is not a claim about guest
|
|
# networking. It exists for a formation-only check, not as a way past a
|
|
# missing credential.
|
|
OCI_NO_EXTERNAL_POOL: ${{ inputs.no_external_pool && '1' || '0' }}
|
|
# Only an API-key run needs one: an instance principal is the node's own
|
|
# certificate, so there is nothing to hand over. Unset is not an error, and
|
|
# the allocator gate then fails rather than passing a node that cannot
|
|
# authenticate.
|
|
OCI_CREDENTIAL_HOOK: ${{ inputs.auth == 'api_key' && vars.OCI_CREDENTIAL_HOOK || '' }}
|
|
# vm-single-principal builds in its own tenancy. The dynamic group it needs
|
|
# is a tenancy-root resource, and the shared test tenancy grants us only
|
|
# compartment rights, so that topology can never pass there however correct
|
|
# the code is. Unset skips it rather than spending a provision to fail.
|
|
OCI_PRINCIPAL_TENANCY_OCID: ${{ secrets.OCI_PRINCIPAL_TENANCY_OCID }}
|
|
OCI_PRINCIPAL_USER_OCID: ${{ secrets.OCI_PRINCIPAL_USER_OCID }}
|
|
OCI_PRINCIPAL_FINGERPRINT: ${{ secrets.OCI_PRINCIPAL_FINGERPRINT }}
|
|
OCI_PRINCIPAL_PRIVATE_KEY: ${{ secrets.OCI_PRINCIPAL_PRIVATE_KEY }}
|
|
OCI_PRINCIPAL_COMPARTMENT_OCID: ${{ secrets.OCI_PRINCIPAL_COMPARTMENT_OCID }}
|
|
OCI_PRINCIPAL_REGION: ${{ vars.OCI_PRINCIPAL_REGION }}
|
|
# Service limits are per tenancy, so the shape the other topologies use
|
|
# can have zero quota in this one. Ours has no E6 cores at all, and the
|
|
# apply then fails before anything about the principal is exercised.
|
|
OCI_PRINCIPAL_SHAPE: ${{ vars.OCI_PRINCIPAL_SHAPE }}
|
|
OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci
|
|
OCI_KEEP_ON_FAIL: ${{ inputs.keep_on_fail && '1' || '0' }}
|
|
# Through the environment, never interpolated into the script below: a
|
|
# dispatch input is attacker-controlled text as far as the shell is
|
|
# concerned, and a quote in it would otherwise end the string.
|
|
WORKBOOKS_INPUT: ${{ inputs.workbooks }}
|
|
run: |
|
|
set -uo pipefail
|
|
# WORKBOOKS is exported only when the dispatch named some: the harness
|
|
# distinguishes unset (the driver's default list) from empty (run none).
|
|
if [ -n "$WORKBOOKS_INPUT" ]; then export WORKBOOKS="$WORKBOOKS_INPUT"; fi
|
|
./e2e-oci.sh 2>&1 | tee "$RUNNER_TEMP/e2e-oci.log"
|
|
exit "${PIPESTATUS[0]}"
|
|
|
|
# Before the sweep, which destroys the state the tables are read from, and
|
|
# always(): a run whose verdict is a failure is exactly the one somebody
|
|
# wants a table for. The successful path writes the same tables itself, so
|
|
# this step is for the run that did not get there.
|
|
#
|
|
# The markdown lives in the script rather than here. A heredoc in a YAML
|
|
# string is the one place in this repository nothing can lint or run.
|
|
- name: Publish the result tables
|
|
if: always()
|
|
working-directory: spinifex/scripts/terraform/oci-spx
|
|
env:
|
|
OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
|
|
OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci
|
|
OCI_TOPOLOGIES: ${{ inputs.topologies }}
|
|
OCI_SOURCE: ${{ inputs.source }}
|
|
OCI_CHANNEL: ${{ inputs.channel }}
|
|
SPX_REF: ${{ github.ref_name }}
|
|
# The same two the run step passes. The header states what was and was not
|
|
# tested, so a summary rendered without them would claim more than the run.
|
|
OCI_INSTANCE_PRINCIPAL: ${{ inputs.auth == 'api_key' && '0' || '1' }}
|
|
OCI_NO_EXTERNAL_POOL: ${{ inputs.no_external_pool && '1' || '0' }}
|
|
run: ./e2e-oci.sh --summary-only
|
|
|
|
# The sweep below deletes the per-topology state directories, so anything
|
|
# still only in one of them at that point is unrecoverable -- and a failing
|
|
# workbook writes its assertion output to exactly there. Copying into the
|
|
# artifact directory first is what makes a failure diagnosable by whoever
|
|
# reads the run page, rather than only on the runner that produced it.
|
|
- name: Collect the per-topology logs
|
|
if: always()
|
|
run: |
|
|
set -uo pipefail
|
|
dest="$RUNNER_TEMP/e2e-oci"
|
|
mkdir -p "$dest"
|
|
# The topologies this run drove, taken from the per-topology log the
|
|
# harness writes rather than from the dispatch input: a run that names
|
|
# none gets the harness's own default, and reading that back here would
|
|
# mean a second copy of the list to disagree with the first. A state
|
|
# directory left by an earlier run is then not collected, so the artifact
|
|
# cannot show a topology this run never attempted.
|
|
topologies=""
|
|
for log in "$dest"/*.log; do
|
|
[ -f "$log" ] || continue
|
|
name="${log##*/}"; name="${name%.log}"
|
|
[ -d "$OCI_STATE_ROOT/.validate-$name" ] && topologies="$topologies $name"
|
|
done
|
|
echo "collecting:${topologies:- nothing}"
|
|
for topology in $topologies; do
|
|
for file in "$OCI_STATE_ROOT/.validate-$topology"/*.log \
|
|
"$OCI_STATE_ROOT/.validate-$topology"/*.txt \
|
|
"$OCI_STATE_ROOT/.validate-$topology"/*.tsv; do
|
|
[ -f "$file" ] || continue
|
|
# The hook handles a private key and the one thing known about an
|
|
# operator's executable is that it is not ours to publish.
|
|
case "${file##*/}" in credential-hook.log) continue ;; esac
|
|
cp -- "$file" "$dest/$topology-${file##*/}"
|
|
done
|
|
done
|
|
ls -l "$dest"
|
|
|
|
# always(), and before the artifact upload: an OCI bare-metal host left
|
|
# running by a cancelled or timed-out job is the most expensive way this
|
|
# workflow can fail. The sweep works from the Terraform state files, so it
|
|
# can only destroy what this run created.
|
|
- name: Sweep anything left behind
|
|
if: always() && !inputs.keep_on_fail
|
|
working-directory: spinifex/scripts/terraform/oci-spx
|
|
env:
|
|
OCI_TENANCY_OCID: ${{ secrets.OCI_TENANCY_OCID }}
|
|
OCI_USER_OCID: ${{ secrets.OCI_USER_OCID }}
|
|
OCI_FINGERPRINT: ${{ secrets.OCI_FINGERPRINT }}
|
|
OCI_PRIVATE_KEY: ${{ secrets.OCI_PRIVATE_KEY }}
|
|
OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
|
|
TF_VAR_compartment_ocid: ${{ secrets.OCI_COMPARTMENT_OCID }}
|
|
# The sweep destroys by state file, so it needs whichever tenancy each
|
|
# topology built in. Without these a leftover principal deployment is
|
|
# swept against a tenancy that never held it and reported clean.
|
|
OCI_PRINCIPAL_TENANCY_OCID: ${{ secrets.OCI_PRINCIPAL_TENANCY_OCID }}
|
|
OCI_PRINCIPAL_USER_OCID: ${{ secrets.OCI_PRINCIPAL_USER_OCID }}
|
|
OCI_PRINCIPAL_FINGERPRINT: ${{ secrets.OCI_PRINCIPAL_FINGERPRINT }}
|
|
OCI_PRINCIPAL_PRIVATE_KEY: ${{ secrets.OCI_PRINCIPAL_PRIVATE_KEY }}
|
|
OCI_PRINCIPAL_COMPARTMENT_OCID: ${{ secrets.OCI_PRINCIPAL_COMPARTMENT_OCID }}
|
|
OCI_PRINCIPAL_REGION: ${{ vars.OCI_PRINCIPAL_REGION }}
|
|
OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci-sweep
|
|
run: ./e2e-oci.sh --sweep-only
|
|
|
|
# Explicit, not left to the runner's own cleanup: a hosted runner is
|
|
# discarded after the job, but a self-hosted one keeps $HOME, so the key
|
|
# would outlive the run that needed it and sit there between nightlies.
|
|
- name: Remove the throwaway SSH key
|
|
if: always()
|
|
run: rm -f ~/.ssh/oci-spx ~/.ssh/oci-spx.pub
|
|
|
|
# Named extensions rather than a directory glob: the artifact directory also
|
|
# holds the distro tarball and the test archive, which are hundreds of
|
|
# megabytes and reproducible from the ref. Everything here is a file the
|
|
# collect step above chose, so terraform.tfstate cannot reach an artifact.
|
|
- name: Upload the run
|
|
if: always()
|
|
uses: actions/upload-artifact@v7.0.2
|
|
with:
|
|
name: e2e-oci-${{ github.run_id }}
|
|
path: |
|
|
${{ runner.temp }}/e2e-oci.log
|
|
${{ runner.temp }}/e2e-oci/*.log
|
|
${{ runner.temp }}/e2e-oci/verdict.txt
|
|
${{ runner.temp }}/e2e-oci/summary.md
|
|
${{ runner.temp }}/e2e-oci/*.txt
|
|
${{ runner.temp }}/e2e-oci/*.tsv
|
|
${{ runner.temp }}/e2e-oci-sweep/*.log
|
|
retention-days: 14
|