name: E2E Cloud Vendor Nightly # Dispatched by the same Cloudflare Worker that drives e2e-nightly.yml, once per # ref, rather than by a `schedule:` here. GitHub registers a schedule only from # the default branch, so a cron in this file could never run `dev` and a feature # branch as separate nightlies, and it would always execute the default branch's # copy of this file -- leaving a fix to the workflow itself untestable until it # had already shipped. A dispatch takes both the file and the source from its ref. on: workflow_dispatch: inputs: topologies: description: "Space-separated topologies (blank = vm-single vm-multi bm vm-single-principal). One name runs one topology." required: false workbooks: description: "Space-separated workbooks (blank = the driver's default list)" required: false source: description: "tree (build this ref) or release (the published installer)" required: false default: tree channel: description: "With source=release: latest, or dev for the newest prerelease" required: false default: latest keep_on_fail: description: "Leave a failed topology up to inspect. It keeps billing." required: false type: boolean default: false auth: description: "How nodes reach the OCI API: instance_principal (needs the tenancy dynamic group) or api_key (needs the credential hook). The vm-single-principal topology always uses instance_principal." required: false default: instance_principal type: choice options: [instance_principal, api_key] no_external_pool: description: "Form without the OCI allocator. Skips every workbook needing a public address. Not applied to vm-single-principal." required: false type: boolean default: false runner: description: "Runner label (blank = the OCI_E2E_RUNNER variable, else ubuntu-26.04)" required: false permissions: contents: read # Every topology is built in one OCI compartment under one set of resource names, # so two runs at once would each see the other's instances and the sweep in # e2e-oci.sh could destroy a cluster the other run was still testing. Queue the # second rather than cancelling it: a dispatch is usually a verification run. concurrency: group: e2e-oci cancel-in-progress: false jobs: oci: # Hosted by default: this job only drives Terraform and SSH, and the work # happens on OCI, so nothing here needs a hypervisor. Overridable because the # argument for a per-provider self-hosted runner is control, not capacity -- # a dedicated label keeps a two-hour cloud wait off the hypervisor labels and # lifts the six-hour cap a hosted job carries. runs-on: ${{ inputs.runner || vars.OCI_E2E_RUNNER || 'ubuntu-26.04' }} # Three topologies including a bare-metal build, each with its own install, # formation and workbook pass. The sweep still runs if this fires. timeout-minutes: 300 steps: # Set here rather than in a job-level env block: the runner context is not # available at job level, only from a step onwards, and a workflow that # references it there is rejected as invalid before any job starts. # # Outside github.workspace on purpose. The checkout wipes untracked files, so # on a persistent runner it would delete the Terraform state of a killed run # before the next sweep could read it, leaving instances nothing can find. - name: Put the Terraform state outside the checkout run: echo "OCI_STATE_ROOT=$RUNNER_WORKSPACE/oci-state" >> "$GITHUB_ENV" - uses: actions/checkout@v7.0.1 with: path: spinifex - uses: actions/create-github-app-token@v3.3.0 id: app-token with: client-id: ${{ secrets.APP_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} # mulga alone. It is the only private repository of the six, and the # five that build-install-artifacts.sh clones are public, so the builder # needs no credential and this token is not widened to reach them. repositories: mulga # build-install-artifacts.sh lives in mulga. The workbooks and their driver # are in spinifex, so only the one directory is needed here. - uses: actions/checkout@v7.0.1 with: repository: mulgadc/mulga path: mulga token: ${{ steps.app-token.outputs.token }} sparse-checkout: scripts/tofu-cluster - uses: hashicorp/setup-terraform@v4.0.1 with: terraform_wrapper: false # Written to the runner's own filesystem and nowhere else. The OCI API # credential is not here at all: it reaches Terraform as OCI_PRIVATE_KEY in # the environment, so no key file is ever created. - name: Install the throwaway SSH key env: OCI_E2E_SSH_PRIVATE_KEY: ${{ secrets.OCI_E2E_SSH_PRIVATE_KEY }} OCI_E2E_SSH_PUBLIC_KEY: ${{ secrets.OCI_E2E_SSH_PUBLIC_KEY }} run: | set -euo pipefail mkdir -p ~/.ssh printf '%s\n' "$OCI_E2E_SSH_PRIVATE_KEY" > ~/.ssh/oci-spx printf '%s\n' "$OCI_E2E_SSH_PUBLIC_KEY" > ~/.ssh/oci-spx.pub chmod 600 ~/.ssh/oci-spx ssh-keygen -y -f ~/.ssh/oci-spx > /dev/null - name: Run the OCI topologies working-directory: spinifex/scripts/terraform/oci-spx env: OCI_TENANCY_OCID: ${{ secrets.OCI_TENANCY_OCID }} OCI_USER_OCID: ${{ secrets.OCI_USER_OCID }} OCI_FINGERPRINT: ${{ secrets.OCI_FINGERPRINT }} OCI_PRIVATE_KEY: ${{ secrets.OCI_PRIVATE_KEY }} OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }} TF_VAR_compartment_ocid: ${{ secrets.OCI_COMPARTMENT_OCID }} MULGA_ROOT: ${{ github.workspace }}/mulga SPX_REF: ${{ github.ref_name }} OCI_TOPOLOGIES: ${{ inputs.topologies }} OCI_SOURCE: ${{ inputs.source }} OCI_CHANNEL: ${{ inputs.channel }} # Independent of the pool, deliberately. These were one switch, which made # the combination we actually need -- an API key with the allocator running # -- impossible to ask for, so public-IP allocation went untested. OCI_INSTANCE_PRINCIPAL: ${{ inputs.auth == 'api_key' && '0' || '1' }} # Skipping the pool records the allocator and every public-address workbook # as SKIPPED, so a green run with this set is not a claim about guest # networking. It exists for a formation-only check, not as a way past a # missing credential. OCI_NO_EXTERNAL_POOL: ${{ inputs.no_external_pool && '1' || '0' }} # Only an API-key run needs one: an instance principal is the node's own # certificate, so there is nothing to hand over. Unset is not an error, and # the allocator gate then fails rather than passing a node that cannot # authenticate. OCI_CREDENTIAL_HOOK: ${{ inputs.auth == 'api_key' && vars.OCI_CREDENTIAL_HOOK || '' }} # vm-single-principal builds in its own tenancy. The dynamic group it needs # is a tenancy-root resource, and the shared test tenancy grants us only # compartment rights, so that topology can never pass there however correct # the code is. Unset skips it rather than spending a provision to fail. OCI_PRINCIPAL_TENANCY_OCID: ${{ secrets.OCI_PRINCIPAL_TENANCY_OCID }} OCI_PRINCIPAL_USER_OCID: ${{ secrets.OCI_PRINCIPAL_USER_OCID }} OCI_PRINCIPAL_FINGERPRINT: ${{ secrets.OCI_PRINCIPAL_FINGERPRINT }} OCI_PRINCIPAL_PRIVATE_KEY: ${{ secrets.OCI_PRINCIPAL_PRIVATE_KEY }} OCI_PRINCIPAL_COMPARTMENT_OCID: ${{ secrets.OCI_PRINCIPAL_COMPARTMENT_OCID }} OCI_PRINCIPAL_REGION: ${{ vars.OCI_PRINCIPAL_REGION }} # Service limits are per tenancy, so the shape the other topologies use # can have zero quota in this one. Ours has no E6 cores at all, and the # apply then fails before anything about the principal is exercised. OCI_PRINCIPAL_SHAPE: ${{ vars.OCI_PRINCIPAL_SHAPE }} OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci OCI_KEEP_ON_FAIL: ${{ inputs.keep_on_fail && '1' || '0' }} # Through the environment, never interpolated into the script below: a # dispatch input is attacker-controlled text as far as the shell is # concerned, and a quote in it would otherwise end the string. WORKBOOKS_INPUT: ${{ inputs.workbooks }} run: | set -uo pipefail # WORKBOOKS is exported only when the dispatch named some: the harness # distinguishes unset (the driver's default list) from empty (run none). if [ -n "$WORKBOOKS_INPUT" ]; then export WORKBOOKS="$WORKBOOKS_INPUT"; fi ./e2e-oci.sh 2>&1 | tee "$RUNNER_TEMP/e2e-oci.log" exit "${PIPESTATUS[0]}" # Before the sweep, which destroys the state the tables are read from, and # always(): a run whose verdict is a failure is exactly the one somebody # wants a table for. The successful path writes the same tables itself, so # this step is for the run that did not get there. # # The markdown lives in the script rather than here. A heredoc in a YAML # string is the one place in this repository nothing can lint or run. - name: Publish the result tables if: always() working-directory: spinifex/scripts/terraform/oci-spx env: OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }} OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci OCI_TOPOLOGIES: ${{ inputs.topologies }} OCI_SOURCE: ${{ inputs.source }} OCI_CHANNEL: ${{ inputs.channel }} SPX_REF: ${{ github.ref_name }} # The same two the run step passes. The header states what was and was not # tested, so a summary rendered without them would claim more than the run. OCI_INSTANCE_PRINCIPAL: ${{ inputs.auth == 'api_key' && '0' || '1' }} OCI_NO_EXTERNAL_POOL: ${{ inputs.no_external_pool && '1' || '0' }} run: ./e2e-oci.sh --summary-only # The sweep below deletes the per-topology state directories, so anything # still only in one of them at that point is unrecoverable -- and a failing # workbook writes its assertion output to exactly there. Copying into the # artifact directory first is what makes a failure diagnosable by whoever # reads the run page, rather than only on the runner that produced it. - name: Collect the per-topology logs if: always() run: | set -uo pipefail dest="$RUNNER_TEMP/e2e-oci" mkdir -p "$dest" # The topologies this run drove, taken from the per-topology log the # harness writes rather than from the dispatch input: a run that names # none gets the harness's own default, and reading that back here would # mean a second copy of the list to disagree with the first. A state # directory left by an earlier run is then not collected, so the artifact # cannot show a topology this run never attempted. topologies="" for log in "$dest"/*.log; do [ -f "$log" ] || continue name="${log##*/}"; name="${name%.log}" [ -d "$OCI_STATE_ROOT/.validate-$name" ] && topologies="$topologies $name" done echo "collecting:${topologies:- nothing}" for topology in $topologies; do for file in "$OCI_STATE_ROOT/.validate-$topology"/*.log \ "$OCI_STATE_ROOT/.validate-$topology"/*.txt \ "$OCI_STATE_ROOT/.validate-$topology"/*.tsv; do [ -f "$file" ] || continue # The hook handles a private key and the one thing known about an # operator's executable is that it is not ours to publish. case "${file##*/}" in credential-hook.log) continue ;; esac cp -- "$file" "$dest/$topology-${file##*/}" done done ls -l "$dest" # always(), and before the artifact upload: an OCI bare-metal host left # running by a cancelled or timed-out job is the most expensive way this # workflow can fail. The sweep works from the Terraform state files, so it # can only destroy what this run created. - name: Sweep anything left behind if: always() && !inputs.keep_on_fail working-directory: spinifex/scripts/terraform/oci-spx env: OCI_TENANCY_OCID: ${{ secrets.OCI_TENANCY_OCID }} OCI_USER_OCID: ${{ secrets.OCI_USER_OCID }} OCI_FINGERPRINT: ${{ secrets.OCI_FINGERPRINT }} OCI_PRIVATE_KEY: ${{ secrets.OCI_PRIVATE_KEY }} OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }} TF_VAR_compartment_ocid: ${{ secrets.OCI_COMPARTMENT_OCID }} # The sweep destroys by state file, so it needs whichever tenancy each # topology built in. Without these a leftover principal deployment is # swept against a tenancy that never held it and reported clean. OCI_PRINCIPAL_TENANCY_OCID: ${{ secrets.OCI_PRINCIPAL_TENANCY_OCID }} OCI_PRINCIPAL_USER_OCID: ${{ secrets.OCI_PRINCIPAL_USER_OCID }} OCI_PRINCIPAL_FINGERPRINT: ${{ secrets.OCI_PRINCIPAL_FINGERPRINT }} OCI_PRINCIPAL_PRIVATE_KEY: ${{ secrets.OCI_PRINCIPAL_PRIVATE_KEY }} OCI_PRINCIPAL_COMPARTMENT_OCID: ${{ secrets.OCI_PRINCIPAL_COMPARTMENT_OCID }} OCI_PRINCIPAL_REGION: ${{ vars.OCI_PRINCIPAL_REGION }} OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci-sweep run: ./e2e-oci.sh --sweep-only # Explicit, not left to the runner's own cleanup: a hosted runner is # discarded after the job, but a self-hosted one keeps $HOME, so the key # would outlive the run that needed it and sit there between nightlies. - name: Remove the throwaway SSH key if: always() run: rm -f ~/.ssh/oci-spx ~/.ssh/oci-spx.pub # Named extensions rather than a directory glob: the artifact directory also # holds the distro tarball and the test archive, which are hundreds of # megabytes and reproducible from the ref. Everything here is a file the # collect step above chose, so terraform.tfstate cannot reach an artifact. - name: Upload the run if: always() uses: actions/upload-artifact@v7.0.2 with: name: e2e-oci-${{ github.run_id }} path: | ${{ runner.temp }}/e2e-oci.log ${{ runner.temp }}/e2e-oci/*.log ${{ runner.temp }}/e2e-oci/verdict.txt ${{ runner.temp }}/e2e-oci/summary.md ${{ runner.temp }}/e2e-oci/*.txt ${{ runner.temp }}/e2e-oci/*.tsv ${{ runner.temp }}/e2e-oci-sweep/*.log retention-days: 14