gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-15 17:11:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge origin/dev into refactor/spinifex-package-boundaries

Browse Source
This commit is contained in:
joshsiv-mulga
2026-10-08 09:28:43 +10:00
parent 8b6840adb0 5529572761
commit 9e77181f21
916 changed files with 17495 additions and 4125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/dev-release.yml
+188
View File
@@ -0,0 +1,188 @@
name: Dev Prerelease
# Publishes the dev channel that install.mulgadc.com already serves. setup.sh has
# understood INSTALL_SPINIFEX_CHANNEL=dev since it was written and the install
# Worker resolves that channel to the newest GitHub prerelease, but nothing has
# ever produced one -- so /download/dev/amd64 returned 404. This is the producer.
#
# A separate workflow rather than conditionals in release.yml, because a dev build
# is a different artifact and not a release with flags off: amd64 only, no ISO, no
# AMI refresh, no bare-metal gate, and nothing that touches a self-hosted runner.
on:
push:
branches: [dev]
# Nothing here changes the binary, and a prerelease per docs commit would
# churn the channel our contacts are installing from.
paths-ignore:
- "docs/**"
- "**/*.md"
workflow_dispatch:
permissions:
contents: write
# Supersede rather than queue: on a run of merges to dev only the newest commit is
# worth publishing, and the older build would land after it and become the channel.
concurrency:
group: dev-release
cancel-in-progress: true
jobs:
prerelease:
runs-on: ubuntu-26.04
steps:
# Full history and tags, because the tag name is git describe and a shallow
# clone has no tag to describe against.
- name: Checkout Spinifex
uses: actions/checkout@v7
with:
path: spinifex
fetch-depth: 0
# dev, not each repository's default branch, which is what release.yml takes.
# A dev build of spinifex against main of the others is a pairing nobody runs.
- name: Checkout Predastore
uses: actions/checkout@v7
with:
repository: mulgadc/predastore
ref: dev
path: predastore
- name: Checkout Viperblock
uses: actions/checkout@v7
with:
repository: mulgadc/viperblock
ref: dev
path: viperblock
- name: Checkout Northstar
uses: actions/checkout@v7
with:
repository: mulgadc/northstar
ref: dev
path: northstar
- name: Checkout Bluebottle
uses: actions/checkout@v7
with:
repository: mulgadc/bluebottle
ref: dev
path: bluebottle
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: spinifex/go.mod
# The tag carries the describe string so the commit is recoverable from the
# version alone, and the -dev suffix is what release.yml's own convention
# already treats as a prerelease. VERSION must equal the tag: the install
# Worker derives the asset name from the tag it resolves.
- name: Resolve the version
id: version
working-directory: spinifex
run: |
set -euo pipefail
# Describing against a dev tag would nest it, growing the name every release.
TAG="$(git describe --tags --always --exclude '*-dev')-dev"
case "$TAG" in
v*) ;;
*) echo "::error::git describe gave '$TAG', which is not a v-prefixed version"; exit 1 ;;
esac
if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
echo "::notice::$TAG already published; nothing to do"
echo "exists=true" >> "$GITHUB_OUTPUT"
fi
echo "tag=$TAG" >> "$GITHUB_OUTPUT"
echo "Publishing $TAG"
env:
GH_TOKEN: ${{ github.token }}
- name: Build microVM artifacts (lb-agent + kernel + initramfs)
if: steps.version.outputs.exists != 'true'
run: cd spinifex && make build-microvm-image
- name: Build amd64 Tarball
if: steps.version.outputs.exists != 'true'
env:
VERSION: ${{ steps.version.outputs.tag }}
run: cd spinifex && make distro-amd64 VERSION="$VERSION"
# --prerelease on create, never a later promote step. release.yml sets it in
# a job that runs after its verification gates, which leaves a window where
# the release is not flagged; GitHub's releases/latest excludes prereleases
# only, so in that window a dev build is what every customer curl installs.
- name: Publish the prerelease
if: steps.version.outputs.exists != 'true'
working-directory: spinifex
env:
GH_TOKEN: ${{ github.token }}
TAG: ${{ steps.version.outputs.tag }}
run: |
set -euo pipefail
# A file, not an inline string: the notes are Markdown, and leading
# whitespace from indenting a shell literal turns them into code blocks.
cat > "$RUNNER_TEMP/notes.md" <<EOF
Automated dev-channel build of \`$GITHUB_SHA\`.
**Not a release.** amd64 only, no ISO and no system images, and it has not been through the release verification gates. Install it with:
\`\`\`
curl -sfL https://install.mulgadc.com | sudo bash -s -- --channel dev
\`\`\`
EOF
sed -i 's/^ //' "$RUNNER_TEMP/notes.md"
gh release create "$TAG" \
--repo "$GITHUB_REPOSITORY" \
--target "$GITHUB_SHA" \
--prerelease \
--title "$TAG" \
--notes-file "$RUNNER_TEMP/notes.md" \
"dist/spinifex-$TAG-linux-amd64.tar.gz" \
"dist/spinifex-$TAG-linux-amd64.tar.gz.sha256" \
"scripts/setup.sh"
# The assertion that matters. A prerelease must never become the latest
# release, because latest is what an unqualified curl | bash installs -- so
# this checks the published state rather than trusting the flag we passed.
- name: Assert the stable channel is untouched
if: steps.version.outputs.exists != 'true'
env:
GH_TOKEN: ${{ github.token }}
TAG: ${{ steps.version.outputs.tag }}
run: |
set -euo pipefail
latest="$(gh api "repos/$GITHUB_REPOSITORY/releases/latest" --jq .tag_name)"
if [ "$latest" = "$TAG" ]; then
echo "::error::$TAG became the latest release: every customer curl | bash would install it"
gh release edit "$TAG" --repo "$GITHUB_REPOSITORY" --prerelease
exit 1
fi
echo "latest is still $latest"
dev="$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=20" --jq 'map(select(.prerelease))[0].tag_name')"
[ "$dev" = "$TAG" ] || { echo "::error::the dev channel resolves to $dev, not $TAG"; exit 1; }
# Kept rather than unbounded: the Worker reads the first page of releases to
# resolve the channel, so enough prereleases would eventually push the real
# ones off it, and each build carries two assets.
#
# Twenty, not five. A pinned dev tag is how an external evaluator installs a
# specific build, and five is about a week of them -- short enough that the tag
# someone was given stops resolving while they are still using it.
- name: Prune old dev prereleases
if: steps.version.outputs.exists != 'true'
env:
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" \
--jq 'map(select(.prerelease and (.tag_name | endswith("-dev")))) | .[20:] | .[].tag_name' \
> stale.txt
while read -r tag; do
[ -n "$tag" ] || continue
echo "deleting $tag"
gh release delete "$tag" --repo "$GITHUB_REPOSITORY" --yes --cleanup-tag
done < stale.txt
.github/workflows/e2e-baremetal.yml
+1 -1
View File
@@ -123,7 +123,7 @@ jobs:
INSTALLER_BIN="${RUNNER_TEMP}/bm-bins/spinifex-installer" \
SPINIFEX_SRC="${GITHUB_WORKSPACE}" \
VIPERBLOCK_SRC="${RUNNER_TEMP}/bm-bins/viperblock" \
VERSION="$(cd "$GITHUB_WORKSPACE" && git describe --tags --always 2>/dev/null || echo dev)"
VERSION="$(cd "$GITHUB_WORKSPACE" && git describe --tags --always --exclude '*-dev' 2>/dev/null || echo dev)"
- name: Write ePDU credentials
run: |
.github/workflows/e2e-cloudvendor-nightly.yml
+287
View File
@@ -0,0 +1,287 @@
name: E2E Cloud Vendor Nightly
# Dispatched by the same Cloudflare Worker that drives e2e-nightly.yml, once per
# ref, rather than by a `schedule:` here. GitHub registers a schedule only from
# the default branch, so a cron in this file could never run `dev` and a feature
# branch as separate nightlies, and it would always execute the default branch's
# copy of this file -- leaving a fix to the workflow itself untestable until it
# had already shipped. A dispatch takes both the file and the source from its ref.
on:
workflow_dispatch:
inputs:
topologies:
description: "Space-separated topologies (blank = vm-single vm-multi bm vm-single-principal). One name runs one topology."
required: false
workbooks:
description: "Space-separated workbooks (blank = the driver's default list)"
required: false
source:
description: "tree (build this ref) or release (the published installer)"
required: false
default: tree
channel:
description: "With source=release: latest, or dev for the newest prerelease"
required: false
default: latest
keep_on_fail:
description: "Leave a failed topology up to inspect. It keeps billing."
required: false
type: boolean
default: false
auth:
description: "How nodes reach the OCI API: instance_principal (needs the tenancy dynamic group) or api_key (needs the credential hook). The vm-single-principal topology always uses instance_principal."
required: false
default: instance_principal
type: choice
options: [instance_principal, api_key]
no_external_pool:
description: "Form without the OCI allocator. Skips every workbook needing a public address. Not applied to vm-single-principal."
required: false
type: boolean
default: false
runner:
description: "Runner label (blank = the OCI_E2E_RUNNER variable, else ubuntu-26.04)"
required: false
permissions:
contents: read
# Every topology is built in one OCI compartment under one set of resource names,
# so two runs at once would each see the other's instances and the sweep in
# e2e-oci.sh could destroy a cluster the other run was still testing. Queue the
# second rather than cancelling it: a dispatch is usually a verification run.
concurrency:
group: e2e-oci
cancel-in-progress: false
jobs:
oci:
# Hosted by default: this job only drives Terraform and SSH, and the work
# happens on OCI, so nothing here needs a hypervisor. Overridable because the
# argument for a per-provider self-hosted runner is control, not capacity --
# a dedicated label keeps a two-hour cloud wait off the hypervisor labels and
# lifts the six-hour cap a hosted job carries.
runs-on: ${{ inputs.runner || vars.OCI_E2E_RUNNER || 'ubuntu-26.04' }}
# Three topologies including a bare-metal build, each with its own install,
# formation and workbook pass. The sweep still runs if this fires.
timeout-minutes: 300
steps:
# Set here rather than in a job-level env block: the runner context is not
# available at job level, only from a step onwards, and a workflow that
# references it there is rejected as invalid before any job starts.
#
# Outside github.workspace on purpose. The checkout wipes untracked files, so
# on a persistent runner it would delete the Terraform state of a killed run
# before the next sweep could read it, leaving instances nothing can find.
- name: Put the Terraform state outside the checkout
run: echo "OCI_STATE_ROOT=$RUNNER_WORKSPACE/oci-state" >> "$GITHUB_ENV"
- uses: actions/checkout@v7
with:
path: spinifex
- uses: actions/create-github-app-token@v3
id: app-token
with:
client-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
# mulga alone. It is the only private repository of the six, and the
# five that build-install-artifacts.sh clones are public, so the builder
# needs no credential and this token is not widened to reach them.
repositories: mulga
# build-install-artifacts.sh lives in mulga. The workbooks and their driver
# are in spinifex, so only the one directory is needed here.
- uses: actions/checkout@v7
with:
repository: mulgadc/mulga
path: mulga
token: ${{ steps.app-token.outputs.token }}
sparse-checkout: scripts/tofu-cluster
- uses: hashicorp/setup-terraform@v4
with:
terraform_wrapper: false
# Written to the runner's own filesystem and nowhere else. The OCI API
# credential is not here at all: it reaches Terraform as OCI_PRIVATE_KEY in
# the environment, so no key file is ever created.
- name: Install the throwaway SSH key
env:
OCI_E2E_SSH_PRIVATE_KEY: ${{ secrets.OCI_E2E_SSH_PRIVATE_KEY }}
OCI_E2E_SSH_PUBLIC_KEY: ${{ secrets.OCI_E2E_SSH_PUBLIC_KEY }}
run: |
set -euo pipefail
mkdir -p ~/.ssh
printf '%s\n' "$OCI_E2E_SSH_PRIVATE_KEY" > ~/.ssh/oci-spx
printf '%s\n' "$OCI_E2E_SSH_PUBLIC_KEY" > ~/.ssh/oci-spx.pub
chmod 600 ~/.ssh/oci-spx
ssh-keygen -y -f ~/.ssh/oci-spx > /dev/null
- name: Run the OCI topologies
working-directory: spinifex/scripts/terraform/oci-spx
env:
OCI_TENANCY_OCID: ${{ secrets.OCI_TENANCY_OCID }}
OCI_USER_OCID: ${{ secrets.OCI_USER_OCID }}
OCI_FINGERPRINT: ${{ secrets.OCI_FINGERPRINT }}
OCI_PRIVATE_KEY: ${{ secrets.OCI_PRIVATE_KEY }}
OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
TF_VAR_compartment_ocid: ${{ secrets.OCI_COMPARTMENT_OCID }}
MULGA_ROOT: ${{ github.workspace }}/mulga
SPX_REF: ${{ github.ref_name }}
OCI_TOPOLOGIES: ${{ inputs.topologies }}
OCI_SOURCE: ${{ inputs.source }}
OCI_CHANNEL: ${{ inputs.channel }}
# Independent of the pool, deliberately. These were one switch, which made
# the combination we actually need -- an API key with the allocator running
# -- impossible to ask for, so public-IP allocation went untested.
OCI_INSTANCE_PRINCIPAL: ${{ inputs.auth == 'api_key' && '0' || '1' }}
# Skipping the pool records the allocator and every public-address workbook
# as SKIPPED, so a green run with this set is not a claim about guest
# networking. It exists for a formation-only check, not as a way past a
# missing credential.
OCI_NO_EXTERNAL_POOL: ${{ inputs.no_external_pool && '1' || '0' }}
# Only an API-key run needs one: an instance principal is the node's own
# certificate, so there is nothing to hand over. Unset is not an error, and
# the allocator gate then fails rather than passing a node that cannot
# authenticate.
OCI_CREDENTIAL_HOOK: ${{ inputs.auth == 'api_key' && vars.OCI_CREDENTIAL_HOOK || '' }}
# vm-single-principal builds in its own tenancy. The dynamic group it needs
# is a tenancy-root resource, and the shared test tenancy grants us only
# compartment rights, so that topology can never pass there however correct
# the code is. Unset skips it rather than spending a provision to fail.
OCI_PRINCIPAL_TENANCY_OCID: ${{ secrets.OCI_PRINCIPAL_TENANCY_OCID }}
OCI_PRINCIPAL_USER_OCID: ${{ secrets.OCI_PRINCIPAL_USER_OCID }}
OCI_PRINCIPAL_FINGERPRINT: ${{ secrets.OCI_PRINCIPAL_FINGERPRINT }}
OCI_PRINCIPAL_PRIVATE_KEY: ${{ secrets.OCI_PRINCIPAL_PRIVATE_KEY }}
OCI_PRINCIPAL_COMPARTMENT_OCID: ${{ secrets.OCI_PRINCIPAL_COMPARTMENT_OCID }}
OCI_PRINCIPAL_REGION: ${{ vars.OCI_PRINCIPAL_REGION }}
# Service limits are per tenancy, so the shape the other topologies use
# can have zero quota in this one. Ours has no E6 cores at all, and the
# apply then fails before anything about the principal is exercised.
OCI_PRINCIPAL_SHAPE: ${{ vars.OCI_PRINCIPAL_SHAPE }}
OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci
OCI_KEEP_ON_FAIL: ${{ inputs.keep_on_fail && '1' || '0' }}
# Through the environment, never interpolated into the script below: a
# dispatch input is attacker-controlled text as far as the shell is
# concerned, and a quote in it would otherwise end the string.
WORKBOOKS_INPUT: ${{ inputs.workbooks }}
run: |
set -uo pipefail
# WORKBOOKS is exported only when the dispatch named some: the harness
# distinguishes unset (the driver's default list) from empty (run none).
if [ -n "$WORKBOOKS_INPUT" ]; then export WORKBOOKS="$WORKBOOKS_INPUT"; fi
./e2e-oci.sh 2>&1 | tee "$RUNNER_TEMP/e2e-oci.log"
exit "${PIPESTATUS[0]}"
# Before the sweep, which destroys the state the tables are read from, and
# always(): a run whose verdict is a failure is exactly the one somebody
# wants a table for. The successful path writes the same tables itself, so
# this step is for the run that did not get there.
#
# The markdown lives in the script rather than here. A heredoc in a YAML
# string is the one place in this repository nothing can lint or run.
- name: Publish the result tables
if: always()
working-directory: spinifex/scripts/terraform/oci-spx
env:
OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci
OCI_TOPOLOGIES: ${{ inputs.topologies }}
OCI_SOURCE: ${{ inputs.source }}
OCI_CHANNEL: ${{ inputs.channel }}
SPX_REF: ${{ github.ref_name }}
# The same two the run step passes. The header states what was and was not
# tested, so a summary rendered without them would claim more than the run.
OCI_INSTANCE_PRINCIPAL: ${{ inputs.auth == 'api_key' && '0' || '1' }}
OCI_NO_EXTERNAL_POOL: ${{ inputs.no_external_pool && '1' || '0' }}
run: ./e2e-oci.sh --summary-only
# The sweep below deletes the per-topology state directories, so anything
# still only in one of them at that point is unrecoverable -- and a failing
# workbook writes its assertion output to exactly there. Copying into the
# artifact directory first is what makes a failure diagnosable by whoever
# reads the run page, rather than only on the runner that produced it.
- name: Collect the per-topology logs
if: always()
run: |
set -uo pipefail
dest="$RUNNER_TEMP/e2e-oci"
mkdir -p "$dest"
# The topologies this run drove, taken from the per-topology log the
# harness writes rather than from the dispatch input: a run that names
# none gets the harness's own default, and reading that back here would
# mean a second copy of the list to disagree with the first. A state
# directory left by an earlier run is then not collected, so the artifact
# cannot show a topology this run never attempted.
topologies=""
for log in "$dest"/*.log; do
[ -f "$log" ] || continue
name="${log##*/}"; name="${name%.log}"
[ -d "$OCI_STATE_ROOT/.validate-$name" ] && topologies="$topologies $name"
done
echo "collecting:${topologies:- nothing}"
for topology in $topologies; do
for file in "$OCI_STATE_ROOT/.validate-$topology"/*.log \
"$OCI_STATE_ROOT/.validate-$topology"/*.txt \
"$OCI_STATE_ROOT/.validate-$topology"/*.tsv; do
[ -f "$file" ] || continue
# The hook handles a private key and the one thing known about an
# operator's executable is that it is not ours to publish.
case "${file##*/}" in credential-hook.log) continue ;; esac
cp -- "$file" "$dest/$topology-${file##*/}"
done
done
ls -l "$dest"
# always(), and before the artifact upload: an OCI bare-metal host left
# running by a cancelled or timed-out job is the most expensive way this
# workflow can fail. The sweep works from the Terraform state files, so it
# can only destroy what this run created.
- name: Sweep anything left behind
if: always() && !inputs.keep_on_fail
working-directory: spinifex/scripts/terraform/oci-spx
env:
OCI_TENANCY_OCID: ${{ secrets.OCI_TENANCY_OCID }}
OCI_USER_OCID: ${{ secrets.OCI_USER_OCID }}
OCI_FINGERPRINT: ${{ secrets.OCI_FINGERPRINT }}
OCI_PRIVATE_KEY: ${{ secrets.OCI_PRIVATE_KEY }}
OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }}
TF_VAR_compartment_ocid: ${{ secrets.OCI_COMPARTMENT_OCID }}
# The sweep destroys by state file, so it needs whichever tenancy each
# topology built in. Without these a leftover principal deployment is
# swept against a tenancy that never held it and reported clean.
OCI_PRINCIPAL_TENANCY_OCID: ${{ secrets.OCI_PRINCIPAL_TENANCY_OCID }}
OCI_PRINCIPAL_USER_OCID: ${{ secrets.OCI_PRINCIPAL_USER_OCID }}
OCI_PRINCIPAL_FINGERPRINT: ${{ secrets.OCI_PRINCIPAL_FINGERPRINT }}
OCI_PRINCIPAL_PRIVATE_KEY: ${{ secrets.OCI_PRINCIPAL_PRIVATE_KEY }}
OCI_PRINCIPAL_COMPARTMENT_OCID: ${{ secrets.OCI_PRINCIPAL_COMPARTMENT_OCID }}
OCI_PRINCIPAL_REGION: ${{ vars.OCI_PRINCIPAL_REGION }}
OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci-sweep
run: ./e2e-oci.sh --sweep-only
# Explicit, not left to the runner's own cleanup: a hosted runner is
# discarded after the job, but a self-hosted one keeps $HOME, so the key
# would outlive the run that needed it and sit there between nightlies.
- name: Remove the throwaway SSH key
if: always()
run: rm -f ~/.ssh/oci-spx ~/.ssh/oci-spx.pub
# Named extensions rather than a directory glob: the artifact directory also
# holds the distro tarball and the test archive, which are hundreds of
# megabytes and reproducible from the ref. Everything here is a file the
# collect step above chose, so terraform.tfstate cannot reach an artifact.
- name: Upload the run
if: always()
uses: actions/upload-artifact@v7
with:
name: e2e-oci-${{ github.run_id }}
path: |
${{ runner.temp }}/e2e-oci.log
${{ runner.temp }}/e2e-oci/*.log
${{ runner.temp }}/e2e-oci/verdict.txt
${{ runner.temp }}/e2e-oci/summary.md
${{ runner.temp }}/e2e-oci/*.txt
${{ runner.temp }}/e2e-oci/*.tsv
${{ runner.temp }}/e2e-oci-sweep/*.log
retention-days: 14
.github/workflows/release.yml
+1 -1
View File
@@ -46,7 +46,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v7
with:
go-version: "1.27.0"
go-version-file: spinifex/go.mod
- name: Build microVM artifacts (lb-agent + kernel + initramfs)
run: cd spinifex && make build-microvm-image
.golangci.yml
+3 -4
View File
@@ -1,5 +1,8 @@
version: "2"
run:
allow-serial-runners: true
issues:
max-same-issues: 10
@@ -142,14 +145,10 @@ linters:
staticcheck:
checks:
- all
- -ST1000 # package comment
- -ST1003 # naming conventions (Id→ID etc.) — too noisy to retrofit
- -ST1012 # error var naming — errname covers this + error type suffixes
- -ST1013 # HTTP status constants — usestdlibvars is broader
- -ST1016 # receiver name consistency (too noisy to retrofit)
- -ST1020 # godoc on exported types — not enforcing yet
- -ST1021 # godoc on exported type groups — not enforcing yet
- -ST1022 # godoc on exported vars/consts — not enforcing yet
- -SA4006 # value overwritten — ineffassign catches strictly more of these
- -SA9005 # marshaling structs with no exported fields — intended {} for empty API-response types
Makefile
+14 -3
View File
@@ -57,7 +57,7 @@ build-ui:
cd spinifex/runtime/roles/spinifexui/frontend && pnpm build
# GO commands
VERSION ?= $(shell git describe --tags --always --dirty)
VERSION ?= $(shell git describe --tags --always --dirty --exclude '*-dev')
COMMIT ?= $(shell git rev-parse --short HEAD)
LDFLAGS := -s -w -X github.com/mulgadc/spinifex/spinifex/operator/cli.Version=$(VERSION) -X github.com/mulgadc/spinifex/spinifex/operator/cli.Commit=$(COMMIT)
@@ -167,7 +167,7 @@ preflight:
# reason: between them they install the software, wire OVN, erase the state and
# remove the install, so a shellcheck warning in one is an outage or a wrong
# deletion, not a style note.
NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/node-reset.sh scripts/uninstall-spx.sh
NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh
test-build-scripts:
@echo -e "\n....Running build/scripts/**/*_test.sh...."
@@ -262,6 +262,17 @@ test-cover:
$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
@scripts/check-coverage.sh $(COVERPROFILE) $(QUIET)
# Refresh the README coverage badge. Measured as awesome-go documents
# (-coverpkg=./... over ./...), with cmd/ entrypoints left out of the total.
coverage-badge:
@echo -e "\n....Measuring whole-module coverage for the README badge...."
LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 600s -covermode=atomic -coverpkg=./... -coverprofile=coverage-badge.out ./...
@grep -v '/cmd/' coverage-badge.out > coverage-badge-nocmd.out
@pct=$$(go tool cover -func=coverage-badge-nocmd.out | tail -1 | awk '{print $$NF}' | tr -d '%'); \
color=$$(awk -v p="$$pct" 'BEGIN { print (p >= 80) ? "brightgreen" : (p >= 70) ? "yellow" : "red" }'); \
sed -i -E "s|badge/coverage-[0-9.]+%25-[a-z]+|badge/coverage-$${pct}%25-$${color}|" README.md; \
echo "Coverage $${pct}% ($${color}); README badge updated"
# Run unit tests with race detector
test-race:
@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
@@ -464,7 +475,7 @@ distro-arm64:
distro-clean:
rm -rf dist/
.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover coverage-badge test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
deploy reinstall clean \
install-system install-go install-aws quickinstall \
lint fix govulncheck nilaway \
README.md
+2
View File
@@ -4,6 +4,8 @@
<p align="center">
<a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.27+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
<a href="https://pkg.go.dev/github.com/mulgadc/spinifex"><img src="https://img.shields.io/badge/pkg.go.dev-reference-007D9C?style=flat-square&logo=go&logoColor=white" alt="Go Reference"></a>
<a href="https://github.com/mulgadc/spinifex/blob/main/Makefile"><img src="https://img.shields.io/badge/coverage-82.9%25-brightgreen?style=flat-square" alt="Coverage"></a>
<a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
<a href="https://mulgadc.com"><img src="https://img.shields.io/badge/home-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
<a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/docs-spinifex-0969da?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTYuNSAySDIwYTEgMSAwIDAgMSAxIDF2MTQuNWExIDEgMCAwIDEtMSAxSDYuNWExLjUgMS41IDAgMCAwIDAgM0gyMWExIDEgMCAwIDEgMCAySDYuNUEzLjUgMy41IDAgMCAxIDMgMjBWNS41QTMuNSAzLjUgMCAwIDEgNi41IDJ6bTEgNGExIDEgMCAwIDAgMCAyaDlhMSAxIDAgMCAwIDAtMmgtOXptMCA0YTEgMSAwIDAgMCAwIDJoOWExIDEgMCAwIDAgMC0yaC05eiIvPjwvc3ZnPg==" alt="docs.mulgadc.com"></a>
docs/COMMANDS.md
+1 -1
View File
@@ -166,7 +166,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
| Command | Flags | Description |
|---------|-------|-------------|
| `spx admin images import` | `--name`, `--file`, `--distro`, `--version`, `--arch`, `--platform`, `--boot-mode` (bios/uefi/uefi-preferred), `--tag`, `--force`, `--skip-verify`, `--checksum`, `--ami-name` (override the registered `DescribeImages` name, which otherwise defaults to `ami-{distro}-{version}-{arch}`; used for locally-built appliances such as `--ami-name spinifex-eks-node`), `--tmp-dir` (default: `/tmp`) | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, verify the SHA-256/SHA-512 digest, and inherit `BootMode` from the catalog entry. `--boot-mode` overrides the catalog value when set. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports require an explicit `--boot-mode` because there is no catalog metadata to inherit from, and are verified only when `--checksum <sums-file>` is passed. `--checksum` takes the sums file the image was published with (`SHA256SUMS`, `SHA512SUMS`, `<image>.sha256`/`.sha512`, coreutils, BSD or bare-digest format), looks up the entry for the image's basename, and infers sha256 or sha512 from the digest length. A mismatch exits 1 before anything is written to storage and leaves the file in place. `--checksum` proves only that the image matches the sums file supplied with it, not that either is authentic; verify the sums file's vendor signature separately. `--checksum` is refused without `--file` or with `--skip-verify`. A `--file` import without `--checksum` prints `Importing <file> without checksum validation` to stderr and proceeds; `--name --file` does not fall back to the catalog digest. Every import records the source artifact's digest on the AMI with who verified it (`operator`, `catalog` or `unverified`, the unverified cases hashed as sha256), prints it on success, and shows it in `spx admin images describe`. The digest is of the file as supplied, not of the stored volume. `--skip-verify` bypasses verification for catalog imports and prints a warning to stderr; use only for debugging or when upstream mirrors are confirmed-broken. Import never modifies the image: the bytes written to storage are the bytes that were verified. A guest that needs to trust the deployment CA for in-guest SDK-over-TLS calls fetches it at runtime from `http://169.254.169.254/spinifex/ca.pem` (see the IMDS section), so a CA rotation reaches guests without re-importing anything. |
| `spx admin images import` | `--name`, `--file`, `--distro`, `--version`, `--arch`, `--platform`, `--boot-mode` (bios/uefi/uefi-preferred), `--tag`, `--force`, `--skip-verify`, `--checksum`, `--ami-name` (override the registered `DescribeImages` name, which otherwise defaults to `ami-{distro}-{version}-{arch}`; used for locally-built appliances such as `--ami-name spinifex-eks-node`), `--tmp-dir` (default: `<spinifex-dir>/images`) | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, verify the SHA-256/SHA-512 digest, and inherit `BootMode` from the catalog entry. `--boot-mode` overrides the catalog value when set. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports require an explicit `--boot-mode` because there is no catalog metadata to inherit from, and are verified only when `--checksum <sums-file>` is passed. `--checksum` takes the sums file the image was published with (`SHA256SUMS`, `SHA512SUMS`, `<image>.sha256`/`.sha512`, coreutils, BSD or bare-digest format), looks up the entry for the image's basename, and infers sha256 or sha512 from the digest length. A mismatch exits 1 before anything is written to storage and leaves the file in place. `--checksum` proves only that the image matches the sums file supplied with it, not that either is authentic; verify the sums file's vendor signature separately. `--checksum` is refused without `--file` or with `--skip-verify`. A `--file` import without `--checksum` prints `Importing <file> without checksum validation` to stderr and proceeds; `--name --file` does not fall back to the catalog digest. Every import records the source artifact's digest on the AMI with who verified it (`operator`, `catalog` or `unverified`, the unverified cases hashed as sha256), prints it on success, and shows it in `spx admin images describe`. The digest is of the file as supplied, not of the stored volume. `--skip-verify` bypasses verification for catalog imports and prints a warning to stderr; use only for debugging or when upstream mirrors are confirmed-broken. Import never modifies the image: the bytes written to storage are the bytes that were verified. Compressed and qcow2 images are extracted into a hidden `.import-*` scratch directory under `--tmp-dir`, which is removed when the import succeeds or fails, so the catalog directory keeps only the downloaded source (for `--name`) and the AMI manifest. Point `--tmp-dir` at a disk-backed filesystem with room for the full raw image, never at a tmpfs such as `/tmp`. A guest that needs to trust the deployment CA for in-guest SDK-over-TLS calls fetches it at runtime from `http://169.254.169.254/spinifex/ca.pem` (see the IMDS section), so a CA rotation reaches guests without re-importing anything. |
| `spx admin images list` | — | Lists available OS images that can be imported or downloaded |
| `spx admin images describe` | `--image-id` (required) | Read-only. Prints the AMI's name, owner, creation date, boot mode and architecture, then its source digest: algorithm and value, verification (`operator` with the sums file name, `catalog` with the sums URL, or `unverified`), and the source file name. AMIs imported before digest recording print `Source digest: not recorded`. The digest is not exposed through `DescribeImages`. |
| `spx admin images promote` | `--image-id` (required), `--yes` | Reads `ami-<id>/config.json`, validates the AMI is account-owned, then rewrites `ImageOwnerAlias` to `"system"` in-place. No block data is copied. The change takes effect immediately — the AMI becomes visible to all accounts via `DescribeImages`. Prompts for confirmation (skipped with `--yes`). Already-system AMIs are refused. |
docs/admin/setting-up-your-cluster/README.md
+17 -11
View File
@@ -71,6 +71,12 @@ ubuntu-26.04-x86_64 | ubuntu | 26.04 | x86_64 | uefi
spx admin images import --name ubuntu-26.04-x86_64
```
Verify the import and note the AMI ID:
```bash
AMI_ID=$(aws ec2 describe-images --query 'Images[0].ImageId' --output text)
```
### Option B: Import a local image file
```bash
@@ -95,12 +101,12 @@ for img in spinifex-rds-postgres spinifex-rds-mariadb spinifex-ecs-node spinifex
done
```
| Image | Serves | Contents |
| --- | --- | --- |
| `spinifex-rds-postgres` | RDS, `--engine postgres` | Alpine 3.24.1 + PostgreSQL 18 + `rds-init` |
| `spinifex-rds-mariadb` | RDS, `--engine mariadb` | Alpine 3.24.1 + MariaDB 11.8 + `rds-init` |
| `spinifex-ecs-node` | ECS container instances | Alpine 3.21.7 + containerd + ecs-agent |
| `spinifex-eks-node` | EKS control plane and nodegroups | Alpine 3.21.7 + K3s v1.32.5 + eks-token-webhook |
| Image | Serves | Contents |
| ----------------------- | -------------------------------- | ----------------------------------------------- |
| `spinifex-rds-postgres` | RDS, `--engine postgres` | Alpine 3.24.1 + PostgreSQL 18 + `rds-init` |
| `spinifex-rds-mariadb` | RDS, `--engine mariadb` | Alpine 3.24.1 + MariaDB 11.8 + `rds-init` |
| `spinifex-ecs-node` | ECS container instances | Alpine 3.21.7 + containerd + ecs-agent |
| `spinifex-eks-node` | EKS control plane and nodegroups | Alpine 3.21.7 + K3s v1.32.5 + eks-token-webhook |
All four boot **BIOS**, unlike the distro images, which boot UEFI. That is a property of how they are built and nothing you need to pass.
@@ -168,7 +174,7 @@ aws ec2 attach-internet-gateway \
### Create a Subnet
Create the subnet your instances will launch into. The routing and public-IP steps below are what make it a *public* subnet.
Create the subnet your instances will launch into. The routing and public-IP steps below are what make it a _public_ subnet.
```bash
SUBNET_ID=$(aws ec2 create-subnet \
@@ -349,17 +355,17 @@ https://YOUR_NODE_IP:3000
On first load, your browser will show a TLS warning — Spinifex generates a self-signed certificate at install time. This is expected.
1. Accept the warning to reach the login page (exact wording varies by browser — e.g. Chrome: *Advanced → Proceed to ...*, Firefox: *Advanced → Accept the Risk and Continue*).
1. Accept the warning to reach the login page (exact wording varies by browser — e.g. Chrome: _Advanced → Proceed to ..._, Firefox: _Advanced → Accept the Risk and Continue_).
2. On the login page, click **Download Certificate** and save `spinifex-ca.pem` to your machine.
3. Install the certificate as a **trusted root** on your workstation, following the steps for your platform below.
4. Restart your browser and reload `https://YOUR_NODE_IP:3000`. The padlock should now show a valid certificate.
Step 3, by platform:
- **macOS:** open `spinifex-ca.pem` in Keychain Access → *System* keychain → set *Trust* to **Always Trust**.
- **macOS:** open `spinifex-ca.pem` in Keychain Access → _System_ keychain → set _Trust_ to **Always Trust**.
- **Linux:** `sudo cp spinifex-ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt && sudo update-ca-certificates`
- **Windows:** double-click the file → *Install Certificate* → *Local Machine* → *Trusted Root Certification Authorities*.
- **Browser-only (Firefox):** *Settings → Privacy & Security → Certificates → View Certificates → Authorities → Import* and tick *Trust this CA to identify websites*.
- **Windows:** double-click the file → _Install Certificate_ → _Local Machine_ → _Trusted Root Certification Authorities_.
- **Browser-only (Firefox):** _Settings → Privacy & Security → Certificates → View Certificates → Authorities → Import_ and tick _Trust this CA to identify websites_.
> **Why this is required:** the UI logs in by reading your AWS credentials through a trusted TLS channel. Browsers refuse to send credentials over an untrusted connection, so the certificate must be installed as trusted — temporary "Proceed anyway" exceptions won't work for login.
docs/containers/eks/README.md
+2 -2
View File
@@ -55,7 +55,7 @@ Authentication uses **EKS access entries** (the API authentication mode) — IAM
**Spinifex specifics**
- **Auth mode is API-only.** `authentication_mode` must be `API`; the legacy `CONFIG_MAP` and `API_AND_CONFIG_MAP` modes are rejected. Grant access with access entries, not an `aws-auth` ConfigMap.
- **Access entries are the only auth path.** `authentication_mode` accepts `API` (the default when unset) and `API_AND_CONFIG_MAP`; bare `CONFIG_MAP` is rejected. Spinifex has no `aws-auth` ConfigMap path, so `API_AND_CONFIG_MAP` behaves as `API`: grant access with access entries.
- **Security groups are auto-managed.** Spinifex creates the cluster and node-group security groups deterministically (e.g. `eks-cluster-<name>-nodegroup-sg`); `vpc_config.security_group_ids` is ignored. To expose a workload you add an ingress rule to the auto-managed node-group SG.
- **The worker image is fixed.** Node groups always boot Spinifex's `eks-node` image. `ami_type` is recorded but does not select the image, and the image must be registered on the cluster before you create a node group.
- **Default Kubernetes version is `1.32`.**
@@ -262,7 +262,7 @@ The workbooks form a ladder. **eks-quickstart** is a NodePort demo; **eks-https-
**Workload unreachable from outside the cluster.** The node-group security group only allows intra-cluster traffic by default. Add an ingress rule for your NodePort to `eks-cluster-<name>-nodegroup-sg`, as shown in the CLI tab.
**`create-cluster` rejected with an authentication-mode error.** `authentication_mode` must be `API`. Remove any `CONFIG_MAP` / `API_AND_CONFIG_MAP` setting and grant access with access entries instead.
**`create-cluster` rejected with an authentication-mode error.** `CONFIG_MAP` is the one mode Spinifex rejects, because it has no `aws-auth` ConfigMap path. Set `authentication_mode` to `API` or `API_AND_CONFIG_MAP`, or leave it unset, and grant access with access entries.
**Addon install fails.** Only addons bundled into the registered `eks-node` image install successfully. If you pin an addon version in Terraform, leave `addon_version` unset so Spinifex selects the catalog default.
docs/coverage/README.md
+6 -6
View File
@@ -17,18 +17,18 @@ tags:
## Overview
The platform serves **397 operations** across the AWS APIs below. Every page names the operations its service implements from the pinned model, generated from the dispatch tables on each build rather than written by hand.
The platform serves **392 operations** across the AWS APIs below. Every page names the operations its service implements from the pinned model, generated from the dispatch tables on each build rather than written by hand.
| Service | Operations |
|---|---:|
| [ACM](/coverage/acm) | 9 |
| [EC2](/coverage/ec2) | 127 |
| [ECR](/coverage/ecr) | 22 |
| [ECR](/coverage/ecr) | 23 |
| [ECS](/coverage/ecs) | 31 |
| [EKS](/coverage/eks) | 36 |
| [EKS](/coverage/eks) | 27 |
| [ELBv2](/coverage/elbv2) | 36 |
| [IAM](/coverage/iam) | 79 |
| [RDS](/coverage/rds) | 26 |
| [IAM](/coverage/iam) | 81 |
| [RDS](/coverage/rds) | 27 |
| [S3](/coverage/s3) | 26 |
| [STS](/coverage/sts) | 5 |
| **Total** | **397** |
| **Total** | **392** |
docs/coverage/coverage.json
+19 -25
View File
@@ -1983,6 +1983,7 @@
"DeleteRepository",
"DeleteRepositoryPolicy",
"DescribeImages",
"DescribeRegistry",
"DescribeRepositories",
"GetAuthorizationToken",
"GetLifecyclePolicy",
@@ -2002,7 +2003,6 @@
"stubbed": [
"BatchCheckLayerAvailability",
"CompleteLayerUpload",
"DescribeRegistry",
"GetDownloadUrlForLayer",
"GetRegistryPolicy",
"InitiateLayerUpload",
@@ -2348,7 +2348,6 @@
],
"registered": [
"AssociateAccessPolicy",
"AssociateIdentityProviderConfig",
"CreateAccessEntry",
"CreateAddon",
"CreateCluster",
@@ -2361,36 +2360,27 @@
"DescribeAddon",
"DescribeAddonVersions",
"DescribeCluster",
"DescribeIdentityProviderConfig",
"DescribeNodegroup",
"DescribeUpdate",
"DisassociateAccessPolicy",
"DisassociateIdentityProviderConfig",
"GetRecoveryDirective",
"ListAccessEntries",
"ListAccessPolicies",
"ListAddons",
"ListAssociatedAccessPolicies",
"ListClusters",
"ListIdentityProviderConfigs",
"ListInternalAddons",
"ListNodegroups",
"ListTagsForResource",
"ListUpdates",
"PublishInternal",
"TagResource",
"UntagResource",
"UpdateAccessEntry",
"UpdateAddon",
"UpdateClusterConfig",
"UpdateClusterVersion",
"UpdateNodegroupConfig",
"UpdateNodegroupVersion",
"WebhookTokenReview"
],
"implemented": [
"AssociateAccessPolicy",
"AssociateIdentityProviderConfig",
"CreateAccessEntry",
"CreateAddon",
"CreateCluster",
@@ -2403,34 +2393,27 @@
"DescribeAddon",
"DescribeAddonVersions",
"DescribeCluster",
"DescribeIdentityProviderConfig",
"DescribeNodegroup",
"DescribeUpdate",
"DisassociateAccessPolicy",
"DisassociateIdentityProviderConfig",
"ListAccessEntries",
"ListAccessPolicies",
"ListAddons",
"ListAssociatedAccessPolicies",
"ListClusters",
"ListIdentityProviderConfigs",
"ListNodegroups",
"ListTagsForResource",
"ListUpdates",
"TagResource",
"UntagResource",
"UpdateAccessEntry",
"UpdateAddon",
"UpdateClusterConfig",
"UpdateClusterVersion",
"UpdateNodegroupConfig",
"UpdateNodegroupVersion"
"UpdateNodegroupConfig"
],
"stubbed": [],
"unsupported": [],
"missing": [
"ActivateCertificateAuthority",
"AssociateEncryptionConfig",
"AssociateIdentityProviderConfig",
"CancelUpdate",
"CreateCapability",
"CreateCertificateAuthority",
@@ -2449,19 +2432,27 @@
"DescribeClusterVersions",
"DescribeEksAnywhereSubscription",
"DescribeFargateProfile",
"DescribeIdentityProviderConfig",
"DescribeInsight",
"DescribeInsightsRefresh",
"DescribePodIdentityAssociation",
"DescribeUpdate",
"DisassociateIdentityProviderConfig",
"ListCapabilities",
"ListCertificateAuthorities",
"ListEksAnywhereSubscriptions",
"ListFargateProfiles",
"ListIdentityProviderConfigs",
"ListInsights",
"ListPodIdentityAssociations",
"ListUpdates",
"RegisterCluster",
"StartInsightsRefresh",
"UpdateCapability",
"UpdateClusterConfig",
"UpdateClusterVersion",
"UpdateEksAnywhereSubscription",
"UpdateNodegroupVersion",
"UpdatePodIdentityAssociation"
],
"extra": [
@@ -2862,6 +2853,7 @@
"GetUser",
"GetUserPolicy",
"ListAccessKeys",
"ListAccountAliases",
"ListAttachedGroupPolicies",
"ListAttachedRolePolicies",
"ListAttachedUserPolicies",
@@ -2901,7 +2893,8 @@
"UntagUser",
"UpdateAccessKey",
"UpdateAssumeRolePolicy",
"UpdateRole"
"UpdateRole",
"UpdateRoleDescription"
],
"implemented": [
"AddRoleToInstanceProfile",
@@ -2943,6 +2936,7 @@
"GetUser",
"GetUserPolicy",
"ListAccessKeys",
"ListAccountAliases",
"ListAttachedGroupPolicies",
"ListAttachedRolePolicies",
"ListAttachedUserPolicies",
@@ -2982,7 +2976,8 @@
"UntagUser",
"UpdateAccessKey",
"UpdateAssumeRolePolicy",
"UpdateRole"
"UpdateRole",
"UpdateRoleDescription"
],
"stubbed": [],
"unsupported": [],
@@ -3042,7 +3037,6 @@
"GetServiceLastAccessedDetails",
"GetServiceLastAccessedDetailsWithEntities",
"GetServiceLinkedRoleDeletionStatus",
"ListAccountAliases",
"ListDelegationRequests",
"ListMFADeviceTags",
"ListMFADevices",
@@ -3078,7 +3072,6 @@
"UpdateGroup",
"UpdateLoginProfile",
"UpdateOpenIDConnectProviderThumbprint",
"UpdateRoleDescription",
"UpdateSAMLProvider",
"UpdateSSHPublicKey",
"UpdateServerCertificate",
@@ -3297,6 +3290,7 @@
"ModifyDBCluster",
"ModifyDBInstance",
"ModifyDBParameterGroup",
"ModifyDBSubnetGroup",
"ModifyOptionGroup",
"PollDBCommands",
"PromoteReadReplica",
@@ -3331,6 +3325,7 @@
"ListTagsForResource",
"ModifyDBInstance",
"ModifyDBParameterGroup",
"ModifyDBSubnetGroup",
"RebootDBInstance",
"RemoveTagsFromResource",
"RestoreDBInstanceFromDBSnapshot",
@@ -3448,7 +3443,6 @@
"ModifyDBShardGroup",
"ModifyDBSnapshot",
"ModifyDBSnapshotAttribute",
"ModifyDBSubnetGroup",
"ModifyEventSubscription",
"ModifyGlobalCluster",
"ModifyIntegration",
docs/coverage/ecr/README.md
+3 -2
View File
@@ -18,7 +18,7 @@ tags:
## Overview
Spinifex implements **22 operations** in the ECR `2015-09-21` API model.
Spinifex implements **23 operations** in the ECR `2015-09-21` API model.
### Two endpoints, one registry
@@ -26,7 +26,7 @@ Repository metadata is served over the AWS API on the gateway endpoint. Image da
The split explains the stubs below. The layer-transfer operations — `BatchCheckLayerAvailability`, `InitiateLayerUpload`, `UploadLayerPart`, `CompleteLayerUpload` and `GetDownloadUrlForLayer` — are registered stubs because the `/v2/` endpoint carries that traffic instead. A client using `docker` or any OCI-compatible tool never calls them.
Registry replication is a stub for the same kind of reason: a deployment is a single registry, with no cross-region peer to replicate to.
Registry replication is a stub for the same kind of reason: a deployment is a single registry, with no cross-region peer to replicate to. `DescribeRegistry` reports this as a replication configuration with no rules.
### Operations
@@ -39,6 +39,7 @@ Registry replication is a stub for the same kind of reason: a deployment is a si
| `DeleteRepository` |
| `DeleteRepositoryPolicy` |
| `DescribeImages` |
| `DescribeRegistry` |
| `DescribeRepositories` |
| `GetAuthorizationToken` |
| `GetLifecyclePolicy` |
docs/coverage/ecr/intro.md
+1 -1
View File
@@ -4,4 +4,4 @@ Repository metadata is served over the AWS API on the gateway endpoint. Image da
The split explains the stubs below. The layer-transfer operations — `BatchCheckLayerAvailability`, `InitiateLayerUpload`, `UploadLayerPart`, `CompleteLayerUpload` and `GetDownloadUrlForLayer` — are registered stubs because the `/v2/` endpoint carries that traffic instead. A client using `docker` or any OCI-compatible tool never calls them.
Registry replication is a stub for the same kind of reason: a deployment is a single registry, with no cross-region peer to replicate to.
Registry replication is a stub for the same kind of reason: a deployment is a single registry, with no cross-region peer to replicate to. `DescribeRegistry` reports this as a replication configuration with no rules.
docs/coverage/eks/README.md
+1 -10
View File
@@ -18,14 +18,13 @@ tags:
## Overview
Spinifex implements **36 operations** in the EKS `2017-11-01` API model.
Spinifex implements **27 operations** in the EKS `2017-11-01` API model.
### Operations
| Operation |
|---|
| `AssociateAccessPolicy` |
| `AssociateIdentityProviderConfig` |
| `CreateAccessEntry` |
| `CreateAddon` |
| `CreateCluster` |
@@ -38,25 +37,17 @@ Spinifex implements **36 operations** in the EKS `2017-11-01` API model.
| `DescribeAddon` |
| `DescribeAddonVersions` |
| `DescribeCluster` |
| `DescribeIdentityProviderConfig` |
| `DescribeNodegroup` |
| `DescribeUpdate` |
| `DisassociateAccessPolicy` |
| `DisassociateIdentityProviderConfig` |
| `ListAccessEntries` |
| `ListAccessPolicies` |
| `ListAddons` |
| `ListAssociatedAccessPolicies` |
| `ListClusters` |
| `ListIdentityProviderConfigs` |
| `ListNodegroups` |
| `ListTagsForResource` |
| `ListUpdates` |
| `TagResource` |
| `UntagResource` |
| `UpdateAccessEntry` |
| `UpdateAddon` |
| `UpdateClusterConfig` |
| `UpdateClusterVersion` |
| `UpdateNodegroupConfig` |
| `UpdateNodegroupVersion` |
docs/coverage/iam/README.md
+3 -1
View File
@@ -18,7 +18,7 @@ tags:
## Overview
Spinifex implements **79 operations** in the IAM `2010-05-08` API model.
Spinifex implements **81 operations** in the IAM `2010-05-08` API model.
### Scope
@@ -67,6 +67,7 @@ All IAM operations are account-scoped. The root user of an account bypasses poli
| `GetUser` |
| `GetUserPolicy` |
| `ListAccessKeys` |
| `ListAccountAliases` |
| `ListAttachedGroupPolicies` |
| `ListAttachedRolePolicies` |
| `ListAttachedUserPolicies` |
@@ -107,3 +108,4 @@ All IAM operations are account-scoped. The root user of an account bypasses poli
| `UpdateAccessKey` |
| `UpdateAssumeRolePolicy` |
| `UpdateRole` |
| `UpdateRoleDescription` |
docs/coverage/internal-report.md
+13 -8
View File
@@ -762,7 +762,7 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
## ecr
`2015-09-21` — 22 of 58 modelled operations implemented (37.9%), 11 stubbed, 0 not applicable, 3 registered outside the pinned model.
`2015-09-21` — 23 of 58 modelled operations implemented (39.7%), 10 stubbed, 0 not applicable, 3 registered outside the pinned model.
### Not implemented
@@ -793,7 +793,6 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
- `BatchCheckLayerAvailability`
- `CompleteLayerUpload`
- `DescribeRegistry`
- `GetDownloadUrlForLayer`
- `GetRegistryPolicy`
- `InitiateLayerUpload`
@@ -882,12 +881,13 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
## eks
`2017-11-01` — 36 of 70 modelled operations implemented (51.4%), 0 stubbed, 0 not applicable, 4 registered outside the pinned model.
`2017-11-01` — 27 of 70 modelled operations implemented (38.6%), 0 stubbed, 0 not applicable, 4 registered outside the pinned model.
### Not implemented
- `ActivateCertificateAuthority`
- `AssociateEncryptionConfig`
- `AssociateIdentityProviderConfig`
- `CancelUpdate`
- `CreateCapability`
- `CreateCertificateAuthority`
@@ -906,19 +906,27 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
- `DescribeClusterVersions`
- `DescribeEksAnywhereSubscription`
- `DescribeFargateProfile`
- `DescribeIdentityProviderConfig`
- `DescribeInsight`
- `DescribeInsightsRefresh`
- `DescribePodIdentityAssociation`
- `DescribeUpdate`
- `DisassociateIdentityProviderConfig`
- `ListCapabilities`
- `ListCertificateAuthorities`
- `ListEksAnywhereSubscriptions`
- `ListFargateProfiles`
- `ListIdentityProviderConfigs`
- `ListInsights`
- `ListPodIdentityAssociations`
- `ListUpdates`
- `RegisterCluster`
- `StartInsightsRefresh`
- `UpdateCapability`
- `UpdateClusterConfig`
- `UpdateClusterVersion`
- `UpdateEksAnywhereSubscription`
- `UpdateNodegroupVersion`
- `UpdatePodIdentityAssociation`
### Registered outside the pinned model
@@ -957,7 +965,7 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
## iam
`2010-05-08` — 79 of 180 modelled operations implemented (43.9%), 0 stubbed, 19 not applicable, 0 registered outside the pinned model.
`2010-05-08` — 81 of 180 modelled operations implemented (45.0%), 0 stubbed, 19 not applicable, 0 registered outside the pinned model.
### Not implemented
@@ -1006,7 +1014,6 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
- `GetServerCertificate`
- `GetServiceLastAccessedDetails`
- `GetServiceLastAccessedDetailsWithEntities`
- `ListAccountAliases`
- `ListDelegationRequests`
- `ListMFADeviceTags`
- `ListMFADevices`
@@ -1038,7 +1045,6 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
- `UpdateGroup`
- `UpdateLoginProfile`
- `UpdateOpenIDConnectProviderThumbprint`
- `UpdateRoleDescription`
- `UpdateSAMLProvider`
- `UpdateServerCertificate`
- `UpdateUser`
@@ -1079,7 +1085,7 @@ signing-certificates: X.509 signing certificates are an EC2-Classic SOAP credent
## rds
`2014-10-31` — 26 of 164 modelled operations implemented (15.9%), 0 stubbed, 4 not applicable, 5 registered outside the pinned model.
`2014-10-31` — 27 of 164 modelled operations implemented (16.5%), 0 stubbed, 4 not applicable, 5 registered outside the pinned model.
### Not implemented
@@ -1178,7 +1184,6 @@ signing-certificates: X.509 signing certificates are an EC2-Classic SOAP credent
- `ModifyDBShardGroup`
- `ModifyDBSnapshot`
- `ModifyDBSnapshotAttribute`
- `ModifyDBSubnetGroup`
- `ModifyEventSubscription`
- `ModifyGlobalCluster`
- `ModifyIntegration`
docs/coverage/rds/README.md
+2 -1
View File
@@ -19,7 +19,7 @@ tags:
## Overview
Spinifex implements **26 operations** in the RDS `2014-10-31` API model.
Spinifex implements **27 operations** in the RDS `2014-10-31` API model.
### Engines
@@ -95,6 +95,7 @@ A parameter whose omission would create a false safety, security or availability
| `ListTagsForResource` |
| `ModifyDBInstance` |
| `ModifyDBParameterGroup` |
| `ModifyDBSubnetGroup` |
| `RebootDBInstance` |
| `RemoveTagsFromResource` |
| `RestoreDBInstanceFromDBSnapshot` |
docs/install/install-multi-node/README.md
+7
View File
@@ -213,6 +213,13 @@ Choose one method and apply it to **every** server in the cluster.
curl -fsSL https://install.mulgadc.com | bash
```
This installs the latest release. To try the newest development build instead, set `INSTALL_SPINIFEX_CHANNEL=dev` on **every** node — a cluster must not mix versions. It carries unreleased work and has not been through release testing.
```bash
curl -fsSL https://install.mulgadc.com -o /tmp/spx-install.sh
sudo INSTALL_SPINIFEX_CHANNEL=dev bash /tmp/spx-install.sh
```
**Option B — bare metal, from the ISO.** Boot each server from the Spinifex ISO and follow [Bootable USB Install](/docs/install-usb). This installs the operating system, partitions the disks, and configures the hostname and network interfaces alongside Spinifex. The ISO installer does not form a cluster — that is what the remaining steps do.
Complete this step on all three servers before continuing. Step 4 requires every node to be installed, reachable, and available at the same time.
docs/install/install/README.md
+7
View File
@@ -75,6 +75,13 @@ curl -fsSL https://install.mulgadc.com | bash
The installer downloads the Spinifex binary and bootstraps all dependencies (QEMU, OVN/OVS, AWS CLI).
This installs the latest release. To try the newest development build instead, set the channel — it carries unreleased work and has not been through release testing, so use it to evaluate a fix or a new feature rather than for anything you depend on.
```bash
curl -fsSL https://install.mulgadc.com -o /tmp/spx-install.sh
sudo INSTALL_SPINIFEX_CHANNEL=dev bash /tmp/spx-install.sh
```
## Step 2. Setup OVN Networking
If your WAN interface is already a bridge (e.g. `br-wan`), setup-ovn.sh auto-detects it:
docs/oci-integration/README.md
+522 -857
View File
File diff suppressed because it is too large Load Diff
docs/oci-integration/oci-architecture/README.md
+483
View File
@@ -0,0 +1,483 @@
---
title: "OCI Architecture"
seoTitle: "Spinifex on OCI: Architecture and Operations — Spinifex Docs"
description: "How Spinifex is wired on OCI: both VNICs, the iSCSI data volume and the public-address allocator, with the IAM policy, the quotas and the troubleshooting."
category: "Cloud Install"
sections:
- overview
- troubleshooting
tags:
- install
- oci
- oracle cloud
- terraform
- architecture
- troubleshooting
resources:
- title: "Spinifex on Oracle Cloud"
url: "/docs/oci-integration"
- title: "Host Firewall"
url: "/docs/host-firewall"
- title: "VPC Networking"
url: "/docs/vpc-networking"
- title: "Multi-Node Install"
url: "/docs/install-multi-node"
---
# Spinifex on OCI: Architecture and Operations
> The reference half of the OCI integration. [Spinifex on Oracle Cloud](/docs/oci-integration) is the step-by-step deployment; everything here is what sits underneath it, plus the troubleshooting.
## Table of Contents
- [Why run Spinifex on OCI](#why-run-spinifex-on-oci)
- [Sizing](#sizing)
- [How it fits together](#how-it-fits-together)
- [Credentials: an API key, or an instance principal](#credentials-an-api-key-or-an-instance-principal)
- [The IAM policy](#the-iam-policy)
- [Quotas](#quotas)
- [Firewalls, in all three layers](#firewalls-in-all-three-layers)
- [What Terraform builds, per node](#what-terraform-builds-per-node)
- [Every Terraform variable](#every-terraform-variable)
- [The `oci` CLI](#the-oci-cli)
- [Deploying without Terraform](#deploying-without-terraform)
- [Troubleshooting](#troubleshooting)
---
## Overview
## Why run Spinifex on OCI
**Lower cost for the same shape of workload.** OCI prices compute and block storage well below the large US clouds for equivalent OCPU and IOPS, and Spinifex turns one large instance into a whole EC2/EBS/S3 surface. A tenant's twenty small instances become twenty QEMU guests on hardware you are billed for once, instead of twenty billable cloud instances.
**Egress stops being the line item that decides the architecture.** Oracle publishes a large monthly outbound-transfer allowance per tenancy, then charges per GB beyond it at roughly an order of magnitude below the majors. Check the current pricing page rather than a number in a document, but that gap is the reason this integration exists. Traffic _between_ guests never leaves the host, or on a cluster never leaves the VCN: it is Geneve over the private plane, which is not billed as internet egress at all.
**The workload stops being AWS-shaped and becomes yours.** Your tenants code against Spinifex's API, so one set of Terraform, SDK calls and AMIs runs on OCI today and on bare metal in your own rack whenever you decide to move it. Leaving AWS becomes a deployment decision rather than a rewrite, and the second move costs nothing extra.
**You keep the parts of OCI worth keeping.** Bare-metal shapes, fault domains, block volumes with a chosen performance tier, and real public addresses out of OCI's own pool. Spinifex registers each guest's address with OCI rather than hiding the cloud underneath it, so the guest is genuinely reachable on the internet instead of sitting behind a shared NAT.
## Sizing
**Bare metal is the recommendation.** `BM.Standard.E2.64` or similar gives you the hardware directly, with no nested virtualisation, full NIC performance, and nothing between your guests and the wire. Where bare metal is out of scope, a single `VM.Standard.E6.Flex` suits testing and development, or three of them production.
| | Minimum | Recommended |
| --------------- | --------------------- | ----------------------------------------------------------------- |
| **Shape** | `VM.Standard.E6.Flex` | Bare metal, e.g. `BM.Standard.E2.64` |
| **OCPU** | 8 (16 vCPU) | 32+ |
| **RAM** | 32 GB | 128 GB+ |
| **Data volume** | 256 GiB block volume | 1 TB+ NVMe-backed |
| **VNICs** | 2 | 2 |
| **Nodes** | 1 | 3, see [Cluster sizing](/docs/install-multi-node#cluster-sizing) |
## How it fits together
Everything in [Single-Node Install](/docs/install) and [Multi-Node Install](/docs/install-multi-node) still applies: formation, storage and OVN behave exactly as they do on hardware you own. The layer underneath is what changes. An OCI VCN is not an Ethernet segment, so the external datapath is wired differently, and public addresses come from OCI's API rather than from a range you write in a config file.
### Single node: the whole AWS surface inside one OCI instance
One instance runs the control plane and every tenant guest. Customers reach the AWS APIs on the node's own address, while their instances reach the internet through addresses registered on the node's second VNIC.
<p align="center">
<img src="../../../.github/assets/diagrams/oci-single-node.svg" alt="Single node on OCI: the instance inside the VCN public subnet, both VNICs, br-wan, the routed-NAT transit veth, the control plane, tenant VPCs and the iSCSI data volume" width="900">
</p>
Two things in that picture are the whole integration. The guest's private address (`10.0.1.4`) never leaves the host, because OVN and the host route both hold it. The address OCI delivers to is a **secondary private IP registered on VNIC 1**, appearing on `br-wan`, and the customer's public address (`150.230.13.131`) is a reserved public IP that OCI 1:1-NATs onto it upstream. `describe-instances` reports the public one while every datapath object holds the private one, which is why the two never appear together on the host.
The third is storage. Every guest disk, every S3 object and the JetStream state all live under `/var/lib/spinifex`, which is an **OCI block volume attached over iSCSI**, not the boot disk. Size it for the guests you intend to run. A boot volume that quietly ends up carrying the stack is the most common way an install on OCI fails later rather than immediately.
### Three nodes: the same surface, distributed
Each node registers the addresses for **its own** guests on **its own** second VNIC, and the allocator runs per node, so nothing has to know another node's OCIDs. When a guest moves, whether through a stop/start that lands elsewhere or a host failure, the node it lands on claims the address's private half from the old VNIC with a single OCI call. The OCID and the public half are untouched, so the customer's address never changes.
<p align="center">
<img src="../../../.github/assets/diagrams/oci-three-node.svg" alt="Three nodes on OCI: three instances inside the VCN public subnet, one per fault domain, each with its own VNICs, addresses and iSCSI volume, joined east-west by Geneve, NATS, predastore and the OVN raft" width="900">
</p>
The overlay, the object shards and the OVN databases all cross **VNIC 0**, inside the VCN. Size that plane rather than the external one: guest-to-guest traffic between nodes is Geneve over the VCN, and it is the link every distributed layer shares.
> [!NOTE]
> **The Spinifex region is your own naming and has nothing to do with the OCI region.** `ap-southeast-2` in `spinifex.toml` beside `ap-sydney-1` in OCI is correct and expected. Do not read one as evidence about the other.
## Credentials: an API key, or an instance principal
The allocator needs OCI credentials at runtime. **A node with none forms, passes every health check, and then refuses every launch that wants a public address** with `InsufficientAddressCapacity`, naming no cause outside its own journal.
### An API key, the documented path
An API key needs nothing from a tenancy admin, which also makes it the only option when the compartment belongs to someone else's tenancy. The cost is key material on disk, with a rotation obligation, on every node:
```bash
# On your workstation, not the node.
openssl genrsa -out ~/.oci/oci_api_key.pem 2048
chmod 600 ~/.oci/oci_api_key.pem
openssl rsa -pubout -in ~/.oci/oci_api_key.pem -out ~/.oci/oci_api_key_public.pem
openssl rsa -pubout -outform DER -in ~/.oci/oci_api_key.pem \
| openssl md5 -c # this is the fingerprint
```
Upload the public key to the user under **Identity → Users → API Keys**, and record the user OCID, the fingerprint and the tenancy OCID.
Each node then needs **two** files, and a node holding only the key is the silent failure described above.
| Path | Mode | Contents |
| ----------------------------------- | -------------------- | ------------------------------------------------------------------------ |
| `/etc/spinifex/oci/oci_api_key.pem` | `0640 root:spinifex` | The private key |
| `/etc/spinifex/oci/config` | `0640 root:spinifex` | An ordinary OCI SDK config, profile `[spinifex]`, naming the key by path |
The profile name must be `spinifex` to match `oci_config_profile` in the pool:
```ini
[spinifex]
user=ocid1.user.oc1..aaaa...
fingerprint=12:34:56:78:90:ab:cd:ef:12:34:56:78:90:ab:cd:ef
tenancy=ocid1.tenancy.oc1..aaaa...
region=ap-sydney-1
key_file=/etc/spinifex/oci/oci_api_key.pem
```
**`spx-oci-config.sh` writes both files on every node, and the deploy calls it for you.** It runs after formation and before the pool is configured, which is the only window where a node has `/etc/spinifex` but has not started the allocator, and it reads the credential from your `[spinifex]` profile, a dedicated key, a set of `OCI_SPX_*` environment variables, or failing all three the credential Terraform itself used — resolved through the same `scripts/oci_env.py` as the apply, so the two cannot disagree.
**The key reaches the nodes over SSH and goes nowhere else.** It is never rendered into user-data or Terraform state, where it would be readable from instance metadata for the life of the instance, and it is carried in the remote shell's stdin rather than its arguments, where `ps` would expose it to every other tenant on the host. `--credential-hook PATH` still takes any executable called as `hook <ssh-key> <host>...`, so a tenancy that keeps credentials in a vault can substitute its own.
### An instance principal: no key material, but it needs a tenancy admin
The instance certificate is served at `/opc/v2/identity/cert.pem` and the Go SDK authenticates as the instance with no key on disk. It must also be _authorised_, by a dynamic group and a policy, and **both are tenancy-root resources**: `oci_identity_dynamic_group` is created with `compartment_id` set to the tenancy OCID. A compartment-scoped user cannot create them, and the attempt fails with `404-NotAuthorizedOrNotFound` on `CreateDynamicGroup`, which reads like a misconfiguration and is not one.
```bash
cd scripts/terraform/oci-spx
./setup-identity.sh --dry-run # always first
./setup-identity.sh # needs a tenancy-admin OCI profile
```
Then set `instance_principal = "adopt"` in your tfvars and pass `--instance-principal` to the deploy instead of `--credential-hook`. The dynamic group matches on `instance.compartment.id`, so it covers every node you ever build in that compartment and nothing needs updating when nodes are replaced.
## The IAM policy
Spinifex calls exactly ten operations, all of them in `spinifex/cloud/oci/client.go`. Grant no more than these:
| Operation | Why |
| ------------------------------------------ | -------------------------------------------------------------------------------- |
| `CreatePrivateIp` | Register the secondary private IP the datapath carries |
| `DeletePrivateIp` | Release it |
| `GetPrivateIp` | Poll for `AVAILABLE` after an asynchronous assign |
| `ListPrivateIps` | The startup reconcile, and the affinity pass that moves an address between nodes |
| `UpdatePrivateIp` | Reassign a private IP to this node's VNIC when a guest moves here |
| `CreatePublicIp` | Create the RESERVED public IP |
| `DeletePublicIp` | Release it |
| `GetPublicIp` / `GetPublicIpByPrivateIpId` | Poll lifecycle state, and reconcile a private IP back to its public half |
| `UpdatePublicIp` | Attach and detach the public IP from a private IP |
The matching policy, scoped to the one compartment Spinifex allocates in:
```text
Allow group SpinifexOperators to use vnics in compartment <compartment-name>
Allow group SpinifexOperators to manage private-ips in compartment <compartment-name>
Allow group SpinifexOperators to manage public-ips in compartment <compartment-name>
Allow group SpinifexOperators to use subnets in compartment <compartment-name>
```
`instance-principal.tf` grants the dynamic group these same four statements, so an instance principal can allocate a public address exactly as an IAM user can. The two credential routes differ in who is authorised, never in what.
> [!IMPORTANT]
> **Allocating one address needs permissions on three resource types, and a 404 names none of them.** `CreatePrivateIp` is checked against `PRIVATE_IP_CREATE`, `PRIVATE_IP_ASSIGN`, `VNIC_ASSIGN` and `SUBNET_ATTACH`, and `DeletePrivateIp` against `SUBNET_DETACH`. `VNIC_ASSIGN` lives in the vnics `use` verb, so `read vnics` is not enough; `SUBNET_ATTACH` and `SUBNET_DETACH` live in the subnets `use` verb, so a policy naming only vnics and IPs authorises nothing. Either omission fails every allocation with `NotAuthorizedOrNotFound` on a node that looks entirely healthy, and OCI uses that one error for "you may not" and "it is not there" alike, so it reads as a wrong compartment OCID rather than a missing verb.
To tell the two apart, make one call per permission class and see which is the first to fail. `GetVnic` tests `VNIC_READ`, `ListPrivateIps` tests `PRIVATE_IP_READ`, and `GetSubnet` tests `SUBNET_READ`. If the first two succeed and the third does not, the credential is authentic and in the right compartment, and the policy is missing its subnet statement.
**Scope it to a compartment, not the tenancy.** `manage public-ips` at tenancy level lets a compromised node consume the whole regional reserved-public-IP quota.
**Do not grant `manage vnics` or `manage instances`.** Spinifex never creates, attaches or deletes a VNIC, and never touches an instance. It only adds addresses to a VNIC that already exists, which `use` permits. `manage` also permits VNIC lifecycle, which is strictly more than the integration can use.
## Quotas
Two limits bound how many public addresses a deployment can hand out, and they do not scale the same way:
| Quota | Default | Scope | Notes |
| ---------------------------------- | ------- | ------------------------- | ---------------------------------------------------------------------------------------------- |
| **Reserved public IPs** | 50 | Per region, whole tenancy | Shared by every node, so it does **not** grow with node count. This is the wall you hit first. |
| **Secondary private IPs per VNIC** | 64 | Per VNIC, so per node | Not raisable. Three nodes gives you 192, so this is the limit that does scale. |
**On a new tenancy, check the compute limits too.** A fresh account commonly carries a service limit of zero for bare-metal shapes and a low one for flex VMs, which can leave `BM.Standard.E2.64` unavailable until you request it. That refusal arrives at apply time, not at plan time.
A raise is a support request, not a setting: Console → Governance & Administration → Limits, Quotas and Usage → "Request a service limit increase". Cite the exact limit name from `oci limits definition list`, not a name from documentation.
```bash
oci limits definition list --service-name vcn --compartment-id <tenancy-ocid>
oci limits value list --service-name vcn --compartment-id <tenancy-ocid>
oci limits resource-availability get --service-name vcn \
--limit-name <name-from-above> --compartment-id <tenancy-ocid>
```
**These calls need a grant the node does not have, and must not be given.** Limits and usage are tenancy-level, which cannot be compartment-scoped the way the policy above insists the node's is. Create a **second, read-only principal** for an operator or a reporting job, and keep it off every node:
```text
Allow group SpinifexReporting to read limits in tenancy
Allow group SpinifexReporting to read usage-reports in tenancy
```
The second verb is what Cost Analysis and `oci usage-api usage-summary request-summarized-usages` read. Adding either to `SpinifexOperators` would hand every node tenancy-wide read, which is exactly what the node policy refuses.
## Firewalls, in all three layers
Three independent things filter traffic to an OCI deployment, and they are easy to mistake for one. Nearly every "my port is closed" report on OCI turns out to be the wrong layer being adjusted.
| Layer | What it guards | Who configures it |
| --- | --- | --- |
| **OCI security list** | The whole public subnet, including the addresses guests are reached on | `node_client_cidr_allow_list` |
| **The host firewall** | The node's own listeners: console, S3 gate, AWS gateway, SSH, the cluster plane | `node_service_ports`, then the Spinifex nft policy |
| **OVN security groups** | Each guest, per AWS security-group rules | Your tenants, through the EC2 API |
### The OCI security list is deliberately open, and should stay that way
```hcl
# network.tf, security_list.public
ingress_security_rules { protocol = "all", source = var.vcn_cidr } # intra-VCN
dynamic "ingress_security_rules" { ... source = each allow-list CIDR } # protocol = "all"
```
A guest's public address is a **secondary private IP on VNIC 1, in the public subnet**, so inbound traffic to a guest crosses this security list. Narrowing `node_client_cidr_allow_list` to your own office `/32` therefore cuts internet ingress to **every guest**, not just to the node. A tenant who then opens 443 on their own security group finds it does nothing, with no error anywhere naming the cause.
That is also why the rule is `protocol = "all"` rather than the three service ports. Enumerating ports here would mean a Terraform change for every customer port.
**So leave it at `0.0.0.0/0` for any deployment that serves guests**, and do the restricting in the host firewall, which is the layer that can tell the node's own listeners apart from a guest's. Narrow the allow list only for a private lab with no public guest services.
The separate intra-VCN rule exists so that narrowing the allow list can never cut Geneve, OVN, NATS or predastore between nodes.
### The host firewall is what you harden
Terraform's cloud-init runs `open-service-ports.sh`, which opens all protocols from the peer CIDR plus `node_service_ports` (`3000 8443 9999`) to any source, in the Ubuntu image's **iptables**. SSH is already open to the world from the image's own INPUT rule, and nothing in that script narrows it.
The Spinifex nft policy is installed but **not armed** on this path, because the installer arms it only when asked. Arming it is what gives you the proper policy: public plane, peer-scoped cluster plane, and an SSH scope you own.
```bash
# On each node. Arms the policy the ISO installer arms by default.
sudo env SETUP_STAGES=firewall /usr/local/share/spinifex/setup.sh --firewall on
```
Then narrow SSH in the one file `setup.sh` creates and never overwrites:
```bash
# /etc/spinifex/firewall/custom.nft is yours, and upgrades leave it alone.
define trusted_ssh_peers = { 203.0.113.10/32 }
```
```bash
sudo nft -c -f /etc/spinifex/firewall/spinifex.nft # check before applying
sudo /usr/local/lib/spinifex/spinifex-firewall-apply
```
What the armed policy allows, and why each port is where it is, is in [Host Firewall](/docs/host-firewall), along with [Narrowing SSH access](/docs/host-firewall#narrowing-ssh-access) and the rule that **forming or expanding a cluster starts by turning the policy off**, because nodes mid-formation are not yet each other's peers.
Two OCI-specific notes on top of that guide:
- **Northstar's DNS on 53 is in the armed policy's public plane, but `node_service_ports` does not include it.** Until the nft policy is armed, a node serving a public zone does not answer on 53 from the internet.
- **Never flush iptables on an OCI node.** The image's `InstanceServices` chain in OUTPUT is what permits iSCSI to `169.254.2.0/24:3260` and the metadata service, so clearing it takes the data volume and IMDS with it.
## What Terraform builds, per node
| Resource | Detail |
| ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- |
| `oci_core_instance` | One per `node_count`, round-robined across **fault domains** so three nodes land on three sets of racks rather than wherever placement puts them |
| Primary VNIC | Public IP, `hostname_label`, the host plane: SSH, the advertise address, the Geneve endpoint |
| `oci_core_vnic_attachment` | A **second VNIC** in the same subnet. Every Spinifex external address becomes a secondary private IP here at runtime |
| `oci_core_volume` | A block volume per node, `data_volume_size_in_gbs` at `data_volume_vpus_per_gb` |
| `oci_core_volume_attachment` | **`attachment_type = "iscsi"`**, with the Oracle Cloud Agent's Block Volume Management plugin enabled so the node logs the iSCSI session in itself |
| cloud-init | Partitions, formats, mounts and `fstab`s the volume; builds `br-wan` over the second VNIC; opens the service ports |
The resource-by-resource architecture is in [`scripts/terraform/oci-spx`](https://github.com/mulgadc/spinifex/tree/dev/scripts/terraform/oci-spx).
**The data volume is iSCSI, and that is the detail that bites.** OCI presents the volume as an iSCSI target reachable at `169.254.2.2:3260`, so attaching it in the API does not put a block device on the host. `is_agent_auto_iscsi_login_enabled` makes the Oracle Cloud Agent do the login, and it does that _asynchronously_, so the device is not there when cloud-init first runs. The mount script waits up to ten minutes for it, formats it **only if `blkid` reports no filesystem** so that a re-run cannot erase a populated volume, and writes an fstab entry by UUID:
```text
UUID=<uuid> /var/lib/spinifex ext4 defaults,_netdev,nofail 0 2
```
`_netdev` orders the mount after the network, and therefore after `iscsid` has a session. `nofail` keeps a missing volume from wedging the boot. Mounting `/var/lib/spinifex` _itself_ rather than somewhere else and symlinking is deliberate: viperblock, predastore and JetStream all land on the volume with no configuration pointing anywhere unusual.
## Every Terraform variable
All of these have defaults that build a working single node, except the first:
| Variable | Default | What it decides |
| ----------------------------- | --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `compartment_ocid` | none | **Required.** An OCID, not a name: this deploys into a compartment that already exists, because creating one needs tenancy-root rights the API user must not have |
| `deployment_name` | `spinifex` | Prefixes every resource and becomes the VCN `dns_label`, so 1–15 lowercase alphanumerics starting with a letter |
| `node_count` | `1` | Nodes, 1–25. `1` for a single node, `3` for a cluster. Each gets its own volume, its own second VNIC and its own fault domain |
| `compute_shape` | `VM.Standard.E6.Flex` | **`BM.*` for bare metal, `VM.*.Flex` for a VM.** A fixed bare-metal shape carries its own size, so `shape_config` is emitted only when the name contains `Flex`. Set a BM shape and the two sizing variables below are ignored rather than rejected |
| `compute_ocpus` | `8` | OCPUs on a flex shape. Validated at ≥ 8 (16 vCPU), the minimum recommended for a node |
| `compute_memory_in_gbs` | `32` | GiB on a flex shape. Validated at ≥ 32 |
| `data_volume_size_in_gbs` | `256` | Size of each node's data volume. Guest disks, S3 objects and JetStream all live here, so size it for the guests rather than for the control plane |
| `data_volume_vpus_per_gb` | `120` | Performance tier, 0–120 VPUs/GB. 120 is Ultra High Performance; 10 is Balanced. Higher is more IOPS and more cost |
| `data_mountpoint` | `/var/lib/spinifex` | Where cloud-init mounts it. Change only if you know why |
| `vcn_cidr` | `10.200.0.0/22` | VCN address space. A `/22` leaves room for the secondary private IPs Spinifex registers per node |
| `public_subnet_cidr` | `10.200.0.0/23` | The subnet every node sits in, **on both VNICs**. Nodes serve the UI, the S3 gate and the AWS gateway directly, so they cannot live in a private subnet |
| `node_client_cidr_allow_list` | `["0.0.0.0/0"]` | OCI security-list ingress, **all protocols**, for the whole public subnet. Guest public addresses cross it too, so read [Firewalls](#firewalls-in-all-three-layers) before narrowing it |
| `node_service_ports` | `[3000, 8443, 9999]` | Console, predastore gate, AWS gateway, opened in the host's iptables by cloud-init. SSH comes from the image's own rule |
| `instance_principal` | unset | `"adopt"` to authenticate the allocator as the instance rather than with a key file. Needs the tenancy dynamic group and policy to exist |
| `ubuntu_version` | `26.04` | Canonical platform image version. The newest image for the shape and region is resolved at plan time rather than pinned to a regional OCID |
| `wan_bridge_name` | `br-wan` | The Linux bridge built over the second VNIC |
| `wan_bridge_mtu` | `9000` | The OCI VCN carries 9000 |
**Six more exist and are not yours to set**: `tenancy_ocid`, `user_ocid`, `fingerprint`, `private_key`, `private_key_path` and `region`. `scripts/oci_env.py` fills them as `TF_VAR_*` from the profile in `~/.oci/config` on every `terraform` call the driver makes, so **the region you deploy into is the profile's region** and the credential never reaches a `.tfvars` file. Setting them by hand works, and is how a run with no config file at all would have to work, but then two sources disagree about which tenancy this is.
The profile is resolved as `--oci-profile`, then `$OCI_CLI_PROFILE`, then the reference tenancy, then `DEFAULT`. A name passed explicitly and absent from the config is an error that lists the profiles present; only an unasked-for name falls through. CI takes a different route entirely, reading `OCI_TENANCY_OCID`, `OCI_USER_OCID`, `OCI_FINGERPRINT`, `OCI_PRIVATE_KEY` and `OCI_REGION` from the environment so a runner writes no key to disk. A partial set there is an error rather than a fallback, because half a credential means a secret failed to reach the job.
## The `oci` CLI
**The `oci` CLI is not on the Ubuntu image**, and `apt` has no `python3-oci-cli` package. Nothing in the deployment needs it; install it when you want to inspect allocations by hand:
```bash
sudo apt-get update && sudo apt-get install -y python3-venv
python3 -m venv ~/.oci-cli-venv
~/.oci-cli-venv/bin/pip install --upgrade pip oci-cli
sudo ln -sf ~/.oci-cli-venv/bin/oci /usr/local/bin/oci
oci --version
```
**What the CLI is and is not for.** Spinifex's daemon uses the OCI **Go SDK** for every allocation, so the CLI is not on that path at all. You need it for bootstrap and for operational inspection. Do not build automation that shells out to it during an allocation: allocations run inside a request budget that a forked Python process will not respect.
---
## Deploying without Terraform
**Terraform automates every step of a normal install, and that is the point of it.** The `oci-spx` configuration builds the VCN, the subnets, the gateways, the block volumes and both VNICs per node, then renders the cloud-init that installs Spinifex and sets up OVN. The sequence below happens without anyone typing it, in the same order every time, on one node or three.
If you do need to install by hand, whether for an existing OCI tenancy you cannot run Terraform against, an unsupported shape, or a stage you are debugging, the install itself is not OCI-specific. Follow the standard guides:
- [Installing Spinifex](../../install/install/README.md) for a single node.
- [Multi-Node Installation](../../install/install-multi-node/README.md) for a cluster, which covers `spx admin init` on the leader and `spx admin join` on the rest.
Four things about OCI are not in those guides, and each has its own section above:
- **The external pool** is `source = "oci"`, which needs the API key or instance principal from [Credentials](#credentials-an-api-key-or-an-instance-principal) and the [IAM policy](#the-iam-policy). Omit it and you get the silent failure that section opens with: a healthy node that cannot hand out an address.
- **`br-wan` stays a Linux bridge** owned by netplan, linked to OVS `br-ext` by a veth pair. Pass `setup-ovn.sh --wan-bridge=br-wan` and it detects this itself. The NIC never becomes an OVS port, so the VNIC keeps its MAC identity. See [How it fits together](#how-it-fits-together).
- **The block volume must be mounted with `_netdev`**, because it arrives over iSCSI and the mount is otherwise attempted before a session exists.
- **IMDS needs the 169.254 remap** described under [Guests cannot reach instance metadata](#guests-cannot-reach-instance-metadata), since the guest's metadata address collides with OCI's own.
---
## Troubleshooting
### If a deploy stage fails
The driver stops at the first failure, names the log, and leaves the rest alone. Logs are all under `.validate-<topology>/`:
| Stage | Log |
| ----------------------------------------------- | ---------------------------------------- |
| Building the infrastructure | `apply.log` |
| Installing Spinifex | `install-<host>.log` |
| Forming the cluster | `form.log` |
| Installing the credential, configuring the pool | `credential-hook.log`, `pool-<host>.log` |
| Verifying public addressing | `allocator-<host>.log` |
| The Terraform workbooks | `workbooks.log`, `workbooks.tsv` |
The allocator log is the one worth knowing about, because public addressing is the part with the most moving pieces. If the allocator never comes up, the driver captures the daemon journal, the pool as configured, the bridge and OCI MAC addresses, and whether each credential file exists and is readable, never the credentials themselves, before anything is torn down.
Add `--keep-on-fail` to leave a failed deployment running so you can log in and look. **It keeps billing** until you run `--destroy-only`.
### Addresses and quotas
**Spinifex enforces no limit of its own.** It attempts the allocation and reports what OCI says, so the ceiling you hit is always your tenancy's and never a number baked into the software. A quota is raised with Oracle, not reconfigured here, and there is nothing to restart afterwards.
**`InsufficientAddressCapacity` on `allocate-address` is what exhaustion looks like.** It is the AWS error code for "out of addresses", and Spinifex returns it for either quota, mapped from OCI's own refusal. On a cluster the regional one is the likelier of the two.
**A detached reserved public IP still bills.** `disassociate-address` returns the address to `AVAILABLE` without deleting it, exactly as an unassociated AWS Elastic IP behaves, and it keeps consuming your regional quota as well as your bill. `release-address` frees both.
**A stopped instance costs nothing for its auto-assigned address.** Stopping an instance returns its auto-assigned address to the pool and deletes the reserved public IP object behind it, as AWS does, and the start that follows creates a new one. An **Elastic IP** is yours and survives the stop untouched. That is the whole distinction between the two, and on OCI it is also the difference between a stopped fleet that bills for addresses and one that does not.
### A guest's public address is unreachable
**Check where you are probing from first.** A node is a poor vantage point for a public address, for two independent reasons: the security list may not admit the node's own public address as a source, and a VCN may or may not reflect its public addresses back into itself. Either one fails the probe on a perfectly healthy cluster. Test from outside the VCN, which is where a customer is.
Then, in order:
1. `sudo ovn-nbctl lr-nat-list <router>` for a `dnat_and_snat` row, holding the _private_ address.
2. `ip route get <private-addr>` to confirm it routes via the transit veth.
3. `sudo iptables -S FORWARD | grep spinifex-eip-ingress` for both directions.
4. `oci network private-ip list --vnic-id <vnic>` to confirm the private address is registered with OCI, **and on this node's VNIC**. If it is not, nothing else matters.
**Egress works from one address and not another on the same NIC.** OCI enforces the source address per VNIC, so that means the address is not a registered private-IP object on it. It is never a routing problem on the host.
**An address went dark after a guest moved node.** The affinity pass should have claimed it within 15 seconds. Check the new node's daemon journal for `ocinet affinity`, then use step 4 above to confirm which VNIC OCI thinks holds the private half.
### A node that was fine and then was not
Terraform's cloud-init does the host jobs on each node: the iSCSI volume, `br-wan` and its source routing, the firewall. When one of them did not take, the symptom arrives much later and rarely looks like a networking or storage problem. This one command answers all of it:
```bash
ssh -i ~/.ssh/oci-spx ubuntu@<node-public-ip> '
cloud-init status # done
sudo iscsiadm -m session # a session to 169.254.2.2:3260
findmnt /var/lib/spinifex # mounted by UUID, with _netdev
ip -br addr show br-wan # UP, second VNIC address as a /32
ip -br addr show enp1s0 # the bridge MEMBER holds NO address
ip route show default # still on the PRIMARY interface
ls -l /dev/kvm # present
'
```
Three answers are worth knowing in advance, because each looks wrong and is not:
- **`br-wan` holds a `/32` and the default route is on the other interface.** That is how two VNICs share one subnet without the second stealing the first one's traffic, and it is what satisfies OCI's per-VNIC source check. `br-wan`'s address appears in `ip rule` instead of in the default route.
- **`br-wan`'s address is not one you chose.** It is the second VNIC's own private IP, assigned by OCI, and not predictable before the apply.
- **`enp1s0` carrying an address is the one real fault here.** The kernel derives an on-link route from it that beats the primary VNIC's, so every packet to another node leaves the wrong VNIC and OCI drops it. The node still forms a cluster and still reports `Ready`, because formation uses outbound connections, and it surfaces later as `predastore … the stripe is short one holder` on the first AMI import. `sudo /usr/local/sbin/spinifex-setup-wan-bridge` repairs it in place.
### The allocator
**Confirm it came up**, which `spx get nodes` cannot tell you:
```bash
sudo journalctl -u spinifex-vpcd --since -5m | grep -i ocinet
```
```text
"msg":"ocinet resolved the external VNIC from its interface","pool":"oci-public","iface":"br-wan",
"vnic_id":"ocid1.vnic.oc1.ap-sydney-1.abzxsljrjwlf...kimuq","private_ip":"10.200.1.31","subnet":"10.200.0.0/23"
"msg":"ocinet credential authorised to allocate","pool":"oci-public",
"vnic_id":"ocid1.vnic.oc1.ap-sydney-1.abzxsljrjwlf...kimuq","compartment":"ocid1.compartment.oc1..aaaa"
"msg":"OCI allocator ready","pool":"oci-public","collected":0,"stale_bindings":0,"skipped":1
```
**`credential authorised to allocate` is the line that matters, and `resolved the external VNIC` is not.** The authorisation line is logged after the node has made one real read in each permission class an allocation needs, so it proves the credential is accepted, the compartment is right and every verb in the policy is held. The resolution line above it comes from the instance metadata service and needs no IAM policy at all, so a node with a policy missing a verb logs it and still refuses every allocation.
The refusal has its own line, which names the verb:
```text
"msg":"ocinet credential cannot allocate an external address","pool":"oci-public",
"error":"reading subnet ocid1.subnet... failed, which needs `use subnets` in compartment ocid1...;
AssignPrivateIP requires SUBNET_ATTACH from that same verb: ... NotAuthorizedOrNotFound"
```
On a cold cluster start you may instead see `OCI allocator reconcile failed … nats: no responders available for request` on some nodes. That is vpcd racing JetStream's KV at boot, where the startup reconcile is skipped and not retried. It is harmless on a new cluster where nothing has been allocated, and it is tracked; restart `spinifex-vpcd` on that node to run it.
### Host
**`RunInstances` returns `ServerInternal`.** Check the daemon journal for the real error with `journalctl -u spinifex-daemon --since -10m`. An OCI allocation failure surfaces this way.
**The stack ends up on the boot disk and the node fills.** The data volume was mounted without `_netdev` and lost the race with `iscsid`. `findmnt /var/lib/spinifex` is the check, and `ls` cannot tell the difference.
### Guests cannot reach instance metadata
**Guests boot but cloud-init hangs on metadata**, or the host stops answering its own. OCI uses `169.254.169.254` for its instance metadata, and so does every AWS-compatible guest, so the host and the guests want the same address.
Spinifex resolves this by moving the _host_ side of the per-guest metadata link to a different link-local pair, leaving guests to ask `169.254.169.254` exactly as they do on AWS. The deploy sets `imds_host_meta_ip` and `imds_host_dns_ip` for you. **Set both or neither**, because a half-configured pair is ignored and the node behaves as if the remap were off, which is what this symptom looks like.
Check it after your first guest launches:
```bash
ip route get 169.254.169.254 # your real NIC, not lo
curl -sH 'Authorization: Bearer Oracle' -o /dev/null -w '%{http_code}\n' \
http://169.254.169.254/opc/v2/instance/ # 200
ip -br addr show | grep ime- # holds 169.254.42.x, not .169.x
```
`ip route get 169.254.169.254` returning `dev lo` is the signature of a missing remap. The same signature can appear after a guest terminates, from a leftover `ime-` endpoint still holding the address; with the remap configured that is harmless, and removing the stale OVS port clears it.
### An OVS problem, when `ovs-vsctl` says nothing is wrong
`ovs-vsctl` exits 0 even when a bridge could not be instantiated. Read the error column and the log, never the exit status:
```bash
sudo ovs-vsctl --columns=name,error list Interface
sudo journalctl -u ovs-vswitchd --since -10m
```
docs/terraform-workbooks/eks-quickstart/README.md
+1 -1
View File
@@ -66,7 +66,7 @@ Under the hood it keeps things minimal: a VPC with two public subnets, an IAM ro
- The cluster's security groups are **auto-managed** — `vpc_config.security_group_ids` is ignored. To reach a NodePort, this workbook looks the worker SG up by its deterministic name (`eks-cluster-<name>-nodegroup-sg`) and adds a single ingress rule.
- The worker AMI is always Spinifex's `eks-node` image; `ami_type` is recorded but does **not** select the image.
- `authentication_mode` must be `"API"` (the `API_AND_CONFIG_MAP` mode is rejected).
- `authentication_mode` can be `"API"` or `"API_AND_CONFIG_MAP"`, and bare `"CONFIG_MAP"` is rejected. Access is granted through access entries, as there is no `aws-auth` ConfigMap path.
- The workers need outbound internet (here via the IGW) to pull the demo container image.
**Prerequisites:**
go.mod
+24 -24
View File
@@ -4,33 +4,33 @@ go 1.27.0
require (
github.com/aws/aws-sdk-go v1.55.8
github.com/aws/aws-sdk-go-v2 v1.47.0
github.com/aws/aws-sdk-go-v2 v1.47.1
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20
github.com/aws/aws-sdk-go-v2/config v1.33.5
github.com/aws/aws-sdk-go-v2/credentials v1.20.5
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0
github.com/aws/aws-sdk-go-v2/config v1.33.6
github.com/aws/aws-sdk-go-v2/credentials v1.20.6
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1
github.com/cenkalti/backoff/v4 v4.3.0
github.com/charmbracelet/bubbles v1.0.0
github.com/charmbracelet/bubbletea v1.3.10
github.com/charmbracelet/lipgloss v1.1.0
github.com/containerd/containerd/v2 v2.4.0
github.com/containerd/containerd/v2 v2.4.1
github.com/distribution/reference v0.6.0
github.com/go-chi/chi/v5 v5.3.2
github.com/golang-jwt/jwt/v5 v5.3.1
github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82
github.com/jackc/pgx/v5 v5.11.0
github.com/klauspost/compress v1.20.0
github.com/klauspost/compress v1.20.1
github.com/klauspost/cpuid/v2 v2.4.0
github.com/miekg/dns v1.1.73
github.com/minio/highwayhash v1.0.4
github.com/mulgadc/bluebottle v1.21.1-0.20261001020941-036da4967e1c
github.com/mulgadc/northstar v1.21.0
github.com/mulgadc/predastore v1.21.0
github.com/mulgadc/viperblock v1.21.0
github.com/mulgadc/bluebottle v1.22.0
github.com/mulgadc/northstar v1.22.0
github.com/mulgadc/predastore v1.22.0
github.com/mulgadc/viperblock v1.22.0
github.com/nats-io/nats-server/v2 v2.15.0
github.com/nats-io/nats.go v1.54.0
github.com/opencontainers/runtime-spec v1.3.0
github.com/oracle/oci-go-sdk/v65 v65.126.0
github.com/oracle/oci-go-sdk/v65 v65.126.1
github.com/ovn-kubernetes/libovsdb v0.8.1
github.com/pelletier/go-toml/v2 v2.4.3
github.com/pterm/pterm v0.12.83
@@ -48,6 +48,7 @@ require (
golang.org/x/crypto v0.57.0
golang.org/x/sync v0.23.0
golang.org/x/sys v0.48.0
golang.org/x/term v0.46.0
golang.org/x/time v0.16.0
golang.org/x/tools v0.50.0
gopkg.in/ini.v1 v1.67.3
@@ -61,18 +62,18 @@ require (
github.com/Microsoft/hcsshim v0.15.0-rc.4 // indirect
github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
github.com/atotto/clipboard v0.1.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.3 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.113.3 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.51.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 // indirect
github.com/aws/smithy-go v1.28.2 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
@@ -177,7 +178,7 @@ require (
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tidwall/btree v1.8.1 // indirect
github.com/tidwall/btree v1.8.2 // indirect
github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
@@ -197,7 +198,6 @@ require (
golang.org/x/mod v0.41.0 // indirect
golang.org/x/net v0.59.0 // indirect
golang.org/x/telemetry v0.0.0-20260908163034-4bcc4b2ee518 // indirect
golang.org/x/term v0.46.0 // indirect
golang.org/x/text v0.42.0 // indirect
golang.org/x/vuln v1.6.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260825221802-da73d73af1c5 // indirect
go.sum
+46 -46
View File
@@ -30,40 +30,40 @@ github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
github.com/aws/aws-sdk-go-v2 v1.47.0 h1:0jsHallhJCeaU0Ko48c/3FK1ctOQ7NpzggxriJOQ8MQ=
github.com/aws/aws-sdk-go-v2 v1.47.0/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU=
github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw=
github.com/aws/aws-sdk-go-v2 v1.47.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8=
github.com/aws/aws-sdk-go-v2/config v1.33.5 h1:UA1dmokBFOLFoOyVBhO6HjM6edy0MIk5AZSkJVcksQw=
github.com/aws/aws-sdk-go-v2/config v1.33.5/go.mod h1:Dop8axzz0xx38GExIYWXdeyc8QQ7Cr+nPsxpD/LYy4U=
github.com/aws/aws-sdk-go-v2/credentials v1.20.5 h1:wklUVvHMc9xTQ3rcp49/ISpiMnhbCicJcA6n6S8m7J8=
github.com/aws/aws-sdk-go-v2/credentials v1.20.5/go.mod h1:fyEdrn6ccLFOkoK84j5bQyGTxp9zPt5l2XMhxf4DVZs=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0 h1:AM4hHjww+PSFtt6E+UrBrPlZkWsePCLEt9AjkfQX+yM=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0/go.mod h1:3x/yXezeQjpOvBb4jEMxrS8SXvpdvJ5abv6l5c1gWM8=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 h1:Hp/VgjP0BysR3OgLlR057Vz2LcbbVnoWeJ+3qWiS/fY=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3/go.mod h1:nwGV5qw7F1IZPgxCvA/ph8N2TAuz+BkRG/bXn808qMA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 h1:MUaM4f+kj1ZIBPZfUS8cxP1GKXXZtHJjAthy93AN7SM=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3/go.mod h1:6YmVmEVRI5ZZzRjCSsb9SryKH0hAlMRdgA7kG9aDvBU=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 h1:fuSCw4Z2qfRCztMPO3GXJNSiEp6Wee+WOLwrHHUMy9c=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3/go.mod h1:6SxcHheD1pPR5+kWm1wGvjlL/YqUsh267sAfEmN4K7A=
github.com/aws/aws-sdk-go-v2/config v1.33.6 h1:MBjkSTLczek/UgiK+EYPIoRTqE7gP8vtW3OFbFo7Nug=
github.com/aws/aws-sdk-go-v2/config v1.33.6/go.mod h1:grRAFzdAZJrwcbasJRg2MPvIrVjtlfXllHssN6+E1JE=
github.com/aws/aws-sdk-go-v2/credentials v1.20.6 h1:NpAFXCU7NzXNkdGK3zQTtsRJ+3v9tZQV0xcdRw8uBdw=
github.com/aws/aws-sdk-go-v2/credentials v1.20.6/go.mod h1:mcZCoiPnyMvP8VMNbygNX5lLqSlkYJIMPODylQMurOk=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 h1:8gALAAmacnIXh+z6VkdDanv4/IkG5APdg4DZLDTmLog=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1/go.mod h1:Z7IJhJU+poOdJjUR2wpyY21ossQ1XS/R3Lk9Msq5kM4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 h1:CLq4+8UHCI+ZZYl/EuJxXovaIVN2xeeT8JV+dsApQ5E=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4/go.mod h1:Wv4q5sAM04xAMkoOedxLx2inVf6K5FdxYp+A61L+q/0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP9rocC2QnT3qVuXwzlYTtfGEs=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4/go.mod h1:EcXV1kAFd5XwSkDHlj94gnF3q5CkJyYiIJfH8N0VmrE=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 h1:7Wo47d/xn/7KttCSBd8EGYeZ7ULRFRkUHr6vkZPBzVQ=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4/go.mod h1:tDB2IVC1xC3vX8o+6uRlzhTxP3g1b77CZXFX/oD2FnQ=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.4 h1:7TXEkbDzy4BkYYTfTVjKcADTfkDnvHxDwcF1pJG1yF0=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.4/go.mod h1:GqWeeKfYfezihA2KfFL9l7ohEdZWe1tuFWh3GfyNSnE=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 h1:bON1rJf67TSTDCKg816AAIE4xSTtoo9tl0XRkO72R+I=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3/go.mod h1:c5BBpjJcQXpfeq9iASyVKA3T6vX6B6LEXY4mL/gklDY=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.3 h1:L8vIOxylma91TcR96NFTEC07G3JDwSl+CvK2b+IODms=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.3/go.mod h1:fmPIZQzTExYuBNWFyi1P7IoDjvskgphXqK1yObMzusM=
github.com/aws/aws-sdk-go-v2/service/s3 v1.113.3 h1:IDIchqG5C/o/JdprtYn1NirCi7iUx7XQ8+WwZ6odFX8=
github.com/aws/aws-sdk-go-v2/service/s3 v1.113.3/go.mod h1:iJ69H4lkK6a9zQ+L5i9pDERMok5Jvts0iZaMjWEi/78=
github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 h1:ZD5qFpWcaOKdTuhBi431pIDkCgrMkMlMT6jlpSPoIRI=
github.com/aws/aws-sdk-go-v2/service/signin v1.10.0/go.mod h1:8Nuuf+tR346PjJ3MvZPh9pekbLiLQFWJhzMXfwy7alA=
github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 h1:JGeeBcMlhg1xtOXYpeCaTQBZObtXMPQCUqBcmr65NRA=
github.com/aws/aws-sdk-go-v2/service/sso v1.38.0/go.mod h1:XwteswG9EOMRFm73UT0t+MbTwyLxMrEXkU6e+v92Lzo=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 h1:obhahQXDEdVEv8y5bTKXR30LVaxYe1kyYM0L7l2Iq+k=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0/go.mod h1:6twZZ/aXHNy1vXUO8koUbp++MYzMASkOgEBdkbJYmO0=
github.com/aws/aws-sdk-go-v2/service/sts v1.51.0 h1:Zpnqa6XtrNzXZnwbdCqHOXpXhMsa01ql/pcRQ1sb4hk=
github.com/aws/aws-sdk-go-v2/service/sts v1.51.0/go.mod h1:/8JRcdTt//hG0Q4BTmGbuOplT7ABe+5rdtqUHqXvYIM=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5/go.mod h1:qPqp1Uwd/BqdhPufv6oem9j5J7HNsgc2V22dUiDPn+s=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 h1:29SvnfGhXjTl8ONxFwbj2rs6lbhiFXD2CgFQmbT/bXY=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4/go.mod h1:wm04I5DMuNVvZHFe/dHnUxincvNbbK7AiNBbYsQivek=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 h1:VMAdYqr4Jn/8ATs9BHC5riwrs0d6m1Z2ohFriSwZwm0=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI=
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 h1:DzCCWLzcIRQ77F3DEUljud7bEjTgFOIKXP52NmVRyhU=
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1/go.mod h1:xpo/geVldu8payT375WekctUzopG/hBU7miiqItMUlw=
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 h1:Umtl/0YZhng4xndfW3lKJrYYP7NLEjI6bGXVomwLcs0=
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1/go.mod h1:rRD/dnm7q0HYE/I5TMaPgkWyyUGLcwuxHLABsLnQ3e0=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 h1:orIWdNiLgzrhu/11RcPPKO/SBzUUymbUQuZbSPImghg=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1/go.mod h1:skwM/xsbR/1ReUTesv9BhpJp1VjajR7DWQnuVLwiXsQ=
github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 h1:0HOqZXRvMytH6bFHVIc0oJX07sZjfhz0zXtjs6gdE8s=
github.com/aws/aws-sdk-go-v2/service/sts v1.51.1/go.mod h1:26zA0GhDrLo+yiLI2yXWxqB1PdsShfLikoI7GOEgugM=
github.com/aws/smithy-go v1.28.2 h1:myhcykQcatTul2B/zITjDk203G7t0awUAs1hVry5Bvg=
github.com/aws/smithy-go v1.28.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
@@ -111,8 +111,8 @@ github.com/containerd/console v1.0.5 h1:R0ymNeydRqH2DmakFNdmjR2k0t7UPuiOV/N/27/q
github.com/containerd/console v1.0.5/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk=
github.com/containerd/containerd/api v1.12.0 h1:kuQm82SbDrCuO4n7hf2L8zsBtZLuympyq5X/VotfX2A=
github.com/containerd/containerd/api v1.12.0/go.mod h1:EBcSzoi9Vl18cdODaXUCskf3D2NT8lsSXeZJnU5jIUc=
github.com/containerd/containerd/v2 v2.4.0 h1:mZLYWkPAgF4tfTjinClrRfvUxmxHxg1PQL7cjrrpTtk=
github.com/containerd/containerd/v2 v2.4.0/go.mod h1:gHZz+v5y8mGt9grF3ynuaa3r6bXLmghBZWTAftHAUtg=
github.com/containerd/containerd/v2 v2.4.1 h1:DUx/ZJN7cEu0WuzHClDB+68H/bqMEH5pWoEjf0ae4hc=
github.com/containerd/containerd/v2 v2.4.1/go.mod h1:vgLdtvl3prFk1d3ZVqQcsDD5Q9IKM+vAIdU54U85w+I=
github.com/containerd/continuity v0.5.0 h1:7a85HZpCSs+1Zps0Ee3DPSuAWY+0SJM1JNM51nlEVDg=
github.com/containerd/continuity v0.5.0/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
@@ -251,8 +251,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC
github.com/josharian/native v1.0.1-0.20221213033349-c1e37c09b531/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA=
github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
@@ -312,14 +312,14 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
github.com/mulgadc/bluebottle v1.21.1-0.20261001020941-036da4967e1c h1:oRb9PlvooMenQjKU9dD76/bxUOvJaYi6taxm3famoj4=
github.com/mulgadc/bluebottle v1.21.1-0.20261001020941-036da4967e1c/go.mod h1:LH0MuQcFxet0dDEISlZsuULIYj592eCN2Fv39A6EH5o=
github.com/mulgadc/northstar v1.21.0 h1:LE/5kiT3kBelCg02cXa3b7awhaYkGHsOXeJ4nHRLNaQ=
github.com/mulgadc/northstar v1.21.0/go.mod h1:O446cApNMB6qebqieBhDZfCIGVWX++v3QdOz+wIGhUk=
github.com/mulgadc/predastore v1.21.0 h1:pAm4fFWf0vnlwRgnoL2oJ+KSCw8mtjT2rQoBHFmnjzk=
github.com/mulgadc/predastore v1.21.0/go.mod h1:icCfKGnyqDaJ/Jlbp2aYeo2qwrKHZfxmPHDYK1MM7wc=
github.com/mulgadc/viperblock v1.21.0 h1:RbQ/GpHCg/uoPffYioSKqzqyoz22Uc7wpQ7XY7etLg0=
github.com/mulgadc/viperblock v1.21.0/go.mod h1:n0wCOGV58Lt2IVPXEvpbHDkTITlXtFw+2CMrQXgbJ7s=
github.com/mulgadc/bluebottle v1.22.0 h1:pGZ5/Ly3CYem62pPu2zKBRpvMpAG0LHMIMw9Yt/lkIg=
github.com/mulgadc/bluebottle v1.22.0/go.mod h1:DwT91bJyUsrpg1gCJmcCOTG+tgw5J124LloHYvGAQu0=
github.com/mulgadc/northstar v1.22.0 h1:fTndlk47hK76fcNr2MqsyeKhBNVEDt8+BC6Z30ohokA=
github.com/mulgadc/northstar v1.22.0/go.mod h1:yLH8waqfHZx177Ae6dGQGTDt/02rPE9alnUKyxDNhVs=
github.com/mulgadc/predastore v1.22.0 h1:mUQJMTxMiRDVvemMP+34BP/+r1U0iCptc9B++u4NJ8o=
github.com/mulgadc/predastore v1.22.0/go.mod h1:/cMIoKtnQG/DWyeX/0matEQrG2/abY60tNHWcBKS9Kc=
github.com/mulgadc/viperblock v1.22.0 h1:8vS2AtpIDkIhGNOazQPicKXV2y/8PAy8BNrVWIOCP6I=
github.com/mulgadc/viperblock v1.22.0/go.mod h1:Hq8yMagAX8c4wZ68oUkwZPyxTNMwvRwB685eHFewisc=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/nats-io/jwt/v2 v2.8.2 h1:XXRgB60MSTnqsRwejQurVDs/hcv2dkt+86GjI+I/bMc=
@@ -338,8 +338,8 @@ github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJw
github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg=
github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
github.com/oracle/oci-go-sdk/v65 v65.126.0 h1:RuV0MEcLOOgNOBadYbbkUQriCK4Gm5348F/GdWvYPcI=
github.com/oracle/oci-go-sdk/v65 v65.126.0/go.mod h1:Pzy+BpgkDesvGZXEHgslwhIYobHCPHg6wRta1mWnlqQ=
github.com/oracle/oci-go-sdk/v65 v65.126.1 h1:WmQ2Igq7/L/cHlR2jZnJkBC2UI51Ams4DJZW3CZGVBo=
github.com/oracle/oci-go-sdk/v65 v65.126.1/go.mod h1:YmvgbsnUfrsiJ410XAGQ1BRsrUYJJ7W3O5wUikmeI8w=
github.com/ovn-kubernetes/libovsdb v0.8.1 h1:M2J8bcJt5mXCom0HqzfEtuHkT80CTSQRcYG7acT8gf4=
github.com/ovn-kubernetes/libovsdb v0.8.1/go.mod h1:ZlnHLzagmLOSvyd9qfxBIZp6wOSOw0IsRsc+6lNUGbU=
github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
@@ -417,8 +417,8 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/tidwall/btree v1.8.1 h1:27ehoXvm5AG/g+1VxLS1SD3vRhp/H7LuEfwNvddEdmA=
github.com/tidwall/btree v1.8.1/go.mod h1:jBbTdUWhSZClZWoDg54VnvV7/54modSOzDN7VXftj1A=
github.com/tidwall/btree v1.8.2 h1:mTY5iM4UTh2e80TG7RHDQQmnAY3xGNQA/DbU0olkhkA=
github.com/tidwall/btree v1.8.2/go.mod h1:jBbTdUWhSZClZWoDg54VnvV7/54modSOzDN7VXftj1A=
github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923 h1:tHNk7XK9GkmKUR6Gh8gVBKXc2MVSZ4G/NnWLtzw4gNA=
github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923/go.mod h1:eLL9Nub3yfAho7qB0MzZizFhTU2QkLeoVsWdHtDW264=
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
internal/guestenv/guestenv.go
+8 -5
View File
@@ -10,11 +10,13 @@ import (
"strings"
)
// Loader holds the KEY=value settings read from an agent.env file, keyed by
// variable name. Read values through Get so the process environment applies.
type Loader map[string]string
// A missing or unreadable file yields an empty Loader rather than an error: the
// settings are all optional, and a guest whose file never arrived must still
// start far enough to report why.
// Load parses the KEY=value file at path, skipping blanks and # comments. A
// missing or unreadable file yields an empty Loader rather than an error: the
// settings are optional, and a guest without its file must still report why.
func Load(path string) Loader {
out := Loader{}
f, err := os.Open(path)
@@ -38,8 +40,9 @@ func Load(path string) Loader {
return out
}
// A real environment variable wins, so an operator or test can override what
// cloud-init wrote. An empty value is treated as unset rather than as a mask.
// Get returns the setting for key. A real environment variable wins, so an
// operator or test can override what cloud-init wrote. An empty value is
// treated as unset rather than as a mask.
func (l Loader) Get(key string) string {
if v := os.Getenv(key); v != "" {
return v
internal/rdsgw/client.go
+9 -6
View File
@@ -37,7 +37,8 @@ const (
defaultTimeout = 40 * time.Second
)
// One client is reused for register/heartbeat/bootstrap/poll.
// Client sends SigV4-signed RDS Query calls to the gateway. One client is
// reused for register/heartbeat/bootstrap/poll.
type Client struct {
baseURL string
signer *gwsign.Signer
@@ -45,8 +46,9 @@ type Client struct {
httpClient *http.Client
}
// caPath optionally pins the gateway TLS CA; empty relies on the system trust
// store. region defaults to us-east-1, since SigV4 requires a non-empty one.
// New builds a TLS 1.3 client for baseURL. caPath optionally pins the gateway
// TLS CA; empty relies on the system trust store. region defaults to us-east-1,
// since SigV4 requires a non-empty one.
func New(baseURL, caPath string, signer *gwsign.Signer, region string, timeout time.Duration) (*Client, error) {
if baseURL == "" {
return nil, fmt.Errorf("rdsgw: baseURL is required")
@@ -88,8 +90,8 @@ func New(baseURL, caPath string, signer *gwsign.Signer, region string, timeout t
}, nil
}
// Code is the AWS error code, so a caller branches on the failure class rather
// than matching message text.
// APIError is a non-2xx gateway response. Code is the AWS error code, so a
// caller branches on the failure class rather than matching message text.
type APIError struct {
Action string
StatusCode int
@@ -113,8 +115,9 @@ type errorResponse struct {
} `xml:"Error"`
}
// Call POSTs action and decodes its <ActionResult> into out, which may be nil.
// params carries the action's own arguments; Action and Version are set here.
// out may be nil. A non-2xx yields an *APIError. No retry; callers wrap.
// A non-2xx yields an *APIError. No retry; callers wrap.
func (c *Client) Call(ctx context.Context, action string, params url.Values, out any) error {
form := make(url.Values, len(params)+2)
maps.Copy(form, params)
internal/testkit/ebsfake/ebsfake.go
+2
View File
@@ -1,3 +1,5 @@
// Package ebsfake provides an in-memory EBS provider for tests that also
// deletes a volume's objects from the store, as viperblockd does.
package ebsfake
import (
internal/testkit/nats.go
+2
View File
@@ -1,3 +1,5 @@
// Package testutil provides test helpers shared across Spinifex packages:
// embedded NATS and JetStream servers, stub responders and span recorders.
package testutil
import (
internal/testkit/tracing.go
+19 -4
View File
@@ -2,6 +2,7 @@ package testutil
import (
"testing"
"time"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
@@ -42,12 +43,26 @@ func SpanAttribute(t *testing.T, span sdktrace.ReadOnlySpan, key string) attribu
return attribute.Value{}
}
// SpansByKind returns the last recorded span of each kind. Owner-first
// routing can produce more than one client span for a single call, and the
// leg that answered is the last one.
// SpansByKind returns the last recorded span of each kind (owner-first routing
// can record several client legs; the last answered). Servers reply before
// ending their span, so this waits for one and fails the test if none ends.
func SpansByKind(t *testing.T, recorder *tracetest.SpanRecorder) (client, server sdktrace.ReadOnlySpan) {
t.Helper()
for _, span := range recorder.Ended() {
deadline := time.Now().Add(5 * time.Second)
for {
client, server = lastSpansByKind(recorder.Ended())
if server != nil {
return client, server
}
if time.Now().After(deadline) {
t.Fatal("no server span recorded within 5s")
}
time.Sleep(time.Millisecond)
}
}
func lastSpansByKind(spans []sdktrace.ReadOnlySpan) (client, server sdktrace.ReadOnlySpan) {
for _, span := range spans {
switch span.SpanKind() {
case trace.SpanKindClient:
client = span
scripts/setup.sh
+23
View File
@@ -1,6 +1,7 @@
#!/bin/bash
# Spinifex binary installer
# Usage: curl -sfL https://install.mulgadc.com | bash
# curl -sfL https://install.mulgadc.com | sudo bash -s -- --channel dev
#
# Environment variables:
# INSTALL_SPINIFEX_CHANNEL Release channel: latest (default), dev
@@ -897,6 +898,17 @@ install_files() {
info " /usr/local/share/spinifex/setup.sh"
fi
# Formation. Shipping these means a multi-node cluster can be formed from a
# `curl | bash` install with no checkout, which is what a cloud operator has.
# smoke-test.sh comes too: install-node.sh scp's it from its own directory for
# the post-formation verification, so without it formation fails at the last step.
for formation in install-node.sh smoke-test.sh; do
if [ -f "$EXTRACT_DIR/$formation" ]; then
$SUDO install -m 0755 "$EXTRACT_DIR/$formation" "/usr/local/share/spinifex/$formation"
info " /usr/local/share/spinifex/$formation"
fi
done
# Teardown scripts. An install delivered by `curl | bash` with no way to
# undo it is a gap in the product, not just in our tooling, and node-reset.sh
# ships alongside because uninstall-spx.sh delegates its state teardown to it
@@ -1390,11 +1402,22 @@ main() {
case "$1" in
--firewall=*) INSTALL_SPINIFEX_FIREWALL="${1#*=}" ;;
--firewall) INSTALL_SPINIFEX_FIREWALL="${2:-}"; shift ;;
--channel=*) INSTALL_SPINIFEX_CHANNEL="${1#*=}" ;;
--channel) INSTALL_SPINIFEX_CHANNEL="${2:-}"; shift ;;
--version=*) INSTALL_SPINIFEX_VERSION="${1#*=}" ;;
--version) INSTALL_SPINIFEX_VERSION="${2:-}"; shift ;;
*) fatal "unknown option: $1" ;;
esac
shift
done
case "$INSTALL_SPINIFEX_CHANNEL" in
latest|dev) ;;
# Anything else is passed to the install endpoint as a literal tag, so a
# typo would 404 on the download rather than here. Name the two channels.
*) fatal "--channel must be 'latest' or 'dev', got: ${INSTALL_SPINIFEX_CHANNEL} (use --version for a specific tag)" ;;
esac
case "${INSTALL_SPINIFEX_FIREWALL}" in
on|off) ;;
# Unset resolves by install path: the ISO arms, everything else does not.
scripts/terraform/oci-spx/.gitignore
+5
View File
@@ -8,3 +8,8 @@ terraform.tfstate.*
crash.log
terraform.tfvars
*.pem
*.log
.validate-*/
# Per-run artifacts and the identity state, which is not a topology's to destroy.
.e2e-oci-*/
.identity/
scripts/terraform/oci-spx/README.md
+73 -8
View File
@@ -31,7 +31,7 @@ Every resource is prefixed with `deployment_name` (default `spinifex`), so nothi
## Quick Start
Prerequisites: Terraform, Python 3, OCI credentials in `~/.oci/config`, and the public and private halves of the SSH key. The Python helper creates and uses this repository's isolated `.venv` and installs [requirements.txt](requirements.txt).
Prerequisites: Terraform, Python 3, OCI credentials in `~/.oci/config` or the environment, and the public and private halves of the SSH key. The Python helper is standard library only, so there is nothing to install: it once used the OCI SDK, which meant a virtualenv and therefore `python3-venv` on every host that runs it.
```bash
cd scripts/terraform/oci-spx
@@ -53,13 +53,11 @@ python3 scripts/oci_env.py --ssh-public-key-path <path_to_public_ssh_key> -- ter
| Item | Purpose |
| --- | --- |
| `scripts/oci_env.py` | Loads OCI profile inputs, discovers the tenancy home region, exports Terraform variables, and runs Terraform in the repository `.venv`. |
| `requirements.txt` | Python dependency set for the environment helper. |
| `scripts/oci_env.py` | Loads the credential from a profile or the environment, checks its shape, exports Terraform variables, and runs Terraform. |
| `terraform.tfvars.example` | Example non-secret Terraform inputs. Copy it to untracked `terraform.auto.tfvars` for local overrides. |
| `.venv/` | Repository-local virtual environment; generated locally and not committed. |
| `terraform.tfstate` | Terraform state; generated locally unless a remote backend is configured. Do not commit it. |
The helper defaults to OCI profile `apacanzset03child03`. If it is absent and `apacanzset03child3` is locally configured, the helper explicitly falls back to that profile. Override either setting when needed:
The helper reads the `DEFAULT` profile, which is what `oci setup config` writes. Set `OCI_CLI_PROFILE` or pass `--profile` to read a different one; a named profile the config does not hold is an error rather than a fallback, so a deployment cannot land in a tenancy nobody chose.
```bash
python3 scripts/oci_env.py --profile my-profile --region ap-sydney-1 -- terraform plan
@@ -125,7 +123,6 @@ flowchart TB
├── outputs.tf # OCIDs and policy outputs
├── versions.tf # Terraform and provider version constraints
├── terraform.tfvars.example
├── requirements.txt
└── scripts/oci_env.py # Python environment and Terraform command helper
```
@@ -208,9 +205,77 @@ The private subnet is **unused today** and kept only so it exists: Spinifex gues
node_client_cidr_allow_list = ["203.0.113.10/32"]
```
## The API user this seeds
## How each node authenticates to OCI
The OCI provider integration needs credentials on each node so Spinifex can allocate addresses at runtime. **Grant that user only the operations Spinifex performs on addresses** — create/get/list/delete private IPs, create/get/update/delete public IPs — and nothing else. It is not a tenancy admin. Broader rights widen the blast radius of a node compromise for no benefit.
The allocator needs OCI credentials at runtime, and there are two ways to give it them. A node with neither forms, passes every health check, and then refuses every launch that wants a public address with `InsufficientAddressCapacity` — the cause appears only in the node's journal, so this is worth getting right before first start.
**Instance principal is the better one.** Each node authenticates with the certificate its own metadata service serves, so no key material exists on any node, there is nothing to rotate, and nothing sensitive reaches Terraform state. The cost is a dynamic group and a policy, which are tenancy-root resources — so creating them needs a tenancy-admin principal, which is why the rest of this configuration deliberately creates nothing at tenancy root.
`instance_principal` has three values, because using an instance principal and being allowed to create one are different rights:
| Value | Pool auth | Creates the dynamic group and policy | Credential needed |
| --- | --- | --- | --- |
| `off` (default) | API key file | No | Compartment-scoped |
| `adopt` | `instance_principal` | No — assumes they exist | Compartment-scoped |
| `create` | `instance_principal` | Yes | **Tenancy admin** |
Create them once per tenancy, then every deployment and rebuild afterwards uses `adopt`:
```bash
./setup-identity.sh --dry-run # always first
./setup-identity.sh
```
`setup-identity.sh` targets only those two resources and keeps them in `.identity/terraform.tfstate`, separate from every topology's state. That separation is load-bearing: `validate-topology.sh` destroys its own state at the end of each run, so holding tenancy resources there would let a nightly teardown delete the tenancy's policy.
**`adopt` references the dynamic group by nothing at all.** Its matching rule is `instance.compartment.id`, so it covers every instance in the compartment and names no OCID — which is why adopting needs no read on an identity resource and no tenancy rights. The cost is that a missing policy is invisible at apply time: the node forms, passes every health check, and then refuses every launch wanting a public address. The allocator gate in `validate-topology.sh` is what catches that, by requiring `ocinet credential authorised to allocate` in each node's `spinifex-daemon` journal.
The policy grants four verbs in one compartment: `use vnics`, `manage private-ips`, `manage public-ips` and `use subnets`.
That is exactly what allocating an external address does and nothing more.
`use subnets` looks unrelated and is not: `CreatePrivateIp` is checked against `SUBNET_ATTACH` and `CreatePrivateIp` is how an address is registered, so a policy without it authorises nothing and every allocation returns a 404.
**An API key is the fallback**, and the default because it needs nothing from a tenancy admin. Grant that user only the operations Spinifex performs on addresses and nothing else; it is not a tenancy admin, and broader rights widen the blast radius of a node compromise for no benefit.
`spx-oci-config.sh` installs it on **every** node, as `/etc/spinifex/oci/{oci_api_key.pem,config}` — not a home directory, because the daemon's unit sets `ProtectHome=yes`. Both files are needed: `oci_config_file` names an ordinary OCI SDK config, so a node holding only the PEM forms and then fails every allocation. The profile it writes is `spinifex`, matching `oci_config_profile`:
```ini
[spinifex]
user=ocid1.user.oc1..<yours>
fingerprint=<yours>
tenancy=ocid1.tenancy.oc1..<yours>
region=ap-sydney-1
key_file=/etc/spinifex/oci/oci_api_key.pem
```
The script takes `validate-topology.sh`'s hook contract, `hook <ssh-key> <host>...`, so it is both the default thing to pass to `--credential-hook` and runnable on its own against an existing cluster. `--dry-run` resolves and validates the credential without touching a host.
It resolves the credential in four steps and takes the first complete one: the `OCI_SPX_*` environment variables, the `[spinifex]` profile in `~/.oci/config`, `~/.oci/oci_api_key_spx.pem` with the Terraform profile's identifiers, then the Terraform credential itself with a warning that it is wider than needed. Steps 2 to 4 go through `scripts/oci_env.py`, the same resolver the apply uses, so the node credential and the one that built the infrastructure cannot come from different profiles. The fingerprint is checked against the key before anything is written, because a mismatched pair is otherwise an OCI 401 at the first allocation.
Credential material reaches the nodes over SSH, in the remote shell's stdin rather than its arguments, and goes into neither user-data nor Terraform state. `--credential-hook` still accepts any executable with that argument shape, so a tenancy holding credentials in a vault can substitute its own. Instance principal needs no hook at all, which is the reason to prefer it wherever a dynamic group can be created.
Either way, Terraform stages the matching pool block at `/etc/spinifex/oci/external-pool.toml` on each node, with `oci_auth` set to match. Append it to `/etc/spinifex/spinifex.toml` after `spx admin init` and restart `spinifex.target`. Under instance principal that file holds no secret at all, which is the point of it.
## Validating a topology end to end
`validate-topology.sh` builds one topology from nothing, installs the published Spinifex release, forms the cluster, runs a Terraform workbook against it, proves the workbook serves traffic, and destroys everything. Three topologies, because each breaks differently — bare metal presents VNICs unlike a VM, a single node has no Geneve underlay to get wrong, and only a cluster exercises RAFT, the gateway chassis and cross-node allocation.
```bash
./validate-topology.sh --topology bm --dry-run
./validate-topology.sh --topology vm-single --instance-principal
./validate-topology.sh --topology vm-multi --instance-principal
```
`--topology` has no default on purpose: a command aimed at the wrong one is the easiest expensive mistake here. Each topology keeps its own state under `.validate-<topology>/`, so two can be built from one checkout without either destroying the other's instances, and every log from the run lands there.
**A topology's shape and node count are defaults, not settings.** They go to Terraform as `TF_VAR_compute_shape` and `TF_VAR_node_count`, which is the weakest source Terraform reads, so a `terraform.auto.tfvars` in the checkout outranks them and a deployment sizes itself in that file with no flag to pass. CI has no such file — `*.auto.tfvars` is gitignored — so a named topology stays the same every run. `instance_principal` is the exception and remains a `-var`, because `--instance-principal` is a choice about the run rather than about the infrastructure's size, and a stale tfvars must not quietly contradict it.
The count that later gates read is the number of addresses the `hosts_file` output names, not the number asked for, and the run logs the shape and count it built. A tfvars that changes either is reported rather than silently diverging from the topology's name.
**The teardown decides the verdict.** A topology or workbook that cannot be destroyed is half proved, and has been a real defect before, so `destroy` runs from an `EXIT` trap even on failure and a teardown failure fails the run. `--keep` leaves everything up and records no verdict. Other flags: `--skip-workload` (form and verify, launch no guests), `--workbook NAME`, `--ssh-public-key` / `--ssh-private-key`.
**A clean teardown can still leave public IPs behind, and they count against the tenancy.** The addresses Spinifex allocates for guests are created by the running node, so they are in no Terraform state and `destroy` neither sees nor removes them. The node's own reconcile collects detached ones every ten minutes, which is exactly the sweep a destroy takes away. Twenty accumulated in `spxbm` over four runs and exhausted the tenancy's 50 reserved-public-IP limit, at which point every later launch in every compartment failed. [Check for leftover public IPs afterwards](../../../docs/oci-integration/README.md#check-for-leftover-public-ips-afterwards) has the query and the rule for reading it.
The workbook runs **on the node** against `127.0.0.1`, because the node certificate carries no SAN for its public address — a workbook driven from outside the VCN is still blocked.
## Common Commands
scripts/terraform/oci-spx/cloud-init/mount-data-volume.sh
+22 -1
View File
@@ -18,18 +18,39 @@ die() {
exit 1
}
MULTIPATHD_UNIT=/usr/lib/systemd/system/multipathd.service
MULTIPATH_ALIAS=/etc/systemd/system/multipath-tools.service
# OCI makes a bare-metal attachment multipath, and the agent refuses to log one in
# unless multipathd runs. It probes "multipath-tools", the name Debian retired, so
# on Ubuntu 26.04 it gets exit 5 forever while multipathd is active all along. A
# symlink in /etc/systemd/system is a systemd alias, which is all the agent needs.
#
# Asserted on every poll rather than written once: on the first bare-metal run the
# symlink was created and then vanished, cause unknown, and the volume only logged
# in once it was back. Converging on each pass beats trusting a single write.
assert_multipath_alias() {
[ -e "$MULTIPATHD_UNIT" ] || return 0
[ "$(readlink -f "$MULTIPATH_ALIAS" 2>/dev/null)" = "$MULTIPATHD_UNIT" ] && return 0
ln -sfn "$MULTIPATHD_UNIT" "$MULTIPATH_ALIAS"
systemctl daemon-reload
log "aliased multipath-tools.service to multipathd.service for the Oracle agent"
}
# The Oracle Cloud Agent logs the iSCSI session in asynchronously, so the device
# is not present when cloud-init first runs. Wait for it instead of racing it.
wait_for_device() {
local i
for i in $(seq 1 120); do
assert_multipath_alias
if [ -b "$DEVICE" ]; then
log "$DEVICE appeared after $((i * 5))s"
return 0
fi
sleep 5
done
die "$DEVICE did not appear within 600s; check the iSCSI session with 'iscsiadm -m session'"
die "$DEVICE did not appear within 600s; check 'iscsiadm -m session' and the agent log at /var/log/oracle-cloud-agent/plugins/oci-blockautoconfig/"
}
# blkid reporting a TYPE is the only thing standing between a re-run and an erased
scripts/terraform/oci-spx/cloud-init/setup-wan-bridge.sh
+32 -20
View File
@@ -28,41 +28,56 @@ primary_mac() {
cat "/sys/class/net/$dev/address"
}
# Prints shell assignments for the first VNIC that is not the primary, or exits 1.
imds_secondary() {
# One tab-separated record per VNIC that is not the primary.
imds_secondaries() {
curl -sf --max-time 10 -H 'Authorization: Bearer Oracle' "$IMDS/vnics/" |
python3 -c '
import ipaddress, json, sys
primary = sys.argv[1].lower()
for vnic in json.load(sys.stdin):
if vnic["macAddr"].lower() == primary:
mac = vnic["macAddr"].lower()
if mac == primary:
continue
net = ipaddress.ip_network(vnic["subnetCidrBlock"])
print("WAN_MAC=%s" % vnic["macAddr"].lower())
print("WAN_IP=%s" % vnic["privateIp"])
print("WAN_PREFIX=%d" % net.prefixlen)
print("WAN_CIDR=%s" % vnic["subnetCidrBlock"])
print("WAN_ROUTER=%s" % vnic["virtualRouterIp"])
sys.exit(0)
sys.exit(1)
print("\t".join([mac, vnic["privateIp"], str(net.prefixlen),
vnic["subnetCidrBlock"], vnic["virtualRouterIp"]]))
' "$1"
}
# Picks the secondary VNIC the kernel actually has an interface for, and sets the
# WAN_* variables from it. Deliberately not "the first non-primary": IMDS order is
# not documented, and on bare metal a VNIC sharing a port has no interface at all.
select_secondary() {
local primary="$1" mac ip prefix cidr router iface
while IFS=$'\t' read -r mac ip prefix cidr router; do
[ -n "$mac" ] || continue
if iface="$(iface_for_mac "$mac")"; then
WAN_MAC="$mac"
WAN_IP="$ip"
WAN_PREFIX="$prefix"
WAN_CIDR="$cidr"
WAN_ROUTER="$router"
WAN_IFACE="$iface"
return 0
fi
done < <(imds_secondaries "$primary")
return 1
}
wait_for_secondary() {
local primary i out
local primary i
primary="$(primary_mac)"
log "primary VNIC MAC is $primary"
for i in $(seq 1 60); do
if out="$(imds_secondary "$primary")" && [ -n "$out" ]; then
eval "$out"
log "secondary VNIC after $((i * 5))s: $WAN_MAC $WAN_IP/$WAN_PREFIX via $WAN_ROUTER"
if select_secondary "$primary"; then
log "secondary VNIC after $((i * 5))s: $WAN_MAC on $WAN_IFACE, $WAN_IP/$WAN_PREFIX via $WAN_ROUTER"
return 0
fi
sleep 5
done
die "no secondary VNIC appeared within 300s; check the VNIC attachment in OCI"
die "no secondary VNIC with an interface appeared within 300s; on bare metal check it is attached at nic_index 1, because a VNIC sharing the primary's port never becomes an interface"
}
# The physical interface for a MAC. Matching on the address rather than trusting a
@@ -184,16 +199,13 @@ PY
main() {
wait_for_secondary
local iface
iface="$(iface_for_mac "$WAN_MAC")" || die "no interface carries $WAN_MAC"
log "bridging $iface into $BRIDGE"
log "bridging $WAN_IFACE into $BRIDGE"
strip_cloud_init_secondary
local previous=""
[ -f "$NETPLAN_FILE" ] && previous="$(cat "$NETPLAN_FILE")"
write_netplan "$iface"
write_netplan "$WAN_IFACE"
if [ "$STRIPPED" = 0 ] && [ "$previous" = "$(cat "$NETPLAN_FILE")" ] && ip link show "$BRIDGE" >/dev/null 2>&1; then
log "$BRIDGE is already configured and unchanged"
scripts/terraform/oci-spx/cloud-init/user-data.yaml.tftpl
+57 -4
View File
@@ -21,15 +21,68 @@ write_files:
content: |
${indent(6, service_ports_script)}
# The secondary VNIC is attached after the instance reaches RUNNING, so it can
# Staged rather than applied: spinifex.toml does not exist until spx admin init
# has run. Holds no secret under instance principal, which is the point of it --
# the instance's own certificate is the credential.
- path: /etc/spinifex/oci/external-pool.toml
owner: root:root
permissions: "0644"
content: |
${indent(6, oci_pool_fragment)}
# A unit rather than a runcmd so a boot that raced the iSCSI login retries on the
# next one, and so the ports below can declare a dependency on it. The script
# polls for the device itself and is safe to re-run.
- path: /etc/systemd/system/spinifex-data-volume.service
owner: root:root
permissions: "0644"
content: |
[Unit]
Description=Spinifex data volume on the OCI block volume
Wants=network-online.target
After=network-online.target remote-fs.target
ConditionPathIsMountPoint=!${data_mountpoint}
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/usr/local/sbin/spinifex-mount-data-volume ${data_device} ${data_mountpoint}
[Install]
WantedBy=multi-user.target
# Requires= is the point: a node with no data volume has nothing behind these
# ports, so it must not publish them. systemd enforces that ordering, which a
# list of runcmd entries cannot -- runcmd continues past a failed step.
- path: /etc/systemd/system/spinifex-service-ports.service
owner: root:root
permissions: "0644"
content: |
[Unit]
Description=Spinifex node service ports
Requires=spinifex-data-volume.service
After=spinifex-data-volume.service
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/usr/local/sbin/spinifex-open-service-ports ${peer_cidr} ${service_ports}
[Install]
WantedBy=multi-user.target
# The external VNIC is attached after the instance reaches RUNNING, so it can
# arrive late. This unit retries on the next boot if the first attempt missed it,
# and stops running once the netplan file it writes exists.
#
# Deliberately independent of the data volume: losing the external datapath
# because a disk is late would make the node more broken, not safer.
- path: /etc/systemd/system/spinifex-wan-bridge.service
owner: root:root
permissions: "0644"
content: |
[Unit]
Description=Spinifex WAN bridge over the secondary OCI VNIC
Description=Spinifex WAN bridge over the external OCI VNIC
Wants=network-online.target
After=network-online.target
ConditionPathExists=!/etc/netplan/60-spinifex-wan.yaml
@@ -45,6 +98,6 @@ write_files:
WantedBy=multi-user.target
runcmd:
- [/usr/local/sbin/spinifex-mount-data-volume, "${data_device}", "${data_mountpoint}"]
- [/usr/local/sbin/spinifex-open-service-ports, "${peer_cidr}", ${service_ports}]
- [systemctl, enable, --now, spinifex-data-volume.service]
- [systemctl, enable, --now, spinifex-service-ports.service]
- [systemctl, enable, --now, spinifex-wan-bridge.service]
scripts/terraform/oci-spx/compute.tf
+36 -1
View File
@@ -30,6 +30,10 @@ locals {
# one, so it is emitted only when the shape name says Flex.
shape_is_flex = strcontains(var.compute_shape, "Flex")
# Bare metal presents VNICs differently from a VM, which decides where the
# external VNIC is attached. See the nic_index comment below.
shape_is_bare_metal = startswith(var.compute_shape, "BM.")
# Round-robin, so three nodes land on three fault domains rather than wherever
# OCI's placement happens to put them. Left to itself OCI spreads on a
# best-effort basis and says nothing about the result, which is not something
@@ -41,12 +45,38 @@ locals {
# after the boot disk (mulga-poc has oraclevda1 pointing at sdb1).
data_device = "/dev/oracleoci/oraclevdb"
# adopt and create differ only in who creates the dynamic group and policy; a node
# authenticates identically either way, because the matching rule covers the whole
# compartment and so names no instance.
use_instance_principal = var.instance_principal != "off"
# The allocator's pool, staged on each node for spx admin init to pick up. Under
# instance principal there is no oci_config_file key, because naming one beside a
# certificate credential is a second source of truth that config load rejects.
oci_pool_fragment = join("\n", concat([
"# Generated by the oci-spx Terraform. Append to /etc/spinifex/spinifex.toml",
"# after spx admin init, then restart spinifex.target.",
"[[network.external_pools]]",
"name = \"oci-public\"",
"source = \"oci\"",
"oci_compartment_id = \"${var.compartment_ocid}\"",
"oci_vnic_iface = \"${var.wan_bridge_name}\"",
], local.use_instance_principal ? [
"oci_auth = \"instance_principal\"",
] : [
"oci_config_file = \"/etc/spinifex/oci/config\"",
"oci_config_profile = \"spinifex\"",
], [
"dns_servers = [\"169.254.169.253\"]",
]))
# Identical on every node, so a change to it replaces all of them together.
user_data = templatefile("${path.module}/cloud-init/user-data.yaml.tftpl", {
data_device = local.data_device
data_mountpoint = var.data_mountpoint
peer_cidr = var.vcn_cidr
service_ports = join(", ", [for p in var.node_service_ports : "\"${p}\""])
service_ports = join(" ", var.node_service_ports)
oci_pool_fragment = local.oci_pool_fragment
wan_bridge_name = var.wan_bridge_name
wan_bridge_mtu = var.wan_bridge_mtu
mount_script = file("${path.module}/cloud-init/mount-data-volume.sh")
@@ -105,6 +135,11 @@ resource "oci_core_vnic_attachment" "mulgadc_external" {
count = var.node_count
instance_id = oci_core_instance.mulgadc[count.index].id
# On bare metal only the first VNIC on a physical port becomes a Linux device;
# a second one is a tagged overlay with no device. The second port keeps this
# VNIC first, so it is untagged and present at boot exactly as it is on a VM.
nic_index = local.shape_is_bare_metal ? 1 : 0
create_vnic_details {
subnet_id = oci_core_subnet.public.id
display_name = format("%s-node-%02d-external", var.deployment_name, count.index + 1)
scripts/terraform/oci-spx/e2e-oci.sh
Executable
+464
View File
@@ -0,0 +1,464 @@
#!/bin/bash
# e2e-oci.sh — prove a ref works on OCI, across every topology, and leave nothing
# running. One entry point for a workstation, a Cloudflare-dispatched nightly and
# the GitHub workflow, because three ways of starting it would drift into three
# different answers about whether OCI works.
#
# Configured entirely by environment variables so a dispatch needs no arguments:
#
# SPX_REF Ref to prove. Default: the spinifex checkout's current branch.
# OCI_TOPOLOGIES Space-separated. Default: "vm-single vm-multi bm
# vm-single-principal". Set one name to test one thing: a
# single VM topology is roughly a quarter of the full run.
# OCI_SOURCE tree (build SPX_REF) or release (the published installer).
# OCI_VERSION With OCI_SOURCE=release, install this exact release tag and
# ignore OCI_CHANNEL. Preferred for anything repeatable: a tag
# resolves by redirect, while the dev channel resolves through a
# rate-limited GitHub API call that 404s when it trips.
# OCI_CHANNEL With OCI_SOURCE=release, which published channel to install:
# latest (default) or dev, the newest prerelease. dev is the
# path our cloud contacts are given, so it is worth testing.
# Default tree: a nightly exists to judge a ref, and the
# published installer says nothing about one.
# WORKBOOKS Workbooks to run on each topology. Default: the driver's own
# list. Set to "" to skip the workbook phase entirely.
# OCI_INSTANCE_PRINCIPAL Authenticate the node's allocator as the instance
# principal. Default 1, and it needs the tenancy's dynamic
# group to exist. The vm-single-principal topology ignores it
# and always authenticates that way, which is its purpose.
#
# A tenancy where setup-identity.sh cannot run has no such
# group, and then no principal run can pass: the node forms,
# authenticates, and is refused every allocation. That now
# fails at the allocator gate within a minute of the daemon
# starting rather than an hour later inside a workbook, but
# it still costs a full provision, so drop the topology from
# OCI_TOPOLOGIES until the group exists.
# OCI_NO_EXTERNAL_POOL 1 to form with no allocator at all, for a tenancy whose
# dynamic group does not exist yet. The allocator and every
# workbook are then recorded SKIPPED, never PASS, so a green
# run with this set is not a claim about guest networking.
# Not applied to vm-single-principal, which the allocator is
# the entire test of; that pairing is refused rather than run.
# OCI_SSH_PUBLIC_KEY / OCI_SSH_PRIVATE_KEY Paths. Default ~/.ssh/oci-spx[.pub].
# OCI_CREDENTIAL_HOOK Executable run on each topology after formation and
# before the pool, as "hook <ssh-key> <host>...". An API-key
# deployment installs its credential here; instance principal
# needs none, so it is skipped unless the pool is key-based.
# OCI_PRINCIPAL_TENANCY_OCID, OCI_PRINCIPAL_USER_OCID,
# OCI_PRINCIPAL_FINGERPRINT, OCI_PRINCIPAL_PRIVATE_KEY, OCI_PRINCIPAL_REGION,
# OCI_PRINCIPAL_COMPARTMENT_OCID
# The credential vm-single-principal builds with, replacing the
# run's own for that one topology. The dynamic group is a
# tenancy-root resource, so a tenancy that will not let
# setup-identity.sh create one refuses every allocation however
# correct the code is, and the principal route has to be proved
# in a tenancy we administer while the rest of the run stays
# where it is.
#
# All six or none. A partial set builds one tenancy's instances
# against another's compartment, which is refused rather than
# reported; with none set the topology is recorded SKIPPED
# rather than costing a provision to fail. Running it in the
# run's own tenancy is still available by pointing all six
# there, which says so explicitly instead of by default.
# OCI_PRINCIPAL_SHAPE Compute shape for vm-single-principal. Its tenancy
# carries its own service limits, so the shape the other
# topologies use can have zero quota there and fail the apply
# before anything about the principal is exercised. Default
# VM.Standard.E6.Flex, as the others.
# OCI_ARTIFACT_DIR Where logs and the verdict land. Default ./.e2e-oci-<stamp>.
# OCI_KEEP_ON_FAIL 1 to leave a failed topology up for inspection. Off by
# default: an OCI bare-metal host left overnight is expensive.
#
# The credential comes from OCI_TENANCY_OCID, OCI_USER_OCID, OCI_FINGERPRINT,
# OCI_PRIVATE_KEY and OCI_REGION when they are set, and from ~/.oci/config when
# they are not. scripts/oci_env.py owns that choice; nothing here reads a key.
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SPINIFEX_ROOT="$(cd "$HERE/../../.." && pwd)"
# On a workstation spinifex is a submodule of mulga; in CI the two are siblings
# under the workspace. Deriving one from the other is only right in the first case.
MULGA_ROOT="${MULGA_ROOT:-$SPINIFEX_ROOT/..}"
TOPOLOGIES="${OCI_TOPOLOGIES:-vm-single vm-multi bm vm-single-principal}"
SOURCE="${OCI_SOURCE:-tree}"
CHANNEL="${OCI_CHANNEL:-latest}"
INSTALL_VERSION="${OCI_VERSION:-}"
REF="${SPX_REF:-$(git -C "$SPINIFEX_ROOT" rev-parse --abbrev-ref HEAD)}"
SSH_PUBLIC_KEY="${OCI_SSH_PUBLIC_KEY:-$HOME/.ssh/oci-spx.pub}"
SSH_PRIVATE_KEY="${OCI_SSH_PRIVATE_KEY:-$HOME/.ssh/oci-spx}"
ARTIFACT_DIR="${OCI_ARTIFACT_DIR:-$HERE/.e2e-oci-$(date -u +%Y%m%dT%H%M%SZ)}"
# Exported so validate-topology.sh agrees with the sweep about where state lives.
# On a persistent runner this belongs outside the checkout: actions/checkout runs
# git clean -ffdx, which would delete the state of a killed run before the sweep.
export OCI_STATE_ROOT="${OCI_STATE_ROOT:-$HERE}"
STATE_ROOT="$OCI_STATE_ROOT"
SWEEP_ONLY=0
SUMMARY_ONLY=0
DRY_RUN=0
log() { printf '[e2e-oci] %s\n' "$*"; }
die() {
printf '[e2e-oci] ERROR: %s\n' "$*" >&2
exit 1
}
usage() {
cat >&2 <<EOF
usage: ${0##*/} [--sweep-only] [--summary-only] [--dry-run]
Everything else is an environment variable; see the header of this script.
--sweep-only Destroy anything an earlier run left behind, then stop.
--summary-only Write the result tables from the state files, then stop. For
a job that was cancelled or killed before it got there.
--dry-run Print what each topology would do, build nothing.
EOF
exit 2
}
while [ $# -gt 0 ]; do
case "$1" in
--sweep-only) SWEEP_ONLY=1; shift ;;
--summary-only) SUMMARY_ONLY=1; shift ;;
--dry-run) DRY_RUN=1; shift ;;
-h | --help) usage ;;
*) die "unknown option $1" ;;
esac
done
case "$SOURCE" in
tree | release) ;;
*) die "OCI_SOURCE must be tree or release, got '$SOURCE'" ;;
esac
case "$CHANNEL" in
latest | dev) ;;
*) die "OCI_CHANNEL must be latest or dev, got '$CHANNEL'" ;;
esac
# Saying release+dev and tree at once is two different artifacts in one verdict.
[ "$SOURCE" = tree ] && [ -n "$INSTALL_VERSION" ] \
&& die "OCI_VERSION needs OCI_SOURCE=release: a tree build installs the ref, not a published tag"
[ "$SOURCE" = tree ] && [ "$CHANNEL" != latest ] \
&& die "OCI_CHANNEL=$CHANNEL needs OCI_SOURCE=release: a tree build installs the ref, not a channel"
mkdir -p "$ARTIFACT_DIR"
VERDICT="$ARTIFACT_DIR/verdict.txt"
: > "$VERDICT"
# vm-single-principal is the one topology that cannot run wherever the others do.
# It needs the tenancy's dynamic group, and creating that is a tenancy-root write,
# so the tenancy a shared test account gives us can never pass it. These let it
# build somewhere else without moving the rest of the run.
PRINCIPAL_CRED_VARS="TENANCY_OCID USER_OCID FINGERPRINT PRIVATE_KEY REGION COMPARTMENT_OCID"
# principal_cred_state is none, all or partial. Partial is the one worth a stop:
# one tenancy's OCIDs with another's compartment builds in a compartment nobody
# chose, and the apply reports success doing it.
principal_cred_state() {
local name var have=0 miss=0
for name in $PRINCIPAL_CRED_VARS; do
var="OCI_PRINCIPAL_$name"
if [ -n "${!var:-}" ]; then
have=$((have + 1))
else
miss=$((miss + 1))
fi
done
if [ "$have" = 0 ]; then
printf 'none\n'
elif [ "$miss" = 0 ]; then
printf 'all\n'
else
printf 'partial\n'
fi
}
# Into an array rather than a rendered string, because the private key is a
# multi-line PEM and any line-based plumbing silently truncates it to its header.
PRINCIPAL_ENV=()
build_principal_env() {
local name var
PRINCIPAL_ENV=()
for name in $PRINCIPAL_CRED_VARS; do
var="OCI_PRINCIPAL_$name"
case "$name" in
COMPARTMENT_OCID) PRINCIPAL_ENV+=("TF_VAR_compartment_ocid=${!var}") ;;
*) PRINCIPAL_ENV+=("OCI_$name=${!var}") ;;
esac
done
}
# Here rather than at the topology, because the build runs first and takes minutes,
# and a credential that cannot work is knowable before any of it. The loop checks
# again, since it is what reads the variables.
case " $TOPOLOGIES " in
*" vm-single-principal "*)
[ "$(principal_cred_state)" != partial ] \
|| die "the OCI_PRINCIPAL_* credential is incomplete: set all of $PRINCIPAL_CRED_VARS or none of them"
;;
esac
# The run page is the evidence, and its readers are not us: someone assessing
# whether Spinifex works on their cloud should get the answer without opening a
# log or knowing what this harness is. Markdown on stdout as well as to the step
# summary, so a workstation run shows the same thing CI publishes.
mark() {
case "$1" in
PASS) printf '✅ PASS' ;;
FAIL) printf '❌ FAIL' ;;
SKIPPED) printf '⏭️ SKIPPED' ;;
*) printf 'ℹ️ %s' "$1" ;;
esac
}
summary() {
printf '## Spinifex on OCI — %s\n\n' "$REF"
# Said, not implied. A tree build and a published release answer different
# questions, and a table headed "published release" over a build of somebody's
# branch is the one error here that would mislead a reader who trusted it.
if [ "$SOURCE" = tree ]; then
printf 'Built from this ref, region `%s`.\n\n' "${OCI_REGION:-ap-sydney-1}"
else
printf 'Published release `%s`, region `%s`.\n\n' \
"${INSTALL_VERSION:-$CHANNEL channel}" "${OCI_REGION:-ap-sydney-1}"
fi
# In the header, not left to a SKIPPED row further down. A run with no allocator
# is not evidence about public addressing, and that is precisely the claim a
# reader of a green table would otherwise take from it.
# Said per topology rather than once for the run: vm-single-principal pins its
# own credential, so a single claim over a mixed run is wrong for one of them.
if [ "${OCI_NO_EXTERNAL_POOL:-0}" = 1 ]; then
printf '> **No external address pool.** Public-address allocation and every workbook needing a public address were skipped, so nothing here speaks to guest ingress. `vm-single-principal`, if it ran, is the exception and did build one.\n\n'
elif [ "${OCI_INSTANCE_PRINCIPAL:-1}" = 1 ]; then
printf 'Allocator authenticated as the instance principal.\n\n'
else
printf 'Allocator authenticated with an API key, except `vm-single-principal` which authenticates as the instance.\n\n'
fi
# Said in the summary because the row is otherwise read as a result about the
# same tenancy as every row above it, which is the one thing it is not.
if [ "$(principal_cred_state)" = all ]; then
printf '`vm-single-principal` built in its own tenancy in `%s`, from the `OCI_PRINCIPAL_*` credential.\n\n' \
"$OCI_PRINCIPAL_REGION"
fi
local topology results workbooks gate status detail name secs
for topology in $TOPOLOGIES; do
results="$STATE_ROOT/.validate-$topology/results.tsv"
workbooks="$STATE_ROOT/.validate-$topology/workbooks.tsv"
printf '### `%s`\n\n' "$topology"
# A topology the loop never reached is said so rather than omitted: a
# reader takes an absent row for a passing one.
if [ ! -s "$results" ]; then
printf 'Did not run.\n\n'
continue
fi
printf '| Gate | Result | Detail |\n| --- | --- | --- |\n'
while IFS=$'\t' read -r gate status detail; do
[ -n "$gate" ] || continue
printf '| %s | %s | %s |\n' "$gate" "$(mark "$status")" "$detail"
done < "$results"
printf '\n'
if [ ! -s "$workbooks" ]; then
printf 'No published workbooks ran.\n\n'
continue
fi
printf '| Terraform workbook | Result | Duration |\n| --- | --- | --- |\n'
while IFS=$'\t' read -r name status secs; do
[ -n "$name" ] || continue
printf '| `%s` | %s | %ss |\n' "$name" "$(mark "$status")" "$secs"
done < "$workbooks"
printf '\n'
done
}
# GITHUB_STEP_SUMMARY is per step, so appending from both the run and the dedicated
# publish step would put two copies of the tables on the run page. One owner: the run
# writes the file and prints it, and only --summary-only publishes it.
emit_summary() {
summary | tee "$ARTIFACT_DIR/summary.md"
}
publish_summary() {
emit_summary
if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
cat "$ARTIFACT_DIR/summary.md" >> "$GITHUB_STEP_SUMMARY"
fi
}
# A topology destroys itself from its own EXIT trap, so this only catches a run
# the trap never reached: a killed process, a runner reclaimed mid-job, a reboot.
# It works from the state files rather than by name filter, so it can only ever
# destroy what one of these runs created.
sweep() {
local found=0 dir topology
local sweep_env=(env)
for dir in "$STATE_ROOT"/.validate-*; do
[ -s "$dir/terraform.tfstate" ] || continue
python3 -c '
import json, sys
state = json.load(open(sys.argv[1]))
sys.exit(0 if any(r["instances"] for r in state.get("resources", [])) else 1)
' "$dir/terraform.tfstate" 2>/dev/null || continue
topology="${dir##*/.validate-}"
# The child reads OCI_STATE_ROOT from the environment, so it looks in the
# same place this loop found the state rather than beside its own script.
found=1
# The same credential the topology built with, or the destroy authenticates
# against a tenancy that never held these resources and reports a clean
# sweep over a deployment that is still billing.
sweep_env=(env)
if [ "$topology" = vm-single-principal ] && [ "$(principal_cred_state)" = all ]; then
build_principal_env
sweep_env=(env "${PRINCIPAL_ENV[@]}")
fi
log "sweep: $topology still holds resources, destroying"
if "${sweep_env[@]}" "$HERE/validate-topology.sh" --topology "$topology" --destroy-only \
>> "$ARTIFACT_DIR/sweep.log" 2>&1; then
log "sweep: $topology destroyed"
else
log "sweep: $topology WOULD NOT DESTROY -- it is still billing. See $ARTIFACT_DIR/sweep.log"
echo "sweep $topology: FAIL" >> "$VERDICT"
fi
done
[ "$found" = 0 ] && log "sweep: nothing left behind"
}
# Before the sweep, because the sweep is what destroys the state the tables are
# read from. This is the path a cancelled job takes, so it has to work on whatever
# the run got through rather than assuming a complete set.
if [ "$SUMMARY_ONLY" = 1 ]; then
publish_summary
exit 0
fi
# Run the sweep before the topologies as well as after: a leftover cluster from a
# killed run is both a bill and a second set of resources with our names on it.
log "artifacts in $ARTIFACT_DIR"
sweep
if [ "$SWEEP_ONLY" = 1 ]; then
log "--sweep-only: done"
exit 0
fi
# Built once for every topology. A per-topology build would prove three different
# sets of bytes and could not say which one failed.
DISTRO_TARBALL=""
if [ "$SOURCE" = tree ]; then
log "building $REF"
DISTRO_TARBALL="$ARTIFACT_DIR/spinifex-distro.tar.gz"
BUILDER="$MULGA_ROOT/scripts/tofu-cluster/build-install-artifacts.sh"
[ -x "$BUILDER" ] || die "no artifact builder at $BUILDER (set MULGA_ROOT)"
# It clones each repo from GitHub rather than using the checkout beside it, so a
# ref that exists only on this workstation silently falls back to dev and the
# run would report a verdict about the wrong code. Fail on that instead.
if ! git -C "$SPINIFEX_ROOT" rev-parse --verify --quiet "refs/remotes/origin/$REF" >/dev/null \
&& ! git -C "$SPINIFEX_ROOT" ls-remote --exit-code --tags origin "$REF" >/dev/null 2>&1; then
die "$REF is not on origin: the builder clones from GitHub, so an unpushed ref would fall back to dev"
fi
if [ "$DRY_RUN" = 1 ]; then
log "dry run: would build $REF into $DISTRO_TARBALL"
else
# --skip-e2e-build: the Go suites are not run here. The workbook driver is a
# shell script, so building their binaries would add minutes and prove nothing.
"$BUILDER" "$REF" "$DISTRO_TARBALL" "$ARTIFACT_DIR/setup.sh" \
"$ARTIFACT_DIR/spinifex-tests.tar" --skip-e2e-build \
> "$ARTIFACT_DIR/build.log" 2>&1 \
|| die "building $REF failed; see $ARTIFACT_DIR/build.log"
log "built $(du -h "$DISTRO_TARBALL" | cut -f1) from $REF"
fi
fi
RUN_RC=0
for topology in $TOPOLOGIES; do
# The credential swap happens here and nowhere else, so everything downstream
# of the one validate call is unaware the topology built somewhere different.
# Always a bare `env` otherwise, because an empty array is an unbound variable
# under set -u on the bash 3.2 operators run this with.
run_env=(env)
if [ "$topology" = vm-single-principal ]; then
case "$(principal_cred_state)" in
none)
log "$topology: SKIPPED -- no OCI_PRINCIPAL_* credential, and the run's own tenancy cannot hold the dynamic group this topology needs"
echo "$topology: SKIPPED (no principal credential)" >> "$VERDICT"
continue
;;
partial)
die "the OCI_PRINCIPAL_* credential is incomplete: set all of $PRINCIPAL_CRED_VARS or none of them"
;;
esac
build_principal_env
run_env=(env "${PRINCIPAL_ENV[@]}")
fi
args=(--topology "$topology" --ssh-public-key "$SSH_PUBLIC_KEY" --ssh-private-key "$SSH_PRIVATE_KEY")
[ "$DRY_RUN" = 1 ] && args+=(--dry-run)
# Two separate choices. --no-external-pool is the one that forms without any
# allocator, for a tenancy whose dynamic group does not exist yet; it records
# the allocator and the workbooks as SKIPPED rather than passing them.
#
# vm-single-principal opts out of both: it sets its own credential, and a run
# of it without an allocator would exercise nothing. validate-topology.sh
# refuses that pairing, so skipping it here keeps a mixed run from aborting.
if [ "$topology" = vm-single-principal ]; then
:
elif [ "${OCI_NO_EXTERNAL_POOL:-0}" = 1 ]; then
args+=(--no-external-pool)
elif [ "${OCI_INSTANCE_PRINCIPAL:-1}" = 1 ]; then
args+=(--instance-principal)
fi
# Not under --dry-run, where the build was skipped and naming the tarball it
# would have produced fails the readable check, so every topology reported FAIL
# and the flag operators are told to run first could never pass. Still
# unconditional in a real run, so a build that produced nothing stays loud
# rather than quietly installing a release instead.
if [ -n "$DISTRO_TARBALL" ] && [ "$DRY_RUN" != 1 ]; then
args+=(--distro "$DISTRO_TARBALL" --setup-sh "$ARTIFACT_DIR/setup.sh")
fi
if [ "$SOURCE" = release ]; then
if [ -n "$INSTALL_VERSION" ]; then
args+=(--version "$INSTALL_VERSION")
else
args+=(--channel "$CHANNEL")
fi
fi
[ -n "${OCI_CREDENTIAL_HOOK:-}" ] && args+=(--credential-hook "$OCI_CREDENTIAL_HOOK")
[ "${OCI_KEEP_ON_FAIL:-0}" = 1 ] && args+=(--keep-on-fail)
[ -n "${WORKBOOKS+x}" ] && args+=(--workbooks "$WORKBOOKS")
# go test -v markers, so the same log feeds go-junit-report and the RCA bundle
# the Go suites produce. One testcase per topology.
tname="TestOCITopology_${topology//-/_}"
start=$SECONDS
echo "=== RUN $tname"
if "${run_env[@]}" "$HERE/validate-topology.sh" "${args[@]}" 2>&1 | tee "$ARTIFACT_DIR/$topology.log"; then
printf -- '--- PASS: %s (%d.00s)\n' "$tname" "$((SECONDS - start))"
echo "$topology: PASS" >> "$VERDICT"
else
printf -- '--- FAIL: %s (%d.00s)\n' "$tname" "$((SECONDS - start))"
echo "$topology: FAIL" >> "$VERDICT"
RUN_RC=1
fi
done
# Again at the end, because a topology that failed its own teardown is exactly the
# case that costs money, and the verdict should say so rather than the log.
sweep
emit_summary
echo
log "=== $REF on OCI ($SOURCE build$([ "$SOURCE" = release ] && { [ -n "$INSTALL_VERSION" ] && echo ", $INSTALL_VERSION" || echo ", $CHANNEL channel"; })) ==="
cat "$VERDICT"
if [ "$RUN_RC" = 0 ] && ! grep -q FAIL "$VERDICT"; then
log "PASS"
printf 'ok \te2e-oci\t%d.000s\n' "$SECONDS"
else
log "FAIL"
printf 'FAIL\te2e-oci\t%d.000s\n' "$SECONDS"
fi
exit "$RUN_RC"
scripts/terraform/oci-spx/eip-move-test.sh
+1 -1
View File
@@ -17,7 +17,7 @@ set -euo pipefail
KEY=${KEY:-$HOME/.ssh/oci-poc.pem}
OCI_BIN=${OCI_BIN:-$HOME/bin/oci}
OCI_PROFILE=${OCI_PROFILE:-apacanzset03child03}
OCI_PROFILE=${OCI_PROFILE:-DEFAULT}
SUBNET=${SUBNET:-ocid1.subnet.oc1.ap-sydney-1.aaaaaaaaimrhjgsqgrfg7bnka2d3nbx2n5ss7kurcwneon22rkclqf3ypfqa}
# Public address -> node name. The control plane is reached through node01.
scripts/terraform/oci-spx/instance-principal.tf
+44
View File
@@ -0,0 +1,44 @@
# Lets each node authenticate to the OCI API as itself, so the external-address
# allocator needs no API key on disk. Without one of these two credential routes a
# node forms and passes every health check, then refuses every launch that wants a
# public address with InsufficientAddressCapacity and no stated cause.
#
# Only instance_principal = "create" builds these, and deliberately: a dynamic
# group and a policy are tenancy-root resources, so creating them needs rights the
# compartment-scoped user the rest of this configuration runs as does not have and
# should not be given. Create them once with a tenancy admin, then every later
# deployment uses "adopt", which references nothing and so needs no such rights.
resource "oci_identity_dynamic_group" "nodes" {
provider = oci.home
count = var.instance_principal == "create" ? 1 : 0
compartment_id = var.tenancy_ocid
name = "${var.deployment_name}-nodes"
description = "Spinifex nodes in the ${var.deployment_name} deployment, authenticating as instance principals."
# Scoped to the compartment, not to instance OCIDs: nodes are replaced, and a
# rule listing them by OCID goes stale the first time one is.
matching_rule = "ALL {instance.compartment.id = '${var.compartment_ocid}'}"
}
# Narrower than it looks. The allocator creates, moves and deletes secondary
# private IPs on a node's own VNIC and attaches public IPs to them, which is what
# these four verbs cover; nothing here grants compute, storage or identity.
resource "oci_identity_policy" "nodes" {
provider = oci.home
count = var.instance_principal == "create" ? 1 : 0
compartment_id = var.compartment_ocid
name = "${var.deployment_name}-nodes-network"
description = "Allows Spinifex nodes to manage the private and public IPs backing external addresses."
statements = [
"Allow dynamic-group ${oci_identity_dynamic_group.nodes[0].name} to use vnics in compartment id ${var.compartment_ocid}",
"Allow dynamic-group ${oci_identity_dynamic_group.nodes[0].name} to manage private-ips in compartment id ${var.compartment_ocid}",
"Allow dynamic-group ${oci_identity_dynamic_group.nodes[0].name} to manage public-ips in compartment id ${var.compartment_ocid}",
# CreatePrivateIp needs SUBNET_ATTACH and DeletePrivateIp needs SUBNET_DETACH,
# both of which only the subnets `use` verb carries. Without it every allocation
# is refused as a 404, which reads as a missing resource rather than a missing grant.
"Allow dynamic-group ${oci_identity_dynamic_group.nodes[0].name} to use subnets in compartment id ${var.compartment_ocid}",
]
}
scripts/terraform/oci-spx/outputs.tf
+7
View File
@@ -8,6 +8,13 @@ output "security_list_ocids" { value = { public = oci_core_security_list.public.
output "compute_ocids" { value = [for instance in oci_core_instance.mulgadc : instance.id] }
output "data_volume_ocids" { value = [for volume in oci_core_volume.mulgadc_data : volume.id] }
# Read back rather than assumed: both can come from a tfvars file, so this is the
# only place that knows what was actually built. The deploy driver skips the
# credential handoff on instance_principal, so a wrong guess leaves a node with no
# credential at all.
output "compute_shape" { value = var.compute_shape }
output "instance_principal" { value = var.instance_principal }
# What the Spinifex deploy path consumes. The primary VNIC is the host plane, so
# these are the addresses update-nodes.sh and install-node.sh connect to.
output "nodes" {
scripts/terraform/oci-spx/provider.tf
+18 -3
View File
@@ -1,10 +1,25 @@
# A runner gets the key as a value, a workstation as a path. Passing the value
# means CI never writes key material to disk, so there is nothing to leave behind
# on a shared runner and nothing to clean up after a cancelled job.
provider "oci" {
tenancy_ocid = var.tenancy_ocid
user_ocid = var.user_ocid
fingerprint = var.fingerprint
private_key_path = var.private_key_path
private_key = var.private_key != "" ? var.private_key : null
private_key_path = var.private_key == "" ? var.private_key_path : null
region = var.region
}
# The aliased home-region provider is gone with the identity resources it served.
# Everything here now lives in one region, in an existing compartment.
# OCI serves IAM writes from the tenancy's home region, so the dynamic group and
# policy go through this one. Defaults to var.region, which makes it identical to
# the provider above until OCI_HOME_REGION says otherwise -- so a tenancy whose
# home region is where we build is unaffected, and ours is not.
provider "oci" {
alias = "home"
tenancy_ocid = var.tenancy_ocid
user_ocid = var.user_ocid
fingerprint = var.fingerprint
private_key = var.private_key != "" ? var.private_key : null
private_key_path = var.private_key == "" ? var.private_key_path : null
region = var.home_region != null && var.home_region != "" ? var.home_region : var.region
}
scripts/terraform/oci-spx/requirements.txt
-2
View File
@@ -1,2 +0,0 @@
# Dependencies used only by scripts/oci_env.py in its isolated .venv.
oci>=2.160.0,<3.0.0
scripts/terraform/oci-spx/scripts/oci_env.py
+158 -46
View File
@@ -1,5 +1,9 @@
#!/usr/bin/env python3
"""Prepare OCI-profile Terraform environment variables in this repo's .venv.
"""Prepare OCI Terraform environment variables from a profile or the environment.
Standard library only, deliberately. It once used the OCI SDK, which meant a
virtualenv, which meant python3-venv on every host that runs it -- and banksia
did not have it. Terraform's own provider makes the OCI calls.
Examples:
eval "$(python3 scripts/oci_env.py --shell)"
@@ -12,63 +16,144 @@ import argparse
import configparser
import os
from pathlib import Path
import re
import shlex
import subprocess
import sys
import venv
REPOSITORY = Path(__file__).resolve().parent.parent
VENV_PYTHON = REPOSITORY / ".venv" / "bin" / "python"
REQUIREMENTS = REPOSITORY / "requirements.txt"
REQUESTED_PROFILE = "apacanzset03child03"
FALLBACK_PROFILE = "apacanzset03child3"
# The OCI CLI's own name for "the profile when none is named", so a config written
# by `oci setup config` works with no flag.
DEFAULT_PROFILE = "DEFAULT"
def ensure_venv() -> None:
"""Create, populate, then re-execute from this repository's virtualenv."""
if Path(sys.executable).resolve() == VENV_PYTHON.resolve():
return
if not VENV_PYTHON.exists():
venv.EnvBuilder(with_pip=True).create(VENV_PYTHON.parent.parent)
subprocess.check_call([str(VENV_PYTHON), "-m", "pip", "install", "--quiet", "-r", str(REQUIREMENTS)])
os.execv(str(VENV_PYTHON), [str(VENV_PYTHON), str(Path(__file__).resolve()), *sys.argv[1:]])
def default_profile() -> str:
"""The profile to read when the caller named none.
OCI_CLI_PROFILE is the OCI CLI's own variable, so an operator who has already
selected a profile for `oci` does not select it twice.
"""
return os.environ.get("OCI_CLI_PROFILE") or DEFAULT_PROFILE
def has_profile(parser: configparser.RawConfigParser, name: str) -> bool:
"""Whether the config holds this profile.
DEFAULT needs its own test: configparser treats it as the fallback for every
other section rather than as a section, so has_section and sections() both
deny it exists even when `oci setup config` wrote nothing else.
"""
if name == DEFAULT_PROFILE:
return bool(parser.defaults())
return parser.has_section(name)
def profile_names(parser: configparser.RawConfigParser) -> list[str]:
names = list(parser.sections())
if parser.defaults():
names.insert(0, DEFAULT_PROFILE)
return names
def load_profile(config_path: Path, requested_profile: str) -> tuple[str, dict[str, str]]:
parser = configparser.RawConfigParser()
parser.read(config_path)
profile = requested_profile
if not parser.has_section(profile):
if profile == REQUESTED_PROFILE and parser.has_section(FALLBACK_PROFILE):
print(
f"Profile {REQUESTED_PROFILE} not found; using {FALLBACK_PROFILE}.",
file=sys.stderr,
)
profile = FALLBACK_PROFILE
else:
raise ValueError(f"OCI profile {profile!r} was not found in {config_path}")
# No fallback to another profile. A missing one is an error because the only
# alternative is building a cluster in whichever tenancy a different profile
# happens to point at.
if not has_profile(parser, profile):
raise ValueError(
f"OCI profile {profile!r} was not found in {config_path}; "
f"profiles present: {', '.join(profile_names(parser)) or 'none'}. "
"Name one with --profile or OCI_CLI_PROFILE."
)
# Validate through the OCI SDK installed in the dedicated virtualenv.
import oci
values = oci.config.from_file(str(config_path), profile)
values = {key: value.strip() for key, value in parser.items(profile)}
required = ("tenancy", "user", "fingerprint", "key_file", "region")
missing = [key for key in required if not values.get(key)]
if missing:
raise ValueError(f"Missing OCI profile values: {', '.join(missing)}")
# ~ in key_file is the OCI CLI's own convention and configparser does not expand it.
values["key_file"] = str(Path(values["key_file"]).expanduser())
if not Path(values["key_file"]).is_file():
raise FileNotFoundError(f"key_file in profile {profile} does not exist: {values['key_file']}")
check_credential_shape(values, source=str(config_path), names=CONFIG_SOURCE)
return profile, values
def tenancy_home_region(values: dict[str, str]) -> str:
"""Read the home region from OCI rather than assuming the profile region."""
import oci
ENVIRONMENT_KEYS = ("OCI_TENANCY_OCID", "OCI_USER_OCID", "OCI_FINGERPRINT", "OCI_PRIVATE_KEY", "OCI_REGION")
subscriptions = oci.identity.IdentityClient(values).list_region_subscriptions(values["tenancy"]).data
home = next((item.region_name for item in subscriptions if item.is_home_region), None)
if not home:
raise ValueError("OCI did not return a tenancy home region")
return home
def load_environment() -> dict[str, str] | None:
"""Read the credential from the environment, the way a CI runner holds it.
Returns None when none of the keys are set, so a workstation still falls
through to ~/.oci/config. A partial set is an error rather than a fallback:
half a credential in the environment means a secret failed to reach the job,
and silently using a different one would validate the wrong tenancy.
"""
present = [key for key in ENVIRONMENT_KEYS if os.environ.get(key)]
if not present:
return None
missing = [key for key in ENVIRONMENT_KEYS if not os.environ.get(key)]
if missing:
raise ValueError(
f"OCI credential in the environment is incomplete: {', '.join(present)} set, {', '.join(missing)} missing"
)
key = os.environ["OCI_PRIVATE_KEY"]
# Base64 is accepted because a PEM's newlines do not survive every secret
# store. Detected by shape rather than by a second variable to set wrongly.
if "-----BEGIN" not in key:
import base64
key = base64.b64decode(key).decode()
if "-----BEGIN" not in key:
raise ValueError("OCI_PRIVATE_KEY is neither a PEM nor base64-encoded PEM")
# Stripped because `echo` into a secret store appends a newline, and the SDK
# then calls the value malformed without saying which one or why.
values = {
"tenancy": os.environ["OCI_TENANCY_OCID"].strip(),
"user": os.environ["OCI_USER_OCID"].strip(),
"fingerprint": os.environ["OCI_FINGERPRINT"].strip(),
"key_content": key.strip() + "\n",
"region": os.environ["OCI_REGION"].strip(),
}
check_credential_shape(values, source="the environment", names=ENVIRONMENT_SOURCE)
return values
# Which input to go and fix, named the way the person reading the failure set it.
ENVIRONMENT_SOURCE = {
"tenancy": "OCI_TENANCY_OCID",
"user": "OCI_USER_OCID",
"fingerprint": "OCI_FINGERPRINT",
"key_content": "OCI_PRIVATE_KEY",
"region": "OCI_REGION",
}
CONFIG_SOURCE = {key: key for key in ENVIRONMENT_SOURCE} | {"key_content": "key_file"}
FINGERPRINT = re.compile(r"^[0-9a-f]{2}(:[0-9a-f]{2}){15}$")
def check_credential_shape(values: dict[str, str], source: str, names: dict[str, str]) -> None:
"""Reject a malformed credential here rather than as an OCI 401 later.
Every fault names the input the reader set, because the alternative is a
401 forty seconds into a Terraform run, which says nothing about which of
five values is wrong. A newline from `echo` into a secret store is the
common one, and it is stripped before this runs.
"""
faults = []
for field in ("tenancy", "user"):
if not values.get(field, "").startswith("ocid1."):
faults.append(f"{names[field]} is not an OCID")
if not FINGERPRINT.match(values.get("fingerprint", "")):
faults.append(f"{names['fingerprint']} is not a 16-byte hex fingerprint")
if not values.get("region"):
faults.append(f"{names['region']} is empty")
if faults:
raise ValueError(f"the OCI credential in {source} is not usable: {', '.join(faults)}")
def terraform_environment(
@@ -79,14 +164,22 @@ def terraform_environment(
ssh_private_key_path: str | None,
) -> dict[str, str]:
environment = {
"OCI_CLI_PROFILE": profile,
"TF_VAR_tenancy_ocid": values["tenancy"],
"TF_VAR_user_ocid": values["user"],
"TF_VAR_fingerprint": values["fingerprint"],
"TF_VAR_private_key_path": values["key_file"],
"TF_VAR_region": region_override or values["region"],
"TF_VAR_home_region": tenancy_home_region(values),
# A tenancy's home region is a fixed property of the tenancy, so it is an
# input rather than something to rediscover on every run. IAM resources
# have to be created against it; see the phase 12 note in the plan.
"TF_VAR_home_region": os.environ.get("OCI_HOME_REGION") or region_override or values["region"],
}
# One of the two, never both: the provider rejects a key given twice, and the
# variable validation in variables.tf says so before OCI is called at all.
if values.get("key_content"):
environment["TF_VAR_private_key"] = values["key_content"]
else:
environment["OCI_CLI_PROFILE"] = profile
environment["TF_VAR_private_key_path"] = values["key_file"]
if ssh_public_key_path:
key_path = Path(ssh_public_key_path).expanduser().resolve()
if not key_path.is_file():
@@ -102,7 +195,11 @@ def terraform_environment(
def main() -> int:
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("--profile", default=REQUESTED_PROFILE)
arg_parser.add_argument(
"--profile",
default=default_profile(),
help="Profile in ~/.oci/config. Defaults to $OCI_CLI_PROFILE, else DEFAULT.",
)
arg_parser.add_argument("--region")
arg_parser.add_argument(
"--ssh-public-key-path",
@@ -118,11 +215,21 @@ def main() -> int:
if args.shell and args.command:
arg_parser.error("--shell and a command cannot be used together")
ensure_venv()
config_path = Path(os.environ.get("OCI_CONFIG_FILE", Path.home() / ".oci" / "config"))
if not config_path.is_file():
raise FileNotFoundError(f"OCI configuration was not found at {config_path}")
profile, values = load_profile(config_path, args.profile)
values = load_environment()
if values is not None:
profile = "environment"
# Checked here, before anything calls OCI: --shell prints to stdout, which
# in CI is the job log, and this route holds the PEM itself.
if args.shell:
raise ValueError("--shell cannot be used with a credential from the environment: it would print the key")
else:
config_path = Path(os.environ.get("OCI_CONFIG_FILE", Path.home() / ".oci" / "config"))
if not config_path.is_file():
raise FileNotFoundError(
f"OCI configuration was not found at {config_path} and the environment holds no credential"
f" (set {', '.join(ENVIRONMENT_KEYS)})"
)
profile, values = load_profile(config_path, args.profile)
environment = terraform_environment(
profile,
values,
@@ -137,7 +244,7 @@ def main() -> int:
return 0
if not args.command:
print(f"Loaded OCI profile {profile} for region {environment['TF_VAR_region']}; virtualenv: {VENV_PYTHON.parent.parent}")
print(f"Loaded OCI profile {profile} for region {environment['TF_VAR_region']}")
return 0
command = args.command[1:] if args.command[0] == "--" else args.command
@@ -145,7 +252,12 @@ def main() -> int:
raise ValueError("A command is required after --")
child_environment = os.environ.copy()
child_environment.update(environment)
print(f"Running with OCI profile {profile} in {environment['TF_VAR_region']}: {' '.join(map(shlex.quote, command))}")
# Stderr, because stdout belongs to the wrapped command: a banner there is
# indistinguishable from output and corrupts anything that captures it.
print(
f"Running with OCI profile {profile} in {environment['TF_VAR_region']}: {' '.join(map(shlex.quote, command))}",
file=sys.stderr,
)
return subprocess.call(command, cwd=REPOSITORY, env=child_environment)
scripts/terraform/oci-spx/setup-identity.sh
Executable
+90
View File
@@ -0,0 +1,90 @@
#!/bin/bash
# setup-identity.sh -- create the dynamic group and policy that let Spinifex nodes
# authenticate to the OCI API as themselves, so no API key is installed on any node.
#
# Run once per tenancy, with a tenancy-admin principal. Every deployment afterwards
# passes instance_principal = "adopt", which references these by nothing at all: the
# matching rule covers the whole compartment, so it names no instance and needs no
# OCID from here.
#
# Its own state file, deliberately. validate-topology.sh destroys its state at the
# end of every run, and these two resources outlive every topology -- holding them
# in a per-topology state would let a nightly teardown delete the tenancy's policy.
#
# MUST MAINTAIN BASH 3.X COMPATABILITY. Operators run this on macOS, which ships
# bash 3.2: no declare -A, no mapfile, and under set -u an empty "${arr[@]}" is an
# unbound variable, so guard every such expansion with ${#arr[@]}.
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
STATE_DIR="${OCI_STATE_ROOT:-$HERE}/.identity"
SSH_PUBLIC_KEY="${OCI_SSH_PUBLIC_KEY:-$HOME/.ssh/oci-spx.pub}"
DESTROY=0
DRY_RUN=0
log() { printf '[setup-identity] %s\n' "$*"; }
die() {
printf '[setup-identity] ERROR: %s\n' "$*" >&2
exit 1
}
usage() {
cat >&2 <<EOF
usage: ${0##*/} [--dry-run] [--destroy]
Creates the OCI dynamic group and policy for instance-principal authentication.
Needs a tenancy-admin OCI credential; the compartment-scoped user the rest of this
configuration runs as cannot create either resource.
--dry-run Show the plan, change nothing.
--destroy Remove them again. Every node using "adopt" loses its credential.
EOF
exit 2
}
while [ $# -gt 0 ]; do
case "$1" in
--dry-run) DRY_RUN=1; shift ;;
--destroy) DESTROY=1; shift ;;
-h | --help) usage ;;
*) die "unknown option $1" ;;
esac
done
[ -r "$SSH_PUBLIC_KEY" ] || die "no readable SSH public key at $SSH_PUBLIC_KEY (set OCI_SSH_PUBLIC_KEY)"
mkdir -p "$STATE_DIR"
# The instance and network resources are not targeted, so nothing else in this
# configuration is created or read. A tenancy admin running this does not need to
# hold the compartment rights the rest of it uses.
targets=(-target oci_identity_dynamic_group.nodes -target oci_identity_policy.nodes)
tf() {
python3 "$HERE/scripts/oci_env.py" --ssh-public-key-path "$SSH_PUBLIC_KEY" -- \
terraform -chdir="$HERE" "$@"
}
tf init -input=false -upgrade >/dev/null || die "terraform init failed"
if [ "$DESTROY" = 1 ]; then
log "destroying the dynamic group and policy"
tf destroy -auto-approve -var 'instance_principal=create' \
-state "$STATE_DIR/terraform.tfstate" "${targets[@]}" \
|| die "destroy failed"
log "removed. Any node configured for instance principal can no longer allocate an address."
exit 0
fi
args=(-var 'instance_principal=create' -state "$STATE_DIR/terraform.tfstate" "${targets[@]}")
if [ "$DRY_RUN" = 1 ]; then
log "dry run: planning the dynamic group and policy only"
tf plan "${args[@]}" || die "plan failed"
exit 0
fi
log "creating the dynamic group and policy (tenancy-root resources)"
tf apply -auto-approve "${args[@]}" || die "apply failed -- a tenancy-admin principal is required"
log "done. State in $STATE_DIR/terraform.tfstate -- keep it, it is the only record."
log "Deployments can now use instance_principal = \"adopt\" with a compartment-scoped user."
scripts/terraform/oci-spx/spx-oci-config.sh
Executable
+268
View File
@@ -0,0 +1,268 @@
#!/usr/bin/env bash
# Installs the OCI credential the external-address allocator authenticates with
# on every node, as /etc/spinifex/oci/{config,oci_api_key.pem}.
#
# Usage: spx-oci-config.sh [--dry-run] <ssh-private-key> <host>...
#
# The argument shape is validate-topology.sh's credential-hook contract, so this
# drops straight into --credential-hook or $OCI_CREDENTIAL_HOOK.
#
# MUST MAINTAIN BASH 3.X COMPATABILITY. Operators run this on macOS, which ships
# bash 3.2: no declare -A, no mapfile, and under set -u an empty "${arr[@]}" is an
# unbound variable, so guard every such expansion with ${#arr[@]}.
set -euo pipefail
HERE=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
SELF="$HERE/$(basename -- "$0")"
# What the allocator reads, from the pool fragment Terraform stages on each node.
# The config path and the profile name are the contract; the key filename is only
# ever named by key_file inside that config.
REMOTE_DIR=/etc/spinifex/oci
REMOTE_CONFIG="$REMOTE_DIR/config"
REMOTE_PEM="$REMOTE_DIR/oci_api_key.pem"
REMOTE_PROFILE=spinifex
SPX_KEY="${OCI_SPX_KEY:-$HOME/.oci/oci_api_key_spx.pem}"
SPX_PROFILE="${OCI_SPX_PROFILE:-spinifex}"
SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
-o LogLevel=ERROR -o BatchMode=yes -o ConnectTimeout=15)
DRY_RUN=0
die() { echo "$(basename -- "$0"): $*" >&2; exit 1; }
log() { echo "[spx-oci-config] $*"; }
usage() {
sed -n '2,8p' "$SELF" | sed 's/^# \{0,1\}//'
cat <<'EOF'
Options:
--dry-run Resolve and validate the credential, print what would be written,
and touch no host.
-h, --help This text.
The credential is resolved in this order, and the first one that is complete wins:
1. $OCI_SPX_PRIVATE_KEY with $OCI_SPX_USER_OCID, $OCI_SPX_FINGERPRINT,
$OCI_SPX_TENANCY_OCID and $OCI_SPX_REGION. A dedicated narrow credential,
which is the shape a CI secret store holds. A partial set is an error.
2. The [spinifex] profile in ~/.oci/config, or $OCI_SPX_PROFILE.
3. ~/.oci/oci_api_key_spx.pem, or $OCI_SPX_KEY, with the user, tenancy and
region of the profile Terraform itself uses. The fingerprint is derived
from the key rather than read from the config.
4. The credential Terraform builds with. This works and is wider than the
allocator needs, so it warns.
Rules 2 to 4 resolve the Terraform profile through scripts/oci_env.py, so the
precedence is the same one the apply used and the two cannot disagree.
EOF
}
while [ "$#" -gt 0 ]; do
case "$1" in
--dry-run) DRY_RUN=1; shift ;;
-h|--help) usage; exit 0 ;;
--) shift; break ;;
-*) die "unknown option: $1" ;;
*) break ;;
esac
done
# oci_env.py owns credential resolution for this directory: the environment route
# a runner uses, ~/.oci/config otherwise, a partial set treated as an error, and
# the shape checks. Re-exec under it rather than reimplement any of that here.
if [ -z "${SPX_OCI_ENV_READY:-}" ]; then
[ -x "$HERE/scripts/oci_env.py" ] || [ -r "$HERE/scripts/oci_env.py" ] \
|| die "scripts/oci_env.py was not found next to $SELF"
if [ "$DRY_RUN" = 1 ]; then
exec env SPX_OCI_ENV_READY=1 python3 "$HERE/scripts/oci_env.py" -- "$SELF" --dry-run "$@"
fi
exec env SPX_OCI_ENV_READY=1 python3 "$HERE/scripts/oci_env.py" -- "$SELF" "$@"
fi
# The same argument shape in both modes, so a --dry-run rehearsal and the real run
# differ by one flag. Only the checks that need a host are skipped.
ssh_key="${1:?usage: $(basename -- "$0") [--dry-run] <ssh-private-key> <host>...}"
shift
if [ "$DRY_RUN" != 1 ]; then
[ "$#" -gt 0 ] || die "no hosts given"
[ -r "$ssh_key" ] || die "SSH private key is not readable: $ssh_key"
fi
# A key's fingerprint is a property of the key. Deriving it beats trusting a
# config field, because a stale fingerprint beside a good key is a 401 that names
# neither.
fingerprint_of() {
openssl rsa -pubout -outform DER -in "$1" 2>/dev/null \
| openssl md5 -c 2>/dev/null | awk '{print $NF}'
}
require_complete() {
local source="$1" missing="" name
shift
for name in "$@"; do
[ -n "${!name:-}" ] || missing="$missing $name"
done
[ -z "$missing" ] || die "$source is incomplete, missing:$missing"
}
key_pem="" # the PEM itself, never a path, so nothing is read twice
oci_user=""
oci_fingerprint=""
oci_tenancy=""
oci_region=""
source_name=""
if [ -n "${OCI_SPX_PRIVATE_KEY:-}${OCI_SPX_USER_OCID:-}${OCI_SPX_FINGERPRINT:-}${OCI_SPX_TENANCY_OCID:-}${OCI_SPX_REGION:-}" ]; then
require_complete "the OCI_SPX_* credential in the environment" \
OCI_SPX_PRIVATE_KEY OCI_SPX_USER_OCID OCI_SPX_FINGERPRINT \
OCI_SPX_TENANCY_OCID OCI_SPX_REGION
# Base64 accepted by shape, because a PEM's newlines do not survive every
# secret store, and detected rather than selected by a second variable.
case "$OCI_SPX_PRIVATE_KEY" in
*-----BEGIN*) key_pem="$OCI_SPX_PRIVATE_KEY" ;;
*) key_pem=$(printf '%s' "$OCI_SPX_PRIVATE_KEY" | tr -d ' \n' | base64 -d 2>/dev/null) || true ;;
esac
case "$key_pem" in
*-----BEGIN*) ;;
*) die "OCI_SPX_PRIVATE_KEY is neither a PEM nor a base64-encoded PEM" ;;
esac
oci_user="$OCI_SPX_USER_OCID"
oci_fingerprint="$OCI_SPX_FINGERPRINT"
oci_tenancy="$OCI_SPX_TENANCY_OCID"
oci_region="$OCI_SPX_REGION"
source_name="the OCI_SPX_* environment credential"
elif [ -r "$HOME/.oci/config" ] && python3 - "$SPX_PROFILE" <<'PY'
import configparser, pathlib, sys
cfg = configparser.RawConfigParser()
cfg.read(pathlib.Path.home() / ".oci" / "config")
sys.exit(0 if cfg.has_section(sys.argv[1]) else 1)
PY
then
# One pipe-separated line read into named variables, rather than eval of
# generated shell: none of these fields can contain a pipe.
profile_fields=$(python3 - "$SPX_PROFILE" <<'PY'
import configparser, pathlib, sys
cfg = configparser.RawConfigParser()
cfg.read(pathlib.Path.home() / ".oci" / "config")
section = dict(cfg[sys.argv[1]])
fields = ("user", "fingerprint", "tenancy", "region", "key_file")
missing = [f for f in fields if not section.get(f)]
if missing:
sys.exit(f"profile {sys.argv[1]} has no {', '.join(missing)}")
print("|".join(section[f] for f in fields))
PY
) || die "could not read the [$SPX_PROFILE] profile from ~/.oci/config"
IFS='|' read -r profile_user profile_fingerprint profile_tenancy \
profile_region profile_key_file <<EOF
$profile_fields
EOF
profile_key=$(printf '%s' "$profile_key_file" | sed "s|^~|$HOME|")
[ -r "$profile_key" ] || die "key_file in [$SPX_PROFILE] is not readable: $profile_key"
key_pem=$(cat "$profile_key")
oci_user="$profile_user"
oci_fingerprint="$profile_fingerprint"
oci_tenancy="$profile_tenancy"
oci_region="$profile_region"
source_name="the [$SPX_PROFILE] profile in ~/.oci/config"
elif [ -r "$SPX_KEY" ]; then
require_complete "the credential oci_env.py resolved" \
TF_VAR_user_ocid TF_VAR_tenancy_ocid TF_VAR_region
key_pem=$(cat "$SPX_KEY")
oci_user="${TF_VAR_user_ocid:-}"
oci_fingerprint=$(fingerprint_of "$SPX_KEY")
oci_tenancy="${TF_VAR_tenancy_ocid:-}"
oci_region="${TF_VAR_region:-}"
source_name="$SPX_KEY with the user and tenancy of the Terraform credential"
[ -n "$oci_fingerprint" ] || die "could not read an RSA public key out of $SPX_KEY"
else
require_complete "the credential oci_env.py resolved" \
TF_VAR_user_ocid TF_VAR_fingerprint TF_VAR_tenancy_ocid TF_VAR_region
if [ -n "${TF_VAR_private_key:-}" ]; then
key_pem="$TF_VAR_private_key"
elif [ -n "${TF_VAR_private_key_path:-}" ] && [ -r "$TF_VAR_private_key_path" ]; then
key_pem=$(cat "$TF_VAR_private_key_path")
else
die "no node credential found: set OCI_SPX_PRIVATE_KEY, add a [$SPX_PROFILE] profile to ~/.oci/config, or create $SPX_KEY"
fi
oci_user="${TF_VAR_user_ocid:-}"
oci_fingerprint="${TF_VAR_fingerprint:-}"
oci_tenancy="${TF_VAR_tenancy_ocid:-}"
oci_region="${TF_VAR_region:-}"
source_name="the credential Terraform builds with"
log "WARNING: falling back to $source_name, which is wider than the allocator needs."
log "WARNING: see the OCI guide for a user scoped to vnics, private-ips and public-ips."
fi
case "$oci_user$oci_tenancy" in
ocid1.*) ;;
*) die "$source_name did not yield OCIDs for the user and tenancy" ;;
esac
echo "$oci_fingerprint" | grep -Eq '^[0-9a-f]{2}(:[0-9a-f]{2}){15}$' \
|| die "$source_name has no 16-byte hex fingerprint: '$oci_fingerprint'"
# The fingerprint and the key have to be the same key, and nothing upstream checks
# that they are. Unchecked it is an OCI 401 at the first allocation, long after the
# deploy reports success.
pem_tmp=$(mktemp); trap 'rm -f "$pem_tmp"' EXIT
( umask 077; printf '%s\n' "$key_pem" > "$pem_tmp" )
derived=$(fingerprint_of "$pem_tmp")
[ -n "$derived" ] || die "$source_name does not hold a usable RSA private key"
[ "$derived" = "$oci_fingerprint" ] \
|| die "the key from $source_name has fingerprint $derived, but the credential names $oci_fingerprint"
log "credential: $source_name"
log "user: $oci_user"
log "fingerprint: $oci_fingerprint"
log "region: $oci_region"
if [ "$DRY_RUN" = 1 ]; then
log "would write $REMOTE_CONFIG and $REMOTE_PEM as 0640 root:spinifex on:${*:-" (no hosts given)"}"
exit 0
fi
# base64 so the PEM survives two shells, and carried in the stdin stream rather
# than in the command: an argument is readable from ps on the node for the length
# of the call, on a host that runs other people's guests.
key_b64=$(printf '%s\n' "$key_pem" | base64 | tr -d '\n')
install_on() {
# Unquoted heredoc on purpose: the key and the four credential fields are
# expanded here, so nothing in it is expanded on the node.
# shellcheck disable=SC2087
ssh -i "$ssh_key" "${SSH_OPTS[@]}" "ubuntu@$1" bash -s <<REMOTE
set -eu
getent group spinifex >/dev/null \
|| { echo "no spinifex group yet, so this ran before the install" >&2; exit 3; }
sudo install -d -m 0750 -o root -g spinifex $REMOTE_DIR
# install /dev/null first, then write: the file never exists at a wider mode, and
# sudo's umask is not the caller's.
sudo install -m 0640 -o root -g spinifex /dev/null $REMOTE_PEM
base64 -d <<'PEM_B64' | sudo tee $REMOTE_PEM >/dev/null
$key_b64
PEM_B64
sudo install -m 0640 -o root -g spinifex /dev/null $REMOTE_CONFIG
sudo tee $REMOTE_CONFIG >/dev/null <<'CONFIG'
[$REMOTE_PROFILE]
user=$oci_user
fingerprint=$oci_fingerprint
tenancy=$oci_tenancy
region=$oci_region
key_file=$REMOTE_PEM
CONFIG
REMOTE
}
# Every host is attempted and the failures are named. Stopping at the first leaves
# a half-credentialled cluster and a log that does not say which half.
failed=""
for host in "$@"; do
log "installing the OCI credential on $host"
install_on "$host" || failed="$failed $host"
done
[ -z "$failed" ] || die "failed on:$failed"
log "credential installed on $# host(s)"
scripts/terraform/oci-spx/validate-topology.sh
Executable
+853
View File
@@ -0,0 +1,853 @@
#!/bin/bash
# Builds one OCI topology from nothing, proves Spinifex works on it, and destroys
# it again. Three topologies, because each breaks differently: bare metal presents
# VNICs unlike a VM, a single node has no Geneve underlay to get wrong, and only a
# cluster exercises RAFT, the gateway chassis and cross-node allocation.
#
# Every run is build-prove-destroy. A topology that cannot be torn down is half
# proved, so the teardown decides the verdict and runs even on failure.
#
# MUST MAINTAIN BASH 3.X COMPATABILITY. Operators run this on macOS, which ships
# bash 3.2: no declare -A, no mapfile, and under set -u an empty "${arr[@]}" is an
# unbound variable, so guard every such expansion with ${#arr[@]}.
set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$HERE/../../.." && pwd)"
TOPOLOGY=""
SSH_PUBLIC_KEY="$HOME/.ssh/oci-spx.pub"
SSH_PRIVATE_KEY="$HOME/.ssh/oci-spx"
KEEP=0
KEEP_ON_FAIL=0
SKIP_WORKLOAD=0
SKIP_POOL=0
DRY_RUN=0
DESTROY_ONLY=0
INSTANCE_PRINCIPAL=0
CREDENTIAL_HOOK="${OCI_CREDENTIAL_HOOK:-}"
# Empty means oci_env.py chooses, which is $OCI_CLI_PROFILE, then the reference
# tenancy, then DEFAULT. Named here so a deployment can say which tenancy it is in.
OCI_PROFILE="${OCI_PROFILE:-}"
DISTRO=""
SETUP_SH=""
# Empty means the driver's own default list. Unset is distinguishable from empty,
# so --workbooks "" can deliberately mean "run none".
WORKBOOKS_SET=0
CHANNEL=latest
INSTALL_VERSION=""
WORKBOOKS=""
# Declared here, not at the apply that fills it: the EXIT trap reads it, and under
# set -u an unset array is an error rather than an empty one.
HOSTS=()
TOPOLOGIES="bm vm-single vm-multi vm-single-principal"
# What a topology is worth when nothing says otherwise, so a CI run of a named
# topology is the same every time. Supplied as TF_VAR_, which terraform.auto.tfvars
# outranks, so a deployment sizes itself in that file and no flag has to carry it.
#
# TOPO_FORCE_PRINCIPAL is the exception: vm-single-principal exists to exercise
# instance-principal authentication, so the credential is the topology rather than
# a separate switch that could be left at the other value.
#
# A case rather than an associative array: macOS ships bash 3.2, where declare -A
# is an indexed assignment and the key is read as an arithmetic variable.
topo_defaults() {
TOPO_FORCE_PRINCIPAL=0
case "$1" in
bm) TOPO_DEFAULT_SHAPE="BM.Standard.E2.64"; TOPO_DEFAULT_NODES=1 ;;
vm-single) TOPO_DEFAULT_SHAPE="VM.Standard.E6.Flex"; TOPO_DEFAULT_NODES=1 ;;
vm-multi) TOPO_DEFAULT_SHAPE="VM.Standard.E6.Flex"; TOPO_DEFAULT_NODES=3 ;;
vm-single-principal)
# Its own tenancy carries its own service limits, so the shape every
# other topology uses can have zero quota there and fail the apply
# before a single thing about the principal has been proved.
TOPO_DEFAULT_SHAPE="${OCI_PRINCIPAL_SHAPE:-VM.Standard.E6.Flex}"
TOPO_DEFAULT_NODES=1
TOPO_FORCE_PRINCIPAL=1 ;;
*) return 1 ;;
esac
}
log() { printf '[validate-%s] %s\n' "${TOPOLOGY:-?}" "$*"; }
die() {
printf '[validate-%s] ERROR: %s\n' "${TOPOLOGY:-?}" "$*" >&2
exit 1
}
usage() {
cat >&2 <<EOF
usage: ${0##*/} --topology <bm|vm-single|vm-multi|vm-single-principal> [options]
Builds the topology, installs Spinifex, forms the cluster, runs a Terraform
workbook against it, then destroys everything.
--topology <name> Required. No default: a command aimed at the wrong
topology is the easiest expensive mistake here.
vm-single-principal is vm-single authenticating as the
instance, and implies --instance-principal.
--ssh-public-key PATH Public half installed on each node. Default $SSH_PUBLIC_KEY
--ssh-private-key PATH Private half used to reach them. Default $SSH_PRIVATE_KEY
--instance-principal Configure the pool with oci_auth="instance_principal"
instead of a key file. Needs the dynamic group and
policy to exist already; see instance-principal.tf.
Not needed when instance_principal is set in your
tfvars, which this reads; the flag outranks it.
--credential-hook PATH Executable run after formation, before the pool, as
"hook <ssh-key> <host>...". Where an API-key deployment
installs its credential. Default \$OCI_CREDENTIAL_HOOK.
--oci-profile NAME Profile in ~/.oci/config that Terraform builds with.
Default \$OCI_PROFILE, else \$OCI_CLI_PROFILE, else the
reference tenancy, else DEFAULT. A name that is passed
and absent is an error, never a fallback.
--workbooks LIST Space-separated workbooks for the published driver to
run on the cluster. Empty means run none; omitted means
the driver's own default list.
--version TAG Install this exact release tag. Preferred over
--channel dev for anything repeatable: a tag resolves
by redirect, while the dev channel resolves through a
rate-limited GitHub API call that 404s when it trips.
--distro PATH Install this distro tarball instead of the published
release, so what is proved is the ref it was built from.
--setup-sh PATH setup.sh to pair with --distro. Both or neither.
--no-external-pool Form without an OCI credential: no external pool and no
allocator gate. Implies --skip-workload, because a guest
with no public address proves nothing a customer wants.
--skip-workload Form and verify only; launch no guests.
--keep Leave the infrastructure up. Implies no verdict.
--keep-on-fail Destroy on success, leave a failure up to inspect.
--destroy-only Destroy whatever this topology's state holds, then stop.
--dry-run Print the plan and the per-node steps, change nothing.
EOF
exit 2
}
while [ $# -gt 0 ]; do
case "$1" in
--topology) TOPOLOGY="${2:?}"; shift 2 ;;
--ssh-public-key) SSH_PUBLIC_KEY="${2:?}"; shift 2 ;;
--ssh-private-key) SSH_PRIVATE_KEY="${2:?}"; shift 2 ;;
--workbooks) WORKBOOKS_SET=1; WORKBOOKS="${2-}"; shift 2 ;;
--channel) CHANNEL="${2-}"; shift 2 ;;
--version) INSTALL_VERSION="${2:?}"; shift 2 ;;
--distro) DISTRO="${2:?}"; shift 2 ;;
--setup-sh) SETUP_SH="${2:?}"; shift 2 ;;
--instance-principal) INSTANCE_PRINCIPAL=1; shift ;;
--credential-hook) CREDENTIAL_HOOK="${2:?}"; shift 2 ;;
--oci-profile) OCI_PROFILE="${2:?}"; shift 2 ;;
--no-external-pool) SKIP_POOL=1; SKIP_WORKLOAD=1; shift ;;
--skip-workload) SKIP_WORKLOAD=1; shift ;;
--keep) KEEP=1; shift ;;
--keep-on-fail) KEEP_ON_FAIL=1; shift ;;
--destroy-only) DESTROY_ONLY=1; shift ;;
--dry-run) DRY_RUN=1; shift ;;
-h | --help) usage ;;
*) die "unknown option $1" ;;
esac
done
[ -n "$TOPOLOGY" ] || usage
topo_defaults "$TOPOLOGY" || die "unknown topology $TOPOLOGY; one of: $TOPOLOGIES"
[ -r "$SSH_PUBLIC_KEY" ] || die "no readable SSH public key at $SSH_PUBLIC_KEY"
[ -r "$SSH_PRIVATE_KEY" ] || die "no readable SSH private key at $SSH_PRIVATE_KEY"
# Both or neither: a distro with the published setup.sh installs one ref's bytes
# with another's layout, and that is a cluster nobody can reason about.
if [ -n "$DISTRO" ] || [ -n "$SETUP_SH" ]; then
[ -r "$DISTRO" ] || die "--distro is not readable: '$DISTRO'"
[ -r "$SETUP_SH" ] || die "--setup-sh is not readable: '$SETUP_SH'"
fi
if [ "$TOPO_FORCE_PRINCIPAL" = 1 ]; then
INSTANCE_PRINCIPAL=1
# The pool is the whole point of this topology: with no allocator built, the
# credential is never used and a green run says nothing about it. That is
# exactly how the auth branch stayed broken through passing nightlies.
[ "$SKIP_POOL" != 1 ] \
|| die "--no-external-pool makes $TOPOLOGY meaningless: the allocator is what exercises the instance principal, so a run without it proves nothing"
fi
# Terraform's own order, as far as this script needs it: the last assignment across
# terraform.tfvars then *.auto.tfvars alphabetically is the one that applies. Read
# because the credential-hook gate has to run before the apply spends anything; the
# apply's own output is checked against this afterwards.
tfvars_instance_principal() {
local file mode="" found=""
for file in "$HERE/terraform.tfvars" "$HERE"/*.auto.tfvars; do
[ -r "$file" ] || continue
mode=$(sed -n 's/^[[:space:]]*instance_principal[[:space:]]*=[[:space:]]*"\([a-z]*\)".*/\1/p' \
"$file" | tail -1)
[ -n "$mode" ] && found="$mode"
done
[ -n "$found" ] || return 1
printf '%s\n' "$found"
}
# adopt, never create: this script's state is destroyed at the end of every run, and
# the dynamic group and policy outlive every topology. setup-identity.sh owns them.
#
# A flag goes in as -var and outranks the tfvars, because it is a choice about this
# run. Without one the deployment's own tfvars decides, so a guide can set the
# credential in the file it already writes and no flag has to carry it.
tf_principal_var=()
if [ "$INSTANCE_PRINCIPAL" = 1 ]; then
PRINCIPAL_MODE=adopt
tf_principal_var=(-var "instance_principal=$PRINCIPAL_MODE")
else
PRINCIPAL_MODE="$(tfvars_instance_principal || echo off)"
fi
# What the apply actually built, filled in from the hosts_file output once it has.
# Until then these are what the topology asked for, which is all there is to say.
SHAPE="$TOPO_DEFAULT_SHAPE"
NODES="$TOPO_DEFAULT_NODES"
# Outside the checkout on a persistent runner, because actions/checkout runs
# git clean -ffdx: it would delete the Terraform state of a killed run before the
# sweep could read it, leaving instances running that nothing can find.
STATE_ROOT="${OCI_STATE_ROOT:-$HERE}"
mkdir -p "$STATE_ROOT"
STATE_DIR="$STATE_ROOT/.validate-$TOPOLOGY"
RESULTS="$STATE_DIR/results.txt"
# Tab-separated siblings of the human block, for the run-page tables. Written here
# rather than parsed out of the log later: the status of a gate is then a field
# this script sets, not a regex over prose that reads fine and matches wrong.
RESULTS_TSV="$STATE_DIR/results.tsv"
WORKBOOKS_TSV="$STATE_DIR/workbooks.tsv"
# A separate state directory per topology, so two topologies can be built from one
# checkout without one destroying the other's instances.
tf() {
# The topology's shape and count go in as TF_VAR_, which is the weakest source
# Terraform reads: a terraform.auto.tfvars in the checkout outranks it, so a
# deployment sizes itself in that file and the topology only supplies the
# default a CI run wants.
TF_VAR_compute_shape="$TOPO_DEFAULT_SHAPE" TF_VAR_node_count="$TOPO_DEFAULT_NODES" \
python3 "$HERE/scripts/oci_env.py" --ssh-public-key-path "$SSH_PUBLIC_KEY" \
${OCI_PROFILE:+--profile "$OCI_PROFILE"} -- \
terraform -chdir="$HERE" "$@"
}
# terraform output takes -state but not -var, so the two sets are kept apart. They
# were one set once, and the -var made output fail into the default state file.
tf_state=(-state "$STATE_DIR/terraform.tfstate")
tf_vars=("${tf_state[@]}")
# Expanded only when it holds something. bash before 4.4, which is what macOS
# ships, calls "${arr[@]}" on an empty array an unbound variable under set -u, and
# tf_principal_var is empty on every run that did not pass --instance-principal.
if [ "${#tf_principal_var[@]}" -gt 0 ]; then
tf_vars=("${tf_principal_var[@]}" "${tf_vars[@]}")
fi
ssh_node() {
local host="$1"
shift
# LogLevel=ERROR: the host-key warning is unavoidable with a throwaway known-hosts
# file, and it lands in the middle of whatever the remote command reported.
ssh -i "$SSH_PRIVATE_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-o LogLevel=ERROR -o BatchMode=yes -o ConnectTimeout=15 "ubuntu@$host" "$@"
}
# record <gate> <PASS|FAIL|SKIPPED|INFO> [detail]
# The node's own account of a failed run. A workbook's log says which API call
# failed; only the journal says why, and the nodes are destroyed minutes later.
# Warnings first because that is where a swallowed error surfaces, then a bounded
# tail for context -- unbounded, a multi-hour suite's journal dwarfs the artifact.
capture_journals() {
local host
# A failure before the apply leaves no hosts, and on bash before 4.4 iterating
# the empty array is an unbound-variable error rather than zero passes.
[ "${#HOSTS[@]}" -gt 0 ] || return 0
for host in "${HOSTS[@]}"; do
ssh_node "$host" '
echo "=== spinifex, warning and above ==="
sudo journalctl -u "spinifex-*" --since -4h --priority=warning --no-pager | tail -2000
echo "=== spinifex, all priorities, last 2000 lines ==="
sudo journalctl -u "spinifex-*" --since -4h --no-pager | tail -2000
' > "$STATE_DIR/journal-$host.log" 2>&1 || log "could not collect the journal from $host"
done
}
record() {
local gate="$1" status="$2" detail="${3-}"
printf '%s: %s%s\n' "$gate" "$status" "${detail:+ $detail}" >> "$RESULTS"
printf '%s\t%s\t%s\n' "$gate" "$status" "$detail" >> "$RESULTS_TSV"
}
# The driver's per-workbook lines, which are go test's own shape because
# go-junit-report consumes them downstream. A workbook the driver never reached
# is absent from its log and so absent here, which the summary renders as a row
# rather than dropping — a missing row reads as a pass.
record_workbooks() {
local log="$1"
[ -r "$log" ] || return 0
sed -n 's/^--- \(PASS\|FAIL\): TestTofuWorkbook_\([A-Za-z0-9_]*\) (\([0-9]*\)\.[0-9]*s)$/\2\t\1\t\3/p' \
"$log" | tr '_' '-' > "$WORKBOOKS_TSV"
}
# Teardown is the last assertion, not cleanup: a workbook or a topology that
# cannot be destroyed is a defect, and it has been one before.
DESTROY_RC=""
destroy_topology() {
log "destroying"
if tf destroy -auto-approve -no-color "${tf_vars[@]}" > "$STATE_DIR/destroy.log" 2>&1; then
DESTROY_RC=0
record destroy PASS
else
DESTROY_RC=1
record destroy FAIL "see $STATE_DIR/destroy.log"
log "TEARDOWN FAILED — resources may still be billing. $STATE_DIR/destroy.log"
fi
}
cleanup() {
local rc=$?
if [ "$KEEP" = 1 ]; then
log "--keep: leaving the infrastructure up, no verdict recorded"
# The command to get in, printed rather than left to be assembled from an
# address elsewhere in the log. HOSTS is empty if we never got as far as
# the apply, which is why this is guarded rather than assumed.
if [ "${#HOSTS[@]}" -gt 0 ]; then
# ~ rather than the expanded path, so the line is the same on every
# machine and reads as something a person would type.
local key="$SSH_PRIVATE_KEY"
case "$key" in "$HOME"/*) key="~${key#"$HOME"}" ;; esac
log "ssh in with: ssh -i $key ubuntu@${HOSTS[0]}"
log "destroy it with: $0 --topology $TOPOLOGY --destroy-only"
fi
return
fi
# Only on a failure, and said out loud: the cost of forgetting is an OCI bare
# metal host running overnight.
if [ "$KEEP_ON_FAIL" = 1 ] && [ "$rc" != 0 ]; then
log "--keep-on-fail: $TOPOLOGY FAILED and is being left up. IT IS STILL BILLING."
log "destroy it with: $0 --topology $TOPOLOGY --destroy-only"
verdict "$rc"
return
fi
[ "$DRY_RUN" = 1 ] && return
destroy_topology
verdict "$rc"
}
verdict() {
local run_rc="$1"
echo
log "=== $TOPOLOGY ($SHAPE, $NODES node(s)) ==="
[ -r "$RESULTS" ] && cat "$RESULTS"
if [ "$run_rc" = 0 ] && [ "$DESTROY_RC" = 0 ]; then
log "PASS"
else
log "FAIL"
fi
}
mkdir -p "$STATE_DIR"
if [ "$DESTROY_ONLY" = 1 ]; then
log "destroy only: whatever $STATE_DIR/terraform.tfstate holds"
destroy_topology
[ "$DESTROY_RC" = 0 ] || exit 1
exit 0
fi
: > "$RESULTS"
: > "$RESULTS_TSV"
: > "$WORKBOOKS_TSV"
# Last run's logs go before this one's first line, so what is left in here always
# describes the run that is starting. Without this a topology keeps logs that read
# as current, and anything collecting the directory publishes them as this run's.
# Named globs rather than a find: the tfstate is how the sweep finds hosts to
# destroy, so what is removed here has to be readable at a glance.
# :? so an unset STATE_DIR stops the shell rather than expanding to /*.log.
rm -f "${STATE_DIR:?}"/*.log "${STATE_DIR:?}"/nodes.txt "${STATE_DIR:?}"/cloudinit-*.txt
log "$SHAPE, $NODES node(s), state in $STATE_DIR"
# Checked here, not where the hook is run: nothing about it depends on the apply, and
# the run is otherwise forty minutes and a bare-metal bill from discovering that the
# path in a CI variable does not exist on this runner.
if [ "$SKIP_POOL" != 1 ] && [ "$PRINCIPAL_MODE" = off ]; then
[ -n "$CREDENTIAL_HOOK" ] \
|| die "this deployment authenticates with an API key and no credential hook is set, so no node could reach the OCI API; set --credential-hook or OCI_CREDENTIAL_HOOK, or use an instance principal"
[ -x "$CREDENTIAL_HOOK" ] || die "credential hook is not executable: $CREDENTIAL_HOOK"
fi
# The directory's own terraform.tfstate belongs to hand-driven runs, not to a
# topology, and an instance left in it is both a bill and a name that collides
# with ours in the console. Say so rather than letting it be a surprise.
if [ -s "$HERE/terraform.tfstate" ]; then
stray="$(python3 -c '
import json, sys
state = json.load(open(sys.argv[1]))
for res in state.get("resources", []):
if res["type"] != "oci_core_instance":
continue
for inst in res["instances"]:
attrs = inst["attributes"]
if attrs.get("state") not in ("TERMINATED", None):
print(attrs.get("display_name"), attrs.get("shape"), attrs.get("state"), attrs.get("public_ip"))
' "$HERE/terraform.tfstate" 2>/dev/null || true)"
[ -n "$stray" ] && log "NOTE: the hand-driven state in $HERE still holds: $stray"
fi
# Terraform ranks a *.auto.tfvars file above TF_VAR_*, so a leftover one wins
# silently over an operator targeting a different tenancy by environment. The
# guide tells every operator to write one, which is what makes this likely.
assert_no_tfvars_override() {
local f want got
want="${TF_VAR_compartment_ocid:-}"
[ -n "$want" ] || return 0
for f in "$HERE"/*.auto.tfvars "$HERE"/terraform.tfvars; do
[ -f "$f" ] || continue
got=$(sed -n 's/^[[:space:]]*compartment_ocid[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' "$f" | tail -1)
[ -n "$got" ] || continue
[ "$got" = "$want" ] && continue
die "${f##*/} sets compartment_ocid=$got, which Terraform ranks above the TF_VAR_compartment_ocid=$want this run was given, so it would build in a compartment nobody chose. Remove that file or make it agree."
done
}
assert_no_tfvars_override
if [ "$DRY_RUN" = 1 ]; then
# Whole plan to a file, summary to the terminal. A dry run is here to catch a
# plan aimed at the wrong compartment, and `tail` dropped the resource count
# and every compartment and shape line along with it, leaving only the outputs
# block -- so the one check it exists for could not be made from its output.
mkdir -p "$STATE_DIR"
tf plan -no-color "${tf_vars[@]}" > "$STATE_DIR/plan.txt" 2>&1 \
|| { tail -30 "$STATE_DIR/plan.txt"; die "plan failed; see $STATE_DIR/plan.txt"; }
grep -E '^Plan: |^(No changes)' "$STATE_DIR/plan.txt" || true
printf 'compartment and shape as planned:\n'
# Terraform aligns the = to the widest key in each block, so the same value
# appears several times at several indents unless the spacing is normalised.
grep -hoE '(compartment_id|shape|availability_domain) += +"[^"]*"' "$STATE_DIR/plan.txt" \
| sed -E 's/ +=/ =/' | sort -u | sed 's/^/ /'
log "full plan in $STATE_DIR/plan.txt"
plan="install Spinifex on $NODES node(s), form the cluster"
[ "$SKIP_POOL" = 1 ] && plan="$plan, configure no external pool"
if [ "$SKIP_WORKLOAD" = 1 ]; then
plan="$plan, launch no guests"
elif [ "$WORKBOOKS_SET" = 1 ]; then
plan="$plan, run the workbooks '$WORKBOOKS'"
else
plan="$plan, run the driver's default workbooks"
fi
log "dry run: would $plan, then destroy"
exit 0
fi
trap cleanup EXIT
tf init -input=false > "$STATE_DIR/init.log" 2>&1 || die "terraform init failed; see $STATE_DIR/init.log"
log "building"
tf apply -auto-approve -no-color "${tf_vars[@]}" > "$STATE_DIR/apply.log" 2>&1 \
|| die "terraform apply failed; see $STATE_DIR/apply.log"
record build PASS
# A read loop rather than mapfile, which bash 3.2 on macOS does not have. The
# output is captured first so a failed terraform fails here, not silently as an
# empty host list.
hosts_raw=$(tf output -raw "${tf_state[@]}" hosts_file) \
|| die "could not read the hosts_file output from $STATE_DIR/terraform.tfstate"
HOSTS=()
while IFS= read -r line; do
if [ -n "$line" ]; then HOSTS+=("$line"); fi
done <<EOF
$hosts_raw
EOF
[ "${#HOSTS[@]}" -ge 1 ] || die "the hosts_file output named no hosts, so the apply built nothing to install on"
for host in "${HOSTS[@]}"; do
[[ "$host" =~ ^[0-9]+(\.[0-9]+){3}$ ]] \
|| die "the hosts_file output is not an address: '$host' -- something on the wrapper's stdout is in the capture"
done
# Written for every topology, not just the multi-node one that passes it to
# install-node.sh: it is also how an operator gets the addresses afterwards
# without a terraform invocation and the right -state path.
printf '%s\n' "${HOSTS[@]}" > "$STATE_DIR/hosts"
# node_count can come from a tfvars file, so the hosts the apply produced are the
# authority on how many there are. Every later count check reads this.
if [ "${#HOSTS[@]}" != "$NODES" ]; then
log "note: $NODES node(s) is the $TOPOLOGY default, and the apply built ${#HOSTS[@]}; taking the apply's"
NODES="${#HOSTS[@]}"
fi
SHAPE=$(tf output -raw "${tf_state[@]}" compute_shape) \
|| die "could not read the compute_shape output from $STATE_DIR/terraform.tfstate"
# Terraform is the authority on which credential was built, and the reader above is
# a guess made before the apply. They have to agree: the hook is skipped on the
# strength of that guess, and a node left with no credential at all would otherwise
# fail much later as an allocator that cannot resolve its VNIC.
built_principal=$(tf output -raw "${tf_state[@]}" instance_principal) \
|| die "could not read the instance_principal output from $STATE_DIR/terraform.tfstate"
[ "$built_principal" = "$PRINCIPAL_MODE" ] \
|| die "read instance_principal='$PRINCIPAL_MODE' from the tfvars and the apply built '$built_principal'; set it in one place"
log "hosts: ${HOSTS[*]}"
log "$SHAPE, $NODES node(s), oci_auth $([ "$PRINCIPAL_MODE" = off ] && echo "from a key file" || echo "as the instance principal")"
# cloud-init owns the volume, the ports and br-wan, and all three are units now, so
# the question is whether they converged rather than whether a runcmd happened to
# run. This is the gate the bare-metal findings were about.
log "waiting for cloud-init on each node"
for host in "${HOSTS[@]}"; do
for _ in $(seq 1 120); do
if ssh_node "$host" 'cloud-init status --wait >/dev/null 2>&1 || true; test -f /var/lib/cloud/instance/boot-finished' 2>/dev/null; then
break
fi
sleep 10
done
ssh_node "$host" '
set -e
for u in spinifex-data-volume spinifex-service-ports spinifex-wan-bridge; do
printf "%s=%s " "$u" "$(systemctl is-active "$u" 2>&1)"
done
echo
findmnt --noheadings --output SOURCE,TARGET /var/lib/spinifex
ip -br addr show br-wan
' > "$STATE_DIR/cloudinit-$host.txt" 2>&1 \
|| die "cloud-init did not converge on $host; see $STATE_DIR/cloudinit-$host.txt"
log "$host cloud-init: $(head -1 "$STATE_DIR/cloudinit-$host.txt")"
done
record "cloud-init units" PASS
# With --distro the bytes are the ref under test; without it they are the published
# release, which answers a different question -- what a customer following the
# guide gets today. A nightly judging a branch must pass --distro.
log "installing Spinifex on each node ($([ -n "$DISTRO" ] && echo "$(basename "$DISTRO")" || echo "published release"))"
for host in "${HOSTS[@]}"; do
if [ -n "$DISTRO" ]; then
scp -i "$SSH_PRIVATE_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-o LogLevel=ERROR -q "$DISTRO" "$SETUP_SH" "ubuntu@$host:/tmp/" \
|| die "could not copy the distro to $host"
fi
# cloud-init being finished does not mean apt is: the apt-daily timers and
# unattended-upgrades run on their own schedule and hold the dpkg lock, which
# the installer then fails on. Wait for the lock rather than fight it.
ssh_node "$host" "DISTRO_NAME='$(basename "${DISTRO:-}")' SETUP_NAME='$(basename "${SETUP_SH:-}")' CHANNEL='$CHANNEL' INSTALL_VERSION='$INSTALL_VERSION' bash -s" \
> "$STATE_DIR/install-$host.log" 2>&1 <<'REMOTE' \
|| die "install failed on $host; see $STATE_DIR/install-$host.log"
set -e
for _ in $(seq 1 120); do
sudo fuser /var/lib/dpkg/lock-frontend /var/lib/apt/lists/lock >/dev/null 2>&1 || break
sleep 5
done
sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 \
&& { echo "the dpkg lock was still held after 10 minutes"; exit 1; }
if [ -n "${DISTRO_NAME:-}" ]; then
# The production setup.sh against the ref's own tarball, which is what the
# release and the ISO both run, so there is no separate dev install layout.
# No INSTALL_SPINIFEX_SKIP_APT/SKIP_AWS here, unlike the CI hypervisors: an OCI
# stock image is not pre-baked, so the dependency stages have to run.
sudo env INSTALL_SPINIFEX_TARBALL="/tmp/$DISTRO_NAME" bash "/tmp/$SETUP_NAME"
else
# The real customer path, including the checksum step a local tarball skips.
# The environment variable rather than --channel: the installer served by
# install.mulgadc.com is the latest release's, so it predates the flag.
#
# A tag goes in as VERSION, not CHANNEL. Only the dev channel resolves through
# an unauthenticated GitHub API call, which is rate limited per source address
# and returns 404 once it trips -- indistinguishable from a missing asset. A
# tagged path is a plain redirect, so it does not have that failure mode.
if [ -n "$INSTALL_VERSION" ]; then
curl -sfL https://install.mulgadc.com | sudo env INSTALL_SPINIFEX_VERSION="$INSTALL_VERSION" bash
else
curl -sfL https://install.mulgadc.com | sudo env INSTALL_SPINIFEX_CHANNEL="$CHANNEL" bash
fi
fi
sudo /usr/local/share/spinifex/setup-ovn.sh --management --nat-uplink
REMOTE
INSTALLED_VERSION="$(ssh_node "$host" 'spx version' 2>&1 | head -1)"
log "$host installed: $INSTALLED_VERSION"
done
record install PASS "$INSTALLED_VERSION"
# A single node is its own documented path: install-node.sh refuses fewer than two
# hosts, because there is nothing to join. Both branches use the flags the guide
# names, so what is validated is what the guide tells a customer to type.
if [ "$NODES" = 1 ]; then
log "initializing the single node"
ssh_node "${HOSTS[0]}" '
set -e
sudo spx admin init --node "$(hostname -s)" --nodes 1 \
--region ap-southeast-2 --az ap-southeast-2a \
--external-mode=nat --ipsec=false
sudo systemctl start spinifex.target
' > "$STATE_DIR/form.log" 2>&1 \
|| die "single-node init failed; see $STATE_DIR/form.log"
else
log "forming the cluster"
"$REPO_ROOT/scripts/install-node.sh" \
--hosts-file "$STATE_DIR/hosts" \
--user ubuntu \
--identity "$SSH_PRIVATE_KEY" \
--external-mode nat \
--ipsec off \
--yes > "$STATE_DIR/form.log" 2>&1 \
|| die "formation failed; see $STATE_DIR/form.log"
fi
record formation PASS
# Runs after formation and before the pool is configured, which is the only window
# where a node has /etc/spinifex but has not yet started the allocator. Nothing in
# this repository knows what it does: an API-key deployment needs a credential on
# each node, and a credential belongs to the operator, not to a checked-in script.
# Skipped under instance principal, which needs no handoff at all.
if [ "$SKIP_POOL" != 1 ] && [ "$PRINCIPAL_MODE" = off ]; then
log "running the credential hook"
# Arguments, not a file: the hook is told where the nodes are and how to reach
# them, and decides for itself what to put there.
"$CREDENTIAL_HOOK" "$SSH_PRIVATE_KEY" "${HOSTS[@]}" > "$STATE_DIR/credential-hook.log" 2>&1 \
|| die "the credential hook failed; see $STATE_DIR/credential-hook.log"
log "credential hook ok"
fi
# The IMDS remap and the pool are both set-before-first-start, and the remap is the
# half that is invisible when missing: without it the cloud's metadata service is
# shadowed by Spinifex's own endpoints and the allocator cannot resolve its VNIC.
log "configuring the IMDS remap$([ "$SKIP_POOL" = 1 ] && echo " (no external pool)" || echo " and the external pool")"
for host in "${HOSTS[@]}"; do
ssh_node "$host" "SKIP_POOL=$SKIP_POOL bash -s" > "$STATE_DIR/pool-$host.log" 2>&1 <<'REMOTE' \
|| die "pool configuration failed on $host; see $STATE_DIR/pool-$host.log"
set -e
sudo cp /etc/spinifex/spinifex.toml /etc/spinifex/spinifex.toml.bak-prepool
# subn, not sub: with no [network] section to match, sub rewrites the file unchanged
# and exits 0, leaving the remap silently unapplied. That is the half of this that is
# invisible when missing, so the substitution count is checked rather than assumed.
grep -q imds_host_meta_ip /etc/spinifex/spinifex.toml || sudo python3 - <<"PY"
import re, sys
path = "/etc/spinifex/spinifex.toml"
text, n = re.subn(r"(?m)^\[network\]$",
"[network]\nimds_host_meta_ip = \"169.254.42.254\"\nimds_host_dns_ip = \"169.254.42.253\"",
open(path).read(), count=1)
if n != 1:
sys.exit("no [network] section in %s, so the IMDS remap was not applied" % path)
open(path, "w").write(text)
PY
if [ "${SKIP_POOL:-0}" != 1 ]; then
grep -q 'name = "oci-public"' /etc/spinifex/spinifex.toml \
|| sudo tee -a /etc/spinifex/spinifex.toml < /etc/spinifex/oci/external-pool.toml >/dev/null
fi
# spx has no config subcommand, so the `spx config validate` that used to be here
# could only ever fail, and was swallowed. Parsing the file is the check that was
# wanted: both edits above are textual, and a broken result would otherwise surface
# as every service failing to start with nothing saying why.
sudo python3 -c 'import sys, tomllib; tomllib.load(open(sys.argv[1], "rb"))' \
/etc/spinifex/spinifex.toml \
|| { echo "spinifex.toml is not valid TOML after the IMDS remap and the pool append"; exit 1; }
sudo systemctl restart spinifex.target
REMOTE
done
# Everything needed to tell the three allocator faults apart, and deliberately no
# credential: the config file and the PEM are reported as mode and size only, which
# distinguishes absent from unreadable without copying either into a log that is
# attached to a CI run.
allocator_diagnostics() {
local host="$1"
ssh_node "$host" '
echo "=== spinifex-daemon: the allocator (ocinet) lives here ==="
sudo journalctl -u spinifex-daemon --since -20min --no-pager | grep -i "ocinet\|external VNIC\|allocator\|oci" | tail -40
echo "=== spinifex-vpcd: consumes the addresses, does not allocate them ==="
sudo journalctl -u spinifex-vpcd --since -20min --no-pager | tail -40
echo "=== external_pools as configured ==="
sudo sed -n "/\[\[network.external_pools\]\]/,\$p" /etc/spinifex/spinifex.toml
echo "=== credential files (mode and size, never content) ==="
sudo stat -c "%n %A %s bytes owner=%U:%G" \
/etc/spinifex/oci/config /etc/spinifex/oci/oci_api_key.pem 2>&1
echo "=== the bridge MAC the allocator matches on ==="
ip -br link show
echo "=== MACs IMDS reports for this instance VNICs ==="
curl -sf -H "Authorization: Bearer Oracle" \
http://169.254.169.254/opc/v2/vnics/ | tr "," "\n" | grep -i "macAddr\|privateIp" || \
echo "IMDS unreachable -- check the remap, Spinifex claims 169.254.169.254 itself"
' 2>&1
}
# credential authorised to allocate is the line that proves the credential is
# accepted, in the right compartment, and holds every permission an allocation
# needs. A node missing it accepts allocate-address and then fails it.
#
# "resolved the external VNIC" is not enough and was what this used to wait for.
# That line comes from the metadata service and needs no IAM policy at all, so a
# node with an unauthorised credential passed here and failed hours later inside
# a workbook, reported as capacity.
ALLOCATOR_READY='ocinet credential authorised to allocate'
ALLOCATOR_DENIED='ocinet credential cannot allocate an external address'
if [ "$SKIP_POOL" = 1 ]; then
log "--no-external-pool: no allocator to check"
record "oci allocator" SKIPPED "no external pool configured"
else
log "checking the allocator came up"
for host in "${HOSTS[@]}"; do
found=0
for _ in $(seq 1 30); do
# spinifex-daemon, not spinifex-vpcd: the allocator is built in the
# daemon, and vpcd only consumes the addresses it hands out. Gating on
# vpcd's journal failed a node whose allocator was working.
if ssh_node "$host" "sudo journalctl -u spinifex-daemon --since -10min --no-pager | grep -q '$ALLOCATOR_READY'" 2>/dev/null; then
found=1
break
fi
# A refusal is final, so waiting out the remaining five minutes only
# delays a verdict the node has already reached.
if denial=$(ssh_node "$host" "sudo journalctl -u spinifex-daemon --since -10min --no-pager | grep -m1 '$ALLOCATOR_DENIED'" 2>/dev/null) && [ -n "$denial" ]; then
allocator_diagnostics "$host" > "$STATE_DIR/allocator-$host.log" 2>&1 || true
die "$host refused its own OCI credential: ${denial#*$ALLOCATOR_DENIED}. Diagnostics: $STATE_DIR/allocator-$host.log"
fi
sleep 10
done
if [ "$found" != 1 ]; then
# Before the teardown, because the teardown is what destroys the only
# copy. Three unrelated faults land here -- a credential the SDK would
# not load, a bridge MAC matching no VNIC, and a pool vpcd never read --
# and they are indistinguishable from the missing log line alone.
allocator_diagnostics "$host" > "$STATE_DIR/allocator-$host.log" 2>&1 || true
die "$host never logged '$ALLOCATOR_READY'; the OCI allocator is not up, so no guest can get a public address. Diagnostics: $STATE_DIR/allocator-$host.log"
fi
log "$host allocator ready"
done
record "oci allocator" PASS
fi
# Parse the STATUS column, never grep for Ready: NotReady contains it, and that
# read passed a single-node run whose only node was NotReady on 0.0.0.0. The table
# is colourised, so the escapes come off first.
count_ready() {
sed 's/\x1b\[[0-9;]*m//g' "$STATE_DIR/nodes.txt" \
| awk -F'|' 'NR > 1 { gsub(/^[ \t]+|[ \t]+$/, "", $2); if ($2 == "Ready") n++ } END { print n + 0 }'
}
log "cluster membership"
ready=0
for _ in $(seq 1 30); do
ssh_node "${HOSTS[0]}" 'sudo spx get nodes' > "$STATE_DIR/nodes.txt" 2>&1 || true
ready="$(count_ready)"
[ "$ready" = "$NODES" ] && break
sleep 10
done
cat "$STATE_DIR/nodes.txt"
[ "$ready" = "$NODES" ] || die "expected $NODES Ready node(s), got $ready; see $STATE_DIR/nodes.txt"
record membership PASS "$ready Ready"
if [ "$SKIP_WORKLOAD" = 1 ]; then
log "--skip-workload: stopping before the workbook"
record workbooks SKIPPED "--skip-workload"
exit 0
fi
# The same driver the GitHub nightly runs (cell 17), on the same published
# workbooks, so a workbook that passes on a hypervisor and fails on OCI is a
# difference in OCI and not in the test. It owns its own per-workbook assertions
# and destroys each one it builds.
# The suite used to run with stdout redirected to a file, so a run wedged inside a
# workbook printed nothing at all until it finished. Streamed now, and each of the
# driver's own RUN/PASS/FAIL lines is stamped with elapsed seconds and the workbook.
stream_workbooks() {
local start=$SECONDS line current="(starting)"
while IFS= read -r line; do
case "$line" in
'=== RUN'*)
current="${line##*RUN }"
printf '[validate-%s] %5ds workbook START %s\n' "$TOPOLOGY" "$((SECONDS - start))" "$current" ;;
'--- PASS'*|'--- FAIL'*|'--- SKIP'*)
printf '[validate-%s] %5ds workbook %s\n' "$TOPOLOGY" "$((SECONDS - start))" "$line" ;;
*)
printf '%s\n' "$line" ;;
esac
done
}
log "running the published workbooks"
DRIVER="$REPO_ROOT/tests/e2e/run-tofu-examples-e2e.sh"
[ -r "$DRIVER" ] || die "no workbook driver at $DRIVER"
scp -i "$SSH_PRIVATE_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-o LogLevel=ERROR -q "$DRIVER" "ubuntu@${HOSTS[0]}:~/run-tofu-examples-e2e.sh" \
|| die "could not copy the workbook driver"
scp -i "$SSH_PRIVATE_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-o LogLevel=ERROR -qr "$REPO_ROOT/docs/terraform-workbooks" "ubuntu@${HOSTS[0]}:~/workbooks" \
|| die "could not copy the workbooks"
# Every AMI the workbooks need, imported one at a time so predastore is not asked
# to absorb parallel uploads. Three are appliances rather than distros: RDS, ECS
# and EKS each boot their own, found by a spinifex:managed-by tag and never by name.
log "importing the images the workbooks need"
ssh_node "${HOSTS[0]}" '
set -e
for img in ubuntu-26.04-x86_64 spinifex-rds-postgres spinifex-ecs-node spinifex-eks-node; do
sudo spx admin images import --name "$img" --config /etc/spinifex/spinifex.toml >/dev/null
done
' > "$STATE_DIR/images.log" 2>&1 || die "image import failed; see $STATE_DIR/images.log"
# Every workbook that is a workbook. The shared driver's own default is five, which
# leaves ECS and all three EKS variants untested on every platform -- their
# assertions exist and nothing was running them.
#
# demo-app is absent because it is not a workbook: it has no .tf at all, being the
# container image the EKS workbooks' nested workloads/ modules deploy. Listing it
# here would fail on a missing root module rather than test anything.
OCI_WORKBOOKS="nginx-alb bastion-private-subnet nginx-webserver s3-webapp rds-quickstart"
OCI_WORKBOOKS="$OCI_WORKBOOKS ecs-quickstart eks-quickstart eks-https-ingress eks-gitops-argocd"
[ "$WORKBOOKS_SET" = 1 ] || WORKBOOKS="$OCI_WORKBOOKS"
workbook_env="WORKBOOK_DIR=\$HOME/workbooks $(printf 'WORKBOOKS=%q' "$WORKBOOKS")"
# Whether a node can reach a public address inside its own VCN. Informational: the
# remedy below is right either way, because a customer reaches a guest from outside.
#
# Needs two hosts. With one, HOSTS[0] and HOSTS[-1] are the same address and the node
# connects to itself, which always succeeds and measures nothing — it reported
# "present" on vm-single and bm for exactly that reason before this guard.
#
# A negative result is only interpretable when the security list admits the source,
# so the record says which it was: narrowing node_client_cidr_allow_list makes a
# node's own public address a non-permitted source, and the refusal that follows
# looks identical to a VCN that does not hairpin.
if [ "${#HOSTS[@]}" -lt 2 ]; then
record "vcn hairpin" SKIPPED "needs two nodes; one node can only probe itself"
elif ssh_node "${HOSTS[0]}" \
"bash -c 'exec 3<>/dev/tcp/${HOSTS[${#HOSTS[@]}-1]}/22' 2>/dev/null"; then
record "vcn hairpin" INFO "present: a node reached another node's public address"
else
record "vcn hairpin" INFO "not reached, which is a VCN without hairpin or a security list that excludes the source"
fi
# Remote dynamic forward: the node gets a SOCKS5 proxy on this port whose egress
# is this host, outside the VCN. It gives the driver the customer's vantage point
# for public addresses without moving the driver off the node, where its ovn-nbctl,
# journalctl and spx diagnostics have to run. The subnet's own allow list does not
# need a new entry: ssh from here already works, and its rules are per-CIDR for
# every protocol.
PUBLIC_PROXY_PORT="${PUBLIC_PROXY_PORT:-1080}"
workbook_env="$workbook_env E2E_PUBLIC_PROXY=127.0.0.1:$PUBLIC_PROXY_PORT"
# ExitOnForwardFailure, so a port already in use stops the run here instead of
# handing the driver a proxy that is not there. Keepalives because this one
# connection carries the proxy for the whole suite, and a dropped forward would
# read as every remaining public address going dark at once.
#
# if !, not a $? read after the fact: under set -e a failing ssh never reaches the
# next line, which is the one that prints the driver's own diagnostics.
if ssh -i "$SSH_PRIVATE_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-o LogLevel=ERROR -o BatchMode=yes -o ConnectTimeout=15 \
-o ServerAliveInterval=30 -o ServerAliveCountMax=6 \
-o ExitOnForwardFailure=yes -R "$PUBLIC_PROXY_PORT" "ubuntu@${HOSTS[0]}" \
"chmod +x ~/run-tofu-examples-e2e.sh; $workbook_env ~/run-tofu-examples-e2e.sh" \
2>&1 | tee "$STATE_DIR/workbooks.log" | stream_workbooks; then
record_workbooks "$STATE_DIR/workbooks.log"
passed="$(grep -c '^--- PASS' "$STATE_DIR/workbooks.log" || true)"
ran="$(grep -c '^=== RUN' "$STATE_DIR/workbooks.log" || true)"
record workbooks PASS "$passed of $ran"
else
# Before die, so a failed suite still gets its per-workbook table: which four
# passed is most of what the fifth failing means.
record_workbooks "$STATE_DIR/workbooks.log"
failed="$(grep -c '^--- FAIL' "$STATE_DIR/workbooks.log" || true)"
record workbooks FAIL "$failed failed, see $STATE_DIR/workbooks.log"
capture_journals
tail -60 "$STATE_DIR/workbooks.log"
grep -E '^--- (PASS|FAIL)' "$STATE_DIR/workbooks.log" | sed 's/^/ /' || true
die "the published workbooks failed on $TOPOLOGY; see $STATE_DIR/workbooks.log"
fi
scripts/terraform/oci-spx/variables.tf
+35 -4
View File
@@ -1,17 +1,34 @@
variable "tenancy_ocid" { type = string }
variable "user_ocid" { type = string }
variable "fingerprint" { type = string }
variable "private_key_path" { type = string }
variable "private_key_path" {
type = string
default = ""
}
# The PEM itself, for a CI runner that holds it as a secret in the environment.
# Exactly one of this and private_key_path is set; the check lives here so a run
# fails on the input rather than on an OCI 401 forty seconds later.
variable "private_key" {
type = string
default = ""
sensitive = true
validation {
condition = (var.private_key == "") != (var.private_key_path == "")
error_message = "Set exactly one of private_key (the PEM, for CI) or private_key_path (a file, for a workstation)."
}
}
# OCI's own region name. Spinifex's region is a separate setting that follows AWS
# naming, so ap-sydney-1 here beside ap-southeast-2 in spinifex.toml is correct.
variable "region" {
type = string
default = "ap-sydney-1"
}
# Unused. Kept declared because scripts/oci_env.py discovers and exports it, and
# there are no identity resources left that need the home-region endpoint.
# OCI serves IAM writes from the tenancy home region, which is not necessarily
# where we build: the dynamic group and policy in instance-principal.tf go through
# the oci.home provider. Unset means the two are the same region.
variable "home_region" {
description = "Unused. The tenancy home region, exported by the Python environment tool."
description = "Tenancy home region, for IAM writes. Defaults to var.region; set OCI_HOME_REGION when they differ."
type = string
default = null
nullable = true
@@ -37,6 +54,20 @@ variable "deployment_name" {
error_message = "deployment_name must be 1-15 lowercase alphanumeric characters starting with a letter, because it is used as the VCN dns_label."
}
}
# Three values rather than a bool, because using an instance principal and being
# allowed to create one are different rights. See instance-principal.tf for what
# the policy grants and why it is narrow.
variable "instance_principal" {
description = "off: nodes use an API key. adopt: nodes authenticate as themselves against a dynamic group and policy that already exist. create: also create those two, which needs a tenancy-admin principal."
type = string
default = "off"
validation {
condition = contains(["off", "adopt", "create"], var.instance_principal)
error_message = "instance_principal must be off, adopt or create."
}
}
variable "vcn_cidr" {
description = "VCN address space. A /22 leaves room for the per-node secondary private IPs Spinifex allocates for external addresses."
type = string
spinifex/accountteardown/reapers_account.go
+1
View File
@@ -96,6 +96,7 @@ type IAMAccounts struct{ svc handlers_iam.IAMService }
var _ AccountStore = (*IAMAccounts)(nil)
// NewIAMAccounts wraps svc as the engine's AccountStore.
func NewIAMAccounts(svc handlers_iam.IAMService) *IAMAccounts {
return &IAMAccounts{svc: svc}
}
spinifex/admin/admin.go
+122 -30
View File
@@ -1,3 +1,6 @@
// Package admin holds the node administration helpers behind spx admin: config
// and certificate generation, credential bootstrap, image import and host
// network discovery.
package admin
import (
@@ -19,6 +22,7 @@ import (
"regexp"
"strconv"
"strings"
"testing"
"text/template"
"time"
@@ -43,6 +47,8 @@ type RemoteNode struct {
NorthstarConfigPath string
}
// ConfigSettings is the template data rendered into every generated config file
// (spinifex.toml, predastore.toml, NATS and the rest) by GenerateConfigFile.
type ConfigSettings struct {
AccessKey string
SecretKey string
@@ -148,12 +154,16 @@ type ConfigSettings struct {
ResolverAllowFrom []string
}
// ConfigFile names one config file to render: Template is the text/template
// source and Path the destination it is written to.
type ConfigFile struct {
Name string
Path string
Template string
}
// GenerateConfigFiles renders each of configs with configSettings, printing a
// line per file and stopping at the first failure.
func GenerateConfigFiles(configs []ConfigFile, configSettings ConfigSettings) error {
for _, cfg := range configs {
if err := GenerateConfigFile(cfg.Path, cfg.Template, configSettings); err != nil {
@@ -290,6 +300,8 @@ func PredastoreDataDir(spxRoot string) string {
return filepath.Join(spxRoot, "predastore", "cluster")
}
// CreateServiceDirectories creates the per-service directories under spxRoot.
// A directory that cannot be created is warned about on stderr, not returned.
func CreateServiceDirectories(spxRoot string) {
dirs := []string{
filepath.Join(spxRoot, "images"),
@@ -316,6 +328,8 @@ func CreateServiceDirectories(spxRoot string) {
fmt.Printf("✅ Directory structure created in %s\n", spxRoot)
}
// FileExists reports whether path can be stat'd. Any stat error, including
// permission denied, reads as false.
func FileExists(path string) bool {
_, err := os.Stat(path)
return err == nil
@@ -465,33 +479,111 @@ func SetServiceOwnership() error {
return ownershipErr
}
// awsINIOptions keeps AWS nested settings (an empty key followed by indented
// lines) intact across a rewrite; without it they are flattened into the
// enclosing section and lost.
var awsINIOptions = ini.LoadOptions{AllowNestedValues: true}
// UpdateAWSINIFile updates or creates an AWS INI file section with the given key-value pairs.
func UpdateAWSINIFile(path, section string, values map[string]string) error {
var cfg *ini.File
var err error
if FileExists(path) {
cfg, err = ini.Load(path)
if err != nil {
return fmt.Errorf("failed to load INI file: %w", err)
}
} else {
cfg = ini.Empty()
}
sec, err := cfg.NewSection(section)
cfg, err := loadAWSINIFile(path)
if err != nil {
// Section already exists, get it.
sec, err = cfg.GetSection(section)
if err != nil {
return fmt.Errorf("failed to get section: %w", err)
}
return err
}
sec, err := awsINISection(cfg, section)
if err != nil {
return err
}
for key, value := range values {
sec.Key(key).SetValue(value)
}
return saveAWSINIFile(cfg, path)
}
// WriteAWSProfileConfig points an AWS CLI profile at this cluster. One profile
// endpoint_url cannot send S3 to the predastore gate and everything else to the
// gateway, so S3 goes in the profile's services section, which takes precedence.
func WriteAWSProfileConfig(path, profileName, region, certPath, endpointHost string) error {
cfg, err := loadAWSINIFile(path)
if err != nil {
return err
}
profileSection := profileName
if profileName != "default" {
profileSection = "profile " + profileName
}
profile, err := awsINISection(cfg, profileSection)
if err != nil {
return err
}
for key, value := range map[string]string{
"region": region,
"endpoint_url": "https://" + net.JoinHostPort(endpointHost, "9999"),
"ca_bundle": certPath,
"output": "json",
"services": profileName,
} {
profile.Key(key).SetValue(value)
}
services, err := awsINISection(cfg, "services "+profileName)
if err != nil {
return err
}
s3Endpoint := "https://" + net.JoinHostPort(endpointHost, strconv.Itoa(predastoreGatePort))
if err := setNestedAWSSetting(services, "s3", "endpoint_url", s3Endpoint); err != nil {
return err
}
return saveAWSINIFile(cfg, path)
}
// setNestedAWSSetting sets name inside the nested block under key, keeping any
// other settings an operator nested there.
func setNestedAWSSetting(sec *ini.Section, key, name, value string) error {
var lines []string
if sec.HasKey(key) {
for _, line := range sec.Key(key).NestedValues() {
if setting, _, _ := strings.Cut(line, "="); strings.TrimSpace(setting) != name {
lines = append(lines, line)
}
}
sec.DeleteKey(key)
}
nested, err := sec.NewKey(key, "")
if err != nil {
return fmt.Errorf("failed to create key %s: %w", key, err)
}
for _, line := range append(lines, name+" = "+value) {
if err := nested.AddNestedValue(line); err != nil {
return fmt.Errorf("failed to set %s.%s: %w", key, name, err)
}
}
return nil
}
func loadAWSINIFile(path string) (*ini.File, error) {
if !FileExists(path) {
return ini.Empty(awsINIOptions), nil
}
cfg, err := ini.LoadSources(awsINIOptions, path)
if err != nil {
return nil, fmt.Errorf("failed to load INI file: %w", err)
}
return cfg, nil
}
func awsINISection(cfg *ini.File, name string) (*ini.Section, error) {
sec, err := cfg.NewSection(name)
if err != nil {
// Section already exists, get it.
sec, err = cfg.GetSection(name)
if err != nil {
return nil, fmt.Errorf("failed to get section: %w", err)
}
}
return sec, nil
}
func saveAWSINIFile(cfg *ini.File, path string) error {
// Write atomically: ini.SaveTo uses os.Create (world-readable); render to a
// sibling temp file (0600) and rename to avoid briefly exposing secrets.
dir := filepath.Dir(path)
@@ -590,6 +682,16 @@ func GenerateNATSToken() (string, error) {
// keeps the 4096-bit default.
var certKeyBits = 4096
// ShortenCertKeyBitsForTest generates certificate keys at bits for the rest of
// one test and restores the previous size afterwards. There is one size per
// process, so the test must not run in parallel with anything generating keys.
func ShortenCertKeyBitsForTest(tb testing.TB, bits int) {
tb.Helper()
prev := certKeyBits
certKeyBits = bits
tb.Cleanup(func() { certKeyBits = prev })
}
// GenerateCACert generates a Certificate Authority certificate and key.
func GenerateCACert(caCertPath, caKeyPath string) error {
caPrivateKey, err := rsa.GenerateKey(rand.Reader, certKeyBits)
@@ -879,22 +981,12 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP string)
}
}
configSection := profileName
if profileName != "default" {
configSection = "profile " + profileName
}
endpointHost := bindIP
if endpointHost == "" || endpointHost == "0.0.0.0" {
endpointHost = "localhost"
}
if err := UpdateAWSINIFile(configPath, configSection, map[string]string{
"region": region,
"endpoint_url": "https://" + net.JoinHostPort(endpointHost, "9999"),
"ca_bundle": certPath,
"output": "json",
}); err != nil {
if err := WriteAWSProfileConfig(configPath, profileName, region, certPath, endpointHost); err != nil {
return err
}
spinifex/admin/admin_test.go
+77 -3
View File
@@ -11,6 +11,7 @@ import (
"strings"
"testing"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
"github.com/mulgadc/spinifex/spinifex/runtime/formation"
"github.com/stretchr/testify/assert"
@@ -229,6 +230,79 @@ func TestUpdateAWSINIFile_AddNewSection(t *testing.T) {
assert.Contains(t, content, "spinifex-val")
}
// --- WriteAWSProfileConfig ---
// resolveProfileEndpoint resolves sdkID's endpoint through the Go SDK's own
// shared-config loader, the way a client does: the service's entry in the
// profile's services section, else the profile-wide endpoint_url.
func resolveProfileEndpoint(t *testing.T, configPath, profile, sdkID string) string {
t.Helper()
shared, err := awsconfig.LoadSharedConfigProfile(t.Context(), profile, func(o *awsconfig.LoadSharedConfigOptions) {
o.ConfigFiles = []string{configPath}
o.CredentialsFiles = []string{}
})
require.NoError(t, err)
endpoint, found, err := shared.GetServiceBaseEndpoint(t.Context(), sdkID)
require.NoError(t, err)
if found {
return endpoint
}
return shared.BaseEndpoint
}
func TestWriteAWSProfileConfig_SendsS3ToPredastoreAndTheRestToTheGateway(t *testing.T) {
path := filepath.Join(t.TempDir(), "config")
require.NoError(t, WriteAWSProfileConfig(path, "spinifex", "ap-southeast-2", "/ca.pem", "10.11.12.1"))
assert.Equal(t, "https://10.11.12.1:8443", resolveProfileEndpoint(t, path, "spinifex", "S3"))
assert.Equal(t, "https://10.11.12.1:9999", resolveProfileEndpoint(t, path, "spinifex", "EC2"))
assert.Equal(t, "https://10.11.12.1:9999", resolveProfileEndpoint(t, path, "spinifex", "STS"))
}
// A rewrite must not flatten nested settings an operator added, which is how a
// hand-written s3 block used to be lost on every re-init.
func TestWriteAWSProfileConfig_KeepsNestedSettingsAcrossARewrite(t *testing.T) {
path := filepath.Join(t.TempDir(), "config")
require.NoError(t, os.WriteFile(path, []byte(`[profile spinifex]
s3 =
max_concurrent_requests = 20
services = spinifex
[services spinifex]
s3 =
addressing_style = path
endpoint_url = https://stale:8443
`), 0600))
require.NoError(t, WriteAWSProfileConfig(path, "spinifex", "ap-southeast-2", "/ca.pem", "localhost"))
require.NoError(t, WriteAWSProfileConfig(path, "spinifex", "ap-southeast-2", "/ca.pem", "localhost"))
assert.Equal(t, "https://localhost:8443", resolveProfileEndpoint(t, path, "spinifex", "S3"))
shared, err := awsconfig.LoadSharedConfigProfile(t.Context(), "spinifex", func(o *awsconfig.LoadSharedConfigOptions) {
o.ConfigFiles = []string{path}
o.CredentialsFiles = []string{}
})
require.NoError(t, err)
assert.Equal(t, map[string]string{"addressing_style": "path", "endpoint_url": "https://localhost:8443"},
shared.Services.ServiceValues["s3"])
data, err := os.ReadFile(path)
require.NoError(t, err)
assert.Contains(t, string(data), "max_concurrent_requests = 20")
assert.NotContains(t, string(data), "stale")
}
// Account profiles each get their own services section, so a tenant profile
// resolves S3 the same way as the operator's without sharing its section.
func TestWriteAWSProfileConfig_ProfilesDoNotShareAServicesSection(t *testing.T) {
path := filepath.Join(t.TempDir(), "config")
require.NoError(t, WriteAWSProfileConfig(path, "spinifex", "ap-southeast-2", "/ca.pem", "localhost"))
require.NoError(t, WriteAWSProfileConfig(path, "spinifex-acme", "ap-southeast-2", "/ca.pem", "10.0.0.5"))
assert.Equal(t, "https://localhost:8443", resolveProfileEndpoint(t, path, "spinifex", "S3"))
assert.Equal(t, "https://10.0.0.5:8443", resolveProfileEndpoint(t, path, "spinifex-acme", "S3"))
}
// --- SetupAWSCredentials ---
func TestSetupAWSCredentials_CreatesFiles(t *testing.T) {
@@ -274,9 +348,9 @@ func TestSetupAWSCredentials_UsesBindIP(t *testing.T) {
err := SetupAWSCredentials("AKIATEST123", "secret123", "us-east-1", "/ca.pem", "10.11.12.1")
require.NoError(t, err)
configData, _ := os.ReadFile(filepath.Join(dir, ".aws", "config"))
assert.Contains(t, string(configData), "https://10.11.12.1:9999")
assert.NotContains(t, string(configData), "localhost")
configPath := filepath.Join(dir, ".aws", "config")
assert.Equal(t, "https://10.11.12.1:9999", resolveProfileEndpoint(t, configPath, "spinifex", "EC2"))
assert.Equal(t, "https://10.11.12.1:8443", resolveProfileEndpoint(t, configPath, "spinifex", "S3"))
}
func TestSetupAWSCredentials_FallsBackToLocalhostForWildcard(t *testing.T) {
spinifex/agents/rds/agent/agent.go
+8 -7
View File
@@ -230,7 +230,8 @@ func newAgent(cfg Config, cp controlPlane, probe *engineProbe) (*Agent, error) {
return a, nil
}
// Does not wait for IMDS: the register loop rides out a datapath still coming up.
// New builds the agent from cfg, failing if no gateway URL is set. It does not
// wait for IMDS: the register loop rides out a datapath still coming up.
func New(cfg Config) (*Agent, error) {
if cfg.GatewayURL == "" {
return nil, fmt.Errorf("no gateway URL configured (RDS_GATEWAY_URL)")
@@ -375,16 +376,16 @@ func unescapeMountField(field string) string {
// Boot-retry bounds. Tight to start, since the usual cause is a control plane
// still creating this instance's record, and capped so an outage does not turn
// a booting fleet into a retry storm.
const (
// a booting fleet into a retry storm. Vars so tests can shorten the wait.
var (
retryMin = 1 * time.Second
retryMax = 30 * time.Second
// With the production backoff, attempt five is about 15 seconds into the
// failure and still precedes the first 30-second heartbeat.
retryErrorAttempt = 5
)
// With the production backoff, attempt five is about 15 seconds into the
// failure and still precedes the first 30-second heartbeat.
const retryErrorAttempt = 5
// Wrapping an error in this stops retryObserved. For the failures the control
// plane decides from its own record, no retry can change the answer, and looping
// on one only keeps the caller blocked.
spinifex/agents/rds/agent/config.go
+1 -1
View File
@@ -81,7 +81,7 @@ type Config struct {
AgentVersion string
}
// Reads the cloud-init env file, then lets real env vars override.
// LoadConfig reads the cloud-init env file, then lets real env vars override.
func LoadConfig(envFile string) Config {
get := guestenv.Load(envFile).Get
spinifex/agents/rds/agent/engine_mariadb_test.go
+2 -2
View File
@@ -948,8 +948,8 @@ func TestMariaDBProbe_LiveButUnreachableEngineResetsTheRollbackDeadline(t *testi
recovery := &fakeRecovery{restored: true}
guard := newTestGuard(recovery, probe)
// Bounded so the deadline-resetting loop cannot run forever if it stops
// resetting; the assertion is that the rollback was never reached.
ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
// resetting; four rollback budgets, so a missed reset would roll back.
ctx, cancel := context.WithTimeout(t.Context(), 80*time.Millisecond)
defer cancel()
guard.Run(ctx)
spinifex/agents/rds/agent/main_test.go
+14
View File
@@ -0,0 +1,14 @@
package agent
import (
"os"
"testing"
"time"
)
// TestMain shortens the boot-retry backoff for the whole package: the tests
// assert which calls are retried, not how long the agent waits between them.
func TestMain(m *testing.M) {
retryMin, retryMax = time.Millisecond, time.Millisecond
os.Exit(m.Run())
}
spinifex/agents/rds/agent/paramrecovery_test.go
+2 -2
View File
@@ -460,8 +460,8 @@ func TestParamGuard_LeavesARecoveringEngineAlone(t *testing.T) {
engine := &fakeRecovery{restored: true}
guard := newTestGuard(engine, stubProbe(t, 1, nil))
// Bounded so the deadline-resetting loop cannot run forever if it stops
// resetting; the assertion is that the rollback was never reached.
ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
// resetting; four rollback budgets, so a missed reset would roll back.
ctx, cancel := context.WithTimeout(t.Context(), 80*time.Millisecond)
defer cancel()
guard.Run(ctx)
spinifex/bootstrap/config/config.go
+31 -5
View File
@@ -1,3 +1,5 @@
// Package config defines spinifex.toml, the cluster and per-node configuration
// every Spinifex service is started from, and loads it via LoadConfig.
package config
import (
@@ -13,6 +15,8 @@ import (
"github.com/spf13/viper"
)
// ClusterConfig is the root of spinifex.toml as returned by LoadConfig: the
// cluster-wide settings plus a [nodes.<name>] Config for every node.
type ClusterConfig struct {
Epoch uint64 `mapstructure:"epoch"` // bump when leader commits changes
Node string `mapstructure:"node"` // my node name
@@ -92,6 +96,7 @@ type ExternalPool struct {
OCIVNICIface string `mapstructure:"oci_vnic_iface"` // Host interface to resolve the VNIC OCID from, instead of oci_vnic_id
OCISubnetID string `mapstructure:"oci_subnet_id"` // Subnet OCID the private IPs come from (optional; defaults to the VNIC's)
OCIPublicIPPool string `mapstructure:"oci_public_ip_pool"` // Public IP pool OCID for BYOIP (optional)
OCIAuth string `mapstructure:"oci_auth"` // "config_file" (default) or "instance_principal"
OCIConfigFile string `mapstructure:"oci_config_file"` // API-key config file (optional; defaults to ~/.oci/config)
OCIConfigProfile string `mapstructure:"oci_config_profile"` // Profile within that file (optional; defaults to DEFAULT)
@@ -208,6 +213,8 @@ type Config struct {
WalDir string `json:"WalDir" mapstructure:"wal_dir"`
}
// AWSGWConfig is a node's [awsgw] section: the AWS gateway's listen address,
// TLS key pair and the path of its own config file.
type AWSGWConfig struct {
Host string `json:"Host" mapstructure:"host"`
TLSKey string `json:"TLSKey" mapstructure:"tlskey"`
@@ -218,6 +225,8 @@ type AWSGWConfig struct {
ExpectedNodes int `json:"ExpectedNodes" mapstructure:"expected_nodes"` // TODO: Replace with root cluster config
}
// ViperblockConfig is a node's [viperblock] section, tuning every viperblock
// instance the node constructs for EBS volumes.
type ViperblockConfig struct {
ShardWAL *bool `json:"ShardWAL" mapstructure:"shardwal"` // Enable sharded WAL (default false when nil)
@@ -333,8 +342,9 @@ type NorthstarConfig struct {
InternalDomain string `json:"InternalDomain" mapstructure:"internal_domain"`
}
// Every DB VM's primary NIC lives in the shared system VPC, which gives the
// in-guest agent management egress while the customer ENI stays ingress-only.
// RDSConfig is a node's [rds] section. Every DB VM's primary NIC lives in the
// shared system VPC, which gives the in-guest agent management egress while
// the customer ENI stays ingress-only.
type RDSConfig struct {
// The IPv4 /14 the system VPC's /22 is carved from. It must not overlap the
// EKS control-plane supernet or any customer VPC CIDR.
@@ -380,9 +390,9 @@ type RDSConfig struct {
// supernet, so a name-hash collision can never place an RDS subnet in EKS space.
const RDSDefaultSystemVPCSupernet = "10.248.0.0/14"
// Every self-hosted vLLM serving VM's primary NIC lives in the shared Bedrock
// system VPC, mirroring RDS's DB-VM VPC: one shared VPC per region rather
// than one per endpoint, since a serving VM has no customer ENI to isolate.
// BedrockConfig is a node's [bedrock] section. Every vLLM serving VM's primary
// NIC lives in one shared system VPC per region, not one per endpoint, as with
// RDS DB VMs, since a serving VM has no customer ENI to isolate.
type BedrockConfig struct {
// The IPv4 /14 the system VPC's /22 is carved from. It must not overlap the
// RDS or EKS control-plane supernets or any customer VPC CIDR.
@@ -453,6 +463,8 @@ func ParseEndpoints(addr string) []string {
return out
}
// PredastoreConfig is a node's [predastore] section: the S3 endpoint, bucket
// and credentials Spinifex uses to reach Predastore.
type PredastoreConfig struct {
Host string `json:"Host" mapstructure:"host"`
Bucket string `json:"Bucket" mapstructure:"bucket"`
@@ -728,6 +740,20 @@ func validateClusterConfig(cc *ClusterConfig) error {
if (p.OCIVNICID == "") == (p.OCIVNICIface == "") {
return fmt.Errorf("config: [[network.external_pools]] %q: source=\"oci\" requires exactly one of oci_vnic_id or oci_vnic_iface", p.Name)
}
// Named explicitly rather than inferred, so a node that was meant to
// use its own identity and has no dynamic group fails at config load
// instead of falling back to a key file nobody installed.
switch p.OCIAuth {
case "", "config_file", "instance_principal":
default:
return fmt.Errorf("config: [[network.external_pools]] %q: oci_auth must be \"config_file\" or \"instance_principal\", got %q", p.Name, p.OCIAuth)
}
// Credentials come from the metadata service under instance principal,
// so a config file named beside it is one of two sources of truth and
// the one that would be silently ignored.
if p.OCIAuth == "instance_principal" && (p.OCIConfigFile != "" || p.OCIConfigProfile != "") {
return fmt.Errorf("config: [[network.external_pools]] %q: oci_config_file/oci_config_profile are not valid with oci_auth=\"instance_principal\" (the instance's own certificate is the credential)", p.Name)
}
if p.BindBridge != "" || p.DHCPMAC != "" {
return fmt.Errorf("config: [[network.external_pools]] %q: bind_bridge/dhcp_mac are only valid with source=\"dhcp\"", p.Name)
}
spinifex/bootstrap/config/config_test.go
+52
View File
@@ -1262,6 +1262,58 @@ oci_compartment_id = "ocid1.compartment.oc1..c1"
assert.Contains(t, err.Error(), "exactly one of oci_vnic_id or oci_vnic_iface")
}
func TestLoadConfig_NetworkPoolOCIDefaultsToConfigFileAuth(t *testing.T) {
cfg, err := loadOCIPool(t, "nat", `
source = "oci"
oci_compartment_id = "ocid1.compartment.oc1..c1"
oci_vnic_iface = "br-wan"
`)
require.NoError(t, err)
assert.Empty(t, cfg.Network.ExternalPools[0].OCIAuth)
}
func TestLoadConfig_NetworkPoolOCIAcceptsInstancePrincipal(t *testing.T) {
cfg, err := loadOCIPool(t, "nat", `
source = "oci"
oci_auth = "instance_principal"
oci_compartment_id = "ocid1.compartment.oc1..c1"
oci_vnic_iface = "br-wan"
`)
require.NoError(t, err)
assert.Equal(t, "instance_principal", cfg.Network.ExternalPools[0].OCIAuth)
}
// A typo must not fall back to the key file, because a node meant to use its own
// identity would then fail on a credential nobody installed rather than here.
func TestLoadConfig_NetworkPoolOCIRejectsUnknownAuth(t *testing.T) {
_, err := loadOCIPool(t, "nat", `
source = "oci"
oci_auth = "instance-principal"
oci_compartment_id = "ocid1.compartment.oc1..c1"
oci_vnic_iface = "br-wan"
`)
require.Error(t, err)
assert.Contains(t, err.Error(), "oci_auth must be")
}
// Under instance principal the certificate is the credential, so a config file
// beside it is a second source of truth and the one that is silently ignored.
func TestLoadConfig_NetworkPoolOCIRejectsConfigFileWithInstancePrincipal(t *testing.T) {
for _, key := range []string{
`oci_config_file = "/etc/spinifex/oci/config"`,
`oci_config_profile = "spinifex"`,
} {
_, err := loadOCIPool(t, "nat", `
source = "oci"
oci_auth = "instance_principal"
oci_compartment_id = "ocid1.compartment.oc1..c1"
oci_vnic_iface = "br-wan"
`+key+"\n")
require.Error(t, err, key)
assert.Contains(t, err.Error(), "not valid with oci_auth=\"instance_principal\"", key)
}
}
func TestLoadConfig_NetworkPoolOCIRejectsRange(t *testing.T) {
_, err := loadOCIPool(t, "nat", `
source = "oci"
spinifex/daemon/coverage_fakes_test.go
+231
View File
@@ -0,0 +1,231 @@
package daemon
import (
"context"
"errors"
viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
"net/netip"
"sync"
"github.com/aws/aws-sdk-go/service/ec2"
ec2eip "github.com/mulgadc/spinifex/spinifex/domains/ec2/eip"
"github.com/mulgadc/spinifex/spinifex/domains/network/external"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
)
var errInjected = errors.New("injected failure")
// failingKV wraps a real bucket and fails the methods named in failOn with
// errInjected, so tests can reach the KV error branches.
type failingKV struct {
jetstream.KeyValue
mu sync.Mutex
failOn map[string]bool
}
func newFailingKV(kv jetstream.KeyValue, methods ...string) *failingKV {
f := &failingKV{KeyValue: kv, failOn: map[string]bool{}}
for _, m := range methods {
f.failOn[m] = true
}
return f
}
func (f *failingKV) fails(method string) bool {
f.mu.Lock()
defer f.mu.Unlock()
return f.failOn[method]
}
func (f *failingKV) setFail(method string, fail bool) {
f.mu.Lock()
defer f.mu.Unlock()
f.failOn[method] = fail
}
func (f *failingKV) Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) {
if f.fails("Get") {
return nil, errInjected
}
return f.KeyValue.Get(ctx, key)
}
func (f *failingKV) Put(ctx context.Context, key string, value []byte) (uint64, error) {
if f.fails("Put") {
return 0, errInjected
}
return f.KeyValue.Put(ctx, key, value)
}
func (f *failingKV) Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
if f.fails("Create") {
return 0, errInjected
}
return f.KeyValue.Create(ctx, key, value, opts...)
}
func (f *failingKV) Update(ctx context.Context, key string, value []byte, last uint64) (uint64, error) {
if f.fails("Update") {
return 0, errInjected
}
return f.KeyValue.Update(ctx, key, value, last)
}
func (f *failingKV) Delete(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error {
if f.fails("Delete") {
return errInjected
}
return f.KeyValue.Delete(ctx, key, opts...)
}
func (f *failingKV) Purge(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error {
if f.fails("Purge") {
return errInjected
}
return f.KeyValue.Purge(ctx, key, opts...)
}
func (f *failingKV) Keys(ctx context.Context, opts ...jetstream.WatchOpt) ([]string, error) {
if f.fails("Keys") {
return nil, errInjected
}
return f.KeyValue.Keys(ctx, opts...)
}
func (f *failingKV) ListKeys(ctx context.Context, opts ...jetstream.WatchOpt) (jetstream.KeyLister, error) {
if f.fails("ListKeys") {
return nil, errInjected
}
return f.KeyValue.ListKeys(ctx, opts...)
}
// fakeEIPService answers only the EIP calls a test configures; any other
// method panics through the nil embedded interface.
type fakeEIPService struct {
ec2eip.EIPService
mu sync.Mutex
allocateOut *ec2.AllocateAddressOutput
allocateErr error
associateOut *ec2.AssociateAddressOutput
associateErr error
disassociateErr error
releaseErr error
released []string
disassociated []string
}
func (f *fakeEIPService) AllocateAddress(context.Context, *ec2.AllocateAddressInput, string) (*ec2.AllocateAddressOutput, error) {
if f.allocateErr != nil {
return nil, f.allocateErr
}
return f.allocateOut, nil
}
func (f *fakeEIPService) AssociateAddress(context.Context, *ec2.AssociateAddressInput, string) (*ec2.AssociateAddressOutput, error) {
if f.associateErr != nil {
return nil, f.associateErr
}
if f.associateOut != nil {
return f.associateOut, nil
}
return &ec2.AssociateAddressOutput{}, nil
}
func (f *fakeEIPService) DisassociateAddress(_ context.Context, in *ec2.DisassociateAddressInput, _ string) (*ec2.DisassociateAddressOutput, error) {
f.mu.Lock()
if in != nil && in.AssociationId != nil {
f.disassociated = append(f.disassociated, *in.AssociationId)
}
f.mu.Unlock()
if f.disassociateErr != nil {
return nil, f.disassociateErr
}
return &ec2.DisassociateAddressOutput{}, nil
}
func (f *fakeEIPService) ReleaseAddress(_ context.Context, in *ec2.ReleaseAddressInput, _ string) (*ec2.ReleaseAddressOutput, error) {
f.mu.Lock()
if in != nil && in.AllocationId != nil {
f.released = append(f.released, *in.AllocationId)
}
f.mu.Unlock()
if f.releaseErr != nil {
return nil, f.releaseErr
}
return &ec2.ReleaseAddressOutput{}, nil
}
// fakeAllocator is an external.Allocator whose Release returns releaseErr and
// records each release.
type fakeAllocator struct {
mu sync.Mutex
allocateAddr netip.Addr
allocateErr error
releaseErr error
releases []netip.Addr
}
var _ external.Allocator = (*fakeAllocator)(nil)
func (f *fakeAllocator) Allocate(context.Context, external.AllocateRequest) (netip.Addr, error) {
return f.allocateAddr, f.allocateErr
}
func (f *fakeAllocator) Release(_ context.Context, _ string, ip netip.Addr, _ string) error {
f.mu.Lock()
f.releases = append(f.releases, ip)
f.mu.Unlock()
return f.releaseErr
}
// erroringPlumber is a recordingPlumber whose SetupTap returns setupErr.
type erroringPlumber struct {
recordingPlumber
setupErr error
}
var _ vm.NetworkPlumber = (*erroringPlumber)(nil)
func (p *erroringPlumber) SetupTap(vm.TapSpec) error { return p.setupErr }
// hookVolumeMounter is a vm.VolumeMounter whose Mount runs onMount, letting a
// test set the instance status so launch returns before starting QEMU.
type hookVolumeMounter struct {
onMount func(*vm.VM)
mountErr error
}
var _ vm.VolumeMounter = (*hookVolumeMounter)(nil)
func (m *hookVolumeMounter) Mount(_ context.Context, v *vm.VM) error {
if m.onMount != nil {
m.onMount(v)
}
return m.mountErr
}
func (m *hookVolumeMounter) Unmount(context.Context, *vm.VM) error { return nil }
func (m *hookVolumeMounter) Abandon(context.Context, *vm.VM, string) error { return nil }
func (m *hookVolumeMounter) MountOne(context.Context, string, *viperblocklegacyv1.EBSRequest) error {
return nil
}
func (m *hookVolumeMounter) UnmountOne(context.Context, string, viperblocklegacyv1.EBSRequest) error {
return nil
}
// noReplyMsg builds a message with no reply subject, so msg.Respond fails
// and the handler's "respond failed" branch runs.
func noReplyMsg(subject string, data []byte) *nats.Msg {
return &nats.Msg{Subject: subject, Data: data}
}
spinifex/daemon/daemon.go
+47 -12
View File
@@ -1,3 +1,6 @@
// Package daemon is the per-node Spinifex daemon: it wires the AWS service
// handlers to their NATS topics and owns this node's VMs, volumes, ENIs and
// the reconcilers that keep them converged.
package daemon
import (
@@ -1082,22 +1085,17 @@ func (d *Daemon) subscribeAll() error {
)
}
// EKS gateway → daemon subscriptions. Every handler currently returns
// NotImplemented; topics are subscribed up-front so the wiring layer is
// stable while real bodies land.
// EKS gateway → daemon subscriptions.
if d.eksService != nil {
subs = append(subs,
natsSub{"eks.CreateCluster", handleNATSRequestWithPrincipal(d.node, d.eksService.CreateCluster), "spinifex-workers"},
natsSub{"eks.DescribeCluster", handleNATSRequest(d.node, d.eksService.DescribeCluster), "spinifex-workers"},
natsSub{"eks.ListClusters", handleNATSRequest(d.node, d.eksService.ListClusters), "spinifex-workers"},
natsSub{"eks.UpdateClusterConfig", handleNATSRequest(d.node, d.eksService.UpdateClusterConfig), "spinifex-workers"},
natsSub{"eks.UpdateClusterVersion", handleNATSRequest(d.node, d.eksService.UpdateClusterVersion), "spinifex-workers"},
natsSub{"eks.DeleteCluster", handleNATSRequest(d.node, d.eksService.DeleteCluster), "spinifex-workers"},
natsSub{"eks.CreateNodegroup", handleNATSRequest(d.node, d.eksService.CreateNodegroup), "spinifex-workers"},
natsSub{"eks.DescribeNodegroup", handleNATSRequest(d.node, d.eksService.DescribeNodegroup), "spinifex-workers"},
natsSub{"eks.ListNodegroups", handleNATSRequest(d.node, d.eksService.ListNodegroups), "spinifex-workers"},
natsSub{"eks.UpdateNodegroupConfig", handleNATSRequest(d.node, d.eksService.UpdateNodegroupConfig), "spinifex-workers"},
natsSub{"eks.UpdateNodegroupVersion", handleNATSRequest(d.node, d.eksService.UpdateNodegroupVersion), "spinifex-workers"},
natsSub{"eks.DeleteNodegroup", handleNATSRequest(d.node, d.eksService.DeleteNodegroup), "spinifex-workers"},
natsSub{"eks.CreateAccessEntry", handleNATSRequest(d.node, d.eksService.CreateAccessEntry), "spinifex-workers"},
natsSub{"eks.DescribeAccessEntry", handleNATSRequest(d.node, d.eksService.DescribeAccessEntry), "spinifex-workers"},
@@ -1118,10 +1116,6 @@ func (d *Daemon) subscribeAll() error {
natsSub{"eks.GetRecoveryDirective", handleNATSRequest(d.node, d.eksService.GetRecoveryDirective), "spinifex-workers"},
natsSub{"eks.SetRecoveryDirective", handleNATSRequest(d.node, d.eksService.SetRecoveryDirective), "spinifex-workers"},
natsSub{"eks.RestoreSnapshot", handleNATSRequest(d.node, d.eksService.RestoreSnapshot), "spinifex-workers"},
natsSub{"eks.AssociateIdentityProviderConfig", handleNATSRequest(d.node, d.eksService.AssociateIdentityProviderConfig), "spinifex-workers"},
natsSub{"eks.DescribeIdentityProviderConfig", handleNATSRequest(d.node, d.eksService.DescribeIdentityProviderConfig), "spinifex-workers"},
natsSub{"eks.ListIdentityProviderConfigs", handleNATSRequest(d.node, d.eksService.ListIdentityProviderConfigs), "spinifex-workers"},
natsSub{"eks.DisassociateIdentityProviderConfig", handleNATSRequest(d.node, d.eksService.DisassociateIdentityProviderConfig), "spinifex-workers"},
natsSub{"eks.TagResource", handleNATSRequest(d.node, d.eksService.TagResource), "spinifex-workers"},
natsSub{"eks.UntagResource", handleNATSRequest(d.node, d.eksService.UntagResource), "spinifex-workers"},
natsSub{"eks.ListTagsForResource", handleNATSRequest(d.node, d.eksService.ListTagsForResource), "spinifex-workers"},
@@ -1194,6 +1188,7 @@ func (d *Daemon) subscribeAll() error {
natsSub{handlers_rds.SubjectListTagsForResource, handleNATSRequest(d.node, d.rdsService.ListTagsForResource), "spinifex-workers"},
natsSub{handlers_rds.SubjectCreateDBSubnetGroup, handleNATSRequest(d.node, d.rdsService.CreateDBSubnetGroup), "spinifex-workers"},
natsSub{handlers_rds.SubjectDescribeDBSubnetGroups, handleNATSRequest(d.node, d.rdsService.DescribeDBSubnetGroups), "spinifex-workers"},
natsSub{handlers_rds.SubjectModifyDBSubnetGroup, handleNATSRequest(d.node, d.rdsService.ModifyDBSubnetGroup), "spinifex-workers"},
natsSub{handlers_rds.SubjectDeleteDBSubnetGroup, handleNATSRequest(d.node, d.rdsService.DeleteDBSubnetGroup), "spinifex-workers"},
natsSub{handlers_rds.SubjectCreateDBParameterGroup, handleNATSRequest(d.node, d.rdsService.CreateDBParameterGroup), "spinifex-workers"},
natsSub{handlers_rds.SubjectDescribeDBParameterGroups, handleNATSRequest(d.node, d.rdsService.DescribeDBParameterGroups), "spinifex-workers"},
@@ -1464,6 +1459,7 @@ func (d *Daemon) externalPoolConfigs() (pools []external.ExternalPoolConfig, any
OCIVNICIface: p.OCIVNICIface,
OCISubnetID: p.OCISubnetID,
OCIPublicIPPool: p.OCIPublicIPPool,
OCIAuth: p.OCIAuth,
OCIConfigFile: p.OCIConfigFile,
OCIConfigProfile: p.OCIConfigProfile,
@@ -1514,11 +1510,13 @@ func (d *Daemon) installOCIAllocators(ipam *ec2vpc.ExternalIPAM, js jetstream.Je
continue
}
slog.Info("OCI allocator ready", "pool", p.Name,
"collected", len(res.Collected), "stale_bindings", len(res.Stale), "skipped", res.Skipped)
"collected", len(res.Collected), "stale_bindings", len(res.Stale),
"orphaned_public_ips", len(res.Orphaned), "skipped", res.Skipped)
d.ociAllocators = append(d.ociAllocators, ociPool{name: p.Name, alloc: alloc})
}
if len(d.ociAllocators) > 0 {
go d.runOCIAffinityLoop()
go d.runOCIReconcileLoop()
}
return nil
}
@@ -1563,6 +1561,11 @@ type ociPool struct {
// guest has finished coming up.
const ociAffinityInterval = 15 * time.Second
// ociReconcileInterval is how often leaked OCI objects are collected. Slower
// than the affinity pass by three orders of magnitude: a leak costs money and
// quota, not reachability, and the pass lists a whole compartment.
const ociReconcileInterval = 10 * time.Minute
// runOCIAffinityLoop keeps OCI's idea of where an address lives in step with
// where its guest actually runs. Every node runs its own, like the host EIP
// loop and for the same reason: the question is about this host's guests, and a
@@ -1581,6 +1584,36 @@ func (d *Daemon) runOCIAffinityLoop() {
}
}
// runOCIReconcileLoop repeats the startup reconcile for as long as the node
// runs. An address is leaked by a release that half-failed or a guest that went
// away, both of which happen long after start, and the startup pass alone left
// those to accumulate against the tenancy's reserved-address quota until a
// launch failed.
func (d *Daemon) runOCIReconcileLoop() {
ticker := time.NewTicker(ociReconcileInterval)
defer ticker.Stop()
for {
select {
case <-d.ctx.Done():
return
case <-ticker.C:
}
for _, p := range d.ociAllocators {
res, err := p.alloc.Reconcile(d.ctx)
if err != nil {
slog.Error("OCI allocator reconcile failed; leaked addresses may be billing",
"pool", p.name, "err", err)
continue
}
if len(res.Collected) > 0 || len(res.Stale) > 0 || len(res.Orphaned) > 0 {
slog.Info("OCI allocator reconcile collected leaks", "pool", p.name,
"collected", len(res.Collected), "stale_bindings", len(res.Stale),
"orphaned_public_ips", len(res.Orphaned))
}
}
}
}
// ClaimOCIAddresses runs one affinity pass over every OCI pool and records which
// guests OCI is not delivering an address to.
//
@@ -3220,7 +3253,9 @@ func (rm *ResourceManager) deallocate(instanceType *ec2.InstanceTypeInfo) {
var _ ec2instance.InstanceTypeAllocator = (*ResourceManager)(nil)
// Allocate, Deallocate, CanAllocate satisfy ec2instance.InstanceTypeAllocator.
// Allocate reserves host capacity for one instance of type it, erroring when
// the node lacks room. With Deallocate and CanAllocate it satisfies
// ec2instance.InstanceTypeAllocator.
func (rm *ResourceManager) Allocate(it *ec2.InstanceTypeInfo) error { return rm.allocate(it) }
func (rm *ResourceManager) Deallocate(it *ec2.InstanceTypeInfo) { rm.deallocate(it) }
func (rm *ResourceManager) CanAllocate(it *ec2.InstanceTypeInfo, count int) int {
spinifex/daemon/daemon_handlers_eks_test.go
+2 -10
View File
@@ -63,21 +63,17 @@ func TestDaemonHandleEKS_AllHandlersDispatchToService(t *testing.T) {
invalid := awserrors.ErrorInvalidParameterValue
// Nodegroup mutators gate on orchestration deps, which the shim service
// lacks → ServiceUnavailable. The read paths reach input validation and an
// empty body fails with InvalidParameterValue. UpdateNodegroupVersion stays
// NotImplemented (v1 doesn't do AMI upgrades).
// empty body fails with InvalidParameterValue.
unavailable := awserrors.ErrorServiceUnavailable
cases := []struct {
subject string
handler nats.MsgHandler
wantCode string
}{
{"eks.UpdateClusterConfig", asMsgHandler(handleNATSRequest(d.node, d.eksService.UpdateClusterConfig)), notImpl},
{"eks.UpdateClusterVersion", asMsgHandler(handleNATSRequest(d.node, d.eksService.UpdateClusterVersion)), notImpl},
{"eks.CreateNodegroup", asMsgHandler(handleNATSRequest(d.node, d.eksService.CreateNodegroup)), unavailable},
{"eks.DescribeNodegroup", asMsgHandler(handleNATSRequest(d.node, d.eksService.DescribeNodegroup)), invalid},
{"eks.ListNodegroups", asMsgHandler(handleNATSRequest(d.node, d.eksService.ListNodegroups)), invalid},
{"eks.UpdateNodegroupConfig", asMsgHandler(handleNATSRequest(d.node, d.eksService.UpdateNodegroupConfig)), unavailable},
{"eks.UpdateNodegroupVersion", asMsgHandler(handleNATSRequest(d.node, d.eksService.UpdateNodegroupVersion)), notImpl},
{"eks.DeleteNodegroup", asMsgHandler(handleNATSRequest(d.node, d.eksService.DeleteNodegroup)), unavailable},
{"eks.CreateAccessEntry", asMsgHandler(handleNATSRequest(d.node, d.eksService.CreateAccessEntry)), invalid},
{"eks.DescribeAccessEntry", asMsgHandler(handleNATSRequest(d.node, d.eksService.DescribeAccessEntry)), invalid},
@@ -97,15 +93,11 @@ func TestDaemonHandleEKS_AllHandlersDispatchToService(t *testing.T) {
{"eks.DeleteAddon", asMsgHandler(handleNATSRequest(d.node, d.eksService.DeleteAddon)), invalid},
{"eks.DescribeAddon", asMsgHandler(handleNATSRequest(d.node, d.eksService.DescribeAddon)), invalid},
{"eks.UpdateAddon", asMsgHandler(handleNATSRequest(d.node, d.eksService.UpdateAddon)), invalid},
{"eks.AssociateIdentityProviderConfig", asMsgHandler(handleNATSRequest(d.node, d.eksService.AssociateIdentityProviderConfig)), notImpl},
{"eks.DescribeIdentityProviderConfig", asMsgHandler(handleNATSRequest(d.node, d.eksService.DescribeIdentityProviderConfig)), notImpl},
{"eks.ListIdentityProviderConfigs", asMsgHandler(handleNATSRequest(d.node, d.eksService.ListIdentityProviderConfigs)), notImpl},
{"eks.DisassociateIdentityProviderConfig", asMsgHandler(handleNATSRequest(d.node, d.eksService.DisassociateIdentityProviderConfig)), notImpl},
{"eks.TagResource", asMsgHandler(handleNATSRequest(d.node, d.eksService.TagResource)), notImpl},
{"eks.UntagResource", asMsgHandler(handleNATSRequest(d.node, d.eksService.UntagResource)), notImpl},
{"eks.ListTagsForResource", asMsgHandler(handleNATSRequest(d.node, d.eksService.ListTagsForResource)), notImpl},
}
require.Len(t, cases, 30, "expected exactly one handler per non-lifecycle AWS EKS action")
require.Len(t, cases, 23, "expected exactly one handler per non-lifecycle AWS EKS action")
for _, c := range cases {
t.Run(c.subject, func(t *testing.T) {
spinifex/daemon/daemon_handlers_instance_tags_test.go
+19 -2
View File
@@ -8,6 +8,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/contracts/ec2/v1"
"github.com/mulgadc/spinifex/internal/testkit"
ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
ec2tags "github.com/mulgadc/spinifex/spinifex/domains/ec2/tags"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -155,9 +156,21 @@ func TestHandleSetInstanceTags_CrossAccountRejected(t *testing.T) {
// into the central tag store, so describe-tags sees them from birth. The write
// is best-effort, so the assertion tolerates it landing slightly late.
func TestHandleEC2RunInstances_LaunchTagsWriteCentralStore(t *testing.T) {
daemon, memStore := createFullTestDaemonWithStore(t, sharedNATSURL)
// The launch fails (the AMI has no snapshot), so the manager needs a
// terminated bucket to release the record into for its tags to be checked.
// Its own server keeps that record out of the shared terminated bucket.
ns, _, _ := testutil.StartTestJetStream(t)
daemon, memStore := createFullTestDaemonWithStore(t, ns.ClientURL())
seedTestAMI(t, memStore, daemon.config.Predastore.Bucket, "ami-launchtags")
jsm, err := NewJetStreamManager(daemon.natsConn)
require.NoError(t, err)
require.NoError(t, jsm.InitKVBucket())
require.NoError(t, jsm.InitTerminatedInstanceBucket())
daemon.jsManager = jsm
daemon.stateStore = newStateStoreAdapter(jsm, daemon.persistState)
daemon.vmMgr.SetDeps(vm.Deps{NodeID: daemon.node, StateStore: daemon.stateStore})
sub, err := daemon.natsConn.QueueSubscribe("ec2.RunInstances.launchtags", "spinifex-workers", asMsgHandler(daemon.handleEC2RunInstances))
require.NoError(t, err)
defer func() { _ = sub.Unsubscribe() }()
@@ -187,7 +200,11 @@ func TestHandleEC2RunInstances_LaunchTagsWriteCentralStore(t *testing.T) {
assert.Eventually(t, func() bool {
return assert.ObjectsAreEqual(want, centralTags(t, daemon, testAccountID, id))
}, 5*time.Second, 50*time.Millisecond, "central tag store must receive launch tags")
assert.Equal(t, want, recordTags(t, daemon, id))
assert.Eventually(t, func() bool {
v, err := daemon.jsManager.LoadTerminatedInstance(id)
return err == nil && v != nil && v.Instance != nil &&
assert.ObjectsAreEqual(want, tagsAsMap(v.Instance.Tags))
}, 5*time.Second, 10*time.Millisecond, "the released record must carry the launch tags")
}
// Missing InstanceTagsData, and a set with no tags, are rejected with
spinifex/daemon/daemon_handlers_instance_test.go
+234
View File
@@ -3,13 +3,17 @@ package daemon
import (
"encoding/json"
"errors"
"github.com/mulgadc/spinifex/contracts/ec2/v1"
"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
ec2image "github.com/mulgadc/spinifex/spinifex/domains/ec2/image"
ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
ec2key "github.com/mulgadc/spinifex/spinifex/domains/ec2/key"
ec2tags "github.com/mulgadc/spinifex/spinifex/domains/ec2/tags"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
@@ -487,3 +491,233 @@ func TestHandleEC2DescribeStoppedInstances_CrossAccountIsolation(t *testing.T) {
}
assert.ElementsMatch(t, []string{"i-mine"}, seen, "caller must only see their own instances")
}
// --- handleEC2RunInstances ---
func TestHandleEC2RunInstances_MissingAccountHeader(t *testing.T) {
d := createTestDaemon(t, sharedNATSURL)
body := mustMarshal(t, &ec2.RunInstancesInput{InstanceType: aws.String(getTestInstanceType(t))})
reply := requestHandler(t, d.natsConn, "ec2.RunInstances.p1-noacct", asMsgHandler(d.handleEC2RunInstances), "", body)
assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"])
assert.Zero(t, d.vmMgr.Count(), "an unattributable launch must not reserve anything")
}
func TestHandleEC2RunInstances_InvalidReservationTarget(t *testing.T) {
d := createTestDaemon(t, sharedNATSURL)
body := mustMarshal(t, &ec2.RunInstancesInput{
InstanceType: aws.String(getTestInstanceType(t)),
MinCount: aws.Int64(1),
MaxCount: aws.Int64(1),
CapacityReservationSpecification: &ec2.CapacityReservationSpecification{
CapacityReservationTarget: &ec2.CapacityReservationTarget{CapacityReservationId: aws.String("cr-0000000000missing")},
},
})
reply := requestHandler(t, d.natsConn, "ec2.RunInstances.p1-badcr", asMsgHandler(d.handleEC2RunInstances), testAccountID, body)
assert.Equal(t, awserrors.ErrorInvalidCapacityReservationIdNotFound, decodeError(t, reply.Data)["Code"])
assert.Zero(t, d.vmMgr.Count())
}
// A gateway-minted reservation ID replaces the one prepare generated, and a
// central tag store that cannot take the launch tags does not fail the launch.
func TestHandleEC2RunInstances_GatewayReservationAndTagStoreFailure(t *testing.T) {
d := createTestDaemon(t, sharedNATSURL)
images := objectstore.NewMemoryObjectStore()
seedTestAMI(t, images, d.config.Predastore.Bucket, "ami-p1-gateway")
d.instanceService = ec2instance.NewInstanceServiceImpl(
d.config, d.resourceMgr.instanceTypes, d.natsConn, images, d.vmMgr, d.resourceMgr, nil)
d.instanceService.SetRunInstancesDeps(
ec2image.NewImageServiceImplWithStore(images, d.config.Predastore.Bucket),
ec2key.NewKeyServiceImplWithStore(images, d.config.Predastore.Bucket), nil, nil)
_, tagKV := faultBucket(t)
for _, m := range []string{"Put", "Create", "Update"} {
tagKV.setFail(m, true)
}
d.tagsService = ec2tags.NewTagsServiceImplWithStore(d.config, objectstore.NewMemoryObjectStore(), tagKV)
t.Cleanup(d.vmMgr.WaitForBackgroundWork)
subject := "ec2.RunInstances.p1-gateway"
sub, err := d.natsConn.Subscribe(subject, asMsgHandler(d.handleEC2RunInstances))
require.NoError(t, err)
t.Cleanup(func() { _ = sub.Unsubscribe() })
msg := nats.NewMsg(subject)
msg.Data = mustMarshal(t, &ec2.RunInstancesInput{
ImageId: aws.String("ami-p1-gateway"),
InstanceType: aws.String(getTestInstanceType(t)),
MinCount: aws.Int64(1),
MaxCount: aws.Int64(1),
TagSpecifications: []*ec2.TagSpecification{{
ResourceType: aws.String("instance"),
Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("web")}},
}},
})
msg.Header.Set(natsmsg.AccountIDHeader, testAccountID)
msg.Header.Set(natsmsg.ReservationIDHeader, "r-0gateway000000001")
reply, err := d.natsConn.RequestMsg(msg, 5*time.Second)
require.NoError(t, err)
var reservation ec2.Reservation
require.NoError(t, json.Unmarshal(reply.Data, &reservation), "%s", reply.Data)
assert.Equal(t, "r-0gateway000000001", aws.StringValue(reservation.ReservationId))
require.Len(t, reservation.Instances, 1, "a failed tag projection must not fail the launch")
}
// --- handleEC2StartStoppedInstance forwarding ---
func TestHandleEC2StartStoppedInstance_Forwarding(t *testing.T) {
newFixture := func(t *testing.T, id, lastNode string) (*Daemon, *vmmock.StateStore, []byte) {
store := vmmock.New()
v := stoppedVMFixture(id, testAccountID)
v.InstanceType = "definitely.not.a.real.type"
v.LastNode = lastNode
store.Stopped[v.ID] = v
body, err := json.Marshal(ec2instance.StartStoppedInstanceInput{InstanceID: v.ID})
require.NoError(t, err)
return daemonWithFakeStateStore(t, store), store, body
}
owner := func(t *testing.T, d *Daemon, node string, fn nats.MsgHandler) {
sub, err := d.natsConn.Subscribe("ec2.start."+node, fn)
require.NoError(t, err)
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
t.Run("owner success is relayed", func(t *testing.T) {
d, _, body := newFixture(t, "i-p1-fwd-ok", "node-p1-ok")
owner(t, d, "node-p1-ok", func(m *nats.Msg) { _ = m.Respond([]byte(`{"started":true}`)) })
reply := requestHandler(t, d.natsConn, "ec2.start.p1-ok", asMsgHandler(d.handleEC2StartStoppedInstance), testAccountID, body)
assert.JSONEq(t, `{"started":true}`, string(reply.Data))
})
t.Run("owner at capacity falls back locally", func(t *testing.T) {
d, store, body := newFixture(t, "i-p1-fwd-cap", "node-p1-cap")
// The owner claims the record before answering at capacity, so the local
// attempt that follows finds it gone: a NotFound reply proves the
// fallback ran rather than the capacity error being relayed.
owner(t, d, "node-p1-cap", func(m *nats.Msg) {
_, _ = store.ClaimStoppedInstance("i-p1-fwd-cap")
_ = m.Respond(awserrors.GenerateErrorPayload(awserrors.ErrorInsufficientInstanceCapacity))
})
reply := requestHandler(t, d.natsConn, "ec2.start.p1-cap", asMsgHandler(d.handleEC2StartStoppedInstance), testAccountID, body)
assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, decodeError(t, reply.Data)["Code"])
})
t.Run("owner not subscribed falls back locally", func(t *testing.T) {
d, _, body := newFixture(t, "i-p1-fwd-gone", "node-p1-gone")
reply := requestHandler(t, d.natsConn, "ec2.start.p1-gone", asMsgHandler(d.handleEC2StartStoppedInstance), testAccountID, body)
assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, decodeError(t, reply.Data)["Code"],
"the local attempt must run and reject the unresolvable instance type")
})
t.Run("relay to a departed caller is an error", func(t *testing.T) {
d, _, body := newFixture(t, "i-p1-fwd-relay", "node-p1-relay")
owner(t, d, "node-p1-relay", func(m *nats.Msg) { _ = m.Respond([]byte(`{}`)) })
msg := nats.NewMsg("ec2.start.p1-relay")
msg.Data = body
msg.Header.Set(natsmsg.AccountIDHeader, testAccountID)
assert.Equal(t, outcomeError, d.handleEC2StartStoppedInstance(msg))
})
}
// --- handleSetInstanceTags / handleSetInstanceMonitoring ---
func setTagsCommand(id string) ec2v1.EC2InstanceCommand {
return ec2v1.EC2InstanceCommand{
ID: id,
Attributes: ec2v1.EC2CommandAttributes{SetInstanceTags: true},
InstanceTagsData: &ec2v1.InstanceTagsData{
Tags: map[string]string{"env": "dev"},
},
}
}
func TestHandleSetInstanceTags_Failures(t *testing.T) {
t.Run("record without an instance", func(t *testing.T) {
const id = "i-p1-tags-norecord"
d := tagTestDaemon(t, id, nil)
d.vmMgr.UpdateState(id, func(v *vm.VM) { v.Instance = nil })
reply := requestHandler(t, d.natsConn, "ec2.cmd."+id, d.handleEC2Events, testAccountID, mustMarshal(t, setTagsCommand(id)))
assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"])
})
t.Run("central store write fails", func(t *testing.T) {
const id = "i-p1-tags-central"
d := tagTestDaemon(t, id, map[string]string{"Name": "web"})
_, tagKV := faultBucket(t)
for _, m := range []string{"Put", "Create", "Update"} {
tagKV.setFail(m, true)
}
d.tagsService = ec2tags.NewTagsServiceImplWithStore(d.config, objectstore.NewMemoryObjectStore(), tagKV)
reply := requestHandler(t, d.natsConn, "ec2.cmd."+id, d.handleEC2Events, testAccountID, mustMarshal(t, setTagsCommand(id)))
assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"])
assert.Equal(t, map[string]string{"Name": "web"}, recordTags(t, d, id),
"the record must not move ahead of the central store")
})
t.Run("persist fails", func(t *testing.T) {
const id = "i-p1-tags-persist"
d := tagTestDaemon(t, id, nil)
d.vmMgr.SetDeps(vm.Deps{NodeID: d.node, StateStore: &vmmock.StateStore{SaveRunningErr: errInjected}})
reply := requestHandler(t, d.natsConn, "ec2.cmd."+id, d.handleEC2Events, testAccountID, mustMarshal(t, setTagsCommand(id)))
assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"])
})
t.Run("caller gone after the write", func(t *testing.T) {
const id = "i-p1-tags-noreply"
d := tagTestDaemon(t, id, nil)
instance, ok := d.vmMgr.Get(id)
require.True(t, ok)
msg := nats.NewMsg("ec2.cmd." + id)
msg.Header.Set(natsmsg.AccountIDHeader, testAccountID)
assert.Equal(t, outcomeSuccess, d.handleSetInstanceTags(t.Context(), msg, setTagsCommand(id), instance))
assert.Equal(t, map[string]string{"env": "dev"}, recordTags(t, d, id))
})
}
func TestHandleSetInstanceMonitoring_Failures(t *testing.T) {
enable := func(id string) ec2v1.EC2InstanceCommand {
return ec2v1.EC2InstanceCommand{
ID: id,
Attributes: ec2v1.EC2CommandAttributes{SetInstanceMonitoring: true},
InstanceMonitoringData: &ec2v1.InstanceMonitoringData{Enabled: true},
}
}
t.Run("missing data", func(t *testing.T) {
const id = "i-p1-mon-nodata"
d, _ := monitoringTestDaemon(t, id, false)
cmd := enable(id)
cmd.InstanceMonitoringData = nil
reply := requestHandler(t, d.natsConn, "ec2.cmd."+id, d.handleEC2Events, testAccountID, mustMarshal(t, cmd))
assert.Equal(t, awserrors.ErrorMissingParameter, decodeError(t, reply.Data)["Code"])
})
t.Run("persist fails", func(t *testing.T) {
const id = "i-p1-mon-persist"
d, _ := monitoringTestDaemon(t, id, false)
d.vmMgr.SetDeps(vm.Deps{NodeID: d.node, StateStore: &vmmock.StateStore{SaveRunningErr: errInjected}})
reply := requestHandler(t, d.natsConn, "ec2.cmd."+id, d.handleEC2Events, testAccountID, mustMarshal(t, enable(id)))
assert.Equal(t, awserrors.ErrorServerInternal, decodeError(t, reply.Data)["Code"])
})
t.Run("caller gone after the write", func(t *testing.T) {
const id = "i-p1-mon-noreply"
d, _ := monitoringTestDaemon(t, id, false)
instance, ok := d.vmMgr.Get(id)
require.True(t, ok)
assert.Equal(t, outcomeSuccess, d.handleSetInstanceMonitoring(t.Context(), noReplyMsg("ec2.cmd."+id, nil), enable(id), instance))
assert.True(t, recordMonitoring(t, d, id))
})
}
spinifex/daemon/daemon_handlers_tag_test.go
+83
View File
@@ -6,8 +6,21 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
ec2eigw "github.com/mulgadc/spinifex/spinifex/domains/ec2/eigw"
ec2eip "github.com/mulgadc/spinifex/spinifex/domains/ec2/eip"
ec2igw "github.com/mulgadc/spinifex/spinifex/domains/ec2/igw"
ec2image "github.com/mulgadc/spinifex/spinifex/domains/ec2/image"
ec2key "github.com/mulgadc/spinifex/spinifex/domains/ec2/key"
ec2natgw "github.com/mulgadc/spinifex/spinifex/domains/ec2/natgw"
ec2routetable "github.com/mulgadc/spinifex/spinifex/domains/ec2/routetable"
ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
ec2tags "github.com/mulgadc/spinifex/spinifex/domains/ec2/tags"
ec2volume "github.com/mulgadc/spinifex/spinifex/domains/ec2/volume"
ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
"github.com/nats-io/nats.go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -205,3 +218,73 @@ func TestDeleteTags_Validation(t *testing.T) {
_, err = d.deleteTags(context.Background(), &ec2.DeleteTagsInput{}, testAccountID)
assert.Equal(t, awserrors.ErrorMissingParameter, err.Error())
}
// Every service that stores tags on its own record must be mirrored, or a
// create-tags reaches the central store and leaves that record stale.
func TestRecordTagMirrors_IncludesEveryRecordStore(t *testing.T) {
assert.Empty(t, (&Daemon{eipService: ec2eip.NewDisabledEIPService()}).recordTagMirrors(),
"the disabled EIP service holds no records to mirror")
d := &Daemon{
vpcService: &ec2vpc.VPCServiceImpl{},
imageService: &ec2image.ImageServiceImpl{},
volumeService: &ec2volume.VolumeServiceImpl{},
snapshotService: &ec2snapshot.SnapshotServiceImpl{},
routeTableService: &ec2routetable.RouteTableServiceImpl{},
igwService: &ec2igw.IGWServiceImpl{},
eigwService: &ec2eigw.EgressOnlyIGWServiceImpl{},
eipService: &ec2eip.EIPServiceImpl{},
natGatewayService: &ec2natgw.NatGatewayServiceImpl{},
keyService: &ec2key.KeyServiceImpl{},
}
assert.Len(t, d.recordTagMirrors(), 10)
}
func TestDeleteTags_NonInstanceResource(t *testing.T) {
d := tagTestDaemon(t, "i-p1-tag-unused", nil)
require.NoError(t, d.tagsService.PutResourceTags(t.Context(), testAccountID, "vol-p1-tagdel",
map[string]string{"env": "dev", "team": "infra"}))
_, err := d.deleteTags(context.Background(), &ec2.DeleteTagsInput{
Resources: []*string{nil, aws.String("vol-p1-tagdel")},
Tags: []*ec2.Tag{{Key: nil}, {Key: aws.String("team")}},
}, testAccountID)
require.NoError(t, err)
assert.Equal(t, map[string]string{"env": "dev"}, centralTags(t, d, testAccountID, "vol-p1-tagdel"))
}
func TestTags_CentralStoreFailure(t *testing.T) {
d := tagTestDaemon(t, "i-p1-tag-unused2", nil)
_, tagKV := faultBucket(t)
for _, m := range []string{"Get", "Put", "Create", "Update", "Delete", "Purge"} {
tagKV.setFail(m, true)
}
d.tagsService = ec2tags.NewTagsServiceImplWithStore(d.config, objectstore.NewMemoryObjectStore(), tagKV)
vol := []*string{aws.String("vol-p1-tagfail")}
_, err := d.createTags(context.Background(), &ec2.CreateTagsInput{
Resources: vol, Tags: []*ec2.Tag{{Key: aws.String("k"), Value: aws.String("v")}},
}, testAccountID)
require.Error(t, err)
_, err = d.deleteTags(context.Background(), &ec2.DeleteTagsInput{Resources: vol}, testAccountID)
require.Error(t, err)
}
func TestCreateTags_OwnerRequestFails(t *testing.T) {
d := tagTestDaemon(t, "i-p1-tag-closed", nil)
closed, err := nats.Connect(sharedNATSURL)
require.NoError(t, err)
closed.Close()
orig := d.natsConn
d.natsConn = closed
defer func() { d.natsConn = orig }()
_, err = d.createTags(context.Background(), &ec2.CreateTagsInput{
Resources: []*string{aws.String("i-p1-tag-closed")},
Tags: []*ec2.Tag{{Key: aws.String("k"), Value: aws.String("v")}},
}, testAccountID)
require.Error(t, err)
assert.Equal(t, awserrors.ErrorServerInternal, err.Error(),
"a transport failure is not evidence the instance is gone")
}
spinifex/daemon/daemon_handlers_test.go
+131 -8
View File
@@ -23,6 +23,7 @@ import (
viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
"github.com/mulgadc/spinifex/internal/testkit"
"github.com/mulgadc/spinifex/internal/testkit/ebsfake"
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
ec2account "github.com/mulgadc/spinifex/spinifex/domains/ec2/account"
"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
ec2eigw "github.com/mulgadc/spinifex/spinifex/domains/ec2/eigw"
@@ -696,14 +697,15 @@ func TestHandleEC2Events_StopInstance(t *testing.T) {
daemon := createFullTestDaemonWithJetStream(t, natsURL)
instanceID := "i-test-stop-001"
daemon.vmMgr.Insert(&vm.VM{
instance := &vm.VM{
ID: instanceID,
InstanceType: getTestInstanceType(t),
Status: vm.StateRunning,
Instance: &ec2.Instance{},
QMPClient: &qmp.QMPClient{},
AccountID: testAccountID,
})
}
daemon.vmMgr.Insert(instance)
sub, err := daemon.natsConn.Subscribe(
fmt.Sprintf("ec2.cmd.%s", instanceID),
@@ -731,18 +733,22 @@ func TestHandleEC2Events_StopInstance(t *testing.T) {
// The ack comes first and the transition runs detached, so the state is a
// later fact than the reply — and it does not stop at stopping.
assertLeavesRunning(t, daemon, instanceID, vm.StateStopping, vm.StateStopped)
assertLeavesRunning(t, daemon, instance, vm.StateStopping, vm.StateStopped)
}
// assertLeavesRunning waits for a detached stop or terminate to move the
// instance out of running. Asserting on the transient state alone is a race in
// both directions: too early reads running, too late reads the final state.
func assertLeavesRunning(t *testing.T, daemon *Daemon, instanceID string, want ...vm.InstanceState) {
func assertLeavesRunning(t *testing.T, daemon *Daemon, instance *vm.VM, want ...vm.InstanceState) {
t.Helper()
require.EventuallyWithT(t, func(c *assert.CollectT) {
var status vm.InstanceState
daemon.vmMgr.UpdateState(instanceID, func(v *vm.VM) { status = v.Status })
if !daemon.vmMgr.UpdateState(instance.ID, func(v *vm.VM) { status = v.Status }) {
// A finished stop or terminate drops the instance from the manager
// under the same lock, so the dropped VM holds the state it left in.
status = instance.Status
}
assert.Contains(c, want, status)
}, 10*time.Second, 10*time.Millisecond)
}
@@ -753,14 +759,15 @@ func TestHandleEC2Events_TerminateInstance(t *testing.T) {
daemon := createFullTestDaemonWithJetStream(t, natsURL)
instanceID := "i-test-term-001"
daemon.vmMgr.Insert(&vm.VM{
instance := &vm.VM{
ID: instanceID,
InstanceType: getTestInstanceType(t),
Status: vm.StateRunning,
Instance: &ec2.Instance{},
QMPClient: &qmp.QMPClient{},
AccountID: testAccountID,
})
}
daemon.vmMgr.Insert(instance)
sub, err := daemon.natsConn.Subscribe(
fmt.Sprintf("ec2.cmd.%s", instanceID),
@@ -785,7 +792,7 @@ func TestHandleEC2Events_TerminateInstance(t *testing.T) {
assert.Equal(t, `{}`, string(reply.Data))
assertLeavesRunning(t, daemon, instanceID, vm.StateShuttingDown, vm.StateTerminated)
assertLeavesRunning(t, daemon, instance, vm.StateShuttingDown, vm.StateTerminated)
}
func TestHandleEC2Events_RebootRunningInstance(t *testing.T) {
@@ -4060,3 +4067,119 @@ func TestHandleEC2CreateImage_StoppedInstanceNotInKV(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, errResp["Code"])
}
func TestHandleNATSRequestWithPrincipal(t *testing.T) {
nc, err := nats.Connect(sharedNATSURL)
require.NoError(t, err)
defer nc.Close()
const principal = "arn:aws:iam::000000000001:user/alice"
fn := func(_ context.Context, in *testInput, accountID, principalARN string) (*testOutput, error) {
if in.Name == "fail" {
return nil, errors.New(awserrors.ErrorInvalidParameterValue)
}
return &testOutput{Greeting: in.Name + "@" + accountID + "/" + principalARN}, nil
}
sub, err := nc.Subscribe("test.principal", asMsgHandler(handleNATSRequestWithPrincipal("test-node", fn)))
require.NoError(t, err)
defer func() { _ = sub.Unsubscribe() }()
request := func(body string) *nats.Msg {
msg := nats.NewMsg("test.principal")
msg.Data = []byte(body)
msg.Header.Set(natsmsg.AccountIDHeader, testAccountID)
msg.Header.Set(natsmsg.PrincipalARNHeader, principal)
reply, err := nc.RequestMsg(msg, 5*time.Second)
require.NoError(t, err)
return reply
}
t.Run("caller identity reaches the service", func(t *testing.T) {
var out testOutput
require.NoError(t, json.Unmarshal(request(`{"name":"bob"}`).Data, &out))
assert.Equal(t, "bob@"+testAccountID+"/"+principal, out.Greeting)
})
t.Run("service error", func(t *testing.T) {
assert.Equal(t, awserrors.ErrorInvalidParameterValue, decodeError(t, request(`{"name":"fail"}`).Data)["Code"])
})
t.Run("malformed payload", func(t *testing.T) {
assert.Equal(t, awserrors.ErrorValidationError, decodeError(t, request(`{`).Data)["Code"])
})
}
func TestHandleEC2Events_CommandValidation(t *testing.T) {
d := createTestDaemon(t, sharedNATSURL)
instanceID := "i-p1-cmd-validate"
d.vmMgr.Insert(&vm.VM{
ID: instanceID,
InstanceType: getTestInstanceType(t),
Status: vm.StateRunning,
Instance: &ec2.Instance{},
AccountID: testAccountID,
})
cases := []struct {
name string
attrs ec2v1.EC2CommandAttributes
want string
}{
{"start a running instance", ec2v1.EC2CommandAttributes{StartInstance: true}, awserrors.ErrorIncorrectInstanceState},
{"attach ENI without data", ec2v1.EC2CommandAttributes{AttachENI: true}, awserrors.ErrorInvalidParameterValue},
{"detach ENI without data", ec2v1.EC2CommandAttributes{DetachENI: true}, awserrors.ErrorInvalidParameterValue},
{"associate profile without data", ec2v1.EC2CommandAttributes{AssociateIamInstanceProfile: true}, awserrors.ErrorMissingParameter},
{"no command set", ec2v1.EC2CommandAttributes{}, awserrors.ErrorServerInternal},
}
for i, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
body, err := json.Marshal(ec2v1.EC2InstanceCommand{ID: instanceID, Attributes: tc.attrs})
require.NoError(t, err)
subject := fmt.Sprintf("ec2.cmd.p1-validate-%d", i)
reply := requestHandler(t, d.natsConn, subject, d.handleEC2Events, testAccountID, body)
assert.Equal(t, tc.want, decodeError(t, reply.Data)["Code"])
})
}
var status vm.InstanceState
d.vmMgr.UpdateState(instanceID, func(v *vm.VM) { status = v.Status })
assert.Equal(t, vm.StateRunning, status, "rejected commands must leave the instance alone")
}
// A requester that has gone away cannot be answered, but the admitted reboot
// still counts as a success rather than an error.
func TestDispatchEC2Command_RebootWithoutReplySubject(t *testing.T) {
d := createTestDaemon(t, sharedNATSURL)
instanceID := "i-p1-reboot-noreply"
d.vmMgr.Insert(&vm.VM{
ID: instanceID,
InstanceType: getTestInstanceType(t),
Status: vm.StateRunning,
Instance: &ec2.Instance{},
QMPClient: &qmp.QMPClient{},
AccountID: testAccountID,
})
t.Cleanup(d.vmMgr.WaitForBackgroundWork)
body, err := json.Marshal(ec2v1.EC2InstanceCommand{
ID: instanceID,
Attributes: ec2v1.EC2CommandAttributes{RebootInstance: true},
})
require.NoError(t, err)
msg := nats.NewMsg("ec2.cmd." + instanceID)
msg.Data = body
msg.Header.Set(natsmsg.AccountIDHeader, testAccountID)
name, outcome := d.dispatchEC2Command(msg)
assert.Equal(t, "RebootInstance", name)
assert.Equal(t, outcomeSuccess, outcome)
}
func TestIsOVNDBQuorumMember_NoClusterConfig(t *testing.T) {
assert.False(t, (&Daemon{node: "node-1"}).isOVNDBQuorumMember())
}
func TestQueryNATSRole_NoNATSService(t *testing.T) {
d := &Daemon{config: &config.Config{Services: []string{"daemon"}}}
assert.Empty(t, d.queryNATSRole(), "a node without NATS has no JetStream role to probe")
}
spinifex/daemon/daemon_handlers_vpc_test.go
+72
View File
@@ -15,6 +15,7 @@ import (
ec2routetable "github.com/mulgadc/spinifex/spinifex/domains/ec2/routetable"
ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -284,3 +285,74 @@ func TestEnsureDefaultVPCInfrastructure_SkipsHalfBuiltAccounts(t *testing.T) {
require.NoError(t, err)
assert.Len(t, completed.InternetGateways, 1)
}
// Without persistence the VPC service reports no default VPC and no error; the
// handlers must say so rather than reply with an empty ID.
func TestDefaultVPCHandlers_NoPersistence(t *testing.T) {
d := createTestDaemon(t, sharedNATSURL)
d.vpcService = &ec2vpc.VPCServiceImpl{}
msg, sub := syncReply(t, d.natsConn, "ec2.test.EnsureDefaultVpc.nopersist", []byte(`{"account_id":"000000000771"}`))
assert.Equal(t, outcomeError, d.handleEnsureDefaultVpc(msg))
reply, err := sub.NextMsg(2 * time.Second)
require.NoError(t, err)
assert.JSONEq(t, `{"error":"default VPC has no ID"}`, string(reply.Data))
assert.Equal(t, outcomeSuccess, d.handleAccountCreated(noReplyMsg("account.created", []byte(`{"account_id":"000000000771"}`))),
"with no gateway service there is no infrastructure to build")
}
func TestDefaultVPCHandlers_UnreadableClaim(t *testing.T) {
d, nc := createDefaultVPCTestDaemon(t)
const account = "000000000772"
js, err := jetstream.New(nc)
require.NoError(t, err)
defaults, err := js.KeyValue(t.Context(), ec2vpc.KVBucketDefaultVPCs)
require.NoError(t, err)
_, err = defaults.Put(t.Context(), account, []byte("{"))
require.NoError(t, err)
assert.Equal(t, outcomeError, d.handleAccountCreated(noReplyMsg("account.created", []byte(`{"account_id":"`+account+`"}`))))
msg, sub := syncReply(t, nc, "ec2.test.EnsureDefaultVpc.unreadable", []byte(`{"account_id":"`+account+`"}`))
assert.Equal(t, outcomeError, d.handleEnsureDefaultVpc(msg))
reply, err := sub.NextMsg(2 * time.Second)
require.NoError(t, err)
assert.JSONEq(t, `{"error":"could not create default VPC"}`, string(reply.Data))
// A requester that has gone away gets the same outcome, with no reply sent.
assert.Equal(t, outcomeError, d.handleEnsureDefaultVpc(noReplyMsg("ec2.EnsureDefaultVpc", []byte(`{"account_id":"`+account+`"}`))))
}
func TestEnsureDefaultVPCInfrastructure_Partial(t *testing.T) {
d, _ := createDefaultVPCTestDaemon(t)
t.Run("account without a default VPC", func(t *testing.T) {
const account = "000000000773"
d.ensureDefaultVPCInfrastructureFor(t.Context(), account)
igws, err := d.igwService.DescribeInternetGateways(t.Context(), &ec2.DescribeInternetGatewaysInput{}, account)
require.NoError(t, err)
assert.Empty(t, igws.InternetGateways)
})
t.Run("route waits for a gateway", func(t *testing.T) {
const account = "000000000775"
info, err := d.vpcService.EnsureDefaultVPC(account)
require.NoError(t, err)
require.NotNil(t, info)
d.ensureDefaultIGWRoute(t.Context(), account, info.VpcId)
out, err := d.routeTableService.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{
Filters: []*ec2.Filter{{Name: aws.String("vpc-id"), Values: []*string{aws.String(info.VpcId)}}},
}, account)
require.NoError(t, err)
require.NotEmpty(t, out.RouteTables)
for _, rtb := range out.RouteTables {
for _, r := range rtb.Routes {
assert.NotEqual(t, "0.0.0.0/0", aws.StringValue(r.DestinationCidrBlock),
"a default route with no gateway would black-hole egress")
}
}
})
}
spinifex/daemon/daemon_system_dispatch_test.go
+149
View File
@@ -2,11 +2,14 @@ package daemon
import (
"encoding/json"
"strings"
"testing"
"time"
"github.com/mulgadc/spinifex/spinifex/domains/ec2/systeminstance"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
"github.com/nats-io/nats.go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -546,3 +549,149 @@ func msgWithConn(nc *nats.Conn, msg *nats.Msg) *nats.Msg {
msg.Sub = sub
return msg
}
// syncReply returns a message whose reply lands on a synchronous inbox, so a
// handler called directly can be asserted on without a responder goroutine.
func syncReply(t *testing.T, nc *nats.Conn, subject string, data []byte) (*nats.Msg, *nats.Subscription) {
t.Helper()
inbox := nats.NewInbox()
sub, err := nc.SubscribeSync(inbox)
require.NoError(t, err)
t.Cleanup(func() { _ = sub.Unsubscribe() })
return &nats.Msg{Subject: subject, Reply: inbox, Sub: sub, Data: data}, sub
}
func nextEnvelopeError(t *testing.T, sub *nats.Subscription) string {
t.Helper()
reply, err := sub.NextMsg(2 * time.Second)
require.NoError(t, err)
var env struct {
Error string `json:"error,omitempty"`
}
require.NoError(t, json.Unmarshal(reply.Data, &env))
return env.Error
}
func TestServeSystemLaunchInstance_UnknownType(t *testing.T) {
nc, err := nats.Connect(sharedNATSURL)
require.NoError(t, err)
t.Cleanup(nc.Close)
rm, err := NewResourceManager(nil, nil, nil)
require.NoError(t, err)
d := &Daemon{natsConn: nc, resourceMgr: rm, natsSubscriptions: make(map[string]*nats.Subscription)}
msg, sub := syncReply(t, nc, "system.LaunchInstance.zz9.none", []byte(`{"instance_type":"zz9.none"}`))
assert.Equal(t, outcomeError, d.serveSystemLaunchInstance(msg))
assert.Contains(t, nextEnvelopeError(t, sub), "unknown instance type: zz9.none")
assert.Empty(t, d.natsSubscriptions, "a failed launch must not subscribe a terminate subject")
}
func TestLaunchSystemInstanceOnNode_Failures(t *testing.T) {
nc, err := nats.Connect(sharedNATSURL)
require.NoError(t, err)
t.Cleanup(nc.Close)
rm, err := NewResourceManager(nil, nil, nil)
require.NoError(t, err)
d := &Daemon{natsConn: nc, node: "node-a", resourceMgr: rm}
t.Run("local target validates the type", func(t *testing.T) {
_, err := d.LaunchSystemInstanceOnNode("node-a", &handlers_elbv2.SystemInstanceInput{InstanceType: "zz9.none"})
assert.ErrorContains(t, err, "unknown instance type")
})
t.Run("remote without a type", func(t *testing.T) {
_, err := d.LaunchSystemInstanceOnNode("node-b", &handlers_elbv2.SystemInstanceInput{})
assert.ErrorContains(t, err, "missing InstanceType")
_, err = d.LaunchSystemInstanceOnNode("node-b", nil)
assert.ErrorContains(t, err, "missing InstanceType")
})
t.Run("remote node not listening", func(t *testing.T) {
_, err := d.LaunchSystemInstanceOnNode("node-gone", &handlers_elbv2.SystemInstanceInput{InstanceType: "sys.p1"})
assert.ErrorIs(t, err, nats.ErrNoResponders)
})
replies := map[string]string{
"undecodable reply": "not json",
"empty envelope": "{}",
}
wantErr := map[string]string{
"undecodable reply": "decode launch reply",
"empty envelope": "missing output payload",
}
for name, body := range replies {
t.Run(name, func(t *testing.T) {
node := "node-" + strings.ReplaceAll(name, " ", "-")
sub, err := nc.Subscribe("system.LaunchInstance.sys.p1."+node, func(msg *nats.Msg) {
_ = msg.Respond([]byte(body))
})
require.NoError(t, err)
t.Cleanup(func() { _ = sub.Unsubscribe() })
_, err = d.LaunchSystemInstanceOnNode(node, &handlers_elbv2.SystemInstanceInput{InstanceType: "sys.p1"})
assert.ErrorContains(t, err, wantErr[name])
})
}
}
func TestTerminateSystemInstanceRemote_TransportFailures(t *testing.T) {
t.Run("closed connection is not reported as already gone", func(t *testing.T) {
nc, err := nats.Connect(sharedNATSURL)
require.NoError(t, err)
nc.Close()
err = (&Daemon{natsConn: nc}).terminateSystemInstanceRemote("i-closed")
require.ErrorContains(t, err, "route terminate i-closed")
assert.NotErrorIs(t, err, systeminstance.ErrSystemInstanceNotFound)
})
t.Run("undecodable reply", func(t *testing.T) {
nc, err := nats.Connect(sharedNATSURL)
require.NoError(t, err)
t.Cleanup(nc.Close)
sub, err := nc.Subscribe("system.TerminateInstance.i-garbled", func(msg *nats.Msg) {
_ = msg.Respond([]byte("garbled"))
})
require.NoError(t, err)
t.Cleanup(func() { _ = sub.Unsubscribe() })
err = (&Daemon{natsConn: nc}).terminateSystemInstanceRemote("i-garbled")
assert.ErrorContains(t, err, "decode routed terminate reply i-garbled")
})
}
func TestSubscribeSystemTerminate_ConnFailures(t *testing.T) {
d := &Daemon{natsSubscriptions: make(map[string]*nats.Subscription)}
require.NoError(t, d.subscribeSystemTerminateLocked("i-nil"), "nil natsConn must short-circuit")
nc, err := nats.Connect(sharedNATSURL)
require.NoError(t, err)
nc.Close()
d.natsConn = nc
require.ErrorContains(t, d.subscribeSystemTerminate("i-closed"), "subscribe system.TerminateInstance.i-closed")
assert.Empty(t, d.natsSubscriptions)
}
func TestHandleSystemTerminateInstance_Errors(t *testing.T) {
nc, err := nats.Connect(sharedNATSURL)
require.NoError(t, err)
t.Cleanup(nc.Close)
t.Run("unknown instance", func(t *testing.T) {
d := &Daemon{natsConn: nc, vmMgr: vm.NewManager(), natsSubscriptions: make(map[string]*nats.Subscription)}
msg, sub := syncReply(t, nc, "system.TerminateInstance.i-unknown", nil)
d.serveSystemTerminateInstance(msg)
assert.Contains(t, nextEnvelopeError(t, sub), systeminstance.ErrSystemInstanceNotFound.Error())
})
// No VM manager makes the terminate nil-deref; the dispatch goroutine must
// recover and still answer.
t.Run("panic is recovered", func(t *testing.T) {
d := &Daemon{natsConn: nc, natsSubscriptions: make(map[string]*nats.Subscription)}
msg, sub := syncReply(t, nc, "system.TerminateInstance.i-panic", nil)
d.handleSystemTerminateInstance(msg)
assert.Equal(t, awserrors.ErrorServerInternal, nextEnvelopeError(t, sub))
d.systemDispatchWg.Wait()
})
}
spinifex/daemon/daemon_system_instance_coverage_test.go
+842
View File
@@ -0,0 +1,842 @@
package daemon
import (
"context"
"errors"
"fmt"
awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
"os"
"path/filepath"
"strings"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/internal/testkit"
"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
"github.com/mulgadc/spinifex/spinifex/domains/ec2/systeminstance"
ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
"github.com/mulgadc/spinifex/spinifex/domains/network/external"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2"
"github.com/mulgadc/spinifex/spinifex/providers/ebs"
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// sysInstFixture is a VPC test daemon with one subnet in the customer account
// and one in the system account, so launches can use real ENIs in either.
type sysInstFixture struct {
d *Daemon
js jetstream.JetStream
subnetID string
sysSubnetID string
itype string
}
func newSysInstFixture(t *testing.T) *sysInstFixture {
t.Helper()
t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
d := createTestDaemon(t, sharedNATSURL)
_, nc, js := testutil.StartTestJetStream(t)
testutil.StubVpcdSGResponder(t, nc)
vpcSvc, err := ec2vpc.NewVPCServiceImplWithNATS(t.Context(), d.config, nc)
require.NoError(t, err)
d.vpcService = vpcSvc
// MarkFailed tears down in the background; let it finish before the
// temp dirs and the NATS connection go away.
t.Cleanup(d.vmMgr.WaitForBackgroundWork)
// Launches here never hand capacity back (no resource controller is
// wired), so give the node room for every launch the test makes.
d.resourceMgr.mu.Lock()
d.resourceMgr.hostVCPU += 64
d.resourceMgr.hostMemGB += 256
d.resourceMgr.mu.Unlock()
f := &sysInstFixture{d: d, js: js, itype: getTestInstanceType(t)}
f.subnetID = f.newSubnet(t, testAccountID, "10.81")
f.sysSubnetID = f.newSubnet(t, awsidentifiers.GlobalAccountID, "10.82")
return f
}
func (f *sysInstFixture) newSubnet(t *testing.T, accountID, prefix string) string {
t.Helper()
vpcOut, err := f.d.vpcService.CreateVpc(t.Context(), &ec2.CreateVpcInput{
CidrBlock: aws.String(prefix + ".0.0/16"),
}, accountID)
require.NoError(t, err)
subnetOut, err := f.d.vpcService.CreateSubnet(t.Context(), &ec2.CreateSubnetInput{
VpcId: vpcOut.Vpc.VpcId,
CidrBlock: aws.String(prefix + ".1.0/24"),
}, accountID)
require.NoError(t, err)
return aws.StringValue(subnetOut.Subnet.SubnetId)
}
// newENI creates an available ENI in the customer subnet.
func (f *sysInstFixture) newENI(t *testing.T) *ec2.NetworkInterface {
t.Helper()
out, err := f.d.vpcService.CreateNetworkInterface(t.Context(), &ec2.CreateNetworkInterfaceInput{
SubnetId: aws.String(f.subnetID),
}, testAccountID)
require.NoError(t, err)
return out.NetworkInterface
}
// input returns a direct-boot launch on a pre-created customer ENI with a
// valid single-NIC network config.
func (f *sysInstFixture) input(eni *ec2.NetworkInterface) *handlers_elbv2.SystemInstanceInput {
return &handlers_elbv2.SystemInstanceInput{
InstanceType: f.itype,
SubnetID: f.subnetID,
AccountID: testAccountID,
ENIID: aws.StringValue(eni.NetworkInterfaceId),
ENIMac: aws.StringValue(eni.MacAddress),
ENIIP: aws.StringValue(eni.PrivateIpAddress),
NICs: []handlers_elbv2.NICConfig{{MAC: aws.StringValue(eni.MacAddress), CIDR: "10.81.1.5/24", IsDefault: true}},
}
}
func (f *sysInstFixture) eniRecord(t *testing.T, accountID, eniID string) ec2vpc.ENIRecord {
t.Helper()
rec, err := f.d.vpcService.GetENIRecord(accountID, eniID)
require.NoError(t, err)
return rec
}
// sysInstLaunchRuns swaps the volume mounter for one that marks the instance
// running, so vm.Manager.Run returns before it would start QEMU. It waits out
// earlier teardowns first, since they read the deps it replaces.
func sysInstLaunchRuns(d *Daemon) {
d.vmMgr.WaitForBackgroundWork()
d.vmMgr.SetDeps(vm.Deps{
NodeID: d.node,
VolumeMounter: &hookVolumeMounter{onMount: func(v *vm.VM) {
d.vmMgr.UpdateState(v.ID, func(x *vm.VM) { x.Status = vm.StateRunning })
}},
VolumeStateUpdater: d.volumeService,
})
}
// sysInstLaunchFails swaps the volume mounter for one that refuses every mount, so
// vm.Manager.Run fails without reaching QEMU.
func sysInstLaunchFails(d *Daemon) {
d.vmMgr.WaitForBackgroundWork()
d.vmMgr.SetDeps(vm.Deps{
NodeID: d.node,
VolumeMounter: &hookVolumeMounter{mountErr: errSysInstLaunchRefused},
VolumeStateUpdater: d.volumeService,
})
}
var errSysInstLaunchRefused = errors.New("launch refused")
// sysInstMgmtAllocator returns an allocator bound to the shared cluster-state KV
// on a subnet no other test uses.
func sysInstMgmtAllocator(t *testing.T, bridgeIP string) *MgmtIPAllocator {
t.Helper()
jsm := newTestMgmtJSM(t)
a, err := NewMgmtIPAllocator(bridgeIP)
require.NoError(t, err)
a.BindKV(jsm, "node-1")
cleanupMgmtIPAM(t, jsm, a)
return a
}
// sysInstUnboundMgmtAllocator has no KV, so every Allocate fails.
func sysInstUnboundMgmtAllocator(t *testing.T) *MgmtIPAllocator {
t.Helper()
a, err := NewMgmtIPAllocator("10.94.8.1")
require.NoError(t, err)
return a
}
// sysInstReclaimingEIP adds the KV-backed backstop release that
// reclaimSystemInstanceEIP looks for.
type sysInstReclaimingEIP struct {
fakeEIPService
reclaimErr error
reclaimed []string
}
func (s *sysInstReclaimingEIP) ReleaseAddressByInstanceID(id string) error {
s.reclaimed = append(s.reclaimed, id)
return s.reclaimErr
}
func TestLaunchSystemInstance_EarlyErrors(t *testing.T) {
t.Run("BootAMI dispatches to the AMI path", func(t *testing.T) {
_, err := (&Daemon{}).LaunchSystemInstance(&handlers_elbv2.SystemInstanceInput{BootMode: systeminstance.BootAMI})
require.Error(t, err)
assert.Contains(t, err.Error(), "instance service not initialized")
})
d := createTestDaemon(t, sharedNATSURL)
itype := getTestInstanceType(t)
t.Run("unknown instance type", func(t *testing.T) {
_, err := d.LaunchSystemInstance(&handlers_elbv2.SystemInstanceInput{InstanceType: "zz9.none"})
require.Error(t, err)
assert.Contains(t, err.Error(), "unknown instance type: zz9.none")
})
t.Run("capacity exhausted", func(t *testing.T) {
d.resourceMgr.mu.Lock()
saved := d.resourceMgr.allocatedVCPU
d.resourceMgr.allocatedVCPU = 1 << 20
d.resourceMgr.mu.Unlock()
t.Cleanup(func() {
d.resourceMgr.mu.Lock()
d.resourceMgr.allocatedVCPU = saved
d.resourceMgr.mu.Unlock()
})
_, err := d.LaunchSystemInstance(&handlers_elbv2.SystemInstanceInput{InstanceType: itype})
require.Error(t, err)
assert.Contains(t, err.Error(), "insufficient capacity for "+itype)
})
t.Run("RunInstance failure returns the capacity", func(t *testing.T) {
orig := d.instanceService
t.Cleanup(func() { d.instanceService = orig })
// An instance service that knows no types rejects the launch after the
// daemon has already reserved capacity for it.
d.instanceService = ec2instance.NewInstanceServiceImpl(d.config, map[string]*ec2.InstanceTypeInfo{},
d.natsConn, objectstore.NewMemoryObjectStore(), d.vmMgr, d.resourceMgr, nil)
d.resourceMgr.mu.Lock()
before := d.resourceMgr.allocatedVCPU
d.resourceMgr.mu.Unlock()
_, err := d.LaunchSystemInstance(&handlers_elbv2.SystemInstanceInput{InstanceType: itype})
require.Error(t, err)
assert.Contains(t, err.Error(), "create instance")
d.resourceMgr.mu.Lock()
after := d.resourceMgr.allocatedVCPU
d.resourceMgr.mu.Unlock()
assert.Equal(t, before, after, "the reserved capacity must be released")
})
}
func TestLaunchSystemInstance_PreCreatedENI(t *testing.T) {
f := newSysInstFixture(t)
d := f.d
t.Run("made-up ENI still reaches launch", func(t *testing.T) {
sysInstLaunchFails(d)
in := f.input(f.newENI(t))
in.ENIID = "eni-madeup"
in.IamInstanceProfileArn = "arn:aws:iam::123456789012:instance-profile/lb"
// The attach and owner stamp are best-effort, so only the launch fails.
_, err := d.LaunchSystemInstance(in)
require.ErrorIs(t, err, errSysInstLaunchRefused)
assert.Contains(t, err.Error(), "launch instance")
})
t.Run("cross-account ENI and extras are attached and stamped", func(t *testing.T) {
sysInstLaunchRuns(d)
primary, extra := f.newENI(t), f.newENI(t)
extraID := aws.StringValue(extra.NetworkInterfaceId)
in := f.input(primary)
in.ExtraENIs = []systeminstance.ExtraENIInput{{
ENIID: extraID,
ENIMac: aws.StringValue(extra.MacAddress),
ENIIP: aws.StringValue(extra.PrivateIpAddress),
SubnetID: f.subnetID,
DeleteOnTermination: aws.Bool(false),
}}
in.HostfwdPorts = []int{6443}
out, err := d.LaunchSystemInstance(in)
require.NoError(t, err)
assert.Equal(t, aws.StringValue(primary.PrivateIpAddress), out.PrivateIP)
assert.Empty(t, out.PublicIP)
rec := f.eniRecord(t, testAccountID, aws.StringValue(primary.NetworkInterfaceId))
assert.Equal(t, out.InstanceID, rec.InstanceId)
assert.Equal(t, awsidentifiers.GlobalAccountID, rec.InstanceOwnerId, "a system VM on a customer ENI must stamp its own account")
extraRec := f.eniRecord(t, testAccountID, extraID)
assert.Equal(t, out.InstanceID, extraRec.InstanceId)
assert.Equal(t, int64(1), extraRec.DeviceIndex)
require.NotNil(t, extraRec.DeleteOnTermination)
assert.False(t, *extraRec.DeleteOnTermination)
v, ok := d.vmMgr.Get(out.InstanceID)
require.True(t, ok)
assert.Equal(t, tags.ManagedByELBv2, v.ManagedBy)
assert.Equal(t, awsidentifiers.GlobalAccountID, v.AccountID)
assert.NotEmpty(t, aws.StringValue(v.Instance.VpcId), "the VPC is resolved from the pre-created ENI")
require.Len(t, v.ExtraENIs, 1)
assert.Equal(t, extraID, v.ExtraENIs[0].ENIID)
assert.Equal(t, []int{6443}, v.HostfwdPorts)
assert.True(t, v.DirectBoot)
d.mu.Lock()
_, subscribed := d.natsSubscriptions[out.InstanceID]
d.mu.Unlock()
assert.True(t, subscribed, "the instance must listen for terminate commands")
})
t.Run("failed extra attach fails the launch", func(t *testing.T) {
in := f.input(f.newENI(t))
in.ExtraENIs = []systeminstance.ExtraENIInput{{ENIID: "eni-missing"}}
_, err := d.LaunchSystemInstance(in)
require.Error(t, err)
assert.Contains(t, err.Error(), "attach extra ENI eni-missing")
})
t.Run("same-account ENI skips the owner stamp", func(t *testing.T) {
out, err := d.vpcService.CreateNetworkInterface(t.Context(), &ec2.CreateNetworkInterfaceInput{
SubnetId: aws.String(f.sysSubnetID),
}, awsidentifiers.GlobalAccountID)
require.NoError(t, err)
eni := out.NetworkInterface
in := f.input(eni)
in.AccountID = ""
sysInstLaunchFails(d)
_, err = d.LaunchSystemInstance(in)
require.ErrorIs(t, err, errSysInstLaunchRefused)
rec := f.eniRecord(t, awsidentifiers.GlobalAccountID, aws.StringValue(eni.NetworkInterfaceId))
assert.Empty(t, rec.InstanceOwnerId, "same-account attachments leave the owner empty")
})
}
func TestLaunchSystemInstance_AutoCreateENI(t *testing.T) {
f := newSysInstFixture(t)
d := f.d
sysInstLaunchRuns(d)
t.Run("creates and attaches an ENI in the system account", func(t *testing.T) {
out, err := d.LaunchSystemInstance(&handlers_elbv2.SystemInstanceInput{
InstanceType: f.itype,
SubnetID: f.sysSubnetID,
NICs: []handlers_elbv2.NICConfig{{MAC: "02:00:00:00:00:01", IsDefault: true}},
})
require.NoError(t, err)
assert.True(t, strings.HasPrefix(out.PrivateIP, "10.82.1."), "private IP %q must come from the subnet", out.PrivateIP)
v, ok := d.vmMgr.Get(out.InstanceID)
require.True(t, ok)
require.NotEmpty(t, v.ENIId)
rec := f.eniRecord(t, awsidentifiers.GlobalAccountID, v.ENIId)
assert.Equal(t, out.InstanceID, rec.InstanceId)
assert.Equal(t, f.sysSubnetID, aws.StringValue(v.Instance.SubnetId))
})
t.Run("unknown subnet", func(t *testing.T) {
_, err := d.LaunchSystemInstance(&handlers_elbv2.SystemInstanceInput{
InstanceType: f.itype,
SubnetID: "subnet-madeup",
})
require.Error(t, err)
assert.Contains(t, err.Error(), "create ENI")
})
}
func TestLaunchSystemInstance_EIPService(t *testing.T) {
f := newSysInstFixture(t)
d := f.d
internetFacing := func(t *testing.T) *handlers_elbv2.SystemInstanceInput {
in := f.input(f.newENI(t))
in.Scheme = handlers_elbv2.SchemeInternetFacing
return in
}
allocated := &ec2.AllocateAddressOutput{
PublicIp: aws.String("203.0.113.50"),
PublicIpv4Pool: aws.String("wan"),
AllocationId: aws.String("eipalloc-1"),
}
t.Run("allocate failure", func(t *testing.T) {
d.eipService = &fakeEIPService{allocateErr: errInjected}
_, err := d.LaunchSystemInstance(internetFacing(t))
require.ErrorIs(t, err, errInjected)
assert.Contains(t, err.Error(), "allocate public IP for internet-facing ALB")
})
for _, releaseErr := range []error{nil, errors.New("release failed")} {
t.Run(fmt.Sprintf("associate failure releases the address (release error %v)", releaseErr), func(t *testing.T) {
eip := &fakeEIPService{allocateOut: allocated, associateErr: errInjected, releaseErr: releaseErr}
d.eipService = eip
_, err := d.LaunchSystemInstance(internetFacing(t))
require.ErrorIs(t, err, errInjected)
assert.Contains(t, err.Error(), "associate public IP")
assert.Equal(t, []string{"eipalloc-1"}, eip.released)
})
}
t.Run("success records the address on the ENI and the VM", func(t *testing.T) {
sysInstLaunchRuns(d)
d.eipService = &fakeEIPService{
allocateOut: allocated,
associateOut: &ec2.AssociateAddressOutput{AssociationId: aws.String("eipassoc-1")},
}
in := internetFacing(t)
out, err := d.LaunchSystemInstance(in)
require.NoError(t, err)
assert.Equal(t, "203.0.113.50", out.PublicIP)
v, ok := d.vmMgr.Get(out.InstanceID)
require.True(t, ok)
assert.Equal(t, "203.0.113.50", v.PublicIP)
assert.Equal(t, "wan", v.PublicIPPool)
assert.Equal(t, "eipalloc-1", v.PublicIPAllocID)
assert.Equal(t, "eipassoc-1", v.PublicIPAssocID)
assert.Equal(t, "203.0.113.50", f.eniRecord(t, testAccountID, in.ENIID).PublicIpAddress)
})
t.Run("launch failure releases the associated address", func(t *testing.T) {
sysInstLaunchFails(d)
eip := &fakeEIPService{
allocateOut: allocated,
associateOut: &ec2.AssociateAddressOutput{AssociationId: aws.String("eipassoc-2")},
}
d.eipService = eip
_, err := d.LaunchSystemInstance(internetFacing(t))
require.ErrorIs(t, err, errSysInstLaunchRefused)
assert.Equal(t, []string{"eipassoc-2"}, eip.disassociated)
assert.Equal(t, []string{"eipalloc-1"}, eip.released)
})
}
func (f *sysInstFixture) externalIPAM(t *testing.T, pool external.ExternalPoolConfig) *ec2vpc.ExternalIPAM {
t.Helper()
ipam, err := ec2vpc.NewExternalIPAM(t.Context(), f.js, []external.ExternalPoolConfig{pool})
require.NoError(t, err)
return ipam
}
func TestLaunchSystemInstance_IPAM(t *testing.T) {
f := newSysInstFixture(t)
d := f.d
sysInstLaunchRuns(d)
t.Run("NAT committed", func(t *testing.T) {
d.externalIPAM = f.externalIPAM(t, external.ExternalPoolConfig{
Name: "wan-ok", RangeStart: "198.51.100.10", RangeEnd: "198.51.100.20", Gateway: "198.51.100.1", PrefixLen: 24,
})
sub, err := d.natsConn.Subscribe("vpc.add-nat", func(msg *nats.Msg) {
_ = msg.Respond([]byte(`{"success":true}`))
})
require.NoError(t, err)
t.Cleanup(func() { _ = sub.Unsubscribe() })
in := f.input(f.newENI(t))
in.Scheme = handlers_elbv2.SchemeInternetFacing
out, err := d.LaunchSystemInstance(in)
require.NoError(t, err)
require.True(t, strings.HasPrefix(out.PublicIP, "198.51.100."), "public IP %q must come from the pool", out.PublicIP)
v, ok := d.vmMgr.Get(out.InstanceID)
require.True(t, ok)
assert.Equal(t, "wan-ok", v.PublicIPPool)
assert.Empty(t, v.PublicIPAllocID, "a direct IPAM address has no EIP allocation")
assert.Equal(t, out.PublicIP, f.eniRecord(t, testAccountID, in.ENIID).PublicIpAddress)
})
t.Run("pool exhausted", func(t *testing.T) {
// A one-address range holds only the reserved first address.
d.externalIPAM = f.externalIPAM(t, external.ExternalPoolConfig{
Name: "wan-full", RangeStart: "198.51.101.10", RangeEnd: "198.51.101.10", Gateway: "198.51.101.1", PrefixLen: 24,
})
in := f.input(f.newENI(t))
in.Scheme = handlers_elbv2.SchemeInternetFacing
_, err := d.LaunchSystemInstance(in)
require.Error(t, err)
assert.Contains(t, err.Error(), "allocate public IP for internet-facing ALB")
assert.Empty(t, f.eniRecord(t, testAccountID, in.ENIID).PublicIpAddress)
})
}
func TestLaunchSystemInstance_MgmtNIC(t *testing.T) {
f := newSysInstFixture(t)
d := f.d
_, primed := captureNeighHooks(t)
d.mgmtBridgeIP = "10.93.8.1"
tapErr := errors.New("tap failed")
cases := []struct {
name string
scheme string
alloc func(t *testing.T) *MgmtIPAllocator
tapErr error
wantErr string
wantMgmt bool
wantAlloc int
}{
{name: "internal allocate error", scheme: handlers_elbv2.SchemeInternal, alloc: sysInstUnboundMgmtAllocator,
wantErr: "allocate mgmt IP for internal-scheme ALB"},
{name: "internet-facing allocate error continues", scheme: "", alloc: sysInstUnboundMgmtAllocator},
{name: "internal tap error", scheme: handlers_elbv2.SchemeInternal, tapErr: tapErr,
wantErr: "setup mgmt tap for internal-scheme ALB"},
{name: "internet-facing tap error continues", scheme: "", tapErr: tapErr},
{name: "success", scheme: handlers_elbv2.SchemeInternal, wantMgmt: true, wantAlloc: 1},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
sysInstLaunchRuns(d)
var alloc *MgmtIPAllocator
if tc.alloc != nil {
alloc = tc.alloc(t)
} else {
alloc = sysInstMgmtAllocator(t, "10.93.8.1")
}
d.mgmtIPAllocator = alloc
d.networkPlumber = &erroringPlumber{setupErr: tc.tapErr}
*primed = nil
in := f.input(f.newENI(t))
in.Scheme = tc.scheme
in.NICs = append(in.NICs, handlers_elbv2.NICConfig{})
out, err := d.LaunchSystemInstance(in)
if tc.wantErr != "" {
require.Error(t, err)
assert.Contains(t, err.Error(), tc.wantErr)
assert.Equal(t, 0, alloc.AllocatedCount(), "a failed mgmt NIC must return its address")
return
}
require.NoError(t, err)
v, ok := d.vmMgr.Get(out.InstanceID)
require.True(t, ok)
assert.Equal(t, tc.wantAlloc, alloc.AllocatedCount())
if !tc.wantMgmt {
assert.Empty(t, v.MgmtIP)
assert.Empty(t, v.MgmtMAC)
return
}
assert.True(t, strings.HasPrefix(v.MgmtIP, "10.93.8."))
assert.Equal(t, vm.GenerateMgmtMAC(out.InstanceID), v.MgmtMAC)
assert.Equal(t, v.MgmtMAC, in.NICs[1].MAC, "the allocated MAC is injected into the guest NIC config")
assert.Equal(t, v.MgmtIP+"/24", in.NICs[1].CIDR)
require.Len(t, *primed, 1)
assert.Equal(t, neighCall{dev: defaultMgmtBridge, ip: v.MgmtIP, mac: v.MgmtMAC}, (*primed)[0])
})
}
}
func TestLaunchSystemInstance_ConfigAndRun(t *testing.T) {
f := newSysInstFixture(t)
d := f.d
t.Run("no default NIC fails the direct-boot config", func(t *testing.T) {
in := f.input(f.newENI(t))
in.NICs = nil
_, err := d.LaunchSystemInstance(in)
require.Error(t, err)
assert.Contains(t, err.Error(), "build direct-boot config")
})
t.Run("dev networking derives a dev MAC", func(t *testing.T) {
sysInstLaunchRuns(d)
d.config.Daemon.DevNetworking = true
t.Cleanup(func() { d.config.Daemon.DevNetworking = false })
out, err := d.LaunchSystemInstance(f.input(f.newENI(t)))
require.NoError(t, err)
v, ok := d.vmMgr.Get(out.InstanceID)
require.True(t, ok)
assert.Equal(t, vm.GenerateDevMAC(out.InstanceID), v.DevMAC)
assert.Equal(t, microvmMachineType(), v.Config.MachineType)
})
}
// sysInstAMILoader serves one AMI whose snapshot the memory provider holds.
type sysInstAMILoader struct {
ami ebsmetadata.AMI
sourceVol string
}
func (l *sysInstAMILoader) GetAMIConfig(_ context.Context, imageID string) (ebsmetadata.AMI, error) {
if imageID != l.ami.ImageID {
return ebsmetadata.AMI{}, fmt.Errorf("ami %s not found", imageID)
}
return l.ami, nil
}
func (l *sysInstAMILoader) GetAMISourceVolumeID(context.Context, string) (string, error) {
return l.sourceVol, nil
}
func TestLaunchAMISystemInstance_Guards(t *testing.T) {
d := createTestDaemon(t, sharedNATSURL)
base := systeminstance.SystemInstanceInput{
BootMode: systeminstance.BootAMI, ImageID: "ami-x", AccountID: testAccountID, ENIID: "eni-x", InstanceType: getTestInstanceType(t),
}
cases := []struct {
name string
mutate func(*systeminstance.SystemInstanceInput)
wantErr string
}{
{"missing image", func(in *systeminstance.SystemInstanceInput) { in.ImageID = "" }, "requires ImageID"},
{"missing account", func(in *systeminstance.SystemInstanceInput) { in.AccountID = "" }, "requires AccountID"},
{"missing ENI", func(in *systeminstance.SystemInstanceInput) { in.ENIID = "" }, "requires a pre-created ENI"},
// The test daemon wires no AMI loader, so Prepare refuses the launch.
{"prepare error", func(in *systeminstance.SystemInstanceInput) {
in.UserData = "#cloud-config"
in.IamInstanceProfileArn = "arn:aws:iam::123456789012:instance-profile/cp"
in.ManagedBy = tags.ManagedByEKS
}, "ServerInternal"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
in := base
tc.mutate(&in)
_, err := d.LaunchSystemInstance(&in)
require.Error(t, err)
assert.Contains(t, err.Error(), tc.wantErr)
})
}
}
func TestLaunchAMISystemInstance(t *testing.T) {
f := newSysInstFixture(t)
d := f.d
_, _ = captureNeighHooks(t)
provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
_, err := provider.CreateVolume(t.Context(), ebsprovider.CreateVolumeRequest{
Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-src",
CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30},
})
require.NoError(t, err)
_, err = provider.CreateSnapshot(t.Context(), ebsprovider.CreateSnapshotRequest{
Versioned: ebsprovider.NewVersioned(), SnapshotID: "snap-src", VolumeID: "vol-src",
})
require.NoError(t, err)
loader := &sysInstAMILoader{ami: ebsmetadata.AMI{ImageID: "ami-sys", State: "available", SnapshotID: "snap-src"}, sourceVol: "vol-src"}
d.instanceService.SetRunInstancesDeps(loader, nil, &daemonENICreator{d: d}, nil)
d.instanceService.SetEBSProvider(provider)
sysInstLaunchFails(d)
input := func(t *testing.T) *systeminstance.SystemInstanceInput {
eni := f.newENI(t)
return &systeminstance.SystemInstanceInput{
BootMode: systeminstance.BootAMI,
InstanceType: f.itype,
ImageID: "ami-sys",
AccountID: testAccountID,
ManagedBy: tags.ManagedByEKS,
ENIID: aws.StringValue(eni.NetworkInterfaceId),
}
}
t.Run("no management bridge", func(t *testing.T) {
d.mgmtIPAllocator = nil
_, err := d.LaunchSystemInstance(input(t))
require.Error(t, err)
assert.Contains(t, err.Error(), "management bridge unavailable")
})
d.mgmtBridgeIP = "10.95.8.1"
d.networkPlumber = &recordingPlumber{}
alloc := sysInstMgmtAllocator(t, "10.95.8.1")
d.mgmtIPAllocator = alloc
t.Run("extra ENI attach failure", func(t *testing.T) {
in := input(t)
in.ExtraENIs = []systeminstance.ExtraENIInput{{ENIID: "eni-missing"}}
_, err := d.LaunchSystemInstance(in)
require.Error(t, err)
assert.Contains(t, err.Error(), "attach extra ENI eni-missing")
})
t.Run("launch that never reaches running", func(t *testing.T) {
_, err := d.LaunchSystemInstance(input(t))
require.Error(t, err)
assert.Contains(t, err.Error(), "did not reach running state")
})
t.Run("success", func(t *testing.T) {
sysInstLaunchRuns(d)
in := input(t)
extra := f.newENI(t)
in.ExtraENIs = []systeminstance.ExtraENIInput{{ENIID: aws.StringValue(extra.NetworkInterfaceId), SubnetID: f.subnetID}}
out, err := d.LaunchSystemInstance(in)
require.NoError(t, err)
assert.True(t, strings.HasPrefix(out.MgmtIP, "10.95.8."))
assert.True(t, strings.HasPrefix(out.PrivateIP, "10.81.1."), "private IP %q comes from the pre-created ENI", out.PrivateIP)
v, ok := d.vmMgr.Get(out.InstanceID)
require.True(t, ok)
assert.Equal(t, tags.ManagedByEKS, v.ManagedBy)
assert.Equal(t, testAccountID, v.AccountID)
require.Len(t, v.ExtraENIs, 1)
assert.Equal(t, out.InstanceID, f.eniRecord(t, testAccountID, v.ExtraENIs[0].ENIID).InstanceId)
})
}
func TestAttachSystemMgmtNIC(t *testing.T) {
_, primed := captureNeighHooks(t)
t.Run("no allocator", func(t *testing.T) {
err := (&Daemon{mgmtBridgeIP: "10.96.8.1"}).attachSystemMgmtNIC(&vm.VM{ID: "i-a"})
require.Error(t, err)
assert.Contains(t, err.Error(), "management bridge unavailable")
})
t.Run("allocate error", func(t *testing.T) {
d := &Daemon{mgmtBridgeIP: "10.94.8.1", mgmtIPAllocator: sysInstUnboundMgmtAllocator(t)}
err := d.attachSystemMgmtNIC(&vm.VM{ID: "i-b"})
require.Error(t, err)
assert.Contains(t, err.Error(), "allocate mgmt IP")
})
alloc := sysInstMgmtAllocator(t, "10.96.8.1")
t.Run("tap error releases the address", func(t *testing.T) {
d := &Daemon{mgmtBridgeIP: "10.96.8.1", mgmtIPAllocator: alloc, networkPlumber: &erroringPlumber{setupErr: errInjected}}
inst := &vm.VM{ID: "i-c"}
err := d.attachSystemMgmtNIC(inst)
require.ErrorIs(t, err, errInjected)
assert.Empty(t, inst.MgmtIP)
assert.Empty(t, inst.MgmtMAC)
assert.Equal(t, 0, alloc.AllocatedCount())
})
t.Run("success", func(t *testing.T) {
d := &Daemon{mgmtBridgeIP: "10.96.8.1", mgmtIPAllocator: alloc, networkPlumber: &recordingPlumber{}}
inst := &vm.VM{ID: "i-d"}
require.NoError(t, d.attachSystemMgmtNIC(inst))
assert.Equal(t, "10.96.8.10", inst.MgmtIP)
assert.Equal(t, vm.GenerateMgmtMAC("i-d"), inst.MgmtMAC)
require.Len(t, *primed, 1)
assert.Equal(t, "10.96.8.10", (*primed)[0].ip)
})
}
func TestTerminateSystemInstance(t *testing.T) {
d := createTestDaemon(t, sharedNATSURL)
t.Cleanup(d.vmMgr.WaitForBackgroundWork)
t.Run("local terminate releases the EIP", func(t *testing.T) {
eip := &sysInstReclaimingEIP{fakeEIPService: fakeEIPService{disassociateErr: errInjected}}
d.eipService = eip
inst := &vm.VM{ID: "i-term-ok", Status: vm.StateRunning, AccountID: testAccountID,
PublicIP: "203.0.113.7", PublicIPAllocID: "eipalloc-t", PublicIPAssocID: "eipassoc-t"}
d.vmMgr.Insert(inst)
require.NoError(t, d.TerminateSystemInstance("i-term-ok"))
assert.Equal(t, []string{"eipassoc-t"}, eip.disassociated)
assert.Equal(t, []string{"eipalloc-t"}, eip.released, "release still runs after a failed disassociate")
// Once by the owning-node path and once by the backstop.
assert.Equal(t, []string{"i-term-ok", "i-term-ok"}, eip.reclaimed)
assert.Equal(t, vm.StateTerminated, d.vmMgr.Status(inst))
})
t.Run("local terminate failure", func(t *testing.T) {
d.eipService = nil
d.vmMgr.SetDeps(vm.Deps{NodeID: d.node, TransitionState: func(*vm.VM, vm.InstanceState) error { return errInjected }})
d.vmMgr.Insert(&vm.VM{ID: "i-term-fail", Status: vm.StateRunning})
err := d.TerminateSystemInstance("i-term-fail")
require.Error(t, err)
assert.Contains(t, err.Error(), "terminate:")
})
t.Run("no owner and failing reclaim", func(t *testing.T) {
eip := &sysInstReclaimingEIP{reclaimErr: errInjected}
d.eipService = eip
err := d.TerminateSystemInstance("i-term-nobody")
require.ErrorIs(t, err, systeminstance.ErrSystemInstanceNotFound)
assert.Equal(t, []string{"i-term-nobody"}, eip.reclaimed, "the backstop runs even when no node owns the VM")
})
t.Run("local lookup miss", func(t *testing.T) {
err := d.terminateSystemInstanceLocal("i-term-gone")
require.ErrorIs(t, err, systeminstance.ErrSystemInstanceNotFound)
})
}
func TestReleaseSystemInstanceEIP_Errors(t *testing.T) {
d := newDaemonWithVMs()
eip := &fakeEIPService{disassociateErr: errInjected, releaseErr: errInjected}
d.eipService = eip
inst := &vm.VM{ID: "i-rel", PublicIP: "203.0.113.9", PublicIPPool: "wan", PublicIPAllocID: "eipalloc-r", PublicIPAssocID: "eipassoc-r"}
d.vmMgr.Insert(inst)
d.releaseSystemInstanceEIP(inst)
assert.Equal(t, []string{"eipassoc-r"}, eip.disassociated)
assert.Equal(t, []string{"eipalloc-r"}, eip.released)
// Fields clear even on failure so teardown does not release the IP again.
assert.Empty(t, inst.PublicIP)
assert.Empty(t, inst.PublicIPPool)
assert.Empty(t, inst.PublicIPAllocID)
assert.Empty(t, inst.PublicIPAssocID)
}
func TestBuildDirectBootConfig_InstanceTypeShape(t *testing.T) {
t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
rm, err := NewResourceManager(nil, nil, nil)
require.NoError(t, err)
itype := getTestInstanceType(t)
it := rm.instanceTypes[itype]
d := &Daemon{resourceMgr: rm}
cfg, err := d.buildDirectBootConfig("i-shape", &handlers_elbv2.SystemInstanceInput{
InstanceType: itype,
ENIID: "eni-shape",
NICs: []handlers_elbv2.NICConfig{{MAC: "02:00:00:00:00:02", IsDefault: true}},
})
require.NoError(t, err)
assert.Equal(t, int(instanceTypeVCPUs(it)), cfg.CPUCount)
assert.Equal(t, int(instanceTypeMemoryMiB(it)), cfg.Memory)
assert.Equal(t, aws.StringValue(it.ProcessorInfo.SupportedArchitectures[0]), cfg.Architecture)
assert.Len(t, cfg.FwCfg, 3)
}
func TestBuildDirectBootConfig_FwCfgWriteError(t *testing.T) {
notADir := filepath.Join(t.TempDir(), "file")
require.NoError(t, os.WriteFile(notADir, nil, 0o600))
t.Setenv("XDG_RUNTIME_DIR", notADir)
_, err := (&Daemon{resourceMgr: &ResourceManager{}}).buildDirectBootConfig("i-werr", &handlers_elbv2.SystemInstanceInput{
NICs: []handlers_elbv2.NICConfig{{MAC: "02:00:00:00:00:03", IsDefault: true}},
})
require.Error(t, err)
assert.Contains(t, err.Error(), "write fw_cfg blobs")
}
// A directory at a blob's path fails that write; the blobs written before it
// must be removed so no partial fw_cfg set is left behind.
func TestWriteFwCfgBlobs_LaterWriteErrorsCleanUp(t *testing.T) {
input := &handlers_elbv2.SystemInstanceInput{
NICs: []handlers_elbv2.NICConfig{{MAC: "02:00:00:00:00:04", IsDefault: true}},
}
for _, blob := range []string{"lbenv", "cacert"} {
t.Run(blob, func(t *testing.T) {
dir := t.TempDir()
t.Setenv("XDG_RUNTIME_DIR", dir)
require.NoError(t, os.Mkdir(filepath.Join(dir, "fwcfg-i-blob-"+blob+".tmp"), 0o700))
_, err := (&Daemon{}).writeFwCfgBlobs("i-blob", input)
require.Error(t, err)
_, statErr := os.Stat(filepath.Join(dir, "fwcfg-i-blob-netcfg.tmp"))
assert.ErrorIs(t, statErr, os.ErrNotExist, "the netcfg blob must be removed")
_, statErr = os.Stat(filepath.Join(dir, "fwcfg-i-blob-lbenv.tmp"))
if blob == "cacert" {
assert.ErrorIs(t, statErr, os.ErrNotExist, "the lb-agent-env blob must be removed")
}
})
}
}
spinifex/daemon/dns_reconcile_services_test.go
+53
View File
@@ -0,0 +1,53 @@
package daemon
import (
"bytes"
"testing"
"github.com/mulgadc/bluebottle/pkg/masterkey"
"github.com/mulgadc/spinifex/internal/testkit"
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2"
handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Without a base domain no service can name its records, so none of them may
// claim prune authority over the zone.
func TestDNSDesiredSet_ServicesWithoutBaseDomain(t *testing.T) {
d := &Daemon{
elbv2Service: &handlers_elbv2.ELBv2ServiceImpl{},
eksService: &handlers_eks.EKSServiceImpl{},
rdsService: &handlers_rds.Service{},
}
ds := d.dnsDesiredSet()
assert.False(t, ds.Prunable.ELB)
assert.False(t, ds.Prunable.EKS)
assert.False(t, ds.Prunable.RDS)
assert.Empty(t, ds.Changes)
}
func TestDNSDesiredSet_ServicesEnumerateCompletely(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
cfg := &config.Config{Region: "ap-southeast-2"}
cfg.Northstar.DefaultDomain = "example.internal"
elb, err := handlers_elbv2.NewELBv2ServiceImplWithNATS(cfg, nc, bytes.Repeat([]byte{0x42}, masterkey.MasterKeySize))
require.NoError(t, err)
t.Cleanup(elb.Close)
eks, err := handlers_eks.NewEKSServiceImpl(handlers_eks.EKSServiceDeps{NATSConn: nc, Config: cfg})
require.NoError(t, err)
t.Cleanup(eks.Shutdown)
rds := handlers_rds.NewService(nc, cfg.Region).WithDeps(handlers_rds.Deps{BaseDomain: "example.internal"})
d := &Daemon{elbv2Service: elb, eksService: eks, rdsService: rds}
ds := d.dnsDesiredSet()
assert.True(t, ds.Prunable.ELB, "an empty load balancer store is a complete view")
assert.True(t, ds.Prunable.EKS)
assert.True(t, ds.Prunable.RDS)
assert.False(t, ds.Prunable.EC2, "no instance store was configured")
assert.Empty(t, ds.Changes)
}
spinifex/daemon/eks_deps_test.go
+83
View File
@@ -1,10 +1,18 @@
package daemon
import (
"bytes"
"os"
"path/filepath"
"testing"
"github.com/mulgadc/bluebottle/pkg/masterkey"
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
ec2volume "github.com/mulgadc/spinifex/spinifex/domains/ec2/volume"
"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
"github.com/nats-io/nats.go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestResolveGatewayHost covers all five host-selection branches plus the
@@ -155,3 +163,78 @@ func TestBuildEKSServiceDeps(t *testing.T) {
assert.Equal(t, "https://10.15.8.1:8443", deps.SystemPredastoreURL)
assert.NotNil(t, deps.SnapshotStore)
}
// writeMasterKey drops a valid shared master key beside configPath.
func writeMasterKey(t *testing.T, configPath string) {
t.Helper()
key := bytes.Repeat([]byte{0x42}, masterkey.MasterKeySize)
require.NoError(t, os.WriteFile(filepath.Join(filepath.Dir(configPath), "master.key"), key, 0o640))
}
func TestSystemRoleEnsurer(t *testing.T) {
t.Run("no master key falls back to static creds", func(t *testing.T) {
d := &Daemon{ctx: t.Context(), configPath: filepath.Join(t.TempDir(), "spinifex.toml")}
assert.Nil(t, d.systemRoleEnsurer())
})
t.Run("IAM init failure is retried rather than cached", func(t *testing.T) {
nc, err := nats.Connect(sharedNATSURL)
require.NoError(t, err)
defer nc.Close()
d := &Daemon{ctx: t.Context(), natsConn: nc, configPath: filepath.Join(t.TempDir(), "spinifex.toml")}
writeMasterKey(t, d.configPath)
assert.Nil(t, d.systemRoleEnsurer(), "a NATS server without JetStream cannot back IAM")
assert.Nil(t, d.iamEnsurerCached)
})
t.Run("success is built once and cached", func(t *testing.T) {
clustersize.DeclareForTest(t, 1)
nc, err := nats.Connect(sharedJSNATSURL)
require.NoError(t, err)
defer nc.Close()
d := &Daemon{ctx: t.Context(), natsConn: nc, configPath: filepath.Join(t.TempDir(), "spinifex.toml")}
writeMasterKey(t, d.configPath)
first := d.systemRoleEnsurer()
require.NotNil(t, first)
// The key going away after a successful build must not matter.
require.NoError(t, os.Remove(filepath.Join(filepath.Dir(d.configPath), "master.key")))
assert.Same(t, first, d.systemRoleEnsurer())
})
}
func TestBuildEKSServiceDeps_ConfigDerivedFields(t *testing.T) {
caPath := filepath.Join(t.TempDir(), "ca.pem")
require.NoError(t, os.WriteFile(caPath, []byte("-----BEGIN CERTIFICATE-----"), 0o600))
base := func() *Daemon {
return &Daemon{
mgmtBridgeIP: "10.15.8.1",
clusterConfig: &config.ClusterConfig{AWS: config.AWSConfig{ServicesDomain: "svc.example"}},
volumeService: &ec2volume.VolumeServiceImpl{},
config: &config.Config{
AdvertiseIP: "192.0.2.10",
AWSGW: config.AWSGWConfig{Host: "0.0.0.0:8443"},
NATS: config.NATSConfig{CACert: caPath},
},
}
}
t.Run("readable CA and gateway port", func(t *testing.T) {
deps := base().buildEKSServiceDeps()
assert.Equal(t, "svc.example", deps.InternalSuffix)
assert.Equal(t, "-----BEGIN CERTIFICATE-----", deps.GatewayCACert)
assert.Equal(t, "https://192.0.2.10:8443", deps.GatewayBaseURL)
assert.Equal(t, "https://10.15.8.1:8443", deps.SystemGatewayURL)
assert.NotNil(t, deps.Volume, "a configured volume service must reach CSI reclaim")
})
t.Run("unreadable CA leaves the gateway unverified", func(t *testing.T) {
d := base()
d.config.NATS.CACert = filepath.Join(t.TempDir(), "missing.pem")
deps := d.buildEKSServiceDeps()
assert.Empty(t, deps.GatewayCACert)
})
}
spinifex/daemon/gpu_handlers_test.go
+14
View File
@@ -163,3 +163,17 @@ func TestResolveVMGPU_UnknownPCI_ReturnsNil(t *testing.T) {
func TestResolveVMGPU_EmptyAttachment_ReturnsNil(t *testing.T) {
assert.Nil(t, resolveVMGPU(gpu.GPUAttachment{}, nil, nil))
}
// A partly resolved set is reported short rather than padded or dropped.
func TestResolveVMGPUs_SkipsUnresolvedAttachments(t *testing.T) {
dev := gpu.GPUDevice{PCIAddress: "0000:01:00.0", Model: "NVIDIA A10", MemoryMiB: 23028}
byPCI := map[string]gpu.PoolEntry{"0000:01:00.0": {Device: dev, InstanceID: "i-abc"}}
got := resolveVMGPUs("i-abc", []gpu.GPUAttachment{
{PCIAddress: "0000:01:00.0"},
{PCIAddress: "0000:99:00.0"},
}, nil, byPCI)
require.Len(t, got, 1)
assert.Equal(t, "NVIDIA A10", got[0].Model)
}
spinifex/daemon/heartbeat_test.go
+49
View File
@@ -1,13 +1,16 @@
package daemon
import (
"context"
"encoding/json"
"testing"
"time"
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/cache"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -159,3 +162,49 @@ func TestHeartbeatIntervalMatchesLivenessCopy(t *testing.T) {
heartbeatInterval, instancecache.HeartbeatInterval)
}
}
func heartbeatDaemon(t *testing.T, node string) *Daemon {
t.Helper()
rm, err := NewResourceManager(nil, nil, nil)
require.NoError(t, err)
return &Daemon{
node: node,
clusterConfig: &config.ClusterConfig{Epoch: 7},
config: &config.Config{Services: []string{"daemon"}},
resourceMgr: rm,
vmMgr: vm.NewManager(),
}
}
func TestStartHeartbeat_PublishesImmediately(t *testing.T) {
jsm, _ := newFaultInjectedJSM(t)
d := heartbeatDaemon(t, "hb-start-node")
d.jsManager = jsm
ctx, cancel := context.WithCancel(t.Context())
d.ctx = ctx
defer cancel()
d.startHeartbeat()
var got *Heartbeat
require.Eventually(t, func() bool {
h, err := jsm.ReadHeartbeat("hb-start-node")
got = h
return err == nil
}, 2*time.Second, time.Millisecond, "the first heartbeat must not wait for the ticker")
assert.Equal(t, uint64(7), got.Epoch)
assert.Equal(t, []string{"daemon"}, got.Services)
}
func TestPublishHeartbeat_WriteFailure(t *testing.T) {
jsm, f := newFaultInjectedJSM(t)
d := heartbeatDaemon(t, "hb-fail-node")
d.jsManager = jsm
f.setFail("Put", true)
d.publishHeartbeat()
f.setFail("Put", false)
_, err := jsm.ReadHeartbeat("hb-fail-node")
assert.ErrorIs(t, err, jetstream.ErrKeyNotFound)
}
spinifex/daemon/instance_membership.go
+7
View File
@@ -44,6 +44,10 @@ func runsOn(record *vm.InstanceRecord, nodeID string) bool {
// drained for maintenance leaves them observed-stopped but still wanted — both
// are recoveries. What is excluded is an instance nobody wants running: an
// operator stop, or a terminate already under way.
//
// A fenced instance is excluded for the same reason the restore path excludes
// it. The fence does not seal, so no node can show the volumes are mountable,
// and taking it over is how an unproven state becomes a destroyed one.
func recoverable(record *vm.InstanceRecord) bool {
if record == nil || record.Spec.DesiredState != vm.DesiredRunning {
return false
@@ -55,6 +59,9 @@ func recoverable(record *vm.InstanceRecord) bool {
case vm.StateTerminated, vm.StateShuttingDown:
return false
}
if vm.VolumeFenced(record.Status.Instance) {
return false
}
return !storageFaulted(record)
}
spinifex/daemon/instance_records_migrate_internal_test.go
+235
View File
@@ -0,0 +1,235 @@
package daemon
import (
"context"
"log/slog"
"testing"
"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
"github.com/nats-io/nats.go/jetstream"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// migrateFaultKV fails chosen calls on a real bucket: the key listing, reads
// and creates of named keys, and every delete.
type migrateFaultKV struct {
jetstream.KeyValue
keysErr error
getErr map[string]error
createErr map[string]error
deleteErr error
}
func (k *migrateFaultKV) Keys(ctx context.Context, opts ...jetstream.WatchOpt) ([]string, error) {
if k.keysErr != nil {
return nil, k.keysErr
}
return k.KeyValue.Keys(ctx, opts...)
}
func (k *migrateFaultKV) Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) {
if err := k.getErr[key]; err != nil {
return nil, err
}
return k.KeyValue.Get(ctx, key)
}
func (k *migrateFaultKV) Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
if err := k.createErr[key]; err != nil {
return 0, err
}
return k.KeyValue.Create(ctx, key, value, opts...)
}
func (k *migrateFaultKV) Delete(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error {
if k.deleteErr != nil {
return k.deleteErr
}
return k.KeyValue.Delete(ctx, key, opts...)
}
type migrateCase struct {
name string
seed map[string][]byte
fault migrateFaultKV
wantErr string
check func(t *testing.T, kv jetstream.KeyValue)
}
// runMigrateCases runs each case against its own freshly seeded bucket.
func runMigrateCases(t *testing.T, run func(context.Context, migrate.KVContext) error, cases []migrateCase) {
t.Helper()
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
_, f := faultBucket(t)
for k, v := range tc.seed {
_, err := f.KeyValue.Put(t.Context(), k, v)
require.NoError(t, err)
}
kv := tc.fault
kv.KeyValue = f.KeyValue
err := run(t.Context(), migrate.KVContext{KV: &kv, Logger: slog.Default()})
if tc.wantErr == "" {
require.NoError(t, err)
} else {
require.ErrorContains(t, err, tc.wantErr)
}
if tc.check != nil {
tc.check(t, f.KeyValue)
}
})
}
}
func assertKeyAbsent(t *testing.T, kv jetstream.KeyValue, key string) {
t.Helper()
_, err := kv.Get(context.Background(), key)
assert.ErrorIs(t, err, jetstream.ErrKeyNotFound, "%s must not be written", key)
}
func notFound(key string) map[string]error {
return map[string]error{key: jetstream.ErrKeyNotFound}
}
func injected(key string) map[string]error {
return map[string]error{key: errInjected}
}
func TestRekeyRecordSeparator_Faults(t *testing.T) {
t.Parallel()
slashed := map[string][]byte{"i/i-1": []byte(`{}`)}
runMigrateCases(t, rekeyRecordSeparator, []migrateCase{
{name: "empty bucket"},
{name: "listing fails", seed: slashed, fault: migrateFaultKV{keysErr: errInjected}, wantErr: "list keys"},
{
name: "record vanished mid-pass", seed: slashed, fault: migrateFaultKV{getErr: notFound("i/i-1")},
check: func(t *testing.T, kv jetstream.KeyValue) { assertKeyAbsent(t, kv, "i.i-1") },
},
{name: "read fails", seed: slashed, fault: migrateFaultKV{getErr: injected("i/i-1")}, wantErr: "read i/i-1"},
{name: "write fails", seed: slashed, fault: migrateFaultKV{createErr: injected("i.i-1")}, wantErr: "write i.i-1"},
{
name: "delete fails after the copy", seed: slashed, fault: migrateFaultKV{deleteErr: errInjected},
wantErr: "delete i/i-1",
check: func(t *testing.T, kv jetstream.KeyValue) {
_, err := kv.Get(context.Background(), "i.i-1")
assert.NoError(t, err, "the copy lands before the delete, so a rerun finds it")
},
},
})
}
func TestCarryNodeOwnershipForward_Faults(t *testing.T) {
t.Parallel()
blob := mustMarshal(t, LocalState{SchemaVersion: LocalStateSchemaVersion, VMS: map[string]*vm.VM{"i-1": {ID: "i-1"}}})
withRecord := map[string][]byte{
"node.n1": blob,
"i.i-1": mustMarshal(t, (&vm.VM{ID: "i-1"}).Record()),
}
runMigrateCases(t, carryNodeOwnershipForward, []migrateCase{
{name: "empty bucket"},
{name: "listing fails", seed: withRecord, fault: migrateFaultKV{keysErr: errInjected}, wantErr: "list keys"},
{
name: "marker write fails", seed: withRecord,
fault: migrateFaultKV{createErr: injected("nodepresence.n1")}, wantErr: "write nodepresence.n1",
},
{
name: "blob vanished still seeds the marker", seed: withRecord,
fault: migrateFaultKV{getErr: notFound("node.n1")},
check: func(t *testing.T, kv jetstream.KeyValue) {
_, err := kv.Get(context.Background(), "nodepresence.n1")
assert.NoError(t, err)
},
},
{name: "blob read fails", seed: withRecord, fault: migrateFaultKV{getErr: injected("node.n1")}, wantErr: "read node.n1"},
{name: "blob corrupt", seed: map[string][]byte{"node.n1": []byte("{")}, wantErr: "decode node.n1"},
{
name: "blob from a newer build", seed: map[string][]byte{"node.n1": []byte(`{"schema_version":99}`)},
wantErr: "newer than this node understands",
},
{
name: "instance without a record is skipped", seed: map[string][]byte{"node.n1": blob},
check: func(t *testing.T, kv jetstream.KeyValue) { assertKeyAbsent(t, kv, "i.i-1") },
},
{
name: "record read fails", seed: withRecord, fault: migrateFaultKV{getErr: injected("i.i-1")},
wantErr: "stamp owner on i.i-1",
},
})
}
func TestCopyInstancesForward_Faults(t *testing.T) {
t.Parallel()
stopped := map[string][]byte{"instance.i-1": mustMarshal(t, &vm.VM{ID: "i-1", InstanceType: "t3.nano"})}
run := func(ctx context.Context, kvc migrate.KVContext) error {
return copyInstancesForward(ctx, kvc, StoppedInstancePrefix)
}
runMigrateCases(t, run, []migrateCase{
{name: "empty bucket"},
{name: "listing fails", seed: stopped, fault: migrateFaultKV{keysErr: errInjected}, wantErr: "list keys"},
{
name: "instance vanished mid-pass", seed: stopped, fault: migrateFaultKV{getErr: notFound("instance.i-1")},
check: func(t *testing.T, kv jetstream.KeyValue) { assertKeyAbsent(t, kv, "i.i-1") },
},
{name: "read fails", seed: stopped, fault: migrateFaultKV{getErr: injected("instance.i-1")}, wantErr: "read instance.i-1"},
{name: "corrupt instance", seed: map[string][]byte{"instance.i-1": []byte("{")}, wantErr: "decode instance.i-1"},
{name: "write fails", seed: stopped, fault: migrateFaultKV{createErr: injected("i.i-1")}, wantErr: "write i.i-1"},
{
name: "existing record is kept",
seed: map[string][]byte{
"instance.i-1": stopped["instance.i-1"],
"i.i-1": []byte(`{"kept":true}`),
},
check: func(t *testing.T, kv jetstream.KeyValue) {
entry, err := kv.Get(context.Background(), "i.i-1")
require.NoError(t, err)
assert.JSONEq(t, `{"kept":true}`, string(entry.Value()))
},
},
})
}
func TestCopyRunningSetsForward_Faults(t *testing.T) {
t.Parallel()
set := map[string][]byte{"node.n1": mustMarshal(t, LocalState{
SchemaVersion: LocalStateSchemaVersion,
VMS: map[string]*vm.VM{"i-1": {ID: "i-1"}},
})}
runMigrateCases(t, copyRunningSetsForward, []migrateCase{
{name: "empty bucket"},
{name: "listing fails", seed: set, fault: migrateFaultKV{keysErr: errInjected}, wantErr: "list keys"},
{
name: "set vanished mid-pass", seed: set, fault: migrateFaultKV{getErr: notFound("node.n1")},
check: func(t *testing.T, kv jetstream.KeyValue) { assertKeyAbsent(t, kv, "i.i-1") },
},
{name: "read fails", seed: set, fault: migrateFaultKV{getErr: injected("node.n1")}, wantErr: "read node.n1"},
{name: "corrupt set", seed: map[string][]byte{"node.n1": []byte("{")}, wantErr: "decode node.n1"},
{
name: "set from a newer build", seed: map[string][]byte{"node.n1": []byte(`{"schema_version":99}`)},
wantErr: "newer than this node understands",
},
{
name: "nil instance is skipped",
seed: map[string][]byte{"node.n1": []byte(`{"schema_version":1,"vms":{"i-1":null}}`)},
check: func(t *testing.T, kv jetstream.KeyValue) { assertKeyAbsent(t, kv, "i.i-1") },
},
{name: "write fails", seed: set, fault: migrateFaultKV{createErr: injected("i.i-1")}, wantErr: "write i.i-1"},
{
name: "existing record is kept",
seed: map[string][]byte{"node.n1": set["node.n1"], "i.i-1": []byte(`{"kept":true}`)},
check: func(t *testing.T, kv jetstream.KeyValue) {
entry, err := kv.Get(context.Background(), "i.i-1")
require.NoError(t, err)
assert.JSONEq(t, `{"kept":true}`, string(entry.Value()))
},
},
})
}
spinifex/daemon/instance_recovery_coverage_test.go
+373
View File
@@ -0,0 +1,373 @@
//test:in-package — the recovery loop, its pass and its attempt are unexported,
//and these tests drive them against a real store rather than through a cluster.
package daemon
import (
"cmp"
"context"
"fmt"
"slices"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/cache"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
"github.com/nats-io/nats.go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// recoveryStore is a JetStreamManager over private record and heartbeat
// buckets on the shared server, with a liveness view of the heartbeats.
type recoveryStore struct {
m *JetStreamManager
liveness *instancecache.Liveness
}
func newRecoveryStore(t *testing.T) *recoveryStore {
t.Helper()
nc, err := nats.Connect(sharedJSNATSURL)
require.NoError(t, err)
t.Cleanup(nc.Close)
m, err := NewJetStreamManager(nc)
require.NoError(t, err)
clustersize.RedeclareForTest(t, 1)
name := fmt.Sprintf("recovery-%d", time.Now().UnixNano())
m.setInstanceStateBucket(kvstore.NewBucket(m.js, kvstore.Config{Name: name, History: 1}))
_, err = m.stateB.KV(t.Context())
require.NoError(t, err)
beats := kvstore.Config{Name: name + "-hb", History: 1}
m.clusterKV, err = kvstore.NewBucket(m.js, beats).KV(t.Context())
require.NoError(t, err)
t.Cleanup(func() {
_ = m.js.DeleteKeyValue(context.Background(), name)
_ = m.js.DeleteKeyValue(context.Background(), name+"-hb")
})
return &recoveryStore{m: m, liveness: instancecache.NewLiveness(m.js, beats)}
}
// beat writes node's heartbeat stamped at.
func (s *recoveryStore) beat(t *testing.T, node string, at time.Time) {
t.Helper()
require.NoError(t, s.m.WriteHeartbeat(&Heartbeat{Node: node, Timestamp: at.UTC().Format(time.RFC3339)}))
}
// put writes a desired-running record for id owned by node.
func (s *recoveryStore) put(t *testing.T, id, node, instanceType string) {
t.Helper()
record := runningOn(id, node)
record.Spec.InstanceType = instanceType
record.Status.Instance = &ec2.Instance{InstanceId: aws.String(id)}
require.NoError(t, s.m.records.Set(context.Background(), instanceRecordKey(id), record))
}
func (s *recoveryStore) load(t *testing.T, id string) *vm.InstanceRecord {
t.Helper()
record, err := s.m.LoadInstanceRecord(id)
require.NoError(t, err)
require.NotNil(t, record)
return record
}
// recoveryDaemon is a node-1 daemon whose recovery loop reads s, has already
// settled, and finds its own object store healthy.
func recoveryDaemon(t *testing.T, s *recoveryStore) (*Daemon, *instanceRecovery) {
t.Helper()
t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
origTTL := predastoreHealthCacheTTL
predastoreHealthCacheTTL = time.Hour
t.Cleanup(func() { predastoreHealthCacheTTL = origTTL })
d := createTestDaemon(t, sharedNATSURL)
d.jsManager = s.m
d.predastoreHealth.at = time.Now()
d.predastoreHealth.result = predastoreHealthOK
return d, &instanceRecovery{
daemon: d,
liveness: s.liveness,
startedAt: time.Now(),
settledOnce: true,
staleOnce: map[string]struct{}{},
backoff: map[string]recoveryBackoff{},
}
}
// schedulableType is an instance type this daemon can allocate.
func schedulableType(t *testing.T, d *Daemon) string {
t.Helper()
for name := range d.resourceMgr.instanceTypes {
if strings.HasSuffix(name, ".nano") || strings.HasSuffix(name, ".micro") {
return name
}
}
t.Fatal("no small instance type on this host")
return ""
}
func threeNodes() *config.ClusterConfig {
return &config.ClusterConfig{Nodes: map[string]config.Config{"node-1": {}, "node-2": {}, "node-3": {}}}
}
func TestRecoveryStartsAndStopsOnContext(t *testing.T) {
t.Run("the loop runs until the daemon stops", func(t *testing.T) {
logs := captureSlogForTest(t)
nc, err := nats.Connect(sharedJSNATSURL)
require.NoError(t, err)
t.Cleanup(nc.Close)
m, err := NewJetStreamManager(nc)
require.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background())
d := &Daemon{
node: "node-1",
clusterConfig: threeNodes(),
jsManager: m,
instanceService: &ec2instance.InstanceServiceImpl{},
ctx: ctx,
cancel: cancel,
}
d.startInstanceRecovery()
assert.Contains(t, logs.String(), "Instance recovery started")
cancel()
d.shutdownWg.Wait()
assert.Contains(t, logs.String(), "Instance recovery stopping",
"the loop has to exit with the daemon, not outlive the store it reads")
})
}
func TestRecoveryPassPaths(t *testing.T) {
t.Run("an unreadable record store ends the pass", func(t *testing.T) {
logs := captureSlogForTest(t)
r := recoveryFixture("node-1")
r.settledOnce = true
r.daemon.jsManager = &JetStreamManager{}
r.daemon.predastoreHealth.at = time.Now()
r.daemon.predastoreHealth.result = predastoreHealthOK
r.pass(context.Background())
assert.Contains(t, logs.String(), "could not read the instance records")
})
t.Run("a backed-off candidate is left where it is", func(t *testing.T) {
s := newRecoveryStore(t)
_, r := recoveryDaemon(t, s)
s.beat(t, "node-dead", time.Now().Add(-time.Hour))
s.put(t, "i-backoff", "node-dead", "zz9.none")
r.backoff["i-backoff"] = recoveryBackoff{until: time.Now().Add(time.Hour)}
r.pass(t.Context())
r.pass(t.Context())
assert.Equal(t, "node-dead", s.load(t, "i-backoff").Status.LastNode, "a backed-off instance must not be claimed")
assert.Zero(t, r.backoff["i-backoff"].attempts)
})
t.Run("a cancelled pass stops before claiming", func(t *testing.T) {
s := newRecoveryStore(t)
_, r := recoveryDaemon(t, s)
s.beat(t, "node-dead", time.Now().Add(-time.Hour))
s.put(t, "i-cancel", "node-dead", "zz9.none")
r.pass(t.Context())
ctx, cancel := context.WithCancel(t.Context())
cancel()
r.pass(ctx)
assert.Contains(t, r.staleOnce, "i-cancel", "the second pass derived the candidate from the memoised liveness")
assert.Equal(t, "node-dead", s.load(t, "i-cancel").Status.LastNode, "a cancelled pass must not claim")
assert.Empty(t, r.backoff)
})
t.Run("a candidate seen stale twice is attempted", func(t *testing.T) {
s := newRecoveryStore(t)
_, r := recoveryDaemon(t, s)
s.beat(t, "node-dead", time.Now().Add(-time.Hour))
s.put(t, "i-attempt", "node-dead", "zz9.none")
r.pass(t.Context())
assert.Empty(t, r.backoff, "one stale sample is not enough to act on")
r.pass(t.Context())
assert.Equal(t, 1, r.backoff["i-attempt"].attempts, "the failed launch counts against the budget")
assert.Equal(t, "node-dead", s.load(t, "i-attempt").Status.LastNode,
"a launch that could not run here hands the record back")
})
}
func TestRecoveryAttemptFailureAndBudget(t *testing.T) {
t.Run("an unschedulable type is handed back and backed off", func(t *testing.T) {
s := newRecoveryStore(t)
_, r := recoveryDaemon(t, s)
s.put(t, "i-nofit", "node-dead", "zz9.none")
r.attempt(t.Context(), recoveryCandidate{record: s.load(t, "i-nofit"), from: "node-dead"})
record := s.load(t, "i-nofit")
assert.Equal(t, "node-dead", record.Status.LastNode)
assert.Equal(t, vm.DesiredRunning, record.Spec.DesiredState, "one failure is not a give-up")
held := r.backoff["i-nofit"]
assert.Equal(t, 1, held.attempts)
assert.True(t, held.until.After(time.Now()))
})
t.Run("the last attempt in the budget stops the instance with the reason", func(t *testing.T) {
s := newRecoveryStore(t)
_, r := recoveryDaemon(t, s)
s.put(t, "i-budget", "node-dead", "zz9.none")
r.backoff["i-budget"] = recoveryBackoff{attempts: recoveryMaxAttempts - 1}
r.staleOnce["i-budget"] = struct{}{}
r.attempt(t.Context(), recoveryCandidate{record: s.load(t, "i-budget"), from: "node-dead"})
record := s.load(t, "i-budget")
assert.Equal(t, vm.DesiredStopped, record.Spec.DesiredState)
assert.Equal(t, vm.StateStopped, record.Status.Status)
require.NotNil(t, record.Status.Instance.StateReason)
assert.Equal(t, "Server.InsufficientInstanceCapacity", aws.StringValue(record.Status.Instance.StateReason.Code))
assert.NotContains(t, r.backoff, "i-budget")
assert.NotContains(t, r.staleOnce, "i-budget")
})
}
func TestRecoveryAttemptSucceeds(t *testing.T) {
s := newRecoveryStore(t)
d, r := recoveryDaemon(t, s)
// Marking the guest as terminating before Run makes launch return without
// starting QEMU, which is the one part of a recovery this cannot do.
d.vmMgr.SetDeps(vm.Deps{
NodeID: d.node,
VolumeMounter: newVolumeMounterAdapter(d.natsConn, d.node, d.volumeService),
VolumeStateUpdater: d.volumeService,
Hooks: vm.ManagerHooks{BeforeInstanceRelaunch: func(v *vm.VM) error {
d.vmMgr.Inspect(v, func(v *vm.VM) { v.Status = vm.StateShuttingDown })
return nil
}},
})
s.put(t, "i-recovered", "node-dead", schedulableType(t, d))
r.backoff["i-recovered"] = recoveryBackoff{attempts: 2}
r.staleOnce["i-recovered"] = struct{}{}
r.attempt(t.Context(), recoveryCandidate{record: s.load(t, "i-recovered"), from: "node-dead"})
assert.Equal(t, "node-1", s.load(t, "i-recovered").Status.LastNode, "the claim stands once the launch succeeds")
assertVMPresent(t, d, "i-recovered")
assert.NotContains(t, r.backoff, "i-recovered", "a recovered instance starts a fresh budget")
assert.NotContains(t, r.staleOnce, "i-recovered")
}
func TestAbandonOutcomes(t *testing.T) {
cause := fmt.Errorf("launch refused")
t.Run("a give-up that cannot be written keeps the backoff", func(t *testing.T) {
r := recoveryFixture("node-1")
r.daemon.jsManager = &JetStreamManager{}
r.backoff["i-1"] = recoveryBackoff{attempts: recoveryMaxAttempts}
r.abandon(t.Context(), "i-1", "node-dead", recoveryFaultUnknown, cause, recoveryMaxAttempts)
assert.Contains(t, r.backoff, "i-1", "the next pass retries the launch rather than the give-up")
})
t.Run("an instance another node now owns is left running", func(t *testing.T) {
s := newRecoveryStore(t)
r := recoveryFixture("node-1")
r.daemon.jsManager = s.m
s.put(t, "i-moved", "node-3", "t3.micro")
r.backoff["i-moved"] = recoveryBackoff{attempts: recoveryMaxAttempts}
r.staleOnce["i-moved"] = struct{}{}
r.abandon(t.Context(), "i-moved", "node-dead", recoveryFaultUnknown, cause, recoveryMaxAttempts)
record := s.load(t, "i-moved")
assert.Equal(t, vm.DesiredRunning, record.Spec.DesiredState)
assert.Equal(t, "node-3", record.Status.LastNode)
assert.NotContains(t, r.backoff, "i-moved")
assert.NotContains(t, r.staleOnce, "i-moved")
})
t.Run("an instance still on its dead owner is stopped with the reason", func(t *testing.T) {
s := newRecoveryStore(t)
r := recoveryFixture("node-1")
r.daemon.jsManager = s.m
s.put(t, "i-gone", "node-dead", "t3.micro")
r.backoff["i-gone"] = recoveryBackoff{attempts: recoveryMaxAttempts}
r.abandon(t.Context(), "i-gone", "node-dead", recoveryFaultUnknown, cause, recoveryMaxAttempts)
record := s.load(t, "i-gone")
assert.Equal(t, vm.DesiredStopped, record.Spec.DesiredState)
reason := record.Status.Instance.StateReason
require.NotNil(t, reason)
assert.Equal(t, "Server.HostRecoveryFailed", aws.StringValue(reason.Code))
assert.Contains(t, aws.StringValue(reason.Message), fmt.Sprintf("failed %d times", recoveryMaxAttempts))
assert.NotContains(t, r.backoff, "i-gone")
})
}
// Every survivor derives the same candidates, so each has to try them in an
// order of its own or they all race the same instance first.
func TestCandidateOrderingIsPerNode(t *testing.T) {
var records []*vm.InstanceRecord
for i := range 8 {
records = append(records, runningOn(fmt.Sprintf("i-%d", i), "node-dead"))
}
order := func(self string) []string {
got := twice(recoveryFixture(self), records, stale)
ids := make([]string, len(got))
for i, c := range got {
ids[i] = c.record.Metadata.Name
}
return ids
}
a, b := order("node-1"), order("node-2")
require.Len(t, a, len(records))
assert.Equal(t, a, order("node-1"), "one node's order is stable, so a failure is reproducible")
assert.ElementsMatch(t, a, b)
assert.NotEqual(t, a, b, "two survivors must not share one order")
r := recoveryFixture("node-1")
assert.True(t, slices.IsSortedFunc(a, func(x, y string) int {
return cmp.Compare(r.preference(x), r.preference(y))
}), "candidates are tried in this node's preference order")
}
// A returning node drops only the instances the records say another node now
// runs. Its own, ones with no record, and ones it cannot read are kept.
func TestForgetSupersededInstances(t *testing.T) {
s := newRecoveryStore(t)
d, _ := recoveryDaemon(t, s)
s.put(t, "i-moved", "node-3", "")
s.put(t, "i-mine", "node-1", "")
kv, err := s.m.stateB.KV(t.Context())
require.NoError(t, err)
_, err = kv.Put(t.Context(), instanceRecordKey("i-corrupt"), []byte("{"))
require.NoError(t, err)
for _, id := range []string{"i-moved", "i-mine", "i-corrupt", "i-unrecorded"} {
d.vmMgr.Insert(&vm.VM{ID: id, Status: vm.StateStopped})
}
d.forgetSupersededInstances()
assertVMNotPresent(t, d, "i-moved", "another node owns it now")
assertVMPresent(t, d, "i-mine")
assertVMPresent(t, d, "i-corrupt", "an unreadable record is not evidence the instance moved")
assertVMPresent(t, d, "i-unrecorded")
}
spinifex/daemon/instance_recovery_test.go
+29
View File
@@ -15,6 +15,8 @@ import (
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -149,6 +151,33 @@ func TestAnInstancePausedOnItsStorageIsNotMoved(t *testing.T) {
"only the guest whose storage was answering is worth moving")
}
// A fenced guest is the case the storage-fault guard does not catch: the fence
// kills the export rather than pausing the guest, so it publishes no I/O error
// and reads as an ordinary desired-running instance on a node that has died.
func TestAFencedInstanceIsNotMoved(t *testing.T) {
fenced := runningOn("i-fenced", "node-1")
fenced.Status.Status = vm.StateError
fenced.Status.Instance = &ec2.Instance{StateReason: &ec2.StateReason{
Code: aws.String("Server.VolumeFenced"),
Message: aws.String("the lease moved while this node had it open"),
}}
// Errored for a reason that is not a fence, which is what the exclusion has
// to be narrower than: a launch that failed is still worth another node.
failed := runningOn("i-failed", "node-1")
failed.Status.Status = vm.StateError
failed.Status.Instance = &ec2.Instance{StateReason: &ec2.StateReason{
Code: aws.String("Server.RecoveryFailed"),
}}
r := recoveryFixture("node-2")
got := twice(r, []*vm.InstanceRecord{fenced, failed}, stale)
require.Len(t, got, 1)
assert.Equal(t, "i-failed", got[0].record.Metadata.Name,
"a fenced guest's volumes were never sealed, so moving it is how an unproven state becomes a destroyed one")
}
// A guest that had recovered from an earlier pause is an ordinary candidate.
// IOErrorSince clears on the first running poll, so a non-zero value means the
// storage was still failing when the owner last wrote.
spinifex/daemon/instance_running_set_internal_test.go
+100
View File
@@ -0,0 +1,100 @@
package daemon
import (
"testing"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func runningSetOf(ids ...string) map[string]*vm.VM {
set := make(map[string]*vm.VM, len(ids))
for _, id := range ids {
set[id] = &vm.VM{ID: id, InstanceType: "t3.micro", Status: vm.StateRunning}
}
return set
}
func TestWriteRunningSet_SeedFailure(t *testing.T) {
m, f := newFaultInjectedJSM(t)
f.setFail("ListKeys", true)
f.setFail("Keys", true)
got := m.WriteRunningSet("node-1", "", runningSetOf("i-1"))
assert.Equal(t, RunningSetResult{Failed: 1}, got)
f.setFail("ListKeys", false)
f.setFail("Keys", false)
got = m.WriteRunningSet("node-1", "", runningSetOf("i-1"))
assert.Equal(t, RunningSetResult{Written: 1}, got, "the next write must seed and publish")
}
func TestWriteRunningSet_NilMemberIsSkipped(t *testing.T) {
m, _ := newFaultInjectedJSM(t)
set := runningSetOf("i-1")
set["i-nil"] = nil
assert.Equal(t, RunningSetResult{Written: 1}, m.WriteRunningSet("node-1", "", set))
rec, err := m.LoadInstanceRecord("i-nil")
require.NoError(t, err)
assert.Nil(t, rec)
}
// A failed publish forgets its digest, so an unchanged set retries the write.
func TestWriteRunningSet_FailedWriteIsRetried(t *testing.T) {
m, f := newFaultInjectedJSM(t)
f.setFail("Create", true)
assert.Equal(t, RunningSetResult{Failed: 1}, m.WriteRunningSet("node-1", "", runningSetOf("i-1")))
f.setFail("Create", false)
assert.Equal(t, RunningSetResult{Written: 1}, m.WriteRunningSet("node-1", "", runningSetOf("i-1")))
}
func TestWriteRunningSet_Retirement(t *testing.T) {
t.Run("record already gone", func(t *testing.T) {
m, _ := newFaultInjectedJSM(t)
m.WriteRunningSet("node-1", "", runningSetOf("i-1"))
require.NoError(t, m.DeleteStoppedInstance("i-1"))
assert.Equal(t, RunningSetResult{Retired: 1}, m.WriteRunningSet("node-1", "", nil))
})
t.Run("record moved to another node is left alone", func(t *testing.T) {
m, _ := newFaultInjectedJSM(t)
m.WriteRunningSet("node-1", "", runningSetOf("i-1"))
_, err := m.UpdateInstanceRecord("i-1", func(r *vm.InstanceRecord) { r.Status.LastNode = "node-2" })
require.NoError(t, err)
assert.Equal(t, RunningSetResult{Retired: 1}, m.WriteRunningSet("node-1", "", nil))
rec, err := m.LoadInstanceRecord("i-1")
require.NoError(t, err)
require.NotNil(t, rec, "the new owner's record must survive the old owner's write")
assert.Equal(t, "node-2", rec.Status.LastNode)
})
t.Run("unreadable record is kept until it can be checked", func(t *testing.T) {
m, f := newFaultInjectedJSM(t)
m.WriteRunningSet("node-1", "", runningSetOf("i-1"))
f.setFail("Get", true)
assert.Equal(t, RunningSetResult{Failed: 1}, m.WriteRunningSet("node-1", "", nil))
f.setFail("Get", false)
assert.Equal(t, RunningSetResult{Retired: 1}, m.WriteRunningSet("node-1", "", nil),
"the failed retirement must be retried, not forgotten")
})
t.Run("delete failure keeps the record", func(t *testing.T) {
m, f := newFaultInjectedJSM(t)
m.WriteRunningSet("node-1", "", runningSetOf("i-1"))
f.setFail("Delete", true)
assert.Equal(t, RunningSetResult{Failed: 1}, m.WriteRunningSet("node-1", "", nil))
f.setFail("Delete", false)
rec, err := m.LoadInstanceRecord("i-1")
require.NoError(t, err)
assert.NotNil(t, rec)
})
}
spinifex/daemon/jetstream_test.go
+238
View File
@@ -1,13 +1,17 @@
package daemon
import (
"context"
"encoding/json"
"errors"
"github.com/mulgadc/spinifex/internal/testkit"
"strings"
"sync"
"testing"
"time"
"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/resource"
"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
@@ -1870,3 +1874,237 @@ func TestJetStreamManager_WriteState_StampsSchemaVersion(t *testing.T) {
assert.Empty(t, read("stamp-node").VMS,
"the node key is a presence marker: carrying the running set is the cost the split removes")
}
// faultBucket creates a uniquely named bucket on the shared JetStream server
// and wraps it so a test can fail individual KV methods.
func faultBucket(t *testing.T) (jetstream.JetStream, *failingKV) {
t.Helper()
nc, err := nats.Connect(sharedJSNATSURL)
require.NoError(t, err)
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
require.NoError(t, err)
name := "fault-" + strings.Map(func(r rune) rune {
if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') {
return r
}
return '-'
}, t.Name())
kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: name, History: 1})
require.NoError(t, err)
t.Cleanup(func() { _ = js.DeleteKeyValue(context.Background(), name) })
return js, newFailingKV(kv)
}
// newFaultInjectedJSM returns a manager whose instance-state, terminated and
// cluster buckets all sit on one fault-injectable bucket with recovery off.
func newFaultInjectedJSM(t *testing.T) (*JetStreamManager, *failingKV) {
t.Helper()
js, f := faultBucket(t)
m := &JetStreamManager{js: js, clusterKV: f}
m.setInstanceStateBucket(kvstore.NewOpenBucket(nil, f, instanceStateConfig()))
term := kvstore.NewOpenBucket(nil, f, terminatedInstanceConfig())
m.term = kvstore.On[vm.VM](term)
m.termRecords = kvstore.On[vm.InstanceRecord](term)
return m, f
}
func TestJetStreamManager_NilGuards(t *testing.T) {
m := &JetStreamManager{}
noop := func(*vm.VM) {}
assert.False(t, m.KVHealthy(), "a manager with no JetStream context cannot be healthy")
guards := map[string]func() error{
"WriteHeartbeat": func() error { return m.WriteHeartbeat(&Heartbeat{Node: "n"}) },
"ReadHeartbeat": func() error { _, err := m.ReadHeartbeat("n"); return err },
"WriteClusterShutdown": func() error {
return m.WriteClusterShutdown(&ClusterShutdownState{})
},
"UpdateClusterShutdown": func() error {
_, err := m.UpdateClusterShutdown(func(*ClusterShutdownState) {})
return err
},
"UpdateMgmtIPAM": func() error {
_, err := m.UpdateMgmtIPAM("10.0.0.0/24", func(*MgmtIPRecord) {}, true)
return err
},
"ReadClusterShutdown": func() error { _, err := m.ReadClusterShutdown(); return err },
"DeleteClusterShutdown": m.DeleteClusterShutdown,
"WriteShutdownMarker": func() error { return m.WriteShutdownMarker("n") },
"ReadShutdownMarker": func() error { _, err := m.ReadShutdownMarker("n"); return err },
"DeleteShutdownMarker": func() error { return m.DeleteShutdownMarker("n") },
"WriteNodeMarker": func() error { return m.WriteNodeMarker("n") },
"ClaimStoppedInstance": func() error { _, err := m.ClaimStoppedInstance("i-1"); return err },
"ClaimRecoverableInstance": func() error {
_, err := m.ClaimRecoverableInstance("i-1", "a", "b")
return err
},
"ReleaseRecoveredInstance": func() error { return m.ReleaseRecoveredInstance("i-1", "a", "b") },
"AbandonRecovery": func() error {
_, err := m.AbandonRecovery("i-1", "a", "code", "reason")
return err
},
"UpdateStoppedInstance": func() error { _, err := m.UpdateStoppedInstance("i-1", noop); return err },
"WriteTerminatedInstance": func() error {
return m.WriteTerminatedInstance("i-1", &vm.VM{ID: "i-1"})
},
"UpdateTerminatedInstance": func() error {
_, err := m.UpdateTerminatedInstance("i-1", noop)
return err
},
}
for name, call := range guards {
t.Run(name, func(t *testing.T) {
assert.ErrorContains(t, call(), "not initialized")
})
}
}
func TestJetStreamManager_InitClusterStateBucket_NoJetStream(t *testing.T) {
nc, err := nats.Connect(sharedNATSURL)
require.NoError(t, err)
defer nc.Close()
jsm, err := NewJetStreamManager(nc)
require.NoError(t, err)
require.Error(t, jsm.InitClusterStateBucket())
assert.Nil(t, jsm.clusterKV, "a failed init must not leave a half-open handle")
}
func TestJetStreamManager_ClusterStateReads(t *testing.T) {
m, f := newFaultInjectedJSM(t)
t.Run("heartbeat missing", func(t *testing.T) {
_, err := m.ReadHeartbeat("absent")
assert.ErrorIs(t, err, jetstream.ErrKeyNotFound)
})
t.Run("heartbeat corrupt", func(t *testing.T) {
_, err := f.Put(t.Context(), "heartbeat.corrupt", []byte("{"))
require.NoError(t, err)
_, err = m.ReadHeartbeat("corrupt")
assert.Error(t, err)
})
t.Run("cluster shutdown missing then corrupt", func(t *testing.T) {
_, err := m.ReadClusterShutdown()
require.ErrorIs(t, err, jetstream.ErrKeyNotFound)
_, err = f.Put(t.Context(), "cluster.shutdown", []byte("{"))
require.NoError(t, err)
_, err = m.ReadClusterShutdown()
assert.Error(t, err)
require.NoError(t, m.DeleteClusterShutdown())
})
t.Run("update cluster shutdown merges an ack", func(t *testing.T) {
require.NoError(t, m.WriteClusterShutdown(&ClusterShutdownState{
Initiator: "n1", Phase: "drain", NodesTotal: 2, NodesAcked: map[string]string{},
}))
got, err := m.UpdateClusterShutdown(func(s *ClusterShutdownState) {
s.NodesAcked["n2"] = "done"
})
require.NoError(t, err)
assert.Equal(t, "done", got.NodesAcked["n2"])
stored, err := m.ReadClusterShutdown()
require.NoError(t, err)
assert.Equal(t, "drain", stored.Phase)
assert.Equal(t, map[string]string{"n2": "done"}, stored.NodesAcked)
})
t.Run("kv errors surface", func(t *testing.T) {
f.setFail("Get", true)
f.setFail("Delete", true)
t.Cleanup(func() { f.setFail("Get", false); f.setFail("Delete", false) })
_, err := m.ReadShutdownMarker("n1")
require.ErrorIs(t, err, errInjected, "only a missing key reads as no marker")
require.ErrorIs(t, m.DeleteClusterShutdown(), errInjected)
require.ErrorIs(t, m.DeleteShutdownMarker("n1"), errInjected)
})
}
func TestJetStreamManager_WriteNodeMarkerBestEffort_Timeout(t *testing.T) {
clustersize.DeclareForTest(t, 1)
nc, err := nats.Connect(sharedJSNATSURL)
require.NoError(t, err)
defer nc.Close()
jsm, err := NewJetStreamManager(nc)
require.NoError(t, err)
require.NoError(t, jsm.InitKVBucket())
obs := &fakeKVObserver{}
jsm.SetSyncObserver(obs)
jsm.WriteNodeMarkerBestEffort("obs-timeout", time.Nanosecond)
successes, failures := obs.snapshot()
assert.Empty(t, successes)
require.Len(t, failures, 1)
assert.ErrorIs(t, failures[0].err, context.DeadlineExceeded)
}
func TestJetStreamManager_InstanceRecordKVErrors(t *testing.T) {
m, f := newFaultInjectedJSM(t)
require.NoError(t, m.WriteStoppedInstance("i-stopped", &vm.VM{ID: "i-stopped", InstanceType: "t3.micro"}))
require.NoError(t, m.WriteInstanceRecord("i-run", &vm.InstanceRecord{
Metadata: resource.Metadata{Name: "i-run", AccountID: "111122223333"},
Spec: vm.InstanceSpec{InstanceType: "t3.micro", DesiredState: vm.DesiredRunning},
Status: vm.InstanceStatus{Status: vm.StateRunning, LastNode: "node-1"},
}))
require.NoError(t, m.WriteNodeMarker("node-1"))
t.Run("read failures", func(t *testing.T) {
f.setFail("Get", true)
t.Cleanup(func() { f.setFail("Get", false) })
_, err := m.ClaimStoppedInstance("i-stopped")
require.ErrorIs(t, err, errInjected)
_, err = m.ClaimRecoverableInstance("i-run", "node-1", "node-2")
require.ErrorIs(t, err, errInjected)
require.ErrorIs(t, m.ReleaseRecoveredInstance("i-run", "node-2", "node-1"), errInjected)
_, err = m.AbandonRecovery("i-run", "node-1", "code", "reason")
require.ErrorIs(t, err, errInjected)
})
t.Run("write failures are not lost claims", func(t *testing.T) {
f.setFail("Update", true)
t.Cleanup(func() { f.setFail("Update", false) })
_, err := m.ClaimStoppedInstance("i-stopped")
require.ErrorIs(t, err, errInjected)
assert.NotErrorIs(t, err, vm.ErrStoppedInstanceClaimed)
_, err = m.ClaimRecoverableInstance("i-run", "node-1", "node-2")
require.ErrorIs(t, err, errInjected)
assert.NotErrorIs(t, err, vm.ErrRecoveryClaimLost)
rec, err := m.LoadInstanceRecord("i-run")
require.NoError(t, err)
assert.Equal(t, "node-1", rec.Status.LastNode, "a failed claim must leave the owner unchanged")
})
t.Run("load state listing fails", func(t *testing.T) {
f.setFail("ListKeys", true)
f.setFail("Keys", true)
t.Cleanup(func() { f.setFail("ListKeys", false); f.setFail("Keys", false) })
vms, known, err := m.LoadState("node-1")
require.ErrorIs(t, err, errInjected)
assert.False(t, known, "a node whose records cannot be read is not a node with no instances")
assert.Nil(t, vms)
})
t.Run("terminated write and delete", func(t *testing.T) {
f.setFail("Get", true)
f.setFail("Delete", true)
t.Cleanup(func() { f.setFail("Get", false); f.setFail("Delete", false) })
require.ErrorIs(t, m.WriteTerminatedInstance("i-term", &vm.VM{ID: "i-term"}), errInjected)
require.ErrorIs(t, m.DeleteTerminatedInstance("i-term"), errInjected)
})
}
spinifex/daemon/local_state_test.go
+34
View File
@@ -136,3 +136,37 @@ func TestReadLocalState_NilVMSMapInitialized(t *testing.T) {
require.NotNil(t, state)
assert.NotNil(t, state.VMS)
}
func TestWriteLocalState_Failures(t *testing.T) {
t.Run("parent is a file", func(t *testing.T) {
parent := filepath.Join(t.TempDir(), "file")
require.NoError(t, os.WriteFile(parent, nil, 0o600))
err := writeLocalState(filepath.Join(parent, "instance-state.json"), map[string]*vm.VM{})
assert.ErrorContains(t, err, "mkdir")
})
t.Run("tmp path is a directory", func(t *testing.T) {
path := filepath.Join(t.TempDir(), "instance-state.json")
require.NoError(t, os.Mkdir(path+".tmp", 0o750))
assert.ErrorContains(t, writeLocalState(path, map[string]*vm.VM{}), "open tmp")
_, err := os.Stat(path)
assert.ErrorIs(t, err, os.ErrNotExist)
})
t.Run("destination is a non-empty directory", func(t *testing.T) {
path := filepath.Join(t.TempDir(), "instance-state.json")
require.NoError(t, os.MkdirAll(filepath.Join(path, "occupied"), 0o750))
assert.ErrorContains(t, writeLocalState(path, map[string]*vm.VM{}), "rename")
_, err := os.Stat(path + ".tmp")
assert.ErrorIs(t, err, os.ErrNotExist, "a failed rename must not leave the tmp file behind")
})
}
func TestReadLocalState_Unreadable(t *testing.T) {
state, err := ReadLocalState(t.TempDir())
assert.ErrorContains(t, err, "read local state")
assert.Nil(t, state, "an unreadable file is not a fresh install")
}
spinifex/daemon/main_test.go
+4
View File
@@ -25,6 +25,10 @@ func TestMain(m *testing.M) {
// refuse to start and every daemon-constructing test would fail. 4 keeps the
// default reserve intact with 2 schedulable cores — enough for these tests.
os.Setenv("SPINIFEX_HOST_VCPU", "4")
// Admission otherwise also clamps on the host's live MemAvailable, which a
// loaded box running tests in parallel can drop below the reserve. The gate
// has its own tests that inject a reader.
os.Setenv("SPINIFEX_ADMISSION_LIVE_MEM", "0")
// Start a plain NATS server (no JetStream)
ns, err := server.NewServer(&server.Options{
spinifex/daemon/shutdown_coverage_test.go
+219
View File
@@ -0,0 +1,219 @@
package daemon
import (
"encoding/json"
"errors"
hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
"os"
"path/filepath"
"syscall"
"testing"
"time"
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
"github.com/nats-io/nats.go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// deadPID is above the default pid_max, so it names no process.
const deadPID = 1 << 30
// shutdownPhaseReply sends payload to handler over NATS and returns the ACK it
// answered with and the outcome it reported.
func shutdownPhaseReply(t *testing.T, handler func(*nats.Msg) string, payload []byte) (ShutdownACK, string) {
t.Helper()
nc, err := nats.Connect(sharedNATSURL)
require.NoError(t, err)
t.Cleanup(nc.Close)
outcome := make(chan string, 1)
subject := nats.NewInbox()
sub, err := nc.Subscribe(subject, func(msg *nats.Msg) { outcome <- handler(msg) })
require.NoError(t, err)
t.Cleanup(func() { _ = sub.Unsubscribe() })
reply, err := nc.Request(subject, payload, 5*time.Second)
require.NoError(t, err)
var ack ShutdownACK
require.NoError(t, json.Unmarshal(reply.Data, &ack))
return ack, <-outcome
}
// stubNBDKitProcs makes every pid look like a live nbdkit and records which
// were signalled, so a STORAGE phase can never reach a real process.
func stubNBDKitProcs(t *testing.T, signalErr error) *[]int {
t.Helper()
origComm, origSignal := procComm, signalProcess
t.Cleanup(func() { procComm, signalProcess = origComm, origSignal })
var signalled []int
procComm = func(int) (string, error) { return "nbdkit", nil }
signalProcess = func(pid int, _ syscall.Signal) error {
signalled = append(signalled, pid)
return signalErr
}
return &signalled
}
// shutdownTestDaemon is the plain daemon the phase handlers need: a node name
// and a config whose pid directory is private to the test.
func shutdownTestDaemon(t *testing.T, services ...string) (*Daemon, string) {
t.Helper()
d := &Daemon{node: "node-1", config: &config.Config{Services: services}}
return d, configurePidDir(t, d)
}
func TestShutdownMalformedPayloads(t *testing.T) {
d := &Daemon{node: "node-1", config: &config.Config{}}
for phase, handler := range map[string]func(*nats.Msg) string{
"storage": d.handleShutdownStorage,
"persist": d.handleShutdownPersist,
"infra": d.handleShutdownInfra,
} {
t.Run(phase, func(t *testing.T) {
ack, outcome := shutdownPhaseReply(t, handler, []byte("{"))
assert.Equal(t, outcomeError, outcome)
assert.Equal(t, phase, ack.Phase)
assert.Equal(t, "node-1", ack.Node)
assert.NotEmpty(t, ack.Error, "the coordinator has to be told why the phase did not run")
assert.Empty(t, ack.Stopped)
})
}
t.Run("an ACK that cannot be sent is logged", func(t *testing.T) {
logs := captureSlogForTest(t)
assert.Equal(t, outcomeError, d.handleShutdownPersist(noReplyMsg("shutdown.persist", []byte("{"))))
assert.Contains(t, logs.String(), "Failed to respond with shutdown ACK")
})
}
func TestHandleShutdownStorage(t *testing.T) {
t.Run("a viperblock already gone is not reported stopped", func(t *testing.T) {
runtimeDir := t.TempDir()
t.Setenv("XDG_RUNTIME_DIR", runtimeDir)
signalled := stubNBDKitProcs(t, nil)
require.NoError(t, os.WriteFile(filepath.Join(runtimeDir, "nbdkit-vol-a.pid"), []byte("4242"), 0o600))
d, pidDir := shutdownTestDaemon(t, "viperblock")
require.NoError(t, hostprocess.WritePidFileTo(pidDir, "viperblock", deadPID))
ack, outcome := shutdownPhaseReply(t, d.handleShutdownStorage, []byte(`{"phase":"storage"}`))
assert.Equal(t, outcomeSuccess, outcome)
assert.Equal(t, "storage", ack.Phase)
assert.Empty(t, ack.Error)
assert.Empty(t, ack.Stopped, "a stop that failed must not be reported as one")
assert.NoFileExists(t, filepath.Join(pidDir, "viperblock.pid"), "the stale pid file is cleared")
assert.Equal(t, []int{4242}, *signalled, "this node's nbdkit backends are swept from its runtime dir")
})
t.Run("an unconfigured viperblock is left alone", func(t *testing.T) {
t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
signalled := stubNBDKitProcs(t, nil)
d, pidDir := shutdownTestDaemon(t, "daemon")
require.NoError(t, hostprocess.WritePidFileTo(pidDir, "viperblock", deadPID))
ack, outcome := shutdownPhaseReply(t, d.handleShutdownStorage, []byte(`{"phase":"storage"}`))
assert.Equal(t, outcomeSuccess, outcome)
assert.Empty(t, ack.Stopped)
assert.FileExists(t, filepath.Join(pidDir, "viperblock.pid"))
assert.Empty(t, *signalled)
})
}
func TestHandleShutdownPersist(t *testing.T) {
t.Run("a predastore already gone is not reported stopped", func(t *testing.T) {
d, pidDir := shutdownTestDaemon(t, "predastore")
require.NoError(t, hostprocess.WritePidFileTo(pidDir, "predastore", deadPID))
ack, outcome := shutdownPhaseReply(t, d.handleShutdownPersist, []byte(`{"phase":"persist"}`))
assert.Equal(t, outcomeSuccess, outcome)
assert.Equal(t, "persist", ack.Phase)
assert.Empty(t, ack.Error)
assert.Empty(t, ack.Stopped)
assert.NoFileExists(t, filepath.Join(pidDir, "predastore.pid"))
})
t.Run("an unconfigured predastore is left alone", func(t *testing.T) {
d, pidDir := shutdownTestDaemon(t, "daemon")
require.NoError(t, hostprocess.WritePidFileTo(pidDir, "predastore", deadPID))
ack, outcome := shutdownPhaseReply(t, d.handleShutdownPersist, []byte(`{"phase":"persist"}`))
assert.Equal(t, outcomeSuccess, outcome)
assert.Empty(t, ack.Stopped)
assert.FileExists(t, filepath.Join(pidDir, "predastore.pid"))
})
}
// A service whose process is already gone still lets GATE complete: the daemon
// stops taking work either way, and only real stops are reported.
func TestHandleShutdownGate_DeadServices(t *testing.T) {
d, pidDir := shutdownTestDaemon(t, "ui", "vpcd")
require.NoError(t, hostprocess.WritePidFileTo(pidDir, "spinifex-ui", deadPID))
require.NoError(t, hostprocess.WritePidFileTo(pidDir, "vpcd", deadPID))
ack, outcome := shutdownPhaseReply(t, d.handleShutdownGate, []byte(`{"phase":"gate"}`))
assert.Equal(t, outcomeSuccess, outcome)
assert.Empty(t, ack.Stopped)
assert.True(t, d.shuttingDown.Load())
}
// Neither the marker nor the local state write gates the ACK: by DRAIN the
// guests are already stopped, and refusing to advance would only strand them.
func TestHandleShutdownDrain_MarkerAndStateErrors(t *testing.T) {
logs := captureSlogForTest(t)
d := createTestDaemon(t, sharedNATSURL)
d.jsManager = &JetStreamManager{}
blocker := filepath.Join(t.TempDir(), "file")
require.NoError(t, os.WriteFile(blocker, nil, 0o600))
d.config.DataDir = filepath.Join(blocker, "data")
ack, outcome := shutdownPhaseReply(t, d.handleShutdownDrain, []byte(`{"phase":"drain"}`))
assert.Equal(t, outcomeSuccess, outcome)
assert.Equal(t, "drain", ack.Phase)
assert.Empty(t, ack.Error)
assert.Contains(t, logs.String(), "Failed to write shutdown marker during DRAIN")
assert.Contains(t, logs.String(), "Failed to write state during DRAIN")
}
func TestCleanupOrphanNBDKit_Failures(t *testing.T) {
t.Run("an unlistable runtime dir signals nothing", func(t *testing.T) {
logs := captureSlogForTest(t)
signalled := stubNBDKitProcs(t, nil)
cleanupOrphanNBDKit(filepath.Join(t.TempDir(), "["))
assert.Empty(t, *signalled)
assert.Contains(t, logs.String(), "cannot list pid files")
})
t.Run("a backend that cannot be signalled keeps its pid file", func(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "nbdkit-vol-stuck.pid")
require.NoError(t, os.WriteFile(path, []byte("5151"), 0o600))
signalled := stubNBDKitProcs(t, errors.New("operation not permitted"))
cleanupOrphanNBDKit(dir)
assert.Equal(t, []int{5151}, *signalled)
assert.FileExists(t, path, "the backend may still be running, so its pid file is the only handle left on it")
})
t.Run("an unreadable pid file that cannot be removed is skipped", func(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "nbdkit-vol-dir.pid")
require.NoError(t, os.MkdirAll(filepath.Join(path, "child"), 0o750))
signalled := stubNBDKitProcs(t, nil)
cleanupOrphanNBDKit(dir)
assert.Empty(t, *signalled)
assert.DirExists(t, path)
})
}
func TestPidDirEmpty(t *testing.T) {
assert.Empty(t, (&Daemon{config: &config.Config{}}).pidDir(), "no base dir means no pid dir to look in")
assert.Equal(t, "/data/logs", (&Daemon{config: &config.Config{BaseDir: "/data/spinifex/"}}).pidDir())
}
spinifex/daemon/vm_adapters_coverage_test.go
+798
View File
@@ -0,0 +1,798 @@
package daemon
import (
"context"
"encoding/json"
"fmt"
viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
"net/netip"
"os"
"path/filepath"
"slices"
"sync"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/mulgadc/spinifex/internal/testkit"
"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
ec2eip "github.com/mulgadc/spinifex/spinifex/domains/ec2/eip"
ec2placementgroup "github.com/mulgadc/spinifex/spinifex/domains/ec2/placementgroup"
ec2spotinstance "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
ec2volume "github.com/mulgadc/spinifex/spinifex/domains/ec2/volume"
ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
"github.com/mulgadc/spinifex/spinifex/domains/network/external/dhcp"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
"github.com/mulgadc/spinifex/spinifex/services/viperblockd/vbwire"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// uniqueAdapterID returns an ID no other test in the binary uses, for keys
// written to the shared JetStream buckets.
func uniqueAdapterID(prefix string) string {
return fmt.Sprintf("%s-%d", prefix, time.Now().UnixNano())
}
// connectAdapterTestNATS opens a connection to url that closes with the test.
func connectAdapterTestNATS(t *testing.T, url string) *nats.Conn {
t.Helper()
clustersize.DeclareForTest(t, 1)
nc, err := nats.Connect(url)
require.NoError(t, err)
t.Cleanup(nc.Close)
return nc
}
// closedAdapterTestConn returns a connection that is already closed, so every
// request and subscribe on it fails immediately.
func closedAdapterTestConn(t *testing.T) *nats.Conn {
t.Helper()
nc, err := nats.Connect(sharedNATSURL)
require.NoError(t, err)
nc.Close()
return nc
}
// respondJSON subscribes subject on nc and answers every request with reply.
func respondJSON(t *testing.T, nc *nats.Conn, subject string, reply any) {
t.Helper()
data, err := json.Marshal(reply)
require.NoError(t, err)
sub, err := nc.Subscribe(subject, func(msg *nats.Msg) { _ = msg.Respond(data) })
require.NoError(t, err)
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
// --- stateStoreAdapter ---
func TestStateStoreAdapter_DelegatesToJetStream(t *testing.T) {
nc := connectAdapterTestNATS(t, sharedJSNATSURL)
jsm, err := NewJetStreamManager(nc)
require.NoError(t, err)
require.NoError(t, jsm.InitKVBucket())
require.NoError(t, jsm.InitTerminatedInstanceBucket())
var persistedNode string
adapter := newStateStoreAdapter(jsm, func(nodeID string, _ map[string]*vm.VM) error {
persistedNode = nodeID
return nil
})
t.Run("running set goes through persist", func(t *testing.T) {
require.NoError(t, adapter.SaveRunningState("node-persist", map[string]*vm.VM{}))
assert.Equal(t, "node-persist", persistedNode)
vms, found, err := adapter.LoadRunningState(uniqueAdapterID("node-none"))
require.NoError(t, err)
assert.False(t, found, "a node that never wrote state has none")
assert.Empty(t, vms)
})
t.Run("stopped instance lifecycle", func(t *testing.T) {
id := uniqueAdapterID("i-stopped")
t.Cleanup(func() { _ = jsm.DeleteStoppedInstance(id) })
require.NoError(t, adapter.WriteStoppedInstance(id, &vm.VM{ID: id, InstanceType: "t3.nano"}))
got, err := adapter.LoadStoppedInstance(id)
require.NoError(t, err)
require.NotNil(t, got)
assert.Equal(t, "t3.nano", got.InstanceType)
updated, err := adapter.UpdateStoppedInstance(id, func(v *vm.VM) { v.InstanceType = "t3.micro" })
require.NoError(t, err)
assert.Equal(t, "t3.micro", updated.InstanceType)
stopped, err := adapter.ListStoppedInstances()
require.NoError(t, err)
assert.True(t, slices.ContainsFunc(stopped, func(v *vm.VM) bool { return v.ID == id }))
require.NoError(t, adapter.DeleteStoppedInstance(id))
got, err = adapter.LoadStoppedInstance(id)
require.NoError(t, err)
assert.Nil(t, got, "a deleted stopped instance is gone")
})
t.Run("a stopped instance is claimed once", func(t *testing.T) {
id := uniqueAdapterID("i-claim")
t.Cleanup(func() { _ = jsm.DeleteStoppedInstance(id) })
require.NoError(t, adapter.WriteStoppedInstance(id, &vm.VM{ID: id, InstanceType: "t3.nano"}))
claimed, err := adapter.ClaimStoppedInstance(id)
require.NoError(t, err)
assert.Equal(t, id, claimed.ID)
_, err = adapter.ClaimStoppedInstance(id)
assert.ErrorIs(t, err, vm.ErrStoppedInstanceClaimed)
})
t.Run("terminated instance lifecycle", func(t *testing.T) {
id := uniqueAdapterID("i-term")
require.NoError(t, adapter.WriteTerminatedInstance(id, &vm.VM{ID: id, InstanceType: "t3.nano"}))
updated, err := adapter.UpdateTerminatedInstance(id, func(v *vm.VM) { v.InstanceType = "t3.small" })
require.NoError(t, err)
assert.Equal(t, "t3.small", updated.InstanceType)
hasID := func(v *vm.VM) bool { return v.ID == id }
terminated, err := adapter.ListTerminatedInstances()
require.NoError(t, err)
assert.True(t, slices.ContainsFunc(terminated, hasID))
require.NoError(t, adapter.DeleteTerminatedInstance(id))
terminated, err = adapter.ListTerminatedInstances()
require.NoError(t, err)
assert.False(t, slices.ContainsFunc(terminated, hasID))
})
}
// --- consumeCleanShutdownMarker ---
func TestConsumeCleanShutdownMarker(t *testing.T) {
assert.False(t, (&Daemon{}).consumeCleanShutdownMarker()(),
"without JetStream there is no marker to trust")
nc := connectAdapterTestNATS(t, sharedJSNATSURL)
jsm, err := NewJetStreamManager(nc)
require.NoError(t, err)
require.NoError(t, jsm.InitClusterStateBucket())
node := uniqueAdapterID("node-marker")
d := &Daemon{node: node, jsManager: jsm}
consume := d.consumeCleanShutdownMarker()
assert.False(t, consume(), "no marker means the shutdown was not clean")
require.NoError(t, jsm.WriteShutdownMarker(node))
assert.True(t, consume(), "a recorded clean shutdown is trusted")
present, err := jsm.ReadShutdownMarker(node)
require.NoError(t, err)
assert.False(t, present, "the marker is consumed so a later crash is not mistaken for a clean stop")
assert.False(t, consume())
}
func TestBuildVMManagerDeps_BackingStoreReady(t *testing.T) {
d := &Daemon{config: &config.Config{}, vmMgr: vm.NewManager()}
assert.False(t, d.buildVMManagerDeps().BackingStoreReady(),
"no NATS connection means viperblock is unreachable")
d.natsConn = connectAdapterTestNATS(t, sharedJSNATSURL)
assert.True(t, d.buildVMManagerDeps().BackingStoreReady(),
"no predastore configured and JetStream answering is ready")
}
// --- volumeMounterAdapter ---
func TestVolumeMounterAdapter_Mount_ClosedConnIsNotRetryable(t *testing.T) {
adapter := newVolumeMounterAdapter(closedAdapterTestConn(t), "node-closed", nil)
instance := &vm.VM{ID: "i-mount-closed"}
instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-closed"}}
err := adapter.Mount(t.Context(), instance)
require.ErrorIs(t, err, nats.ErrConnectionClosed)
assert.NotErrorIs(t, err, vm.ErrMountRetryable,
"only a missing responder means viperblockd is still starting")
}
// recordingVolumeState records UpdateVolumeState calls and returns err.
type recordingVolumeState struct {
err error
updated []string
}
func (r *recordingVolumeState) UpdateVolumeState(_, volumeID, state, _, _ string) error {
r.updated = append(r.updated, volumeID+"="+state)
return r.err
}
func TestVolumeMounterAdapter_Unmount_Errors(t *testing.T) {
nc := connectAdapterTestNATS(t, sharedNATSURL)
t.Run("no responder fails the seal and leaves the volume", func(t *testing.T) {
volState := &recordingVolumeState{}
adapter := newVolumeMounterAdapter(nc, uniqueAdapterID("node-unmount"), volState)
instance := &vm.VM{ID: "i-unmount-none"}
instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-a"}, {Name: "vol-b"}}
err := adapter.Unmount(t.Context(), instance)
require.ErrorIs(t, err, nats.ErrNoResponders)
assert.Contains(t, err.Error(), "vol-a")
assert.Contains(t, err.Error(), "vol-b", "every volume is attempted")
assert.Empty(t, volState.updated, "an unsealed volume must not go available")
})
t.Run("state update failure does not fail the unmount", func(t *testing.T) {
node := uniqueAdapterID("node-unmount")
respondJSON(t, nc, "ebs."+node+".unmount", viperblocklegacyv1.EBSUnMountResponse{Volume: "vol-a"})
volState := &recordingVolumeState{err: errInjected}
adapter := newVolumeMounterAdapter(nc, node, volState)
instance := &vm.VM{ID: "i-unmount-state"}
instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-a"}}
require.NoError(t, adapter.Unmount(t.Context(), instance))
assert.Equal(t, []string{"vol-a=available"}, volState.updated)
})
}
func TestVolumeMounterAdapter_Abandon(t *testing.T) {
nc := connectAdapterTestNATS(t, sharedNATSURL)
// abandonResponder answers per volume so one request can mix outcomes.
abandonResponder := func(t *testing.T, node string, replies map[string][]byte) *[]vbwire.VolumeAbandonRequest {
t.Helper()
var (
mu sync.Mutex
seen []vbwire.VolumeAbandonRequest
)
sub, err := nc.Subscribe(vbwire.VolumeAbandonSubject(node), func(msg *nats.Msg) {
var req vbwire.VolumeAbandonRequest
_ = json.Unmarshal(msg.Data, &req)
mu.Lock()
seen = append(seen, req)
mu.Unlock()
_ = msg.Respond(replies[req.Volume])
})
require.NoError(t, err)
t.Cleanup(func() { _ = sub.Unsubscribe() })
return &seen
}
instanceWith := func(volumes ...string) *vm.VM {
v := &vm.VM{ID: "i-abandon", AccountID: testAccountID}
for _, name := range volumes {
v.EBSRequests.Requests = append(v.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{Name: name})
}
return v
}
t.Run("no responder", func(t *testing.T) {
adapter := newVolumeMounterAdapter(nc, uniqueAdapterID("node-abandon"), nil)
err := adapter.Abandon(t.Context(), instanceWith("vol-1"), "superseded")
assert.ErrorIs(t, err, nats.ErrNoResponders)
})
t.Run("outcomes per volume", func(t *testing.T) {
node := uniqueAdapterID("node-abandon")
seen := abandonResponder(t, node, map[string][]byte{
"vol-abandoned": []byte(`{"abandoned":true}`),
"vol-absent": []byte(`{"abandoned":false}`),
})
adapter := newVolumeMounterAdapter(nc, node, nil)
require.NoError(t, adapter.Abandon(t.Context(), instanceWith("vol-abandoned", "vol-absent"), "superseded"),
"a volume not exported here is the ordinary case, not a failure")
require.Len(t, *seen, 2)
assert.Equal(t, vbwire.VolumeAbandonRequest{Volume: "vol-abandoned", Reason: "superseded"}, (*seen)[0])
})
t.Run("failures are joined and do not stop the sweep", func(t *testing.T) {
node := uniqueAdapterID("node-abandon")
seen := abandonResponder(t, node, map[string][]byte{
"vol-garbled": []byte(`not json`),
"vol-refused": []byte(`{"error":"export busy"}`),
"vol-ok": []byte(`{"abandoned":true}`),
})
adapter := newVolumeMounterAdapter(nc, node, nil)
err := adapter.Abandon(t.Context(), instanceWith("vol-garbled", "vol-refused", "vol-ok"), "fenced")
require.Error(t, err)
assert.Contains(t, err.Error(), "unmarshal abandon response for vol-garbled")
assert.Contains(t, err.Error(), "abandon vol-refused: export busy")
assert.NotContains(t, err.Error(), "vol-ok")
assert.Len(t, *seen, 3, "every volume is attempted")
})
}
// --- resourceControllerAdapter ---
func TestResourceControllerAdapter_UnknownAndKnownType(t *testing.T) {
rm := newReservationTestRM()
a := newResourceControllerAdapter(rm)
t.Run("unknown type", func(t *testing.T) {
require.ErrorContains(t, a.Allocate("zz9.none"), "instance type zz9.none not found")
assert.Zero(t, a.CanAllocate("zz9.none", 4))
a.Deallocate("zz9.none")
a.ReleaseToReservation("cr-none", "zz9.none")
assert.Zero(t, rm.allocatedVCPU, "an unknown type must not move the books")
})
t.Run("known type", func(t *testing.T) {
assert.Equal(t, 4, a.CanAllocate("t3.micro", 10))
require.NoError(t, a.Allocate("t3.micro"))
assert.Equal(t, 2, rm.allocatedVCPU)
assert.Equal(t, 3, a.CanAllocate("t3.micro", 10))
a.Deallocate("t3.micro")
assert.Zero(t, rm.allocatedVCPU)
require.NoError(t, a.Allocate("t3.micro"))
a.ReleaseToReservation("cr-gone", "t3.micro")
assert.Zero(t, rm.allocatedVCPU, "a slot from a vanished reservation returns to the general pool")
})
}
// --- per-instance subscription hooks ---
func TestOnInstanceRecoveringHook(t *testing.T) {
t.Run("subscribes the command topic once", func(t *testing.T) {
d, _ := newHookTestDaemon(t)
hook := d.onInstanceRecoveringHook()
instance := &vm.VM{ID: "i-recovering"}
hook(instance)
first, ok := d.natsSubscriptions[instance.ID]
require.True(t, ok)
assert.Equal(t, "ec2.cmd.i-recovering", first.Subject)
hook(instance)
assert.Same(t, first, d.natsSubscriptions[instance.ID], "an existing subscription is kept, not replaced")
})
t.Run("subscribe failure leaves no entry", func(t *testing.T) {
d := &Daemon{natsConn: closedAdapterTestConn(t), natsSubscriptions: map[string]*nats.Subscription{}}
d.onInstanceRecoveringHook()(&vm.VM{ID: "i-recovering-closed"})
assert.Empty(t, d.natsSubscriptions)
})
}
func TestOnInstanceUpHook_SubscribeFailureReturnsError(t *testing.T) {
d := &Daemon{natsConn: closedAdapterTestConn(t), natsSubscriptions: map[string]*nats.Subscription{}}
err := d.onInstanceUpHook()(&vm.VM{ID: "i-up-closed"})
require.ErrorIs(t, err, nats.ErrConnectionClosed)
assert.Empty(t, d.natsSubscriptions, "no topic may stay bound when the set fails")
}
// natEIPResolver answers the associated-EIP lookup the up hook makes.
type natEIPResolver struct {
ec2eip.EIPService
ip string
}
func (r *natEIPResolver) AssociatedPublicIPForInstance(context.Context, string, string) (string, bool) {
return r.ip, r.ip != ""
}
// recordNATEvents answers subject with success and records each event whose
// external IP is externalIP.
func recordNATEvents(t *testing.T, nc *nats.Conn, subject, externalIP string) func() []map[string]string {
t.Helper()
var (
mu sync.Mutex
events []map[string]string
)
sub, err := nc.Subscribe(subject, func(msg *nats.Msg) {
var evt map[string]string
_ = json.Unmarshal(msg.Data, &evt)
if evt["external_ip"] == externalIP {
mu.Lock()
events = append(events, evt)
mu.Unlock()
}
_ = msg.Respond([]byte(`{"success":true}`))
})
require.NoError(t, err)
t.Cleanup(func() { _ = sub.Unsubscribe() })
return func() []map[string]string {
mu.Lock()
defer mu.Unlock()
return slices.Clone(events)
}
}
func TestOnInstanceUpHook_RepublishesNAT(t *testing.T) {
d, nc := newHookTestDaemon(t)
upInstance := func(id, publicIP string) *vm.VM {
return &vm.VM{
ID: id,
ENIId: "eni-" + id,
ENIMac: "02:00:00:00:00:01",
PublicIP: publicIP,
Instance: &ec2.Instance{VpcId: aws.String("vpc-nat"), PrivateIpAddress: aws.String("10.0.1.5")},
}
}
t.Run("elastic IP resolved from the EIP store", func(t *testing.T) {
events := recordNATEvents(t, nc, "vpc.add-nat", "198.51.100.77")
d.eipService = &natEIPResolver{ip: "198.51.100.77"}
require.NoError(t, d.onInstanceUpHook()(upInstance("i-nat-eip", "")))
got := events()
require.Len(t, got, 1, "the EIP's NAT must be re-announced after a reboot")
assert.Equal(t, "vpc-nat", got[0]["vpc_id"])
assert.Equal(t, "10.0.1.5", got[0]["logical_ip"])
assert.Equal(t, "port-eni-i-nat-eip", got[0]["port_name"])
assert.Equal(t, "02:00:00:00:00:01", got[0]["mac"])
})
t.Run("auto-assigned IP wins over the EIP store", func(t *testing.T) {
events := recordNATEvents(t, nc, "vpc.add-nat", "198.51.100.78")
d.eipService = &natEIPResolver{ip: "198.51.100.200"}
require.NoError(t, d.onInstanceUpHook()(upInstance("i-nat-auto", "198.51.100.78")))
assert.Len(t, events(), 1)
})
}
func TestOnInstanceUpHook_ReclaimsMIGSlice(t *testing.T) {
mdev := filepath.Join(t.TempDir(), "mdev-uuid")
require.NoError(t, os.WriteFile(mdev, nil, 0o600))
mgr := gpu.NewManager(nil)
mgr.AddMIGInstances(gpu.GPUDevice{PCIAddress: "0000:41:00.0"}, []gpu.MIGInstance{{MdevPath: mdev}})
d, _ := newHookTestDaemon(t)
d.gpuManager = mgr
instance := &vm.VM{
ID: "i-mig",
GPUAttachments: []gpu.GPUAttachment{
{MdevPath: mdev},
{MdevPath: "/nonexistent/mdev"},
},
}
require.NoError(t, d.onInstanceUpHook()(instance), "a slice that cannot be re-claimed only warns")
assert.Zero(t, mgr.Available(), "the surviving slice is claimed again after a restart")
require.NoError(t, newInstanceCleanerAdapter(d).ReleaseGPU(instance))
assert.Equal(t, 1, mgr.Available(), "release returns the slice to the pool")
}
func TestOnInstanceDownHook_UnsubscribeErrors(t *testing.T) {
d, nc := newHookTestDaemon(t)
id := "i-down-stale"
keys := []string{id, id + ".console", id + ".password", "system.TerminateInstance." + id}
for _, key := range keys {
sub, err := nc.SubscribeSync("test.stale." + key)
require.NoError(t, err)
require.NoError(t, sub.Unsubscribe())
d.natsSubscriptions[key] = sub
}
d.onInstanceDownHook()(id)
assert.Empty(t, d.natsSubscriptions, "a failed unsubscribe still drops the entry")
}
// --- instanceCleanerAdapter ---
// getFailingStore fails every metadata read with a generic error.
type getFailingStore struct {
objectstore.ObjectStore
}
func (getFailingStore) GetObject(context.Context, *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
return nil, errInjected
}
func TestInstanceCleaner_DeleteVolumes_Errors(t *testing.T) {
instanceWith := func(reqs ...viperblocklegacyv1.EBSRequest) *vm.VM {
v := &vm.VM{ID: "i-delvol", AccountID: testAccountID}
v.EBSRequests.Requests = reqs
return v
}
t.Run("EFI delete with no responder", func(t *testing.T) {
a := newInstanceCleanerAdapter(&Daemon{natsConn: connectAdapterTestNATS(t, sharedNATSURL)})
err := a.DeleteVolumes(instanceWith(viperblocklegacyv1.EBSRequest{Name: "vol-efi-" + uniqueAdapterID("x"), EFI: true}))
assert.ErrorIs(t, err, nats.ErrNoResponders)
})
t.Run("no volume service skips both kinds", func(t *testing.T) {
a := newInstanceCleanerAdapter(&Daemon{})
require.NoError(t, a.DeleteVolumes(instanceWith(
viperblocklegacyv1.EBSRequest{Name: "vol-keep", DeleteOnTermination: false},
viperblocklegacyv1.EBSRequest{Name: "vol-delete", DeleteOnTermination: true},
)))
})
t.Run("volume service errors are reported", func(t *testing.T) {
svc := ec2volume.NewVolumeServiceImplWithStore(&config.Config{}, getFailingStore{objectstore.NewMemoryObjectStore()}, nil)
a := newInstanceCleanerAdapter(&Daemon{volumeService: svc})
err := a.DeleteVolumes(instanceWith(viperblocklegacyv1.EBSRequest{Name: "vol-keep", DeleteOnTermination: false}))
require.ErrorIs(t, err, errInjected, "a failed detach must surface")
err = a.DeleteVolumes(instanceWith(viperblocklegacyv1.EBSRequest{Name: "vol-delete", DeleteOnTermination: true}))
require.ErrorIs(t, err, errInjected, "a failed delete must surface")
})
}
// cleanupFailingPlumber fails CleanupTap and records the tap it was given.
type cleanupFailingPlumber struct {
recordingPlumber
}
func (p *cleanupFailingPlumber) CleanupTap(name string) error {
p.cleanedTaps = append(p.cleanedTaps, name)
return errInjected
}
func TestInstanceCleaner_CleanupMgmtNetwork_TapFailureStillReleases(t *testing.T) {
flushed, _ := captureNeighHooks(t)
alloc, err := NewMgmtIPAllocator("10.15.9.1")
require.NoError(t, err)
const ip = "10.15.9.20"
alloc.Claim("i-tapfail", ip)
require.Equal(t, 1, alloc.AllocatedCount())
plumber := &cleanupFailingPlumber{}
d := &Daemon{networkPlumber: plumber, mgmtIPAllocator: alloc, mgmtBridgeIP: "10.15.9.1"}
newInstanceCleanerAdapter(d).CleanupMgmtNetwork(&vm.VM{ID: "i-tapfail", MgmtIP: ip})
assert.Len(t, plumber.cleanedTaps, 1)
assert.Len(t, *flushed, 1, "the neighbour entry is invalidated even when the tap teardown fails")
assert.Zero(t, alloc.AllocatedCount(), "the address goes back to the pool")
}
func TestInstanceCleaner_ReleasePublicIP(t *testing.T) {
const pool = "wan-fake"
newCleaner := func(t *testing.T, releaseErr error, nc *nats.Conn) (*instanceCleanerAdapter, *fakeAllocator) {
t.Helper()
ipam := ec2vpc.NewExternalIPAMWithKV(nil, nil)
alloc := &fakeAllocator{releaseErr: releaseErr}
require.NoError(t, ipam.InstallAllocator(pool, alloc))
return newInstanceCleanerAdapter(&Daemon{externalIPAM: ipam, natsConn: nc}), alloc
}
withIP := func(ip string) *vm.VM {
return &vm.VM{ID: "i-pub", ENIId: "eni-pub", PublicIP: ip, PublicIPPool: pool}
}
t.Run("nothing to release", func(t *testing.T) {
a, alloc := newCleaner(t, nil, nil)
require.NoError(t, a.ReleasePublicIP(&vm.VM{ID: "i-pub", PublicIPPool: pool}))
require.NoError(t, a.ReleasePublicIP(&vm.VM{ID: "i-pub", PublicIP: "203.0.113.5"}))
require.NoError(t, newInstanceCleanerAdapter(&Daemon{}).ReleasePublicIP(withIP("203.0.113.5")))
assert.Empty(t, alloc.releases)
})
t.Run("success tears down NAT and releases", func(t *testing.T) {
nc := connectAdapterTestNATS(t, sharedNATSURL)
events := recordNATEvents(t, nc, "vpc.delete-nat", "203.0.113.6")
a, alloc := newCleaner(t, nil, nc)
inst := withIP("203.0.113.6")
inst.Instance = &ec2.Instance{VpcId: aws.String("vpc-pub"), PrivateIpAddress: aws.String("10.0.0.6")}
require.NoError(t, a.ReleasePublicIP(inst))
assert.Equal(t, []netip.Addr{netip.MustParseAddr("203.0.113.6")}, alloc.releases)
got := events()
require.Len(t, got, 1)
assert.Equal(t, "vpc-pub", got[0]["vpc_id"])
assert.Equal(t, "10.0.0.6", got[0]["logical_ip"])
assert.Equal(t, "port-eni-pub", got[0]["port_name"])
})
t.Run("untracked lease is terminal success", func(t *testing.T) {
a, alloc := newCleaner(t, fmt.Errorf("release: %w", dhcp.ErrLeaseNotTracked), nil)
require.NoError(t, a.ReleasePublicIP(withIP("203.0.113.7")))
assert.Len(t, alloc.releases, 1)
})
t.Run("release failure is returned", func(t *testing.T) {
a, _ := newCleaner(t, errInjected, nil)
assert.ErrorIs(t, a.ReleasePublicIP(withIP("203.0.113.8")), errInjected)
})
t.Run("unparseable address", func(t *testing.T) {
a, alloc := newCleaner(t, nil, nil)
assert.ErrorContains(t, a.ReleasePublicIP(withIP("not-an-ip")), "parse release IP")
assert.Empty(t, alloc.releases)
})
}
// adapterVPCFixture is a VPC service on a private JetStream with one ENI.
type adapterVPCFixture struct {
url string
nc *nats.Conn
js jetstream.JetStream
vpc *ec2vpc.VPCServiceImpl
eniID string
}
func newAdapterVPCFixture(t *testing.T) *adapterVPCFixture {
t.Helper()
ns, nc, js := testutil.StartTestJetStream(t)
testutil.StubVpcdSGResponder(t, nc)
vpcSvc, err := ec2vpc.NewVPCServiceImplWithNATS(t.Context(), &config.Config{}, nc)
require.NoError(t, err)
vpcOut, err := vpcSvc.CreateVpc(t.Context(), &ec2.CreateVpcInput{CidrBlock: aws.String("10.0.0.0/16")}, testAccountID)
require.NoError(t, err)
subnetOut, err := vpcSvc.CreateSubnet(t.Context(), &ec2.CreateSubnetInput{
VpcId: vpcOut.Vpc.VpcId, CidrBlock: aws.String("10.0.1.0/24"),
}, testAccountID)
require.NoError(t, err)
eniOut, err := vpcSvc.CreateNetworkInterface(t.Context(), &ec2.CreateNetworkInterfaceInput{
SubnetId: subnetOut.Subnet.SubnetId,
}, testAccountID)
require.NoError(t, err)
return &adapterVPCFixture{
url: ns.ClientURL(), nc: nc, js: js, vpc: vpcSvc,
eniID: *eniOut.NetworkInterface.NetworkInterfaceId,
}
}
func TestInstanceCleaner_ReleaseAutoAssignedPublicIP(t *testing.T) {
const pool = "wan-auto"
f := newAdapterVPCFixture(t)
eipKV, err := f.js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: "test-adapter-eips"})
require.NoError(t, err)
_, err = eipKV.Put(t.Context(), testAccountID+".eipalloc-owned", []byte(`{"eni_id":"eni-eip-owned"}`))
require.NoError(t, err)
f.vpc.SetExternalIPAM(nil, eipKV)
newCleaner := func(t *testing.T, releaseErr error) (*instanceCleanerAdapter, *fakeAllocator) {
t.Helper()
ipam := ec2vpc.NewExternalIPAMWithKV(nil, nil)
alloc := &fakeAllocator{releaseErr: releaseErr}
require.NoError(t, ipam.InstallAllocator(pool, alloc))
return newInstanceCleanerAdapter(&Daemon{externalIPAM: ipam, vpcService: f.vpc}), alloc
}
stopping := func(eniID string) *vm.VM {
return &vm.VM{ID: "i-auto", AccountID: testAccountID, ENIId: eniID, PublicIP: "203.0.113.20", PublicIPPool: pool}
}
t.Run("nothing to release", func(t *testing.T) {
a, alloc := newCleaner(t, nil)
released, err := a.ReleaseAutoAssignedPublicIP(&vm.VM{ID: "i-auto", PublicIPPool: pool})
require.NoError(t, err)
assert.False(t, released)
inst := stopping(f.eniID)
inst.PublicIPAllocID = "eipalloc-launch"
released, err = a.ReleaseAutoAssignedPublicIP(inst)
require.NoError(t, err)
assert.False(t, released, "an address allocated through the EIP service is the customer's")
assert.Empty(t, alloc.releases)
})
t.Run("address held by an elastic IP stays", func(t *testing.T) {
a, alloc := newCleaner(t, nil)
released, err := a.ReleaseAutoAssignedPublicIP(stopping("eni-eip-owned"))
require.NoError(t, err)
assert.False(t, released)
assert.Empty(t, alloc.releases)
})
t.Run("EIP lookup failure keeps the address", func(t *testing.T) {
failing := newFailingKV(eipKV, "Keys")
f.vpc.SetExternalIPAM(nil, failing)
t.Cleanup(func() { f.vpc.SetExternalIPAM(nil, eipKV) })
a, alloc := newCleaner(t, nil)
released, err := a.ReleaseAutoAssignedPublicIP(stopping(f.eniID))
require.ErrorIs(t, err, errInjected)
assert.False(t, released)
assert.Empty(t, alloc.releases, "an address not proven ours is never released")
})
t.Run("release failure", func(t *testing.T) {
a, _ := newCleaner(t, errInjected)
released, err := a.ReleaseAutoAssignedPublicIP(stopping(f.eniID))
require.ErrorIs(t, err, errInjected)
assert.False(t, released)
})
t.Run("missing ENI only warns", func(t *testing.T) {
a, alloc := newCleaner(t, nil)
released, err := a.ReleaseAutoAssignedPublicIP(stopping("eni-vanished"))
require.NoError(t, err)
assert.True(t, released)
assert.Len(t, alloc.releases, 1)
})
t.Run("success clears the ENI public IP", func(t *testing.T) {
require.NoError(t, f.vpc.UpdateENIPublicIP(testAccountID, f.eniID, "203.0.113.20", pool))
a, alloc := newCleaner(t, nil)
released, err := a.ReleaseAutoAssignedPublicIP(stopping(f.eniID))
require.NoError(t, err)
assert.True(t, released)
assert.Equal(t, []netip.Addr{netip.MustParseAddr("203.0.113.20")}, alloc.releases)
rec, err := f.vpc.GetENIRecord(testAccountID, f.eniID)
require.NoError(t, err)
assert.Empty(t, rec.PublicIpAddress, "a stale address would let the reconciler restore the NAT rule")
assert.Empty(t, rec.PublicIpPool)
})
t.Run("primary ENI delete failure is returned", func(t *testing.T) {
nc, err := nats.Connect(f.url)
require.NoError(t, err)
vpcSvc, err := ec2vpc.NewVPCServiceImplWithNATS(t.Context(), &config.Config{}, nc)
require.NoError(t, err)
nc.Close()
a := newInstanceCleanerAdapter(&Daemon{vpcService: vpcSvc})
err = a.DetachAndDeleteENI(&vm.VM{ID: "i-auto", AccountID: testAccountID, ENIId: f.eniID})
require.Error(t, err, "a primary ENI that could not be deleted must fail terminate")
})
t.Run("spot close failure is returned", func(t *testing.T) {
nc, err := nats.Connect(f.url)
require.NoError(t, err)
spotSvc, err := ec2spotinstance.NewSpotInstanceServiceImplWithNATS(t.Context(), &config.Config{}, nc)
require.NoError(t, err)
a := newInstanceCleanerAdapter(&Daemon{spotInstanceService: spotSvc})
require.NoError(t, a.RemoveFromSpotRequest(&vm.VM{ID: "i-auto", AccountID: testAccountID}),
"an instance no spot request names is a no-op")
nc.Close()
assert.ErrorContains(t, a.RemoveFromSpotRequest(&vm.VM{ID: "i-auto", AccountID: testAccountID}),
awserrors.ErrorServerInternal)
})
}
// failingEIPDisassociator fails every disassociation.
type failingEIPDisassociator struct {
ec2eip.EIPService
calls int
}
func (s *failingEIPDisassociator) DisassociateByENI(context.Context, string, string) (bool, error) {
s.calls++
return false, errInjected
}
func TestInstanceCleaner_DisassociateEIPFailureDoesNotFailTerminate(t *testing.T) {
f := newENIHotPlugFixture(t)
f.vmInst.AccountID = testAccountID
f.vmInst.ENIId = f.eniID
eip := &failingEIPDisassociator{}
f.daemon.eipService = eip
require.NoError(t, newInstanceCleanerAdapter(f.daemon).DetachAndDeleteENI(f.vmInst))
assert.Equal(t, 1, eip.calls)
}
func TestInstanceCleaner_RemoveFromPlacementGroup(t *testing.T) {
inGroup := &vm.VM{ID: "i-pg", AccountID: testAccountID, PlacementGroupName: "pg-spread"}
require.NoError(t, newInstanceCleanerAdapter(&Daemon{}).RemoveFromPlacementGroup(inGroup),
"no placement group service is a no-op")
a := newInstanceCleanerAdapter(&Daemon{placementGroupService: &ec2placementgroup.PlacementGroupServiceImpl{}})
require.NoError(t, a.RemoveFromPlacementGroup(&vm.VM{ID: "i-pg"}), "an ungrouped instance is a no-op")
assert.EqualError(t, a.RemoveFromPlacementGroup(inGroup), awserrors.ErrorMissingParameter,
"a group member with no node recorded cannot be removed")
}
func TestInstanceCleaner_DetachAndDeleteENI_NoVPCServiceIsNoOp(t *testing.T) {
a := newInstanceCleanerAdapter(&Daemon{})
require.NoError(t, a.DetachAndDeleteENI(&vm.VM{ID: "i-novpc", ENIId: "eni-novpc"}))
}
spinifex/domains/acm/certfields.go
+182
View File
@@ -0,0 +1,182 @@
package acm
import (
"crypto/x509"
"fmt"
"log/slog"
"math/big"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/acm"
)
// maxSANSummaries is how many names AWS lists in a ListCertificates summary
// before setting HasAdditionalSubjectAlternativeNames.
const maxSANSummaries = 100
// formatSerial renders a serial number as ACM does: two lowercase hex digits
// per byte, joined by colons.
func formatSerial(serial *big.Int) string {
return strings.ReplaceAll(fmt.Sprintf("% x", serial.Bytes()), " ", ":")
}
// signatureAlgorithms are ACM's spellings of the leaf's signature algorithm.
var signatureAlgorithms = map[x509.SignatureAlgorithm]string{
x509.SHA256WithRSA: "SHA256WITHRSA",
x509.SHA384WithRSA: "SHA384WITHRSA",
x509.SHA512WithRSA: "SHA512WITHRSA",
x509.ECDSAWithSHA256: "SHA256WITHECDSA",
x509.ECDSAWithSHA384: "SHA384WITHECDSA",
x509.ECDSAWithSHA512: "SHA512WITHECDSA",
}
// keyUsageNames is in x509.KeyUsage bit order, which is the order ACM lists them.
var keyUsageNames = []string{
acm.KeyUsageNameDigitalSignature, acm.KeyUsageNameNonRepudiation, acm.KeyUsageNameKeyEncipherment,
acm.KeyUsageNameDataEncipherment, acm.KeyUsageNameKeyAgreement, acm.KeyUsageNameCertificateSigning,
acm.KeyUsageNameCrlSigning, acm.KeyUsageNameEncipherOnly, acm.KeyUsageNameDecipherOnly,
}
var extKeyUsageNames = map[x509.ExtKeyUsage]struct{ name, oid string }{
x509.ExtKeyUsageAny: {acm.ExtendedKeyUsageNameAny, "2.5.29.37.0"},
x509.ExtKeyUsageServerAuth: {acm.ExtendedKeyUsageNameTlsWebServerAuthentication, "1.3.6.1.5.5.7.3.1"},
x509.ExtKeyUsageClientAuth: {acm.ExtendedKeyUsageNameTlsWebClientAuthentication, "1.3.6.1.5.5.7.3.2"},
x509.ExtKeyUsageCodeSigning: {acm.ExtendedKeyUsageNameCodeSigning, "1.3.6.1.5.5.7.3.3"},
x509.ExtKeyUsageEmailProtection: {acm.ExtendedKeyUsageNameEmailProtection, "1.3.6.1.5.5.7.3.4"},
x509.ExtKeyUsageIPSECEndSystem: {acm.ExtendedKeyUsageNameIpsecEndSystem, "1.3.6.1.5.5.7.3.5"},
x509.ExtKeyUsageIPSECTunnel: {acm.ExtendedKeyUsageNameIpsecTunnel, "1.3.6.1.5.5.7.3.6"},
x509.ExtKeyUsageIPSECUser: {acm.ExtendedKeyUsageNameIpsecUser, "1.3.6.1.5.5.7.3.7"},
x509.ExtKeyUsageTimeStamping: {acm.ExtendedKeyUsageNameTimeStamping, "1.3.6.1.5.5.7.3.8"},
x509.ExtKeyUsageOCSPSigning: {acm.ExtendedKeyUsageNameOcspSigning, "1.3.6.1.5.5.7.3.9"},
}
// issuerName is how ACM names the issuer: its O when it has one, else its CN.
func issuerName(leaf *x509.Certificate) string {
if len(leaf.Issuer.Organization) > 0 {
return leaf.Issuer.Organization[0]
}
if leaf.Issuer.CommonName != "" {
return leaf.Issuer.CommonName
}
return leaf.Issuer.String()
}
// importedSANs is what ACM reports for an imported certificate: the CN, then
// any DNS SANs not already listed.
func importedSANs(leaf *x509.Certificate) []string {
return uniqueDomains(leafDomain(leaf), leaf.DNSNames)
}
// keyUsages returns ACM's key-usage names, or ANY when the extension is absent.
func keyUsages(leaf *x509.Certificate) []string {
if leaf.KeyUsage == 0 {
return []string{acm.KeyUsageNameAny}
}
var names []string
for i, name := range keyUsageNames {
if leaf.KeyUsage&(1<<i) != 0 {
names = append(names, name)
}
}
return names
}
// extKeyUsages returns ACM's extended-key-usage names and OIDs, or NONE with an
// empty OID when the extension is absent. Unrecognised OIDs are CUSTOM.
func extKeyUsages(leaf *x509.Certificate) (names, oids []string) {
if len(leaf.ExtKeyUsage)+len(leaf.UnknownExtKeyUsage) == 0 {
return []string{acm.ExtendedKeyUsageNameNone}, []string{""}
}
for _, u := range leaf.ExtKeyUsage {
if e, ok := extKeyUsageNames[u]; ok {
names, oids = append(names, e.name), append(oids, e.oid)
}
}
for _, oid := range leaf.UnknownExtKeyUsage {
names, oids = append(names, acm.ExtendedKeyUsageNameCustom), append(oids, oid.String())
}
return names, oids
}
// parseStoredLeaf parses the stored certificate body, or returns nil for a
// record that has none yet (a requested certificate awaiting issuance).
func parseStoredLeaf(rec *CertRecord) *x509.Certificate {
if rec.Certificate == "" {
return nil
}
leaf, err := parseLeaf([]byte(rec.Certificate))
if err != nil {
slog.Warn("acm: stored certificate body does not parse", "arn", rec.CertificateArn, "err", err)
return nil
}
return leaf
}
// importedCreatedAt falls back to ImportedAt for a record stored before
// CreatedAt was kept; for those, the first import time is no longer known.
func importedCreatedAt(rec *CertRecord) time.Time {
if !rec.CreatedAt.IsZero() {
return rec.CreatedAt
}
return rec.ImportedAt
}
// applyLeafDetail fills the CertificateDetail fields that come from the
// certificate body. They are derived on read, so records stored before these
// fields existed describe the same as new ones.
func applyLeafDetail(detail *acm.CertificateDetail, rec *CertRecord) {
leaf := parseStoredLeaf(rec)
if leaf != nil {
detail.Serial = aws.String(formatSerial(leaf.SerialNumber))
detail.Issuer = aws.String(issuerName(leaf))
if alg, ok := signatureAlgorithms[leaf.SignatureAlgorithm]; ok {
detail.SignatureAlgorithm = aws.String(alg)
}
}
if certTypeOrDefault(rec) != certTypeImported {
return
}
detail.CreatedAt = timePtr(importedCreatedAt(rec))
detail.Options = &acm.CertificateOptions{
CertificateTransparencyLoggingPreference: aws.String(acm.CertificateTransparencyLoggingPreferenceDisabled),
}
if leaf == nil {
return
}
if sans := importedSANs(leaf); len(sans) > 0 {
detail.SubjectAlternativeNames = aws.StringSlice(sans)
detail.DomainValidationOptions = make([]*acm.DomainValidation, len(sans))
for i, d := range sans {
detail.DomainValidationOptions[i] = &acm.DomainValidation{DomainName: aws.String(d)}
}
}
for _, name := range keyUsages(leaf) {
detail.KeyUsages = append(detail.KeyUsages, &acm.KeyUsage{Name: aws.String(name)})
}
names, oids := extKeyUsages(leaf)
for i := range names {
detail.ExtendedKeyUsages = append(detail.ExtendedKeyUsages, &acm.ExtendedKeyUsage{Name: aws.String(names[i]), OID: aws.String(oids[i])})
}
}
// applyLeafSummary fills the ListCertificates summary fields ACM reports for
// an imported certificate, derived on read as in applyLeafDetail.
func applyLeafSummary(sum *acm.CertificateSummary, rec *CertRecord) {
if certTypeOrDefault(rec) != certTypeImported {
return
}
sum.CreatedAt = timePtr(importedCreatedAt(rec))
leaf := parseStoredLeaf(rec)
if leaf == nil {
return
}
if sans := importedSANs(leaf); len(sans) > 0 {
sum.SubjectAlternativeNameSummaries = aws.StringSlice(sans[:min(len(sans), maxSANSummaries)])
sum.HasAdditionalSubjectAlternativeNames = aws.Bool(len(sans) > maxSANSummaries)
}
sum.KeyUsages = aws.StringSlice(keyUsages(leaf))
names, _ := extKeyUsages(leaf)
sum.ExtendedKeyUsages = aws.StringSlice(names)
}
spinifex/domains/acm/certfields_test.go
+210
View File
@@ -0,0 +1,210 @@
package acm
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/json"
"encoding/pem"
"math/big"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/private/protocol/json/jsonutil"
"github.com/aws/aws-sdk-go/service/acm"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// awsAuditSerial is the serial of the certificate imported into real ACM in
// the response audit; awsAuditSerialText is how ACM rendered it.
const (
awsAuditCN = "awsdiff.example.com"
awsAuditSerialHex = "6c42f03e00b6ee54c0740448f8cad7939787c61c"
awsAuditSerialText = "6c:42:f0:3e:00:b6:ee:54:c0:74:04:48:f8:ca:d7:93:97:87:c6:1c"
)
// genAuditShapedCert builds a certificate shaped like the one the audit
// imported into real ACM (openssl req -x509 -newkey rsa:2048 -subj /CN=...):
// RSA-2048, SHA-256 with RSA, CN only, no SAN, key-usage or EKU extension.
func genAuditShapedCert(t *testing.T) (certPEM, keyPEM []byte) {
t.Helper()
key, err := rsa.GenerateKey(rand.Reader, 2048)
require.NoError(t, err)
serial, ok := new(big.Int).SetString(awsAuditSerialHex, 16)
require.True(t, ok)
tmpl := &x509.Certificate{
SerialNumber: serial,
Subject: pkix.Name{CommonName: awsAuditCN},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(30 * 24 * time.Hour),
SignatureAlgorithm: x509.SHA256WithRSA,
BasicConstraintsValid: true,
IsCA: true,
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
require.NoError(t, err)
certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
keyPEM = pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
return certPEM, keyPEM
}
func importCert(t *testing.T, svc *ACMServiceImpl, certPEM, keyPEM []byte) string {
t.Helper()
out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{Certificate: certPEM, PrivateKey: keyPEM}, testAccountID)
require.NoError(t, err)
return aws.StringValue(out.CertificateArn)
}
func describeCert(t *testing.T, svc *ACMServiceImpl, certArn string) *acm.CertificateDetail {
t.Helper()
out, err := svc.DescribeCertificate(context.Background(), &acm.DescribeCertificateInput{CertificateArn: aws.String(certArn)}, testAccountID)
require.NoError(t, err)
return out.Certificate
}
// wireJSON encodes v as the gateway does and decodes it generically, so
// assertions see the members a client receives rather than the Go struct.
func wireJSON(t *testing.T, v any) map[string]any {
t.Helper()
body, err := jsonutil.BuildJSON(v)
require.NoError(t, err)
var m map[string]any
require.NoError(t, json.Unmarshal(body, &m))
return m
}
// Expected values are copied from real ACM's DescribeCertificate response for
// the audit's certificate. Export is absent: aws-sdk-go v1 cannot carry it.
func TestDescribeCertificate_ImportedMatchesAWSAuditResponse(t *testing.T) {
svc := setupACMService(t)
certPEM, keyPEM := genAuditShapedCert(t)
d := describeCert(t, svc, importCert(t, svc, certPEM, keyPEM))
assert.Equal(t, []string{awsAuditCN}, aws.StringValueSlice(d.SubjectAlternativeNames),
"ACM lists the CN as the only SAN when the certificate has no SAN extension")
assert.Equal(t, awsAuditSerialText, aws.StringValue(d.Serial))
assert.Equal(t, awsAuditCN, aws.StringValue(d.Issuer))
assert.Equal(t, "CN="+awsAuditCN, aws.StringValue(d.Subject))
assert.Equal(t, "SHA256WITHRSA", aws.StringValue(d.SignatureAlgorithm))
require.NotNil(t, d.CreatedAt)
require.NotNil(t, d.ImportedAt)
assert.True(t, d.CreatedAt.Equal(*d.ImportedAt), "CreatedAt equals ImportedAt on a first import")
cert := wireJSON(t, &acm.DescribeCertificateOutput{Certificate: d})["Certificate"].(map[string]any)
assert.Equal(t, []any{map[string]any{"DomainName": awsAuditCN}}, cert["DomainValidationOptions"],
"an imported certificate's validation entries carry the domain name and nothing else")
assert.Equal(t, []any{map[string]any{"Name": "ANY"}}, cert["KeyUsages"])
assert.Equal(t, []any{map[string]any{"Name": "NONE", "OID": ""}}, cert["ExtendedKeyUsages"])
assert.Equal(t, map[string]any{"CertificateTransparencyLoggingPreference": "DISABLED"}, cert["Options"])
}
func TestListCertificates_ImportedSummaryMatchesAWSAuditResponse(t *testing.T) {
svc := setupACMService(t)
certPEM, keyPEM := genAuditShapedCert(t)
importCert(t, svc, certPEM, keyPEM)
list, err := svc.ListCertificates(context.Background(), &acm.ListCertificatesInput{}, testAccountID)
require.NoError(t, err)
sums := wireJSON(t, list)["CertificateSummaryList"].([]any)
require.Len(t, sums, 1)
s := sums[0].(map[string]any)
assert.Equal(t, []any{awsAuditCN}, s["SubjectAlternativeNameSummaries"])
assert.Equal(t, false, s["HasAdditionalSubjectAlternativeNames"])
assert.Equal(t, []any{"ANY"}, s["KeyUsages"])
assert.Equal(t, []any{"NONE"}, s["ExtendedKeyUsages"])
assert.Equal(t, "INELIGIBLE", s["RenewalEligibility"])
require.Contains(t, s, "CreatedAt")
assert.Equal(t, s["ImportedAt"], s["CreatedAt"])
}
func TestImportCertificate_ReimportKeepsCreatedAtAndMovesImportedAt(t *testing.T) {
svc := setupACMService(t)
c1, k1 := genAuditShapedCert(t)
certArn := importCert(t, svc, c1, k1)
first := describeCert(t, svc, certArn)
c2, k2 := genAuditShapedCert(t)
_, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{
CertificateArn: aws.String(certArn), Certificate: c2, PrivateKey: k2,
}, testAccountID)
require.NoError(t, err)
second := describeCert(t, svc, certArn)
assert.True(t, second.CreatedAt.Equal(*first.CreatedAt), "re-import must not move CreatedAt")
assert.True(t, second.ImportedAt.After(*first.ImportedAt), "re-import records the new import time")
}
// A record stored before these fields existed holds the old Serial and Issuer
// spellings, no SANs and no CreatedAt; it must still describe as AWS does.
func TestDescribeCertificate_LegacyImportedRecordDerivedFromPEM(t *testing.T) {
svc := setupACMService(t)
certPEM, keyPEM := genAuditShapedCert(t)
certArn := svc.mintCertificateArn(testAccountID)
importedAt := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
require.NoError(t, svc.store.PutCert(context.Background(), &CertRecord{
CertificateArn: certArn,
AccountID: testAccountID,
Certificate: string(certPEM),
PrivateKey: string(keyPEM),
DomainName: awsAuditCN,
Serial: awsAuditSerialHex,
Subject: "CN=" + awsAuditCN,
Issuer: "CN=" + awsAuditCN,
KeyAlgorithm: "RSA_2048",
ImportedAt: importedAt,
}))
d := describeCert(t, svc, certArn)
assert.Equal(t, awsAuditSerialText, aws.StringValue(d.Serial))
assert.Equal(t, awsAuditCN, aws.StringValue(d.Issuer))
assert.Equal(t, []string{awsAuditCN}, aws.StringValueSlice(d.SubjectAlternativeNames))
require.NotNil(t, d.CreatedAt)
assert.True(t, d.CreatedAt.Equal(importedAt), "a legacy record's CreatedAt falls back to ImportedAt")
require.Len(t, d.KeyUsages, 1)
require.Len(t, d.ExtendedKeyUsages, 1)
}
// Expected values are copied from real ACM's DescribeCertificate response for
// a self-signed P-384 import carrying an O, SANs that omit the CN, and both
// usage extensions.
func TestDescribeCertificate_ImportedWithExtensionsMatchesAWS(t *testing.T) {
svc := setupACMService(t)
key, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader)
require.NoError(t, err)
tmpl := &x509.Certificate{
SerialNumber: big.NewInt(1005),
Subject: pkix.Name{Organization: []string{"Probe Org"}, CommonName: "cn.example.com"},
DNSNames: []string{"a.example.com", "b.example.com"},
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(24 * time.Hour),
SignatureAlgorithm: x509.ECDSAWithSHA384,
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
require.NoError(t, err)
keyDER, err := x509.MarshalECPrivateKey(key)
require.NoError(t, err)
d := describeCert(t, svc, importCert(t, svc,
pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}),
pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})))
assert.Equal(t, "EC-secp384r1", aws.StringValue(d.KeyAlgorithm))
assert.Equal(t, "SHA384WITHECDSA", aws.StringValue(d.SignatureAlgorithm))
assert.Equal(t, "03:ed", aws.StringValue(d.Serial))
assert.Equal(t, "Probe Org", aws.StringValue(d.Issuer))
assert.Equal(t, []string{"cn.example.com", "a.example.com", "b.example.com"}, aws.StringValueSlice(d.SubjectAlternativeNames))
cert := wireJSON(t, &acm.DescribeCertificateOutput{Certificate: d})["Certificate"].(map[string]any)
assert.Equal(t, []any{map[string]any{"Name": "DIGITAL_SIGNATURE"}, map[string]any{"Name": "KEY_ENCIPHERMENT"}}, cert["KeyUsages"])
assert.Equal(t, []any{
map[string]any{"Name": "TLS_WEB_SERVER_AUTHENTICATION", "OID": "1.3.6.1.5.5.7.3.1"},
map[string]any{"Name": "TLS_WEB_CLIENT_AUTHENTICATION", "OID": "1.3.6.1.5.5.7.3.2"},
}, cert["ExtendedKeyUsages"])
}
spinifex/domains/acm/getcertificate_test.go
+41 -4
View File
@@ -32,9 +32,10 @@ func TestGetCertificate_ReturnsBodyAndChain(t *testing.T) {
assert.Equal(t, string(chainPEM), aws.StringValue(out.CertificateChain))
}
// A self-signed leaf has no chain; the field is left absent rather than an
// empty string a client would have to distinguish from a real chain.
func TestGetCertificate_OmitsAbsentChain(t *testing.T) {
// AWS answers a certificate imported without a chain with the certificate
// itself as the chain. The substitution is response-only: the stored record
// keeps no chain, which is what ELBv2 builds its leaf+chain bundle from.
func TestGetCertificate_AbsentChainReturnsCertificate(t *testing.T) {
svc := setupACMService(t)
leafPEM, keyPEM := genCert(t, "nochain.example.com")
@@ -49,7 +50,43 @@ func TestGetCertificate_OmitsAbsentChain(t *testing.T) {
}, testAccountID)
require.NoError(t, err)
assert.Equal(t, string(leafPEM), aws.StringValue(out.Certificate))
assert.Nil(t, out.CertificateChain)
require.NotNil(t, out.CertificateChain)
assert.Equal(t, string(leafPEM), aws.StringValue(out.CertificateChain))
rec, err := svc.store.GetCertMetadata(t.Context(), aws.StringValue(imported.CertificateArn))
require.NoError(t, err)
require.NotNil(t, rec)
assert.Empty(t, rec.CertificateChain)
}
// A re-import without a chain drops the chain the first import carried, so
// GetCertificate falls back to the new certificate, not the stale chain.
func TestGetCertificate_ReimportWithoutChainReturnsNewCertificate(t *testing.T) {
svc := setupACMService(t)
leafPEM, keyPEM := genCert(t, "reimport.example.com")
chainPEM, _ := genCert(t, "issuer.example.com")
imported, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{
Certificate: leafPEM,
PrivateKey: keyPEM,
CertificateChain: chainPEM,
}, testAccountID)
require.NoError(t, err)
newLeafPEM, newKeyPEM := genCert(t, "reimport.example.com")
_, err = svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{
CertificateArn: imported.CertificateArn,
Certificate: newLeafPEM,
PrivateKey: newKeyPEM,
}, testAccountID)
require.NoError(t, err)
out, err := svc.GetCertificate(context.Background(), &acm.GetCertificateInput{
CertificateArn: imported.CertificateArn,
}, testAccountID)
require.NoError(t, err)
assert.Equal(t, string(newLeafPEM), aws.StringValue(out.Certificate))
assert.Equal(t, string(newLeafPEM), aws.StringValue(out.CertificateChain))
}
func TestGetCertificate_UnknownArnNotFound(t *testing.T) {
spinifex/domains/acm/keyalgorithm_test.go
+149
View File
@@ -0,0 +1,149 @@
package acm
import (
"context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"math/big"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/acm"
"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// genRSACert returns a self-signed RSA leaf with only a CN, the shape of the
// certificate the AWS conformance audit imported.
func genRSACert(t *testing.T, cn string, bits int) (certPEM, keyPEM []byte) {
t.Helper()
key, err := rsa.GenerateKey(rand.Reader, bits)
require.NoError(t, err)
tmpl := &x509.Certificate{
SerialNumber: big.NewInt(time.Now().UnixNano()),
Subject: pkix.Name{CommonName: cn},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(24 * time.Hour),
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
require.NoError(t, err)
certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
keyPEM = pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
return certPEM, keyPEM
}
// describeAndListKeyAlgorithm returns the KeyAlgorithm that DescribeCertificate
// and the ListCertificates summary report for certArn.
func describeAndListKeyAlgorithm(t *testing.T, svc *ACMServiceImpl, certArn string) (described, listed string) {
t.Helper()
desc, err := svc.DescribeCertificate(context.Background(), &acm.DescribeCertificateInput{
CertificateArn: aws.String(certArn),
}, testAccountID)
require.NoError(t, err)
list, err := svc.ListCertificates(context.Background(), &acm.ListCertificatesInput{}, testAccountID)
require.NoError(t, err)
for _, s := range list.CertificateSummaryList {
if aws.StringValue(s.CertificateArn) == certArn {
return aws.StringValue(desc.Certificate.KeyAlgorithm), aws.StringValue(s.KeyAlgorithm)
}
}
t.Fatalf("ListCertificates did not return %s", certArn)
return "", ""
}
// AWS returns RSA-2048 with a hyphen from both Describe and List, although the
// API enum (and the ListCertificates keyTypes filter) spells it RSA_2048.
func TestImportCertificate_RSAKeyAlgorithmIsHyphenated(t *testing.T) {
svc := setupACMService(t)
certPEM, keyPEM := genRSACert(t, "rsa.example.com", 2048)
out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{
Certificate: certPEM,
PrivateKey: keyPEM,
}, testAccountID)
require.NoError(t, err)
described, listed := describeAndListKeyAlgorithm(t, svc, aws.StringValue(out.CertificateArn))
assert.Equal(t, "RSA-2048", described)
assert.Equal(t, "RSA-2048", listed)
// The read path normalises, so check the stored spelling separately.
raw, _, err := svc.store.store.Get(t.Context(), certKey(aws.StringValue(out.CertificateArn)))
require.NoError(t, err)
assert.Equal(t, "RSA-2048", raw.KeyAlgorithm)
}
// Records written before the spelling fix hold RSA_<bits> or EC_<Go curve>;
// they must read back in the AWS spelling without a migration.
func TestKeyAlgorithm_StoredUnderscoreSpellingReadsBackHyphenated(t *testing.T) {
cases := []struct {
stored, want string
}{
{stored: "RSA_2048", want: "RSA-2048"},
{stored: "RSA_4096", want: "RSA-4096"},
{stored: "RSA-3072", want: "RSA-3072"},
{stored: "EC_P-256", want: "EC-prime256v1"},
}
svc := setupACMService(t)
for i, tc := range cases {
t.Run(tc.stored, func(t *testing.T) {
certArn := "arn:aws:acm:ap-southeast-2:" + testAccountID + ":certificate/legacy-" + string(rune('a'+i))
require.NoError(t, svc.store.PutCert(t.Context(), &CertRecord{
CertificateArn: certArn,
AccountID: testAccountID,
DomainName: "legacy.example.com",
KeyAlgorithm: tc.stored,
}))
described, listed := describeAndListKeyAlgorithm(t, svc, certArn)
assert.Equal(t, tc.want, described)
assert.Equal(t, tc.want, listed)
})
}
}
// AWS refuses a re-import whose key type differs from the current one, naming
// both in the API enum spelling.
func TestImportCertificate_ReimportWithDifferentKeyTypeRejected(t *testing.T) {
svc := setupACMService(t)
c1, k1 := genRSACert(t, "k.example.com", 2048)
out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{Certificate: c1, PrivateKey: k1}, testAccountID)
require.NoError(t, err)
c2, k2 := genCert(t, "k.example.com", "k.example.com")
_, err = svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{
CertificateArn: out.CertificateArn, Certificate: c2, PrivateKey: k2,
}, testAccountID)
code, msg, ok := awserrors.ResolveErrorDetail(err)
require.True(t, ok)
assert.Equal(t, awserrors.ErrorValidationException, code)
assert.Equal(t, "New certificate has a key of EC_prime256v1 which is different from RSA_2048 in the current certificate.", msg)
}
func TestImportCertificate_SerialZeroRejected(t *testing.T) {
svc := setupACMService(t)
key, err := rsa.GenerateKey(rand.Reader, 2048)
require.NoError(t, err)
tmpl := &x509.Certificate{
SerialNumber: big.NewInt(0),
Subject: pkix.Name{CommonName: "zero.example.com"},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(24 * time.Hour),
}
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
require.NoError(t, err)
_, err = svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{
Certificate: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}),
PrivateKey: pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}),
}, testAccountID)
code, msg, ok := awserrors.ResolveErrorDetail(err)
require.True(t, ok)
assert.Equal(t, awserrors.ErrorValidationException, code)
assert.Equal(t, "The serial number in the certificate is not supported by ACM.", msg)
}
spinifex/domains/acm/service_impl.go
+96 -22
View File
@@ -1,3 +1,6 @@
// Package acm is the daemon-side ACM service: certificate records in
// JetStream KV with sealed private keys, issuance from the tenant private CA,
// and the worker that renews those certificates before they expire.
package acm
import (
@@ -11,6 +14,7 @@ import (
"fmt"
"log/slog"
"maps"
"regexp"
"strings"
"time"
"uuid"
@@ -28,8 +32,16 @@ const (
certStatusIssued = "ISSUED"
certTypeImported = "IMPORTED"
// domainNamePattern is AWS's DomainName constraint, quoted verbatim in the
// ValidationException message.
domainNamePattern = `(\*\.)?(((?!-)[A-Za-z0-9-]{0,62}[A-Za-z0-9])\.)+((?!-)[A-Za-z0-9-]{1,62}[A-Za-z0-9])`
)
// domainNameRE is domainNamePattern with its (?!-) lookaheads rewritten, since
// Go regexp has none.
var domainNameRE = regexp.MustCompile(`^(\*\.)?([A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z0-9][A-Za-z0-9-]{0,61}[A-Za-z0-9]$`)
// CertAuthority issues leaf certificates from the tenant private CA and
// authorizes domains against the CA's own x509 name constraints. Satisfied by
// *TenantCA (privateca.go), which is implemented separately;
@@ -207,9 +219,15 @@ func (s *ACMServiceImpl) ImportCertificate(ctx context.Context, input *acm.Impor
if err != nil {
return nil, errors.New(awserrors.ErrorInvalidParameter)
}
if leaf.SerialNumber.Sign() == 0 {
return nil, awserrors.Errorf(awserrors.ErrorValidationException, "The serial number in the certificate is not supported by ACM.")
}
certArn := aws.StringValue(input.CertificateArn)
var inUseBy []string
now := time.Now().UTC()
createdAt := now
tags := tagsToMap(input.Tags)
reimport := certArn != ""
if certArn == "" {
certArn = s.mintCertificateArn(accountID)
@@ -226,9 +244,22 @@ func (s *ACMServiceImpl) ImportCertificate(ctx context.Context, input *acm.Impor
if existing == nil || existing.AccountID != accountID {
return nil, errors.New(awserrors.ErrorResourceNotFound)
}
// AWS refuses tags on re-import, once the ARN resolves. The stored tags
// carry forward below, so a plain re-import does not clear them.
if len(input.Tags) > 0 {
return nil, awserrors.Errorf(awserrors.ErrorValidationException, "Tagging is not permitted on re-import.")
}
// AWS names both keys in its API enum spelling (RSA_2048).
if newAlg := keyAlgorithm(leaf); newAlg != existing.KeyAlgorithm {
return nil, awserrors.Errorf(awserrors.ErrorValidationException,
"New certificate has a key of %s which is different from %s in the current certificate.",
strings.Replace(newAlg, "-", "_", 1), strings.Replace(existing.KeyAlgorithm, "-", "_", 1))
}
// Carry the InUseBy index forward — new material under the same ARN
// must not silently drop the load balancers that reference it.
inUseBy = existing.InUseBy
createdAt = importedCreatedAt(existing)
tags = existing.Tags
}
rec := &CertRecord{
@@ -245,8 +276,9 @@ func (s *ACMServiceImpl) ImportCertificate(ctx context.Context, input *acm.Impor
KeyAlgorithm: keyAlgorithm(leaf),
NotBefore: leaf.NotBefore,
NotAfter: leaf.NotAfter,
ImportedAt: time.Now().UTC(),
Tags: tagsToMap(input.Tags),
CreatedAt: createdAt,
ImportedAt: now,
Tags: tags,
InUseBy: inUseBy,
Type: certTypeImported,
Status: certStatusIssued,
@@ -285,6 +317,17 @@ func (s *ACMServiceImpl) RequestCertificate(ctx context.Context, input *acm.Requ
return nil, errors.New(awserrors.ErrorInvalidParameter)
}
domain := aws.StringValue(input.DomainName)
// Before the mode is derived, so a malformed domain is a ValidationException
// in every mode rather than whatever the mode's own checks answer.
if len(domain) > 253 {
return nil, awserrors.Errorf(awserrors.ErrorValidationException,
"1 validation error detected: Value of the input at 'domainName' failed to satisfy constraint: Member must have length less than or equal to 253")
}
if !domainNameRE.MatchString(domain) {
return nil, awserrors.Errorf(awserrors.ErrorValidationException,
"1 validation error detected: Value of the input at 'domainName' failed to satisfy constraint: Member must satisfy regular expression pattern: %s",
domainNamePattern)
}
sans := aws.StringValueSlice(input.SubjectAlternativeNames)
allDomains := uniqueDomains(domain, sans)
@@ -494,12 +537,17 @@ func (s *ACMServiceImpl) GetCertificate(ctx context.Context, input *acm.GetCerti
if rec.Certificate == "" {
return nil, errors.New(awserrors.ErrorACMRequestInProgress)
}
out := &acm.GetCertificateOutput{Certificate: aws.String(rec.Certificate)}
// Left absent rather than empty for a self-signed leaf with no chain.
if rec.CertificateChain != "" {
out.CertificateChain = aws.String(rec.CertificateChain)
// AWS answers a certificate imported without a chain with the certificate
// itself as the chain. Only the response does this; the stored record keeps
// no chain, so ELBv2's leaf+chain bundle does not repeat the leaf.
chain := rec.CertificateChain
if chain == "" {
chain = rec.Certificate
}
return out, nil
return &acm.GetCertificateOutput{
Certificate: aws.String(rec.Certificate),
CertificateChain: aws.String(chain),
}, nil
}
// ListCertificates returns summaries for every cert owned by accountID.
@@ -515,17 +563,20 @@ func (s *ACMServiceImpl) ListCertificates(ctx context.Context, input *acm.ListCe
}
summaries := make([]*acm.CertificateSummary, 0, len(recs))
for _, rec := range recs {
summaries = append(summaries, &acm.CertificateSummary{
CertificateArn: aws.String(rec.CertificateArn),
DomainName: aws.String(rec.DomainName),
Status: aws.String(certStatusOrDefault(rec)),
Type: aws.String(certTypeOrDefault(rec)),
KeyAlgorithm: aws.String(rec.KeyAlgorithm),
NotBefore: timePtr(rec.NotBefore),
NotAfter: timePtr(rec.NotAfter),
ImportedAt: timePtr(rec.ImportedAt),
InUse: aws.Bool(len(rec.InUseBy) > 0),
})
sum := &acm.CertificateSummary{
CertificateArn: aws.String(rec.CertificateArn),
DomainName: aws.String(rec.DomainName),
Status: aws.String(certStatusOrDefault(rec)),
Type: aws.String(certTypeOrDefault(rec)),
KeyAlgorithm: aws.String(rec.KeyAlgorithm),
NotBefore: timePtr(rec.NotBefore),
NotAfter: timePtr(rec.NotAfter),
ImportedAt: timePtr(rec.ImportedAt),
InUse: aws.Bool(len(rec.InUseBy) > 0),
RenewalEligibility: aws.String(renewalEligibilityOrDefault(rec)),
}
applyLeafSummary(sum, rec)
summaries = append(summaries, sum)
}
return &acm.ListCertificatesOutput{CertificateSummaryList: summaries}, nil
}
@@ -617,6 +668,7 @@ func recordToDetail(rec *CertRecord) *acm.CertificateDetail {
}
detail.RenewalSummary = summary
}
applyLeafDetail(detail, rec)
return detail
}
@@ -764,15 +816,37 @@ func leafDomain(leaf *x509.Certificate) string {
return ""
}
// keyAlgorithm maps the leaf public key to an ACM-style algorithm string
// (RSA_2048, EC_prime256v1, ...).
// ecCurveNames maps Go's curve names to the OpenSSL names ACM's KeyAlgorithm uses.
var ecCurveNames = map[string]string{"P-256": "prime256v1", "P-384": "secp384r1", "P-521": "secp521r1"}
// keyAlgorithm maps the leaf public key to the KeyAlgorithm ACM returns:
// RSA-<bits> or EC-<curve>, hyphenated unlike the RSA_2048 API enum.
func keyAlgorithm(leaf *x509.Certificate) string {
switch pub := leaf.PublicKey.(type) {
case *rsa.PublicKey:
return fmt.Sprintf("RSA_%d", pub.N.BitLen())
return fmt.Sprintf("RSA-%d", pub.N.BitLen())
case *ecdsa.PublicKey:
return "EC_" + pub.Curve.Params().Name
return ecKeyAlgorithm(pub.Curve.Params().Name)
default:
return "UNKNOWN"
}
}
func ecKeyAlgorithm(goCurve string) string {
if name, ok := ecCurveNames[goCurve]; ok {
return "EC-" + name
}
return "EC-" + goCurve
}
// normaliseKeyAlgorithm rewrites the RSA_<bits> and EC_<Go curve> spellings
// stored by earlier builds to the ones keyAlgorithm now writes.
func normaliseKeyAlgorithm(alg string) string {
if bits, ok := strings.CutPrefix(alg, "RSA_"); ok {
return "RSA-" + bits
}
if curve, ok := strings.CutPrefix(alg, "EC_"); ok {
return ecKeyAlgorithm(curve)
}
return alg
}
spinifex/domains/acm/service_impl_test.go
+1 -1
View File
@@ -87,7 +87,7 @@ func TestImportCertificate_MintsArnAndParsesLeaf(t *testing.T) {
assert.Equal(t, "example.com", aws.StringValue(d.DomainName))
assert.Equal(t, certStatusIssued, aws.StringValue(d.Status))
assert.Equal(t, certTypeImported, aws.StringValue(d.Type))
assert.Equal(t, "EC_P-256", aws.StringValue(d.KeyAlgorithm))
assert.Equal(t, "EC-prime256v1", aws.StringValue(d.KeyAlgorithm))
assert.ElementsMatch(t, []string{"example.com", "www.example.com"}, aws.StringValueSlice(d.SubjectAlternativeNames))
require.NotNil(t, d.NotAfter)
}
spinifex/domains/acm/store.go
+6
View File
@@ -111,6 +111,10 @@ type CertRecord struct {
// CertificateDetail.InUseBy field.
InUseBy []string `json:"in_use_by,omitempty"`
// CreatedAt is the first import and survives re-imports, which move
// ImportedAt. Zero on records stored before it was kept.
CreatedAt time.Time `json:"created_at"`
// Type is the ACM certificate type: AMAZON_ISSUED, PRIVATE or IMPORTED.
Type string `json:"type,omitempty"`
// Status is the ACM certificate status (PENDING_VALIDATION, ISSUED,
@@ -273,6 +277,7 @@ func (s *Store) getCert(ctx context.Context, certArn string, decrypt bool) (*Cer
if err != nil {
return nil, err
}
rec.KeyAlgorithm = normaliseKeyAlgorithm(rec.KeyAlgorithm)
if !decrypt {
rec.PrivateKey = ""
return rec, nil
@@ -410,6 +415,7 @@ func (s *Store) listCerts(ctx context.Context, keep func(*CertRecord) bool, decr
if err := json.Unmarshal(entry.Value(), &rec); err != nil {
continue
}
rec.KeyAlgorithm = normaliseKeyAlgorithm(rec.KeyAlgorithm)
if !keep(&rec) {
continue
}
spinifex/domains/acm/tags_test.go
+81
View File
@@ -108,3 +108,84 @@ func TestTagOps_CrossAccountHidden(t *testing.T) {
}, "000000000002")
assert.Equal(t, awserrors.ErrorResourceNotFound, err.Error())
}
func listTagMap(t *testing.T, svc *ACMServiceImpl, arn string) map[string]string {
t.Helper()
out, err := svc.ListTagsForCertificate(context.Background(), &acm.ListTagsForCertificateInput{
CertificateArn: aws.String(arn),
}, testAccountID)
require.NoError(t, err)
got := map[string]string{}
for _, tg := range out.Tags {
got[aws.StringValue(tg.Key)] = aws.StringValue(tg.Value)
}
return got
}
// AWS answers a re-import carrying Tags with ValidationException and leaves
// the stored certificate, material and tags alike, untouched.
func TestImportCertificate_ReimportWithTagsRejected(t *testing.T) {
svc := setupACMService(t)
arn := importTagged(t, svc, &acm.Tag{Key: aws.String("Name"), Value: aws.String("ingress")})
c2, k2 := genCert(t, "rotated.example.com", "rotated.example.com")
_, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{
Certificate: c2,
PrivateKey: k2,
CertificateArn: aws.String(arn),
Tags: []*acm.Tag{{Key: aws.String("env"), Value: aws.String("prod")}},
}, testAccountID)
require.Error(t, err)
code, message, ok := awserrors.ResolveErrorDetail(err)
require.True(t, ok)
assert.Equal(t, awserrors.ErrorValidationException, code)
assert.Equal(t, "Tagging is not permitted on re-import.", message)
assert.Equal(t, 400, awserrors.ErrorLookup[code].HTTPCode)
desc, err := svc.DescribeCertificate(context.Background(), &acm.DescribeCertificateInput{CertificateArn: aws.String(arn)}, testAccountID)
require.NoError(t, err)
assert.Equal(t, "tags.example.com", aws.StringValue(desc.Certificate.DomainName), "a refused re-import must not replace the material")
assert.Equal(t, map[string]string{"Name": "ingress"}, listTagMap(t, svc, arn))
}
// A re-import without Tags keeps the tags already on the certificate rather
// than clearing them, as InUseBy is kept.
func TestImportCertificate_ReimportKeepsStoredTags(t *testing.T) {
svc := setupACMService(t)
arn := importTagged(t, svc, &acm.Tag{Key: aws.String("Name"), Value: aws.String("ingress")})
_, err := svc.AddTagsToCertificate(context.Background(), &acm.AddTagsToCertificateInput{
CertificateArn: aws.String(arn),
Tags: []*acm.Tag{{Key: aws.String("env"), Value: aws.String("dev")}},
}, testAccountID)
require.NoError(t, err)
c2, k2 := genCert(t, "rotated.example.com", "rotated.example.com")
out, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{
Certificate: c2,
PrivateKey: k2,
CertificateArn: aws.String(arn),
}, testAccountID)
require.NoError(t, err)
assert.Equal(t, arn, aws.StringValue(out.CertificateArn))
desc, err := svc.DescribeCertificate(context.Background(), &acm.DescribeCertificateInput{CertificateArn: aws.String(arn)}, testAccountID)
require.NoError(t, err)
assert.Equal(t, "rotated.example.com", aws.StringValue(desc.Certificate.DomainName), "material must be replaced")
assert.Equal(t, map[string]string{"Name": "ingress", "env": "dev"}, listTagMap(t, svc, arn))
}
// AWS resolves the ARN before refusing the tags, so an unknown ARN with tags is
// a ResourceNotFoundException, not the tagging ValidationException.
func TestImportCertificate_ReimportWithTagsUnknownArnIsNotFound(t *testing.T) {
svc := setupACMService(t)
c, k := genCert(t, "x.example.com", "x.example.com")
_, err := svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{
Certificate: c,
PrivateKey: k,
CertificateArn: aws.String(svc.mintCertificateArn(testAccountID)),
Tags: []*acm.Tag{{Key: aws.String("a"), Value: aws.String("b")}},
}, testAccountID)
require.Error(t, err)
assert.Equal(t, awserrors.ErrorResourceNotFound, err.Error())
}
spinifex/domains/acm/tenant_ca_paths.go
+2
View File
@@ -24,6 +24,8 @@ func TenantCACertPath(configDir string) string {
return filepath.Join(configDir, tenantCACertFilename)
}
// TenantCAKeyPath returns the tenant private CA key path under configDir; see
// TenantCACertPath.
func TenantCAKeyPath(configDir string) string {
return filepath.Join(configDir, tenantCAKeyFilename)
}
spinifex/domains/dns/bootstrap_test.go
+16 -4
View File
@@ -69,18 +69,25 @@ secret_key = "READONLY"
// flakyS3 refuses the first failUntil requests (simulating predastore not yet
// listening) then behaves like fakeS3, to exercise the bootstrap retry loop.
func flakyS3(t *testing.T, bucket string, failUntil int) (endpoint string, objects map[string]string) {
// sdkRetried reports whether the SDK itself ever resent a request.
func flakyS3(t *testing.T, bucket string, failUntil int) (endpoint string, objects map[string]string, sdkRetried func() bool) {
t.Helper()
var mu sync.Mutex
objects = map[string]string{}
var calls int
var retried bool
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
defer mu.Unlock()
calls++
if !strings.HasPrefix(r.Header.Get("Amz-Sdk-Request"), "attempt=1;") {
retried = true
}
// 501 rather than 503: the SDK retries 5xx-unavailable itself with
// seconds of jitter, which would hide the bootstrap loop under test.
if calls <= failUntil {
http.Error(w, "unavailable", http.StatusServiceUnavailable)
http.Error(w, "unavailable", http.StatusNotImplemented)
return
}
key := strings.TrimPrefix(r.URL.Path, "/"+bucket+"/")
@@ -100,7 +107,11 @@ func flakyS3(t *testing.T, bucket string, failUntil int) (endpoint string, objec
}
}))
t.Cleanup(srv.Close)
return srv.URL, objects
return srv.URL, objects, func() bool {
mu.Lock()
defer mu.Unlock()
return retried
}
}
// assertZoneNameservers verifies the exact apex NS and glue topology rendered
@@ -182,7 +193,7 @@ func TestBootstrapBaseZoneRetries(t *testing.T) {
t.Cleanup(func() { bootstrapRetryDelay = prev })
// Fail the first two S3 requests, then succeed — the retry must seed.
endpoint, objects := flakyS3(t, "northstar", 2)
endpoint, objects, sdkRetried := flakyS3(t, "northstar", 2)
tomlBody := fmt.Sprintf(`listen = "0.0.0.0:5300"
default_domain = "spx3.net"
[s3]
@@ -208,6 +219,7 @@ secret_key = "READONLY"
require.NoError(t, BootstrapBaseZone(configPath, cluster))
require.Contains(t, objects, "spx3.net.toml")
assert.False(t, sdkRetried(), "failures must be retried by the bootstrap loop, not the SDK")
}
// TestBootstrapBaseZoneSeedsTheServiceEndpointSuffix covers the third zone:
spinifex/domains/ec2/account/service.go
+3
View File
@@ -1,3 +1,6 @@
// Package account implements the EC2 account-settings actions (EBS
// encryption by default, serial console access): the AccountSettingsService
// contract, its daemon-side JetStream KV implementation and its NATS client.
package account
import (
spinifex/domains/ec2/awsapi/account/DescribeAccountAttributes.go
+3
View File
@@ -1,3 +1,6 @@
// Package account implements the EC2 account-level actions: it
// answers DescribeAccountAttributes itself and forwards the EBS-encryption and
// serial-console defaults to the account settings service over NATS.
package account
import (
spinifex/domains/ec2/awsapi/account/DisableEbsEncryptionByDefault.go
+4
View File
@@ -10,6 +10,8 @@ import (
"github.com/nats-io/nats.go"
)
// ValidateDisableEbsEncryptionByDefaultInput rejects a nil input with InvalidParameterValue; the
// action takes no parameters.
func ValidateDisableEbsEncryptionByDefaultInput(input *ec2.DisableEbsEncryptionByDefaultInput) error {
if input == nil {
return errors.New(awserrors.ErrorInvalidParameterValue)
@@ -17,6 +19,8 @@ func ValidateDisableEbsEncryptionByDefaultInput(input *ec2.DisableEbsEncryptionB
return nil
}
// DisableEbsEncryptionByDefault implements the EC2 DisableEbsEncryptionByDefault action, turning
// off default encryption of new EBS volumes for accountID via the NATS account settings service.
func DisableEbsEncryptionByDefault(ctx context.Context, input *ec2.DisableEbsEncryptionByDefaultInput, natsConn *nats.Conn, accountID string) (ec2.DisableEbsEncryptionByDefaultOutput, error) {
var output ec2.DisableEbsEncryptionByDefaultOutput
spinifex/domains/ec2/awsapi/account/DisableSerialConsoleAccess.go
+4
View File
@@ -10,6 +10,8 @@ import (
"github.com/nats-io/nats.go"
)
// ValidateDisableSerialConsoleAccessInput rejects a nil input with InvalidParameterValue; the
// action takes no parameters.
func ValidateDisableSerialConsoleAccessInput(input *ec2.DisableSerialConsoleAccessInput) error {
if input == nil {
return errors.New(awserrors.ErrorInvalidParameterValue)
@@ -17,6 +19,8 @@ func ValidateDisableSerialConsoleAccessInput(input *ec2.DisableSerialConsoleAcce
return nil
}
// DisableSerialConsoleAccess implements the EC2 DisableSerialConsoleAccess action, turning off
// serial console access for accountID via the NATS account settings service.
func DisableSerialConsoleAccess(ctx context.Context, input *ec2.DisableSerialConsoleAccessInput, natsConn *nats.Conn, accountID string) (ec2.DisableSerialConsoleAccessOutput, error) {
var output ec2.DisableSerialConsoleAccessOutput
spinifex/domains/ec2/awsapi/account/EnableEbsEncryptionByDefault.go
+4
View File
@@ -10,6 +10,8 @@ import (
"github.com/nats-io/nats.go"
)
// ValidateEnableEbsEncryptionByDefaultInput rejects a nil input with InvalidParameterValue; the
// action takes no parameters.
func ValidateEnableEbsEncryptionByDefaultInput(input *ec2.EnableEbsEncryptionByDefaultInput) error {
if input == nil {
return errors.New(awserrors.ErrorInvalidParameterValue)
@@ -17,6 +19,8 @@ func ValidateEnableEbsEncryptionByDefaultInput(input *ec2.EnableEbsEncryptionByD
return nil
}
// EnableEbsEncryptionByDefault implements the EC2 EnableEbsEncryptionByDefault action, making new
// EBS volumes in accountID encrypted by default via the NATS account settings service.
func EnableEbsEncryptionByDefault(ctx context.Context, input *ec2.EnableEbsEncryptionByDefaultInput, natsConn *nats.Conn, accountID string) (ec2.EnableEbsEncryptionByDefaultOutput, error) {
var output ec2.EnableEbsEncryptionByDefaultOutput
spinifex/domains/ec2/awsapi/account/EnableSerialConsoleAccess.go
+4
View File
@@ -10,6 +10,8 @@ import (
"github.com/nats-io/nats.go"
)
// ValidateEnableSerialConsoleAccessInput rejects a nil input with InvalidParameterValue; the
// action takes no parameters.
func ValidateEnableSerialConsoleAccessInput(input *ec2.EnableSerialConsoleAccessInput) error {
if input == nil {
return errors.New(awserrors.ErrorInvalidParameterValue)
@@ -17,6 +19,8 @@ func ValidateEnableSerialConsoleAccessInput(input *ec2.EnableSerialConsoleAccess
return nil
}
// EnableSerialConsoleAccess implements the EC2 EnableSerialConsoleAccess action, turning on
// serial console access for accountID via the NATS account settings service.
func EnableSerialConsoleAccess(ctx context.Context, input *ec2.EnableSerialConsoleAccessInput, natsConn *nats.Conn, accountID string) (ec2.EnableSerialConsoleAccessOutput, error) {
var output ec2.EnableSerialConsoleAccessOutput
spinifex/domains/ec2/awsapi/account/GetEbsEncryptionByDefault.go
+4
View File
@@ -10,6 +10,8 @@ import (
"github.com/nats-io/nats.go"
)
// ValidateGetEbsEncryptionByDefaultInput rejects a nil input with InvalidParameterValue; the
// action takes no parameters.
func ValidateGetEbsEncryptionByDefaultInput(input *ec2.GetEbsEncryptionByDefaultInput) error {
if input == nil {
return errors.New(awserrors.ErrorInvalidParameterValue)
@@ -17,6 +19,8 @@ func ValidateGetEbsEncryptionByDefaultInput(input *ec2.GetEbsEncryptionByDefault
return nil
}
// GetEbsEncryptionByDefault implements the EC2 GetEbsEncryptionByDefault action, reporting
// whether new EBS volumes in accountID are encrypted by default.
func GetEbsEncryptionByDefault(ctx context.Context, input *ec2.GetEbsEncryptionByDefaultInput, natsConn *nats.Conn, accountID string) (ec2.GetEbsEncryptionByDefaultOutput, error) {
var output ec2.GetEbsEncryptionByDefaultOutput
spinifex/domains/ec2/awsapi/account/GetSerialConsoleAccessStatus.go
+4
View File
@@ -10,6 +10,8 @@ import (
"github.com/nats-io/nats.go"
)
// ValidateGetSerialConsoleAccessStatusInput rejects a nil input with InvalidParameterValue; the
// action takes no parameters.
func ValidateGetSerialConsoleAccessStatusInput(input *ec2.GetSerialConsoleAccessStatusInput) error {
if input == nil {
return errors.New(awserrors.ErrorInvalidParameterValue)
@@ -17,6 +19,8 @@ func ValidateGetSerialConsoleAccessStatusInput(input *ec2.GetSerialConsoleAccess
return nil
}
// GetSerialConsoleAccessStatus implements the EC2 GetSerialConsoleAccessStatus action, reporting
// whether serial console access is enabled for accountID.
func GetSerialConsoleAccessStatus(ctx context.Context, input *ec2.GetSerialConsoleAccessStatusInput, natsConn *nats.Conn, accountID string) (ec2.GetSerialConsoleAccessStatusOutput, error) {
var output ec2.GetSerialConsoleAccessStatusOutput

Some files were not shown because too many files have changed in this diff Show More