fix: write MariaDB's startup spelling for time_zone into the option file

mariadbd has no time_zone startup option — the name only exists for SET — so
the resolved parameter group aborted the server while it parsed defaults, before
it opened the datadir. No pidfile was ever written and the reconciler failed
every instance at the 20-minute engine-health deadline.

ParameterSpec gains an option-file spelling, used only when rendering the
include. time_zone is the one setting that needs it, as default_time_zone, which
resolves @@global.time_zone to the same value. The customer-facing name is
unchanged: the API still reports time_zone and a live apply still issues
SET GLOBAL time_zone. Names read back out of a generated file return to the
catalog's namespace first, so a startup spelling cannot read as an unknown name
and report a pending restart nothing could clear.

setup.sh now feeds the catalog's resolved defaults to the image's own mariadbd
at build time, so a name the server refuses fails the build rather than every
instance built from it.

.github/dependabot.yml

@@ -5,7 +5,7 @@ updates:
     directory: /
     schedule:
       interval: weekly
-      day: monday
+      day: sunday
       timezone: Australia/Sydney
     cooldown:
       default-days: 3
@@ -19,7 +19,7 @@ updates:
     directory: /
     schedule:
       interval: weekly
-      day: monday
+      day: sunday
       timezone: Australia/Sydney
     cooldown:
       default-days: 3
@@ -38,7 +38,7 @@ updates:
     directory: /spinifex/services/spinifexui/frontend
     schedule:
       interval: weekly
-      day: monday
+      day: sunday
       timezone: Australia/Sydney
     cooldown:
       default-days: 3

.github/workflows/e2e-nightly.yml

@@ -46,7 +46,7 @@ jobs:
           - { cell: 19, install: binary, topo: single,     net: nat,    host: debian-13,    arch: x86_64, runner: ci-single, extra: "nat-single" }
 
     runs-on: [ self-hosted, "${{ matrix.runner }}" ]
-    timeout-minutes: 35
+    timeout-minutes: ${{ matrix.extra == 'tofu-examples' && 45 || 35 }}
     env:
       TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}
@@ -251,6 +251,18 @@ jobs:
                if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
              done; exit \$FAIL"
 
+      - name: Import RDS PostgreSQL AMI
+        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples'
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
+            "spx admin images import --name spinifex-rds-postgres"
+
       - name: Run Tofu Workbooks
         if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples'
         working-directory: mulga/scripts/tofu-cluster

.github/workflows/e2e.yml

@@ -52,7 +52,7 @@ jobs:
     # inputs.suites.
     if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-iam') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-ecs') || contains(github.event.inputs.suites, 'e2e-storagegrowth') || contains(github.event.inputs.suites, 'e2e-partialblock') || contains(github.event.inputs.suites, 'e2e-rds')
     runs-on: [self-hosted, ci-single]
-    timeout-minutes: 50
+    timeout-minutes: 60
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
@@ -250,53 +250,58 @@ jobs:
           EOF
           ok "kubectl installed on VM"
 
-      # CreateDBInstance boots the DB VM from the rds-postgres AMI, so that image
+      # CreateDBInstance boots the DB VM from an RDS AMI, so both engines' images
       # must be in the VM's image store before the suite runs — same split as the
       # eks-node pair above: build on the runner (needs source + network), import
-      # on the VM (needs predastore/NATS). The suite's psql client needs no step
-      # here: it runs inside a client VM that installs postgresql-client from its
-      # own user-data, because a private-only endpoint is unreachable from the
-      # runner and from the cluster host alike.
-      - name: Build rds-postgres AMI (libguestfs, runner)
+      # on the VM (needs predastore/NATS). The suite's psql and mariadb clients
+      # need no step here: they run inside a client VM that installs them from
+      # its own user-data, because a private-only endpoint is unreachable from
+      # the runner and from the cluster host alike.
+      - name: Build RDS AMIs (libguestfs, runner)
         if: contains(env.SUITES, 'e2e-rds')
         working-directory: spinifex
         run: |
           set -euo pipefail
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          banner "build rds-postgres AMI (libguestfs — no host devices)"
           sudo apt-get update -qq
           sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
           export LIBGUESTFS_BACKEND=direct
           sudo chmod 0644 /boot/vmlinuz-* || true
-          group_open "make build-rds-postgres-image"
-          make build-rds-postgres-image
-          group_close
-          ok "raw: $(du -h /tmp/rds-postgres-image-build/rds-postgres-alpine.raw | cut -f1)"
+          for IMAGE in rds-postgres rds-mariadb; do
+            banner "build ${IMAGE} AMI (libguestfs — no host devices)"
+            group_open "make build-${IMAGE}-image"
+            make "build-${IMAGE}-image"
+            group_close
+            ok "raw: $(du -h "/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" | cut -f1)"
+          done
 
-      - name: Import rds-postgres AMI on VM
+      - name: Import RDS AMIs on VM
         if: contains(env.SUITES, 'e2e-rds')
         run: |
           set -euo pipefail
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
-            spinifex/scripts/images/rds-postgres/manifest.conf)"
-          # SYSTEM_TAG carries three tags, not one: managed-by marks it a system
-          # image and engine/engine-version are what an EngineVersion request
-          # resolves against. --tag is repeatable, so expand rather than pass the
-          # whole string as a single value.
-          TAG_ARGS=""
-          for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
           SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
           VM=tf-user@${{ steps.vm.outputs.wan_ip }}
-          banner "import rds-postgres AMI to VM image store"
-          scp "${SSH_OPTS[@]}" \
-            /tmp/rds-postgres-image-build/rds-postgres-alpine.raw \
-            "$VM":/tmp/rds-postgres-alpine.raw
-          ssh "${SSH_OPTS[@]}" "$VM" \
-            "spx admin images import --file /tmp/rds-postgres-alpine.raw \
-               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
-               --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
-          ok "rds-postgres AMI imported"
+          for IMAGE in rds-postgres rds-mariadb; do
+            eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
+              "spinifex/scripts/images/${IMAGE}/manifest.conf")"
+            # SYSTEM_TAG carries four tags, not one: managed-by marks it a system
+            # image, engine/engine-version are what an EngineVersion request
+            # resolves against, and the contract tag gates the data-volume format
+            # grant. --tag is repeatable, so expand rather than pass the whole
+            # string as a single value.
+            TAG_ARGS=""
+            for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
+            banner "import ${IMAGE} AMI to VM image store"
+            scp "${SSH_OPTS[@]}" \
+              "/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" \
+              "$VM":"/tmp/${IMAGE}-alpine.raw"
+            ssh "${SSH_OPTS[@]}" "$VM" \
+              "spx admin images import --file /tmp/${IMAGE}-alpine.raw \
+                 --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
+                 --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
+            ok "${IMAGE} AMI imported"
+          done
 
       - name: Build E2E test binaries on runner
         working-directory: spinifex/tests/e2e
@@ -383,9 +388,10 @@ jobs:
                echo \"::group::suite \$SUITE\"; \
                printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
+               case \$SUITE in rds) TMO=45m ;; *) TMO=30m ;; esac; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
-                 ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
+                 ./\$SUITE.test -test.v -test.timeout=\$TMO ${TEST_RUN_FLAG} \
                  2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \
                       | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
                rc=\${PIPESTATUS[0]}; \

.github/workflows/publish-system-ami.yml

@@ -1,17 +1,21 @@
-name: Publish EKS + ECS node AMIs
+name: Publish System AMIs
 
 # Builds the system AMIs (libguestfs, on the runner) and publishes each
 # compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket that backs
-# iso.mulgadc.com/system-ami/. Four images ship:
+# iso.mulgadc.com/system-ami/. Six images ship:
 #   spinifex-eks-node       Alpine EKS K3s node (server|agent, role at boot)
 #   spinifex-ecs-node       Alpine ECS container instance
 #   spinifex-eks-node-gpu   Ubuntu EKS K3s GPU worker (NVIDIA driver + toolkit)
 #   spinifex-ecs-node-gpu   Ubuntu ECS GPU container instance
+#   spinifex-rds-postgres   Alpine RDS PostgreSQL instance
+#   spinifex-rds-mariadb    Alpine RDS MariaDB instance
 # Operators then install them on any cluster with:
 #   spx admin images import --name spinifex-eks-node
 #   spx admin images import --name spinifex-ecs-node
 #   spx admin images import --name spinifex-eks-node-gpu
 #   spx admin images import --name spinifex-ecs-node-gpu
+#   spx admin images import --name spinifex-rds-postgres
+#   spx admin images import --name spinifex-rds-mariadb
 #
 # Manual-only: publishing overwrites the mirror artifact, so it is a deliberate
 # action, not something every push should trigger. The `variant` input scopes
@@ -29,6 +33,7 @@ on:
           - all
           - cpu
           - gpu
+          - rds
       dry_run:
         description: "Build + convert only; skip the R2 upload"
         type: boolean
@@ -56,10 +61,12 @@ jobs:
           set -euo pipefail
           cpu='{"name":"spinifex-eks-node","manifest":"scripts/images/eks-node/manifest.conf"},{"name":"spinifex-ecs-node","manifest":"scripts/images/ecs-agent/manifest.conf"}'
           gpu='{"name":"spinifex-eks-node-gpu","manifest":"scripts/images/eks-node-gpu/manifest.conf"},{"name":"spinifex-ecs-node-gpu","manifest":"scripts/images/ecs-agent-gpu/manifest.conf"}'
+          rds='{"name":"spinifex-rds-postgres","manifest":"scripts/images/rds-postgres/manifest.conf"},{"name":"spinifex-rds-mariadb","manifest":"scripts/images/rds-mariadb/manifest.conf"}'
           case "${VARIANT}" in
             cpu) items="${cpu}" ;;
             gpu) items="${gpu}" ;;
-            *)   items="${cpu},${gpu}" ;;
+            rds) items="${rds}" ;;
+            *)   items="${cpu},${gpu},${rds}" ;;
           esac
           echo "images={\"include\":[${items}]}" >> "$GITHUB_OUTPUT"
 

Makefile

@@ -49,7 +49,7 @@ else
   _SECQ  = 2>&1 | tee
 endif
 
-build: go_build build-installer build-lb-agent
+build: go_build build-installer build-lb-agent generate-aws-model-coverage
 
 # Build spinifex-ui frontend (requires pnpm)
 build-ui:
@@ -111,6 +111,12 @@ build-rds-postgres-image: ## Build the spinifex-rds-postgres AMI (Alpine + Postg
 import-rds-postgres-image: ## Build + register the rds-postgres AMI (requires a running cluster)
 	$(MAKE) build-system-image IMAGE=rds-postgres IMPORT=1
 
+build-rds-mariadb-image: ## Build the spinifex-rds-mariadb AMI (Alpine + MariaDB 11.8 + rds-init; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=rds-mariadb
+
+import-rds-mariadb-image: ## Build + register the rds-mariadb AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=rds-mariadb IMPORT=1
+
 MICROVM_OUT_DIR := build/microvm
 MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
 MICROVM_INPUTS := scripts/build-microvm-image.sh $(MICROVM_OUT_DIR)/init.sh $(MICROVM_OUT_DIR)/inittab bin/lb-agent
@@ -146,9 +152,22 @@ test-harness:
 
 # In-process integration tier: the real gateway router against embedded NATS
 # JetStream, with only the daemon-side NATS subjects stubbed.
+AWS_MODEL_CONFORMANCE_REPORT ?= $(CURDIR)/.cache/aws-model-conformance-report.txt
+AWS_MODEL_CONFORMANCE_MODE ?= fail
+AWS_MODEL_OPERATION_COVERAGE_REPORT ?= $(CURDIR)/docs/aws-model-operation-coverage.md
 test-integration:
 	@echo -e "\n....Running in-process integration tests...."
-	$(_Q)LOG_IGNORE=1 go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
+	@cat $(AWS_MODEL_CONFORMANCE_REPORT)
+	@$(MAKE) --no-print-directory generate-aws-model-coverage
+	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_REPORT)"
+
+generate-aws-model-coverage:
+	@mkdir -p $(dir $(AWS_MODEL_OPERATION_COVERAGE_REPORT))
+	@go run ./cmd/aws-model-coverage > $(AWS_MODEL_OPERATION_COVERAGE_REPORT)
+
+aws-model-coverage: generate-aws-model-coverage
+	@cat $(AWS_MODEL_OPERATION_COVERAGE_REPORT)
 
 # Segscan storage oracle: needs the mulga umbrella repo's scripts/segscan
 # checked out alongside spinifex (see spinifex/testutil/segscanoracle), which
@@ -199,18 +218,37 @@ test-actions:
 	@echo -e "\n....Running action tests...."
 	LOG_IGNORE=1 go test -timeout 60s ./.github/actions/...
 
-# Shell suites + shellcheck for scripts/images/ helpers baked into system
-# images. Kept out of `preflight` (a dedicated CI job gates it on
-# scripts/images/** changes instead) so image-asset churn doesn't run on
-# every Go contributor's commit.
+# Shell suites + shellcheck for scripts/images/ and images/mkosi.profiles/
+# helpers baked into system images. Kept out of `preflight` (a dedicated CI
+# job gates it on scripts/images/**+images/mkosi.profiles/** changes instead)
+# so image-asset churn doesn't run on every Go contributor's commit.
+#
+# images/mkosi.profiles/ ships scripts with no .sh suffix (mkosi's own
+# mkosi.*.chroot lifecycle hooks, and wrapper binaries like vllm-serve under
+# mkosi.extra/) so a plain `-name '*.sh'` glob silently skips exactly the
+# files most worth linting. Sources there are found by shebang instead of
+# name; this also means a profile that ships no scripts at all yields an
+# empty match rather than an error.
 test-images:
 	@echo -e "\n....Running scripts/images/**/*_test.sh...."
 	@for t in $$(find scripts/images -name '*_test.sh' | sort); do \
 		echo "-- $$t"; \
 		bash "$$t" || exit 1; \
 	done
+	@echo -e "\n....Running images/mkosi.profiles/**/*_test.sh...."
+	@for t in $$(find images/mkosi.profiles -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		bash "$$t" || exit 1; \
+	done
 	@echo -e "\n....Running shellcheck over scripts/images/**/*.sh...."
 	shellcheck -S warning $$(find scripts/images -name '*.sh' | sort)
+	@echo -e "\n....Running shellcheck over images/mkosi.profiles/** shell sources...."
+	@srcs="$$(grep -rlIE '^#!/bin/(bash|sh)' images/mkosi.profiles 2>/dev/null | sort)"; \
+	if [ -n "$$srcs" ]; then \
+		shellcheck -S warning $$srcs; \
+	else \
+		echo "  (no shell sources found under images/mkosi.profiles)"; \
+	fi
 	@echo "  test-images ok"
 
 # Check that new/changed code meets coverage threshold (runs tests first)
@@ -228,6 +266,20 @@ bench:
 # `make install-microvm` explicitly the first time.
 deploy: build
 	sudo install -m 755 bin/spx /usr/local/bin/spx
+	@if [ "$${SKIP_ASSET_PREFLIGHT:-}" != "1" ]; then \
+		if ! sudo /usr/local/bin/spx admin preflight; then \
+			echo ""; \
+			echo "[deploy] host-asset preflight failed: this node's helper scripts or sudoers"; \
+			echo "  grants don't match the binary just installed. Remediate with:"; \
+			echo "    scripts/update-nodes.sh   (from the mulga root)"; \
+			echo "  or:"; \
+			echo "    make reinstall"; \
+			echo "  Set SKIP_ASSET_PREFLIGHT=1 to bypass for a deliberate binary-only push."; \
+			exit 1; \
+		fi; \
+	else \
+		echo "[deploy] SKIP_ASSET_PREFLIGHT=1 — skipping host-asset preflight"; \
+	fi
 	@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \
 		$(MAKE) install-microvm; \
 	else \
@@ -342,7 +394,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
+.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

README.md

@@ -5,7 +5,7 @@
 <p align="center">
   <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.26+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
-  <a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/Docs-docs.mulgadc.com-DC2626?style=flat-square" alt="Docs"></a>
+  <a href="https://mulgadc.com"><img src="https://img.shields.io/badge/home-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
 </p>
 
 <p align="center">
@@ -23,22 +23,13 @@
 
 # Spinifex: the open, AWS-compatible cloud you run yourself
 
-Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your
-software already uses (EC2, EBS, S3, VPC, IAM) on infrastructure you control.
-Move workloads off the hyperscalers, run AI at a fraction of the cost, or take
-the cloud to the edge, without rewriting a line. The exact build you ship to AWS,
-Spinifex serves on your own hardware: same APIs, same Terraform, same SDKs.
+Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your software already uses (EC2, EBS, S3, VPC, IAM) on infrastructure you control. Move workloads off the hyperscalers, run AI at a fraction of the cost, or take the cloud to the edge, without rewriting a line. The exact build you ship to AWS, Spinifex serves on your own hardware: same APIs, same Terraform, same SDKs.
 
 ## What is Spinifex?
 
-Spinifex is an open-source infrastructure platform that recreates the AWS service
-surface (EC2, EBS, S3, VPC, IAM) on bare-metal, edge, and on-premises
-environments. Run cloud-native software without a hyperscaler.
+Spinifex is an open-source infrastructure platform that recreates the AWS service surface (EC2, EBS, S3, VPC, IAM) on bare-metal, edge, and on-premises environments. Run cloud-native software without a hyperscaler.
 
-Most AWS alternatives wrap an API and call it a cloud. We rebuilt the engineering
-underneath: distributed object storage with erasure coding, block storage built
-to survive failure, and compute on bare metal. That depth is the reason your
-software runs unchanged, instead of rewritten.
+Most AWS alternatives wrap an API and call it a cloud. We rebuilt the engineering underneath: distributed object storage with erasure coding, block storage built to survive failure, and compute on bare metal. That depth is the reason your software runs unchanged, instead of rewritten.
 
 It's built for teams that need:
 
@@ -59,34 +50,23 @@ From commodity hardware up to unmodified AWS tooling, every layer is replaceable
 
 ## Three ways to deploy
 
-One AWS-compatible surface, three deployment shapes. Pick the one that matches
-your reality; the platform on top is the same.
+One AWS-compatible surface, three deployment shapes. Pick the one that matches your reality; the platform on top is the same.
 
 ### Neocloud
 
-Lift and shift onto a partner Neocloud. Move workloads off the hyperscalers and
-onto our Neocloud partner ecosystem without rewriting them. GPU capacity
-(H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the
-software.
+Lift and shift onto a partner Neocloud. Move workloads off the hyperscalers and onto our Neocloud partner ecosystem without rewriting them. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
 
 ### On-premise
 
-Bring your own hardware and host Spinifex in your own data centre. A real
-multi-node HA cluster integrated with your storage and networking, with full
-control of stack, data, and jurisdiction. A predictable bill, no egress
-surprises, an auditable open-source core.
+Bring your own hardware and host Spinifex in your own data centre. A real multi-node HA cluster integrated with your storage and networking, with full control of stack, data, and jurisdiction. A predictable bill, no egress surprises, an auditable open-source core.
 
 ### Edge
 
-Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels,
-factories, and clinics. Compute next to the data, running through disconnection,
-on hardware you own. The same AWS APIs your software already uses.
+Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels, factories, and clinics. Compute next to the data, running through disconnection, on hardware you own. The same AWS APIs your software already uses.
 
 ## AWS compatibility
 
-Speak the AWS API surface, natively. The AWS SDKs, AWS CLI, and Terraform:
-everything you deploy on AWS deploys on Spinifex unchanged. At the edge,
-on-premise, or on a partner Neocloud.
+Speak the AWS API surface, natively. The AWS SDKs, AWS CLI, and Terraform: everything you deploy on AWS deploys on Spinifex unchanged. At the edge, on-premise, or on a partner Neocloud.
 
 | Service | What it is | Status |
 | --- | --- | --- |
@@ -102,8 +82,7 @@ on-premise, or on a partner Neocloud.
 | **RDS** | Databases | Available |
 | **Bedrock** | AI Deployment | Q3 2026 |
 
-Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land.
-[Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
+Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
 
 ## Core Components
 
@@ -139,11 +118,7 @@ Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs
   <img src=".github/assets/architecture.svg" alt="Spinifex message-driven architecture: standard AWS clients call the AWS Gateway over HTTPS with SigV4, which publishes to a NATS bus answered by stateless ec2/ebs/s3/vpc/iam daemons over local backends." width="900">
 </p>
 
-Every AWS API call is authenticated at the gateway, published to a NATS subject,
-and answered by whichever daemon claims it. Daemons are stateless: scale
-horizontally by starting more. No etcd, no Kubernetes, no external control plane.
-Just systemd units, a NATS cluster, and your hardware. Deep dive:
-**[`docs/DESIGN.md`](docs/DESIGN.md)**.
+Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless: scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Just systemd units, a NATS cluster, and your hardware. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
 
 ## Key Features
 

cmd/aws-model-coverage/main.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/mulgadc/spinifex/internal/awsmodel"
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+)
+
+func main() {
+	dispatch := gateway.AWSOperationInventory()
+	coverages := make([]awsmodel.OperationCoverage, 0, len(awsmodel.Services()))
+	for _, service := range awsmodel.Services() {
+		inventory, ok := dispatch[string(service)]
+		var modelInventory awsmodel.DispatchInventory
+		switch {
+		case ok:
+			modelInventory = awsmodel.DispatchInventory{
+				Registered:  inventory.Registered,
+				Stubbed:     inventory.Stubbed,
+				Unsupported: inventory.Unsupported,
+			}
+		case service == awsmodel.S3:
+			modelInventory = awsmodel.DispatchInventory{
+				Opaque: true,
+				Note:   "Spinifex delegates the S3 REST surface to Predastore, which has no operation-name dispatch table to compare mechanically.",
+			}
+		default:
+			fmt.Fprintf(os.Stderr, "aws-model-coverage: no gateway inventory for %s\n", service)
+			os.Exit(1)
+		}
+
+		coverage, err := awsmodel.CompareOperations(service, modelInventory)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "aws-model-coverage: %v\n", err)
+			os.Exit(1)
+		}
+		coverages = append(coverages, coverage)
+	}
+
+	fmt.Print(awsmodel.RenderCoverageMarkdown(coverages))
+}

cmd/installer/autoinstall/autoinstall.go

@@ -138,9 +138,11 @@ func buildConfig() (*install.Config, error) {
 
 // resolveStorage builds the disk configuration from the kernel cmdline.
 //
-//	SPINIFEX_FS     ext4 (default) or one of the zfs-* topologies
-//	SPINIFEX_DISKS  explicit ordered member list, required for multi-disk pools
-//	SPINIFEX_DISK   single-disk selector, as before
+//	SPINIFEX_FS               ext4 (default) or one of the zfs-* topologies
+//	SPINIFEX_DISKS            explicit ordered member list, required for multi-disk pools
+//	SPINIFEX_DISK             the OS disk, as before
+//	SPINIFEX_DISK_SPINIFEX    ext4 only: dedicated disk for /var/lib/spinifex
+//	SPINIFEX_DISK_PREDASTORE  ext4 only: dedicated disk for /var/lib/spinifex/predastore
 func resolveStorage() (install.DiskConfig, error) {
 	cfg := install.DiskConfig{FS: install.FSExt4}
 	if v := strings.TrimSpace(os.Getenv("SPINIFEX_FS")); v != "" {
@@ -171,6 +173,9 @@ func resolveStorage() (install.DiskConfig, error) {
 		cfg.Disks = []install.Disk{d}
 	}
 
+	if err := applyDiskRoles(&cfg); err != nil {
+		return cfg, err
+	}
 	if cfg.ZFS, err = parseZFSOpts(); err != nil {
 		return cfg, err
 	}
@@ -181,6 +186,51 @@ func resolveStorage() (install.DiskConfig, error) {
 	return cfg, nil
 }
 
+// roleVars maps each optional data role to the variable that assigns it. The OS
+// role is not here: it always comes from SPINIFEX_DISK.
+var roleVars = []struct {
+	role install.DiskRole
+	env  string
+}{
+	{install.RoleSpinifex, "SPINIFEX_DISK_SPINIFEX"},
+	{install.RolePredastore, "SPINIFEX_DISK_PREDASTORE"},
+}
+
+// applyDiskRoles attaches dedicated data drives to an ext4 install. Roles are
+// always set on ext4, even with no extra drives, so the OS assignment is
+// explicit rather than implied by position.
+func applyDiskRoles(cfg *install.DiskConfig) error {
+	if cfg.FS.IsZFS() {
+		for _, rv := range roleVars {
+			if strings.TrimSpace(os.Getenv(rv.env)) != "" {
+				return fmt.Errorf("%s applies to ext4 only — %s spans every member already", rv.env, cfg.FS.Label())
+			}
+		}
+		return nil
+	}
+	if len(cfg.Disks) != 1 {
+		return fmt.Errorf("ext4 takes one OS disk, %d given — set SPINIFEX_DISK, and SPINIFEX_DISK_SPINIFEX or SPINIFEX_DISK_PREDASTORE for dedicated drives",
+			len(cfg.Disks))
+	}
+
+	roles := []install.RoleMount{{Role: install.RoleOS, Disk: cfg.Disks[0]}}
+	for _, rv := range roleVars {
+		spec := strings.TrimSpace(os.Getenv(rv.env))
+		if spec == "" {
+			continue
+		}
+		// Resolved with the same fail-closed selector as the OS disk, so an
+		// ambiguous value lists the candidates rather than picking one.
+		d, err := resolveDisk(spec)
+		if err != nil {
+			return fmt.Errorf("%s: %w", rv.env, err)
+		}
+		roles = append(roles, install.RoleMount{Role: rv.role, Disk: d})
+	}
+	*cfg = cfg.WithRoles(roles)
+	return nil
+}
+
 // parseZFSOpts reads the advanced pool tunables. Every one is optional; unset
 // values are computed from the hardware at install time.
 func parseZFSOpts() (install.ZFSOpts, error) {

cmd/installer/autoinstall/autoinstall_test.go

@@ -436,10 +436,53 @@ func TestResolveStorage(t *testing.T) {
 			wantErr: `"vdz" is not an available disk`,
 		},
 		{
-			name:    "member list that fails validation",
+			name:    "member list is for pools, not ext4 roles",
 			disks:   four,
 			env:     map[string]string{"SPINIFEX_DISKS": "vdb,vdc"},
-			wantErr: "supports a single disk only",
+			wantErr: "ext4 takes one OS disk",
+		},
+		{
+			name:  "dedicated role drives, os first",
+			disks: four,
+			env: map[string]string{
+				"SPINIFEX_DISK":            "vdb",
+				"SPINIFEX_DISK_SPINIFEX":   "vdc",
+				"SPINIFEX_DISK_PREDASTORE": "vdd",
+			},
+			wantFS:    install.FSExt4,
+			wantPaths: []string{"/dev/vdb", "/dev/vdc", "/dev/vdd"},
+		},
+		{
+			// The two-drive layout: predastore gets its own disk while
+			// /var/lib/spinifex stays on the root filesystem.
+			name:      "predastore alone",
+			disks:     four,
+			env:       map[string]string{"SPINIFEX_DISK": "vdb", "SPINIFEX_DISK_PREDASTORE": "vdc"},
+			wantFS:    install.FSExt4,
+			wantPaths: []string{"/dev/vdb", "/dev/vdc"},
+		},
+		{
+			name:  "role drive on a pool",
+			disks: four,
+			env: map[string]string{
+				"SPINIFEX_FS": "zfs-raid1", "SPINIFEX_DISKS": "vdb,vdc",
+				"SPINIFEX_DISK_SPINIFEX": "vdd",
+			},
+			wantErr: "SPINIFEX_DISK_SPINIFEX applies to ext4 only",
+		},
+		{
+			name:    "role drive that is not present",
+			disks:   four[:1],
+			env:     map[string]string{"SPINIFEX_DISK": "vdb", "SPINIFEX_DISK_PREDASTORE": "vdz"},
+			wantErr: "SPINIFEX_DISK_PREDASTORE",
+		},
+		{
+			// Two roles cannot share a drive; the disk is erased once and the
+			// second mount would land on top of the first.
+			name:    "role drive repeats the os disk",
+			disks:   four,
+			env:     map[string]string{"SPINIFEX_DISK": "vdb", "SPINIFEX_DISK_SPINIFEX": "vdb"},
+			wantErr: "selected more than once",
 		},
 		{
 			name:    "bad zfs tunable",

cmd/installer/firstboot/firstboot.go

@@ -156,7 +156,8 @@ resolvconf -u
 # Write the banner (first time, spinifex-banner service will do this on reboot)
 /usr/local/bin/spx admin banner --boot-check
 
-# Cluster formation — capture credentials to file for display on console.
+# Cluster formation. Output goes to the journal via the systemd unit; spx
+# admin init writes credentials to disk and never prints them.
 %s 2>&1
 
 # Fix ownership of files spx admin init wrote. spx runs as root under

cmd/installer/install/disk.go

@@ -14,6 +14,10 @@ const (
 	biosPartNum = 1
 	espPartNum  = 2
 	rootPartNum = 3
+
+	// dataPartNum is the sole partition on a role drive. Data drives carry no
+	// bootloader, so they start at 1 rather than mirroring the boot layout.
+	dataPartNum = 1
 )
 
 // GPT partition type GUIDs, by their sgdisk short code.
@@ -60,14 +64,43 @@ func partitionDisks(cfg DiskConfig) error {
 	if cfg.FS.IsZFS() {
 		ptype = typeZFS
 	}
-	for _, d := range cfg.Disks {
+	for _, d := range cfg.bootDisks() {
 		if err := partitionOne(d, ptype); err != nil {
 			return fmt.Errorf("partition %s: %w", d.Path, err)
 		}
 	}
+	for _, rm := range cfg.DataMounts() {
+		if err := partitionDataDisk(rm); err != nil {
+			return fmt.Errorf("partition %s (%s): %w", rm.Disk.Path, rm.Role, err)
+		}
+	}
 	return waitForPartitions(cfg)
 }
 
+// partitionDataDisk writes the single-partition GPT for a role drive. It gets
+// no bios_boot and no ESP: nothing installs a bootloader there, and a formatted
+// ESP no firmware entry points at only misleads whoever reads lsblk in an
+// incident.
+func partitionDataDisk(rm RoleMount) error {
+	if err := clearDisk(rm.Disk); err != nil {
+		return fmt.Errorf("clear: %w", err)
+	}
+	// Same tail reserve as a boot drive, for the same reason: a replacement of
+	// the "same" size is routinely a few thousand sectors smaller.
+	if err := run("sgdisk",
+		"-n", fmt.Sprintf("%d:1M:-%dM", dataPartNum, tailReserveMiB),
+		"-t", fmt.Sprintf("%d:%s", dataPartNum, typeLinuxFS),
+		"-c", fmt.Sprintf("%d:%s", dataPartNum, rm.Role.Label()),
+		rm.Disk.Path,
+	); err != nil {
+		return err
+	}
+	if err := run("sgdisk", "-G", rm.Disk.Path); err != nil {
+		slog.Warn("could not randomise GPT GUIDs", "disk", rm.Disk.Path, "err", err)
+	}
+	return nil
+}
+
 // partitionOne writes the GPT for a single disk.
 func partitionOne(d Disk, rootType string) error {
 	// Stale ZFS labels and filesystem signatures survive a new partition table
@@ -115,7 +148,7 @@ func waitForPartitions(cfg DiskConfig) error {
 	}
 
 	deadline := time.Now().Add(15 * time.Second)
-	for _, d := range cfg.Disks {
+	for _, d := range cfg.bootDisks() {
 		wanted := []string{d.PartitionPath(espPartNum), d.PartitionPath(rootPartNum)}
 		// The pool is built from by-id paths, so their symlinks have to exist
 		// too — udev publishes them later than the kernel device node.
@@ -128,6 +161,11 @@ func waitForPartitions(cfg DiskConfig) error {
 			}
 		}
 	}
+	for _, rm := range cfg.DataMounts() {
+		if err := waitForPath(rm.Disk.PartitionPath(dataPartNum), deadline); err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
@@ -146,7 +184,7 @@ func waitForPath(path string, deadline time.Time) error {
 // formatESPs makes a FAT32 filesystem on every disk's ESP. Each one is
 // independent — there is no RAID here, they are kept in step by content.
 func formatESPs(cfg DiskConfig) error {
-	for _, d := range cfg.Disks {
+	for _, d := range cfg.bootDisks() {
 		esp := d.PartitionPath(espPartNum)
 		// A distinct label per member so `lsblk -f` on a degraded machine tells
 		// the operator which disk each ESP belongs to.

cmd/installer/install/diskconfig.go

@@ -82,6 +82,45 @@ func ParseFSType(s string) (FSType, error) {
 	return f, nil
 }
 
+// DiskRole is the purpose a selected drive serves on an ext4 install. A ZFS
+// pool carries every workload on one set of members, so roles do not apply
+// there — the topology already spans all the disks.
+type DiskRole string
+
+const (
+	RoleOS         DiskRole = "os"
+	RoleSpinifex   DiskRole = "spinifex"
+	RolePredastore DiskRole = "predastore"
+)
+
+// AllDiskRoles is canonical order, outermost mountpoint first. Mounting, fstab
+// emission and rsync protect filters all walk it forwards; teardown walks it
+// back, because predastore nests inside spinifex.
+var AllDiskRoles = []DiskRole{RoleOS, RoleSpinifex, RolePredastore}
+
+// roleMountpoints is where each role lands on the installed system. A role left
+// unassigned simply stays on the filesystem above it.
+var roleMountpoints = map[DiskRole]string{
+	RoleOS:         "/",
+	RoleSpinifex:   "/var/lib/spinifex",
+	RolePredastore: "/var/lib/spinifex/predastore",
+}
+
+// Label is the role name as shown in the UI and written to partition labels.
+func (r DiskRole) Label() string { return string(r) }
+
+// Mountpoint is the role's path on the installed system.
+func (r DiskRole) Mountpoint() string { return roleMountpoints[r] }
+
+// RoleMount binds one drive to one role.
+type RoleMount struct {
+	Role DiskRole
+	Disk Disk
+}
+
+// Mountpoint is where this drive's filesystem is mounted on the target.
+func (r RoleMount) Mountpoint() string { return r.Role.Mountpoint() }
+
 // ZFSOpts are the advanced tunables. A zero value means "use the computed
 // default", which is why Ashift and ARCMaxMiB are plain ints — 0 is not legal
 // for either, so it is unambiguous as a sentinel.
@@ -94,14 +133,81 @@ type ZFSOpts struct {
 }
 
 // DiskConfig is the storage half of the installer's configuration. Disks is
-// ordered and the order is significant: RAID10 pairs members two at a time.
+// ordered and the order is significant: RAID10 pairs members two at a time, and
+// on ext4 the first entry is the OS drive.
 type DiskConfig struct {
 	FS    FSType
 	Disks []Disk
 	ZFS   ZFSOpts
+
+	// Roles assigns ext4 drives to mountpoints, in AllDiskRoles order. Empty
+	// means a single-disk install with everything on the root filesystem.
+	Roles []RoleMount
 }
 
-// Primary is the first selected disk — the whole target in ext4 mode, and the
+// WithRoles returns a copy with ext4 drive roles assigned and Disks rebuilt
+// from them in canonical order, so the list of drives to erase and the list of
+// roles cannot drift apart — there is one place a drive can be named, not two.
+func (d DiskConfig) WithRoles(roles []RoleMount) DiskConfig {
+	d.Roles, d.Disks = nil, nil
+	add := func(rm RoleMount) {
+		d.Roles = append(d.Roles, rm)
+		d.Disks = append(d.Disks, rm.Disk)
+	}
+	for _, want := range AllDiskRoles {
+		for _, rm := range roles {
+			if rm.Role == want {
+				add(rm)
+			}
+		}
+	}
+	// Anything with an unknown or duplicate role is kept so Validate can name
+	// it, rather than being dropped into a silently smaller selection.
+	for _, rm := range roles {
+		if _, known := roleMountpoints[rm.Role]; !known {
+			add(rm)
+		}
+	}
+	return d
+}
+
+// RolesForDisks assigns roles positionally, which is the default the UI offers:
+// first disk picked is the OS, second is spinifex, third is predastore.
+func RolesForDisks(disks []Disk) []RoleMount {
+	out := make([]RoleMount, 0, len(disks))
+	for i, disk := range disks {
+		var role DiskRole
+		if i < len(AllDiskRoles) {
+			role = AllDiskRoles[i]
+		}
+		out = append(out, RoleMount{Role: role, Disk: disk})
+	}
+	return out
+}
+
+// DataMounts is every role except the OS drive, shallowest mountpoint first.
+// This is the traversal every stage of the install shares.
+func (d DiskConfig) DataMounts() []RoleMount {
+	var out []RoleMount
+	for _, rm := range d.Roles {
+		if rm.Role != RoleOS {
+			out = append(out, rm)
+		}
+	}
+	return out
+}
+
+// bootDisks are the drives a bootloader is installed to. Every ZFS member gets
+// its own ESP; on ext4 only the OS drive does, because a data drive holding a
+// bootloader that nothing points at is a trap during recovery.
+func (d DiskConfig) bootDisks() []Disk {
+	if d.FS.IsZFS() || len(d.Disks) == 0 {
+		return d.Disks
+	}
+	return d.Disks[:1]
+}
+
+// Primary is the first selected disk — the OS drive in ext4 mode, and the
 // disk shown in single-disk summaries.
 func (d DiskConfig) Primary() Disk {
 	if len(d.Disks) == 0 {
@@ -222,9 +328,8 @@ func (d DiskConfig) Validate() error {
 	if len(d.Disks) == 0 {
 		return fmt.Errorf("%s: select at least one disk", d.FS.Label())
 	}
-	if !d.FS.IsZFS() && len(d.Disks) > 1 {
-		return fmt.Errorf("%s: supports a single disk only, %d selected — choose a zfs mode to use them all",
-			d.FS.Label(), len(d.Disks))
+	if err := d.validateRoles(); err != nil {
+		return err
 	}
 	if n := len(d.Disks); n < d.FS.MinDisks() {
 		return fmt.Errorf("%s: needs at least %d disks, %d selected", d.FS.Label(), d.FS.MinDisks(), n)
@@ -243,21 +348,66 @@ func (d DiskConfig) Validate() error {
 		if disk.LiveMedia {
 			return fmt.Errorf("%s is the installer's own boot media and cannot be erased", disk.Path)
 		}
-		// GRUB's i386-pc target cannot read a 4Kn drive, so on legacy BIOS the
-		// installed system would have no reachable bootloader.
-		if !bootedEFI() && disk.LogicalBlockSize == 4096 {
-			return fmt.Errorf("%s is a 4Kn drive (4096-byte sectors) and this machine booted in legacy BIOS mode — booting from it is not supported; enable UEFI in firmware",
-				disk.Path)
-		}
 		if disk.Bytes < minRootBytes {
 			return fmt.Errorf("%s is too small (%s, need at least %dGiB)",
 				disk.Path, disk.SizeHuman(), minRootBytes>>30)
 		}
 	}
 
+	// GRUB's i386-pc target cannot read a 4Kn drive, so on legacy BIOS a drive
+	// holding a bootloader would leave the installed system unreachable. Only
+	// boot drives are constrained: a data role is never booted from.
+	for _, disk := range d.bootDisks() {
+		if !bootedEFI() && disk.LogicalBlockSize == 4096 {
+			return fmt.Errorf("%s is a 4Kn drive (4096-byte sectors) and this machine booted in legacy BIOS mode — booting from it is not supported; enable UEFI in firmware",
+				disk.Path)
+		}
+	}
+
 	return d.validateSizes()
 }
 
+// validateRoles checks the ext4 role assignment. Roles are the only way to use
+// more than one disk without ZFS, so a multi-disk selection without them is
+// rejected rather than quietly installed onto the first drive.
+func (d DiskConfig) validateRoles() error {
+	if d.FS.IsZFS() {
+		if len(d.Roles) > 0 {
+			return fmt.Errorf("%s: drive roles apply to ext4 only — a pool spans every member already", d.FS.Label())
+		}
+		return nil
+	}
+	if len(d.Roles) == 0 {
+		if len(d.Disks) > 1 {
+			return fmt.Errorf("%s: %d disks selected but no roles assigned — assign os, spinifex and predastore, or choose a zfs mode",
+				d.FS.Label(), len(d.Disks))
+		}
+		return nil
+	}
+	if len(d.Roles) > len(AllDiskRoles) {
+		return fmt.Errorf("%s: at most %d disks, %d selected — the roles are os, spinifex and predastore",
+			d.FS.Label(), len(AllDiskRoles), len(d.Roles))
+	}
+
+	seen := map[DiskRole]bool{}
+	for _, rm := range d.Roles {
+		if _, ok := roleMountpoints[rm.Role]; !ok {
+			return fmt.Errorf("%s has no role assigned — every selected disk needs one", rm.Disk.Path)
+		}
+		if seen[rm.Role] {
+			return fmt.Errorf("two disks are assigned the %s role", rm.Role)
+		}
+		seen[rm.Role] = true
+	}
+	if !seen[RoleOS] {
+		return fmt.Errorf("%s: no disk assigned the os role — one disk must hold the operating system", d.FS.Label())
+	}
+	if len(d.Roles) != len(d.Disks) {
+		return fmt.Errorf("internal: %d roles for %d disks", len(d.Roles), len(d.Disks))
+	}
+	return nil
+}
+
 // minRootBytes matches Proxmox's hard floor for a root disk.
 const minRootBytes = 2 << 30
 
@@ -314,5 +464,20 @@ func (d DiskConfig) Warnings() []string {
 	if d.FS.IsZFS() && d.FS != FSZFSRAID0 && d.Buildable() && d.Tolerated() == 0 {
 		out = append(out, "this topology stores no redundancy")
 	}
+	// A three-drive role layout looks like an array and is not one: each drive
+	// is an independent point of failure, so the node loses whatever sat on it.
+	if len(d.DataMounts()) > 0 {
+		out = append(out, "role-assigned drives store no redundancy — losing any one drive loses what it held")
+	}
+	for _, rm := range d.DataMounts() {
+		if rm.Role == RolePredastore && rm.Disk.Bytes < smallPredastoreBytes {
+			out = append(out, fmt.Sprintf("%s is small for object storage (%s) — predastore will fill quickly",
+				rm.Disk.Path, rm.Disk.SizeHuman()))
+		}
+	}
 	return out
 }
+
+// smallPredastoreBytes is where a dedicated object-store drive stops being
+// worth the drive. Advisory only: the install is legal below it.
+const smallPredastoreBytes = 64 << 30

cmd/installer/install/diskconfig_test.go

@@ -20,7 +20,7 @@ func TestDiskConfigValidate(t *testing.T) {
 
 		{"no filesystem chosen", DiskConfig{Disks: disks(1, 40)}, "no filesystem selected"},
 		{"no disks", DiskConfig{FS: FSZFSRAIDZ1}, "select at least one disk"},
-		{"ext4 cannot span disks", DiskConfig{FS: FSExt4, Disks: disks(2, 40)}, "single disk only"},
+		{"ext4 spans disks only with roles", DiskConfig{FS: FSExt4, Disks: disks(2, 40)}, "no roles assigned"},
 		{"raidz1 needs three", DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(2, 40)}, "at least 3 disks"},
 		{"raidz2 needs four", DiskConfig{FS: FSZFSRAIDZ2, Disks: disks(3, 40)}, "at least 4 disks"},
 		{"raidz3 needs five", DiskConfig{FS: FSZFSRAIDZ3, Disks: disks(4, 40)}, "at least 5 disks"},

cmd/installer/install/install.go

@@ -61,6 +61,7 @@ func Run(cfg *Config) error {
 	}
 	steps = append(steps,
 		step{"copy rootfs", func() error { return copyRootfs(cfg.Storage) }},
+		step{"verify role layout", func() error { return verifyRoleLayout(cfg.Storage) }},
 		step{"create swap", func() error { return createSwap(cfg.Storage) }},
 		step{"write fstab", func() error { return writeFstab(cfg.Storage) }},
 		step{"install spinifex", func() error { return installSpinifex(cfg) }},
@@ -84,7 +85,7 @@ func Run(cfg *Config) error {
 	return reboot()
 }
 
-// ext4RootSteps formats and mounts a single-disk ext4 root.
+// ext4RootSteps formats and mounts the ext4 root and any role filesystems.
 func ext4RootSteps(cfg *Config) []step {
 	return []step{
 		{"format partitions", func() error { return formatPartitions(cfg.Storage) }},
@@ -124,15 +125,36 @@ func cleanupTarget(cfg DiskConfig) {
 		exportPool()
 		return
 	}
+	// Deepest first: predastore nests inside spinifex, and unmounting the outer
+	// filesystem fails with EBUSY while the inner one is still attached.
+	for _, rm := range slices.Backward(cfg.DataMounts()) {
+		_ = runQuiet("umount", targetPath(rm.Mountpoint()))
+	}
 	_ = runQuiet("umount", efiPart())
 	_ = runQuiet("umount", mountRoot)
 }
 
+// targetPath maps a path on the installed system to its location under the
+// install mount root.
+func targetPath(mountpoint string) string {
+	return filepath.Join(mountRoot, strings.TrimPrefix(mountpoint, "/"))
+}
+
 func formatPartitions(cfg DiskConfig) error {
 	if err := formatESPs(cfg); err != nil {
 		return err
 	}
-	return run("mkfs.ext4", "-F", cfg.Primary().PartitionPath(rootPartNum))
+	if err := run("mkfs.ext4", "-F", cfg.Primary().PartitionPath(rootPartNum)); err != nil {
+		return err
+	}
+	for _, rm := range cfg.DataMounts() {
+		// Labelled with the role so `lsblk -f` identifies each drive on a
+		// console that has no access to the installer log.
+		if err := run("mkfs.ext4", "-F", "-L", rm.Role.Label(), rm.Disk.PartitionPath(dataPartNum)); err != nil {
+			return fmt.Errorf("format %s (%s): %w", rm.Disk.Path, rm.Role, err)
+		}
+	}
+	return nil
 }
 
 func mountPartitions(cfg DiskConfig) error {
@@ -146,7 +168,36 @@ func mountPartitions(cfg DiskConfig) error {
 	if err := os.MkdirAll(efiPart(), 0o755); err != nil {
 		return err
 	}
-	return run("mount", d.PartitionPath(espPartNum), efiPart())
+	if err := run("mount", d.PartitionPath(espPartNum), efiPart()); err != nil {
+		return err
+	}
+	// Role filesystems are mounted here, before the rootfs copy, so rsync lays
+	// each tree down on its own drive with the ownership setup.sh gave it at
+	// ISO build time. Mounting afterwards hides those trees and loses it.
+	for _, rm := range cfg.DataMounts() {
+		target := targetPath(rm.Mountpoint())
+		if err := os.MkdirAll(target, 0o755); err != nil {
+			return err
+		}
+		if err := run("mount", rm.Disk.PartitionPath(dataPartNum), target); err != nil {
+			return fmt.Errorf("mount %s at %s: %w", rm.Disk.Path, rm.Mountpoint(), err)
+		}
+	}
+	return nil
+}
+
+// mountProtectFilters stops rsync's --delete from acting across a mount
+// boundary. Every mountpoint the installer created before the copy needs one,
+// whether it is a ZFS dataset or a role filesystem.
+func mountProtectFilters(cfg DiskConfig) []string {
+	if cfg.FS.IsZFS() {
+		return datasetProtectFilters(cfg)
+	}
+	var out []string
+	for _, rm := range cfg.DataMounts() {
+		out = append(out, "--filter=P "+rm.Mountpoint())
+	}
+	return out
 }
 
 // copyRootfs copies the live squashfs environment onto the target disk using
@@ -171,7 +222,7 @@ func copyRootfs(cfg DiskConfig) error {
 		"--exclude=/lost+found",
 		"--exclude=/boot/efi",
 	}
-	args = append(args, datasetProtectFilters(cfg)...)
+	args = append(args, mountProtectFilters(cfg)...)
 	args = append(args, "/", mountRoot+"/")
 	if err := run("rsync", args...); err != nil {
 		return err
@@ -213,6 +264,71 @@ func copyRootfs(cfg DiskConfig) error {
 	return nil
 }
 
+// verifyRoleLayout asserts the rootfs copy landed on the role drives rather
+// than on top of them. A mount that happened in the wrong order produces a node
+// that boots, starts every service, and has each one writing to an empty
+// directory on the wrong drive — healthy-looking and silently wrong.
+func verifyRoleLayout(cfg DiskConfig) error {
+	for _, rm := range cfg.DataMounts() {
+		target := targetPath(rm.Mountpoint())
+		dev, err := deviceOf(target)
+		if err != nil {
+			return fmt.Errorf("verify %s: %w", rm.Mountpoint(), err)
+		}
+		parentDev, err := deviceOf(filepath.Dir(target))
+		if err != nil {
+			return fmt.Errorf("verify %s: %w", rm.Mountpoint(), err)
+		}
+		if dev == parentDev {
+			return fmt.Errorf("%s is not a separate filesystem — %s was not mounted before the rootfs copy",
+				rm.Mountpoint(), rm.Disk.Path)
+		}
+		if err := matchesLiveOwnership(target, rm.Mountpoint()); err != nil {
+			return err
+		}
+		// Logged so the layout is recoverable from the install log alone, which
+		// on a headless node is the only record of where each workload landed.
+		slog.Info("role mounted", "role", rm.Role, "mountpoint", rm.Mountpoint(), "disk", rm.Disk.Path)
+	}
+	return nil
+}
+
+// matchesLiveOwnership compares a copied directory against the live rootfs,
+// which is where setup.sh set every service owner at ISO build time. Comparing
+// rather than hardcoding uids keeps this correct as services are added.
+func matchesLiveOwnership(target, source string) error {
+	src, err := os.Stat(source)
+	if err != nil {
+		// Nothing to compare against; the mount check above still holds.
+		slog.Warn("skipping ownership check, path absent in live environment", "path", source, "err", err)
+		return nil
+	}
+	dst, err := os.Stat(target)
+	if err != nil {
+		return fmt.Errorf("verify %s: %w", source, err)
+	}
+	srcSt, dstSt := src.Sys().(*syscall.Stat_t), dst.Sys().(*syscall.Stat_t)
+	if srcSt.Uid != dstSt.Uid || srcSt.Gid != dstSt.Gid || src.Mode().Perm() != dst.Mode().Perm() {
+		return fmt.Errorf("%s copied as %d:%d %o, expected %d:%d %o — the drive was mounted after the rootfs copy",
+			source, dstSt.Uid, dstSt.Gid, dst.Mode().Perm(), srcSt.Uid, srcSt.Gid, src.Mode().Perm())
+	}
+	return nil
+}
+
+// deviceOf returns the filesystem a path lives on, for distinguishing a real
+// mount from a directory of the same name.
+func deviceOf(path string) (uint64, error) {
+	fi, err := os.Stat(path)
+	if err != nil {
+		return 0, err
+	}
+	st, ok := fi.Sys().(*syscall.Stat_t)
+	if !ok {
+		return 0, fmt.Errorf("%s: no stat information", path)
+	}
+	return st.Dev, nil
+}
+
 func installSpinifex(cfg *Config) error {
 	// The rootfs copy already contains spx and spinifex-installer at
 	// /usr/local/bin/ — no binary copy needed. Regenerate machine-specific
@@ -808,6 +924,18 @@ func writeFstab(cfg DiskConfig) error {
 		}
 		fstab += fmt.Sprintf("UUID=%s / ext4 errors=remount-ro 0 1\nUUID=%s /boot/efi vfat umask=0077 0 1\n",
 			rootUUID, efiUUID)
+
+		// Shallowest first, so `mount -a` in a rescue shell works top-down.
+		// Deliberately no nofail: a missing data drive must stop the boot for
+		// an admin, not let predastore write objects onto the OS drive while
+		// every service reports healthy. fsck pass 2 keeps root first.
+		for _, rm := range cfg.DataMounts() {
+			uuid, err := partUUID(rm.Disk.PartitionPath(dataPartNum))
+			if err != nil {
+				return fmt.Errorf("get %s UUID: %w", rm.Role, err)
+			}
+			fstab += fmt.Sprintf("UUID=%s %s ext4 errors=remount-ro 0 2\n", uuid, rm.Mountpoint())
+		}
 	}
 
 	// Only claim the swap the previous step actually produced; systemd fails the
@@ -1014,7 +1142,9 @@ func parseSize(s string) (int64, error) {
 	return n * mult, nil
 }
 
-func partUUID(dev string) (string, error) {
+// partUUID reads a partition's filesystem UUID. A variable so fstab generation
+// can be tested without real block devices.
+var partUUID = func(dev string) (string, error) {
 	out, err := exec.Command("blkid", "-s", "UUID", "-o", "value", dev).Output()
 	if err != nil {
 		return "", fmt.Errorf("blkid %s: %w", dev, err)

cmd/installer/install/roles_test.go

@@ -0,0 +1,391 @@
+package install
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+)
+
+// roleCfg builds an ext4 configuration from role/disk-name pairs, in whatever
+// order they are given — WithRoles is responsible for normalising them.
+func roleCfg(pairs ...any) DiskConfig {
+	var roles []RoleMount
+	for i := 0; i+1 < len(pairs); i += 2 {
+		roles = append(roles, RoleMount{
+			Role: pairs[i].(DiskRole),
+			Disk: disk(pairs[i+1].(string), 100),
+		})
+	}
+	return DiskConfig{FS: FSExt4}.WithRoles(roles)
+}
+
+// fakeUUIDs makes partUUID return a value derived from the device path, so a
+// generated fstab can be checked without touching a real disk.
+func fakeUUIDs(t *testing.T) {
+	t.Helper()
+	prev := partUUID
+	partUUID = func(dev string) (string, error) { return "uuid" + strings.ReplaceAll(dev, "/", "-"), nil }
+	t.Cleanup(func() { partUUID = prev })
+}
+
+func TestWithRolesNormalisesOrderAndDerivesDisks(t *testing.T) {
+	// Given deepest-first, which is the order that breaks mounting.
+	cfg := roleCfg(RolePredastore, "c", RoleSpinifex, "b", RoleOS, "a")
+
+	wantRoles := []DiskRole{RoleOS, RoleSpinifex, RolePredastore}
+	for i, rm := range cfg.Roles {
+		if rm.Role != wantRoles[i] {
+			t.Fatalf("role %d = %s, want %s", i, rm.Role, wantRoles[i])
+		}
+	}
+	// Disks is derived, so the OS drive is first and every role drive appears
+	// exactly once in the list the installer erases.
+	if got := cfg.Paths(); !slices.Equal(got, []string{"/dev/a", "/dev/b", "/dev/c"}) {
+		t.Fatalf("Paths() = %v, want the roles in canonical order", got)
+	}
+	if cfg.Primary().Path != "/dev/a" {
+		t.Errorf("Primary() = %s, want the os drive", cfg.Primary().Path)
+	}
+}
+
+func TestDataMountsExcludeTheOSDriveAndNestShallowestFirst(t *testing.T) {
+	got := roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c").DataMounts()
+	want := []string{"/var/lib/spinifex", "/var/lib/spinifex/predastore"}
+	for i, rm := range got {
+		if rm.Mountpoint() != want[i] {
+			t.Fatalf("mount %d = %s, want %s", i, rm.Mountpoint(), want[i])
+		}
+	}
+	if len(got) != len(want) {
+		t.Fatalf("got %d data mounts, want %d", len(got), len(want))
+	}
+}
+
+// A predastore drive without a spinifex drive is the two-drive layout, and it
+// nests a mount inside the root filesystem rather than inside another mount.
+func TestPredastoreRoleWithoutSpinifexIsValid(t *testing.T) {
+	cfg := roleCfg(RoleOS, "a", RolePredastore, "b")
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("Validate() = %v, want nil", err)
+	}
+	if got := cfg.DataMounts(); len(got) != 1 || got[0].Mountpoint() != "/var/lib/spinifex/predastore" {
+		t.Fatalf("DataMounts() = %v, want predastore only", got)
+	}
+}
+
+func TestValidateRoles(t *testing.T) {
+	twoRoles := func(a, b DiskRole) DiskConfig { return roleCfg(a, "a", b, "b") }
+
+	tests := []struct {
+		name    string
+		cfg     DiskConfig
+		wantErr string
+	}{
+		{"single disk needs no roles", DiskConfig{FS: FSExt4, Disks: disks(1, 40)}, ""},
+		{"os alone", roleCfg(RoleOS, "a"), ""},
+		{"all three", roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c"), ""},
+		{"no os role", twoRoles(RoleSpinifex, RolePredastore), "no disk assigned the os role"},
+		{"duplicate role", twoRoles(RoleOS, RoleOS), "two disks are assigned the os role"},
+		{"unknown role", roleCfg(RoleOS, "a", DiskRole("cache"), "b"), "no role assigned"},
+		{"more disks than roles", roleCfg(
+			RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c", DiskRole("cache"), "d"), "at most 3 disks"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.cfg.Validate()
+			switch {
+			case tt.wantErr == "" && err != nil:
+				t.Fatalf("Validate() = %v, want nil", err)
+			case tt.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tt.wantErr)):
+				t.Fatalf("Validate() = %v, want error containing %q", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+// Roles describe where a workload lives on a plain filesystem. A pool already
+// spans every member, so accepting them there would silently do nothing.
+func TestValidateRejectsRolesOnZFS(t *testing.T) {
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+	cfg.Roles = []RoleMount{{Role: RoleOS, Disk: cfg.Disks[0]}}
+	if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "ext4 only") {
+		t.Fatalf("Validate() = %v, want a roles-are-ext4-only error", err)
+	}
+}
+
+func TestBootDisksAreOnlyTheOSDriveOnExt4(t *testing.T) {
+	cfg := roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c")
+	if got := cfg.bootDisks(); len(got) != 1 || got[0].Path != "/dev/a" {
+		t.Fatalf("bootDisks() = %v, want the os drive only", got)
+	}
+	// Every ZFS member carries its own ESP, so all of them stay boot disks.
+	zfs := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+	if got := zfs.bootDisks(); len(got) != 2 {
+		t.Fatalf("bootDisks() = %v, want every pool member", got)
+	}
+}
+
+func TestWriteFstabEmitsRoleMountsShallowestFirst(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+	fakeUUIDs(t)
+
+	cfg := roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c")
+	if err := writeFstab(cfg); err != nil {
+		t.Fatalf("writeFstab: %v", err)
+	}
+	got := readFile(t, root, "etc/fstab")
+
+	// Order matters: `mount -a` in a rescue shell walks the file top-down, and
+	// predastore cannot be mounted before the filesystem it sits inside.
+	wantOrder := []string{" / ", "/boot/efi", " /var/lib/spinifex ", " /var/lib/spinifex/predastore "}
+	at := -1
+	for _, want := range wantOrder {
+		i := strings.Index(got, want)
+		if i < 0 {
+			t.Fatalf("fstab is missing %q:\n%s", want, got)
+		}
+		if i < at {
+			t.Fatalf("%q is out of order:\n%s", want, got)
+		}
+		at = i
+	}
+
+	for _, dev := range []string{"/dev/b1", "/dev/c1"} {
+		if want := "UUID=uuid-dev-" + strings.TrimPrefix(dev, "/dev/"); !strings.Contains(got, want) {
+			t.Errorf("fstab does not mount %s by UUID:\n%s", dev, got)
+		}
+	}
+	// Data mounts are fsck'd after root, never alongside it.
+	for line := range strings.SplitSeq(got, "\n") {
+		if strings.Contains(line, "/var/lib/spinifex") && !strings.HasSuffix(line, "0 2") {
+			t.Errorf("data mount should be fsck pass 2: %q", line)
+		}
+	}
+}
+
+// Fail-closed by design: a missing data drive must stop the boot for an admin,
+// not let predastore write objects onto the OS drive while everything reports
+// healthy. nofail would silently produce exactly that.
+func TestWriteFstabNeverMarksRoleMountsNofail(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+	fakeUUIDs(t)
+
+	if err := writeFstab(roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c")); err != nil {
+		t.Fatalf("writeFstab: %v", err)
+	}
+	if got := readFile(t, root, "etc/fstab"); strings.Contains(got, "nofail") {
+		t.Errorf("role mounts must not be nofail:\n%s", got)
+	}
+}
+
+func TestWriteFstabSingleDiskIsUnchanged(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+	fakeUUIDs(t)
+
+	if err := writeFstab(roleCfg(RoleOS, "a")); err != nil {
+		t.Fatalf("writeFstab: %v", err)
+	}
+	if got := readFile(t, root, "etc/fstab"); strings.Contains(got, "/var/lib/spinifex") {
+		t.Errorf("a single-disk install should mount nothing extra:\n%s", got)
+	}
+}
+
+// rsync runs with --delete, so every mountpoint that exists before the copy
+// needs protecting or the copy can delete across a mount boundary on a retry.
+func TestMountProtectFiltersCoverEveryRoleMount(t *testing.T) {
+	got := mountProtectFilters(roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c"))
+	want := []string{"--filter=P /var/lib/spinifex", "--filter=P /var/lib/spinifex/predastore"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("mountProtectFilters() = %v, want %v", got, want)
+	}
+	if got := mountProtectFilters(roleCfg(RoleOS, "a")); len(got) != 0 {
+		t.Errorf("a single-disk install has no mounts to protect, got %v", got)
+	}
+	// ZFS keeps its dataset filters — the union is by mode, not additive.
+	if got := mountProtectFilters(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}); len(got) == 0 {
+		t.Error("zfs mode should still protect its dataset mountpoints")
+	}
+}
+
+func TestMountPartitionsMountsRolesAfterRootAndBeforeTheCopy(t *testing.T) {
+	f := fakeCommands(t)
+	root := withMountRoot(t)
+
+	cfg := roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c")
+	if err := mountPartitions(cfg); err != nil {
+		t.Fatalf("mountPartitions: %v", err)
+	}
+
+	// The nested mount cannot precede the one it sits inside, or it is masked.
+	want := []string{
+		"mount /dev/a3 " + root,
+		"mount /dev/a2 " + root + "/boot/efi",
+		"mount /dev/b1 " + root + "/var/lib/spinifex",
+		"mount /dev/c1 " + root + "/var/lib/spinifex/predastore",
+	}
+	var mounts []string
+	for _, c := range f.calls {
+		if strings.HasPrefix(c, "mount ") {
+			mounts = append(mounts, c)
+		}
+	}
+	if !slices.Equal(mounts, want) {
+		t.Fatalf("mount order:\n got %v\nwant %v", mounts, want)
+	}
+}
+
+func TestCleanupTargetUnmountsDeepestFirst(t *testing.T) {
+	f := fakeCommands(t)
+	root := withMountRoot(t)
+
+	cleanupTarget(roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c"))
+
+	// Unmounting the outer filesystem first fails with EBUSY and leaves a
+	// half-mounted target behind for the next attempt.
+	want := []string{
+		"umount " + root + "/var/lib/spinifex/predastore",
+		"umount " + root + "/var/lib/spinifex",
+		"umount " + root + "/boot/efi",
+		"umount " + root,
+	}
+	// The chroot binds are released first and are not part of the target order.
+	var got []string
+	for _, c := range f.calls {
+		if strings.HasPrefix(c, "umount ") && slices.Contains(want, c) {
+			got = append(got, c)
+		}
+	}
+	if !slices.Equal(got, want) {
+		t.Fatalf("unmount order:\n got %v\nwant %v", got, want)
+	}
+}
+
+func TestPartitionDisksGivesDataDrivesOnePartitionAndNoESP(t *testing.T) {
+	f := fakeCommands(t)
+	nodes := tempDisks(t, 2, 200)
+	cfg := DiskConfig{FS: FSExt4}.WithRoles(
+		[]RoleMount{{Role: RoleOS, Disk: nodes[0]}, {Role: RoleSpinifex, Disk: nodes[1]}})
+	osDisk, data := nodes[0].Path, nodes[1].Path
+
+	if err := partitionDisks(cfg); err != nil {
+		t.Fatalf("partitionDisks: %v", err)
+	}
+
+	f.mustRun(t, "sgdisk", osDisk, "-n 3:0:-1024M", "-t 3:8300")
+	f.mustRun(t, "sgdisk", osDisk, "-t 2:EF00")
+	// A data drive is never booted from, so a bios_boot or an ESP there would
+	// only mislead whoever reads lsblk during an incident.
+	for _, c := range f.calls {
+		if strings.Contains(c, data) && (strings.Contains(c, "EF02") || strings.Contains(c, "EF00")) {
+			t.Errorf("data drive must get no bios_boot or ESP: %q", c)
+		}
+	}
+	f.mustRun(t, "sgdisk", data, "-n 1:1M:-1024M", "-t 1:8300", "-c 1:spinifex")
+}
+
+func TestFormatPartitionsLabelsEachRoleFilesystem(t *testing.T) {
+	f := fakeCommands(t)
+	cfg := roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c")
+
+	if err := formatPartitions(cfg); err != nil {
+		t.Fatalf("formatPartitions: %v", err)
+	}
+	// One ESP only: a data drive holding a bootloader nothing points at would
+	// mislead whoever reads lsblk during an incident.
+	esps := 0
+	for _, c := range f.calls {
+		if strings.HasPrefix(c, "mkfs.fat") {
+			esps++
+		}
+	}
+	if esps != 1 {
+		t.Errorf("want one ESP, got %d:\n%v", esps, f.calls)
+	}
+	f.mustRun(t, "mkfs.ext4", "-L", "spinifex", "/dev/b1")
+	f.mustRun(t, "mkfs.ext4", "-L", "predastore", "/dev/c1")
+}
+
+func TestWarningsFlagTheAbsenceOfRedundancy(t *testing.T) {
+	got := strings.Join(roleCfg(RoleOS, "a", RoleSpinifex, "b").Warnings(), "\n")
+	if !strings.Contains(got, "no redundancy") {
+		t.Errorf("a multi-drive role layout must warn it is not an array, got %q", got)
+	}
+	if got := roleCfg(RoleOS, "a").Warnings(); len(got) != 0 {
+		t.Errorf("a single-disk install has nothing to warn about, got %v", got)
+	}
+}
+
+func TestWarningsFlagASmallPredastoreDrive(t *testing.T) {
+	cfg := DiskConfig{FS: FSExt4}.WithRoles([]RoleMount{
+		{Role: RoleOS, Disk: disk("a", 500)},
+		{Role: RolePredastore, Disk: disk("b", 8)},
+	})
+	got := strings.Join(cfg.Warnings(), "\n")
+	if !strings.Contains(got, "small for object storage") {
+		t.Errorf("an 8GiB predastore drive should be flagged, got %q", got)
+	}
+	if err := cfg.Validate(); err != nil {
+		t.Errorf("the warning must not block the install: %v", err)
+	}
+}
+
+func TestRolesForDisksAssignsPositionally(t *testing.T) {
+	got := RolesForDisks(disks(3, 40))
+	for i, rm := range got {
+		if want := AllDiskRoles[i]; rm.Role != want {
+			t.Errorf("disk %d got role %s, want %s", i, rm.Role, want)
+		}
+	}
+	// A fourth disk has no role, which Validate is left to reject by name.
+	if extra := RolesForDisks(disks(4, 40)); extra[3].Role != "" {
+		t.Errorf("disk 4 got role %q, want none", extra[3].Role)
+	}
+}
+
+func TestFourKnDataDriveIsAllowedWhereABootDriveIsNot(t *testing.T) {
+	if bootedEFI() {
+		t.Skip("machine booted in EFI mode; the 4Kn rule does not apply")
+	}
+	fourKn := func(name string) Disk {
+		d := disk(name, 100)
+		d.LogicalBlockSize = 4096
+		return d
+	}
+	// GRUB's i386-pc target cannot read a 4Kn drive, but nothing boots from a
+	// data drive, so only the OS role is constrained.
+	ok := DiskConfig{FS: FSExt4}.WithRoles(
+		[]RoleMount{{Role: RoleOS, Disk: disk("a", 100)}, {Role: RoleSpinifex, Disk: fourKn("b")}})
+	if err := ok.Validate(); err != nil {
+		t.Errorf("a 4Kn data drive should be accepted: %v", err)
+	}
+
+	bad := DiskConfig{FS: FSExt4}.WithRoles(
+		[]RoleMount{{Role: RoleOS, Disk: fourKn("a")}, {Role: RoleSpinifex, Disk: disk("b", 100)}})
+	if err := bad.Validate(); err == nil || !strings.Contains(err.Error(), "4Kn") {
+		t.Errorf("Validate() = %v, want a 4Kn rejection for the os drive", err)
+	}
+}
+
+func TestVerifyRoleLayoutCatchesAMountThatNeverHappened(t *testing.T) {
+	root := withMountRoot(t)
+	// Plain directories on one filesystem: exactly what a mount performed after
+	// the rootfs copy, or not at all, leaves behind.
+	if err := os.MkdirAll(filepath.Join(root, "var/lib/spinifex/predastore"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+
+	err := verifyRoleLayout(roleCfg(RoleOS, "a", RoleSpinifex, "b"))
+	if err == nil || !strings.Contains(err.Error(), "not a separate filesystem") {
+		t.Fatalf("verifyRoleLayout() = %v, want a missing-mount error", err)
+	}
+	if !strings.Contains(fmt.Sprint(err), "/dev/b") {
+		t.Errorf("the error must name the drive that was not mounted: %v", err)
+	}
+}

cmd/installer/systemd/systemd.go

@@ -68,6 +68,8 @@ ExecStopPost=/bin/sh -c 'if [ "$EXIT_STATUS" != "0" ]; then printf "\n*** Spinif
 # ovn-central startup cannot wedge multi-user.target and keep getty from
 # ever reaching the login prompt.
 TimeoutStartSec=300s
+# Install diagnostics land in the journal, which persists for the life of the
+# node — nothing this script runs may print secrets to stdout or stderr.
 StandardOutput=journal
 StandardError=journal
 

cmd/installer/ui/storage_test.go

@@ -0,0 +1,139 @@
+package ui
+
+import (
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/installer/install"
+)
+
+// diskModel is a storage screen with n unselected disks and ext4 chosen, which
+// is where the wizard starts.
+func diskModel(n int) model {
+	m := model{fsCursor: slices.Index(install.AllFSTypes, install.FSExt4)}
+	for i := range n {
+		m.disks = append(m.disks, install.Disk{
+			Path:              string(rune('a'+i)) + "-disk",
+			Bytes:             200 << 30,
+			LogicalBlockSize:  512,
+			PhysicalBlockSize: 512,
+		})
+	}
+	return m
+}
+
+func rolePaths(cfg install.DiskConfig) map[install.DiskRole]string {
+	out := map[install.DiskRole]string{}
+	for _, rm := range cfg.Roles {
+		out[rm.Role] = rm.Disk.Path
+	}
+	return out
+}
+
+// A single-disk machine should need no storage input at all, so the disk that
+// newModel preselects has to arrive already holding the os role.
+func TestPreselectedDiskIsAValidConfiguration(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda", Bytes: 200 << 30, LogicalBlockSize: 512}}, nil)
+	cfg := m.storage()
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("the preselected disk must be installable as-is: %v", err)
+	}
+	if cfg.Primary().Path != "/dev/sda" {
+		t.Errorf("Primary() = %q, want the preselected disk", cfg.Primary().Path)
+	}
+}
+
+// Picking disks in order gives the layout an operator almost always wants, so
+// the role key is only needed for the exceptions.
+func TestSelectingDisksAssignsRolesInOrder(t *testing.T) {
+	m := diskModel(3)
+	for i := range 3 {
+		m = m.toggleDisk(i)
+	}
+	got := rolePaths(m.storage())
+	for role, want := range map[install.DiskRole]string{
+		install.RoleOS: "a-disk", install.RoleSpinifex: "b-disk", install.RolePredastore: "c-disk",
+	} {
+		if got[role] != want {
+			t.Errorf("%s = %q, want %q", role, got[role], want)
+		}
+	}
+	if err := m.storage().Validate(); err != nil {
+		t.Errorf("the default assignment must be valid: %v", err)
+	}
+}
+
+// Deselecting has to drop the role with the disk, or every later role shifts
+// onto the wrong drive.
+func TestDeselectingADiskReleasesItsRole(t *testing.T) {
+	m := diskModel(3)
+	for i := range 3 {
+		m = m.toggleDisk(i)
+	}
+	m = m.toggleDisk(1) // drop the spinifex drive
+
+	got := rolePaths(m.storage())
+	if got[install.RoleOS] != "a-disk" || got[install.RolePredastore] != "c-disk" {
+		t.Fatalf("remaining roles moved: %v", got)
+	}
+	if _, ok := got[install.RoleSpinifex]; ok {
+		t.Errorf("the spinifex role should be free again: %v", got)
+	}
+}
+
+// The two-drive predastore layout is only reachable by reassigning, so cycling
+// swaps rather than refusing a role another disk already holds.
+func TestCycleRoleSwapsWithTheDiskThatHoldsIt(t *testing.T) {
+	m := diskModel(2)
+	m = m.toggleDisk(0).toggleDisk(1)
+	// b-disk: spinifex → predastore, which no disk holds, so no swap.
+	m = m.cycleRole(1)
+
+	got := rolePaths(m.storage())
+	if got[install.RoleOS] != "a-disk" || got[install.RolePredastore] != "b-disk" {
+		t.Fatalf("roles = %v, want os on a-disk and predastore on b-disk", got)
+	}
+
+	// Cycling past the end wraps onto os, which a-disk holds, so they trade.
+	m = m.cycleRole(1)
+	got = rolePaths(m.storage())
+	if got[install.RoleOS] != "b-disk" || got[install.RolePredastore] != "a-disk" {
+		t.Fatalf("roles = %v, want the two disks to have swapped", got)
+	}
+	if err := m.storage().Validate(); err != nil {
+		t.Errorf("a swap must always leave a valid assignment: %v", err)
+	}
+}
+
+// ZFS pairs mirrors by selection order and has no roles to assign.
+func TestStorageLeavesRolesUnsetOnZFS(t *testing.T) {
+	m := diskModel(2)
+	m.fsCursor = slices.Index(install.AllFSTypes, install.FSZFSRAID1)
+	m = m.toggleDisk(0).toggleDisk(1)
+
+	cfg := m.storage()
+	if len(cfg.Roles) != 0 {
+		t.Errorf("Roles = %v, want none on a pool", cfg.Roles)
+	}
+	if !slices.Equal(cfg.Paths(), []string{"a-disk", "b-disk"}) {
+		t.Errorf("Paths() = %v, want the selection order preserved", cfg.Paths())
+	}
+}
+
+// A three-drive selection looks like an array and is not one, so the mapping
+// and the absence of redundancy both have to be on screen before the erase.
+func TestConfirmScreenStatesTheLayoutAndTheRisk(t *testing.T) {
+	m := diskModel(3)
+	m.height = 40
+	for i := range 3 {
+		m = m.toggleDisk(i)
+	}
+	got := m.viewDiskConfirm(100)
+
+	for _, want := range []string{"/var/lib/spinifex/predastore", "c-disk", "losing one loses"} {
+		if !strings.Contains(got, want) {
+			t.Errorf("confirm screen is missing %q:\n%s", want, got)
+		}
+	}
+}

cmd/installer/ui/ui.go

@@ -47,9 +47,13 @@ type model struct {
 	//
 	// selected holds indexes into disks in the order they were picked, which is
 	// what pairs members for RAID10 — so it is a slice, not a set.
+	//
+	// roleOf runs parallel to selected and holds the ext4 drive role, so the
+	// two are edited together and cannot drift.
 	disks         []install.Disk
 	storageCursor int
 	selected      []int
+	roleOf        []install.DiskRole
 	fsCursor      int
 	zfsCursor     int
 	zfs           install.ZFSOpts
@@ -173,11 +177,14 @@ func newModel(disks []install.Disk, nics []netprobe.NIC) model {
 	}
 
 	// The first non-live, non-removable disk is preselected so a single-disk
-	// machine needs no storage input at all.
+	// machine needs no storage input at all. It takes the os role, since on ext4
+	// a selected disk with no role is not a valid configuration.
 	var preselected []int
+	var preselectedRoles []install.DiskRole
 	for i, d := range disks {
 		if !d.LiveMedia && !d.Removable {
 			preselected = []int{i}
+			preselectedRoles = []install.DiskRole{install.RoleOS}
 			break
 		}
 	}
@@ -186,6 +193,7 @@ func newModel(disks []install.Disk, nics []netprobe.NIC) model {
 		screen:               screenWelcome,
 		disks:                disks,
 		selected:             preselected,
+		roleOf:               preselectedRoles,
 		nics:                 nics,
 		eraseInput:           eraseIn,
 		roles:                roles,
@@ -439,11 +447,22 @@ func (m model) fsType() install.FSType { return install.AllFSTypes[m.fsCursor] }
 // storage assembles the disk configuration from the current selection.
 func (m model) storage() install.DiskConfig {
 	cfg := install.DiskConfig{FS: m.fsType(), ZFS: m.zfs}
-	for _, i := range m.selected {
-		if i < len(m.disks) {
-			cfg.Disks = append(cfg.Disks, m.disks[i])
+	if cfg.FS.IsZFS() {
+		for _, i := range m.selected {
+			if i < len(m.disks) {
+				cfg.Disks = append(cfg.Disks, m.disks[i])
+			}
 		}
+		return cfg
 	}
+	roles := make([]install.RoleMount, 0, len(m.selected))
+	for si, i := range m.selected {
+		if i >= len(m.disks) || si >= len(m.roleOf) {
+			continue
+		}
+		roles = append(roles, install.RoleMount{Role: m.roleOf[si], Disk: m.disks[i]})
+	}
+	cfg = cfg.WithRoles(roles)
 	return cfg
 }
 
@@ -479,6 +498,10 @@ func (m model) handleDiskKey(key string) (tea.Model, tea.Cmd) {
 			m.zfsCursor = 0
 			m.screen = screenZFSOptions
 		}
+	case "r":
+		if !m.fsType().IsZFS() && m.storageCursor != fsRow {
+			m = m.cycleRole(slices.Index(m.selected, m.storageCursor-1))
+		}
 	case "esc":
 		m.validationErr = ""
 		m.screen = screenWelcome
@@ -502,12 +525,45 @@ func (m model) handleDiskKey(key string) (tea.Model, tea.Cmd) {
 // array with the version bubbletea still holds.
 func (m model) toggleDisk(i int) model {
 	sel := slices.Clone(m.selected)
+	roles := slices.Clone(m.roleOf)
 	if at := slices.Index(sel, i); at >= 0 {
 		sel = slices.Delete(sel, at, at+1)
+		roles = slices.Delete(roles, at, at+1)
 	} else {
 		sel = append(sel, i)
+		roles = append(roles, freeRole(roles))
 	}
 	m.selected = sel
+	m.roleOf = roles
+	m.validationErr = ""
+	return m
+}
+
+// freeRole is the first role no selected disk holds yet, so picking disks in
+// order produces the usual layout without touching the role key at all.
+func freeRole(taken []install.DiskRole) install.DiskRole {
+	for _, r := range install.AllDiskRoles {
+		if !slices.Contains(taken, r) {
+			return r
+		}
+	}
+	return ""
+}
+
+// cycleRole moves the disk at selection index si to the next role, swapping
+// with whichever disk already holds it. Swapping keeps every press legal, so
+// the operator never has to unassign one drive before reassigning another.
+func (m model) cycleRole(si int) model {
+	if si < 0 || si >= len(m.roleOf) {
+		return m
+	}
+	roles := slices.Clone(m.roleOf)
+	next := install.AllDiskRoles[(slices.Index(install.AllDiskRoles, roles[si])+1)%len(install.AllDiskRoles)]
+	if other := slices.Index(roles, next); other >= 0 {
+		roles[other] = roles[si]
+	}
+	roles[si] = next
+	m.roleOf = roles
 	m.validationErr = ""
 	return m
 }
@@ -685,7 +741,7 @@ func (m model) viewDisk(w int) string {
 	subtitle := styleMuted.Render("All data on the selected disks will be permanently erased.")
 
 	fs := m.fsType()
-	req := "single disk"
+	req := "1-3 disks, one per role"
 	if n := fs.MinDisks(); n > 1 {
 		req = fmt.Sprintf("%d+ disks, same size", n)
 	} else if fs.IsZFS() {
@@ -702,8 +758,12 @@ func (m model) viewDisk(w int) string {
 	for i, d := range m.disks {
 		marker := "[ ]"
 		if at := slices.Index(m.selected, i); at >= 0 {
-			// Numbered, not ticked: the order is what pairs mirrors.
+			// Numbered, not ticked: the order is what pairs mirrors. On ext4 the
+			// role is what matters instead, so it takes the same column.
 			marker = fmt.Sprintf("[%d]", at+1)
+			if !fs.IsZFS() && at < len(m.roleOf) {
+				marker = fmt.Sprintf("[%s]", m.roleOf[at])
+			}
 		}
 		note := d.Content
 		switch {
@@ -712,7 +772,7 @@ func (m model) viewDisk(w int) string {
 		case d.Removable:
 			note += ", removable"
 		}
-		line := fmt.Sprintf("  %s %-16s %-8s %-20s %s", marker, d.Path, d.SizeHuman(), truncate(d.Model, 20), note)
+		line := fmt.Sprintf("  %-12s %-16s %-8s %-20s %s", marker, d.Path, d.SizeHuman(), truncate(d.Model, 20), note)
 		if m.storageCursor == i+1 {
 			line = styleSelected.Render("> " + strings.TrimPrefix(line, "  "))
 		} else {
@@ -736,7 +796,7 @@ func (m model) viewDisk(w int) string {
 		rows = append(rows, styleError.Render("  "+m.validationErr))
 	}
 
-	keys := "↑/↓ to move • ←/→ filesystem • Space to select disk • Enter to continue"
+	keys := "↑/↓ to move • ←/→ filesystem • Space to select disk • R to change its role • Enter to continue"
 	if fs.IsZFS() {
 		keys = "↑/↓ to move • ←/→ filesystem • Space to select disk • A for ZFS options • Enter to continue"
 	}
@@ -761,6 +821,11 @@ func (m model) geometryPreview() string {
 		return styleMuted.Render(fmt.Sprintf("  %s %s — %d selected.",
 			cfg.FS.Label(), cfg.Requirement(), len(cfg.Disks)))
 	}
+	// A role layout has no single capacity worth quoting — what matters is
+	// which drive each workload lands on.
+	if mounts := cfg.DataMounts(); len(mounts) > 0 {
+		return styleLabel.Render("  " + roleLayoutLine(cfg))
+	}
 	line := fmt.Sprintf("  %s across %d disk(s) — %s usable",
 		cfg.FS.Label(), len(cfg.Disks), humanBytes(cfg.UsableBytes()))
 	if n := cfg.Tolerated(); n > 0 {
@@ -769,6 +834,16 @@ func (m model) geometryPreview() string {
 	return styleLabel.Render(line)
 }
 
+// roleLayoutLine renders the mountpoint each drive will carry, which is the
+// only summary of a role install an operator can check at a glance.
+func roleLayoutLine(cfg install.DiskConfig) string {
+	parts := make([]string, 0, len(cfg.Roles))
+	for _, rm := range cfg.Roles {
+		parts = append(parts, fmt.Sprintf("%s → %s (%s)", rm.Mountpoint(), rm.Disk.Path, rm.Disk.SizeHuman()))
+	}
+	return strings.Join(parts, "   ")
+}
+
 func (m model) viewZFSOptions(w int) string {
 	title := styleTitle.Render("ZFS Options")
 	subtitle := styleMuted.Render("Defaults are computed from the selected disks and this machine's memory.")
@@ -793,12 +868,20 @@ func (m model) viewZFSOptions(w int) string {
 }
 
 func (m model) viewDiskConfirm(w int) string {
+	cfg := m.storage()
 	title := styleTitle.Render("Confirm Disk Erasure")
-	disk := styleLabel.Render(strings.Join(m.storage().Paths(), ", "))
+	disk := styleLabel.Render(strings.Join(cfg.Paths(), ", "))
 	msg := fmt.Sprintf("All data on %s will be permanently erased.\nType 'yes' to confirm:", disk)
 
 	var lines []string
-	lines = append(lines, title, msg, "", m.eraseInput.View())
+	lines = append(lines, title)
+	// A multi-drive ext4 layout resembles an array and is not one, so the
+	// mapping and the absence of redundancy are both stated before the erase.
+	if len(cfg.DataMounts()) > 0 {
+		lines = append(lines, styleLabel.Render(roleLayoutLine(cfg)),
+			styleWarning.Render("Each drive is independent — losing one loses everything it held."), "")
+	}
+	lines = append(lines, msg, "", m.eraseInput.View())
 	if m.validationErr != "" {
 		lines = append(lines, "", styleError.Render(m.validationErr))
 	}

cmd/rds-agent/acknowledge.go

@@ -0,0 +1,155 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mulgadc/spinifex/internal/rdsgw"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+)
+
+// The completion receipt rds-init writes once the master role is durably
+// applied. Deliberately outside PGDATA: clear_unfinished_datadir does an rm -rf
+// of PGDATA, so keeping the receipt out of it means no initdb-adjacent path can
+// touch it and the datadir stays a byte-for-byte stock one.
+const (
+	receiptRelDir  = ".spinifex-rds/bootstrap"
+	receiptFile    = "receipt.env"
+	receiptPayload = "RDS_RECEIPT_PAYLOAD_ID"
+	receiptDBID    = "RDS_RECEIPT_DB_INSTANCE_IDENTIFIER"
+)
+
+func (a *Agent) receiptPath() string {
+	return filepath.Join(a.cfg.DataMount, receiptRelDir, receiptFile)
+}
+
+// Confirms the initial bootstrap, which is what destroys the staged ciphertext.
+// A no-op on the attach path, and deliberately synchronous: it inherits the
+// cancellation semantics register, bootstrap and the data-mount wait already
+// share, and blocking the stateful loops behind it is correct because on a first
+// boot the engine is not bootstrapped until rds-init has finished anyway.
+//
+// It does not wait for the engine to be healthy. The receipt already proves the
+// master role was durably applied, and acknowledging early shortens the window
+// in which the ciphertext exists.
+func (a *Agent) acknowledgeBootstrap(ctx context.Context) error {
+	pending := a.pending
+	if pending == nil {
+		return nil
+	}
+
+	// Polled rather than waited on: rds-init runs after this agent registered,
+	// so the file appears some time into the boot. There is no give-up deadline
+	// here — the control plane's bootstrap timeout already fails the instance,
+	// and a second one in the guest would only give the two a way to disagree.
+	if err := retryObserved(ctx, "bootstrap receipt", func(context.Context) error {
+		return a.checkBootstrapReceipt(pending)
+	}, func(err error) {
+		// An absent receipt is the expected state for the whole initdb window,
+		// so reporting it would put a failure on every healthy create's beat and
+		// then into the reason a timed-out create records.
+		if errors.Is(err, fs.ErrNotExist) {
+			return
+		}
+		a.hb.setBootstrapFailure("bootstrap receipt", err)
+	}); err != nil {
+		return err
+	}
+	a.hb.clearBootstrapFailure()
+
+	if err := retryObserved(ctx, "bootstrap acknowledgement", func(ctx context.Context) error {
+		return terminalOnDenial(a.cp.AcknowledgeBootstrap(ctx, a.id, pending))
+	}, func(err error) {
+		a.hb.setBootstrapFailure("bootstrap acknowledgement", err)
+	}); err != nil {
+		if !errors.Is(err, errRetryTerminal) {
+			return err
+		}
+		// The staged payload will never be delivered to this VM, and blocking the
+		// command poller and the parameter guard on that would leave a serving
+		// database no one can administer. The reason rides the heartbeat instead.
+		slog.ErrorContext(ctx, "rds-agent: the initial bootstrap cannot be acknowledged",
+			"payloadId", pending.payloadID, "err", err)
+		return nil
+	}
+	a.hb.clearBootstrapFailure()
+
+	// AcknowledgedAt on the record is the durable audit trail, so the receipt is
+	// removed rather than archived: keeping it would only grow the surface of
+	// stale receipts riding along in data-volume snapshots. A removal failure is
+	// harmless, since a later boot fetches attach and never consults one.
+	if err := os.Remove(a.receiptPath()); err != nil && !os.IsNotExist(err) {
+		slog.WarnContext(ctx, "rds-agent: could not remove the bootstrap receipt",
+			"receipt", a.receiptPath(), "err", err)
+	}
+	slog.InfoContext(ctx, "rds-agent: initial bootstrap acknowledged", "payloadId", pending.payloadID)
+	return nil
+}
+
+// The control plane resolved these from its own record — a superseded
+// generation, a payload it never staged, a data volume that is not this
+// instance's. Asking again cannot change any of them.
+func terminalOnDenial(err error) error {
+	var apiErr *rdsgw.APIError
+	if !errors.As(err, &apiErr) {
+		return err
+	}
+	switch apiErr.Code {
+	case awserrors.ErrorAccessDenied, awserrors.ErrorInvalidParameterValue:
+		return fmt.Errorf("%w: %w", errRetryTerminal, err)
+	}
+	return err
+}
+
+// A receipt naming another payload or another DB instance is treated as absent.
+// The receipt lives on the data volume, so it rides along in every snapshot of
+// it and a restored instance's volume carries the source instance's receipt.
+func (a *Agent) checkBootstrapReceipt(pending *pendingBootstrap) error {
+	path := a.receiptPath()
+	values, err := readReceipt(path)
+	if err != nil {
+		return err
+	}
+	if got := values[receiptPayload]; got != pending.payloadID {
+		return fmt.Errorf("%s names bootstrap payload %q, not %q", path, got, pending.payloadID)
+	}
+	if got := values[receiptDBID]; got != a.id.DBInstanceIdentifier {
+		return fmt.Errorf("%s names DB instance %q, not %q", path, got, a.id.DBInstanceIdentifier)
+	}
+	return nil
+}
+
+// KEY=value, the one format both readers can parse: rds-init in POSIX shell
+// with no guaranteed jq, and this.
+func readReceipt(path string) (map[string]string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	values := make(map[string]string)
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		key, value, ok := strings.Cut(line, "=")
+		if !ok {
+			continue
+		}
+		values[strings.TrimSpace(key)] = strings.TrimSpace(value)
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("read %s: %w", path, err)
+	}
+	return values, nil
+}

cmd/rds-agent/agent.go

@@ -1,11 +1,15 @@
 package main
 
 import (
+	"bufio"
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
 	"net/url"
+	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/mulgadc/spinifex/internal/gwsign"
@@ -30,9 +34,12 @@ type identity struct {
 type controlPlane interface {
 	Register(ctx context.Context, id identity) (*handlers_rds.RegisterDBInstanceOutput, error)
 	SubmitState(ctx context.Context, id identity, health handlers_rds.EngineHealth, message string) (*handlers_rds.SubmitDBStateChangeOutput, error)
-	// The first call of an instance's life returns Mode=initialize with the
-	// master password; every later call returns Mode=attach without it.
+	// Returns Mode=initialize with the master password for as long as a payload
+	// staged for this VM generation is unacknowledged, and Mode=attach after.
 	GetBootstrapConfig(ctx context.Context, id identity) (*handlers_rds.GetDBBootstrapConfigOutput, error)
+	// Reports that PostgreSQL durably applied the staged password, which is what
+	// lets the control plane destroy it.
+	AcknowledgeBootstrap(ctx context.Context, id identity, pending *pendingBootstrap) error
 	PollCommands(ctx context.Context, id identity, replies []handlers_rds.CommandReply, wait time.Duration) ([]handlers_rds.Command, error)
 }
 
@@ -112,6 +119,27 @@ func (g *gatewayControlPlane) GetBootstrapConfig(ctx context.Context, id identit
 	return &out, nil
 }
 
+func (g *gatewayControlPlane) AcknowledgeBootstrap(ctx context.Context, id identity, pending *pendingBootstrap) error {
+	params := url.Values{
+		"PayloadId":    {pending.payloadID},
+		"VMGeneration": {strconv.FormatInt(pending.vmGeneration, 10)},
+	}
+	setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
+	setIfPresent(params, "DataVolumeId", pending.dataVolumeID)
+
+	ctx, cancel := context.WithTimeout(ctx, callTimeout)
+	defer cancel()
+
+	var out handlers_rds.AcknowledgeDBBootstrapOutput
+	if err := g.client.Call(ctx, "AcknowledgeDBBootstrap", params, &out); err != nil {
+		return err
+	}
+	if !out.Acknowledged {
+		return fmt.Errorf("the control plane did not acknowledge bootstrap payload %s", pending.payloadID)
+	}
+	return nil
+}
+
 // Declared here so the in-guest binary does not link the server side of the API.
 type pollDBCommandsOutput struct {
 	Commands []handlers_rds.Command `xml:"Commands>member"`
@@ -148,12 +176,17 @@ func setIfPresent(params url.Values, key, value string) {
 }
 
 type Agent struct {
-	cfg           config
-	id            identity
-	cp            controlPlane
-	probe         *engineProbe
-	engine        *postgresEngine
-	handoffWriter func(string, *handlers_rds.GetDBBootstrapConfigOutput) error
+	cfg             config
+	id              identity
+	cp              controlPlane
+	probe           *engineProbe
+	engine          engine
+	handoffWriter   func(string, *handlers_rds.GetDBBootstrapConfigOutput) error
+	dataMountWaiter func(context.Context, string, string) error
+
+	// Set by bootstrap when the fetch replayed a staged payload; nil leaves the
+	// acknowledgement step a no-op, which is every attach boot.
+	pending *pendingBootstrap
 
 	hb    *heartbeater
 	cmd   *commander
@@ -162,18 +195,28 @@ type Agent struct {
 
 // Assembles from already-built parts, so tests can pass fakes; New builds the
 // production control plane and delegates here.
-func newAgent(cfg config, cp controlPlane, probe *engineProbe) *Agent {
+func newAgent(cfg config, cp controlPlane, probe *engineProbe) (*Agent, error) {
+	if cfg.MountsFile == "" {
+		cfg.MountsFile = defaultMountsFile
+	}
 	id := identity{
 		DBInstanceIdentifier: cfg.DBInstanceIdentifier,
 		AgentVersion:         version,
 		EngineVersion:        cfg.EngineVersion,
 	}
-	a := &Agent{cfg: cfg, id: id, cp: cp, probe: probe, handoffWriter: writeHandoff}
-	a.engine = newPostgresEngine(cfg, execCommandRunner, execSessionRunner)
-	a.hb = newHeartbeater(cp, probe, handlers_rds.HeartbeatInterval)
+	a := &Agent{
+		cfg: cfg, id: id, cp: cp, probe: probe,
+		handoffWriter: writeHandoff, dataMountWaiter: waitForDataMount,
+	}
+	eng, err := newEngine(cfg, execCommandRunner, execSessionRunner, probe)
+	if err != nil {
+		return nil, err
+	}
+	a.engine = eng
+	a.hb = newHeartbeater(cp, probe, a.engine, handlers_rds.HeartbeatInterval)
 	a.cmd = newCommander(cp, newCommandRegistry(a.engine, newGuestStorage(cfg, execCommandRunner)), cfg.PollWait)
-	a.guard = newParamGuard(a.engine, probe)
-	return a
+	a.guard = newParamGuard(a.engine, probe, cp)
+	return a, nil
 }
 
 // Does not wait for IMDS: the register loop rides out a datapath still coming up.
@@ -185,7 +228,11 @@ func New(cfg config) (*Agent, error) {
 	if err != nil {
 		return nil, err
 	}
-	return newAgent(cfg, cp, newEngineProbe(cfg, execProbeRunner)), nil
+	probe, err := newProbe(cfg, execProbeRunner)
+	if err != nil {
+		return nil, err
+	}
+	return newAgent(cfg, cp, probe)
 }
 
 func (a *Agent) Run(ctx context.Context) error {
@@ -201,7 +248,14 @@ func (a *Agent) Run(ctx context.Context) error {
 	if err := a.register(ctx); err != nil {
 		return err
 	}
-	a.hb.id, a.cmd.id = a.id, a.id
+	a.hb.id, a.cmd.id, a.guard.id = a.id, a.id, a.id
+
+	// Ahead of the handoff, so rds-init never unblocks and no datadir is touched.
+	// An agent running on the wrong image would initialise one engine over
+	// another's data, coming up healthy and empty beside data nothing references.
+	if err := a.refuseEngineMismatch(ctx); err != nil {
+		return err
+	}
 
 	// Beating before the bootstrap keeps a stuck boot visible as a live VM with
 	// a down engine rather than as silence.
@@ -211,7 +265,22 @@ func (a *Agent) Run(ctx context.Context) error {
 		return err
 	}
 
-	// Directives are only meaningful against a bootstrapped engine.
+	// The handoff must land before this wait: rds-datadir needs it to identify
+	// and authorize the volume. Commands and the parameter guard wait because
+	// their durable state belongs on that mount, never on the disposable boot disk.
+	if err := a.dataMountWaiter(ctx, a.cfg.MountsFile, a.cfg.DataMount); err != nil {
+		// A shutdown cancels this wait; main treats the wrapped context.Canceled
+		// as a clean stop, exactly as it does for register and bootstrap above.
+		return fmt.Errorf("wait for data mount %s: %w", a.cfg.DataMount, err)
+	}
+
+	// The receipt it waits on lives on that mount, so this cannot run any earlier.
+	// A cancelled wait is a clean stop here too.
+	if err := a.acknowledgeBootstrap(ctx); err != nil {
+		return err
+	}
+
+	// Directives are only meaningful against a bootstrapped, mounted engine.
 	go a.cmd.Run(ctx)
 
 	// Started after the handoff, so the window it measures covers the engine's
@@ -222,17 +291,104 @@ func (a *Agent) Run(ctx context.Context) error {
 	return nil
 }
 
+// The engine cloud-init says this VM was launched as, against the engine the
+// image bakes. An empty expectation is no assertion rather than a disagreement:
+// a VM launched before the control plane emitted RDS_ENGINE carries none.
+func (a *Agent) refuseEngineMismatch(ctx context.Context) error {
+	expected := strings.ToLower(strings.TrimSpace(a.cfg.Engine))
+	if expected == "" || expected == a.cfg.BakedEngine {
+		return nil
+	}
+
+	message := fmt.Sprintf("this VM was launched as engine %q but its image bakes %q; refusing to bootstrap",
+		expected, a.cfg.BakedEngine)
+	slog.ErrorContext(ctx, "rds-agent: "+message)
+	// Reported before returning, so the control plane learns why the instance
+	// never came up rather than only that it never did.
+	if _, err := a.cp.SubmitState(ctx, a.id, handlers_rds.EngineHealthUnhealthy, message); err != nil {
+		slog.ErrorContext(ctx, "rds-agent: reporting the engine mismatch failed", "err", err)
+	}
+	return errors.New(message)
+}
+
+const dataMountPollInterval = time.Second
+
+// waitForDataMount blocks the stateful agent loops until the kernel reports the
+// configured data mount. Registration, heartbeat, and bootstrap run before it.
+func waitForDataMount(ctx context.Context, mountsFile, target string) error {
+	ticker := time.NewTicker(dataMountPollInterval)
+	defer ticker.Stop()
+
+	for {
+		mounted, err := mountTableContains(mountsFile, target)
+		if err != nil {
+			slog.WarnContext(ctx, "rds-agent: could not read mount table while waiting for data volume",
+				"mountsFile", mountsFile, "dataMount", target, "err", err)
+		} else if mounted {
+			slog.InfoContext(ctx, "rds-agent: data volume mount is ready", "dataMount", target)
+			return nil
+		}
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+		}
+	}
+}
+
+func mountTableContains(path, target string) (bool, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		fields := strings.Fields(scanner.Text())
+		if len(fields) >= 2 && unescapeMountField(fields[1]) == target {
+			return true, nil
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return false, err
+	}
+	return false, nil
+}
+
+// Linux mount tables encode whitespace and backslashes as octal escapes.
+func unescapeMountField(field string) string {
+	replacer := strings.NewReplacer(`\134`, `\`, `\040`, " ", `\011`, "\t", `\012`, "\n")
+	return replacer.Replace(field)
+}
+
 // Boot-retry bounds. Tight to start, since the usual cause is a control plane
 // still creating this instance's record, and capped so an outage does not turn
 // a booting fleet into a retry storm.
 const (
 	retryMin = 1 * time.Second
 	retryMax = 30 * time.Second
+
+	// With the production backoff, attempt five is about 15 seconds into the
+	// failure and still precedes the first 30-second heartbeat.
+	retryErrorAttempt = 5
 )
 
+// Wrapping an error in this stops retryObserved. For the failures the control
+// plane decides from its own record, no retry can change the answer, and looping
+// on one only keeps the caller blocked.
+var errRetryTerminal = errors.New("terminal failure")
+
 // Never gives up on its own: an agent that stopped trying would leave the
 // control plane with no signal at all.
 func retry(ctx context.Context, what string, fn func(context.Context) error) error {
+	return retryObserved(ctx, what, fn, nil)
+}
+
+// onFailure lets boot-critical callers publish the latest error while the retry
+// remains blocked. It must not block because it runs on the retry path itself.
+func retryObserved(ctx context.Context, what string, fn func(context.Context) error, onFailure func(error)) error {
 	delay := retryMin
 	for attempt := 1; ; attempt++ {
 		err := fn(ctx)
@@ -242,7 +398,20 @@ func retry(ctx context.Context, what string, fn func(context.Context) error) err
 		if ctx.Err() != nil {
 			return fmt.Errorf("%s: %w", what, ctx.Err())
 		}
-		slog.Warn("rds-agent: "+what+" failed, retrying", "attempt", attempt, "retryIn", delay, "err", err)
+		if onFailure != nil {
+			onFailure(err)
+		}
+		if errors.Is(err, errRetryTerminal) {
+			slog.ErrorContext(ctx, "rds-agent: "+what+" failed terminally, not retrying", "err", err)
+			return fmt.Errorf("%s: %w", what, err)
+		}
+		if attempt >= retryErrorAttempt {
+			slog.ErrorContext(ctx, "rds-agent: "+what+" still failing, retrying",
+				"attempt", attempt, "retryIn", delay, "err", err)
+		} else {
+			slog.WarnContext(ctx, "rds-agent: "+what+" failed, retrying",
+				"attempt", attempt, "retryIn", delay, "err", err)
+		}
 
 		select {
 		case <-ctx.Done():

cmd/rds-agent/agent_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -15,9 +16,12 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/gwsign"
 	"github.com/mulgadc/spinifex/internal/rdsgw"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
 	gateway_rds "github.com/mulgadc/spinifex/spinifex/gateway/rds"
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 // fakeControlPlane is a scriptable controlPlane. Every call records what the
@@ -32,6 +36,12 @@ type fakeControlPlane struct {
 	bootstrapOut  *handlers_rds.GetDBBootstrapConfigOutput
 	bootstrapErrs []error
 	bootstrapReqs []identity
+	// Scripted responses ahead of bootstrapOut, so a test can answer the first
+	// fetch differently from the ones the agent retries with.
+	bootstrapQueue []*handlers_rds.GetDBBootstrapConfigOutput
+
+	ackErrs []error
+	ackReqs []pendingBootstrap
 
 	states []submittedState
 
@@ -95,9 +105,21 @@ func (f *fakeControlPlane) GetBootstrapConfig(_ context.Context, id identity) (*
 	if err := nextErr(&f.bootstrapErrs); err != nil {
 		return nil, err
 	}
+	if len(f.bootstrapQueue) > 0 {
+		out := f.bootstrapQueue[0]
+		f.bootstrapQueue = f.bootstrapQueue[1:]
+		return out, nil
+	}
 	return f.bootstrapOut, nil
 }
 
+func (f *fakeControlPlane) AcknowledgeBootstrap(_ context.Context, _ identity, pending *pendingBootstrap) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.ackReqs = append(f.ackReqs, *pending)
+	return nextErr(&f.ackErrs)
+}
+
 func (f *fakeControlPlane) PollCommands(ctx context.Context, _ identity, replies []handlers_rds.CommandReply, _ time.Duration) ([]handlers_rds.Command, error) {
 	f.mu.Lock()
 	f.pollReplies = append(f.pollReplies, slices.Clone(replies))
@@ -124,6 +146,12 @@ func (f *fakeControlPlane) PollCommands(ctx context.Context, _ identity, replies
 	return nil, ctx.Err()
 }
 
+func (f *fakeControlPlane) ackRequests() []pendingBootstrap {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return slices.Clone(f.ackReqs)
+}
+
 func (f *fakeControlPlane) snapshotStates() []submittedState {
 	f.mu.Lock()
 	defer f.mu.Unlock()
@@ -140,18 +168,26 @@ func (f *fakeControlPlane) snapshotReplies() [][]handlers_rds.CommandReply {
 	return out
 }
 
-// testConfig points the agent's file output at a temp dir and its probe at a
-// name no real binary answers to.
+// testConfig stamps the engine a real rds-postgres image bakes, so the agent
+// resolves the same guest layout it would in a guest, and points its file output
+// at a temp dir.
 func testConfig(t *testing.T) config {
 	t.Helper()
-	return config{
-		GatewayURL: "https://gw.test",
-		HandoffDir: filepath.Join(t.TempDir(), "spinifex-rds"),
-		EngineHost: defaultEngineHost,
-		EnginePort: defaultEnginePort,
-		PGIsReady:  "pg_isready",
-		PollWait:   time.Second,
+	cfg := testLoadConfig(t, enginePostgres)
+	cfg.GatewayURL = "https://gw.test"
+	cfg.HandoffDir = filepath.Join(t.TempDir(), "spinifex-rds")
+	cfg.PollWait = time.Second
+	return cfg
+}
+
+// newTestAgent builds the agent over a probe the test drives directly.
+func newTestAgent(t *testing.T, cfg config, cp controlPlane, run probeRunner) *Agent {
+	t.Helper()
+	a, err := newAgent(cfg, cp, newPostgresProbe(cfg, run))
+	if err != nil {
+		t.Fatalf("newAgent: %v", err)
 	}
+	return a
 }
 
 // staticProbe is a probeRunner that always reports the same exit code.
@@ -171,6 +207,27 @@ func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput
 		MasterUsername:       "master",
 		MasterUserPassword:   password,
 		Port:                 5432,
+		DataVolumeID:         "vol-data-01",
+		DataVolumeSerial:     "voldata01",
+		VMGeneration:         1,
+		PayloadID:            testPayloadID,
+		BootstrapPending:     password != nil,
+	}
+}
+
+const testPayloadID = "bp-0123456789abcdef"
+
+// Writes the completion receipt rds-init leaves behind, so the acknowledgement
+// step finds what a finished initialization would have produced.
+func writeReceipt(t *testing.T, a *Agent, payloadID, dbInstanceIdentifier string) {
+	t.Helper()
+	path := a.receiptPath()
+	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		t.Fatalf("mkdir receipt dir: %v", err)
+	}
+	body := fmt.Sprintf("%s=%s\n%s=%s\n", receiptPayload, payloadID, receiptDBID, dbInstanceIdentifier)
+	if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
+		t.Fatalf("write receipt: %v", err)
 	}
 }
 
@@ -192,6 +249,11 @@ func runAgent(t *testing.T, a *Agent) error {
 	cancel()
 	select {
 	case err := <-done:
+		// The cancellation above is this helper's own, so Run reporting it is the
+		// expected clean stop rather than a failure, exactly as main treats it.
+		if errors.Is(err, context.Canceled) {
+			return nil
+		}
 		return err
 	case <-time.After(5 * time.Second):
 		t.Fatal("Run did not exit after cancellation")
@@ -218,7 +280,7 @@ func TestRun_RegistersThenDeliversBootstrap(t *testing.T) {
 	cp.bootstrapOut = bootstrapOutput(&password)
 
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	if err := runAgent(t, a); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
@@ -249,7 +311,7 @@ func TestRun_AttachModeNeedsNoPassword(t *testing.T) {
 	cp.bootstrapOut = bootstrapOutput(nil)
 
 	cfg := testConfig(t)
-	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
 
@@ -271,7 +333,7 @@ func TestRun_RetriesRegisterAndBootstrap(t *testing.T) {
 	cp.bootstrapErrs = []error{errors.New("ServerInternal")}
 
 	cfg := testConfig(t)
-	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
 
@@ -283,12 +345,14 @@ func TestRun_RetriesRegisterAndBootstrap(t *testing.T) {
 	}
 }
 
-func TestBootstrap_RetriesHandoffWithoutRefetchingConsumedPassword(t *testing.T) {
+// The fetch mutates nothing, so a handoff that cannot be written is retried
+// against the payload already in hand rather than costing a second fetch.
+func TestBootstrap_RetriesHandoffWithoutRefetching(t *testing.T) {
 	cp := newFakeControlPlane()
-	password := "one-shot-secret"
+	password := "staged-secret"
 	cp.bootstrapOut = bootstrapOutput(&password)
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 
 	writes := 0
 	a.handoffWriter = func(_ string, got *handlers_rds.GetDBBootstrapConfigOutput) error {
@@ -311,6 +375,163 @@ func TestBootstrap_RetriesHandoffWithoutRefetchingConsumedPassword(t *testing.T)
 	if writes != 2 {
 		t.Errorf("handoff writes = %d, want 2", writes)
 	}
+	if a.pending == nil || a.pending.payloadID != testPayloadID {
+		t.Errorf("pending bootstrap = %+v, want the staged payload %s", a.pending, testPayloadID)
+	}
+}
+
+// A daemon that predates the encrypted payload answers initialize with nothing
+// to initialise with. Writing that handoff would let rds-init run initdb with an
+// empty master password, so it is retried until an upgraded node answers.
+func TestBootstrap_RetriesInitializeWithNoPassword(t *testing.T) {
+	cp := newFakeControlPlane()
+	empty := ""
+	password := "staged-secret"
+	cp.bootstrapQueue = []*handlers_rds.GetDBBootstrapConfigOutput{{
+		Mode: handlers_rds.BootstrapModeInitialize, DBInstanceIdentifier: "db-resolved",
+		MasterUsername: "master", MasterUserPassword: &empty,
+	}}
+	cp.bootstrapOut = bootstrapOutput(&password)
+	cfg := testConfig(t)
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+
+	var handed []*handlers_rds.GetDBBootstrapConfigOutput
+	a.handoffWriter = func(_ string, got *handlers_rds.GetDBBootstrapConfigOutput) error {
+		handed = append(handed, got)
+		return nil
+	}
+
+	if err := a.bootstrap(t.Context()); err != nil {
+		t.Fatalf("bootstrap: %v", err)
+	}
+	if len(handed) != 1 {
+		t.Fatalf("handoff writes = %d, want 1 once a usable payload arrived", len(handed))
+	}
+	if handed[0].MasterUserPassword == nil || *handed[0].MasterUserPassword != password {
+		t.Error("the agent wrote a handoff for a fetch that carried no master password")
+	}
+}
+
+func TestAcknowledgeBootstrap_WaitsForTheReceiptThenRetiresIt(t *testing.T) {
+	cp := newFakeControlPlane()
+	cfg := testConfig(t)
+	cfg.DataMount = t.TempDir()
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+	a.id = identity{DBInstanceIdentifier: "db-resolved"}
+	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1, dataVolumeID: "vol-data-01"}
+
+	// rds-init runs after the agent registers, so the receipt is absent when the
+	// acknowledgement step first looks and appears some way into the boot.
+	if err := a.checkBootstrapReceipt(a.pending); err == nil {
+		t.Fatal("a missing receipt must not read as a completed bootstrap")
+	}
+	writeReceipt(t, a, testPayloadID, "db-resolved")
+
+	if err := a.acknowledgeBootstrap(t.Context()); err != nil {
+		t.Fatalf("acknowledgeBootstrap: %v", err)
+	}
+	if len(cp.ackRequests()) != 1 {
+		t.Fatalf("acknowledgements = %d, want 1", len(cp.ackRequests()))
+	}
+	if got := cp.ackRequests()[0]; got != *a.pending {
+		t.Errorf("acknowledged %+v, want %+v", got, *a.pending)
+	}
+	if _, err := os.Stat(a.receiptPath()); !os.IsNotExist(err) {
+		t.Error("the receipt must be removed once the record holds the audit trail")
+	}
+}
+
+// The receipt rides along in every snapshot of the data volume, so a restored
+// instance's volume carries the source instance's receipt. Acknowledging on one
+// would destroy the ciphertext this instance still needs.
+func TestAcknowledgeBootstrap_IgnoresAForeignReceipt(t *testing.T) {
+	tests := map[string]struct{ payloadID, dbID string }{
+		"another payload":  {payloadID: "bp-somebody-else", dbID: "db-resolved"},
+		"another instance": {payloadID: testPayloadID, dbID: "db-the-snapshot-source"},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			cp := newFakeControlPlane()
+			cfg := testConfig(t)
+			cfg.DataMount = t.TempDir()
+			a := newTestAgent(t, cfg, cp, staticProbe(0))
+			a.id = identity{DBInstanceIdentifier: "db-resolved"}
+			a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
+			writeReceipt(t, a, tc.payloadID, tc.dbID)
+
+			ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
+			defer cancel()
+			if err := a.acknowledgeBootstrap(ctx); err == nil {
+				t.Fatal("acknowledgeBootstrap returned nil, want it still waiting on a real receipt")
+			}
+			if n := len(cp.ackRequests()); n != 0 {
+				t.Errorf("acknowledgements = %d, want 0", n)
+			}
+		})
+	}
+}
+
+// An attach boot has nothing staged, so it must not go looking for a receipt the
+// datadir will never grow.
+func TestAcknowledgeBootstrap_IsANoOpOnAttach(t *testing.T) {
+	cp := newFakeControlPlane()
+	cfg := testConfig(t)
+	cfg.DataMount = t.TempDir()
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+
+	if err := a.acknowledgeBootstrap(t.Context()); err != nil {
+		t.Fatalf("acknowledgeBootstrap: %v", err)
+	}
+	if n := len(cp.ackRequests()); n != 0 {
+		t.Errorf("acknowledgements = %d, want 0 when nothing was staged", n)
+	}
+}
+
+// The image is the authority on which engine the agent runs. A VM launched as
+// another engine has to stop ahead of the handoff: rds-init would otherwise
+// initialise one engine over the other's datadir and come up healthy and empty
+// beside data nothing references.
+func TestRun_RefusesAnEngineTheImageDoesNotBake(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+
+	cfg := testConfig(t)
+	cfg.Engine = "mariadb"
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+
+	err := a.Run(t.Context())
+	if err == nil {
+		t.Fatal("Run returned nil for a VM launched as an engine this image does not bake")
+	}
+	if !strings.Contains(err.Error(), "mariadb") {
+		t.Errorf("error = %v, want it to name the engine the VM was launched as", err)
+	}
+	if _, statErr := os.Stat(filepath.Join(cfg.HandoffDir, handoffEnvFile)); !os.IsNotExist(statErr) {
+		t.Error("the agent wrote a bootstrap handoff for an engine it refused to run")
+	}
+	if n := len(cp.bootstrapReqs); n != 0 {
+		t.Errorf("bootstrap fetches = %d, want none before the refusal", n)
+	}
+	// Reported rather than only logged, so the control plane learns why the
+	// instance never came up rather than only that it never did.
+	states := cp.snapshotStates()
+	if len(states) != 1 || states[0].health != handlers_rds.EngineHealthUnhealthy {
+		t.Fatalf("submitted states = %+v, want a single unhealthy report", states)
+	}
+}
+
+// The same assertion when it agrees is not a gate: the boot goes on exactly as
+// it does for a VM launched before the control plane sent one at all.
+func TestRun_ProceedsWhenTheDeliveredEngineMatchesTheImage(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+
+	cfg := testConfig(t)
+	cfg.Engine = enginePostgres
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
 }
 
 // A configured identifier is asserted on the wire so a mis-provisioned VM is
@@ -322,7 +543,7 @@ func TestRun_SendsConfiguredIdentifier(t *testing.T) {
 	cfg := testConfig(t)
 	cfg.DBInstanceIdentifier = "db-configured"
 	cfg.EngineVersion = "18.1"
-	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
 
@@ -332,6 +553,108 @@ func TestRun_SendsConfiguredIdentifier(t *testing.T) {
 	}
 }
 
+func TestRun_WaitsForDataMountBeforePollingCommands(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+	cfg := testConfig(t)
+	a := newTestAgent(t, cfg, cp, staticProbe(2))
+
+	waiting := make(chan struct{})
+	release := make(chan struct{})
+	a.dataMountWaiter = func(ctx context.Context, mountsFile, target string) error {
+		if mountsFile != defaultMountsFile || target != cfg.DataMount {
+			t.Errorf("mount waiter got %q/%q, want configured defaults", mountsFile, target)
+		}
+		close(waiting)
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-release:
+			return nil
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	select {
+	case <-waiting:
+	case <-ctx.Done():
+		t.Fatal("agent did not reach the data-mount wait after writing the handoff")
+	}
+	if _, err := os.Stat(filepath.Join(cfg.HandoffDir, handoffEnvFile)); err != nil {
+		t.Fatalf("handoff was not present before mount wait: %v", err)
+	}
+	select {
+	case <-cp.polled:
+		t.Fatal("command poll started before the data volume was mounted")
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	close(release)
+	select {
+	case <-cp.polled:
+	case <-ctx.Done():
+		t.Fatal("command poll did not start after the data mount became ready")
+	}
+	cancel()
+	if err := <-done; err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+}
+
+func TestMountTableContains(t *testing.T) {
+	mounts := filepath.Join(t.TempDir(), "mounts")
+	require.NoError(t, os.WriteFile(mounts, []byte(
+		"/dev/vda1 / ext4 rw 0 0\n/dev/vdb /var/lib/postgresql ext4 rw 0 0\n"+
+			"/dev/vdc /path\\040with\\040spaces xfs rw 0 0\nmalformed\n"), 0o600))
+
+	tests := []struct {
+		target string
+		want   bool
+	}{
+		{target: "/var/lib/postgresql", want: true},
+		{target: "/path with spaces", want: true},
+		{target: "/missing", want: false},
+	}
+	for _, tc := range tests {
+		t.Run(tc.target, func(t *testing.T) {
+			got, err := mountTableContains(mounts, tc.target)
+			require.NoError(t, err)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}
+
+func TestRun_HeartbeatIncludesBootstrapFailure(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapErrs = []error{errors.New("403 AccessDenied: instance profile is not authorised")}
+	cp.registerOut.HeartbeatIntervalSeconds = 0
+
+	cfg := testConfig(t)
+	a := newTestAgent(t, cfg, cp, staticProbe(2))
+	a.hb.interval = 10 * time.Millisecond
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	waitFor(t, func() bool {
+		for _, state := range cp.snapshotStates() {
+			if strings.Contains(state.message, "bootstrap fetch failed") &&
+				strings.Contains(state.message, "403 AccessDenied") {
+				return true
+			}
+		}
+		return false
+	}, "the bootstrap failure to reach a heartbeat")
+	cancel()
+	<-done
+}
+
 func TestRun_HeartbeatsEngineHealth(t *testing.T) {
 	cp := newFakeControlPlane()
 	cp.bootstrapOut = bootstrapOutput(nil)
@@ -340,7 +663,7 @@ func TestRun_HeartbeatsEngineHealth(t *testing.T) {
 	cp.registerOut.HeartbeatIntervalSeconds = 0
 
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	a.hb.interval = 10 * time.Millisecond
 
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
@@ -370,7 +693,7 @@ func TestRegister_AdoptsControlPlaneCadence(t *testing.T) {
 	cp.registerOut.HeartbeatIntervalSeconds = 45
 
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	if err := runAgent(t, a); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
@@ -387,12 +710,15 @@ func TestRun_HeartbeatReportsDownEngine(t *testing.T) {
 	cp.registerOut.HeartbeatIntervalSeconds = 0
 
 	cfg := testConfig(t)
-	probe := newEngineProbe(cfg, staticProbe(2))
+	probe := newPostgresProbe(cfg, staticProbe(2))
 	// The engine has served before, so a silent one now is a failure rather
 	// than a boot still in progress.
 	probe.seenHealthy = true
 
-	a := newAgent(cfg, cp, probe)
+	a, err := newAgent(cfg, cp, probe)
+	if err != nil {
+		t.Fatalf("newAgent: %v", err)
+	}
 	a.hb.interval = 10 * time.Millisecond
 
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
@@ -470,3 +796,69 @@ func readFile(t *testing.T, path string) string {
 	}
 	return string(body)
 }
+
+// AccessDenied is decided by the control plane's own record, so no retry can
+// change it. Looping on one would leave the command poller and the parameter
+// guard unstarted for the life of a VM whose engine is serving normally.
+func TestAcknowledgeBootstrap_StopsOnATerminalDenial(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.ackErrs = []error{&rdsgw.APIError{
+		Action: "AcknowledgeDBBootstrap", StatusCode: 403,
+		Code:    awserrors.ErrorAccessDenied,
+		Message: "bootstrap payload bp-x at generation 1 does not match the current state",
+	}}
+	cfg := testConfig(t)
+	cfg.DataMount = t.TempDir()
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+	a.id = identity{DBInstanceIdentifier: "db-resolved"}
+	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
+	writeReceipt(t, a, testPayloadID, "db-resolved")
+
+	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+	defer cancel()
+	if err := a.acknowledgeBootstrap(ctx); err != nil {
+		t.Fatalf("acknowledgeBootstrap = %v, want nil so the stateful loops still start", err)
+	}
+	if n := len(cp.ackRequests()); n != 1 {
+		t.Errorf("acknowledgements = %d, want 1 attempt and no retry", n)
+	}
+	if a.hb.bootstrapFailure.Load() == nil {
+		t.Error("a terminal denial must still reach the operator through the heartbeat")
+	}
+	if _, err := os.Stat(a.receiptPath()); err != nil {
+		t.Error("a denied acknowledgement must leave the receipt for a later attempt")
+	}
+}
+
+// rds-init runs after this agent, so the receipt is absent for the whole initdb
+// window. Reporting that would put a failure on every healthy create's beat, and
+// then into the reason a create that timed out for some other cause records.
+func TestAcknowledgeBootstrap_DoesNotReportAnAbsentReceiptAsAFailure(t *testing.T) {
+	cp := newFakeControlPlane()
+	cfg := testConfig(t)
+	cfg.DataMount = t.TempDir()
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+	a.id = identity{DBInstanceIdentifier: "db-resolved"}
+	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
+
+	ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
+	defer cancel()
+	if err := a.acknowledgeBootstrap(ctx); err == nil {
+		t.Fatal("acknowledgeBootstrap returned nil, want it still waiting for rds-init")
+	}
+	if got := a.hb.bootstrapFailure.Load(); got != nil {
+		t.Errorf("bootstrap failure = %q, want none while the receipt has simply not been written", *got)
+	}
+
+	// A receipt that exists but names another instance is a real mismatch and is
+	// reported, which is what separates the two cases.
+	writeReceipt(t, a, testPayloadID, "db-the-snapshot-source")
+	mismatch, cancelMismatch := context.WithTimeout(t.Context(), 50*time.Millisecond)
+	defer cancelMismatch()
+	if err := a.acknowledgeBootstrap(mismatch); err == nil {
+		t.Fatal("a foreign receipt must not be acknowledged")
+	}
+	if a.hb.bootstrapFailure.Load() == nil {
+		t.Error("a receipt naming another instance must be reported")
+	}
+}

cmd/rds-agent/bootstrap.go

@@ -12,11 +12,19 @@ import (
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
 
+// The staged payload this boot has to acknowledge once PostgreSQL has applied
+// it. Nil on an attach boot, which makes the acknowledgement step a no-op.
+type pendingBootstrap struct {
+	payloadID    string
+	vmGeneration int64
+	dataVolumeID string
+}
+
 // Fetches the boot material and writes the handoff rds-init is blocked waiting
 // for, then points the health probe at the assigned port.
 func (a *Agent) bootstrap(ctx context.Context) error {
 	var cfg *handlers_rds.GetDBBootstrapConfigOutput
-	if err := retry(ctx, "bootstrap fetch", func(ctx context.Context) error {
+	if err := retryObserved(ctx, "bootstrap fetch", func(ctx context.Context) error {
 		fetched, err := a.cp.GetBootstrapConfig(ctx, a.id)
 		if err != nil {
 			return err
@@ -24,28 +32,49 @@ func (a *Agent) bootstrap(ctx context.Context) error {
 		if fetched == nil {
 			return fmt.Errorf("bootstrap fetch returned no config")
 		}
+		// A control plane that predates the encrypted payload answers initialize
+		// with nothing to initialise with. Retrying re-dials the worker queue
+		// group until an upgraded node answers, so a mixed-version fleet heals
+		// within one boot rather than needing a reboot.
+		if fetched.Mode == handlers_rds.BootstrapModeInitialize &&
+			(fetched.MasterUserPassword == nil || *fetched.MasterUserPassword == "") {
+			return fmt.Errorf("bootstrap fetch returned mode=%s with no master password", fetched.Mode)
+		}
 		cfg = fetched
 		return nil
+	}, func(err error) {
+		a.hb.setBootstrapFailure("bootstrap fetch", err)
 	}); err != nil {
 		return err
 	}
+	a.hb.clearBootstrapFailure()
 
-	// Once an initialize response reaches the guest, keep retrying that same
-	// payload. Re-fetching after a local write failure returns attach because the
-	// control plane has already consumed the one-shot password.
-	if err := retry(ctx, "bootstrap handoff", func(context.Context) error {
+	// The fetch mutates nothing, so a handoff that cannot be written is retried
+	// against the same staged payload rather than against an attach response.
+	if err := retryObserved(ctx, "bootstrap handoff", func(context.Context) error {
 		return a.handoffWriter(a.cfg.HandoffDir, cfg)
+	}, func(err error) {
+		a.hb.setBootstrapFailure("bootstrap handoff", err)
 	}); err != nil {
 		return err
 	}
+	a.hb.clearBootstrapFailure()
 	if cfg.Port > 0 {
 		a.probe.setPort(int(cfg.Port))
 		a.engine.setPort(int(cfg.Port))
 	}
+	if cfg.BootstrapPending && cfg.PayloadID != "" {
+		a.pending = &pendingBootstrap{
+			payloadID:    cfg.PayloadID,
+			vmGeneration: cfg.VMGeneration,
+			dataVolumeID: cfg.DataVolumeID,
+		}
+	}
 	// Mode is logged but never branched on: rds-init decides whether to initdb
 	// from the state of the datadir.
 	slog.Info("rds-agent: bootstrap config delivered",
-		"mode", cfg.Mode, "port", cfg.Port, "parameters", len(cfg.Parameters))
+		"mode", cfg.Mode, "port", cfg.Port, "parameters", len(cfg.Parameters),
+		"bootstrapPending", cfg.BootstrapPending)
 	return nil
 }
 
@@ -78,7 +107,11 @@ func writeHandoff(dir string, cfg *handlers_rds.GetDBBootstrapConfigOutput) erro
 		return fmt.Errorf("secure handoff dir %s: %w", dir, err)
 	}
 
-	if err := writeHandoffFile(dir, handoffParamsFile, renderParameters(cfg.Parameters)); err != nil {
+	params, err := renderParameters(cfg.Engine, cfg.Parameters)
+	if err != nil {
+		return err
+	}
+	if err := writeHandoffFile(dir, handoffParamsFile, params); err != nil {
 		return err
 	}
 	// Cert and key are written as a pair; half of one would start the engine
@@ -101,7 +134,17 @@ func renderBootstrapEnv(cfg *handlers_rds.GetDBBootstrapConfigOutput) string {
 	var b strings.Builder
 	b.WriteString("# Written by rds-agent. Regenerated on every boot; edits are lost.\n")
 	writeEnvLine(&b, "RDS_MODE", cfg.Mode)
+	writeEnvLine(&b, "RDS_DB_INSTANCE_IDENTIFIER", cfg.DBInstanceIdentifier)
 	writeEnvLine(&b, "RDS_MASTER_USERNAME", cfg.MasterUsername)
+	writeEnvLine(&b, "RDS_DATA_VOLUME_ID", cfg.DataVolumeID)
+	writeEnvLine(&b, "RDS_DATA_VOLUME_SERIAL", cfg.DataVolumeSerial)
+	writeEnvLine(&b, "RDS_VM_GENERATION", strconv.FormatInt(cfg.VMGeneration, 10))
+	writeEnvLine(&b, "RDS_FORMAT_AUTHORIZED", strconv.FormatBool(cfg.FormatAuthorized))
+	// Neither needs volume access, which is why the assertion is made here: the
+	// agent runs before rds-datadir mounts the volume, so rds-init is the first
+	// component that can compare it against the completion receipt on disk.
+	writeEnvLine(&b, "RDS_BOOTSTRAP_PENDING", boolFlag(cfg.BootstrapPending))
+	writeEnvLine(&b, "RDS_PAYLOAD_ID", cfg.PayloadID)
 	// Present only in initialize mode: an attach fetch has no password to write
 	// and rds-init must not find a stale one.
 	if cfg.MasterUserPassword != nil {
@@ -116,6 +159,15 @@ func renderBootstrapEnv(cfg *handlers_rds.GetDBBootstrapConfigOutput) string {
 	return b.String()
 }
 
+// rds-init compares against 1 rather than sourcing a Go boolean, so the two
+// halves of the guard cannot drift on spelling.
+func boolFlag(v bool) string {
+	if v {
+		return "1"
+	}
+	return "0"
+}
+
 func writeEnvLine(b *strings.Builder, key, value string) {
 	b.WriteString(key)
 	b.WriteString("=")
@@ -129,18 +181,27 @@ func shellQuote(s string) string {
 	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
 }
 
-// postgresql.conf syntax. Values are quoted so a setting with a space or unit
-// suffix survives; the engine accepts quoted numerics and booleans too.
-func renderParameters(params []handlers_rds.Parameter) string {
+// postgresql.conf syntax, which MariaDB's option files also parse. Values are
+// quoted so a setting with a space or unit suffix survives; the engine accepts
+// quoted numerics and booleans too.
+//
+// Names are the engine's rather than the customer's: a setting the server takes
+// only through SET has a different spelling at startup, and writing the wrong one
+// leaves the engine refusing to boot on a file already on the data volume.
+func renderParameters(engineName string, params []handlers_rds.Parameter) (string, error) {
+	engine, err := handlers_rds.LookupEngine(engineName)
+	if err != nil {
+		return "", fmt.Errorf("render the resolved parameters: %w", err)
+	}
 	var b strings.Builder
 	b.WriteString("# Resolved parameter group, written by rds-agent.\n")
 	for _, p := range params {
 		if p.Name == "" {
 			continue
 		}
-		fmt.Fprintf(&b, "%s = '%s'\n", p.Name, strings.ReplaceAll(p.Value, "'", "''"))
+		fmt.Fprintf(&b, "%s = '%s'\n", engine.OptionFileName(p.Name), strings.ReplaceAll(p.Value, "'", "''"))
 	}
-	return b.String()
+	return b.String(), nil
 }
 
 // Atomic: a temp file in the same directory, renamed over the target. The mode

cmd/rds-agent/bootstrap_test.go

@@ -16,10 +16,15 @@ func TestWriteHandoff_WritesEveryFileRootOnly(t *testing.T) {
 
 	err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:               handlers_rds.BootstrapModeInitialize,
+		Engine:             "postgres",
 		MasterUsername:     "master",
 		MasterUserPassword: &password,
 		DBName:             "appdb",
 		Port:               5433,
+		DataVolumeID:       "vol-data-01",
+		DataVolumeSerial:   "voldata01",
+		VMGeneration:       4,
+		FormatAuthorized:   true,
 		Parameters: []handlers_rds.Parameter{
 			{Name: "max_connections", Value: "200"},
 			{Name: "shared_buffers", Value: "256MB"},
@@ -38,6 +43,10 @@ func TestWriteHandoff_WritesEveryFileRootOnly(t *testing.T) {
 		"RDS_MASTER_PASSWORD='s3cret'",
 		"RDS_DB_NAME='appdb'",
 		"RDS_PORT='5433'",
+		"RDS_DATA_VOLUME_ID='vol-data-01'",
+		"RDS_DATA_VOLUME_SERIAL='voldata01'",
+		"RDS_VM_GENERATION='4'",
+		"RDS_FORMAT_AUTHORIZED='true'",
 	} {
 		if !strings.Contains(env, want) {
 			t.Errorf("handoff env missing %s:\n%s", want, env)
@@ -74,15 +83,27 @@ func TestWriteHandoff_AttachOmitsPassword(t *testing.T) {
 	dir := filepath.Join(t.TempDir(), "handoff")
 	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:           handlers_rds.BootstrapModeAttach,
+		Engine:         "postgres",
 		MasterUsername: "master",
 		Port:           5432,
 	}); err != nil {
 		t.Fatalf("writeHandoff: %v", err)
 	}
 
-	if env := readFile(t, filepath.Join(dir, handoffEnvFile)); strings.Contains(env, "RDS_MASTER_PASSWORD") {
+	env := readFile(t, filepath.Join(dir, handoffEnvFile))
+	if strings.Contains(env, "RDS_MASTER_PASSWORD") {
 		t.Errorf("handoff env = %q, want no RDS_MASTER_PASSWORD key", env)
 	}
+	for _, want := range []string{
+		"RDS_DATA_VOLUME_ID=''",
+		"RDS_DATA_VOLUME_SERIAL=''",
+		"RDS_VM_GENERATION='0'",
+		"RDS_FORMAT_AUTHORIZED='false'",
+	} {
+		if !strings.Contains(env, want) {
+			t.Errorf("handoff env missing fail-closed field %s: %q", want, env)
+		}
+	}
 }
 
 // The env file is sourced by a shell, so a password holding shell metacharacters
@@ -93,6 +114,7 @@ func TestWriteHandoff_QuotesShellMetacharacters(t *testing.T) {
 
 	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:               handlers_rds.BootstrapModeInitialize,
+		Engine:             "postgres",
 		MasterUsername:     "master",
 		MasterUserPassword: &password,
 		Port:               5432,
@@ -113,6 +135,7 @@ func TestWriteHandoff_SkipsHalfATLSPair(t *testing.T) {
 	dir := filepath.Join(t.TempDir(), "handoff")
 	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:               handlers_rds.BootstrapModeAttach,
+		Engine:             "postgres",
 		MasterUsername:     "master",
 		Port:               5432,
 		ServingCertificate: "CERT",
@@ -130,8 +153,9 @@ func TestWriteHandoff_SkipsHalfATLSPair(t *testing.T) {
 func TestWriteHandoff_RejectsConfigWithNoMasterUser(t *testing.T) {
 	dir := filepath.Join(t.TempDir(), "handoff")
 	err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
-		Mode: handlers_rds.BootstrapModeAttach,
-		Port: 5432,
+		Mode:   handlers_rds.BootstrapModeAttach,
+		Engine: "postgres",
+		Port:   5432,
 	})
 	if err == nil {
 		t.Fatal("writeHandoff accepted a config with no master username, want an error")
@@ -151,6 +175,7 @@ func TestWriteHandoff_TightensExistingDirectory(t *testing.T) {
 
 	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:           handlers_rds.BootstrapModeAttach,
+		Engine:         "postgres",
 		MasterUsername: "master",
 		Port:           5432,
 	}); err != nil {
@@ -200,3 +225,46 @@ func findLine(t *testing.T, body, prefix string) string {
 	t.Fatalf("no line starting %q in:\n%s", prefix, body)
 	return ""
 }
+
+// The include rds-init installs is what the engine starts on, so the spelling
+// written here is the one that has to boot.
+func TestRenderParameters_UsesTheEngineStartupSpelling(t *testing.T) {
+	params := []handlers_rds.Parameter{
+		{Name: "time_zone", Value: "SYSTEM"},
+		{Name: "max_connections", Value: "85"},
+	}
+
+	rendered, err := renderParameters("mariadb", params)
+	if err != nil {
+		t.Fatalf("renderParameters: %v", err)
+	}
+	if !strings.Contains(rendered, "default_time_zone = 'SYSTEM'\n") {
+		t.Errorf("time_zone was not written under its startup spelling:\n%s", rendered)
+	}
+	if strings.Contains(rendered, "\ntime_zone = ") {
+		t.Errorf("time_zone reached the option file, which mariadbd refuses:\n%s", rendered)
+	}
+	if !strings.Contains(rendered, "max_connections = '85'\n") {
+		t.Errorf("an unmapped parameter was not written unchanged:\n%s", rendered)
+	}
+}
+
+func TestRenderParameters_PostgresKeepsEveryName(t *testing.T) {
+	rendered, err := renderParameters("postgres", []handlers_rds.Parameter{
+		{Name: "shared_buffers", Value: "262144"},
+	})
+	if err != nil {
+		t.Fatalf("renderParameters: %v", err)
+	}
+	if !strings.Contains(rendered, "shared_buffers = '262144'\n") {
+		t.Errorf("postgres name was rewritten:\n%s", rendered)
+	}
+}
+
+// Failing closed beats writing a file the engine may refuse: the bad include
+// would already be on the data volume by the time the server rejected it.
+func TestRenderParameters_RefusesAnUnknownEngine(t *testing.T) {
+	if _, err := renderParameters("", nil); err == nil {
+		t.Fatal("renderParameters accepted a config naming no engine, want an error")
+	}
+}

cmd/rds-agent/config.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/mulgadc/spinifex/internal/guestenv"
@@ -12,17 +14,11 @@ const (
 	defaultGatewayCA  = "/etc/spinifex-rds/gateway-ca.pem"
 	defaultHandoffDir = "/run/spinifex-rds"
 	defaultEngineHost = "127.0.0.1"
-	defaultEnginePort = 5432
-	defaultPGIsReady  = "pg_isready"
-
-	// The guest layout rds-init lays down. Overridable for the same reason it is
-	// there: a sibling engine preset points at its own paths.
-	defaultPGBin         = "/usr/libexec/postgresql18"
-	defaultPGData        = "/var/lib/postgresql/18/data"
-	defaultSocketDir     = "/run/postgresql"
-	defaultPGUser        = "postgres"
-	defaultRCService     = "rc-service"
-	defaultEngineSvcName = "postgresql"
+	// Where setup.sh stamps the engine the image bakes. The agent builds its
+	// engine implementation from this file rather than from anything delivered.
+	defaultEngineFile         = "/etc/spinifex-rds/engine"
+	defaultRCService          = "rc-service"
+	defaultEngineProbeTimeout = 5 * time.Second
 	// The long-poll window the agent asks the gateway to hold a request open
 	// for. The gateway caps it at 20s.
 	defaultPollWait = 20 * time.Second
@@ -44,19 +40,29 @@ type config struct {
 	HandoffDir    string
 	EngineHost    string
 	EnginePort    int
-	// Overridable so a test or a sibling engine preset can point at its own.
-	PGIsReady string
-	PollWait  time.Duration
+	PollWait      time.Duration
+
+	// The engine the control plane launched this VM as, delivered by cloud-init,
+	// and the engine the image itself bakes. Two independent statements of the
+	// same fact: the agent refuses to bootstrap when they disagree.
+	Engine      string
+	EngineFile  string
+	BakedEngine string
 
 	// Where the command handlers reach the engine: the client binaries, the
 	// datadir the parameter file is installed into, the socket they connect
 	// over, the OS user they drop to, and the service the stop goes through.
-	PGBin         string
-	PGData        string
+	EngineBinDir  string
+	EngineDataDir string
 	SocketDir     string
-	PGUser        string
+	EngineUser    string
 	RCService     string
 	EngineService string
+	// Where the engine records the pid its probe checks for liveness, for an
+	// engine whose own client cannot tell a server still recovering from one that
+	// is not running at all.
+	EnginePidFile      string
+	EngineProbeTimeout time.Duration
 
 	// Where the data volume is mounted, and the two kernel surfaces a storage
 	// grow resolves its device from. Overridable so a test can point them at
@@ -75,19 +81,21 @@ func loadConfig(envFile string) config {
 		GatewayCA:            get("RDS_GATEWAY_CA"),
 		Region:               get("RDS_REGION"),
 		DBInstanceIdentifier: get("RDS_DB_INSTANCE_IDENTIFIER"),
+		Engine:               get("RDS_ENGINE"),
+		EngineFile:           get("RDS_ENGINE_FILE"),
 		EngineVersion:        get("RDS_ENGINE_VERSION"),
 		HandoffDir:           get("RDS_HANDOFF_DIR"),
 		EngineHost:           get("RDS_ENGINE_HOST"),
-		PGIsReady:            get("RDS_PG_ISREADY"),
-		PGBin:                get("RDS_PG_BIN"),
-		PGData:               get("RDS_PGDATA"),
+		EngineBinDir:         get("RDS_ENGINE_BIN"),
+		EngineDataDir:        get("RDS_ENGINE_DATA"),
 		SocketDir:            get("RDS_SOCKET_DIR"),
-		PGUser:               get("RDS_PG_USER"),
+		EngineUser:           get("RDS_ENGINE_USER"),
 		RCService:            get("RDS_RC_SERVICE"),
 		EngineService:        get("RDS_ENGINE_SERVICE"),
+		EnginePidFile:        get("RDS_ENGINE_PIDFILE"),
 		DataMount:            get("RDS_DATA_MOUNT"),
-		EnginePort:           defaultEnginePort,
 		PollWait:             defaultPollWait,
+		EngineProbeTimeout:   defaultEngineProbeTimeout,
 	}
 	if cfg.GatewayCA == "" {
 		cfg.GatewayCA = defaultGatewayCA
@@ -98,32 +106,20 @@ func loadConfig(envFile string) config {
 	if cfg.EngineHost == "" {
 		cfg.EngineHost = defaultEngineHost
 	}
-	if cfg.PGIsReady == "" {
-		cfg.PGIsReady = defaultPGIsReady
-	}
-	if cfg.PGBin == "" {
-		cfg.PGBin = defaultPGBin
-	}
-	if cfg.PGData == "" {
-		cfg.PGData = defaultPGData
-	}
-	if cfg.SocketDir == "" {
-		cfg.SocketDir = defaultSocketDir
-	}
-	if cfg.PGUser == "" {
-		cfg.PGUser = defaultPGUser
+	if cfg.EngineFile == "" {
+		cfg.EngineFile = defaultEngineFile
 	}
 	if cfg.RCService == "" {
 		cfg.RCService = defaultRCService
 	}
-	if cfg.EngineService == "" {
-		cfg.EngineService = defaultEngineSvcName
-	}
+	// The image's own stamp rather than the delivered engine: an agent that took
+	// its layout from the payload could run one engine's implementation against
+	// another engine's datadir, which is the failure the stamp exists to prevent.
+	cfg.BakedEngine = readBakedEngine(cfg.EngineFile)
+	cfg.applyLayout(engineLayouts[cfg.BakedEngine])
+
 	// rds-datadir reads the same RDS_DATA_MOUNT, so the two agree on where the
 	// volume landed without either asserting it to the other.
-	if cfg.DataMount == "" {
-		cfg.DataMount = defaultDataMount
-	}
 	cfg.MountsFile = defaultMountsFile
 	cfg.SysBlock = defaultSysBlock
 	// The authoritative port comes from the bootstrap config; this is only what
@@ -138,5 +134,50 @@ func loadConfig(envFile string) config {
 			cfg.PollWait = d
 		}
 	}
+	if v := get("RDS_ENGINE_PROBE_TIMEOUT"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.EngineProbeTimeout = d
+		}
+	}
 	return cfg
 }
+
+// Fills in whatever the delivered configuration left unset. An override always
+// wins, so a test can point any of these at a fixture. An unrecognised engine
+// leaves the layout empty, and New refuses rather than guessing at one.
+func (c *config) applyLayout(layout engineLayout) {
+	if c.EngineBinDir == "" {
+		c.EngineBinDir = layout.binDir
+	}
+	if c.EngineDataDir == "" {
+		c.EngineDataDir = layout.dataDir
+	}
+	if c.SocketDir == "" {
+		c.SocketDir = layout.socketDir
+	}
+	if c.EngineUser == "" {
+		c.EngineUser = layout.osUser
+	}
+	if c.EngineService == "" {
+		c.EngineService = layout.service
+	}
+	if c.EnginePidFile == "" {
+		c.EnginePidFile = layout.pidFile
+	}
+	if c.DataMount == "" {
+		c.DataMount = layout.dataMount
+	}
+	if c.EnginePort == 0 {
+		c.EnginePort = layout.port
+	}
+}
+
+// The engine the image bakes. An absent or unreadable stamp reads as empty,
+// which New refuses: an agent that guessed would defeat the stamp.
+func readBakedEngine(path string) string {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return ""
+	}
+	return strings.ToLower(strings.TrimSpace(string(raw)))
+}

cmd/rds-agent/config_test.go

@@ -3,21 +3,34 @@ package main
 import (
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 )
 
+// Stamps the engine a real image bakes and loads the configuration against it,
+// which is what selects the guest layout the agent's defaults come from.
+func testLoadConfig(t *testing.T, engine string) config {
+	t.Helper()
+	stamp := filepath.Join(t.TempDir(), "engine")
+	if err := os.WriteFile(stamp, []byte(engine+"\n"), 0o444); err != nil {
+		t.Fatalf("write the engine stamp: %v", err)
+	}
+	t.Setenv("RDS_ENGINE_FILE", stamp)
+	return loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+}
+
 func TestLoadConfig_Defaults(t *testing.T) {
-	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+	cfg := testLoadConfig(t, enginePostgres)
 
 	if cfg.GatewayCA != defaultGatewayCA || cfg.HandoffDir != defaultHandoffDir {
 		t.Errorf("CA/handoff = %q/%q, want the built-in defaults", cfg.GatewayCA, cfg.HandoffDir)
 	}
-	if cfg.EngineHost != defaultEngineHost || cfg.EnginePort != defaultEnginePort {
-		t.Errorf("engine endpoint = %s:%d, want the local default", cfg.EngineHost, cfg.EnginePort)
+	if cfg.EngineHost != defaultEngineHost || cfg.PollWait != defaultPollWait {
+		t.Errorf("engine host/pollWait = %q/%v, want the built-in defaults", cfg.EngineHost, cfg.PollWait)
 	}
-	if cfg.PGIsReady != defaultPGIsReady || cfg.PollWait != defaultPollWait {
-		t.Errorf("probe/pollWait = %q/%v, want the built-in defaults", cfg.PGIsReady, cfg.PollWait)
+	if cfg.EngineProbeTimeout != defaultEngineProbeTimeout {
+		t.Errorf("engine probe timeout = %v, want %v", cfg.EngineProbeTimeout, defaultEngineProbeTimeout)
 	}
 	// Identity is optional: the gateway resolves it from the caller's
 	// credentials, so an agent with no configured identifier is normal.
@@ -26,11 +39,94 @@ func TestLoadConfig_Defaults(t *testing.T) {
 	}
 }
 
+// The guest layout is the image's to state, not the control plane's: everything
+// the agent reaches the engine through comes from the stamp it was built with.
+func TestLoadConfig_TakesTheLayoutFromTheBakedEngine(t *testing.T) {
+	cfg := testLoadConfig(t, enginePostgres)
+
+	if cfg.BakedEngine != enginePostgres {
+		t.Fatalf("baked engine = %q, want %q", cfg.BakedEngine, enginePostgres)
+	}
+	layout := engineLayouts[enginePostgres]
+	if cfg.EngineBinDir != layout.binDir || cfg.EngineDataDir != layout.dataDir {
+		t.Errorf("bin/data = %q/%q, want the PostgreSQL layout", cfg.EngineBinDir, cfg.EngineDataDir)
+	}
+	if cfg.EngineUser != layout.osUser || cfg.EngineService != layout.service {
+		t.Errorf("user/service = %q/%q, want the PostgreSQL layout", cfg.EngineUser, cfg.EngineService)
+	}
+	if cfg.SocketDir != layout.socketDir || cfg.DataMount != layout.dataMount {
+		t.Errorf("socket/mount = %q/%q, want the PostgreSQL layout", cfg.SocketDir, cfg.DataMount)
+	}
+	if cfg.EnginePort != layout.port {
+		t.Errorf("port = %d, want the PostgreSQL default %d", cfg.EnginePort, layout.port)
+	}
+}
+
+// An image with no stamp cannot say which engine it is, and an agent that
+// guessed would be the failure the stamp exists to prevent.
+func TestNewEngine_RefusesAnImageWithNoEngineStamp(t *testing.T) {
+	t.Setenv("RDS_ENGINE_FILE", filepath.Join(t.TempDir(), "absent-engine"))
+	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+
+	if _, err := newProbe(cfg, staticProbe(0)); err == nil {
+		t.Error("newProbe built a probe for an image carrying no engine stamp")
+	}
+	if _, err := newEngine(cfg, nil, nil, nil); err == nil {
+		t.Error("newEngine built an engine for an image carrying no engine stamp")
+	}
+}
+
+func TestNewEngine_RefusesAnEngineItDoesNotImplement(t *testing.T) {
+	cfg := testLoadConfig(t, "oracle")
+
+	_, err := newEngine(cfg, nil, nil, nil)
+	if err == nil {
+		t.Fatal("newEngine built an engine this agent does not implement")
+	}
+	if !strings.Contains(err.Error(), "oracle") {
+		t.Errorf("error = %v, want it to name the engine the image bakes", err)
+	}
+}
+
+// The implementation is the agent's, but the definition it validates names and
+// classifies parameters against is the control plane's, which now offers MariaDB.
+func TestNewEngine_BuildsMariaDBForAMariaDBImage(t *testing.T) {
+	cfg := testLoadConfig(t, engineMariaDB)
+	probe, err := newProbe(cfg, staticProbe(0))
+	if err != nil {
+		t.Fatalf("newProbe: %v", err)
+	}
+
+	built, err := newEngine(cfg, nil, nil, probe)
+	if err != nil {
+		t.Fatalf("newEngine: %v", err)
+	}
+	if _, ok := built.(*mariadbEngine); !ok {
+		t.Errorf("newEngine built %T for a MariaDB image", built)
+	}
+}
+
+// The probe reads the pidfile Alpine's packaged service starts mariadbd with,
+// because nothing in the generated configuration can move it: the service passes
+// --pid-file on the command line, which beats an option file.
+func TestLoadConfig_TakesTheMariaDBPidfileFromTheLayout(t *testing.T) {
+	cfg := testLoadConfig(t, engineMariaDB)
+
+	if cfg.EnginePidFile != engineLayouts[engineMariaDB].pidFile {
+		t.Errorf("pidfile = %q, want the MariaDB layout's", cfg.EnginePidFile)
+	}
+	if cfg.EnginePidFile == "" {
+		t.Error("an engine whose probe needs a pidfile was given none")
+	}
+}
+
 func TestLoadConfig_ReadsEnvFile(t *testing.T) {
+	t.Setenv("RDS_ENGINE_FILE", filepath.Join(t.TempDir(), "absent-engine"))
 	path := filepath.Join(t.TempDir(), "agent.env")
 	body := "RDS_GATEWAY_URL=https://gw.internal:9999\n" +
 		"RDS_REGION=ap-southeast-2\n" +
 		"RDS_DB_INSTANCE_IDENTIFIER=db-1\n" +
+		"RDS_ENGINE=postgres\n" +
 		"RDS_ENGINE_VERSION=18.1\n" +
 		"RDS_ENGINE_PORT=6543\n" +
 		"RDS_POLL_WAIT=5s\n"
@@ -45,19 +141,22 @@ func TestLoadConfig_ReadsEnvFile(t *testing.T) {
 	if cfg.DBInstanceIdentifier != "db-1" || cfg.EngineVersion != "18.1" {
 		t.Errorf("identity = %q/%q, want db-1/18.1", cfg.DBInstanceIdentifier, cfg.EngineVersion)
 	}
+	if cfg.Engine != enginePostgres {
+		t.Errorf("delivered engine = %q, want the one cloud-init carried", cfg.Engine)
+	}
 	if cfg.EnginePort != 6543 || cfg.PollWait != 5*time.Second {
 		t.Errorf("port/pollWait = %d/%v, want 6543/5s", cfg.EnginePort, cfg.PollWait)
 	}
 }
 
-// An unparseable override falls back to the default rather than leaving the
-// agent with a zero port or a zero-length long poll.
+// An unparseable override falls back to the layout's own default rather than
+// leaving the agent with a zero port or a zero-length long poll.
 func TestLoadConfig_IgnoresUnusableOverrides(t *testing.T) {
 	t.Setenv("RDS_ENGINE_PORT", "not-a-port")
 	t.Setenv("RDS_POLL_WAIT", "forever")
 
-	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
-	if cfg.EnginePort != defaultEnginePort || cfg.PollWait != defaultPollWait {
+	cfg := testLoadConfig(t, enginePostgres)
+	if cfg.EnginePort != engineLayouts[enginePostgres].port || cfg.PollWait != defaultPollWait {
 		t.Errorf("port/pollWait = %d/%v, want the defaults", cfg.EnginePort, cfg.PollWait)
 	}
 }

cmd/rds-agent/defaultparams_test.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The include the image build validates against a real mariadbd. It is checked
+// in rather than generated during the build because the build runs inside the
+// libguestfs appliance, where there is no Go toolchain to resolve the catalog.
+const mariadbDefaultParametersFixture = "../../scripts/images/rds-mariadb/default-parameters.cnf"
+
+// Byte-for-byte what rds-init installs on a smallest-class instance that
+// overrides nothing. Regenerate with RDS_UPDATE_FIXTURES=1 go test ./cmd/rds-agent/.
+//
+// Keeping it in step matters because setup.sh feeds exactly this file to
+// mariadbd: a catalog that grew a name the server refuses at startup would
+// otherwise reach a customer as an instance that never boots.
+func TestMariaDBDefaultParametersFixture_MatchesTheCatalog(t *testing.T) {
+	engine, err := handlers_rds.LookupEngine("mariadb")
+	if err != nil {
+		t.Fatalf("LookupEngine: %v", err)
+	}
+	class := handlers_rds.SmallestInstanceClass()
+	params, err := engine.ResolveEffectiveParameters(class, nil)
+	if err != nil {
+		t.Fatalf("ResolveEffectiveParameters(%s): %v", class, err)
+	}
+	body, err := renderParameters("mariadb", params)
+	if err != nil {
+		t.Fatalf("renderParameters: %v", err)
+	}
+	want := mariadbParametersHeader + body
+
+	path := filepath.FromSlash(mariadbDefaultParametersFixture)
+	if os.Getenv("RDS_UPDATE_FIXTURES") == "1" {
+		if err := os.WriteFile(path, []byte(want), 0o644); err != nil {
+			t.Fatalf("update %s: %v", path, err)
+		}
+		return
+	}
+
+	got, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read %s: %v", path, err)
+	}
+	if string(got) != want {
+		t.Errorf("%s is stale for class %s.\nRegenerate with RDS_UPDATE_FIXTURES=1 go test ./cmd/rds-agent/\n\ngot:\n%s\nwant:\n%s",
+			path, class, got, want)
+	}
+}
+
+// The one thing the fixture exists to prove cannot regress without the image
+// build catching it, asserted here too so a plain preflight names it.
+func TestMariaDBDefaultParametersFixture_CarriesNoSetOnlyName(t *testing.T) {
+	got, err := os.ReadFile(filepath.FromSlash(mariadbDefaultParametersFixture))
+	if err != nil {
+		t.Fatalf("read the fixture: %v", err)
+	}
+	for _, name := range []string{"\ntime_zone ", "\ntx_isolation ", "\ninnodb_buffer_pool_instances "} {
+		if strings.Contains(string(got), name) {
+			t.Errorf("the default include carries %q, which mariadbd refuses at startup", name)
+		}
+	}
+}

cmd/rds-agent/engine.go

@@ -3,25 +3,21 @@ package main
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
-	"os"
 	"os/exec"
 	"os/user"
 	"path/filepath"
 	"strconv"
 	"strings"
-	"sync"
-	"sync/atomic"
 	"syscall"
 	"time"
 
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
 
-// The in-guest half of the engine seam. Everything PostgreSQL-shaped that a
-// control-plane command needs lives behind this, so the command registry above
-// stays engine-agnostic.
+// The in-guest half of the engine seam. Everything engine-shaped that a
+// control-plane command needs lives behind this, so the command registry stays
+// engine-agnostic.
 type engineOps interface {
 	// Rotates the master role's password live. Never persisted anywhere in the
 	// guest: it exists only for the length of this call.
@@ -45,10 +41,123 @@ type engineOps interface {
 // engine fails to come up after a parameter change.
 type parameterRecovery interface {
 	// Reports whether the installed set differed and was replaced.
-	RestoreLastKnownGoodParameters() (bool, error)
+	RestoreLastKnownGoodParameters(ctx context.Context) (bool, error)
 	Restart(ctx context.Context) error
 }
 
+// Records the parameter file only after the probe observes the engine serving
+// it. This is separate from apply, where static values are installed but not
+// adopted until a later restart.
+type servingParameterRecorder interface {
+	RecordServingParameters(ctx context.Context) error
+}
+
+// Everything one engine implementation owns. The narrow interfaces above are
+// what the heartbeat, the commander and the rollback guard each hold; this is
+// only what the agent has to construct.
+type engine interface {
+	engineOps
+	parameterRecovery
+	servingParameterRecorder
+	// Follows the port the control plane assigned, once the bootstrap fetch
+	// lands. Called from a different goroutine than the commands that read it.
+	setPort(port int)
+}
+
+// The guest layout an engine preset lays down, keyed by the engine the image
+// bakes. Every default that belongs to one engine lives here, so nothing
+// outside an implementation names its paths, service, port or probe.
+type engineLayout struct {
+	binDir    string
+	dataDir   string
+	socketDir string
+	osUser    string
+	service   string
+	dataMount string
+	port      int
+	// Where the engine records the pid the probe checks for liveness. Empty for
+	// an engine whose probe answers without one.
+	pidFile string
+	// The engine's own liveness signal, which is the only part of engineProbe
+	// that is not shared.
+	newProbe func(cfg config, run probeRunner) *engineProbe
+	// The implementation every control-plane directive is served by. It fails
+	// when the control plane this agent was built against does not offer the
+	// engine the image bakes, rather than serving one it has no definition for.
+	newEngine func(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error)
+}
+
+// The names the control plane knows these implementations by, which are also
+// what setup.sh stamps into each image.
+const (
+	enginePostgres = "postgres"
+	engineMariaDB  = "mariadb"
+)
+
+var engineLayouts = map[string]engineLayout{
+	enginePostgres: {
+		binDir:    "/usr/libexec/postgresql18",
+		dataDir:   "/var/lib/postgresql/18/data",
+		socketDir: "/run/postgresql",
+		osUser:    "postgres",
+		service:   "postgresql",
+		dataMount: "/var/lib/postgresql",
+		port:      5432,
+		newProbe:  newPostgresProbe,
+		newEngine: func(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+			return newPostgresEngine(cfg, run, startSess, probe), nil
+		},
+	},
+	engineMariaDB: {
+		binDir:    "/usr/bin",
+		dataDir:   "/var/lib/mysql/data",
+		socketDir: "/run/mysqld",
+		osUser:    "mysql",
+		service:   "mariadb",
+		dataMount: "/var/lib/mysql",
+		port:      3306,
+		// Alpine's packaged service starts mysqld_safe with
+		// --pid-file=/run/mysqld/$RC_SVCNAME.pid, and a command-line option beats
+		// the configuration file, so this is the path rather than anything the
+		// platform drop-in could name. setup.sh asserts it against the packaged
+		// init script at build time.
+		pidFile:   "/run/mysqld/mariadb.pid",
+		newProbe:  newMariaDBProbe,
+		newEngine: newMariaDBEngineFromCatalog,
+	},
+}
+
+// Builds the health probe the image's engine supplies.
+func newProbe(cfg config, run probeRunner) (*engineProbe, error) {
+	layout, err := layoutFor(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return layout.newProbe(cfg, run), nil
+}
+
+// Builds the implementation the image bakes. Nothing here consults the control
+// plane for which engine to be: the agent runs the engine it is made of, and the
+// delivered engine is only ever checked against it.
+func newEngine(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+	layout, err := layoutFor(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return layout.newEngine(cfg, run, startSess, probe)
+}
+
+func layoutFor(cfg config) (engineLayout, error) {
+	if cfg.BakedEngine == "" {
+		return engineLayout{}, fmt.Errorf("this image carries no engine stamp at %s", cfg.EngineFile)
+	}
+	layout, ok := engineLayouts[cfg.BakedEngine]
+	if !ok {
+		return engineLayout{}, fmt.Errorf("this image bakes engine %q, which this agent does not implement", cfg.BakedEngine)
+	}
+	return layout, nil
+}
+
 // One child process. Env replaces the agent's own environment rather than
 // extending it, so a secret placed here reaches only the process that needs it.
 type command struct {
@@ -58,10 +167,14 @@ type command struct {
 	Stdin string
 	// The OS user to drop to before exec. Empty runs as the agent itself.
 	User string
+	// The statement that makes this client print the session sentinel. Read only
+	// when the command is started as a held session, whose reader has to know
+	// when the engine has finished each script it was fed.
+	SentinelStatement string
 }
 
-// Returns stdout. stderr is folded into the error, because psql reports the
-// actual SQL failure there while exiting with a bare status.
+// Returns stdout. stderr is folded into the error, because both clients report
+// the actual SQL failure there while exiting with a bare status.
 type commandRunner func(ctx context.Context, c command) (string, error)
 
 func execCommandRunner(ctx context.Context, c command) (string, error) {
@@ -106,270 +219,16 @@ func lookupCredential(username string) (*syscall.Credential, error) {
 	return &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)}, nil
 }
 
-// The engine is reached over its unix socket under peer authentication, so the
-// agent drops to the postgres OS user rather than holding a password of its own.
-type postgresEngine struct {
-	run       commandRunner
-	startSess sessionRunner
-	psql      string
-	rcService string
-	service   string
-	pgData    string
-	socketDir string
-	osUser    string
-	// Set from the bootstrap config, on a different goroutine than the commands
-	// that read it.
-	port atomic.Int64
-
-	// The backup mode currently held, if any. Guarded because the expiry timer
-	// releases it on its own goroutine.
-	mu   sync.Mutex
-	held *quiesceHold
-}
-
-var _ engineOps = (*postgresEngine)(nil)
-
-func newPostgresEngine(cfg config, run commandRunner, startSess sessionRunner) *postgresEngine {
-	e := &postgresEngine{
-		run:       run,
-		startSess: startSess,
-		psql:      filepath.Join(cfg.PGBin, "psql"),
-		rcService: cfg.RCService,
-		service:   cfg.EngineService,
-		pgData:    cfg.PGData,
-		socketDir: cfg.SocketDir,
-		osUser:    cfg.PGUser,
-	}
-	e.port.Store(int64(cfg.EnginePort))
-	return e
-}
-
-func (e *postgresEngine) setPort(port int) {
-	e.port.Store(int64(port))
-}
-
-// The role name and the password ride the environment and are re-quoted by
-// psql, so neither reaches a shell word or an argv another process can read.
-func (e *postgresEngine) SetPassword(ctx context.Context, username, password string) error {
-	if username == "" || password == "" {
-		return fmt.Errorf("set-password requires both %s and %s",
-			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
-	}
-	// The statement below runs as the cluster superuser over the socket under
-	// peer auth, so a reserved name in the command payload would hand the
-	// customer the bootstrap superuser rather than rotate their own role.
-	if err := handlers_rds.EnginePostgres().ValidateUsernameNotReserved(username); err != nil {
-		return fmt.Errorf("refusing to set the password of a role the engine reserves: %w", err)
-	}
-	const sql = `\getenv master RDS_MASTER_USERNAME
-\getenv password RDS_MASTER_PASSWORD
-ALTER ROLE :"master" WITH LOGIN PASSWORD :'password';
-`
-	_, err := e.psqlRun(ctx, sql,
-		"RDS_MASTER_USERNAME="+username,
-		"RDS_MASTER_PASSWORD="+password,
-	)
-	if err != nil {
-		// psql echoes the failing statement, which here would carry the new
-		// password back to the control plane and into the event ring.
-		return fmt.Errorf("apply the master password: %s", redact(err.Error(), password))
-	}
-	return nil
-}
-
-// The include the resolved parameter set is rendered to, and the copy of the
-// last one the engine accepted. Both live beside the data rather than in /etc,
-// matching rds-init: a class change boots a fresh root volume, which would
-// otherwise revert them when a fresh root volume is used.
-const (
-	parametersFileName = "10-rds-parameters.conf"
-	// Deliberately not a .conf name: include_dir globs *.conf, so the rollback
-	// copy must not be read as a second set of settings.
-	lastGoodFileName = "10-rds-parameters.last-good"
-)
-
-// Installs the resolved set, validates it with the engine's own config parser
-// before the engine ever adopts it, and reloads. A value the engine refuses is
-// rolled back here rather than left on the data volume, where it would survive
-// every VM replace and turn the next restart into a boot loop.
-func (e *postgresEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
-	// The check has to run against the file in place, because the engine parses
-	// the datadir's own include_dir. The window that leaves is closed from both
-	// ends: the rollback below, and the last-known-good restore the agent runs at
-	// boot when the engine does not come up.
-	previous, restore, err := e.installParameters(params)
-	if err != nil {
-		return nil, err
-	}
-	if err := e.checkConfig(ctx); err != nil {
-		return nil, errors.Join(fmt.Errorf("the engine rejected the parameter set: %w", err), restore())
-	}
-	if _, err := e.psqlRun(ctx, "SELECT pg_reload_conf();\n"); err != nil {
-		return nil, errors.Join(fmt.Errorf("reload the engine configuration: %w", err), restore())
-	}
-
-	// Recorded only once the engine has both parsed and adopted it, so the
-	// rollback target is a set that is known to work rather than the last one
-	// written.
-	if err := e.recordLastKnownGood(previous); err != nil {
-		return nil, err
-	}
-
-	// Read after the reload: a setting only becomes pending_restart once the
-	// engine has seen the new value and declined to adopt it live.
-	out, err := e.psqlRun(ctx, "SELECT name FROM pg_settings WHERE pending_restart ORDER BY name;\n")
-	if err != nil {
-		return nil, fmt.Errorf("read the settings pending a restart: %w", err)
-	}
-	var pending []string
-	for line := range strings.SplitSeq(out, "\n") {
-		if name := strings.TrimSpace(line); name != "" {
-			pending = append(pending, name)
-		}
-	}
-	return pending, nil
-}
-
-func (e *postgresEngine) parametersPath() string {
-	return filepath.Join(e.pgData, "conf.d", parametersFileName)
-}
-
-func (e *postgresEngine) lastGoodPath() string {
-	return filepath.Join(e.pgData, "conf.d", lastGoodFileName)
-}
-
-// Written as root and handed to the engine's user, at the same path and mode
-// rds-init installs it at, so a later boot overwrites rather than shadows it.
-// Returns the content that was there and a func putting it back, so the caller
-// can undo the install without re-rendering anything.
-func (e *postgresEngine) installParameters(params []handlers_rds.Parameter) (previous []byte, restore func() error, err error) {
-	path := e.parametersPath()
-	// A missing file is the first-ever apply, and its rollback is a removal.
-	previous, readErr := os.ReadFile(path)
-	if readErr != nil && !os.IsNotExist(readErr) {
-		return nil, nil, fmt.Errorf("read the installed parameters at %s: %w", path, readErr)
-	}
-	existed := readErr == nil
-
-	if err := e.writeEngineFile(path, []byte(renderParameters(params))); err != nil {
-		return nil, nil, err
-	}
-	return previous, func() error {
-		if !existed {
-			if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
-				return fmt.Errorf("withdraw the rejected parameters at %s: %w", path, err)
-			}
-			return nil
-		}
-		return e.writeEngineFile(path, previous)
-	}, nil
-}
-
-// Keeps the rollback target in step with what the engine is actually running.
-// The first apply of an instance's life has no predecessor, so the freshly
-// accepted set becomes the target instead.
-func (e *postgresEngine) recordLastKnownGood(previous []byte) error {
-	content := previous
-	if content == nil {
-		var err error
-		if content, err = os.ReadFile(e.parametersPath()); err != nil {
-			return fmt.Errorf("read the accepted parameters: %w", err)
-		}
-	}
-	return e.writeEngineFile(e.lastGoodPath(), content)
-}
-
-// Puts the last set the engine accepted back in place, for a restart that failed
-// after a parameter change: the instance comes back on the old parameters rather
-// than boot-looping into recovery. Reports whether anything was restored.
-func (e *postgresEngine) RestoreLastKnownGoodParameters() (bool, error) {
-	lastGood, err := os.ReadFile(e.lastGoodPath())
-	if err != nil {
-		if os.IsNotExist(err) {
-			return false, nil
-		}
-		return false, fmt.Errorf("read the last known good parameters: %w", err)
-	}
-	current, err := os.ReadFile(e.parametersPath())
-	if err != nil && !os.IsNotExist(err) {
-		return false, fmt.Errorf("read the installed parameters: %w", err)
-	}
-	if bytes.Equal(current, lastGood) {
-		return false, nil
-	}
-	if err := e.writeEngineFile(e.parametersPath(), lastGood); err != nil {
-		return false, err
-	}
-	return true, nil
-}
-
-// Atomic: a temp file in the same directory, renamed over the target. The temp
-// name deliberately does not end in .conf, so a crash between write and rename
-// cannot leave the engine reading a half-written include.
-func (e *postgresEngine) writeEngineFile(path string, content []byte) error {
-	tmp := path + ".new"
-	if err := os.WriteFile(tmp, content, 0o600); err != nil {
-		return fmt.Errorf("write %s: %w", tmp, err)
-	}
-	if err := e.chownToEngine(tmp); err != nil {
-		os.Remove(tmp)
-		return err
-	}
-	if err := os.Rename(tmp, path); err != nil {
-		os.Remove(tmp)
-		return fmt.Errorf("install %s: %w", path, err)
-	}
-	return nil
-}
-
-// The engine's own parser, run offline against the datadir. Reading one setting
-// back is enough: postgres parses postgresql.conf and every include first, and
-// exits non-zero naming the file and line of an unknown parameter or a value
-// outside its range.
-func (e *postgresEngine) checkConfig(ctx context.Context) error {
-	if _, err := e.run(ctx, command{
-		Name: filepath.Join(filepath.Dir(e.psql), "postgres"),
-		Args: []string{"-D", e.pgData, "-C", "shared_buffers"},
-		Env:  []string{"PATH=" + defaultGuestPath},
-		User: e.osUser,
-	}); err != nil {
-		return err
-	}
-	return nil
-}
-
-// The engine reads its own config, so a root-owned file would be unreadable to
-// it. A guest with no such user is a broken image, not something to work around.
-func (e *postgresEngine) chownToEngine(path string) error {
-	credential, err := lookupCredential(e.osUser)
-	if err != nil {
-		return err
-	}
-	if err := os.Chown(path, int(credential.Uid), int(credential.Gid)); err != nil {
-		return fmt.Errorf("hand %s to %s: %w", path, e.osUser, err)
-	}
-	return nil
-}
-
-// Through the service manager rather than pg_ctl, so the supervisor records the
-// engine as stopped and does not restart it underneath a VM that is going down.
-func (e *postgresEngine) Stop(ctx context.Context) error {
-	return e.serviceAction(ctx, "stop")
-}
-
-// Only the parameter rollback calls this: a restart the control plane wants goes
-// through RebootDBInstance, which cycles the VM.
-func (e *postgresEngine) Restart(ctx context.Context) error {
-	return e.serviceAction(ctx, "restart")
-}
-
-func (e *postgresEngine) serviceAction(ctx context.Context, action string) error {
-	if _, err := e.run(ctx, command{
-		Name: e.rcService,
-		Args: []string{e.service, action},
+// Through the service manager rather than the engine's own control tool, so the
+// supervisor records the state and does not restart the engine underneath a VM
+// that is going down.
+func serviceAction(ctx context.Context, run commandRunner, rcService, service, action string) error {
+	if _, err := run(ctx, command{
+		Name: rcService,
+		Args: []string{service, action},
 		Env:  []string{"PATH=" + defaultGuestPath},
 	}); err != nil {
-		return fmt.Errorf("%s the %s service: %w", action, e.service, err)
+		return fmt.Errorf("%s the %s service: %w", action, service, err)
 	}
 	return nil
 }
@@ -377,32 +236,6 @@ func (e *postgresEngine) serviceAction(ctx context.Context, action string) error
 // A minimal environment for a child that is not carrying secrets.
 const defaultGuestPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 
-// Feeds sql on stdin rather than as an argument, so a statement is never
-// visible in the process table.
-func (e *postgresEngine) psqlRun(ctx context.Context, sql string, env ...string) (string, error) {
-	return e.run(ctx, command{
-		Name:  e.psql,
-		Args:  e.psqlArgs(),
-		Env:   append([]string{"PATH=" + defaultGuestPath}, env...),
-		Stdin: sql,
-		User:  e.osUser,
-	})
-}
-
-// Reading the script from stdin is what lets one invocation serve both a
-// one-shot run and a session held open across several statements.
-func (e *postgresEngine) psqlArgs() []string {
-	return []string{
-		"--no-psqlrc", "--quiet", "--no-align", "--tuples-only",
-		"-v", "ON_ERROR_STOP=1",
-		"-h", e.socketDir,
-		"-p", strconv.FormatInt(e.port.Load(), 10),
-		"-U", e.osUser,
-		"-d", "postgres",
-		"-f", "-",
-	}
-}
-
 func redact(s, secret string) string {
 	if secret == "" {
 		return s

cmd/rds-agent/engine_contract_test.go

@@ -0,0 +1,329 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// A probe the test drives directly, in place of whichever binaries the engine's
+// own state function would have run. Each implementation's state function is
+// covered on its own; what these cases exercise is what the engine does with the
+// answer.
+func drivenProbe(state *engineState) *engineProbe {
+	return newEngineProbe(0, func(context.Context, int64) (engineState, string) {
+		return *state, "driven by the test"
+	})
+}
+
+// One engine behind the interfaces the heartbeat, the commander and the rollback
+// guard hold, plus the few facts a shared case needs to say what it means.
+type engineHarness struct {
+	engine  engine
+	runner  *recordingRunner
+	params  parameterStore
+	service string
+	// The state the probe reports, moved by the test rather than by the engine.
+	state *engineState
+	// Points the next quiesce at a session the test can inspect, and returns the
+	// commands those sessions were opened with.
+	useSession func(*fakeSession) *[]command
+	// A setting this engine's catalog really carries, so a shared case installs
+	// something the implementation would genuinely be handed.
+	parameter handlers_rds.Parameter
+	// What the engine's own include directive globs. Neither the rollback copy
+	// nor the serving copy may end in it.
+	includeSuffix string
+}
+
+func engineHarnesses() []struct {
+	name  string
+	build func(*testing.T) *engineHarness
+} {
+	return []struct {
+		name  string
+		build func(*testing.T) *engineHarness
+	}{
+		{name: enginePostgres, build: buildPostgresHarness},
+		{name: engineMariaDB, build: buildMariaDBHarness},
+	}
+}
+
+func forEachEngine(t *testing.T, run func(*testing.T, *engineHarness)) {
+	t.Helper()
+	for _, fixture := range engineHarnesses() {
+		t.Run(fixture.name, func(t *testing.T) { run(t, fixture.build(t)) })
+	}
+}
+
+func buildPostgresHarness(t *testing.T) *engineHarness {
+	t.Helper()
+	runner := &recordingRunner{}
+	e := newTestEngine(t, runner.run)
+	state := engineServing
+	e.probe = drivenProbe(&state)
+
+	return &engineHarness{
+		engine: e, runner: runner, params: e.params, service: e.service, state: &state,
+		useSession: func(session *fakeSession) *[]command {
+			started := &[]command{}
+			e.startSess = func(_ context.Context, c command) (engineSession, error) {
+				*started = append(*started, c)
+				return session, nil
+			}
+			return started
+		},
+		parameter:     handlers_rds.Parameter{Name: "work_mem", Value: "4096"},
+		includeSuffix: ".conf",
+	}
+}
+
+func buildMariaDBHarness(t *testing.T) *engineHarness {
+	t.Helper()
+	runner := &recordingRunner{}
+	e := newTestMariaDBEngine(t, runner.run)
+	state := engineServing
+	e.probe = drivenProbe(&state)
+
+	return &engineHarness{
+		engine: e, runner: runner, params: e.params, service: e.service, state: &state,
+		useSession: func(session *fakeSession) *[]command {
+			started := &[]command{}
+			e.startSess = func(_ context.Context, c command) (engineSession, error) {
+				*started = append(*started, c)
+				return session, nil
+			}
+			return started
+		},
+		parameter:     handlers_rds.Parameter{Name: "max_connections", Value: "200"},
+		includeSuffix: ".cnf",
+	}
+}
+
+// The parameters land where rds-init installs them, so the next boot overwrites
+// the file rather than shadowing it with a second copy.
+func TestEngineContract_ApplyParametersInstallsWhereRdsInitDoes(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		if _, err := h.engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{h.parameter}); err != nil {
+			t.Fatalf("ApplyParameters: %v", err)
+		}
+
+		installed, err := os.ReadFile(h.params.installedPath())
+		if err != nil {
+			t.Fatalf("read the installed parameters: %v", err)
+		}
+		if !strings.Contains(string(installed), h.parameter.Name+" = '"+h.parameter.Value+"'") {
+			t.Errorf("installed parameters = %q, want the resolved value", installed)
+		}
+		if !strings.HasSuffix(h.params.installedPath(), h.includeSuffix) {
+			t.Errorf("the installed set at %s is not a file the engine includes", h.params.installedPath())
+		}
+		// Comparison material, not a second set of settings: an include glob that
+		// read either of these would apply a withdrawn configuration.
+		copies := []string{h.params.lastGoodPath()}
+		if h.params.serving != "" {
+			copies = append(copies, h.params.servingPath())
+		}
+		for _, path := range copies {
+			if strings.HasSuffix(path, h.includeSuffix) {
+				t.Errorf("the copy at %s would be included by the engine", path)
+			}
+		}
+	})
+}
+
+// An engine still starting or replaying has not finished adopting the set it is
+// on, so replacing that set underneath it would be installing over a moving
+// target.
+func TestEngineContract_ApplyParametersRefusesWhileTheEngineIsRecovering(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		*h.state = engineRecovering
+
+		if _, err := h.engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{h.parameter}); err == nil {
+			t.Fatal("ApplyParameters succeeded against an engine that was still coming up")
+		}
+		if _, err := os.Stat(h.params.installedPath()); !os.IsNotExist(err) {
+			t.Errorf("the refused set was installed anyway (stat err = %v)", err)
+		}
+	})
+}
+
+// The boot-time half of the parameter safety net: the last set the engine
+// accepted goes back only when the engine is genuinely down, and only once.
+func TestEngineContract_RestoreLastKnownGoodOnlyRollsBackADownEngine(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		accepted := h.parameter
+		if _, err := h.engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{accepted}); err != nil {
+			t.Fatalf("ApplyParameters: %v", err)
+		}
+		if err := h.engine.RecordServingParameters(t.Context()); err != nil {
+			t.Fatalf("RecordServingParameters: %v", err)
+		}
+		if _, err := h.engine.ApplyParameters(t.Context(),
+			[]handlers_rds.Parameter{{Name: accepted.Name, Value: accepted.Value + "0"}}); err != nil {
+			t.Fatalf("second ApplyParameters: %v", err)
+		}
+
+		// A guard whose deadline expired while an apply brought the engine back must
+		// not reverse that repair.
+		restored, err := h.engine.RestoreLastKnownGoodParameters(t.Context())
+		if err != nil {
+			t.Fatalf("RestoreLastKnownGoodParameters against a serving engine: %v", err)
+		}
+		if restored {
+			t.Fatal("rolled the parameters back while the engine was serving")
+		}
+
+		*h.state = engineAbsent
+		restored, err = h.engine.RestoreLastKnownGoodParameters(t.Context())
+		if err != nil {
+			t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
+		}
+		if !restored {
+			t.Fatal("did not roll back to a set that differs from the last accepted one")
+		}
+		installed, err := os.ReadFile(h.params.installedPath())
+		if err != nil {
+			t.Fatalf("read the installed parameters: %v", err)
+		}
+		if !strings.Contains(string(installed), accepted.Name+" = '"+accepted.Value+"'") {
+			t.Errorf("installed = %q, want the last accepted set", installed)
+		}
+
+		// Repeating a rollback would only churn a cluster that is failing for
+		// another reason.
+		restored, err = h.engine.RestoreLastKnownGoodParameters(t.Context())
+		if err != nil {
+			t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
+		}
+		if restored {
+			t.Error("rolled back again when the installed set already matched")
+		}
+	})
+}
+
+// Through the service manager rather than the engine's own control tool, so the
+// supervisor records the engine as stopped and does not restart it underneath a
+// VM that is going down.
+func TestEngineContract_StopGoesThroughTheServiceManager(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		if err := h.engine.Stop(t.Context()); err != nil {
+			t.Fatalf("Stop: %v", err)
+		}
+		if len(h.runner.calls) != 1 {
+			t.Fatalf("ran %d commands, want the service stop", len(h.runner.calls))
+		}
+		call := h.runner.calls[0]
+		if !slices.Equal(call.Args, []string{h.service, "stop"}) {
+			t.Errorf("stop ran %s %v, want the service manager", call.Name, call.Args)
+		}
+	})
+}
+
+// One backup at a time, released by Unquiesce or by its own deadline, whichever
+// comes first. The deadline is what keeps a control plane that dies mid-snapshot
+// from leaving the engine held.
+func TestEngineContract_QuiesceHoldsUntilReleased(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		session := &fakeSession{}
+		started := h.useSession(session)
+
+		if err := h.engine.Quiesce(t.Context(), "orders-db-pre-upgrade", time.Minute); err != nil {
+			t.Fatalf("Quiesce: %v", err)
+		}
+		if len(*started) != 1 {
+			t.Fatalf("started %d sessions, want 1", len(*started))
+		}
+		if session.closes() != 0 {
+			t.Fatal("the session was closed while the backup was meant to be held")
+		}
+
+		err := h.engine.Quiesce(t.Context(), "second", time.Minute)
+		if err == nil {
+			t.Fatal("a second quiesce was accepted while the first was held")
+		}
+		if !strings.Contains(err.Error(), "orders-db-pre-upgrade") {
+			t.Errorf("error %q does not name the backup already holding the engine", err)
+		}
+		if len(*started) != 1 {
+			t.Errorf("started %d sessions, want the second refused before opening one", len(*started))
+		}
+
+		if err := h.engine.Unquiesce(t.Context()); err != nil {
+			t.Fatalf("Unquiesce: %v", err)
+		}
+		if session.closes() != 1 {
+			t.Errorf("closed %d times, want the session ended exactly once", session.closes())
+		}
+	})
+}
+
+// A hold that expired mid-snapshot means the snapshot was not taken against a
+// held checkpoint, so the release reports it rather than succeeding silently.
+func TestEngineContract_QuiesceReleasesItselfOnItsDeadline(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		session := &fakeSession{}
+		h.useSession(session)
+
+		if err := h.engine.Quiesce(t.Context(), "orders-db-pre-upgrade", 10*time.Millisecond); err != nil {
+			t.Fatalf("Quiesce: %v", err)
+		}
+		deadline := time.Now().Add(2 * time.Second)
+		for session.closes() == 0 && time.Now().Before(deadline) {
+			time.Sleep(time.Millisecond)
+		}
+		if session.closes() != 1 {
+			t.Fatal("the backup session outlived its deadline")
+		}
+
+		err := h.engine.Unquiesce(t.Context())
+		if err == nil {
+			t.Fatal("Unquiesce succeeded after the hold expired")
+		}
+		if !strings.Contains(err.Error(), "expired") {
+			t.Errorf("error %q does not say the hold had expired", err)
+		}
+	})
+}
+
+// A failed start leaves nothing held, so a retry is not refused by a hold that
+// never took effect.
+func TestEngineContract_QuiesceEndsTheSessionWhenTheBackupWillNotStart(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		session := &fakeSession{execErr: errors.New("the engine refused the backup")}
+		h.useSession(session)
+
+		if err := h.engine.Quiesce(t.Context(), "orders-db-pre-upgrade", time.Minute); err == nil {
+			t.Fatal("Quiesce succeeded against an engine that refused the backup")
+		}
+		if session.closes() != 1 {
+			t.Errorf("closed %d times, want the failed session ended", session.closes())
+		}
+		if err := h.engine.Unquiesce(t.Context()); err == nil {
+			t.Error("a failed quiesce left a hold behind")
+		}
+	})
+}
+
+// Every command the control plane can issue is served by both implementations,
+// so a directive cannot be answered as unsupported on one engine and honoured on
+// the other.
+func TestEngineContract_ServesEveryRegisteredCommand(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		registry := newCommandRegistry(h.engine, &fakeStorage{})
+		for _, name := range []string{
+			handlers_rds.CommandSetPassword, handlers_rds.CommandApplyParams, handlers_rds.CommandStopEngine,
+			handlers_rds.CommandGrowFilesystem, handlers_rds.CommandQuiesce, handlers_rds.CommandUnquiesce,
+		} {
+			if _, ok := registry[name]; !ok {
+				t.Errorf("command %q has no handler", name)
+			}
+		}
+	})
+}

cmd/rds-agent/engine_mariadb.go

@@ -0,0 +1,638 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The control plane's own statements about this engine, resolved once at
+// startup. Held as functions rather than as the Engine value so nothing here
+// re-derives a rule the API already owns.
+type controlPlaneRules struct {
+	// The whole master-username rule rather than only the reserved set. MariaDB's
+	// client has no identifier quoting, so the name is interpolated into SQL by
+	// this process and every part of the rule is load-bearing here.
+	validateUsername func(username string) error
+	// Whether a setting takes effect only at a restart. Already customer-facing
+	// and authoritative, since the API refuses ApplyMethod=immediate on one.
+	isStatic func(name string) bool
+	// The catalog name behind a spelling read back out of an option file. Only a
+	// setting whose startup spelling differs from the customer's moves.
+	catalogName func(optionFileName string) string
+}
+
+func controlPlaneRulesFrom(meta handlers_rds.Engine) controlPlaneRules {
+	// Built once per engine: everything read back out of a generated file has to
+	// return to the catalog's namespace before it is classified or reported, or a
+	// startup spelling would read as an unknown name.
+	catalogNames := map[string]string{}
+	for _, name := range meta.CatalogParameterNames() {
+		if optionFileName := meta.OptionFileName(name); optionFileName != name {
+			catalogNames[optionFileName] = name
+		}
+	}
+
+	return controlPlaneRules{
+		validateUsername: meta.ValidateMasterUsername,
+		isStatic: func(name string) bool {
+			spec, ok := meta.LookupParameter(name)
+			// A name the catalog does not carry cannot be shown to have been adopted
+			// without a restart, and is never issued as a live SET GLOBAL either.
+			return !ok || spec.ApplyType == handlers_rds.ApplyTypeStatic
+		},
+		catalogName: func(optionFileName string) string {
+			if name, ok := catalogNames[optionFileName]; ok {
+				return name
+			}
+			return optionFileName
+		},
+	}
+}
+
+// The engine is reached over its unix socket as root, which the datadir's
+// unix_socket plugin authenticates from the connecting process' own uid — the
+// direct analogue of PostgreSQL's peer auth, and the reason the agent holds no
+// password of its own.
+type mariadbEngine struct {
+	quiesceState
+
+	rules     controlPlaneRules
+	run       commandRunner
+	startSess sessionRunner
+	client    string
+	admin     string
+	socket    string
+	rcService string
+	service   string
+	probe     *engineProbe
+
+	params parameterStore
+	// Serializes parameter installs, serving snapshots and rollback restores so
+	// none can copy or replace an intermediate configuration.
+	paramMu       sync.Mutex
+	repairTimeout time.Duration
+	repairPoll    time.Duration
+}
+
+var _ engine = (*mariadbEngine)(nil)
+
+const (
+	mariadbClientBinary = "mariadb"
+	mariadbAdminBinary  = "mariadb-admin"
+	// The account mariadb-install-db creates for the unix_socket plugin, which
+	// rds-init keeps for the platform and never hands to the customer.
+	mariadbSuperuser                  = "root"
+	mariadbProbeConnectTimeoutSeconds = 3
+	// Named by the platform drop-in rds-init writes, so both halves reach the
+	// same socket without either asserting it to the other.
+	mariadbSocketFile = "mysqld.sock"
+	// The customer reaches the instance over its customer ENI, so the master is
+	// created and rotated on the wildcard host rather than on localhost.
+	mariadbMasterHost = "%"
+)
+
+// The generated drop-ins on the data volume. The rollback and serving copies
+// deliberately do not end in .cnf: !includedir reads *.cnf, and neither is a
+// second set of settings.
+const (
+	mariadbParametersFile = "10-rds-parameters.cnf"
+	mariadbLastGoodFile   = "10-rds-parameters.last-good"
+	mariadbServingFile    = "10-rds-parameters.serving"
+	// MariaDB treats a setting before any group as a fatal parsing error, so
+	// every generated file carries the header rds-init writes.
+	mariadbParametersHeader = "[mysqld]\n"
+)
+
+// The layout's factory. The rules come from the control plane's own definition
+// of this engine, so a build whose control plane does not offer MariaDB refuses
+// to run this implementation rather than inventing a definition for it.
+func newMariaDBEngineFromCatalog(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+	meta, err := handlers_rds.LookupEngine(engineMariaDB)
+	if err != nil {
+		return nil, fmt.Errorf("this image bakes %s, which this build's control plane does not offer: %w", engineMariaDB, err)
+	}
+	return newMariaDBEngine(cfg, controlPlaneRulesFrom(meta), run, startSess, probe), nil
+}
+
+func newMariaDBEngine(cfg config, rules controlPlaneRules, run commandRunner, startSess sessionRunner, probe *engineProbe) *mariadbEngine {
+	return &mariadbEngine{
+		rules:     rules,
+		run:       run,
+		startSess: startSess,
+		client:    filepath.Join(cfg.EngineBinDir, mariadbClientBinary),
+		admin:     filepath.Join(cfg.EngineBinDir, mariadbAdminBinary),
+		socket:    mariadbSocketPath(cfg),
+		rcService: cfg.RCService,
+		service:   cfg.EngineService,
+		probe:     probe,
+		params: parameterStore{
+			// The mount point rather than the datadir one level inside it: the
+			// include directory has to outlive the sweep a failed bootstrap runs.
+			dir:       filepath.Join(cfg.DataMount, "conf.d"),
+			installed: mariadbParametersFile,
+			lastGood:  mariadbLastGoodFile,
+			serving:   mariadbServingFile,
+			header:    mariadbParametersHeader,
+			osUser:    cfg.EngineUser,
+			engine:    engineMariaDB,
+		},
+		repairTimeout: parameterRepairTimeout,
+		repairPoll:    parameterRepairPoll,
+	}
+}
+
+func mariadbSocketPath(cfg config) string {
+	return filepath.Join(cfg.SocketDir, mariadbSocketFile)
+}
+
+// Nothing this implementation runs needs the assigned port: every connection it
+// makes is over the engine's unix socket, and the platform drop-in rds-init
+// writes is what puts the server on the port. The probe follows it separately.
+func (e *mariadbEngine) setPort(int) {}
+
+// Reports whether a pid is a live process. EPERM is a process this agent may not
+// signal rather than one that is gone, which matters because a stale pidfile
+// read as "alive" is far less dangerous than a live engine read as "absent".
+type processLivenessFn func(pid int) bool
+
+func processAlive(pid int) bool {
+	if pid <= 0 {
+		return false
+	}
+	err := syscall.Kill(pid, 0)
+	return err == nil || errors.Is(err, syscall.EPERM)
+}
+
+func newMariaDBProbe(cfg config, run probeRunner) *engineProbe {
+	return newEngineProbe(cfg.EnginePort, mariadbProbeState(cfg, run, processAlive))
+}
+
+// Three stages, because MariaDB has no single signal that separates an engine
+// still coming up from one that is not there at all: during InnoDB crash
+// recovery mariadbd opens neither its socket nor its port, so a ping fails
+// exactly as it would against nothing.
+//
+// Losing that distinction would break the rollback guard, which treats a
+// recovering engine as a reason to reset its deadline. An instance killed hard
+// mid-write can spend minutes replaying its redo log; read as absent, the guard
+// would roll the parameter file back and restart the server mid-recovery,
+// discarding the work — a rollback that exists to break a boot loop would create
+// one, on an instance whose parameters were never at fault.
+//
+// The port is not consulted. Both stages below reach the server over its unix
+// socket, whose path does not move with the port.
+func mariadbProbeState(cfg config, run probeRunner, alive processLivenessFn) probeStateFn {
+	pidFile, socket := cfg.EnginePidFile, mariadbSocketPath(cfg)
+	admin, client := filepath.Join(cfg.EngineBinDir, mariadbAdminBinary), filepath.Join(cfg.EngineBinDir, mariadbClientBinary)
+	connect := []string{
+		"--no-defaults", "--protocol=socket", "--socket=" + socket,
+		"--user=" + mariadbSuperuser,
+		fmt.Sprintf("--connect-timeout=%d", mariadbProbeConnectTimeoutSeconds),
+	}
+	probeTimeout := cfg.EngineProbeTimeout
+	if probeTimeout <= 0 {
+		probeTimeout = defaultEngineProbeTimeout
+	}
+	runBounded := func(ctx context.Context, name string, args ...string) (int, error) {
+		probeCtx, cancel := context.WithTimeout(ctx, probeTimeout)
+		defer cancel()
+		return run(probeCtx, name, args...)
+	}
+
+	return func(ctx context.Context, _ int64) (engineState, string) {
+		pid, err := readPidFile(pidFile)
+		switch {
+		case os.IsNotExist(err):
+			return engineAbsent, fmt.Sprintf("the engine has written no pidfile at %s", pidFile)
+		case err != nil:
+			return engineAbsent, fmt.Sprintf("the engine pidfile %s could not be read: %v", pidFile, err)
+		case !alive(pid):
+			return engineAbsent, fmt.Sprintf("the engine pidfile %s names pid %d, which is not running", pidFile, pid)
+		}
+
+		// The process is up from here on, so every remaining failure is an engine
+		// that is not serving yet rather than one that is gone — including a probe
+		// binary that will not run at all. Reporting absent against a live server
+		// would have the rollback guard restart one that is making progress.
+		switch code, err := runBounded(ctx, admin, append(slices.Clone(connect), "ping")...); {
+		case err != nil:
+			return engineRecovering, fmt.Sprintf("engine probe could not run: %v", err)
+		case code != 0:
+			return engineRecovering, "engine is not answering on its socket yet (startup or crash recovery)"
+		}
+
+		// ping answers successfully even on ER_ACCESS_DENIED, so on its own it
+		// certifies a server that may be unable to execute anything at all. The
+		// statement is a literal in the argv rather than on stdin because it
+		// carries nothing secret and a probe reads no result back.
+		query := append(slices.Clone(connect), "--batch", "--skip-column-names", "--execute=SELECT 1")
+		switch code, err := runBounded(ctx, client, query...); {
+		case err != nil:
+			return engineRecovering, fmt.Sprintf("engine probe could not run: %v", err)
+		case code != 0:
+			return engineRecovering, "engine answered its socket but would not execute a statement"
+		}
+		return engineServing, ""
+	}
+}
+
+func readPidFile(path string) (int, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return 0, err
+	}
+	pid, err := strconv.Atoi(strings.TrimSpace(string(raw)))
+	if err != nil {
+		return 0, fmt.Errorf("%s does not hold a pid: %w", path, err)
+	}
+	return pid, nil
+}
+
+func (e *mariadbEngine) Stop(ctx context.Context) error {
+	return serviceAction(ctx, e.run, e.rcService, e.service, "stop")
+}
+
+// Only the parameter rollback and a failed apply call this: a restart the
+// control plane wants goes through RebootDBInstance, which cycles the VM.
+//
+// The serving copy is refreshed first, because the server is about to start on
+// whatever is installed. That is exactly what rds-init does on every boot, and
+// it is what keeps the two equal by construction whenever the engine starts.
+func (e *mariadbEngine) Restart(ctx context.Context) error {
+	if err := e.recordServingCopy(); err != nil {
+		return err
+	}
+	return serviceAction(ctx, e.run, e.rcService, e.service, "restart")
+}
+
+// The installed set copied to the name the pending-restart comparison reads. A
+// live SET GLOBAL apply deliberately does not touch it: that value was adopted
+// without a restart, which is what "not pending" means.
+func (e *mariadbEngine) recordServingCopy() error {
+	content, err := os.ReadFile(e.params.installedPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("read the installed parameters: %w", err)
+	}
+	return e.params.write(e.params.servingPath(), content)
+}
+
+// A value safe to interpolate into a single-quoted SQL literal. The quote is
+// doubled, which holds in every sql_mode, and the backslash is doubled, which is
+// why every statement built here pins a mode with backslash escapes on.
+func sqlLiteral(value string) string {
+	return strings.ReplaceAll(strings.ReplaceAll(value, `\`, `\\`), `'`, `''`)
+}
+
+// Turns off the two logs that would otherwise copy a statement's text, and pins
+// the mode the escaping above assumes. All three are session-scoped: a rotation
+// must not leave the customer's own general log switched off behind it.
+const mariadbSessionGuard = `SET SESSION sql_log_off = 1;
+SET SESSION slow_query_log = 0;
+SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';
+`
+
+// Rotates the master's password. The statement is built here rather than by the
+// client, which has neither psql's identifier interpolation nor its parameter
+// quoting, and it rides stdin so it is never visible in the process table.
+func (e *mariadbEngine) SetPassword(ctx context.Context, username, password string) error {
+	if username == "" || password == "" {
+		return fmt.Errorf("set-password requires both %s and %s",
+			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
+	}
+	// This runs as the server's own superuser over the socket, so a reserved name
+	// in the command payload would rotate the platform's account rather than the
+	// customer's. The rest of the rule matters just as much here, because the name
+	// is interpolated into the statement below rather than quoted by the client.
+	if err := e.rules.validateUsername(username); err != nil {
+		return fmt.Errorf("refusing to set the password of role %q: %w", username, err)
+	}
+
+	escaped := sqlLiteral(password)
+	sql := fmt.Sprintf("%sALTER USER '%s'@'%s' IDENTIFIED BY '%s';\n",
+		mariadbSessionGuard, username, mariadbMasterHost, escaped)
+	if _, err := e.clientRun(ctx, sql); err != nil {
+		// The client echoes the failing statement, which carries the escaped form
+		// of the password rather than the raw one — so redacting only the raw one
+		// would leak exactly the passwords that needed escaping.
+		return fmt.Errorf("apply the master password: %s", redact(redact(err.Error(), password), escaped))
+	}
+	return nil
+}
+
+// Installs the resolved set and applies the half of it a running server can
+// adopt. MariaDB re-reads no configuration file while it is up, so SET GLOBAL is
+// the only way an immediate apply reaches it; the static half waits in the
+// installed file for the next start, and is what comes back as pending.
+//
+// There is no offline parser to check the set with first — MariaDB ships no
+// equivalent of postgres -C — so the safety net for a static value the server
+// will not start on is the boot-time rollback to the last accepted set.
+func (e *mariadbEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+
+	initialState, _ := e.probe.state(ctx)
+	if initialState == engineRecovering {
+		return nil, errors.New("apply parameters while the engine is still starting or recovering")
+	}
+	if initialState == engineServing {
+		// A command can arrive before the first heartbeat. Seed the configuration
+		// currently being served before replacing its file in that window.
+		if err := e.recordServingParametersLocked(ctx); err != nil {
+			return nil, fmt.Errorf("record the parameters serving before the apply: %w", err)
+		}
+	}
+
+	restore, err := e.params.install(params)
+	if err != nil {
+		return nil, err
+	}
+
+	state, _ := e.probe.state(ctx)
+	switch state {
+	case engineAbsent:
+		return e.restartOnRepairSetLocked(ctx)
+	case engineRecovering:
+		return nil, errors.Join(errors.New("the engine entered startup or recovery during the parameter apply"), restore())
+	}
+
+	if err := e.applyDynamicParameters(ctx, params); err != nil {
+		// The engine may have gone down after the first probe. In that case start it
+		// on the newly installed set rather than restoring one that already left it
+		// unable to serve.
+		if state, _ := e.probe.state(ctx); state == engineAbsent {
+			return e.restartOnRepairSetLocked(ctx)
+		}
+		return nil, errors.Join(err, restore())
+	}
+	return e.pendingRestartParameters(ctx)
+}
+
+// The dynamic half of the set, in one invocation that stops at the first refusal
+// so the server's own message names the setting it would not take.
+//
+// Only names the catalog carries are emitted, which is also what keeps the
+// statement safe to build: a name is an identifier here, and the classification
+// that selects it comes from the same table the API validated the request
+// against. MariaDB cannot set several globals atomically, so a value set before
+// a refusal stays live until the next restart, when the restored file wins.
+func (e *mariadbEngine) applyDynamicParameters(ctx context.Context, params []handlers_rds.Parameter) error {
+	var b strings.Builder
+	b.WriteString("SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';\n")
+	applied := 0
+	for _, p := range params {
+		if p.Name == "" || e.rules.isStatic(p.Name) {
+			continue
+		}
+		fmt.Fprintf(&b, "SET GLOBAL %s = '%s';\n", p.Name, sqlLiteral(p.Value))
+		applied++
+	}
+	if applied == 0 {
+		return nil
+	}
+	if _, err := e.clientRun(ctx, b.String()); err != nil {
+		return fmt.Errorf("apply the dynamic parameters: %w", err)
+	}
+	return nil
+}
+
+func (e *mariadbEngine) restartOnRepairSetLocked(ctx context.Context) ([]string, error) {
+	return awaitRepairedEngine(ctx, e.probe, e.Restart, e.pendingRestartParameters, e.repairTimeout, e.repairPoll)
+}
+
+// MariaDB has no pending_restart. Nothing in information_schema reports a
+// setting the server has stored but not adopted, so the answer is computed from
+// the files instead: the catalog-static keys whose value in the installed
+// drop-in differs from the one the server actually started on.
+//
+// Comparing live values against the desired set was rejected. mysqld silently
+// rewrites values at startup — rounding the buffer pool and log file sizes, and
+// downgrading max_connections when it cannot obtain enough file descriptors —
+// and each rewrite would read as a permanent pending restart: un-clearable
+// pending-reboot state, permanent drift, and a last-known-good copy frozen for
+// good, silently disabling a recovery mechanism only exercised during an
+// incident.
+func (e *mariadbEngine) pendingRestartParameters(context.Context) ([]string, error) {
+	installed, err := readOptionFile(e.params.installedPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			// Nothing is installed, so nothing is waiting to be adopted.
+			return nil, nil
+		}
+		return nil, fmt.Errorf("read the installed parameters: %w", err)
+	}
+	serving, err := readOptionFile(e.params.servingPath())
+	if err != nil && !os.IsNotExist(err) {
+		return nil, fmt.Errorf("read the parameters the engine started on: %w", err)
+	}
+	// Back to the catalog's names before anything is classified or reported: the
+	// files are written in the engine's startup spellings, and the answer is a
+	// customer-facing list the API refuses ApplyMethod=immediate against.
+	installed = e.catalogKeyed(installed)
+	serving = e.catalogKeyed(serving)
+
+	// An absent serving copy is a set the engine has not started on: rds-init
+	// writes the two together on every boot, so the whole static half counts as
+	// pending rather than being promoted on the strength of a missing file.
+	var pending []string
+	for name, value := range installed {
+		if !e.rules.isStatic(name) {
+			continue
+		}
+		if served, ok := serving[name]; !ok || served != value {
+			pending = append(pending, name)
+		}
+	}
+	// A static setting the group stopped naming reverts to its default at the next
+	// start, which the running server has not adopted either.
+	for name := range serving {
+		if _, ok := installed[name]; !ok && e.rules.isStatic(name) {
+			pending = append(pending, name)
+		}
+	}
+	slices.Sort(pending)
+	return pending, nil
+}
+
+// Snapshots the installed set as the rollback target, but only when nothing in
+// it is still waiting for a restart. The parameter mutex keeps an apply from
+// replacing the file between that check and the copy.
+func (e *mariadbEngine) RecordServingParameters(ctx context.Context) error {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return e.recordServingParametersLocked(ctx)
+}
+
+func (e *mariadbEngine) recordServingParametersLocked(ctx context.Context) error {
+	return recordLastGood(ctx, e.params, e.pendingRestartParameters)
+}
+
+// Puts the last set the engine accepted back in place, for a start that failed
+// after a parameter change.
+func (e *mariadbEngine) RestoreLastKnownGoodParameters(ctx context.Context) (bool, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return restoreLastGood(ctx, e.params, e.probe)
+}
+
+// The same settings under the names the catalog and the API know them by, since
+// a file is written in the engine's startup spellings and those are not always
+// the customer's.
+func (e *mariadbEngine) catalogKeyed(values map[string]string) map[string]string {
+	out := make(map[string]string, len(values))
+	for name, value := range values {
+		out[e.rules.catalogName(name)] = value
+	}
+	return out
+}
+
+// The subset of MariaDB's option-file syntax the generated drop-ins are written
+// in: a group header, comments, and one `name = value` per line. Nothing else
+// reaches these two files, because only rds-init and this agent write them.
+func readOptionFile(path string) (map[string]string, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	values := make(map[string]string)
+	for line := range strings.SplitSeq(string(raw), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") || strings.HasPrefix(line, "[") {
+			continue
+		}
+		// A bare option such as skip-log-bin carries no value; it is still a setting
+		// whose presence or absence differs.
+		name, value, _ := strings.Cut(line, "=")
+		values[normaliseOptionName(name)] = strings.Trim(strings.TrimSpace(value), `"'`)
+	}
+	return values, nil
+}
+
+// MariaDB reads - and _ as the same character in an option name, so the two
+// spellings of one setting must not compare as two settings.
+func normaliseOptionName(name string) string {
+	return strings.ReplaceAll(strings.TrimSpace(name), "-", "_")
+}
+
+// How long any one BACKUP STAGE waits for the metadata locks it needs. Well
+// under the control plane's own quiesce timeout, so a stage that cannot take its
+// lock fails and is reported rather than being abandoned mid-wait with the
+// request still queued in front of live traffic.
+const mariadbQuiesceLockWait = 20 * time.Second
+
+// MariaDB's own backup API rather than FLUSH TABLES WITH READ LOCK. FTWRL turns
+// a snapshot into a write outage: the whole database is read-only until it is
+// released, so a control plane that died after issuing quiesce would leave the
+// customer read-only for the full hold, on a schedule and unattended. Its
+// acquisition phase is unbounded too, with writes queueing behind a pending lock
+// request. BACKUP STAGE blocks commits rather than all writes and its flush
+// stage covers Aria and MyISAM, so it yields a better consistency point while
+// being far less disruptive — and it is what mariadb-backup itself uses.
+func (e *mariadbEngine) Quiesce(ctx context.Context, label string, hold time.Duration) error {
+	if err := validateQuiesceRequest(label, hold); err != nil {
+		return err
+	}
+
+	// Held across the whole start, so a second quiesce waits and then finds the
+	// first one's hold rather than opening a concurrent backup alongside it.
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.held != nil {
+		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)
+	}
+
+	// Deliberately not the command's context: the session has to outlive the call
+	// that started it, and its own deadline is what bounds it instead.
+	session, err := e.startSess(context.WithoutCancel(ctx), command{
+		Name:              e.client,
+		Args:              e.clientArgs(),
+		Env:               []string{"PATH=" + defaultGuestPath},
+		SentinelStatement: "SELECT '" + sessionSentinel + "';\n",
+	})
+	if err != nil {
+		return fmt.Errorf("open a backup session: %w", err)
+	}
+
+	// The stages run in order and on one connection: MariaDB releases the whole
+	// hold with the session that took it, which is what bounds a control plane
+	// that dies mid-snapshot. Fed one at a time so a stage that cannot take its
+	// locks is named rather than reported as a line number.
+	stages := []string{
+		fmt.Sprintf("SET SESSION lock_wait_timeout = %d;\n", int(mariadbQuiesceLockWait.Seconds())),
+		"BACKUP STAGE START;\n",
+		"BACKUP STAGE FLUSH;\n",
+		"BACKUP STAGE BLOCK_DDL;\n",
+		"BACKUP STAGE BLOCK_COMMIT;\n",
+	}
+	for _, sql := range stages {
+		if err := session.Exec(ctx, sql); err != nil {
+			if closeErr := session.Close(); closeErr != nil {
+				slog.Warn("rds-agent: closing a failed backup session", "err", closeErr)
+			}
+			return fmt.Errorf("put the engine into backup mode at %q: %w", strings.TrimSpace(sql), err)
+		}
+	}
+
+	e.beginHoldLocked(label, session, hold)
+	return nil
+}
+
+// Ends the backup cleanly. A missing hold is an error rather than a silent
+// success: it means the deadline fired first, so the snapshot the control plane
+// just took was not taken against a held stage.
+func (e *mariadbEngine) Unquiesce(ctx context.Context) error {
+	held := e.takeHold()
+	if held == nil {
+		return errors.New("the engine is not quiesced; the backup hold had already expired")
+	}
+
+	// The release has to run on the session that took the stages, and the session
+	// is closed either way — the engine ends an unreleased backup with it.
+	execErr := held.session.Exec(ctx, "BACKUP STAGE END;\n")
+	closeErr := held.session.Close()
+	if execErr != nil {
+		return fmt.Errorf("take the engine out of backup mode: %w", errors.Join(execErr, closeErr))
+	}
+	if closeErr != nil {
+		return fmt.Errorf("close the backup session: %w", closeErr)
+	}
+	slog.Info("rds-agent: engine released from backup mode", "label", held.label)
+	return nil
+}
+
+// Feeds sql on stdin rather than as an argument, so a statement is never visible
+// in the process table.
+func (e *mariadbEngine) clientRun(ctx context.Context, sql string) (string, error) {
+	return e.run(ctx, command{
+		Name:  e.client,
+		Args:  e.clientArgs(),
+		Env:   []string{"PATH=" + defaultGuestPath},
+		Stdin: sql,
+	})
+}
+
+// --no-defaults so no option file can move the connection this agent makes, and
+// --unbuffered so a held session's output reaches the reader as each statement
+// finishes rather than when a pipe buffer fills. No User: the agent is root, and
+// root is what the datadir's unix_socket plugin authenticates.
+func (e *mariadbEngine) clientArgs() []string {
+	return []string{
+		"--no-defaults", "--batch", "--skip-column-names", "--unbuffered",
+		"--protocol=socket", "--socket=" + e.socket, "--user=" + mariadbSuperuser,
+	}
+}

cmd/rds-agent/engine_mariadb_test.go

@@ -0,0 +1,796 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/user"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// Stands in for the control plane's own definition of the engine. It mirrors the
+// production wiring exactly, including that a name the catalog does not carry
+// counts as static: a rule the fake inverted would hide the failure it exists to
+// prevent.
+func testMariaDBRules() controlPlaneRules {
+	applyTypes := map[string]string{
+		"innodb_buffer_pool_size": handlers_rds.ApplyTypeStatic,
+		"innodb_log_file_size":    handlers_rds.ApplyTypeStatic,
+		"max_connections":         handlers_rds.ApplyTypeDynamic,
+		"long_query_time":         handlers_rds.ApplyTypeDynamic,
+		"time_zone":               handlers_rds.ApplyTypeDynamic,
+	}
+	reserved := []string{"root", "mysql", "mariadb.sys", "rdsadmin", "public"}
+
+	return controlPlaneRules{
+		validateUsername: func(username string) error {
+			lower := strings.ToLower(username)
+			if slices.Contains(reserved, lower) || strings.HasPrefix(lower, "mysql.") || strings.HasPrefix(lower, "mariadb.") {
+				return fmt.Errorf("MasterUsername %q is reserved by mariadb", username)
+			}
+			for _, r := range username {
+				if !isTestIdentifierRune(r) {
+					return fmt.Errorf("MasterUsername may contain only letters, digits and underscores")
+				}
+			}
+			return nil
+		},
+		isStatic: func(name string) bool {
+			applyType, ok := applyTypes[name]
+			return !ok || applyType == handlers_rds.ApplyTypeStatic
+		},
+		catalogName: func(optionFileName string) string {
+			if optionFileName == "default_time_zone" {
+				return "time_zone"
+			}
+			return optionFileName
+		},
+	}
+}
+
+func isTestIdentifierRune(r rune) bool {
+	return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_'
+}
+
+func newTestMariaDBEngine(t *testing.T, run commandRunner) *mariadbEngine {
+	t.Helper()
+	cfg := testLoadConfig(t, engineMariaDB)
+	cfg.DataMount = t.TempDir()
+	if err := os.MkdirAll(filepath.Join(cfg.DataMount, "conf.d"), 0o750); err != nil {
+		t.Fatalf("create conf.d: %v", err)
+	}
+	// The guest's mysql user does not exist here, and the chown of the installed
+	// parameter file has to resolve to something real.
+	current, err := user.Current()
+	if err != nil {
+		t.Fatalf("resolve the current user: %v", err)
+	}
+	cfg.EngineUser = current.Username
+
+	state := engineServing
+	return newMariaDBEngine(cfg, testMariaDBRules(), run, nil, drivenProbe(&state))
+}
+
+// The wiring the guest takes its classification from, checked against a catalog
+// that is registered: what a spec says about a parameter is what the guest must
+// see, and an unlisted name must read as static rather than as dynamic.
+func TestControlPlaneRulesFrom_TakesTheClassificationFromTheCatalog(t *testing.T) {
+	rules := controlPlaneRulesFrom(postgresEngineMeta)
+
+	for name, want := range map[string]bool{"shared_buffers": true, "work_mem": false} {
+		spec, ok := postgresEngineMeta.LookupParameter(name)
+		if !ok {
+			t.Fatalf("the catalog no longer carries %s", name)
+		}
+		if (spec.ApplyType == handlers_rds.ApplyTypeStatic) != want {
+			t.Fatalf("%s is %s in the catalog, so this test is asserting the wrong way round", name, spec.ApplyType)
+		}
+		if got := rules.isStatic(name); got != want {
+			t.Errorf("isStatic(%s) = %v, want %v", name, got, want)
+		}
+	}
+	if !rules.isStatic("not_a_parameter_any_catalog_carries") {
+		t.Error("a name the catalog does not carry read as dynamic, so it would be issued as a live SET GLOBAL")
+	}
+	if rules.validateUsername("postgres") == nil {
+		t.Error("the username rule does not refuse a reserved role")
+	}
+}
+
+// The client has no parameter quoting of its own, so the password is built into
+// the statement here. It must reach the process on stdin and nowhere else.
+func TestMariaDBEngine_SetPasswordKeepsTheSecretOutOfArgvAndEnv(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", "n3w-pw"); err != nil {
+		t.Fatalf("SetPassword: %v", err)
+	}
+	if len(runner.calls) != 1 {
+		t.Fatalf("ran %d commands, want 1", len(runner.calls))
+	}
+	call := runner.calls[0]
+	for _, arg := range call.Args {
+		if strings.Contains(arg, "n3w-pw") {
+			t.Fatalf("argv %v carries the password", call.Args)
+		}
+	}
+	for _, env := range call.Env {
+		if strings.Contains(env, "n3w-pw") {
+			t.Fatalf("env %v carries the password, which the client would not read anyway", call.Env)
+		}
+	}
+	if !strings.Contains(call.Stdin, "ALTER USER 'mulgamaster'@'%' IDENTIFIED BY 'n3w-pw';") {
+		t.Errorf("SQL %q does not rotate the master on the wildcard host", call.Stdin)
+	}
+}
+
+// ValidateMasterUserPassword permits both of these, and the client offers no
+// equivalent of psql's parameter quoting, so the escaping here is what keeps a
+// password from ending the literal it is inside.
+func TestMariaDBEngine_SetPasswordEscapesQuotesAndBackslashes(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", `a'b\c`); err != nil {
+		t.Fatalf("SetPassword: %v", err)
+	}
+	sql := runner.calls[0].Stdin
+	if !strings.Contains(sql, `IDENTIFIED BY 'a''b\\c';`) {
+		t.Errorf("SQL %q does not escape both the quote and the backslash", sql)
+	}
+	// The escaping assumes a mode where backslash escapes are on, so the statement
+	// cannot inherit whatever sql_mode the customer's parameter group set.
+	if !strings.Contains(sql, "SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';") {
+		t.Errorf("SQL %q does not pin the mode its escaping assumes", sql)
+	}
+}
+
+// The password is interpolated into the statement, so a parameter group that
+// turned the general or slow log on would copy it into a log file every snapshot
+// then carries. Both are silenced for this session only: a rotation must not
+// leave the customer's own logging switched off behind it.
+func TestMariaDBEngine_SetPasswordSilencesStatementLoggingForItsSessionOnly(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", "n3w-pw"); err != nil {
+		t.Fatalf("SetPassword: %v", err)
+	}
+	sql := runner.calls[0].Stdin
+	alter := strings.Index(sql, "ALTER USER")
+	if alter < 0 {
+		t.Fatalf("SQL %q does not alter the user", sql)
+	}
+	for _, guard := range []string{"SET SESSION sql_log_off = 1;", "SET SESSION slow_query_log = 0;"} {
+		at := strings.Index(sql, guard)
+		if at < 0 {
+			t.Errorf("SQL %q is missing %q", sql, guard)
+			continue
+		}
+		if at > alter {
+			t.Errorf("SQL %q sets %q only after the statement has already been logged", sql, guard)
+		}
+	}
+	if strings.Contains(sql, "SET GLOBAL general_log") || strings.Contains(sql, "SET GLOBAL slow_query_log") {
+		t.Errorf("SQL %q turns logging off globally, which outlives the rotation", sql)
+	}
+}
+
+// The client echoes the statement it failed on, which carries the escaped form
+// of the password rather than the raw one — so redacting only the raw one would
+// leak exactly the passwords that needed escaping.
+func TestMariaDBEngine_SetPasswordRedactsBothFormsOfTheSecret(t *testing.T) {
+	secret := `a'b\c`
+	runner := &recordingRunner{err: errors.New(`ERROR 1064 near "IDENTIFIED BY 'a''b\\c'"`)}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	err := engine.SetPassword(context.Background(), "mulgamaster", secret)
+	if err == nil {
+		t.Fatal("SetPassword succeeded against a failing client")
+	}
+	if strings.Contains(err.Error(), secret) || strings.Contains(err.Error(), sqlLiteral(secret)) {
+		t.Errorf("error %q leaks the password", err)
+	}
+	if !strings.Contains(err.Error(), "[REDACTED]") {
+		t.Errorf("error %q does not mark the redaction", err)
+	}
+}
+
+// The apply runs as the server's own superuser over the socket, and the name is
+// interpolated into the statement rather than quoted by the client, so both
+// halves of the rule are refused before any SQL is built.
+func TestMariaDBEngine_SetPasswordRefusesNamesTheEngineWillNotTake(t *testing.T) {
+	for _, username := range []string{"root", "mysql", "mariadb.sys", "rdsadmin", "PUBLIC", "mysql.infoschema", `ma'ster`} {
+		t.Run(username, func(t *testing.T) {
+			runner := &recordingRunner{}
+			engine := newTestMariaDBEngine(t, runner.run)
+
+			if err := engine.SetPassword(context.Background(), username, "n3w-pw"); err == nil {
+				t.Fatalf("SetPassword accepted %q", username)
+			}
+			if len(runner.calls) != 0 {
+				t.Errorf("ran %d commands for a refused name, want 0", len(runner.calls))
+			}
+		})
+	}
+}
+
+func TestMariaDBEngine_SetPasswordRejectsAnIncompleteCommand(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", ""); err == nil {
+		t.Error("SetPassword accepted an empty password")
+	}
+	if len(runner.calls) != 0 {
+		t.Errorf("ran %d commands for an incomplete request, want 0", len(runner.calls))
+	}
+}
+
+// A running server re-reads no configuration file, so the dynamic half has to
+// reach it as SET GLOBAL. The static half is installed and waits for a restart,
+// which is what comes back as pending.
+func TestMariaDBEngine_ApplyParametersSetsOnlyTheDynamicHalfLive(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	pending, err := engine.ApplyParameters(context.Background(), []handlers_rds.Parameter{
+		{Name: "max_connections", Value: "200"},
+		{Name: "innodb_buffer_pool_size", Value: "1073741824"},
+	})
+	if err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+
+	installed, err := os.ReadFile(engine.params.installedPath())
+	if err != nil {
+		t.Fatalf("read the installed parameters: %v", err)
+	}
+	if !strings.HasPrefix(string(installed), "[mysqld]\n") {
+		t.Errorf("installed parameters = %q, want the group header rds-init writes", installed)
+	}
+	for _, want := range []string{"max_connections = '200'", "innodb_buffer_pool_size = '1073741824'"} {
+		if !strings.Contains(string(installed), want) {
+			t.Errorf("installed parameters = %q, want %q", installed, want)
+		}
+	}
+
+	if len(runner.calls) != 1 {
+		t.Fatalf("ran %d commands, want the one live apply", len(runner.calls))
+	}
+	sql := runner.calls[0].Stdin
+	if !strings.Contains(sql, "SET GLOBAL max_connections = '200';") {
+		t.Errorf("SQL %q does not set the dynamic value live", sql)
+	}
+	if strings.Contains(sql, "innodb_buffer_pool_size") {
+		t.Errorf("SQL %q sets a static value live, which the server would refuse", sql)
+	}
+	if !slices.Equal(pending, []string{"innodb_buffer_pool_size"}) {
+		t.Errorf("pending = %v, want the static setting the engine has not adopted", pending)
+	}
+}
+
+// A set with nothing dynamic in it must not open a connection at all: the
+// installed file is the whole of the change.
+func TestMariaDBEngine_ApplyParametersRunsNoStatementForAStaticOnlySet(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "innodb_log_file_size", Value: "268435456"}}); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if len(runner.calls) != 0 {
+		t.Errorf("ran %d commands for a static-only set, want 0", len(runner.calls))
+	}
+}
+
+// A value the server refuses leaves the installed file back where it was, so it
+// does not sit on the data volume turning the next restart into a boot loop.
+func TestMariaDBEngine_ApplyParametersRollsBackAValueTheServerRefuses(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "max_connections", Value: "200"}}); err != nil {
+		t.Fatalf("first ApplyParameters: %v", err)
+	}
+
+	runner.err = errors.New("ERROR 1231 (42000): Variable 'max_connections' can't be set to the value of '0'")
+	if _, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "max_connections", Value: "0"}}); err == nil {
+		t.Fatal("ApplyParameters succeeded against a value the server refused")
+	}
+
+	installed, err := os.ReadFile(engine.params.installedPath())
+	if err != nil {
+		t.Fatalf("read the installed parameters: %v", err)
+	}
+	if !strings.Contains(string(installed), "max_connections = '200'") {
+		t.Errorf("installed parameters = %q, want the previous accepted set restored", installed)
+	}
+}
+
+// The whole of the pending-restart answer, since MariaDB has none of its own:
+// the installed drop-in against the one the server actually started on, over the
+// static keys only.
+func TestMariaDBEngine_PendingRestartComparesTheInstalledSetAgainstTheServingCopy(t *testing.T) {
+	engine := newTestMariaDBEngine(t, (&recordingRunner{}).run)
+	write := func(path, body string) {
+		t.Helper()
+		if err := os.WriteFile(path, []byte("[mysqld]\n"+body), 0o600); err != nil {
+			t.Fatalf("write %s: %v", path, err)
+		}
+	}
+	pending := func() []string {
+		t.Helper()
+		names, err := engine.pendingRestartParameters(t.Context())
+		if err != nil {
+			t.Fatalf("pendingRestartParameters: %v", err)
+		}
+		return names
+	}
+
+	// A fresh boot: rds-init wrote the two together, so nothing is waiting.
+	write(engine.params.installedPath(), "innodb_buffer_pool_size = '1073741824'\nmax_connections = '100'\n")
+	write(engine.params.servingPath(), "innodb_buffer_pool_size = '1073741824'\nmax_connections = '100'\n")
+	if names := pending(); len(names) != 0 {
+		t.Errorf("pending = %v, want nothing on a set the engine started on", names)
+	}
+
+	// A dynamic value the server took live is not pending: it was adopted without
+	// a restart, which is exactly what the serving copy not moving records.
+	write(engine.params.installedPath(), "innodb_buffer_pool_size = '1073741824'\nmax_connections = '200'\n")
+	if names := pending(); len(names) != 0 {
+		t.Errorf("pending = %v, want nothing for a value adopted live", names)
+	}
+
+	// A static value is.
+	write(engine.params.installedPath(), "innodb_buffer_pool_size = '2147483648'\nmax_connections = '200'\n")
+	if names := pending(); !slices.Equal(names, []string{"innodb_buffer_pool_size"}) {
+		t.Errorf("pending = %v, want the static setting", names)
+	}
+
+	// So is a static setting the group stopped naming, which reverts to its
+	// default at the next start.
+	write(engine.params.installedPath(), "max_connections = '200'\n")
+	if names := pending(); !slices.Equal(names, []string{"innodb_buffer_pool_size"}) {
+		t.Errorf("pending = %v, want the withdrawn static setting", names)
+	}
+
+	// MariaDB reads - and _ as the same character, so one setting spelled two ways
+	// is not two settings.
+	write(engine.params.installedPath(), "innodb-buffer-pool-size = '1073741824'\nmax_connections = '200'\n")
+	if names := pending(); len(names) != 0 {
+		t.Errorf("pending = %v, want the two spellings of one setting to compare equal", names)
+	}
+}
+
+// rds-init writes the serving copy beside the installed one on every boot, so a
+// set with no copy beside it is one the engine has not started on. Promoting it
+// would point the rollback at a configuration that has never served.
+func TestMariaDBEngine_PendingRestartTreatsAMissingServingCopyAsUnadopted(t *testing.T) {
+	engine := newTestMariaDBEngine(t, (&recordingRunner{}).run)
+	if err := os.WriteFile(engine.params.installedPath(),
+		[]byte("[mysqld]\ninnodb_buffer_pool_size = '1073741824'\nmax_connections = '100'\n"), 0o600); err != nil {
+		t.Fatalf("write installed parameters: %v", err)
+	}
+
+	names, err := engine.pendingRestartParameters(t.Context())
+	if err != nil {
+		t.Fatalf("pendingRestartParameters: %v", err)
+	}
+	if !slices.Equal(names, []string{"innodb_buffer_pool_size"}) {
+		t.Errorf("pending = %v, want the static half of a set the engine never started on", names)
+	}
+}
+
+// The restart the agent performs is the only thing that moves the serving copy,
+// which is what makes a fresh start clear whatever was pending.
+func TestMariaDBEngine_RestartRefreshesTheServingCopy(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+	installed := []byte("[mysqld]\ninnodb_buffer_pool_size = '2147483648'\n")
+	if err := os.WriteFile(engine.params.installedPath(), installed, 0o600); err != nil {
+		t.Fatalf("write installed parameters: %v", err)
+	}
+	if err := os.WriteFile(engine.params.servingPath(), []byte("[mysqld]\ninnodb_buffer_pool_size = '1073741824'\n"), 0o600); err != nil {
+		t.Fatalf("write serving parameters: %v", err)
+	}
+
+	if err := engine.Restart(context.Background()); err != nil {
+		t.Fatalf("Restart: %v", err)
+	}
+
+	serving, err := os.ReadFile(engine.params.servingPath())
+	if err != nil {
+		t.Fatalf("read the serving copy: %v", err)
+	}
+	if string(serving) != string(installed) {
+		t.Errorf("serving copy = %q, want the set the engine was just started on", serving)
+	}
+	names, err := engine.pendingRestartParameters(t.Context())
+	if err != nil {
+		t.Fatalf("pendingRestartParameters: %v", err)
+	}
+	if len(names) != 0 {
+		t.Errorf("pending = %v, want a restart to have cleared it", names)
+	}
+	// The copy the comparison reads must never be parsed as a second set of
+	// settings by the engine's own include glob.
+	if strings.HasSuffix(engine.params.servingPath(), ".cnf") {
+		t.Errorf("the serving copy at %s would be included by the engine", engine.params.servingPath())
+	}
+}
+
+// The rollback target advances only once a restart has proved the installed
+// static values can serve, not when a dynamic apply merely set them live.
+func TestMariaDBEngine_LastKnownGoodWaitsForARestartToAdoptAStaticSet(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(t.Context(),
+		[]handlers_rds.Parameter{{Name: "innodb_buffer_pool_size", Value: "1073741824"}}); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("RecordServingParameters: %v", err)
+	}
+	if _, err := os.Stat(engine.params.lastGoodPath()); !os.IsNotExist(err) {
+		t.Errorf("a set the engine has not restarted onto became the rollback target (stat err = %v)", err)
+	}
+
+	if err := engine.Restart(t.Context()); err != nil {
+		t.Fatalf("Restart: %v", err)
+	}
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("RecordServingParameters after the restart: %v", err)
+	}
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
+	if err != nil {
+		t.Fatalf("read the last known good parameters: %v", err)
+	}
+	if !strings.Contains(string(lastGood), "innodb_buffer_pool_size = '1073741824'") {
+		t.Errorf("last known good = %q, want the set the engine restarted onto", lastGood)
+	}
+}
+
+// A dynamic apply is adopted without a restart, so it may advance the rollback
+// target on the next healthy beat rather than waiting for one.
+func TestMariaDBEngine_LastKnownGoodAdvancesOnADynamicApply(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+	if err := os.WriteFile(engine.params.servingPath(), []byte("[mysqld]\n"), 0o600); err != nil {
+		t.Fatalf("write serving parameters: %v", err)
+	}
+
+	if _, err := engine.ApplyParameters(t.Context(),
+		[]handlers_rds.Parameter{{Name: "max_connections", Value: "200"}}); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("RecordServingParameters: %v", err)
+	}
+
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
+	if err != nil {
+		t.Fatalf("read the last known good parameters: %v", err)
+	}
+	if !strings.Contains(string(lastGood), "max_connections = '200'") {
+		t.Errorf("last known good = %q, want the set the engine is already running", lastGood)
+	}
+}
+
+// A probe that reports what the test hands it, in place of whichever binaries
+// the engine's own state function would have run.
+func mariadbTestProbeConfig(t *testing.T) config {
+	t.Helper()
+	cfg := testLoadConfig(t, engineMariaDB)
+	cfg.EnginePidFile = filepath.Join(t.TempDir(), "mariadb.pid")
+	return cfg
+}
+
+func writePid(t *testing.T, path string, pid int) {
+	t.Helper()
+	if err := os.WriteFile(path, []byte(strconv.Itoa(pid)+"\n"), 0o644); err != nil {
+		t.Fatalf("write pidfile: %v", err)
+	}
+}
+
+// The three stages, and what each one separates. During InnoDB crash recovery
+// mariadbd opens neither its socket nor its port, so nothing the client can ask
+// distinguishes a server replaying its redo log from no server at all.
+func TestMariaDBProbe_SeparatesRecoveringFromAbsent(t *testing.T) {
+	tests := []struct {
+		name     string
+		writePid bool
+		alive    bool
+		pingCode int
+		pingErr  error
+		execCode int
+		want     engineState
+	}{
+		{name: "no pidfile at all", want: engineAbsent},
+		{name: "pidfile names a dead process", writePid: true, want: engineAbsent},
+		{name: "process alive but socket silent", writePid: true, alive: true, pingCode: 1, want: engineRecovering},
+		{name: "process alive but probe cannot run", writePid: true, alive: true, pingErr: errors.New("no such file"), want: engineRecovering},
+		{name: "answers ping but executes nothing", writePid: true, alive: true, execCode: 1, want: engineRecovering},
+		{name: "serving", writePid: true, alive: true, want: engineServing},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := mariadbTestProbeConfig(t)
+			if tt.writePid {
+				writePid(t, cfg.EnginePidFile, 4242)
+			}
+			run := func(_ context.Context, name string, args ...string) (int, error) {
+				if strings.HasSuffix(name, mariadbAdminBinary) {
+					return tt.pingCode, tt.pingErr
+				}
+				if !slices.Contains(args, "--execute=SELECT 1") {
+					t.Errorf("the second stage ran %s %v, want a statement the server has to execute", name, args)
+				}
+				return tt.execCode, nil
+			}
+			stateFn := mariadbProbeState(cfg, run, func(int) bool { return tt.alive })
+
+			got, message := stateFn(context.Background(), int64(cfg.EnginePort))
+			if got != tt.want {
+				t.Errorf("state = %v, want %v (%s)", got, tt.want, message)
+			}
+			if got != engineServing && message == "" {
+				t.Error("a non-serving result carried no message explaining it")
+			}
+		})
+	}
+}
+
+// The failure the guard exists for still behaves correctly: a static value
+// mariadbd will not accept makes it exit during startup, so the pidfile names a
+// process that is gone and the deadline runs.
+func TestMariaDBProbe_BoundsAStalledClient(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	cfg.EngineProbeTimeout = 20 * time.Millisecond
+	writePid(t, cfg.EnginePidFile, 4242)
+
+	started := time.Now()
+	state, message := mariadbProbeState(cfg,
+		func(ctx context.Context, _ string, args ...string) (int, error) {
+			if !slices.Contains(args, "--connect-timeout=3") {
+				t.Errorf("probe args = %v, want a client connection timeout", args)
+			}
+			<-ctx.Done()
+			return -1, ctx.Err()
+		},
+		func(int) bool { return true })(t.Context(), int64(cfg.EnginePort))
+
+	if state != engineRecovering {
+		t.Errorf("state = %v, want recovering after a live engine's probe timed out", state)
+	}
+	if !strings.Contains(message, context.DeadlineExceeded.Error()) {
+		t.Errorf("message = %q, want the probe deadline", message)
+	}
+	if elapsed := time.Since(started); elapsed > time.Second {
+		t.Errorf("probe took %v, want the configured deadline to bound it", elapsed)
+	}
+}
+
+func TestMariaDBProbe_MapsToTheHealthTheControlPlaneReads(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	writePid(t, cfg.EnginePidFile, 4242)
+	alive := true
+	serving := true
+	probe := newEngineProbe(cfg.EnginePort, mariadbProbeState(cfg,
+		func(context.Context, string, ...string) (int, error) {
+			if serving {
+				return 0, nil
+			}
+			return 1, nil
+		},
+		func(int) bool { return alive }))
+
+	if got, _ := probe.Check(context.Background()); got != handlers_rds.EngineHealthHealthy {
+		t.Fatalf("health = %q, want healthy", got)
+	}
+	serving = false
+	if got, _ := probe.Check(context.Background()); got != handlers_rds.EngineHealthStarting {
+		t.Errorf("health while recovering = %q, want starting", got)
+	}
+	alive = false
+	if got, _ := probe.Check(context.Background()); got != handlers_rds.EngineHealthUnhealthy {
+		t.Errorf("health after the engine went away = %q, want unhealthy", got)
+	}
+}
+
+// An instance killed hard mid-write can spend minutes replaying its redo log. If
+// that read as absent, the guard would roll the parameter file back and restart
+// the server mid-recovery — a rollback that exists to break a boot loop would
+// create one, on an instance whose parameters were never at fault.
+func TestMariaDBProbe_LiveButUnreachableEngineResetsTheRollbackDeadline(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	writePid(t, cfg.EnginePidFile, 4242)
+	probe := newEngineProbe(cfg.EnginePort, mariadbProbeState(cfg,
+		func(context.Context, string, ...string) (int, error) { return 1, nil },
+		func(int) bool { return true }))
+
+	recovery := &fakeRecovery{restored: true}
+	guard := newTestGuard(recovery, probe)
+	// Bounded so the deadline-resetting loop cannot run forever if it stops
+	// resetting; the assertion is that the rollback was never reached.
+	ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
+	defer cancel()
+	guard.Run(ctx)
+
+	if recovery.restarts != 0 {
+		t.Errorf("restarts = %d, want none while the engine is still recovering", recovery.restarts)
+	}
+}
+
+// MariaDB releases a BACKUP STAGE with the connection that took it, so the whole
+// sequence has to run on one held session — which is also what bounds a control
+// plane that dies mid-snapshot.
+func TestMariaDBEngine_QuiesceTakesTheBackupStagesOnAHeldSession(t *testing.T) {
+	session := &fakeSession{}
+	engine := newTestMariaDBEngine(t, (&recordingRunner{}).run)
+	var started []command
+	engine.startSess = func(_ context.Context, c command) (engineSession, error) {
+		started = append(started, c)
+		return session, nil
+	}
+
+	if err := engine.Quiesce(context.Background(), "orders-db-pre-upgrade", time.Minute); err != nil {
+		t.Fatalf("Quiesce: %v", err)
+	}
+
+	if len(started) != 1 {
+		t.Fatalf("started %d sessions, want 1", len(started))
+	}
+	if !slices.Contains(started[0].Args, "--unbuffered") {
+		t.Errorf("session argv %v does not flush after each statement, so the sentinel could sit in a pipe buffer", started[0].Args)
+	}
+	statements := session.statements()
+	want := []string{
+		"SET SESSION lock_wait_timeout = 20;",
+		"BACKUP STAGE START;",
+		"BACKUP STAGE FLUSH;",
+		"BACKUP STAGE BLOCK_DDL;",
+		"BACKUP STAGE BLOCK_COMMIT;",
+	}
+	if len(statements) != len(want) {
+		t.Fatalf("statements = %q, want the bounded stage sequence %q", statements, want)
+	}
+	for i, sql := range want {
+		if strings.TrimSpace(statements[i]) != sql {
+			t.Errorf("statement %d = %q, want %q", i, statements[i], sql)
+		}
+	}
+	// FLUSH TABLES WITH READ LOCK would make the whole database read-only for the
+	// length of the hold rather than blocking commits.
+	if strings.Contains(strings.Join(statements, " "), "FLUSH TABLES") {
+		t.Error("the quiesce took a read lock over the whole database")
+	}
+	if session.closes() != 0 {
+		t.Error("the session was closed while the backup was meant to be held")
+	}
+}
+
+// The stage bound has to leave the control plane's own quiesce timeout room for
+// the whole sequence, or a stage abandoned mid-wait leaves its lock request
+// queued in front of live traffic.
+func TestMariaDBEngine_QuiesceLockWaitFitsInsideTheControlPlaneTimeout(t *testing.T) {
+	const stages = 4
+	if stages*mariadbQuiesceLockWait >= 2*time.Minute {
+		t.Errorf("%d stages of %s can outlast the control plane's quiesce timeout", stages, mariadbQuiesceLockWait)
+	}
+}
+
+func TestMariaDBEngine_UnquiesceEndsTheBackupAndTheSession(t *testing.T) {
+	session := &fakeSession{}
+	engine := newTestMariaDBEngine(t, (&recordingRunner{}).run)
+	engine.startSess = func(context.Context, command) (engineSession, error) { return session, nil }
+	if err := engine.Quiesce(context.Background(), "orders-db-pre-upgrade", time.Minute); err != nil {
+		t.Fatalf("Quiesce: %v", err)
+	}
+
+	if err := engine.Unquiesce(context.Background()); err != nil {
+		t.Fatalf("Unquiesce: %v", err)
+	}
+
+	statements := session.statements()
+	if last := strings.TrimSpace(statements[len(statements)-1]); last != "BACKUP STAGE END;" {
+		t.Errorf("last statement = %q, want the stage released", last)
+	}
+	if session.closes() != 1 {
+		t.Errorf("closed %d times, want the session ended exactly once", session.closes())
+	}
+}
+
+// The two halves of a time_zone apply use different names: SET GLOBAL takes the
+// customer's, the option file the server's startup spelling. Getting either
+// backwards is a statement the server refuses or a boot it refuses.
+func TestMariaDBEngine_TimeZoneAppliesLiveByItsCustomerFacingName(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(context.Background(), []handlers_rds.Parameter{
+		{Name: "time_zone", Value: "+10:00"},
+	}); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+
+	if len(runner.calls) != 1 {
+		t.Fatalf("ran %d commands, want the one live apply", len(runner.calls))
+	}
+	sql := runner.calls[0].Stdin
+	if !strings.Contains(sql, "SET GLOBAL time_zone = '+10:00';") {
+		t.Errorf("SQL %q does not set time_zone under the name MariaDB accepts for SET", sql)
+	}
+	if strings.Contains(sql, "default_time_zone") {
+		t.Errorf("SQL %q names the startup spelling, which is not a settable variable", sql)
+	}
+
+	installed, err := os.ReadFile(engine.params.installedPath())
+	if err != nil {
+		t.Fatalf("read the installed parameters: %v", err)
+	}
+	if !strings.Contains(string(installed), "default_time_zone = '+10:00'") {
+		t.Errorf("installed parameters = %q, want the startup spelling", installed)
+	}
+}
+
+// A dynamic setting applied live deliberately leaves the serving copy alone, so
+// the two files differ by design. Read back under the startup spelling the
+// catalog does not carry, that difference would classify as static and report a
+// pending restart that nothing could ever clear.
+func TestMariaDBEngine_ALiveTimeZoneChangeLeavesNothingPendingRestart(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if err := os.WriteFile(engine.params.servingPath(),
+		[]byte("[mysqld]\ndefault_time_zone = 'SYSTEM'\n"), 0o600); err != nil {
+		t.Fatalf("seed the serving copy: %v", err)
+	}
+
+	pending, err := engine.ApplyParameters(context.Background(), []handlers_rds.Parameter{
+		{Name: "time_zone", Value: "+10:00"},
+	})
+	if err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if len(pending) != 0 {
+		t.Errorf("pending = %v, want nothing: time_zone is dynamic and was applied live", pending)
+	}
+}
+
+// The list is customer-facing, so a static setting still has to be reported
+// under the name the customer set rather than the engine's startup spelling.
+func TestMariaDBEngine_PendingRestartReportsCatalogNames(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if err := os.WriteFile(engine.params.servingPath(),
+		[]byte("[mysqld]\ninnodb_buffer_pool_size = '536870912'\ndefault_time_zone = 'SYSTEM'\n"), 0o600); err != nil {
+		t.Fatalf("seed the serving copy: %v", err)
+	}
+
+	pending, err := engine.ApplyParameters(context.Background(), []handlers_rds.Parameter{
+		{Name: "innodb_buffer_pool_size", Value: "1073741824"},
+		{Name: "time_zone", Value: "SYSTEM"},
+	})
+	if err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if !slices.Equal(pending, []string{"innodb_buffer_pool_size"}) {
+		t.Errorf("pending = %v, want only the static setting under its catalog name", pending)
+	}
+}

cmd/rds-agent/engine_postgres.go

@@ -0,0 +1,380 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The engine is reached over its unix socket under peer authentication, so the
+// agent drops to the postgres OS user rather than holding a password of its own.
+type postgresEngine struct {
+	quiesceState
+
+	// The control plane's own metadata for this engine, resolved once at
+	// startup: the live password apply runs as the cluster superuser, so it
+	// re-checks the role name against the same reserved set the API validates.
+	meta      handlers_rds.Engine
+	run       commandRunner
+	startSess sessionRunner
+	psql      string
+	rcService string
+	service   string
+	pgData    string
+	socketDir string
+	osUser    string
+	probe     *engineProbe
+	// Set from the bootstrap config, on a different goroutine than the commands
+	// that read it.
+	port atomic.Int64
+
+	params parameterStore
+	// Serializes parameter installs, serving snapshots and rollback restores so
+	// none can copy or replace an intermediate configuration.
+	paramMu       sync.Mutex
+	repairTimeout time.Duration
+	repairPoll    time.Duration
+}
+
+var _ engine = (*postgresEngine)(nil)
+
+// pg_isready is what gives PostgreSQL three states for free: exit 0 is serving,
+// exit 1 is a postmaster up and rejecting connections, and anything else is an
+// engine that is not there at all. Resolved on PATH, where the client package
+// puts it.
+const postgresProbeBinary = "pg_isready"
+
+func newPostgresProbe(cfg config, run probeRunner) *engineProbe {
+	return newEngineProbe(cfg.EnginePort, postgresProbeState(cfg.EngineHost, run))
+}
+
+func postgresProbeState(host string, run probeRunner) probeStateFn {
+	return func(ctx context.Context, port int64) (engineState, string) {
+		portArg := strconv.FormatInt(port, 10)
+		code, err := run(ctx, postgresProbeBinary, "-h", host, "-p", portArg, "-q")
+		switch {
+		case err != nil:
+			// A missing binary or broken image. Reporting healthy on the strength of
+			// nothing would hide it, so this reads as absent like an engine that did
+			// not answer.
+			return engineAbsent, fmt.Sprintf("engine probe could not run: %v", err)
+		case code == 0:
+			return engineServing, ""
+		case code == 1:
+			return engineRecovering, "engine is rejecting connections (startup or recovery)"
+		default:
+			return engineAbsent, fmt.Sprintf("engine did not respond on %s:%s", host, portArg)
+		}
+	}
+}
+
+// Resolved once, under the name the control plane knows this implementation by.
+// A lookup failure is a mismatch between the agent and the control plane it was
+// built against, so it fails at startup rather than at the first rotation.
+var postgresEngineMeta = mustLookupEngine(enginePostgres)
+
+func mustLookupEngine(name string) handlers_rds.Engine {
+	engine, err := handlers_rds.LookupEngine(name)
+	if err != nil {
+		panic("rds-agent: " + err.Error())
+	}
+	return engine
+}
+
+// The include the resolved parameter set is rendered to, and the copy of the
+// last one the engine accepted. Both live beside the data rather than in /etc,
+// matching rds-init: a class change boots a fresh root volume, which would
+// otherwise revert them.
+const (
+	postgresParametersFile = "10-rds-parameters.conf"
+	// Deliberately not a .conf name: include_dir globs *.conf, so the rollback
+	// copy must not be read as a second set of settings.
+	postgresLastGoodFile = "10-rds-parameters.last-good"
+)
+
+func newPostgresEngine(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) *postgresEngine {
+	e := &postgresEngine{
+		meta:      postgresEngineMeta,
+		run:       run,
+		startSess: startSess,
+		psql:      filepath.Join(cfg.EngineBinDir, "psql"),
+		rcService: cfg.RCService,
+		service:   cfg.EngineService,
+		pgData:    cfg.EngineDataDir,
+		socketDir: cfg.SocketDir,
+		osUser:    cfg.EngineUser,
+		probe:     probe,
+		params: parameterStore{
+			dir:       filepath.Join(cfg.EngineDataDir, "conf.d"),
+			installed: postgresParametersFile,
+			lastGood:  postgresLastGoodFile,
+			osUser:    cfg.EngineUser,
+			engine:    enginePostgres,
+		},
+		repairTimeout: parameterRepairTimeout,
+		repairPoll:    parameterRepairPoll,
+	}
+	e.port.Store(int64(cfg.EnginePort))
+	return e
+}
+
+func (e *postgresEngine) setPort(port int) {
+	e.port.Store(int64(port))
+}
+
+// The role name and the password ride the environment and are re-quoted by
+// psql, so neither reaches a shell word or an argv another process can read.
+func (e *postgresEngine) SetPassword(ctx context.Context, username, password string) error {
+	if username == "" || password == "" {
+		return fmt.Errorf("set-password requires both %s and %s",
+			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
+	}
+	// The statement below runs as the cluster superuser over the socket under
+	// peer auth, so a reserved name in the command payload would hand the
+	// customer the bootstrap superuser rather than rotate their own role.
+	if err := e.meta.ValidateUsernameNotReserved(username); err != nil {
+		return fmt.Errorf("refusing to set the password of a role the engine reserves: %w", err)
+	}
+	// psql interpolates the password into the ALTER ROLE before the server sees
+	// it, and these three are what would write it to the log — the last on any
+	// failure at its own default. SUSET, so the parameter group cannot win.
+	const sql = `SET log_statement = 'none';
+SET log_min_duration_statement = -1;
+SET log_min_error_statement = 'panic';
+\getenv master RDS_MASTER_USERNAME
+\getenv password RDS_MASTER_PASSWORD
+ALTER ROLE :"master" WITH LOGIN PASSWORD :'password';
+`
+	_, err := e.psqlRun(ctx, sql,
+		"RDS_MASTER_USERNAME="+username,
+		"RDS_MASTER_PASSWORD="+password,
+	)
+	if err != nil {
+		// psql echoes the failing statement, which here would carry the new
+		// password back to the control plane and into the event ring.
+		return fmt.Errorf("apply the master password: %s", redact(err.Error(), password))
+	}
+	return nil
+}
+
+// Installs the resolved set, validates it with the engine's own config parser
+// before the engine ever adopts it, and reloads. A value the engine refuses is
+// rolled back here rather than left on the data volume, where it would survive
+// every VM replace and turn the next restart into a boot loop.
+func (e *postgresEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+
+	initialState, _ := e.probe.state(ctx)
+	if initialState == engineRecovering {
+		return nil, errors.New("apply parameters while the engine is still starting or recovering")
+	}
+	if initialState == engineServing {
+		// A command can arrive before the first heartbeat. Seed the configuration
+		// currently being served before replacing its file in that window.
+		if err := e.recordServingParametersLocked(ctx); err != nil {
+			return nil, fmt.Errorf("record the parameters serving before the apply: %w", err)
+		}
+	}
+
+	// The check has to run against the file in place, because the engine parses
+	// the datadir's own include_dir. The window that leaves is closed from both
+	// ends: the rollback below, and the last-known-good restore the agent runs at
+	// boot when the engine does not come up.
+	restore, err := e.params.install(params)
+	if err != nil {
+		return nil, err
+	}
+	if err := e.checkConfig(ctx); err != nil {
+		return nil, errors.Join(fmt.Errorf("the engine rejected the parameter set: %w", err), restore())
+	}
+
+	state, _ := e.probe.state(ctx)
+	switch state {
+	case engineAbsent:
+		return e.restartOnRepairSetLocked(ctx)
+	case engineRecovering:
+		return nil, errors.Join(errors.New("the engine entered startup or recovery during the parameter apply"), restore())
+	}
+
+	if _, err := e.psqlRun(ctx, "SELECT pg_reload_conf();\n"); err != nil {
+		// The engine may have gone down after the first probe. In that case start
+		// it on the new, parser-checked repair set rather than restoring the set
+		// that already left it unable to serve.
+		if state, _ := e.probe.state(ctx); state == engineAbsent {
+			return e.restartOnRepairSetLocked(ctx)
+		}
+		return nil, errors.Join(fmt.Errorf("reload the engine configuration: %w", err), restore())
+	}
+	return e.pendingRestartParameters(ctx)
+}
+
+func (e *postgresEngine) restartOnRepairSetLocked(ctx context.Context) ([]string, error) {
+	return awaitRepairedEngine(ctx, e.probe, e.Restart, e.pendingRestartParameters, e.repairTimeout, e.repairPoll)
+}
+
+// The postmaster's own answer: a static setting it has stored but not adopted is
+// what it reports here, so nothing in the guest has to classify one.
+func (e *postgresEngine) pendingRestartParameters(ctx context.Context) ([]string, error) {
+	out, err := e.psqlRun(ctx, "SELECT name FROM pg_settings WHERE pending_restart ORDER BY name;\n")
+	if err != nil {
+		return nil, fmt.Errorf("read the settings pending a restart: %w", err)
+	}
+	var pending []string
+	for line := range strings.SplitSeq(out, "\n") {
+		if name := strings.TrimSpace(line); name != "" {
+			pending = append(pending, name)
+		}
+	}
+	return pending, nil
+}
+
+// Snapshots the include only when the running postmaster has no settings still
+// pending a restart. The parameter mutex keeps an apply from replacing the file
+// between that check and the copy.
+func (e *postgresEngine) RecordServingParameters(ctx context.Context) error {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return e.recordServingParametersLocked(ctx)
+}
+
+func (e *postgresEngine) recordServingParametersLocked(ctx context.Context) error {
+	return recordLastGood(ctx, e.params, e.pendingRestartParameters)
+}
+
+// Puts the last set the engine accepted back in place, for a restart that failed
+// after a parameter change.
+func (e *postgresEngine) RestoreLastKnownGoodParameters(ctx context.Context) (bool, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return restoreLastGood(ctx, e.params, e.probe)
+}
+
+// The engine's own parser, run offline against the datadir. Reading one setting
+// back is enough: postgres parses postgresql.conf and every include first, and
+// exits non-zero naming the file and line of an unknown parameter or a value
+// outside its range.
+func (e *postgresEngine) checkConfig(ctx context.Context) error {
+	if _, err := e.run(ctx, command{
+		Name: filepath.Join(filepath.Dir(e.psql), "postgres"),
+		Args: []string{"-D", e.pgData, "-C", "shared_buffers"},
+		Env:  []string{"PATH=" + defaultGuestPath},
+		User: e.osUser,
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (e *postgresEngine) Stop(ctx context.Context) error {
+	return serviceAction(ctx, e.run, e.rcService, e.service, "stop")
+}
+
+// Only the parameter rollback calls this: a restart the control plane wants goes
+// through RebootDBInstance, which cycles the VM.
+func (e *postgresEngine) Restart(ctx context.Context) error {
+	return serviceAction(ctx, e.run, e.rcService, e.service, "restart")
+}
+
+// Feeds sql on stdin rather than as an argument, so a statement is never
+// visible in the process table.
+func (e *postgresEngine) psqlRun(ctx context.Context, sql string, env ...string) (string, error) {
+	return e.run(ctx, command{
+		Name:  e.psql,
+		Args:  e.psqlArgs(),
+		Env:   append([]string{"PATH=" + defaultGuestPath}, env...),
+		Stdin: sql,
+		User:  e.osUser,
+	})
+}
+
+// Reading the script from stdin is what lets one invocation serve both a
+// one-shot run and a session held open across several statements.
+func (e *postgresEngine) psqlArgs() []string {
+	return []string{
+		"--no-psqlrc", "--quiet", "--no-align", "--tuples-only",
+		"-v", "ON_ERROR_STOP=1",
+		"-h", e.socketDir,
+		"-p", strconv.FormatInt(e.port.Load(), 10),
+		"-U", e.osUser,
+		"-d", "postgres",
+		"-f", "-",
+	}
+}
+
+// Puts the engine into backup mode: the datadir is checkpointed and the engine
+// stops writing over the pages a snapshot is about to read. The hold is released
+// by Unquiesce, or by its own deadline, whichever comes first.
+func (e *postgresEngine) Quiesce(ctx context.Context, label string, hold time.Duration) error {
+	if err := validateQuiesceRequest(label, hold); err != nil {
+		return err
+	}
+
+	// Held across the whole start, so a second quiesce waits and then finds the
+	// first one's hold rather than opening a concurrent backup alongside it.
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.held != nil {
+		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)
+	}
+
+	// Deliberately not the command's context: the session has to outlive the call
+	// that started it, and its own deadline is what bounds it instead.
+	session, err := e.startSess(context.WithoutCancel(ctx), command{
+		Name:              e.psql,
+		Args:              e.psqlArgs(),
+		Env:               []string{"PATH=" + defaultGuestPath, "RDS_BACKUP_LABEL=" + label},
+		User:              e.osUser,
+		SentinelStatement: `\echo ` + sessionSentinel + "\n",
+	})
+	if err != nil {
+		return fmt.Errorf("open a backup session: %w", err)
+	}
+
+	// fast forces an immediate checkpoint rather than spreading it over the
+	// checkpoint interval, which would hold the snapshot open for minutes.
+	const sql = `\getenv label RDS_BACKUP_LABEL
+SELECT pg_backup_start(:'label', fast => true);
+`
+	if err := session.Exec(ctx, sql); err != nil {
+		if closeErr := session.Close(); closeErr != nil {
+			slog.Warn("rds-agent: closing a failed backup session", "err", closeErr)
+		}
+		return fmt.Errorf("put the engine into backup mode: %w", err)
+	}
+
+	e.beginHoldLocked(label, session, hold)
+	return nil
+}
+
+// Ends the backup cleanly. A missing hold is an error rather than a silent
+// success: it means the deadline fired first, so the snapshot the control plane
+// just took was not taken against a held checkpoint.
+func (e *postgresEngine) Unquiesce(ctx context.Context) error {
+	held := e.takeHold()
+	if held == nil {
+		return errors.New("the engine is not quiesced; the backup hold had already expired")
+	}
+
+	// The stop has to run on the session that started the backup, and the session
+	// is closed either way — the engine aborts an unstopped backup with it.
+	execErr := held.session.Exec(ctx, "SELECT pg_backup_stop();\n")
+	closeErr := held.session.Close()
+	if execErr != nil {
+		return fmt.Errorf("take the engine out of backup mode: %w", errors.Join(execErr, closeErr))
+	}
+	if closeErr != nil {
+		return fmt.Errorf("close the backup session: %w", closeErr)
+	}
+	slog.Info("rds-agent: engine released from backup mode", "label", held.label)
+	return nil
+}

cmd/rds-agent/engine_test.go

@@ -141,9 +141,9 @@ func (r *recordingRunner) run(_ context.Context, c command) (string, error) {
 
 func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
 	t.Helper()
-	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
-	cfg.PGData = t.TempDir()
-	if err := os.MkdirAll(filepath.Join(cfg.PGData, "conf.d"), 0o700); err != nil {
+	cfg := testLoadConfig(t, enginePostgres)
+	cfg.EngineDataDir = t.TempDir()
+	if err := os.MkdirAll(filepath.Join(cfg.EngineDataDir, "conf.d"), 0o700); err != nil {
 		t.Fatalf("create conf.d: %v", err)
 	}
 	// The guest's postgres user does not exist here, and the chown of the
@@ -152,8 +152,8 @@ func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
 	if err != nil {
 		t.Fatalf("resolve the current user: %v", err)
 	}
-	cfg.PGUser = current.Username
-	return newPostgresEngine(cfg, run, nil)
+	cfg.EngineUser = current.Username
+	return newPostgresEngine(cfg, run, nil, newPostgresProbe(cfg, staticProbe(0)))
 }
 
 // The password reaches psql through the environment and is re-quoted there,
@@ -185,6 +185,40 @@ func TestPostgresEngine_SetPasswordKeepsTheSecretOutOfArgv(t *testing.T) {
 	}
 }
 
+// psql interpolates the password into the ALTER ROLE before the server sees it,
+// so a parameter group that turns statement logging on would write the rotated
+// password into a postmaster.log every snapshot then carries.
+func TestPostgresEngine_SetPasswordSilencesStatementLogging(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", "n3w-pw"); err != nil {
+		t.Fatalf("SetPassword: %v", err)
+	}
+	if len(runner.calls) != 1 {
+		t.Fatalf("ran %d commands, want 1", len(runner.calls))
+	}
+	sql := runner.calls[0].Stdin
+	alter := strings.Index(sql, "ALTER ROLE")
+	if alter < 0 {
+		t.Fatalf("SQL %q does not alter the role", sql)
+	}
+	for _, guard := range []string{
+		"SET log_statement = 'none';",
+		"SET log_min_duration_statement = -1;",
+		"SET log_min_error_statement = 'panic';",
+	} {
+		at := strings.Index(sql, guard)
+		if at < 0 {
+			t.Errorf("SQL %q is missing %q", sql, guard)
+			continue
+		}
+		if at > alter {
+			t.Errorf("SQL %q sets %q only after the ALTER ROLE has already been logged", sql, guard)
+		}
+	}
+}
+
 // psql echoes the failing statement, which here would carry the new password
 // back into a reply the control plane records.
 func TestPostgresEngine_SetPasswordRedactsTheSecretFromAFailure(t *testing.T) {
@@ -276,7 +310,7 @@ func TestPostgresEngine_ApplyParametersInstallsAndReloads(t *testing.T) {
 // A reload that failed leaves the engine on its old values, so reporting the
 // apply as successful would tell the customer a change took effect that did not.
 func TestPostgresEngine_ApplyParametersSurfacesAFailedReload(t *testing.T) {
-	runner := &recordingRunner{err: errors.New("could not connect to server")}
+	runner := &reloadFailingRunner{reloadErr: errors.New("reload was rejected")}
 	engine := newTestEngine(t, runner.run)
 
 	if _, err := engine.ApplyParameters(context.Background(),
@@ -285,23 +319,84 @@ func TestPostgresEngine_ApplyParametersSurfacesAFailedReload(t *testing.T) {
 	}
 }
 
-// Through the service manager, so the supervisor records the engine as stopped
-// and does not restart it underneath a VM that is going down.
-func TestPostgresEngine_StopGoesThroughTheServiceManager(t *testing.T) {
+type reloadFailingRunner struct {
+	recordingRunner
+
+	reloadErr error
+}
+
+func (r *reloadFailingRunner) run(ctx context.Context, c command) (string, error) {
+	if strings.Contains(c.Stdin, "pg_reload_conf") {
+		r.calls = append(r.calls, c)
+		return "", r.reloadErr
+	}
+	return r.recordingRunner.run(ctx, c)
+}
+
+// A failed instance cannot reload through psql. Its corrected, parser-checked
+// include must start the service and remain installed as the serving set.
+func TestPostgresEngine_ApplyParametersRestartsOnARepairSetWhileEngineIsDown(t *testing.T) {
 	runner := &recordingRunner{}
 	engine := newTestEngine(t, runner.run)
+	codes := []int{2, 2, 0}
+	probeCall := 0
+	cfg := testProbeConfig()
+	engine.probe = newPostgresProbe(cfg, func(context.Context, string, ...string) (int, error) {
+		code := codes[min(probeCall, len(codes)-1)]
+		probeCall++
+		return code, nil
+	})
+	engine.repairTimeout = 100 * time.Millisecond
+	engine.repairPoll = time.Millisecond
+	params := []handlers_rds.Parameter{
+		{Name: "shared_buffers", Value: "32768"},
+		{Name: "work_mem", Value: "8192"},
+	}
 
-	if err := engine.Stop(context.Background()); err != nil {
-		t.Fatalf("Stop: %v", err)
+	pending, err := engine.ApplyParameters(t.Context(), params)
+	if err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
 	}
-	if len(runner.calls) != 1 {
-		t.Fatalf("ran %d commands, want 1", len(runner.calls))
+	if len(pending) != 0 {
+		t.Errorf("pending = %v, want the restarted engine's actual pending set", pending)
 	}
-	call := runner.calls[0]
-	if filepath.Base(call.Name) != "rc-service" {
-		t.Errorf("ran %q, want rc-service", call.Name)
+	if len(runner.calls) != 3 {
+		t.Fatalf("calls = %+v, want config check, service restart and pending read", runner.calls)
 	}
-	if len(call.Args) != 2 || call.Args[0] != "postgresql" || call.Args[1] != "stop" {
-		t.Errorf("args = %v, want [postgresql stop]", call.Args)
+	if !slices.Contains(runner.calls[0].Args, "-C") {
+		t.Errorf("first call = %+v, want the offline config check", runner.calls[0])
+	}
+	if !slices.Equal(runner.calls[1].Args, []string{"postgresql", "restart"}) {
+		t.Errorf("second call args = %v, want the service restart", runner.calls[1].Args)
+	}
+	if !strings.Contains(runner.calls[2].Stdin, "pending_restart") {
+		t.Errorf("third call = %+v, want the pending-restart read", runner.calls[2])
+	}
+	installed, err := os.ReadFile(engine.params.installedPath())
+	if err != nil {
+		t.Fatalf("read installed parameters: %v", err)
+	}
+	if !strings.Contains(string(installed), "work_mem = '8192'") {
+		t.Errorf("installed parameters = %q, want the repair set retained", installed)
+	}
+}
+
+func TestPostgresEngine_ApplyParametersKeepsRepairSetWhenRestartTimesOut(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+	engine.probe = stubProbe(t, 2, nil)
+	engine.repairTimeout = 10 * time.Millisecond
+	engine.repairPoll = time.Millisecond
+
+	_, err := engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{{Name: "work_mem", Value: "8192"}})
+	if err == nil || !strings.Contains(err.Error(), "wait for the engine") {
+		t.Fatalf("ApplyParameters error = %v, want the repair wait failure", err)
+	}
+	installed, readErr := os.ReadFile(engine.params.installedPath())
+	if readErr != nil {
+		t.Fatalf("read installed parameters: %v", readErr)
+	}
+	if !strings.Contains(string(installed), "work_mem = '8192'") {
+		t.Errorf("installed parameters = %q, want the checked repair set retained", installed)
 	}
 }

cmd/rds-agent/health.go

@@ -3,9 +3,7 @@ package main
 import (
 	"context"
 	"errors"
-	"fmt"
 	"os/exec"
-	"strconv"
 	"sync/atomic"
 
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
@@ -20,6 +18,9 @@ func execProbeRunner(ctx context.Context, name string, args ...string) (int, err
 	if err == nil {
 		return 0, nil
 	}
+	if ctxErr := ctx.Err(); ctxErr != nil {
+		return -1, ctxErr
+	}
 	var exitErr *exec.ExitError
 	if errors.As(err, &exitErr) {
 		return exitErr.ExitCode(), nil
@@ -27,12 +28,16 @@ func execProbeRunner(ctx context.Context, name string, args ...string) (int, err
 	return -1, err
 }
 
-// Shells out to pg_isready rather than dialling the port, since only a startup
-// exchange separates an engine that is serving from one still in recovery.
+// What the engine reports about itself, which is the only part of the probe
+// that differs between engines. The port is passed in rather than read from the
+// engine, so the atomic below stays the one place it is followed from.
+type probeStateFn func(ctx context.Context, port int64) (engineState, string)
+
+// Engine-neutral: the latch, the port, the degraded mapping and the translation
+// to a reported health are shared, and only the state function is supplied by
+// the engine the image bakes.
 type engineProbe struct {
-	run    probeRunner
-	binary string
-	host   string
+	stateFn probeStateFn
 	// Set from the bootstrap config once it lands, from a different goroutine
 	// than the heartbeat that reads it.
 	port atomic.Int64
@@ -41,9 +46,9 @@ type engineProbe struct {
 	seenHealthy bool
 }
 
-func newEngineProbe(cfg config, run probeRunner) *engineProbe {
-	p := &engineProbe{run: run, binary: cfg.PGIsReady, host: cfg.EngineHost}
-	p.port.Store(int64(cfg.EnginePort))
+func newEngineProbe(port int, stateFn probeStateFn) *engineProbe {
+	p := &engineProbe{stateFn: stateFn}
+	p.port.Store(int64(port))
 	return p
 }
 
@@ -60,7 +65,7 @@ type engineState int
 const (
 	// Not answering at all, or the probe could not run.
 	engineAbsent engineState = iota
-	// The postmaster is up and rejecting connections: startup or crash recovery.
+	// The process is up but not yet serving: startup or crash recovery.
 	engineRecovering
 	engineServing
 )
@@ -83,21 +88,7 @@ func (p *engineProbe) Check(ctx context.Context) (handlers_rds.EngineHealth, str
 // The raw probe result. Safe to call from a goroutine other than the
 // heartbeat's: it touches no latched state.
 func (p *engineProbe) state(ctx context.Context) (engineState, string) {
-	port := strconv.FormatInt(p.port.Load(), 10)
-	code, err := p.run(ctx, p.binary, "-h", p.host, "-p", port, "-q")
-	switch {
-	case err != nil:
-		// A missing binary or broken image. Reporting healthy on the strength of
-		// nothing would hide it, so this reads as absent like an engine that did
-		// not answer.
-		return engineAbsent, fmt.Sprintf("engine probe could not run: %v", err)
-	case code == 0:
-		return engineServing, ""
-	case code == 1:
-		return engineRecovering, "engine is rejecting connections (startup or recovery)"
-	default:
-		return engineAbsent, fmt.Sprintf("engine did not respond on %s:%s", p.host, port)
-	}
+	return p.stateFn(ctx, p.port.Load())
 }
 
 // A non-answering engine: starting until it has answered once, unhealthy after.

cmd/rds-agent/health_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"strings"
 	"testing"
+	"time"
 
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
@@ -28,7 +29,7 @@ func TestEngineProbe_MapsExitCodesToHealth(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			probe := newEngineProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
+			probe := newPostgresProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
 				return tt.code, tt.runErr
 			})
 			probe.seenHealthy = tt.seenHealthy
@@ -51,7 +52,7 @@ func TestEngineProbe_MapsExitCodesToHealth(t *testing.T) {
 // has to flip the first time the engine answers.
 func TestEngineProbe_LatchesAfterFirstHealthy(t *testing.T) {
 	code := 0
-	probe := newEngineProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
+	probe := newPostgresProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
 		return code, nil
 	})
 
@@ -68,7 +69,7 @@ func TestEngineProbe_LatchesAfterFirstHealthy(t *testing.T) {
 // must follow it rather than keep asking the default.
 func TestEngineProbe_ProbesTheAssignedPort(t *testing.T) {
 	var gotArgs []string
-	probe := newEngineProbe(testProbeConfig(), func(_ context.Context, _ string, args ...string) (int, error) {
+	probe := newPostgresProbe(testProbeConfig(), func(_ context.Context, _ string, args ...string) (int, error) {
 		gotArgs = args
 		return 0, nil
 	})
@@ -81,6 +82,19 @@ func TestEngineProbe_ProbesTheAssignedPort(t *testing.T) {
 	}
 }
 
-func testProbeConfig() config {
-	return config{EngineHost: defaultEngineHost, EnginePort: defaultEnginePort, PGIsReady: defaultPGIsReady}
+func TestExecProbeRunner_PreservesTheProbeDeadline(t *testing.T) {
+	ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
+	defer cancel()
+
+	code, err := execProbeRunner(ctx, "sh", "-c", "while :; do :; done")
+	if code != -1 {
+		t.Errorf("exit code = %d, want -1 for a timed-out probe", code)
+	}
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Errorf("error = %v, want the context deadline", err)
+	}
+}
+
+func testProbeConfig() config {
+	return config{EngineHost: defaultEngineHost, EnginePort: engineLayouts[enginePostgres].port}
 }

cmd/rds-agent/heartbeat.go

@@ -3,25 +3,37 @@ package main
 import (
 	"context"
 	"log/slog"
+	"sync/atomic"
 	"time"
+	"unicode/utf8"
 
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
 
+const (
+	servingParameterRecordTimeout = 15 * time.Second
+	maxBootstrapFailureBytes      = 512
+)
+
 // The beat carries the probe's result, not the agent's own liveness: an agent
 // up while the engine is down is what the recovery reconciler must see.
 type heartbeater struct {
 	cp       controlPlane
 	probe    *engineProbe
+	recorder servingParameterRecorder
 	id       identity
 	interval time.Duration
+
+	// Bootstrap and heartbeat run concurrently. Keeping the latest failure here
+	// makes a VM stuck before engine startup explain why through its next beat.
+	bootstrapFailure atomic.Pointer[string]
 }
 
-func newHeartbeater(cp controlPlane, probe *engineProbe, interval time.Duration) *heartbeater {
+func newHeartbeater(cp controlPlane, probe *engineProbe, recorder servingParameterRecorder, interval time.Duration) *heartbeater {
 	if interval <= 0 {
 		interval = handlers_rds.HeartbeatInterval
 	}
-	return &heartbeater{cp: cp, probe: probe, interval: interval}
+	return &heartbeater{cp: cp, probe: probe, recorder: recorder, interval: interval}
 }
 
 // Called before Run starts and from Run's own goroutine after, so the interval
@@ -32,8 +44,40 @@ func (h *heartbeater) setInterval(d time.Duration) {
 	}
 }
 
+func (h *heartbeater) setBootstrapFailure(what string, err error) {
+	message := what + " failed: " + err.Error()
+	if len(message) > maxBootstrapFailureBytes {
+		limit := maxBootstrapFailureBytes - len("...")
+		for limit > 0 && !utf8.RuneStart(message[limit]) {
+			limit--
+		}
+		message = message[:limit] + "..."
+	}
+	h.bootstrapFailure.Store(&message)
+}
+
+func (h *heartbeater) clearBootstrapFailure() {
+	h.bootstrapFailure.Store(nil)
+}
+
 func (h *heartbeater) beat(ctx context.Context) {
 	health, message := h.probe.Check(ctx)
+	if failure := h.bootstrapFailure.Load(); failure != nil {
+		if message != "" {
+			message += "; "
+		}
+		message += *failure
+	}
+	if health == handlers_rds.EngineHealthHealthy && h.recorder != nil {
+		// Checking every healthy beat also observes a restart that completed
+		// between probes. The recorder skips unchanged and pending-restart sets.
+		recordCtx, cancel := context.WithTimeout(ctx, servingParameterRecordTimeout)
+		err := h.recorder.RecordServingParameters(recordCtx)
+		cancel()
+		if err != nil {
+			slog.Warn("rds-agent: recording the serving parameters failed", "err", err)
+		}
+	}
 
 	out, err := h.cp.SubmitState(ctx, h.id, health, message)
 	if err != nil {

cmd/rds-agent/heartbeat_test.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+	"strings"
+	"testing"
+	"unicode/utf8"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The first healthy probe is the first proof that the installed include can
+// start PostgreSQL, so it must seed recovery without an apply command.
+func TestHeartbeater_FirstServingProbeSeedsLastKnownGood(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+	content := []byte("work_mem = '4096'\n")
+	if err := os.WriteFile(engine.params.installedPath(), content, 0o600); err != nil {
+		t.Fatalf("write installed parameters: %v", err)
+	}
+
+	cp := newFakeControlPlane()
+	h := newHeartbeater(cp, engine.probe, engine, 0)
+	h.beat(context.Background())
+
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
+	if err != nil {
+		t.Fatalf("read last known good: %v", err)
+	}
+	if string(lastGood) != string(content) {
+		t.Errorf("last known good = %q, want %q", lastGood, content)
+	}
+	states := cp.snapshotStates()
+	if len(states) != 1 || states[0].health != handlers_rds.EngineHealthHealthy {
+		t.Errorf("states = %+v, want one healthy heartbeat", states)
+	}
+}
+
+func TestHeartbeater_BoundsBootstrapFailure(t *testing.T) {
+	h := newHeartbeater(newFakeControlPlane(), newPostgresProbe(testProbeConfig(), staticProbe(2)), nil, 0)
+	h.setBootstrapFailure("bootstrap fetch", errors.New(strings.Repeat("界", 1000)))
+
+	failure := h.bootstrapFailure.Load()
+	if failure == nil {
+		t.Fatal("bootstrap failure was not stored")
+	}
+	if len(*failure) > maxBootstrapFailureBytes {
+		t.Errorf("bootstrap failure is %d bytes, want at most %d", len(*failure), maxBootstrapFailureBytes)
+	}
+	if !utf8.ValidString(*failure) {
+		t.Error("bootstrap failure was truncated inside a UTF-8 encoding")
+	}
+	if !strings.HasSuffix(*failure, "...") {
+		t.Errorf("bootstrap failure %q does not show that it was truncated", *failure)
+	}
+}
+
+func TestHeartbeater_ChecksServingParametersOnEveryHealthyProbe(t *testing.T) {
+	code := 0
+	cfg := testProbeConfig()
+	probe := newPostgresProbe(cfg, func(context.Context, string, ...string) (int, error) {
+		return code, nil
+	})
+	recorder := &countingServingRecorder{}
+	h := newHeartbeater(newFakeControlPlane(), probe, recorder, 0)
+
+	h.beat(context.Background())
+	h.beat(context.Background())
+	code = 2
+	h.beat(context.Background())
+	code = 0
+	h.beat(context.Background())
+
+	if recorder.calls != 3 {
+		t.Errorf("record calls = %d, want one per healthy probe", recorder.calls)
+	}
+}
+
+type countingServingRecorder struct {
+	calls int
+}
+
+var _ servingParameterRecorder = (*countingServingRecorder)(nil)
+
+func (r *countingServingRecorder) RecordServingParameters(context.Context) error {
+	r.calls++
+	return nil
+}

cmd/rds-agent/parameters.go

@@ -0,0 +1,223 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The generated parameter files and what it takes to install one. They live on
+// the data volume rather than under /etc, at the same paths rds-init writes them
+// to: a class change is a VM replacement, so a set under /etc would be discarded
+// with the boot volume and the instance would silently revert to catalog
+// defaults while the API still reported the customer's values.
+type parameterStore struct {
+	dir string
+	// The include the resolved set is rendered to.
+	installed string
+	// The copy of the last set the engine accepted, which a boot loop is rolled
+	// back to. Deliberately not a name the engine's own include glob reads, so it
+	// is never parsed as a second set of settings.
+	lastGood string
+	// What the engine actually started on, for an engine that cannot be asked
+	// which of its settings are still pending a restart. Empty for one that can.
+	serving string
+	// Prepended to every file written here, for a configuration format that has
+	// no setting outside a group.
+	header string
+	// The engine parses its own configuration, so a root-owned file would be
+	// unreadable to it.
+	osUser string
+	// Names the engine whose option-file spellings these files are written in.
+	engine string
+}
+
+func (s parameterStore) installedPath() string { return filepath.Join(s.dir, s.installed) }
+func (s parameterStore) lastGoodPath() string  { return filepath.Join(s.dir, s.lastGood) }
+func (s parameterStore) servingPath() string   { return filepath.Join(s.dir, s.serving) }
+
+// Byte-for-byte what rds-init writes for the same set, so the copies compared
+// against it are comparing values rather than formatting.
+func (s parameterStore) render(params []handlers_rds.Parameter) ([]byte, error) {
+	body, err := renderParameters(s.engine, params)
+	if err != nil {
+		return nil, err
+	}
+	return []byte(s.header + body), nil
+}
+
+// Installs the resolved set at the path rds-init uses, so a later boot
+// overwrites rather than shadows it. Returns a func putting back whatever was
+// there, so the caller can undo the install without re-rendering anything.
+func (s parameterStore) install(params []handlers_rds.Parameter) (restore func() error, err error) {
+	path := s.installedPath()
+	// A missing file is the first-ever apply, and its rollback is a removal.
+	previous, readErr := os.ReadFile(path)
+	if readErr != nil && !os.IsNotExist(readErr) {
+		return nil, fmt.Errorf("read the installed parameters at %s: %w", path, readErr)
+	}
+	existed := readErr == nil
+
+	rendered, err := s.render(params)
+	if err != nil {
+		return nil, err
+	}
+	if err := s.write(path, rendered); err != nil {
+		return nil, err
+	}
+	return func() error {
+		if !existed {
+			if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+				return fmt.Errorf("withdraw the rejected parameters at %s: %w", path, err)
+			}
+			return nil
+		}
+		return s.write(path, previous)
+	}, nil
+}
+
+// Atomic: a temp file in the same directory, renamed over the target. The temp
+// name deliberately does not end in the engine's include suffix, so a crash
+// between write and rename cannot leave the engine reading a half-written file.
+func (s parameterStore) write(path string, content []byte) error {
+	tmp := path + ".new"
+	if err := os.WriteFile(tmp, content, 0o600); err != nil {
+		return fmt.Errorf("write %s: %w", tmp, err)
+	}
+	if err := s.chownToEngine(tmp); err != nil {
+		os.Remove(tmp)
+		return err
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("install %s: %w", path, err)
+	}
+	return nil
+}
+
+// A guest with no engine user is a broken image, not something to work around.
+func (s parameterStore) chownToEngine(path string) error {
+	credential, err := lookupCredential(s.osUser)
+	if err != nil {
+		return err
+	}
+	if err := os.Chown(path, int(credential.Uid), int(credential.Gid)); err != nil {
+		return fmt.Errorf("hand %s to %s: %w", path, s.osUser, err)
+	}
+	return nil
+}
+
+// Snapshots the installed set as the rollback target, but only once the engine
+// is running all of it. A set with a setting still pending a restart has not
+// been shown to serve, and promoting it would leave the rollback pointing at the
+// configuration that is about to fail.
+func recordLastGood(ctx context.Context, store parameterStore, pending pendingRestartFn) error {
+	if _, err := os.Stat(store.installedPath()); err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("inspect the serving parameters: %w", err)
+	}
+	names, err := pending(ctx)
+	if err != nil {
+		return err
+	}
+	if len(names) > 0 {
+		return nil
+	}
+
+	content, err := os.ReadFile(store.installedPath())
+	if err != nil {
+		return fmt.Errorf("read the serving parameters: %w", err)
+	}
+	lastGood, err := os.ReadFile(store.lastGoodPath())
+	if err == nil && bytes.Equal(content, lastGood) {
+		return nil
+	}
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("read the last known good parameters: %w", err)
+	}
+	return store.write(store.lastGoodPath(), content)
+}
+
+// Puts the last set the engine accepted back in place, for a restart that failed
+// after a parameter change. The probe is checked again under the caller's
+// parameter lock, so a repair that just brought the engine back is not reversed.
+func restoreLastGood(ctx context.Context, store parameterStore, probe *engineProbe) (bool, error) {
+	if state, _ := probe.state(ctx); state != engineAbsent {
+		return false, nil
+	}
+	lastGood, err := os.ReadFile(store.lastGoodPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, fmt.Errorf("read the last known good parameters: %w", err)
+	}
+	current, err := os.ReadFile(store.installedPath())
+	if err != nil && !os.IsNotExist(err) {
+		return false, fmt.Errorf("read the installed parameters: %w", err)
+	}
+	if bytes.Equal(current, lastGood) {
+		return false, nil
+	}
+	if err := store.write(store.installedPath(), lastGood); err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+// The settings the engine has accepted but will not honour until it restarts.
+// Each implementation answers it its own way, and both the apply and the serving
+// snapshot are gated on the same answer.
+type pendingRestartFn func(ctx context.Context) ([]string, error)
+
+const (
+	parameterRepairTimeout = 90 * time.Second
+	parameterRepairPoll    = time.Second
+)
+
+// Starts a down engine on the set that was just installed and waits for it to
+// serve. Reached when the engine went away during an apply: the installed set is
+// the newer of the two, so starting on it beats restoring one that already left
+// the engine unable to serve.
+func awaitRepairedEngine(ctx context.Context, probe *engineProbe, restart func(context.Context) error,
+	pending pendingRestartFn, timeout, poll time.Duration) ([]string, error) {
+	repairCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	if err := restart(repairCtx); err != nil {
+		return nil, fmt.Errorf("start the engine on the repaired parameter set: %w", err)
+	}
+
+	timer := time.NewTimer(0)
+	defer timer.Stop()
+	lastMessage := "engine did not respond"
+	for {
+		select {
+		case <-repairCtx.Done():
+			if ctx.Err() != nil {
+				return nil, fmt.Errorf("wait for the engine on the repaired parameter set: %w", ctx.Err())
+			}
+			return nil, fmt.Errorf("wait for the engine on the repaired parameter set: %s: %w", lastMessage, repairCtx.Err())
+		case <-timer.C:
+		}
+
+		state, message := probe.state(repairCtx)
+		if state == engineServing {
+			names, err := pending(repairCtx)
+			if err == nil {
+				return names, nil
+			}
+			lastMessage = err.Error()
+		} else if message != "" {
+			lastMessage = message
+		}
+		timer.Reset(poll)
+	}
+}

cmd/rds-agent/paramrecovery.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"log/slog"
 	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
 
 // How long the engine is given to come up after a boot before the installed
@@ -29,12 +31,14 @@ const parameterRollbackPoll = 10 * time.Second
 type paramGuard struct {
 	engine parameterRecovery
 	probe  *engineProbe
+	cp     controlPlane
+	id     identity
 	after  time.Duration
 	poll   time.Duration
 }
 
-func newParamGuard(engine parameterRecovery, probe *engineProbe) *paramGuard {
-	return &paramGuard{engine: engine, probe: probe, after: parameterRollbackAfter, poll: parameterRollbackPoll}
+func newParamGuard(engine parameterRecovery, probe *engineProbe, cp controlPlane) *paramGuard {
+	return &paramGuard{engine: engine, probe: probe, cp: cp, after: parameterRollbackAfter, poll: parameterRollbackPoll}
 }
 
 // Runs once per agent lifetime: a rollback that did not bring the engine back
@@ -45,20 +49,23 @@ func (g *paramGuard) Run(ctx context.Context) {
 		return
 	}
 
-	restored, err := g.engine.RestoreLastKnownGoodParameters()
+	restored, err := g.engine.RestoreLastKnownGoodParameters(ctx)
 	if err != nil {
 		slog.Error("rds-agent: could not restore the last known good parameters", "err", err)
 		return
 	}
 	if !restored {
-		// The engine is already running the set it last accepted, so whatever is
-		// keeping it down is not the parameters.
-		slog.Warn("rds-agent: engine is down but its parameters are the last accepted set; not rolling back")
+		slog.Warn("rds-agent: engine is down but no different last accepted parameter set is available; not rolling back")
 		return
 	}
 
-	slog.Error("rds-agent: engine did not start after a parameter change; rolled back to the last accepted set",
-		"after", g.after)
+	message := handlers_rds.ParameterRollbackMessage
+	slog.Error("rds-agent: "+message, "after", g.after)
+	if g.cp != nil {
+		if _, err := g.cp.SubmitState(ctx, g.id, handlers_rds.EngineHealthUnhealthy, message); err != nil {
+			slog.Error("rds-agent: reporting the parameter rollback failed", "err", err)
+		}
+	}
 	if err := g.engine.Restart(ctx); err != nil {
 		slog.Error("rds-agent: restart after the parameter rollback failed", "err", err)
 	}

cmd/rds-agent/paramrecovery_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"os"
-	"path/filepath"
 	"slices"
 	"strings"
 	"testing"
@@ -50,7 +49,7 @@ func TestPostgresEngine_ApplyParametersRollsBackAValueTheEngineRejects(t *testin
 		t.Fatal("ApplyParameters succeeded against a config the engine rejected")
 	}
 
-	installed, err := os.ReadFile(engine.parametersPath())
+	installed, err := os.ReadFile(engine.params.installedPath())
 	if err != nil {
 		t.Fatalf("read the installed parameters: %v", err)
 	}
@@ -69,36 +68,221 @@ func TestPostgresEngine_ApplyParametersWithdrawsTheFirstRejectedSet(t *testing.T
 		[]handlers_rds.Parameter{{Name: "work_mem", Value: "0"}}); err == nil {
 		t.Fatal("ApplyParameters succeeded against a config the engine rejected")
 	}
-	if _, err := os.Stat(engine.parametersPath()); !os.IsNotExist(err) {
+	if _, err := os.Stat(engine.params.installedPath()); !os.IsNotExist(err) {
 		t.Errorf("the rejected include is still installed (stat err = %v)", err)
 	}
 }
 
-// The rollback target has to be a set the engine actually adopted, so a second
-// accepted apply leaves the first one as what a failed restart falls back to.
-func TestPostgresEngine_LastKnownGoodTracksTheAcceptedSet(t *testing.T) {
+type pendingAfterReloadRunner struct {
+	recordingRunner
+
+	pendingReads int
+}
+
+func (r *pendingAfterReloadRunner) run(ctx context.Context, c command) (string, error) {
+	if strings.Contains(c.Stdin, "pending_restart") {
+		r.pendingReads++
+		if r.pendingReads > 1 {
+			r.calls = append(r.calls, c)
+			return "shared_buffers\n", nil
+		}
+	}
+	return r.recordingRunner.run(ctx, c)
+}
+
+// A command can beat the first heartbeat. The apply must preserve the file the
+// current postmaster started with before installing a static replacement.
+func TestPostgresEngine_ApplyBeforeFirstHeartbeatSeedsLastKnownGood(t *testing.T) {
+	runner := &pendingAfterReloadRunner{}
+	engine := newTestEngine(t, runner.run)
+	first := []byte("shared_buffers = '32768'\n")
+	if err := os.WriteFile(engine.params.installedPath(), first, 0o600); err != nil {
+		t.Fatalf("write serving parameter set: %v", err)
+	}
+
+	pending, err := engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{{Name: "shared_buffers", Value: "65536"}})
+	if err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if !slices.Equal(pending, []string{"shared_buffers"}) {
+		t.Errorf("pending = %v, want shared_buffers", pending)
+	}
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
+	if err != nil {
+		t.Fatalf("read last known good: %v", err)
+	}
+	if !slices.Equal(lastGood, first) {
+		t.Errorf("last known good = %q, want the pre-apply set %q", lastGood, first)
+	}
+}
+
+type blockingParameterRunner struct {
+	recordingRunner
+
+	checkStarted  chan struct{}
+	continueCheck chan struct{}
+	pendingReads  int
+}
+
+func (r *blockingParameterRunner) run(ctx context.Context, c command) (string, error) {
+	if slices.Contains(c.Args, "-C") {
+		r.calls = append(r.calls, c)
+		close(r.checkStarted)
+		select {
+		case <-ctx.Done():
+			return "", ctx.Err()
+		case <-r.continueCheck:
+			return "", nil
+		}
+	}
+	if strings.Contains(c.Stdin, "pending_restart") {
+		r.pendingReads++
+		if r.pendingReads > 1 {
+			r.calls = append(r.calls, c)
+			return "shared_buffers\n", nil
+		}
+	}
+	return r.recordingRunner.run(ctx, c)
+}
+
+func TestPostgresEngine_ServingSnapshotWaitsForParameterApply(t *testing.T) {
+	runner := &blockingParameterRunner{
+		checkStarted:  make(chan struct{}),
+		continueCheck: make(chan struct{}),
+	}
+	engine := newTestEngine(t, runner.run)
+	first := []byte("shared_buffers = '32768'\n")
+	if err := os.WriteFile(engine.params.installedPath(), first, 0o600); err != nil {
+		t.Fatalf("write serving parameter set: %v", err)
+	}
+	if err := os.WriteFile(engine.params.lastGoodPath(), first, 0o600); err != nil {
+		t.Fatalf("write last known good: %v", err)
+	}
+
+	applyDone := make(chan error, 1)
+	go func() {
+		_, err := engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{{Name: "shared_buffers", Value: "65536"}})
+		applyDone <- err
+	}()
+	<-runner.checkStarted
+
+	recordDone := make(chan error, 1)
+	go func() {
+		recordDone <- engine.RecordServingParameters(t.Context())
+	}()
+	var earlyRecordErr error
+	recordedEarly := false
+	select {
+	case earlyRecordErr = <-recordDone:
+		recordedEarly = true
+	case <-time.After(10 * time.Millisecond):
+	}
+	close(runner.continueCheck)
+	if err := <-applyDone; err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if !recordedEarly {
+		earlyRecordErr = <-recordDone
+	}
+	if earlyRecordErr != nil {
+		t.Fatalf("RecordServingParameters: %v", earlyRecordErr)
+	}
+	if recordedEarly {
+		t.Error("serving snapshot completed while the parameter apply held its transaction")
+	}
+
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
+	if err != nil {
+		t.Fatalf("read last known good: %v", err)
+	}
+	if !slices.Equal(lastGood, first) {
+		t.Errorf("last known good = %q, want the set served before the apply %q", lastGood, first)
+	}
+}
+
+// The rollback target moves only when a restart proves the installed static
+// values can serve, not when ApplyParameters merely reloads them.
+func TestPostgresEngine_LastKnownGoodTracksTheServingSet(t *testing.T) {
 	runner := &recordingRunner{}
 	engine := newTestEngine(t, runner.run)
 
-	for _, value := range []string{"4096", "8192"} {
+	apply := func(value string) {
+		t.Helper()
 		if _, err := engine.ApplyParameters(context.Background(),
 			[]handlers_rds.Parameter{{Name: "work_mem", Value: value}}); err != nil {
 			t.Fatalf("ApplyParameters(%s): %v", value, err)
 		}
 	}
+	assertLastGood := func(value string) {
+		t.Helper()
+		lastGood, err := os.ReadFile(engine.params.lastGoodPath())
+		if err != nil {
+			t.Fatalf("read the last known good parameters: %v", err)
+		}
+		if !strings.Contains(string(lastGood), "work_mem = '"+value+"'") {
+			t.Errorf("last known good = %q, want work_mem %s", lastGood, value)
+		}
+	}
 
-	lastGood, err := os.ReadFile(engine.lastGoodPath())
+	apply("4096")
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("RecordServingParameters: %v", err)
+	}
+	apply("8192")
+	assertLastGood("4096")
+
+	// Simulate the restart that adopts 8192, then another apply before the next
+	// restart. The target remains the set the postmaster actually started with.
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("RecordServingParameters after restart: %v", err)
+	}
+	apply("16384")
+	assertLastGood("8192")
+
+	if strings.HasSuffix(engine.params.lastGoodPath(), ".conf") {
+		t.Errorf("the rollback copy at %s would be included by the engine", engine.params.lastGoodPath())
+	}
+}
+
+func TestPostgresEngine_RecordServingParametersSkipsPendingRestart(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+	first := []byte("shared_buffers = '32768'\n")
+	if err := os.WriteFile(engine.params.installedPath(), first, 0o600); err != nil {
+		t.Fatalf("write first parameter set: %v", err)
+	}
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("record first parameter set: %v", err)
+	}
+
+	second := []byte("shared_buffers = '65536'\n")
+	if err := os.WriteFile(engine.params.installedPath(), second, 0o600); err != nil {
+		t.Fatalf("write pending parameter set: %v", err)
+	}
+	runner.out = "shared_buffers\n"
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("record with a pending restart: %v", err)
+	}
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
 	if err != nil {
-		t.Fatalf("read the last known good parameters: %v", err)
+		t.Fatalf("read last known good: %v", err)
 	}
-	if !strings.Contains(string(lastGood), "work_mem = '4096'") {
-		t.Errorf("last known good = %q, want the set the engine ran before the latest apply", lastGood)
+	if !slices.Equal(lastGood, first) {
+		t.Errorf("last known good = %q, want the previously served set %q", lastGood, first)
 	}
 
-	// include_dir globs *.conf, so the rollback copy must not read as a second
-	// set of settings alongside the live one.
-	if strings.HasSuffix(engine.lastGoodPath(), ".conf") {
-		t.Errorf("the rollback copy at %s would be included by the engine", engine.lastGoodPath())
+	// Once a restart has adopted the installed set, pending_restart clears and
+	// the same healthy-heartbeat path may advance the rollback target.
+	runner.out = ""
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("record the restarted parameter set: %v", err)
+	}
+	lastGood, err = os.ReadFile(engine.params.lastGoodPath())
+	if err != nil {
+		t.Fatalf("read advanced last known good: %v", err)
+	}
+	if !slices.Equal(lastGood, second) {
+		t.Errorf("last known good = %q, want the restarted set %q", lastGood, second)
 	}
 }
 
@@ -107,7 +291,7 @@ func TestPostgresEngine_RestoreLastKnownGoodParameters(t *testing.T) {
 	engine := newTestEngine(t, runner.run)
 
 	// Nothing to restore before anything has ever been applied.
-	restored, err := engine.RestoreLastKnownGoodParameters()
+	restored, err := engine.RestoreLastKnownGoodParameters(t.Context())
 	if err != nil {
 		t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
 	}
@@ -115,21 +299,37 @@ func TestPostgresEngine_RestoreLastKnownGoodParameters(t *testing.T) {
 		t.Error("restored a set with no last known good on disk")
 	}
 
-	for _, value := range []string{"4096", "8192"} {
-		if _, err := engine.ApplyParameters(context.Background(),
-			[]handlers_rds.Parameter{{Name: "work_mem", Value: value}}); err != nil {
-			t.Fatalf("ApplyParameters(%s): %v", value, err)
-		}
+	if _, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "work_mem", Value: "4096"}}); err != nil {
+		t.Fatalf("ApplyParameters(4096): %v", err)
+	}
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("RecordServingParameters: %v", err)
+	}
+	if _, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "work_mem", Value: "8192"}}); err != nil {
+		t.Fatalf("ApplyParameters(8192): %v", err)
 	}
 
-	restored, err = engine.RestoreLastKnownGoodParameters()
+	// A guard whose deadline expired while an apply restarted the engine must
+	// recheck health under the parameter lock rather than reverse that repair.
+	restored, err = engine.RestoreLastKnownGoodParameters(t.Context())
+	if err != nil {
+		t.Fatalf("RestoreLastKnownGoodParameters against serving engine: %v", err)
+	}
+	if restored {
+		t.Fatal("restored the old set while the engine was serving")
+	}
+
+	engine.probe = stubProbe(t, 2, nil)
+	restored, err = engine.RestoreLastKnownGoodParameters(t.Context())
 	if err != nil {
 		t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
 	}
 	if !restored {
 		t.Fatal("did not restore a set that differs from the last known good")
 	}
-	installed, err := os.ReadFile(engine.parametersPath())
+	installed, err := os.ReadFile(engine.params.installedPath())
 	if err != nil {
 		t.Fatalf("read the installed parameters: %v", err)
 	}
@@ -139,7 +339,7 @@ func TestPostgresEngine_RestoreLastKnownGoodParameters(t *testing.T) {
 
 	// Idempotent: the second call finds them already equal and does nothing, so a
 	// repeated rollback cannot churn a cluster that is failing for another reason.
-	restored, err = engine.RestoreLastKnownGoodParameters()
+	restored, err = engine.RestoreLastKnownGoodParameters(t.Context())
 	if err != nil {
 		t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
 	}
@@ -156,7 +356,7 @@ type fakeRecovery struct {
 	restarts   int
 }
 
-func (f *fakeRecovery) RestoreLastKnownGoodParameters() (bool, error) {
+func (f *fakeRecovery) RestoreLastKnownGoodParameters(context.Context) (bool, error) {
 	return f.restored, f.restoreErr
 }
 
@@ -168,14 +368,14 @@ func (f *fakeRecovery) Restart(context.Context) error {
 // A probe whose result the test drives directly, standing in for pg_isready.
 func stubProbe(t *testing.T, code int, err error) *engineProbe {
 	t.Helper()
-	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
-	return newEngineProbe(cfg, func(context.Context, string, ...string) (int, error) {
+	cfg := testLoadConfig(t, enginePostgres)
+	return newPostgresProbe(cfg, func(context.Context, string, ...string) (int, error) {
 		return code, err
 	})
 }
 
 func newTestGuard(engine parameterRecovery, probe *engineProbe) *paramGuard {
-	g := newParamGuard(engine, probe)
+	g := newParamGuard(engine, probe, nil)
 	// Real budgets are minutes, which is the point of them; the behaviour under
 	// test is the decision, not the wait.
 	g.after, g.poll = 20*time.Millisecond, time.Millisecond
@@ -184,13 +384,24 @@ func newTestGuard(engine parameterRecovery, probe *engineProbe) *paramGuard {
 
 // pg_isready exit 2 is an engine that is not there at all, which after a
 // parameter change is the boot-loop shape this breaks.
-func TestParamGuard_RollsBackAndRestartsWhenTheEngineNeverComesUp(t *testing.T) {
+func TestParamGuard_RollsBackReportsAndRestartsWhenTheEngineNeverComesUp(t *testing.T) {
 	engine := &fakeRecovery{restored: true}
-	newTestGuard(engine, stubProbe(t, 2, nil)).Run(t.Context())
+	cp := newFakeControlPlane()
+	guard := newParamGuard(engine, stubProbe(t, 2, nil), cp)
+	guard.id = identity{DBInstanceIdentifier: "db-1"}
+	guard.after, guard.poll = 20*time.Millisecond, time.Millisecond
+	guard.Run(t.Context())
 
 	if engine.restarts != 1 {
 		t.Errorf("restarts = %d, want the engine restarted on the rolled-back set", engine.restarts)
 	}
+	states := cp.snapshotStates()
+	if len(states) != 1 {
+		t.Fatalf("submitted states = %d, want the rollback report", len(states))
+	}
+	if states[0].health != handlers_rds.EngineHealthUnhealthy || states[0].message != handlers_rds.ParameterRollbackMessage {
+		t.Errorf("rollback state = %+v, want the unhealthy rollback report", states[0])
+	}
 }
 
 func TestParamGuard_DoesNothingWhenTheEngineIsServing(t *testing.T) {

cmd/rds-agent/quiesce.go

@@ -15,10 +15,11 @@ import (
 	"time"
 )
 
-// PostgreSQL's non-exclusive backup API ties the backup to the session that
-// started it: when that session ends, for any reason, the backup is aborted.
-// That is what makes the hold self-expiring — a control plane that dies mid
-// snapshot cannot leave the engine in backup mode forever.
+// Both engines tie a backup hold to the session that took it: PostgreSQL aborts
+// a non-exclusive backup when its session ends, and MariaDB releases a BACKUP
+// STAGE with the connection that started it. That is what makes the hold
+// self-expiring — a control plane that dies mid snapshot cannot leave the engine
+// in backup mode forever.
 type engineSession interface {
 	// Runs sql and waits for the engine to finish it.
 	Exec(ctx context.Context, sql string) error
@@ -31,7 +32,8 @@ type engineSession interface {
 type sessionRunner func(ctx context.Context, c command) (engineSession, error)
 
 // Written after every statement so the reader knows the engine is done with it.
-// The statements this carries return LSNs, so it cannot appear in a result.
+// The statements this carries return LSNs and backup stages, so it cannot appear
+// in a result.
 const sessionSentinel = "--rds-agent-ready--"
 
 // The backup mode currently held, and the timer that ends it regardless of what
@@ -42,98 +44,65 @@ type quiesceHold struct {
 	expiry  *time.Timer
 }
 
-// Puts the engine into backup mode: the datadir is checkpointed and the engine
-// stops writing over the pages a snapshot is about to read. The hold is released
-// by Unquiesce, or by its own deadline, whichever comes first.
-func (e *postgresEngine) Quiesce(ctx context.Context, label string, hold time.Duration) error {
+// The hold bookkeeping, which is the same for both engines: only the statements
+// that take and release it differ, not the single-hold rule or the deadline that
+// releases it whatever happens to the caller.
+type quiesceState struct {
+	// Guarded because the expiry timer releases the hold on its own goroutine.
+	mu   sync.Mutex
+	held *quiesceHold
+}
+
+// The two things every quiesce needs, checked before a session is opened so a
+// malformed command costs the engine nothing.
+func validateQuiesceRequest(label string, hold time.Duration) error {
 	if label == "" {
 		return errors.New("quiesce requires a backup label")
 	}
 	if hold <= 0 {
 		return errors.New("quiesce requires a positive hold deadline")
 	}
+	return nil
+}
 
-	// Held across the whole start, so a second quiesce waits and then finds the
-	// first one's hold rather than opening a concurrent backup alongside it.
-	e.mu.Lock()
-	defer e.mu.Unlock()
-	if e.held != nil {
-		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)
-	}
-
-	// Deliberately not the command's context: the session has to outlive the call
-	// that started it, and its own deadline is what bounds it instead.
-	session, err := e.startSess(context.WithoutCancel(ctx), command{
-		Name:  e.psql,
-		Args:  e.psqlArgs(),
-		Env:   []string{"PATH=" + defaultGuestPath, "RDS_BACKUP_LABEL=" + label},
-		Stdin: "",
-		User:  e.osUser,
-	})
-	if err != nil {
-		return fmt.Errorf("open a backup session: %w", err)
-	}
-
-	// fast forces an immediate checkpoint rather than spreading it over the
-	// checkpoint interval, which would hold the snapshot open for minutes.
-	const sql = `\getenv label RDS_BACKUP_LABEL
-SELECT pg_backup_start(:'label', fast => true);
-`
-	if err := session.Exec(ctx, sql); err != nil {
-		if closeErr := session.Close(); closeErr != nil {
-			slog.Warn("rds-agent: closing a failed backup session", "err", closeErr)
-		}
-		return fmt.Errorf("put the engine into backup mode: %w", err)
-	}
-
-	e.held = &quiesceHold{
+// Records the hold and arms its deadline. Called with mu held, so a second
+// quiesce waits and then finds this hold rather than opening a concurrent
+// backup alongside it.
+func (q *quiesceState) beginHoldLocked(label string, session engineSession, hold time.Duration) {
+	q.held = &quiesceHold{
 		label:   label,
 		session: session,
-		expiry:  time.AfterFunc(hold, func() { e.expireQuiesce(label) }),
+		expiry:  time.AfterFunc(hold, func() { q.expire(label) }),
 	}
 	slog.Info("rds-agent: engine quiesced for backup", "label", label, "hold", hold)
-	return nil
 }
 
-// Ends the backup cleanly. A missing hold is an error rather than a silent
-// success: it means the deadline fired first, so the snapshot the control plane
-// just took was not taken against a held checkpoint.
-func (e *postgresEngine) Unquiesce(ctx context.Context) error {
-	e.mu.Lock()
-	held := e.held
-	e.held = nil
-	e.mu.Unlock()
+// Takes the hold off the engine, if one is still there, and disarms its
+// deadline. The caller ends the session, because the statement that releases the
+// backup has to run on it first.
+func (q *quiesceState) takeHold() *quiesceHold {
+	q.mu.Lock()
+	defer q.mu.Unlock()
 
-	if held == nil {
-		return errors.New("the engine is not quiesced; the backup hold had already expired")
+	held := q.held
+	q.held = nil
+	if held != nil {
+		held.expiry.Stop()
 	}
-	held.expiry.Stop()
-
-	// The stop has to run on the session that started the backup, and the session
-	// is closed either way — the engine aborts an unstopped backup with it.
-	execErr := held.session.Exec(ctx, "SELECT pg_backup_stop();\n")
-	closeErr := held.session.Close()
-	if execErr != nil {
-		return fmt.Errorf("take the engine out of backup mode: %w", errors.Join(execErr, closeErr))
-	}
-	if closeErr != nil {
-		return fmt.Errorf("close the backup session: %w", closeErr)
-	}
-	slog.Info("rds-agent: engine released from backup mode", "label", held.label)
-	return nil
+	return held
 }
 
-// The deadline. Ending the session is enough — the engine aborts the backup it
-// was holding — so this never has to reach the engine itself.
-func (e *postgresEngine) expireQuiesce(label string) {
-	e.mu.Lock()
-	held := e.held
+// The deadline. Ending the session is enough — both engines release the hold
+// with it — so this never has to reach the engine itself.
+func (q *quiesceState) expire(label string) {
+	q.mu.Lock()
+	held := q.held
 	if held == nil || held.label != label {
-		e.mu.Unlock()
+		q.mu.Unlock()
 		return
 	}
-	e.held = nil
-	e.mu.Unlock()
+	q.held = nil
+	q.mu.Unlock()
 
 	slog.Warn("rds-agent: backup hold expired; releasing the engine", "label", label)
 	if err := held.session.Close(); err != nil {
@@ -141,21 +110,27 @@ func (e *postgresEngine) expireQuiesce(label string) {
 	}
 }
 
-// A psql child with its stdin held open, so statements can be fed to it one at a
-// time and the engine keeps seeing a single session.
-type psqlSession struct {
+// A client child with its stdin held open, so statements can be fed to it one at
+// a time and the engine keeps seeing a single session.
+type clientSession struct {
 	cmd    *exec.Cmd
 	stdin  io.WriteCloser
 	stdout *bufio.Reader
 	stderr *bytes.Buffer
+	// The statement that makes this client print the sentinel, which is the only
+	// part of a held session that differs between engines.
+	sentinel string
 
 	mu     sync.Mutex
 	closed bool
 }
 
-var _ engineSession = (*psqlSession)(nil)
+var _ engineSession = (*clientSession)(nil)
 
 func execSessionRunner(ctx context.Context, c command) (engineSession, error) {
+	if c.SentinelStatement == "" {
+		return nil, fmt.Errorf("%s was started as a session with no sentinel statement", c.Name)
+	}
 	cmd := exec.CommandContext(ctx, c.Name, c.Args...)
 	cmd.Env = c.Env
 	if c.User != "" {
@@ -180,24 +155,30 @@ func execSessionRunner(ctx context.Context, c command) (engineSession, error) {
 	if err := cmd.Start(); err != nil {
 		return nil, fmt.Errorf("start %s: %w", c.Name, err)
 	}
-	return &psqlSession{cmd: cmd, stdin: stdin, stdout: bufio.NewReader(stdout), stderr: &stderr}, nil
+	return &clientSession{
+		cmd:      cmd,
+		stdin:    stdin,
+		stdout:   bufio.NewReader(stdout),
+		stderr:   &stderr,
+		sentinel: c.SentinelStatement,
+	}, nil
 }
 
-// Feeds sql and waits for the sentinel that follows it. ON_ERROR_STOP makes psql
-// exit on a failed statement, so a sentinel that never arrives is the failure
-// itself and the engine's own message is on stderr.
-func (s *psqlSession) Exec(ctx context.Context, sql string) error {
+// Feeds sql and waits for the sentinel that follows it. Both clients are run in
+// a mode that exits on a failed statement, so a sentinel that never arrives is
+// the failure itself and the engine's own message is on stderr.
+func (s *clientSession) Exec(ctx context.Context, sql string) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.closed {
 		return errors.New("the backup session is closed")
 	}
-	if _, err := io.WriteString(s.stdin, sql+"\\echo "+sessionSentinel+"\n"); err != nil {
+	if _, err := io.WriteString(s.stdin, sql+s.sentinel); err != nil {
 		return fmt.Errorf("%w: %s", err, strings.TrimSpace(s.stderr.String()))
 	}
 
-	// Read on its own goroutine so a psql that never answers costs the caller its
-	// deadline rather than blocking it forever.
+	// Read on its own goroutine so a client that never answers costs the caller
+	// its deadline rather than blocking it forever.
 	done := make(chan error, 1)
 	go func() { done <- s.awaitSentinel() }()
 	select {
@@ -208,7 +189,7 @@ func (s *psqlSession) Exec(ctx context.Context, sql string) error {
 	}
 }
 
-func (s *psqlSession) awaitSentinel() error {
+func (s *clientSession) awaitSentinel() error {
 	for {
 		line, err := s.stdout.ReadString('\n')
 		if strings.TrimSpace(line) == sessionSentinel {
@@ -226,7 +207,7 @@ func (s *psqlSession) awaitSentinel() error {
 // Closing stdin ends the script, and the wait reaps the child. A session that
 // will not end is killed, because leaving it is what would keep the engine in
 // backup mode.
-func (s *psqlSession) Close() error {
+func (s *clientSession) Close() error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.closed {

cmd/rds-agent/quiesce_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"errors"
 	"slices"
 	"strings"
 	"sync"
@@ -113,81 +112,6 @@ func TestPostgresEngine_UnquiesceStopsTheBackupAndEndsTheSession(t *testing.T) {
 	}
 }
 
-// A hold that expired mid-snapshot means the snapshot was not taken against a
-// held checkpoint, so the release reports it rather than succeeding silently.
-func TestPostgresEngine_UnquiesceWithoutAHoldIsAnError(t *testing.T) {
-	engine, _ := newQuiesceEngine(t, &fakeSession{})
-
-	err := engine.Unquiesce(context.Background())
-	if err == nil {
-		t.Fatal("Unquiesce succeeded with no hold to release")
-	}
-	if !strings.Contains(err.Error(), "expired") {
-		t.Errorf("error %q does not say the hold had expired", err)
-	}
-}
-
-// The engine aborts the backup when the session ends, which is what makes the
-// hold self-expiring: a control plane that dies cannot leave it held forever.
-func TestPostgresEngine_QuiesceReleasesItselfOnItsDeadline(t *testing.T) {
-	session := &fakeSession{}
-	engine, _ := newQuiesceEngine(t, session)
-
-	if err := engine.Quiesce(context.Background(), "orders-db-pre-upgrade", 10*time.Millisecond); err != nil {
-		t.Fatalf("Quiesce: %v", err)
-	}
-	deadline := time.Now().Add(2 * time.Second)
-	for session.closes() == 0 && time.Now().Before(deadline) {
-		time.Sleep(time.Millisecond)
-	}
-
-	if session.closes() != 1 {
-		t.Fatal("the backup session outlived its deadline")
-	}
-	// The hold is gone with it, so a later release is honest about that.
-	if err := engine.Unquiesce(context.Background()); err == nil {
-		t.Error("Unquiesce succeeded after the hold expired")
-	}
-}
-
-// One backup at a time: a second start would open a concurrent backup alongside
-// the first rather than extending it.
-func TestPostgresEngine_QuiesceRefusesASecondHold(t *testing.T) {
-	session := &fakeSession{}
-	engine, started := newQuiesceEngine(t, session)
-	if err := engine.Quiesce(context.Background(), "first", time.Minute); err != nil {
-		t.Fatalf("Quiesce: %v", err)
-	}
-
-	err := engine.Quiesce(context.Background(), "second", time.Minute)
-	if err == nil {
-		t.Fatal("a second quiesce was accepted while the first was held")
-	}
-	if !strings.Contains(err.Error(), "first") {
-		t.Errorf("error %q does not name the backup already holding the engine", err)
-	}
-	if len(*started) != 1 {
-		t.Errorf("started %d sessions, want the second refused before opening one", len(*started))
-	}
-}
-
-// A failed start leaves nothing held, so a retry is not refused by a hold that
-// never took effect.
-func TestPostgresEngine_QuiesceEndsTheSessionWhenTheBackupWillNotStart(t *testing.T) {
-	session := &fakeSession{execErr: errors.New(`ERROR:  a backup is already in progress`)}
-	engine, _ := newQuiesceEngine(t, session)
-
-	if err := engine.Quiesce(context.Background(), "orders-db-pre-upgrade", time.Minute); err == nil {
-		t.Fatal("Quiesce succeeded against an engine that refused the backup")
-	}
-	if session.closes() != 1 {
-		t.Errorf("closed %d times, want the failed session ended", session.closes())
-	}
-	if engine.held != nil {
-		t.Error("a failed quiesce left a hold behind")
-	}
-}
-
 func TestCommandRegistry_QuiesceCarriesTheLabelAndDeadline(t *testing.T) {
 	engine := &fakeEngine{}
 	reply := newCommander(nil, newCommandRegistry(engine, &fakeStorage{}), 0).execute(context.Background(),

cmd/rds-agent/storage.go

@@ -28,8 +28,6 @@ const (
 	defaultXFSGrowfs  = "xfs_growfs"
 	defaultMountsFile = "/proc/mounts"
 	defaultSysBlock   = "/sys/class/block"
-	// Where rds-datadir mounts the data volume; the datadir sits one level in.
-	defaultDataMount = "/var/lib/postgresql"
 	// growpart says this when the partition already fills the disk, which a
 	// resumed grow is the normal way to reach.
 	growpartNoChange = "NOCHANGE"

cmd/rds-agent/storage_test.go

@@ -61,12 +61,12 @@ func newTestStorage(t *testing.T, device, fstype string, run commandRunner) *gue
 	mounts := filepath.Join(dir, "mounts")
 	content := "proc /proc proc rw 0 0\n"
 	if device != "" {
-		content += device + " " + defaultDataMount + " " + fstype + " rw,relatime 0 0\n"
+		content += device + " " + testDataMount + " " + fstype + " rw,relatime 0 0\n"
 	}
 	if err := os.WriteFile(mounts, []byte(content), 0o600); err != nil {
 		t.Fatalf("write the mount table: %v", err)
 	}
-	cfg := config{DataMount: defaultDataMount, MountsFile: mounts, SysBlock: filepath.Join(dir, "block")}
+	cfg := config{DataMount: testDataMount, MountsFile: mounts, SysBlock: filepath.Join(dir, "block")}
 	if err := os.MkdirAll(cfg.SysBlock, 0o755); err != nil {
 		t.Fatalf("create the sysfs tree: %v", err)
 	}
@@ -93,6 +93,9 @@ func seedPartition(t *testing.T, g *guestStorage, disk, name, number string) {
 
 // The data volume is formatted whole, so the usual grow is resize2fs against
 // the device with no partition to push out first.
+// Where rds-datadir mounts the data volume in the PostgreSQL preset.
+const testDataMount = "/var/lib/postgresql"
+
 func TestGrowFilesystem_ResizesAnExt4DeviceOnline(t *testing.T) {
 	log := &runLog{}
 	g := newTestStorage(t, "/dev/vdb", "ext4", log.run)
@@ -108,7 +111,7 @@ func TestGrowFilesystem_ResizesAnExt4DeviceOnline(t *testing.T) {
 	if args := log.calls[0].Args; len(args) != 1 || args[0] != "/dev/vdb" {
 		t.Errorf("resize2fs args = %v, want the device", args)
 	}
-	if !strings.Contains(message, "/dev/vdb") || !strings.Contains(message, defaultDataMount) {
+	if !strings.Contains(message, "/dev/vdb") || !strings.Contains(message, testDataMount) {
 		t.Errorf("reply message = %q, want the device and mount it grew", message)
 	}
 }
@@ -125,7 +128,7 @@ func TestGrowFilesystem_GrowsXFSByMountPoint(t *testing.T) {
 	if got := log.names(); len(got) != 1 || got[0] != defaultXFSGrowfs {
 		t.Fatalf("ran %v, want just %s", got, defaultXFSGrowfs)
 	}
-	if args := log.calls[0].Args; len(args) != 1 || args[0] != defaultDataMount {
+	if args := log.calls[0].Args; len(args) != 1 || args[0] != testDataMount {
 		t.Errorf("xfs_growfs args = %v, want the mount point", args)
 	}
 }

cmd/spinifex/cmd/admin.go

@@ -1405,9 +1405,10 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	fmt.Printf("   Key: %s\n", viperblockKeyPath)
 
 	if !masterKeyExisted {
-		fmt.Printf("\n🔑 Generated admin credentials (save these — they won't be shown again):\n")
-		fmt.Printf("   Access Key:  %s\n", adminAccessKey)
-		fmt.Printf("   Secret Key:  %s\n", adminSecretKey)
+		// Never echo the keys: on an ISO install stdout is the systemd journal,
+		// which persists them for the life of the node. They are recoverable from
+		// bootstrap.json and ~/.aws/credentials.
+		fmt.Printf("\n🔑 Generated admin credentials (written to ~/.aws/credentials)\n")
 		fmt.Printf("   Account:     %s (%s)\n", admin.DefaultAccountName(), admin.DefaultAccountID())
 		fmt.Printf("   AWS Profile: spinifex\n")
 	}
@@ -1524,7 +1525,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	northstarConfigPath := filepath.Join(dirs.Northstar, "northstar.toml")
 
 	// Parse multi-node predastore configuration (legacy flag-based approach for single-node)
-	var predastoreNodeID int
+	var predastoreHostID int
 	if predastoreNodesStr != "" {
 		ips := strings.Split(predastoreNodesStr, ",")
 		if len(ips) < 2 {
@@ -1545,14 +1546,16 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 			})
 		}
 
-		predastoreNodeID = admin.FindNodeIDByIP(predastoreNodes, bindIP)
-		if predastoreNodeID == 0 {
+		// One machine is one predastore host, so the node list index that
+		// matches this bind IP is this node's host ID.
+		predastoreHostID = admin.FindNodeIDByIP(predastoreNodes, bindIP)
+		if predastoreHostID == 0 {
 			fmt.Fprintf(os.Stderr, "❌ Error: --bind IP %s not found in --predastore-nodes list\n", bindIP)
 			os.Exit(1)
 		}
 
 		// Generate multi-node predastore.toml
-		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP, compactionInterval, northstarCreds)
+		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, spxRoot, bindIP, compactionInterval, northstarCreds)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
 			os.Exit(1)
@@ -1563,7 +1566,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 			fmt.Fprintf(os.Stderr, "Error writing predastore config: %v\n", err)
 			os.Exit(1)
 		}
-		fmt.Printf("✅ Created: multi-node predastore.toml (node ID: %d)\n", predastoreNodeID)
+		fmt.Printf("✅ Created: multi-node predastore.toml (host ID: %d)\n", predastoreHostID)
 	}
 
 	// Pre-generate default VPC/subnet/IGW IDs for bootstrap config.
@@ -1593,7 +1596,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		ClusterRoutes: clusterRoutes,
 		ClusterName:   clusterName,
 
-		PredastoreNodeID:          predastoreNodeID,
+		PredastoreHostID:          predastoreHostID,
 		CompactionIntervalSeconds: compactionInterval,
 		Services:                  services,
 
@@ -1840,10 +1843,10 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	northstarConfigPath := filepath.Join(dirs.Northstar, "northstar.toml")
 
 	// Generate multi-node predastore config
-	var predastoreNodeID int
+	var predastoreHostID int
 	hasPredastoreConfig := len(predastoreNodes) >= 2
 	if hasPredastoreConfig {
-		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP, compactionInterval, northstarCreds)
+		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, spxRoot, bindIP, compactionInterval, northstarCreds)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
 			os.Exit(1)
@@ -1855,8 +1858,8 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 			os.Exit(1)
 		}
 
-		predastoreNodeID = admin.FindNodeIDByIP(predastoreNodes, bindIP)
-		fmt.Printf("✅ Created: multi-node predastore.toml (node ID: %d)\n", predastoreNodeID)
+		predastoreHostID = admin.FindNodeIDByIP(predastoreNodes, bindIP)
+		fmt.Printf("✅ Created: multi-node predastore.toml (host ID: %d)\n", predastoreHostID)
 	}
 
 	spinifexTomlPath := filepath.Join(configDir, "spinifex.toml")
@@ -1880,7 +1883,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		ClusterRoutes: clusterRoutes,
 		ClusterName:   clusterName,
 
-		PredastoreNodeID:          predastoreNodeID,
+		PredastoreHostID:          predastoreHostID,
 		CompactionIntervalSeconds: compactionInterval,
 		Services:                  services,
 		RemoteNodes:               buildRemoteNodes(allNodes, node, northstarConfigPath),
@@ -2390,11 +2393,11 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	northstarCreds, northstarConfigPath := northstarFromFormation(creds, dirs)
 
 	// Generate multi-node predastore config
-	var predastoreNodeID int
+	var predastoreHostID int
 	hasPredastoreConfig := len(predastoreNodes) >= 2
 
 	if hasPredastoreConfig {
-		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, creds.AccessKey, creds.SecretKey, creds.Region, creds.NatsToken, configDir, bindIP, compactionInterval,
+		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, creds.AccessKey, creds.SecretKey, creds.Region, creds.NatsToken, configDir, dataDir, bindIP, compactionInterval,
 			northstarCreds)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
@@ -2407,12 +2410,12 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 			os.Exit(1)
 		}
 
-		predastoreNodeID = admin.FindNodeIDByIP(predastoreNodes, bindIP)
-		if predastoreNodeID == 0 {
+		predastoreHostID = admin.FindNodeIDByIP(predastoreNodes, bindIP)
+		if predastoreHostID == 0 {
 			fmt.Fprintf(os.Stderr, "❌ Error: bind IP %s not found in predastore node list\n", bindIP)
 			os.Exit(1)
 		}
-		fmt.Printf("✅ Created: multi-node predastore.toml (node ID: %d)\n", predastoreNodeID)
+		fmt.Printf("✅ Created: multi-node predastore.toml (host ID: %d)\n", predastoreHostID)
 	}
 
 	spinifexTomlPath := filepath.Join(configDir, "spinifex.toml")
@@ -2436,7 +2439,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		ClusterRoutes: clusterRoutes,
 		ClusterName:   creds.ClusterName,
 
-		PredastoreNodeID:          predastoreNodeID,
+		PredastoreHostID:          predastoreHostID,
 		CompactionIntervalSeconds: compactionInterval,
 		Services:                  services,
 		RemoteNodes:               buildRemoteNodes(statusResp.Nodes, node, northstarConfigPath),

cmd/spinifex/cmd/admin_ochre.go

@@ -9,6 +9,7 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"sort"
 	"strings"
 	"time"
 
@@ -397,6 +398,11 @@ func registerWeightsSnapshot(store objectstore.ObjectStore, bucket, snapshotID,
 // which needs a live environment (see materializeWeightsVolume).
 type weightsMaterializer func(ctx context.Context, downloadDir string, contentBytes int64) (snapshotID string, err error)
 
+// weightsSnapshotChecker reports whether snapshotID still exists in the
+// snapshot store, so runStageWeights can tell a healthy KV record from one
+// whose snapshot was lost to a host rebuild or volume GC.
+type weightsSnapshotChecker func(ctx context.Context, snapshotID string) (bool, error)
+
 // runStageWeights holds 'ochre weights stage' decision and side-effect logic:
 // catalog validation, S3 URI parsing, idempotency/replacement detection
 // against the existing KV record, prefix validation, download, materialisation
@@ -404,7 +410,7 @@ type weightsMaterializer func(ctx context.Context, downloadDir string, contentBy
 // dependency of its own, so tests drive it directly against a fake object
 // store, a real WeightsStore backed by an embedded JetStream server, and a
 // fake materializer.
-func runStageWeights(ctx context.Context, store objectstore.ObjectStore, weightsStore *gateway_bedrock.WeightsStore, tmpDirFlag, modelID, s3URI string, materialize weightsMaterializer) (string, error) {
+func runStageWeights(ctx context.Context, store objectstore.ObjectStore, weightsStore *gateway_bedrock.WeightsStore, tmpDirFlag, modelID, s3URI string, materialize weightsMaterializer, checkSnapshotLive weightsSnapshotChecker) (string, error) {
 	if _, found, selfHost := gateway_bedrock.LookupServingSpec(modelID); !found || !selfHost {
 		if !found {
 			return "", fmt.Errorf("unknown model ID %q: not present in the Ochre catalog", modelID)
@@ -422,8 +428,18 @@ func runStageWeights(ctx context.Context, store objectstore.ObjectStore, weights
 	if err != nil {
 		return "", err
 	}
+	var replacingMissingSnapshot bool
 	if hadPrevious && existing.SourceURI == sourceURI {
-		return fmt.Sprintf("%s is already staged from %s (snapshot %s); nothing to do.", modelID, sourceURI, existing.SnapshotID), nil
+		live, err := checkSnapshotLive(ctx, existing.SnapshotID)
+		if err != nil {
+			return "", err
+		}
+		if live {
+			return fmt.Sprintf("%s is already staged from %s (snapshot %s); nothing to do.", modelID, sourceURI, existing.SnapshotID), nil
+		}
+		// The KV record is otherwise up to date; only its snapshot is gone, so
+		// fall through and self-heal rather than making the operator run remove.
+		replacingMissingSnapshot = true
 	}
 
 	fmt.Printf("Validating %s ...\n", sourceURI)
@@ -457,6 +473,10 @@ func runStageWeights(ctx context.Context, store objectstore.ObjectStore, weights
 		return "", err
 	}
 
+	if replacingMissingSnapshot {
+		return fmt.Sprintf("✅ Staged %s from %s (snapshot %s). Replaced MISSING snapshot %s -- it was no longer present in the snapshot store.",
+			modelID, sourceURI, snapshotID, existing.SnapshotID), nil
+	}
 	if hadPrevious {
 		return fmt.Sprintf("✅ Staged %s from %s (snapshot %s). Replaced previous snapshot %s -- reclaim it separately if no longer needed.",
 			modelID, sourceURI, snapshotID, existing.SnapshotID), nil
@@ -557,6 +577,23 @@ func materializeWeightsVolume(node config.Config, downloadDir string, contentByt
 	return snapshotID, nil
 }
 
+// newWeightsSnapshotChecker is the real weightsSnapshotChecker: it reads the
+// EC2 control plane's metadata.json for snapshotID from the node's
+// predastore bucket -- the same record registerWeightsSnapshot writes at
+// stage time -- and treats a missing object as gone. Any other read failure
+// is a real error, not a signal to guess the snapshot's existence.
+func newWeightsSnapshotChecker(store objectstore.ObjectStore, bucket string) weightsSnapshotChecker {
+	return func(_ context.Context, snapshotID string) (bool, error) {
+		if _, err := handlers_ec2_snapshot.ReadSnapshotConfig(store, bucket, snapshotID); err != nil {
+			if objectstore.IsNoSuchKeyError(err) {
+				return false, nil
+			}
+			return false, err
+		}
+		return true, nil
+	}
+}
+
 // loadConfigAndConnectFn and ochreExit indirect the two effects that make
 // the ochre weights Run functions otherwise untestable: a live NATS
 // connection and a process-terminating exit. Tests substitute both -- a fake
@@ -602,8 +639,9 @@ func runOchreWeightsStage(cmd *cobra.Command, _ []string) {
 	materialize := func(_ context.Context, downloadDir string, contentBytes int64) (string, error) {
 		return materializeWeightsVolume(node, downloadDir, contentBytes, mkey)
 	}
+	checkSnapshotLive := newWeightsSnapshotChecker(store, node.Predastore.Bucket)
 
-	msg, err := runStageWeights(context.Background(), store, weightsStore, tmpDirFlag, modelID, s3URI, materialize)
+	msg, err := runStageWeights(context.Background(), store, weightsStore, tmpDirFlag, modelID, s3URI, materialize, checkSnapshotLive)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 		ochreExit(1)
@@ -612,11 +650,21 @@ func runOchreWeightsStage(cmd *cobra.Command, _ []string) {
 	fmt.Println(msg)
 }
 
+// weightsStatusLive and weightsStatusDangling are the STATUS column values
+// listWeightsOutput renders, so a dangling row (KV record present, snapshot
+// gone) reads distinctly from a healthy one at a glance.
+const (
+	weightsStatusLive     = "OK"
+	weightsStatusDangling = "DANGLING (snapshot missing)"
+)
+
 // listWeightsOutput renders 'ochre weights list': a friendly message when
-// nothing is staged, else a MODEL ID / SOURCE URI / SNAPSHOT ID table. Split
-// out from runOchreWeightsList so it is testable against a real WeightsStore
-// without a NATS connection.
-func listWeightsOutput(ctx context.Context, weightsStore *gateway_bedrock.WeightsStore) (string, error) {
+// nothing is staged, else a MODEL ID / SOURCE URI / SNAPSHOT ID / STATUS
+// table. checkSnapshotLive marks a row dangling when its KV record survives
+// but the snapshot it names does not, so an operator sees at a glance which
+// staged models can actually serve. Split out from runOchreWeightsList so it
+// is testable against a real WeightsStore without a NATS connection.
+func listWeightsOutput(ctx context.Context, weightsStore *gateway_bedrock.WeightsStore, checkSnapshotLive weightsSnapshotChecker) (string, error) {
 	entries, err := weightsStore.ListWeights(ctx)
 	if err != nil {
 		return "", err
@@ -625,9 +673,17 @@ func listWeightsOutput(ctx context.Context, weightsStore *gateway_bedrock.Weight
 		return "No models staged.", nil
 	}
 
-	tableData := pterm.TableData{{"MODEL ID", "SOURCE URI", "SNAPSHOT ID"}}
+	tableData := pterm.TableData{{"MODEL ID", "SOURCE URI", "SNAPSHOT ID", "STATUS"}}
 	for _, e := range entries {
-		tableData = append(tableData, []string{e.ModelID, e.SourceURI, e.SnapshotID})
+		live, err := checkSnapshotLive(ctx, e.SnapshotID)
+		if err != nil {
+			return "", err
+		}
+		status := weightsStatusLive
+		if !live {
+			status = weightsStatusDangling
+		}
+		tableData = append(tableData, []string{e.ModelID, e.SourceURI, e.SnapshotID, status})
 	}
 	return pterm.DefaultTable.WithHasHeader().WithLeftAlignment().WithData(tableData).Srender()
 }
@@ -640,6 +696,7 @@ func runOchreWeightsList(_ *cobra.Command, _ []string) {
 		return
 	}
 	defer nc.Close()
+	node := appConfig.Nodes[appConfig.Node]
 
 	js, err := jetstream.New(nc)
 	if err != nil {
@@ -648,8 +705,10 @@ func runOchreWeightsList(_ *cobra.Command, _ []string) {
 		return
 	}
 	weightsStore := gateway_bedrock.NewWeightsStore(js, len(appConfig.Nodes))
+	store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
+	checkSnapshotLive := newWeightsSnapshotChecker(store, node.Predastore.Bucket)
 
-	out, err := listWeightsOutput(context.Background(), weightsStore)
+	out, err := listWeightsOutput(context.Background(), weightsStore, checkSnapshotLive)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
 		ochreExit(1)
@@ -706,3 +765,180 @@ func runOchreWeightsRemove(cmd *cobra.Command, _ []string) {
 	}
 	fmt.Printf("✅ Removed staged-weights entry for %s (snapshot %s and source objects untouched).\n", modelID, entry.SnapshotID)
 }
+
+var adminOchreAccessCmd = &cobra.Command{
+	Use:   "access",
+	Short: "Manage per-account model access grants",
+	Long: `Manage which accounts may see and invoke which Ochre models.
+
+Access is deny-by-default: an account with no grants sees an empty model
+catalog and every invocation is refused. Grants are cluster state held in the
+bedrock-model-access KV bucket, so these commands need a running cluster but
+may be run from any node.`,
+}
+
+var adminOchreAccessGrantCmd = &cobra.Command{
+	Use:   "grant",
+	Short: "Grant an account access to a model",
+	Run:   runOchreAccessGrant,
+}
+
+var adminOchreAccessRevokeCmd = &cobra.Command{
+	Use:   "revoke",
+	Short: "Revoke an account's access to a model",
+	Run:   runOchreAccessRevoke,
+}
+
+var adminOchreAccessListCmd = &cobra.Command{
+	Use:   "list",
+	Short: "List the models an account has been granted",
+	Run:   runOchreAccessList,
+}
+
+var adminOchreModelsCmd = &cobra.Command{
+	Use:   "models",
+	Short: "List every model in the platform catalog",
+	Long: `List the full platform catalog, ignoring grants. This is what an operator can
+grant from; it is not what any account can see.`,
+	Run: runOchreModels,
+}
+
+func init() {
+	ochreCmd.AddCommand(adminOchreAccessCmd)
+	ochreCmd.AddCommand(adminOchreModelsCmd)
+	adminOchreAccessCmd.AddCommand(adminOchreAccessGrantCmd)
+	adminOchreAccessCmd.AddCommand(adminOchreAccessRevokeCmd)
+	adminOchreAccessCmd.AddCommand(adminOchreAccessListCmd)
+
+	for _, c := range []*cobra.Command{adminOchreAccessGrantCmd, adminOchreAccessRevokeCmd} {
+		c.Flags().String("account-id", "", "12-digit account ID to change (required)")
+		c.Flags().String("model-id", "", "Model ID to change (e.g. meta.llama3-2-1b-instruct-v1:0)")
+		c.Flags().Bool("all-models", false, "Apply to every model in the platform catalog")
+		if err := c.MarkFlagRequired("account-id"); err != nil {
+			panic(err)
+		}
+	}
+
+	adminOchreAccessListCmd.Flags().String("account-id", "", "12-digit account ID to inspect (required)")
+	if err := adminOchreAccessListCmd.MarkFlagRequired("account-id"); err != nil {
+		panic(err)
+	}
+}
+
+// ochreAccessStore connects to the cluster and returns the grant store along
+// with a cleanup that closes the connection.
+func ochreAccessStore() (*gateway_bedrock.ModelAccessStore, func(), error) {
+	cfg, nc, err := loadConfigAndConnectFn()
+	if err != nil {
+		return nil, nil, fmt.Errorf("connect to cluster: %w", err)
+	}
+	js, err := jetstream.New(nc)
+	if err != nil {
+		nc.Close()
+		return nil, nil, fmt.Errorf("jetstream context: %w", err)
+	}
+	return gateway_bedrock.NewModelAccessStore(js, len(cfg.Nodes)), func() { nc.Close() }, nil
+}
+
+// ochreTargetModels resolves the --model-id / --all-models pair into the model
+// set to act on. Exactly one of the two must be given: defaulting either way
+// would make a mistyped flag silently change more or less than intended.
+func ochreTargetModels(cmd *cobra.Command) ([]string, error) {
+	modelID, _ := cmd.Flags().GetString("model-id")
+	allModels, _ := cmd.Flags().GetBool("all-models")
+
+	switch {
+	case allModels && modelID != "":
+		return nil, fmt.Errorf("--model-id and --all-models are mutually exclusive")
+	case allModels:
+		return gateway_bedrock.CatalogModelIDs(), nil
+	case modelID != "":
+		return []string{modelID}, nil
+	default:
+		return nil, fmt.Errorf("one of --model-id or --all-models is required")
+	}
+}
+
+func runOchreAccessGrant(cmd *cobra.Command, _ []string) {
+	runOchreAccessChange(cmd, true)
+}
+
+func runOchreAccessRevoke(cmd *cobra.Command, _ []string) {
+	runOchreAccessChange(cmd, false)
+}
+
+// runOchreAccessChange applies a grant or revoke across the resolved model set.
+// Both operations are idempotent, so re-running after a partial failure is safe
+// and this is callable from provisioning that runs on every deploy.
+func runOchreAccessChange(cmd *cobra.Command, grant bool) {
+	accountID, _ := cmd.Flags().GetString("account-id")
+
+	models, err := ochreTargetModels(cmd)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+
+	store, cleanup, err := ochreAccessStore()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+	defer cleanup()
+
+	ctx := context.Background()
+	verb := "Granted"
+	for _, modelID := range models {
+		if grant {
+			err = store.Grant(ctx, accountID, modelID)
+		} else {
+			verb = "Revoked"
+			err = store.Revoke(ctx, accountID, modelID)
+		}
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+			ochreExit(1)
+			return
+		}
+		fmt.Printf("✅ %s %s → %s\n", verb, accountID, modelID)
+	}
+}
+
+func runOchreAccessList(cmd *cobra.Command, _ []string) {
+	accountID, _ := cmd.Flags().GetString("account-id")
+
+	store, cleanup, err := ochreAccessStore()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+	defer cleanup()
+
+	models, err := store.List(context.Background(), accountID)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+	if len(models) == 0 {
+		fmt.Printf("Account %s has no model grants (it can see and invoke nothing).\n", accountID)
+		return
+	}
+
+	// KV key order is not meaningful, so sort for a stable, diffable listing.
+	sort.Strings(models)
+	for _, modelID := range models {
+		fmt.Println(modelID)
+	}
+}
+
+func runOchreModels(_ *cobra.Command, _ []string) {
+	models := gateway_bedrock.CatalogModelIDs()
+	sort.Strings(models)
+	for _, modelID := range models {
+		fmt.Println(modelID)
+	}
+}

cmd/spinifex/cmd/admin_ochre_endpoint.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strconv"
 	"time"
 
 	handlers_bedrock "github.com/mulgadc/spinifex/spinifex/handlers/bedrock"
@@ -19,8 +20,11 @@ var ochreEndpointCmd = &cobra.Command{
 	Long: `Operator surface over the daemon's serving-endpoint lifecycle: request an
 endpoint for a staged model, inspect its state, and tear it down.
 
-The gateway does not yet request endpoints itself, so until it does this is
-the only way to start a serving VM.`,
+The gateway requests endpoints on its own: an invoke for a model with no
+endpoint returns ModelNotReadyException and launches one in the background,
+and the daemon reclaims an endpoint that has been idle past its TTL. These
+commands drive the same lifecycle by hand, for staging a model ahead of first
+use or taking its GPU back early.`,
 }
 
 var ochreEndpointEnsureCmd = &cobra.Command{
@@ -71,6 +75,7 @@ func init() {
 	_ = ochreEndpointEnsureCmd.MarkFlagRequired("model-id")
 
 	ochreEndpointDescribeCmd.Flags().String("model-id", "", "Model ID to describe (required)")
+	ochreEndpointDescribeCmd.Flags().String("account", "", "Account ID scoping the lookup (defaults to the shared platform account; set to see a pinned provisioned-throughput endpoint)")
 	_ = ochreEndpointDescribeCmd.MarkFlagRequired("model-id")
 
 	ochreEndpointDeleteCmd.Flags().String("model-id", "", "Model ID whose endpoint to tear down (required)")
@@ -78,10 +83,12 @@ func init() {
 }
 
 const (
-	// defaultEndpointWaitTimeout is deliberately generous: the cold start this
-	// waits on has never been measured, so a tight default would report a
-	// timeout for what is really a slow first launch.
-	defaultEndpointWaitTimeout = 15 * time.Minute
+	// defaultEndpointWaitTimeout must stay comfortably above the daemon's own
+	// readiness bound. The daemon starts its clock once the VM is launched,
+	// which is up to a minute after this one starts, so an equal value expires
+	// here first and reports a bare client timeout instead of the daemon's
+	// abort, which says why the launch actually failed.
+	defaultEndpointWaitTimeout = 20 * time.Minute
 	// endpointPollInterval trades responsiveness against KV read volume. A
 	// cold start is minutes, so seconds of granularity is plenty.
 	endpointPollInterval = 2 * time.Second
@@ -139,7 +146,13 @@ func waitForEndpointReady(ctx context.Context, svc handlers_bedrock.EndpointServ
 func formatEndpointRecord(rec handlers_bedrock.EndpointRecord) string {
 	rows := [][2]string{
 		{"Model ID", rec.ModelID},
-		{"State", string(rec.State)},
+	}
+	if rec.AccountID != "" {
+		rows = append(rows, [2]string{"Account", rec.AccountID})
+	}
+	rows = append(rows, [2]string{"State", string(rec.State)})
+	if rec.Pinned {
+		rows = append(rows, [2]string{"Pinned", "yes"})
 	}
 	if rec.InstanceID != "" {
 		rows = append(rows, [2]string{"Instance ID", rec.InstanceID})
@@ -162,6 +175,7 @@ func formatEndpointRecord(rec handlers_bedrock.EndpointRecord) string {
 			rows = append(rows, [2]string{"Startup", rec.ReadyAt.Sub(rec.CreatedAt).Round(time.Second).String()})
 		}
 	}
+	rows = append(rows, reclaimRows(rec)...)
 
 	out := ""
 	for _, row := range rows {
@@ -170,8 +184,36 @@ func formatEndpointRecord(rec handlers_bedrock.EndpointRecord) string {
 	return out
 }
 
+// reclaimRows renders what the daemon's idle sweep last observed, so an
+// operator asking why an endpoint was or was not reclaimed can see the inputs
+// rather than infer them. Only meaningful for a READY endpoint: no other state
+// is swept. The Pinned flag itself is rendered unconditionally in
+// formatEndpointRecord, since it is meaningful regardless of state.
+//
+// "Idle for" is measured from the record's LastActive, which falls back to
+// ReadyAt, so an endpoint that has been quiet since launch reads as idle since
+// launch rather than since the zero time.
+func reclaimRows(rec handlers_bedrock.EndpointRecord) [][2]string {
+	if rec.State != handlers_bedrock.StateReady {
+		return nil
+	}
+	rows := [][2]string{{"In flight", strconv.Itoa(rec.InFlight)}}
+	if since := rec.LastActive(); !since.IsZero() {
+		rows = append(rows,
+			[2]string{"Last active", since.Format(time.RFC3339)},
+			[2]string{"Idle for", time.Since(since).Round(time.Second).String()})
+	}
+	if rec.ScrapeFailures > 0 {
+		rows = append(rows, [2]string{"Scrape failures", strconv.Itoa(rec.ScrapeFailures)})
+	}
+	return rows
+}
+
 // listEndpointsOutput renders 'ochre endpoint list'. Split from its Run
 // function so it is testable against a fake service with no NATS connection.
+// ACCOUNT and PINNED distinguish a pinned, account-scoped endpoint from a
+// shared platform one — List itself now returns every account's records, not
+// just the shared platform account's.
 func listEndpointsOutput(ctx context.Context, svc handlers_bedrock.EndpointService) (string, error) {
 	out, err := svc.List(ctx, &handlers_bedrock.ListEndpointsInput{}, utils.GlobalAccountID)
 	if err != nil {
@@ -181,9 +223,13 @@ func listEndpointsOutput(ctx context.Context, svc handlers_bedrock.EndpointServi
 		return "No serving endpoints.", nil
 	}
 
-	tableData := pterm.TableData{{"MODEL ID", "STATE", "INSTANCE ID", "BASE URL"}}
+	tableData := pterm.TableData{{"MODEL ID", "STATE", "ACCOUNT", "PINNED", "INSTANCE ID", "BASE URL"}}
 	for _, e := range out.Endpoints {
-		tableData = append(tableData, []string{e.ModelID, string(e.State), e.InstanceID, e.BaseURL})
+		pinned := ""
+		if e.Pinned {
+			pinned = "yes"
+		}
+		tableData = append(tableData, []string{e.ModelID, string(e.State), e.AccountID, pinned, e.InstanceID, e.BaseURL})
 	}
 	return pterm.DefaultTable.WithHasHeader().WithData(tableData).Srender()
 }
@@ -248,6 +294,7 @@ func runOchreEndpointEnsure(cmd *cobra.Command, _ []string) {
 
 func runOchreEndpointDescribe(cmd *cobra.Command, _ []string) {
 	modelID, _ := cmd.Flags().GetString("model-id")
+	accountID, _ := cmd.Flags().GetString("account")
 
 	svc, closeFn, err := endpointServiceFn()
 	if err != nil {
@@ -257,7 +304,7 @@ func runOchreEndpointDescribe(cmd *cobra.Command, _ []string) {
 	}
 	defer closeFn()
 
-	out, err := svc.Describe(context.Background(), &handlers_bedrock.DescribeEndpointInput{ModelID: modelID}, utils.GlobalAccountID)
+	out, err := svc.Describe(context.Background(), &handlers_bedrock.DescribeEndpointInput{ModelID: modelID, AccountID: accountID}, utils.GlobalAccountID)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 		ochreExit(1)

cmd/spinifex/cmd/admin_ochre_endpoint_test.go

@@ -22,8 +22,9 @@ type fakeEndpointService struct {
 	listErr     error
 	deleteErr   error
 
-	describeCalls int
-	deleteCalls   int
+	describeCalls  int
+	deleteCalls    int
+	describeInputs []*handlers_bedrock.DescribeEndpointInput
 }
 
 func (f *fakeEndpointService) Ensure(_ context.Context, _ *handlers_bedrock.EnsureEndpointInput, _ string) (*handlers_bedrock.EnsureEndpointOutput, error) {
@@ -33,7 +34,8 @@ func (f *fakeEndpointService) Ensure(_ context.Context, _ *handlers_bedrock.Ensu
 	return &handlers_bedrock.EnsureEndpointOutput{Endpoint: f.ensure}, nil
 }
 
-func (f *fakeEndpointService) Describe(_ context.Context, _ *handlers_bedrock.DescribeEndpointInput, _ string) (*handlers_bedrock.DescribeEndpointOutput, error) {
+func (f *fakeEndpointService) Describe(_ context.Context, in *handlers_bedrock.DescribeEndpointInput, _ string) (*handlers_bedrock.DescribeEndpointOutput, error) {
+	f.describeInputs = append(f.describeInputs, in)
 	if f.describeErr != nil {
 		return nil, f.describeErr
 	}
@@ -72,6 +74,10 @@ func fakeClock(now *time.Time) endpointWaitClock {
 
 const testModelID = "meta.llama3-2-1b-instruct-v1:0"
 
+// testAccountID stands in for a caller's real (non-Global) account, mirroring
+// handlers_bedrock's own testAccountID.
+const testAccountID = "111111111111"
+
 func TestWaitForEndpointReady_ReachesReadyAndReportsElapsed(t *testing.T) {
 	svc := &fakeEndpointService{describes: []handlers_bedrock.EndpointRecord{
 		{ModelID: testModelID, State: handlers_bedrock.StateStarting},
@@ -215,6 +221,24 @@ func TestListEndpointsOutput_ListsEndpoints(t *testing.T) {
 	require.Contains(t, msg, "STARTING")
 }
 
+// TestListEndpointsOutput_ShowsAccountAndPinnedColumns is Bug 2's list seam:
+// a pinned, account-scoped endpoint must render its own account and a PINNED
+// indicator, while a bare GlobalAccountID endpoint keeps listing unchanged
+// (its ACCOUNT cell just reads GlobalAccountID, PINNED empty).
+func TestListEndpointsOutput_ShowsAccountAndPinnedColumns(t *testing.T) {
+	svc := &fakeEndpointService{list: []handlers_bedrock.EndpointRecord{
+		{ModelID: testModelID, State: handlers_bedrock.StateReady, AccountID: "000000000001"},
+		{ModelID: "meta.llama3-2-3b-instruct-v1:0", State: handlers_bedrock.StateReady, AccountID: testAccountID, Pinned: true},
+	}}
+
+	msg, err := listEndpointsOutput(context.Background(), svc)
+	require.NoError(t, err)
+	require.Contains(t, msg, "ACCOUNT")
+	require.Contains(t, msg, "PINNED")
+	require.Contains(t, msg, "000000000001")
+	require.Contains(t, msg, testAccountID)
+}
+
 func TestListEndpointsOutput_ErrorSurfaces(t *testing.T) {
 	svc := &fakeEndpointService{listErr: errors.New("nats: no responders")}
 	_, err := listEndpointsOutput(context.Background(), svc)
@@ -310,6 +334,50 @@ func TestRunOchreEndpointDescribe_PrintsRecord(t *testing.T) {
 	require.Contains(t, out, "i-abc")
 }
 
+// TestRunOchreEndpointDescribe_DefaultAccountIsEmpty covers the regression
+// guard: a bare (GlobalAccountID) describe, with no --account flag set, must
+// still send an empty AccountID — resolveAccountID's own GlobalAccountID
+// fallback, unchanged from before --account existed.
+func TestRunOchreEndpointDescribe_DefaultAccountIsEmpty(t *testing.T) {
+	svc := &fakeEndpointService{describes: []handlers_bedrock.EndpointRecord{
+		{ModelID: testModelID, State: handlers_bedrock.StateReady},
+	}}
+	withEndpointService(t, svc, nil)
+
+	cmd := *ochreEndpointDescribeCmd
+	require.NoError(t, cmd.Flags().Set("model-id", testModelID))
+
+	withOchreExitCapture(t, func() {
+		captureStdout(t, func() { runOchreEndpointDescribe(&cmd, nil) })
+	})
+	require.Len(t, svc.describeInputs, 1)
+	require.Empty(t, svc.describeInputs[0].AccountID)
+}
+
+// TestRunOchreEndpointDescribe_AccountFlagScopesLookup is Bug 2's describe
+// seam: --account must reach DescribeEndpointInput.AccountID, which is how an
+// operator sees a pinned, account-scoped endpoint the GlobalAccountID lookup
+// would otherwise report ABSENT.
+func TestRunOchreEndpointDescribe_AccountFlagScopesLookup(t *testing.T) {
+	svc := &fakeEndpointService{describes: []handlers_bedrock.EndpointRecord{
+		{ModelID: testModelID, State: handlers_bedrock.StateReady, AccountID: testAccountID, Pinned: true},
+	}}
+	withEndpointService(t, svc, nil)
+
+	cmd := *ochreEndpointDescribeCmd
+	require.NoError(t, cmd.Flags().Set("model-id", testModelID))
+	require.NoError(t, cmd.Flags().Set("account", testAccountID))
+
+	var out string
+	withOchreExitCapture(t, func() {
+		out = captureStdout(t, func() { runOchreEndpointDescribe(&cmd, nil) })
+	})
+	require.Len(t, svc.describeInputs, 1)
+	require.Equal(t, testAccountID, svc.describeInputs[0].AccountID)
+	require.Contains(t, out, testAccountID)
+	require.Contains(t, out, "Pinned")
+}
+
 func TestRunOchreEndpointDescribe_ErrorExits1(t *testing.T) {
 	withEndpointService(t, &fakeEndpointService{describeErr: errors.New("nats: timeout")}, nil)
 
@@ -365,3 +433,61 @@ func TestRunOchreEndpointDelete_ErrorExits1(t *testing.T) {
 	code := withOchreExitCapture(t, func() { runOchreEndpointDelete(&cmd, nil) })
 	require.Equal(t, 1, code)
 }
+
+// The reclaim inputs are what an operator needs to answer "why was this
+// endpoint reaped" or "why was it not", so a READY record must show them.
+func TestFormatEndpointRecord_ShowsReclaimInputs(t *testing.T) {
+	ready := time.Now().UTC().Add(-30 * time.Minute)
+	out := formatEndpointRecord(handlers_bedrock.EndpointRecord{
+		ModelID:      testModelID,
+		State:        handlers_bedrock.StateReady,
+		ReadyAt:      ready,
+		LastActiveAt: ready.Add(20 * time.Minute),
+		InFlight:     3,
+	})
+
+	require.Contains(t, out, "In flight:")
+	require.Contains(t, out, "3")
+	require.Contains(t, out, "Last active:")
+	require.Contains(t, out, "Idle for:")
+}
+
+// An endpoint quiet since launch is idle since launch, not since the zero
+// time: the fallback is what keeps the reported figure meaningful.
+func TestFormatEndpointRecord_NeverActiveFallsBackToReadyAt(t *testing.T) {
+	out := formatEndpointRecord(handlers_bedrock.EndpointRecord{
+		ModelID: testModelID,
+		State:   handlers_bedrock.StateReady,
+		ReadyAt: time.Now().UTC().Add(-90 * time.Second),
+	})
+
+	require.Contains(t, out, "Idle for:")
+	require.NotContains(t, out, "0001-01-01", "an unsampled record must not report the zero time")
+	require.NotContains(t, out, "Scrape failures", "a healthy endpoint must not carry a failure row")
+}
+
+func TestFormatEndpointRecord_SurfacesPinnedAndScrapeFailures(t *testing.T) {
+	out := formatEndpointRecord(handlers_bedrock.EndpointRecord{
+		ModelID:        testModelID,
+		State:          handlers_bedrock.StateReady,
+		ReadyAt:        time.Now().UTC().Add(-time.Hour),
+		Pinned:         true,
+		ScrapeFailures: 2,
+	})
+
+	require.Contains(t, out, "Pinned:")
+	require.Contains(t, out, "Scrape failures:")
+	require.Contains(t, out, "2")
+}
+
+// Only READY endpoints are swept, so reclaim rows on any other state would be
+// reporting a decision nothing is making.
+func TestFormatEndpointRecord_NoReclaimRowsWhenNotReady(t *testing.T) {
+	out := formatEndpointRecord(handlers_bedrock.EndpointRecord{
+		ModelID: testModelID,
+		State:   handlers_bedrock.StateStarting,
+	})
+
+	require.NotContains(t, out, "In flight")
+	require.NotContains(t, out, "Idle for")
+}

cmd/spinifex/cmd/admin_ochre_test.go

@@ -5,8 +5,11 @@ import (
 	"context"
 	"errors"
 	"io"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -91,6 +94,23 @@ func explodingMaterializer(t *testing.T) weightsMaterializer {
 	}
 }
 
+// explodingSnapshotChecker fails the test immediately if invoked, standing
+// in for weightsSnapshotChecker on paths that must never need to consult
+// snapshot liveness: a different source URI, or refusal before the
+// idempotency branch is even reached.
+func explodingSnapshotChecker(t *testing.T) weightsSnapshotChecker {
+	t.Helper()
+	return func(context.Context, string) (bool, error) {
+		t.Fatal("checkSnapshotLive called on a path that must not need it")
+		return false, nil
+	}
+}
+
+// fixedSnapshotChecker always reports live, unconditionally.
+func fixedSnapshotChecker(live bool) weightsSnapshotChecker {
+	return func(context.Context, string) (bool, error) { return live, nil }
+}
+
 func TestParseWeightsS3URI_ValidWithTrailingSlash(t *testing.T) {
 	bucket, prefix, err := parseWeightsS3URI("s3://models/llama-3.2-1b/")
 	require.NoError(t, err)
@@ -264,15 +284,16 @@ func TestDownloadWeightsPrefix_FlattensToBasename(t *testing.T) {
 }
 
 // TestRunStageWeights_IdempotentReStageIsNoop covers the no-op path: staging
-// the same --s3-uri a second time must not touch the object store or
-// materialize anything at all, and must report the existing snapshot.
+// the same --s3-uri a second time, with the recorded snapshot still live,
+// must not touch the object store or materialize anything at all, and must
+// report the existing snapshot.
 func TestRunStageWeights_IdempotentReStageIsNoop(t *testing.T) {
 	weightsStore := newWeightsStoreForTest(t)
 	ctx := context.Background()
 	require.NoError(t, weightsStore.PutWeights(ctx, selfHostModelID, "s3://models/llama-3.2-1b/", "snap-0001"))
 
 	msg, err := runStageWeights(ctx, explodingObjectStore{t: t}, weightsStore, t.TempDir(),
-		selfHostModelID, "s3://models/llama-3.2-1b/", explodingMaterializer(t))
+		selfHostModelID, "s3://models/llama-3.2-1b/", explodingMaterializer(t), fixedSnapshotChecker(true))
 
 	require.NoError(t, err)
 	assert.Contains(t, msg, "already staged")
@@ -285,6 +306,59 @@ func TestRunStageWeights_IdempotentReStageIsNoop(t *testing.T) {
 	assert.Equal(t, "snap-0001", entry.SnapshotID)
 }
 
+// TestRunStageWeights_SelfHealsWhenSnapshotGone covers the same --s3-uri
+// re-stage when the recorded snapshot no longer exists (host rebuild, volume
+// GC): stage must not short-circuit on the stale KV record. It must
+// re-materialise, write the new snapshot, and name the missing snapshot it
+// replaced -- distinctly from the different-source-URI replace wording --
+// so an operator never has to run 'weights remove' first.
+func TestRunStageWeights_SelfHealsWhenSnapshotGone(t *testing.T) {
+	weightsStore := newWeightsStoreForTest(t)
+	ctx := context.Background()
+	require.NoError(t, weightsStore.PutWeights(ctx, selfHostModelID, "s3://models/llama-3.2-1b/", "snap-missing"))
+
+	store := objectstore.NewMemoryObjectStore()
+	seedCompleteWeightsPrefix(t, store, "models", "llama-3.2-1b/")
+
+	materializeCalled := false
+	materialize := func(context.Context, string, int64) (string, error) {
+		materializeCalled = true
+		return "snap-healed", nil
+	}
+
+	msg, err := runStageWeights(ctx, store, weightsStore, t.TempDir(),
+		selfHostModelID, "s3://models/llama-3.2-1b/", materialize, fixedSnapshotChecker(false))
+
+	require.NoError(t, err)
+	assert.True(t, materializeCalled, "self-heal must re-materialize rather than short-circuit")
+	assert.Contains(t, msg, "MISSING")
+	assert.Contains(t, msg, "snap-missing")
+	assert.Contains(t, msg, "snap-healed")
+
+	entry, ok, err := weightsStore.GetWeights(ctx, selfHostModelID)
+	require.NoError(t, err)
+	require.True(t, ok)
+	assert.Equal(t, "snap-healed", entry.SnapshotID, "PutWeights must record the new snapshot")
+}
+
+// TestRunStageWeights_LiveCheckErrorSurfaces covers the checker's own error
+// path: a real liveness-check failure (as opposed to a definitive not-found)
+// must be returned rather than guessed at either way.
+func TestRunStageWeights_LiveCheckErrorSurfaces(t *testing.T) {
+	weightsStore := newWeightsStoreForTest(t)
+	ctx := context.Background()
+	require.NoError(t, weightsStore.PutWeights(ctx, selfHostModelID, "s3://models/llama-3.2-1b/", "snap-0001"))
+
+	checkErr := errors.New("predastore unreachable")
+	checker := func(context.Context, string) (bool, error) { return false, checkErr }
+
+	_, err := runStageWeights(ctx, explodingObjectStore{t: t}, weightsStore, t.TempDir(),
+		selfHostModelID, "s3://models/llama-3.2-1b/", explodingMaterializer(t), checker)
+
+	require.Error(t, err)
+	assert.ErrorIs(t, err, checkErr)
+}
+
 // TestRunStageWeights_ReStageDifferentSourceReplacesAndReportsPrevious
 // covers a re-stage from a new source: the KV entry is replaced with the new
 // source/snapshot, and the result reports the previous snapshot ID so an
@@ -300,7 +374,7 @@ func TestRunStageWeights_ReStageDifferentSourceReplacesAndReportsPrevious(t *tes
 	materialize := func(context.Context, string, int64) (string, error) { return "snap-new", nil }
 
 	msg, err := runStageWeights(ctx, store, weightsStore, t.TempDir(),
-		selfHostModelID, "s3://models/llama-3.2-1b-new/", materialize)
+		selfHostModelID, "s3://models/llama-3.2-1b-new/", materialize, explodingSnapshotChecker(t))
 
 	require.NoError(t, err)
 	assert.Contains(t, msg, "snap-new")
@@ -320,7 +394,7 @@ func TestRunStageWeights_UnknownModelIDRefused(t *testing.T) {
 	weightsStore := newWeightsStoreForTest(t)
 
 	_, err := runStageWeights(context.Background(), explodingObjectStore{t: t}, weightsStore, t.TempDir(),
-		"not-a-real-model", "s3://models/llama-3.2-1b/", explodingMaterializer(t))
+		"not-a-real-model", "s3://models/llama-3.2-1b/", explodingMaterializer(t), explodingSnapshotChecker(t))
 
 	require.Error(t, err)
 	assert.ErrorContains(t, err, "unknown model ID")
@@ -339,7 +413,7 @@ func TestRunStageWeights_ProviderModelIDRefused(t *testing.T) {
 	weightsStore := newWeightsStoreForTest(t)
 
 	_, err := runStageWeights(context.Background(), explodingObjectStore{t: t}, weightsStore, t.TempDir(),
-		providerModelID, "s3://models/claude/", explodingMaterializer(t))
+		providerModelID, "s3://models/claude/", explodingMaterializer(t), explodingSnapshotChecker(t))
 
 	require.Error(t, err)
 	assert.ErrorContains(t, err, "provider-served")
@@ -360,7 +434,7 @@ func TestRunStageWeights_MissingRequiredFilesRefusedBeforeMaterializing(t *testi
 	putObject(t, store, "models", "incomplete/README.md", []byte("nothing useful"))
 
 	_, err := runStageWeights(context.Background(), store, weightsStore, t.TempDir(),
-		selfHostModelID, "s3://models/incomplete/", explodingMaterializer(t))
+		selfHostModelID, "s3://models/incomplete/", explodingMaterializer(t), explodingSnapshotChecker(t))
 
 	require.Error(t, err)
 	assert.ErrorContains(t, err, "missing required file")
@@ -374,7 +448,7 @@ func TestRunStageWeights_MissingRequiredFilesRefusedBeforeMaterializing(t *testi
 func TestListWeightsOutput_NoModelsStaged(t *testing.T) {
 	weightsStore := newWeightsStoreForTest(t)
 
-	out, err := listWeightsOutput(context.Background(), weightsStore)
+	out, err := listWeightsOutput(context.Background(), weightsStore, explodingSnapshotChecker(t))
 	require.NoError(t, err)
 	assert.Equal(t, "No models staged.", out)
 }
@@ -388,7 +462,7 @@ func TestListWeightsOutput_ListsSeveralStagedModels(t *testing.T) {
 	require.NoError(t, weightsStore.PutWeights(ctx, selfHostModelID, "s3://models/llama-3.2-1b/", "snap-0001"))
 	require.NoError(t, weightsStore.PutWeights(ctx, "amazon.titan-text-lite-v1", "s3://models/titan-text-lite/", "snap-0002"))
 
-	out, err := listWeightsOutput(ctx, weightsStore)
+	out, err := listWeightsOutput(ctx, weightsStore, fixedSnapshotChecker(true))
 	require.NoError(t, err)
 	assert.Contains(t, out, selfHostModelID)
 	assert.Contains(t, out, "s3://models/llama-3.2-1b/")
@@ -398,6 +472,38 @@ func TestListWeightsOutput_ListsSeveralStagedModels(t *testing.T) {
 	assert.Contains(t, out, "snap-0002")
 }
 
+// TestListWeightsOutput_MarksDanglingEntry covers the annotation this bug is
+// about: an entry whose snapshot is gone must be marked dangling, distinctly
+// from a live entry, so an operator sees at a glance which staged models can
+// actually serve.
+func TestListWeightsOutput_MarksDanglingEntry(t *testing.T) {
+	weightsStore := newWeightsStoreForTest(t)
+	ctx := context.Background()
+	require.NoError(t, weightsStore.PutWeights(ctx, selfHostModelID, "s3://models/llama-3.2-1b/", "snap-live"))
+	require.NoError(t, weightsStore.PutWeights(ctx, "amazon.titan-text-lite-v1", "s3://models/titan-text-lite/", "snap-gone"))
+
+	checker := func(_ context.Context, snapshotID string) (bool, error) {
+		return snapshotID != "snap-gone", nil
+	}
+
+	out, err := listWeightsOutput(ctx, weightsStore, checker)
+	require.NoError(t, err)
+
+	var liveLine, danglingLine string
+	for line := range strings.SplitSeq(out, "\n") {
+		switch {
+		case strings.Contains(line, "snap-live"):
+			liveLine = line
+		case strings.Contains(line, "snap-gone"):
+			danglingLine = line
+		}
+	}
+	require.NotEmpty(t, liveLine, "live entry must appear in output")
+	require.NotEmpty(t, danglingLine, "dangling entry must appear in output")
+	assert.NotContains(t, liveLine, weightsStatusDangling, "a live snapshot must not be marked dangling")
+	assert.Contains(t, danglingLine, weightsStatusDangling, "a missing snapshot must be marked dangling")
+}
+
 // TestRemoveWeights_RoundTrip covers remove: it returns the dropped entry,
 // and afterward the model has no staged-weights record. It must not delete
 // anything outside the KV bucket -- removeWeights takes no object store at
@@ -643,12 +749,36 @@ func TestRunOchreWeightsList_ConnectFailureExits1(t *testing.T) {
 	assert.Equal(t, 1, code)
 }
 
+// notFoundPredastoreServer answers every request with a 404 NoSuchKey S3
+// error, so a weightsSnapshotChecker built against it deterministically
+// reports a snapshot as gone without needing a real predastore.
+func notFoundPredastoreServer(t *testing.T) *httptest.Server {
+	t.Helper()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/xml")
+		w.WriteHeader(http.StatusNotFound)
+		io.WriteString(w, `<?xml version="1.0" encoding="UTF-8"?>`+
+			`<Error><Code>NoSuchKey</Code>`+
+			`<Message>The specified key does not exist.</Message></Error>`)
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
 // TestRunOchreWeightsList_PrintsStagedModels drives the real Run function
 // end to end: it must print what a separately-constructed WeightsStore
-// against the same embedded JetStream server sees.
+// against the same embedded JetStream server sees, and mark the entry
+// dangling since the mock predastore below has no such snapshot.
 func TestRunOchreWeightsList_PrintsStagedModels(t *testing.T) {
 	_, nc, js := testutil.StartTestJetStream(t)
-	stubConnect(t, fakeClusterConfig(), nc, nil)
+	srv := notFoundPredastoreServer(t)
+	cfg := &config.ClusterConfig{
+		Node: "node1",
+		Nodes: map[string]config.Config{
+			"node1": {Predastore: config.PredastoreConfig{Host: srv.URL, Region: "us-east-1", Bucket: "test-bucket", AccessKey: "test", SecretKey: "test"}},
+		},
+	}
+	stubConnect(t, cfg, nc, nil)
 
 	seedStore := gateway_bedrock.NewWeightsStore(js, 1)
 	require.NoError(t, seedStore.PutWeights(context.Background(), selfHostModelID, "s3://models/llama-3.2-1b/", "snap-0001"))
@@ -656,6 +786,7 @@ func TestRunOchreWeightsList_PrintsStagedModels(t *testing.T) {
 	out := captureStdout(t, func() { runOchreWeightsList(nil, nil) })
 	assert.Contains(t, out, selfHostModelID)
 	assert.Contains(t, out, "snap-0001")
+	assert.Contains(t, out, weightsStatusDangling, "snapshot absent from the mock predastore must be marked dangling")
 }
 
 // TestRunOchreWeightsList_PrintsNoModelsStaged covers the empty case through
@@ -1000,3 +1131,161 @@ func TestRegisterWeightsSnapshot_WriteFailureSurfaces(t *testing.T) {
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "register snapshot metadata")
 }
+
+// ochreFlagCmd builds a bare command carrying the model-selection flags, so
+// the resolver can be exercised without the cluster connection the real
+// subcommands make.
+func ochreFlagCmd(t *testing.T, modelID string, allModels bool) *cobra.Command {
+	t.Helper()
+	cmd := &cobra.Command{}
+	cmd.Flags().String("model-id", modelID, "")
+	cmd.Flags().Bool("all-models", allModels, "")
+	return cmd
+}
+
+func TestOchreTargetModels_SingleModel(t *testing.T) {
+	models, err := ochreTargetModels(ochreFlagCmd(t, selfHostModelID, false))
+	require.NoError(t, err)
+	assert.Equal(t, []string{selfHostModelID}, models)
+}
+
+func TestOchreTargetModels_AllModels(t *testing.T) {
+	models, err := ochreTargetModels(ochreFlagCmd(t, "", true))
+	require.NoError(t, err)
+	assert.Equal(t, gateway_bedrock.CatalogModelIDs(), models)
+	assert.NotEmpty(t, models)
+}
+
+// TestOchreTargetModels_NeitherFlag pins the refusal to guess: silently
+// defaulting to the whole catalog would over-grant on a typo.
+func TestOchreTargetModels_NeitherFlag(t *testing.T) {
+	_, err := ochreTargetModels(ochreFlagCmd(t, "", false))
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "--model-id or --all-models")
+}
+
+func TestOchreTargetModels_BothFlags(t *testing.T) {
+	_, err := ochreTargetModels(ochreFlagCmd(t, selfHostModelID, true))
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "mutually exclusive")
+}
+
+// ochreAccessCmd builds a command carrying the access subcommands' full flag
+// set, so the grant/revoke/list runners can be driven without the cobra
+// command tree.
+func ochreAccessCmd(t *testing.T, accountID, modelID string, allModels bool) *cobra.Command {
+	t.Helper()
+	cmd := ochreFlagCmd(t, modelID, allModels)
+	cmd.Flags().String("account-id", accountID, "")
+	return cmd
+}
+
+const accessTestAccount = "000000000001"
+
+// stubAccessConnect dials a fresh connection per call, because each access
+// runner closes the connection it was handed; a single shared one would leave
+// the next runner in a sequence talking to a closed conn.
+func stubAccessConnect(t *testing.T, clientURL string) {
+	t.Helper()
+	orig := loadConfigAndConnectFn
+	t.Cleanup(func() { loadConfigAndConnectFn = orig })
+	loadConfigAndConnectFn = func() (*config.ClusterConfig, *nats.Conn, error) {
+		nc, err := nats.Connect(clientURL)
+		if err != nil {
+			return nil, nil, err
+		}
+		t.Cleanup(nc.Close)
+		return fakeClusterConfig(), nc, nil
+	}
+}
+
+// TestRunOchreAccessChange_ModelFlagFailureExits1 pins that the flag refusal
+// happens before any cluster connection: an operator who mistypes a flag gets
+// told so without a NATS dial.
+func TestRunOchreAccessChange_ModelFlagFailureExits1(t *testing.T) {
+	stubConnect(t, nil, nil, errors.New("connect must not be reached"))
+
+	cmd := ochreAccessCmd(t, accessTestAccount, "", false)
+	code := withOchreExitCapture(t, func() { runOchreAccessGrant(cmd, nil) })
+	assert.Equal(t, 1, code)
+}
+
+func TestRunOchreAccessGrant_ConnectFailureExits1(t *testing.T) {
+	stubConnect(t, nil, nil, errors.New("dial failed"))
+
+	cmd := ochreAccessCmd(t, accessTestAccount, selfHostModelID, false)
+	code := withOchreExitCapture(t, func() { runOchreAccessGrant(cmd, nil) })
+	assert.Equal(t, 1, code)
+}
+
+func TestRunOchreAccessList_ConnectFailureExits1(t *testing.T) {
+	stubConnect(t, nil, nil, errors.New("dial failed"))
+
+	cmd := ochreAccessCmd(t, accessTestAccount, "", false)
+	code := withOchreExitCapture(t, func() { runOchreAccessList(cmd, nil) })
+	assert.Equal(t, 1, code)
+}
+
+// TestRunOchreAccessGrantListRevoke drives the three runners in sequence
+// against an embedded JetStream server, so the CLI is checked against the same
+// grant store the gateway resolves against rather than a fake.
+func TestRunOchreAccessGrantListRevoke(t *testing.T) {
+	srv, _, _ := testutil.StartTestJetStream(t)
+	stubAccessConnect(t, srv.ClientURL())
+
+	grantCmd := ochreAccessCmd(t, accessTestAccount, selfHostModelID, false)
+	out := captureStdout(t, func() { runOchreAccessGrant(grantCmd, nil) })
+	assert.Contains(t, out, "Granted")
+	assert.Contains(t, out, selfHostModelID)
+
+	listCmd := ochreAccessCmd(t, accessTestAccount, "", false)
+	out = captureStdout(t, func() { runOchreAccessList(listCmd, nil) })
+	assert.Contains(t, out, selfHostModelID)
+	assert.NotContains(t, out, providerModelID, "list must show only the granted model")
+
+	revokeCmd := ochreAccessCmd(t, accessTestAccount, selfHostModelID, false)
+	out = captureStdout(t, func() { runOchreAccessRevoke(revokeCmd, nil) })
+	assert.Contains(t, out, "Revoked")
+
+	out = captureStdout(t, func() { runOchreAccessList(listCmd, nil) })
+	assert.Contains(t, out, "no model grants")
+}
+
+// TestRunOchreAccessGrant_AllModelsGrantsWholeCatalog covers the --all-models
+// fan-out, which is what provisioning uses.
+func TestRunOchreAccessGrant_AllModelsGrantsWholeCatalog(t *testing.T) {
+	srv, _, _ := testutil.StartTestJetStream(t)
+	stubAccessConnect(t, srv.ClientURL())
+
+	grantCmd := ochreAccessCmd(t, accessTestAccount, "", true)
+	captureStdout(t, func() { runOchreAccessGrant(grantCmd, nil) })
+
+	listCmd := ochreAccessCmd(t, accessTestAccount, "", false)
+	out := captureStdout(t, func() { runOchreAccessList(listCmd, nil) })
+	for _, modelID := range gateway_bedrock.CatalogModelIDs() {
+		assert.Contains(t, out, modelID)
+	}
+}
+
+// TestRunOchreAccessList_OtherAccountSeesNothing is the deny-by-default rule
+// as an operator observes it: an account nobody granted has an empty listing.
+func TestRunOchreAccessList_OtherAccountSeesNothing(t *testing.T) {
+	srv, _, _ := testutil.StartTestJetStream(t)
+	stubAccessConnect(t, srv.ClientURL())
+
+	grantCmd := ochreAccessCmd(t, accessTestAccount, selfHostModelID, false)
+	captureStdout(t, func() { runOchreAccessGrant(grantCmd, nil) })
+
+	listCmd := ochreAccessCmd(t, "000000000002", "", false)
+	out := captureStdout(t, func() { runOchreAccessList(listCmd, nil) })
+	assert.Contains(t, out, "no model grants")
+}
+
+// TestRunOchreModels_PrintsSortedCatalog checks the listing operators consult
+// before granting: every catalog ID, one per line.
+func TestRunOchreModels_PrintsSortedCatalog(t *testing.T) {
+	out := captureStdout(t, func() { runOchreModels(nil, nil) })
+	for _, modelID := range gateway_bedrock.CatalogModelIDs() {
+		assert.Contains(t, out, modelID)
+	}
+}

cmd/spinifex/cmd/admin_preflight.go

@@ -0,0 +1,50 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"text/tabwriter"
+
+	"github.com/mulgadc/spinifex/spinifex/preflight"
+	"github.com/spf13/cobra"
+)
+
+var adminPreflightQuiet bool
+
+var adminPreflightCmd = &cobra.Command{
+	Use:   "preflight",
+	Short: "Verify this host's managed assets against what this build expects",
+	Long: `spx knows its own build (spx version) but not whether this host already
+has the privileged helper scripts and sudoers grants that build depends on —
+setup.sh and setup-ovn.sh install them, and a binary-swap deploy (make
+deploy or a manual binary copy) skips those steps.
+
+Checks compare content, not mere existence, so a present-but-stale asset is
+caught too. Exits non-zero if any asset is Missing, Stale, or Ungranted.`,
+	RunE:         runAdminPreflight,
+	SilenceUsage: true,
+}
+
+func init() {
+	adminCmd.AddCommand(adminPreflightCmd)
+	adminPreflightCmd.Flags().BoolVar(&adminPreflightQuiet, "quiet", false, "print only non-OK rows")
+}
+
+func runAdminPreflight(_ *cobra.Command, _ []string) error {
+	results := preflight.CheckHost()
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0)
+	fmt.Fprintln(w, "PATH\tKIND\tSTATUS\tDETAIL")
+	for _, r := range results {
+		if adminPreflightQuiet && r.Status == preflight.OK {
+			continue
+		}
+		fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", r.Path, r.Kind, r.Status, r.Detail)
+	}
+	w.Flush()
+
+	if preflight.HasProblem(results) {
+		return fmt.Errorf("host-asset preflight found problems — see table above")
+	}
+	return nil
+}

cmd/spinifex/cmd/admin_test.go

@@ -316,7 +316,7 @@ func TestPredastoreMultinodeTemplate_NatsURLLoopbackShim(t *testing.T) {
 		{ID: 3, Host: "10.0.0.3"},
 	}
 	content, err := admin.GenerateMultiNodePredastoreConfig(
-		predastoreMultiNodeTemplate, nodes, "AK", "SK", "ap-southeast-2", "token", "/tmp", "0.0.0.0", 0, admin.NorthstarCredentials{},
+		predastoreMultiNodeTemplate, nodes, "AK", "SK", "ap-southeast-2", "token", "/tmp", "/var/lib/spinifex", "0.0.0.0", 0, admin.NorthstarCredentials{},
 	)
 	require.NoError(t, err)
 	assert.Contains(t, content, `nats_url = "nats://localhost:4222"`)
@@ -324,7 +324,7 @@ func TestPredastoreMultinodeTemplate_NatsURLLoopbackShim(t *testing.T) {
 
 	// Specific bind IP → stays as-is.
 	content2, err := admin.GenerateMultiNodePredastoreConfig(
-		predastoreMultiNodeTemplate, nodes, "AK", "SK", "ap-southeast-2", "token", "/tmp", "10.11.12.1", 0, admin.NorthstarCredentials{},
+		predastoreMultiNodeTemplate, nodes, "AK", "SK", "ap-southeast-2", "token", "/tmp", "/var/lib/spinifex", "10.11.12.1", 0, admin.NorthstarCredentials{},
 	)
 	require.NoError(t, err)
 	assert.Contains(t, content2, `nats_url = "nats://10.11.12.1:4222"`)
@@ -813,7 +813,7 @@ func (c predastoreBucketAuth) hasBucket(name string) bool {
 // them.
 func TestPredastoreMultinodeTemplate_NorthstarProvisioned(t *testing.T) {
 	content, err := admin.GenerateMultiNodePredastoreConfig(
-		predastoreMultiNodeTemplate, predastoreMultinodeNodes(), "AK", "SK", "ap-southeast-2", "tok", "/cfg", "10.0.0.1", 0,
+		predastoreMultiNodeTemplate, predastoreMultinodeNodes(), "AK", "SK", "ap-southeast-2", "tok", "/cfg", "/var/lib/spinifex", "10.0.0.1", 0,
 		admin.NorthstarCredentials{AccessKey: "NSAK", SecretKey: "NSSK", Bucket: admin.NorthstarBucketName},
 	)
 	require.NoError(t, err)
@@ -857,7 +857,7 @@ func TestPredastoreMultinodeTemplate_NorthstarProvisioned(t *testing.T) {
 // zone bucket.
 func TestPredastoreMultinodeTemplate_NorthstarOmittedWithoutCredentials(t *testing.T) {
 	content, err := admin.GenerateMultiNodePredastoreConfig(
-		predastoreMultiNodeTemplate, predastoreMultinodeNodes(), "AK", "SK", "ap-southeast-2", "tok", "/cfg", "10.0.0.1", 0,
+		predastoreMultiNodeTemplate, predastoreMultinodeNodes(), "AK", "SK", "ap-southeast-2", "tok", "/cfg", "/var/lib/spinifex", "10.0.0.1", 0,
 		admin.NorthstarCredentials{},
 	)
 	require.NoError(t, err)
@@ -1067,7 +1067,7 @@ func TestPredastoreTemplates_UnsetCompactionEmitsNoBlock(t *testing.T) {
 		"unset single-node tail changed: %q", single[len(single)-40:])
 
 	multi, err := admin.GenerateMultiNodePredastoreConfig(
-		predastoreMultiNodeTemplate, predastoreMultinodeNodes(), "AK", "SK", "ap-southeast-2", "tok", "/cfg", "10.0.0.1", 0, admin.NorthstarCredentials{},
+		predastoreMultiNodeTemplate, predastoreMultinodeNodes(), "AK", "SK", "ap-southeast-2", "tok", "/cfg", "/var/lib/spinifex", "10.0.0.1", 0, admin.NorthstarCredentials{},
 	)
 	require.NoError(t, err)
 	assert.NotContains(t, multi, "[compaction]")
@@ -1088,7 +1088,7 @@ func TestPredastoreTemplates_SetCompactionEmitsParseableBlock(t *testing.T) {
 	assert.Equal(t, interval, singleCfg.Compaction.IntervalSeconds)
 
 	multi, err := admin.GenerateMultiNodePredastoreConfig(
-		predastoreMultiNodeTemplate, predastoreMultinodeNodes(), "AK", "SK", "ap-southeast-2", "tok", "/cfg", "10.0.0.1", interval, admin.NorthstarCredentials{},
+		predastoreMultiNodeTemplate, predastoreMultinodeNodes(), "AK", "SK", "ap-southeast-2", "tok", "/cfg", "/var/lib/spinifex", "10.0.0.1", interval, admin.NorthstarCredentials{},
 	)
 	require.NoError(t, err)
 	var multiCfg compactionConfig

cmd/spinifex/cmd/admin_upgrade.go

@@ -7,6 +7,11 @@ import (
 	"strings"
 
 	"github.com/mulgadc/spinifex/spinifex/migrate"
+
+	// Config migrations that need more than the framework register themselves
+	// here rather than in migrate, which every handler imports.
+	_ "github.com/mulgadc/spinifex/spinifex/migrate/predastoretopology"
+
 	"github.com/spf13/cobra"
 )
 

cmd/spinifex/cmd/admin_upgrade_test.go

@@ -0,0 +1,39 @@
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/migrate"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestUpgradeSeesEveryConfigTarget pins the registrations `spx admin upgrade`
+// depends on. A migration whose package is no longer linked here would leave
+// the command reporting nothing pending on an install that needs migrating.
+func TestUpgradeSeesEveryConfigTarget(t *testing.T) {
+	configDir := t.TempDir()
+
+	// Both targets must exist on disk to be reported at all.
+	require.NoError(t, os.MkdirAll(filepath.Join(configDir, "predastore"), 0750))
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640))
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "predastore", "predastore.toml"), []byte("version = \"4\"\n"), 0640))
+
+	versions, err := migrate.DefaultRegistry.ConfigVersions(configDir)
+	require.NoError(t, err)
+	assert.Equal(t, map[string]int{"spinifex.toml": 3, "predastore.toml": 4}, versions)
+
+	pending, err := migrate.DefaultRegistry.PendingConfig(configDir)
+	require.NoError(t, err)
+
+	targets := make(map[string][2]int, len(pending))
+	for _, p := range pending {
+		targets[p.Target] = [2]int{p.FromVersion, p.ToVersion}
+	}
+	assert.Equal(t, map[string][2]int{
+		"spinifex.toml":   {3, 4},
+		"predastore.toml": {4, 5},
+	}, targets)
+}

cmd/spinifex/cmd/predastore_bind_test.go

@@ -28,7 +28,7 @@ func writeSpinifexToml(t *testing.T, body string) string {
 	return path
 }
 
-func TestDerivePredastoreBind_NodeIDConfigured(t *testing.T) {
+func TestDerivePredastoreBind_HostIDConfigured(t *testing.T) {
 	resetGlobalViper(t)
 
 	path := writeSpinifexToml(t, `
@@ -45,7 +45,7 @@ az = "us-east-1a"
 host = "0.0.0.0:8443"
 bucket = "predastore"
 region = "us-east-1"
-node_id = 3
+host_id = 3
 `)
 
 	clusterConfig, err := config.LoadConfig(path)
@@ -55,16 +55,14 @@ node_id = 3
 	require.NoError(t, err)
 	assert.Equal(t, "0.0.0.0", bind.Host)
 	assert.Equal(t, 8443, bind.Port)
-	assert.Equal(t, 3, bind.NodeID)
+	assert.Equal(t, 3, bind.HostID)
 }
 
-// TestDerivePredastoreBind_NodeIDAbsentDefaultsToColocated covers the
-// co-located NODE_ID=-1 rule: single-node spinifex.toml templates omit
-// node_id entirely, since every configured DB peer runs in this one process.
-// predastore-start.sh defaulted to -1 for exactly that reason, and predastore
-// itself rejects node_id=0, so an absent key must never silently resolve to
-// the Go zero value instead.
-func TestDerivePredastoreBind_NodeIDAbsentDefaultsToColocated(t *testing.T) {
+// TestDerivePredastoreBind_HostIDAbsentRunsWholeTopology covers the
+// single-node rule: those spinifex.toml templates omit host_id entirely,
+// pinning every node of the topology to one host, which only a
+// whole-topology process can serve.
+func TestDerivePredastoreBind_HostIDAbsentRunsWholeTopology(t *testing.T) {
 	resetGlobalViper(t)
 
 	path := writeSpinifexToml(t, `
@@ -90,7 +88,7 @@ region = "us-east-1"
 	require.NoError(t, err)
 	assert.Equal(t, "0.0.0.0", bind.Host)
 	assert.Equal(t, 8443, bind.Port)
-	assert.Equal(t, -1, bind.NodeID)
+	assert.Equal(t, 0, bind.HostID)
 }
 
 // TestDerivePredastoreBind_BindHostSurvivesLoadConfigNormalization asserts

cmd/spinifex/cmd/service.go

@@ -89,12 +89,12 @@ var spinifexUICmd = &cobra.Command{
 	Short:   "Manage the spinifex-ui service",
 }
 
-// predastoreBind is the local predastore bind host, port and node_id derived
+// predastoreBind is the local predastore bind host, port and host_id derived
 // directly from spinifex.toml.
 type predastoreBind struct {
 	Host   string
 	Port   int
-	NodeID int
+	HostID int
 }
 
 // derivePredastoreBind reads this node's [nodes.<node>.predastore] section
@@ -104,9 +104,9 @@ type predastoreBind struct {
 // normalization for callers that DIAL predastore, not for the address
 // predastore itself binds to.
 //
-// node_id defaults to -1 (co-located: every configured DB peer runs in this
-// one process) when spinifex.toml omits the key. predastore rejects
-// node_id=0, so an absent key must never silently resolve to that.
+// host_id defaults to 0 when spinifex.toml omits the key, which runs the whole
+// predastore topology in this one process — the single-node deployment. Only a
+// multi-node config names a host (>= 1), selecting just that host's nodes.
 func derivePredastoreBind(clusterConfig *config.ClusterConfig) (predastoreBind, error) {
 	node := clusterConfig.Node
 	bindKey := "nodes." + node + ".predastore.host"
@@ -124,12 +124,7 @@ func derivePredastoreBind(clusterConfig *config.ClusterConfig) (predastoreBind,
 		return predastoreBind{}, fmt.Errorf("parse nodes.%s.predastore.host port %q: %w", node, portStr, err)
 	}
 
-	nodeID := -1
-	if viper.IsSet("nodes." + node + ".predastore.node_id") {
-		nodeID = clusterConfig.Nodes[node].Predastore.NodeID
-	}
-
-	return predastoreBind{Host: host, Port: port, NodeID: nodeID}, nil
+	return predastoreBind{Host: host, Port: port, HostID: clusterConfig.Nodes[node].Predastore.HostID}, nil
 }
 
 var predastoreStartCmd = &cobra.Command{
@@ -144,9 +139,9 @@ var predastoreStartCmd = &cobra.Command{
 		host := viper.GetString("predastore-host")
 		basePath := viper.GetString("predastore-base-path")
 		debug := viper.GetBool("predastore-debug")
-		nodeID := viper.GetInt("predastore-node-id")
+		hostID := viper.GetInt("predastore-host-id")
 
-		// Derive bind host/port/node-id from spinifex.toml when its path is
+		// Derive bind host/port/host-id from spinifex.toml when its path is
 		// known and the caller hasn't explicitly overridden them — replaces
 		// predastore-start.sh, which used to do this derivation and exec us.
 		if cfgFile := viper.GetString("config"); cfgFile != "" {
@@ -166,8 +161,8 @@ var predastoreStartCmd = &cobra.Command{
 			if !viper.IsSet("predastore-port") {
 				port = bind.Port
 			}
-			if !viper.IsSet("predastore-node-id") {
-				nodeID = bind.NodeID
+			if !viper.IsSet("predastore-host-id") {
+				hostID = bind.HostID
 			}
 		}
 
@@ -221,7 +216,7 @@ var predastoreStartCmd = &cobra.Command{
 
 			EncryptionKeyFile: encryptionKeyFile,
 
-			NodeID: nodeID,
+			HostID: hostID,
 
 			PprofEnabled:    pprofEnabled,
 			PprofOutputPath: pprofOutput,
@@ -1034,7 +1029,6 @@ func runNorthstarStart(options northstarStartOptions, deps northstarStartDepende
 	svc, err := deps.newService(&northstar.Config{
 		ConfigPath: configPath,
 		BasePath:   baseDir,
-		NodeID:     nodeConfig.Predastore.NodeID,
 		NatsHost:   nodeConfig.NATS.Host,
 		NatsToken:  nodeConfig.NATS.ACL.Token,
 		NatsCACert: nodeConfig.NATS.CACert,
@@ -1182,7 +1176,7 @@ func bindPredastoreNamespacedEnv() {
 }
 
 // bindPredastoreCollisionEnv namespaces predastore's port, debug, tls-cert,
-// tls-key, encryption-key-file and node-id keys, which nats and awsgw also
+// tls-key, encryption-key-file and host-id keys, which nats and awsgw also
 // bind bare. Each derived env name now matches its own BindEnv target.
 func bindPredastoreCollisionEnv() {
 	viper.BindEnv("predastore-port", "SPINIFEX_PREDASTORE_PORT")
@@ -1200,8 +1194,8 @@ func bindPredastoreCollisionEnv() {
 	viper.BindEnv("predastore-encryption-key-file", "SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE")
 	viper.BindPFlag("predastore-encryption-key-file", predastoreCmd.PersistentFlags().Lookup("encryption-key-file"))
 
-	viper.BindEnv("predastore-node-id", "SPINIFEX_PREDASTORE_NODE_ID")
-	viper.BindPFlag("predastore-node-id", predastoreCmd.PersistentFlags().Lookup("node-id"))
+	viper.BindEnv("predastore-host-id", "SPINIFEX_PREDASTORE_HOST_ID")
+	viper.BindPFlag("predastore-host-id", predastoreCmd.PersistentFlags().Lookup("host-id"))
 }
 
 // bindNatsCollisionEnv namespaces nats's port, host and debug keys, which
@@ -1281,12 +1275,13 @@ func init() {
 	// Predastore at-rest encryption master key (per node; mode 0600)
 	predastoreCmd.PersistentFlags().String("encryption-key-file", "", "Path to this node's 32-byte AES-256 master key file (required)")
 
-	// Predastore Node ID. Default -1 is dev mode (launch every configured
-	// QUIC node in-process). Production deployments set this to the node's
-	// real ID (>= 1) via SPINIFEX_PREDASTORE_NODE_ID or --node-id.
-	predastoreCmd.PersistentFlags().Int("node-id", -1, "Predastore (S3) node ID (-1 = dev mode, >= 1 = production)")
+	// Predastore host ID: which [[host]] of the predastore topology this
+	// process is. Default 0 runs every node of the topology in this process,
+	// which is the single-node deployment. Multi-node deployments set the
+	// host's real ID (>= 1) via SPINIFEX_PREDASTORE_HOST_ID or --host-id.
+	predastoreCmd.PersistentFlags().Int("host-id", 0, "Predastore cluster host ID (0 = run the whole topology in this process, >= 1 = this host only)")
 
-	// Namespaced viper keys for port/debug/tls-cert/tls-key/encryption-key-file/node-id
+	// Namespaced viper keys for port/debug/tls-cert/tls-key/encryption-key-file/host-id
 	// (see bindPredastoreCollisionEnv doc comment)
 	bindPredastoreCollisionEnv()
 

cmd/spinifex/cmd/templates/predastore-multinode.toml

@@ -1,10 +1,10 @@
 # Predastore S3-compatible storage configuration file in TOML format.
 # Generated for multi-node cluster
-version = "4"
+version = "5"
 region = "{{.Region}}"
 
-host = "0.0.0.0"
-port = 8443
+# The gateway listen address comes from the daemon CLI options: a top-level
+# host key would collide with the [[host]] topology tables below.
 
 debug = false
 
@@ -16,26 +16,28 @@ parity = 1
 data = 2
 parity = 1
 {{- end}}
+
+# One host per machine: a predastore process owning a socket and a data
+# directory. bind_addr binds every interface; public_addr is what the other
+# hosts dial.
 {{- range .Nodes}}
 
-[[db]]
+[[host]]
 id = {{.ID}}
-host = "{{.Host}}"
-port = 6660
-path = "distributed/db/node-{{.ID}}/"
-access_key_id = "{{$.AccessKey}}"
-secret_access_key = "{{$.SecretKey}}"
-{{- if eq .ID 1}}
-leader = true
+bind_addr = "0.0.0.0:6660"
+public_addr = "{{.Host}}:6660"
+data_dir = "{{$.PredastoreDataDir}}"
 {{- end}}
-{{- end}}
-{{- range .Nodes}}
 
-[[nodes]]
+# Nodes are roles pinned to a host: each machine carries one shard-storage
+# node for erasure-coded object shards and one state replica for the Raft
+# quorum over global state.
+{{- range .ClusterNodes}}
+
+[[node]]
 id = {{.ID}}
-host = "{{.Host}}"
-port = 9991
-path = "distributed/nodes/node-{{.ID}}/"
+host_id = {{.HostID}}
+role = "{{.Role}}"
 {{- end}}
 
 [[buckets]]

cmd/spinifex/cmd/templates/predastore.toml

@@ -1,11 +1,10 @@
 # Predastore S3-compatible storage configuration file in TOML format.
 # MULTI NODE used for testing
-version = "4"
+version = "5"
 region = "{{.Region}}"
 
-# Optionally configure port and host, or specify via daemon CLI options
-host = "0.0.0.0"
-port = 8443
+# The gateway listen address comes from the daemon CLI options: a top-level
+# host key would collide with the [[host]] topology table below.
 
 # Enable verbose debugging
 debug = false
@@ -20,52 +19,50 @@ debug = false
 data = 2
 parity = 1
 
-[[db]]
+# A host is one predastore process: it owns a socket and a data directory,
+# and runs the nodes pinned to it. A single-node deployment declares one host
+# and colocates every node in it, so all intra-cluster traffic goes over the
+# in-process pipe and no socket is bound. bind_addr/public_addr are unused
+# while everything is colocated; they only matter once nodes are split across
+# processes.
+[[host]]
 id = 1
-host = "127.0.0.1"
-port = 6660
-#raft_port = 7660       # Optional: Raft consensus port (default: port + 1000)
-#advertise_host = ""    # Optional: Address to advertise to other Raft nodes
-path = "distributed/db/node-1/"
-access_key_id = "{{.AccessKey}}"
-secret_access_key = "{{.SecretKey}}"
-leader = true      # Bootstrap leader hint
+bind_addr = "127.0.0.1:6660"
+public_addr = "127.0.0.1:6660"
+data_dir = "{{.PredastoreDataDir}}"
 
-[[db]]
-id = 2
-host = "127.0.0.1"
-port = 6661
-path = "distributed/db/node-2/"
-access_key_id = "{{.AccessKey}}"
-secret_access_key = "{{.SecretKey}}"
-
-[[db]]
-id = 3
-host = "127.0.0.1"
-port = 6662
-path = "distributed/db/node-3/"
-access_key_id = "{{.AccessKey}}"
-secret_access_key = "{{.SecretKey}}"
-
-# Shard nodes
-# Using distributed hash ring for object > shard placement.
-[[nodes]]
+# Shard nodes: the distributed hash ring places object shards across these.
+# RS(2,1) needs 3.
+[[node]]
 id = 1
-host = "127.0.0.1"
-port = 9991
-path = "distributed/nodes/node-1/"
+host_id = 1
+role = "shard-storage"
 
-[[nodes]]
+[[node]]
 id = 2
-host = "127.0.0.1"
-port = 9992
-path = "distributed/nodes/node-2/"
+host_id = 1
+role = "shard-storage"
 
-[[nodes]]
+[[node]]
 id = 3
-host = "127.0.0.1"
-port = 9993
-path = "distributed/nodes/node-3/"
+host_id = 1
+role = "shard-storage"
+
+# State replicas: the Raft quorum over global state (buckets, object index).
+[[node]]
+id = 4
+host_id = 1
+role = "state-replica"
+
+[[node]]
+id = 5
+host_id = 1
+role = "state-replica"
+
+[[node]]
+id = 6
+host_id = 1
+role = "state-replica"
 
 [[buckets]]
 name = "predastore"

cmd/spinifex/cmd/templates/spinifex.toml

@@ -1,5 +1,5 @@
 # Domain configuration file in TOML format.
-version = "3"
+version = "4"
 epoch = 1
 node = "{{.Node}}"
 {{- if .OperatorEmail}}
@@ -140,8 +140,8 @@ region = "{{.Region}}"
 bucket = "predastore"
 accesskey = "{{.AccessKey}}"
 secretkey = "{{.SecretKey}}"
-{{- if gt .PredastoreNodeID 0}}
-node_id = {{.PredastoreNodeID}}
+{{- if gt .PredastoreHostID 0}}
+host_id = {{.PredastoreHostID}}
 {{- end}}
 
 {{- if .NorthstarConfigPath}}

docs/COMMANDS.md

@@ -10,7 +10,7 @@ Service lifecycle commands for starting, stopping, and checking status of all Sp
 
 | Command | Flags | Description |
 |---------|-------|-------------|
-| `spx service predastore start` | `--port` (default: 8443), `--host` (default: 0.0.0.0), `--base-path`, `--config-path`, `--debug`, `--tls-cert`, `--tls-key`, `--backend` (distributed/filesystem, default: distributed), `--node-id` (default: -1), `--pprof`, `--pprof-output` | Creates predastore service instance with S3-compatible storage backend → starts service. When `--config`/`SPINIFEX_CONFIG_PATH` points at a cluster spinifex.toml, `--host`/`--port`/`--node-id` default to that node's `[nodes.<node>.predastore]` section instead of the flag defaults above; an explicit flag or `SPINIFEX_PREDASTORE_*` env var still overrides it. `node_id` absent from spinifex.toml resolves to -1 (co-located mode), never to 0. |
+| `spx service predastore start` | `--port` (default: 8443), `--host` (default: 0.0.0.0), `--base-path`, `--config-path`, `--debug`, `--tls-cert`, `--tls-key`, `--encryption-key-file`, `--host-id` (default: 0 = run the whole topology in this process), `--pprof`, `--pprof-output` | Creates predastore service instance with S3-compatible storage backend → starts service. When `--config`/`SPINIFEX_CONFIG_PATH` points at a cluster spinifex.toml, `--host`/`--port`/`--host-id` default to that node's `[nodes.<node>.predastore]` section instead of the flag defaults above; an explicit flag or `SPINIFEX_PREDASTORE_*` env var still overrides it. `host_id` absent from spinifex.toml resolves to 0, running the whole topology in this process. |
 | `spx service predastore stop` | — | Stops the predastore service |
 | `spx service predastore status` | — | Reports predastore service status |
 | `spx service viperblock start` | `--s3-host` (default: 0.0.0.0:8443), `--s3-bucket` (default: predastore), `--s3-region` (default: ap-southeast-2), `--plugin-path` (auto-detected via `nbdkit --dump-config plugindir`; typically `/usr/lib/x86_64-linux-gnu/nbdkit/plugins/nbdkit-viperblock-plugin.so` on amd64, overridable via `SPINIFEX_VIPERBLOCK_PLUGIN_PATH` in `/etc/spinifex/systemd.env`) | Loads cluster config → connects to NATS and Predastore → starts viperblock block storage service with NBD plugin |
@@ -119,8 +119,8 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 | Command | Flags | Description |
 |---------|-------|-------------|
-| `spx admin ochre weights stage` | `--model-id` (required), `--s3-uri` (required, `s3://bucket/prefix/`), `--tmp-dir` (default: OS temp dir) | Stages a self-host model's weights for serving. Refuses first if `--model-id` is not a self-host catalog entry (unknown or provider-served), or if `bedrock-weights` already has this model staged from the same `--s3-uri` (no-op). Then validates the prefix holds `config.json`, `tokenizer_config.json`, at least one `*.safetensors` file, and at least one of `tokenizer.json` or `tokenizer.model`, failing before downloading anything if any are missing. Downloads the prefix's objects from predastore into `--tmp-dir`, packs them into an ext4 image (`mkfs.ext4 -d`), imports it into a new viperblock volume via `v_utils.ImportDiskImage` with `AMIMetadata` left empty (plain volume, no AMI registration), snapshots the closed volume, and records the source URI and snapshot ID against `--model-id` in the `bedrock-weights` KV bucket. Re-staging a different `--s3-uri` replaces the entry and reports the previous snapshot ID for separate reclamation. |
-| `spx admin ochre weights list` | — | Lists every staged model with its source URI and snapshot ID, so an operator can see why a model is (or isn't) advertised via `ListFoundationModels`. |
+| `spx admin ochre weights stage` | `--model-id` (required), `--s3-uri` (required, `s3://bucket/prefix/`), `--tmp-dir` (default: OS temp dir) | Stages a self-host model's weights for serving. Refuses first if `--model-id` is not a self-host catalog entry (unknown or provider-served), or if `bedrock-weights` already has this model staged from the same `--s3-uri` and that snapshot still exists (no-op). If the recorded snapshot is gone (host rebuild, volume GC), it re-stages instead of refusing, so no manual `remove` is needed. Then validates the prefix holds `config.json`, `tokenizer_config.json`, at least one `*.safetensors` file, and at least one of `tokenizer.json` or `tokenizer.model`, failing before downloading anything if any are missing. Downloads the prefix's objects from predastore into `--tmp-dir`, packs them into an ext4 image (`mkfs.ext4 -d`), imports it into a new viperblock volume via `v_utils.ImportDiskImage` with `AMIMetadata` left empty (plain volume, no AMI registration), snapshots the closed volume, and records the source URI and snapshot ID against `--model-id` in the `bedrock-weights` KV bucket. Re-staging a different `--s3-uri` replaces the entry and reports the previous snapshot ID for separate reclamation. |
+| `spx admin ochre weights list` | — | Lists every staged model with its source URI, snapshot ID and a STATUS column, so an operator can see why a model is (or isn't) advertised via `ListFoundationModels`. STATUS reads `OK` when the recorded snapshot still exists, or `DANGLING (snapshot missing)` when the KV record survives but its snapshot is gone. |
 | `spx admin ochre weights remove` | `--model-id` (required) | Drops `--model-id`'s entry from `bedrock-weights`, hiding it from `ListFoundationModels` again. Refuses if the model has no staged entry. Never deletes the underlying snapshot or source S3 objects; reclaiming that storage is a separate, explicit act. |
 
 ### Ochre Serving Endpoints
@@ -134,6 +134,8 @@ Operator surface over the daemon's `bedrock.endpoint.*` subjects. The gateway do
 | `spx admin ochre endpoint list` | — | Lists every endpoint record with its state, instance ID and base URL. |
 | `spx admin ochre endpoint delete` | `--model-id` (required) | Moves a `READY` endpoint to `DRAINING` and tears its VM down, releasing the GPU. Idempotent: an already-absent endpoint reports success. |
 
+Self-hosted models are served from the `ubuntu-26.04-vllm-serving-x86_64` system image (`spx admin images import --name ubuntu-26.04-vllm-serving-x86_64`), built by `scripts/mkosi-build.sh --image ubuntu-vllm-serving` (`gpu-nvidia` + `vllm` profiles). It carries the tags `spinifex:managed-by=bedrock` and `spinifex:bedrock-role=vllm-serving`, which is what `LaunchServingVM` filters on rather than resolving by name. The image bakes vLLM's OpenAI-compatible server into a uv-managed venv — no Docker daemon, no runtime container pull — and boots `vllm-serve.service`, which waits for cloud-init's `/etc/conf.d/vllm-serve` handoff, resolves and mounts the endpoint's cloned weights volume read-only, then execs `vllm serve` on port 8000.
+
 ### EKS Control-Plane Disaster Recovery
 
 | Command | Flags | Description |
@@ -155,6 +157,7 @@ Operator surface over the daemon's `bedrock.endpoint.*` subjects. The gateway do
 | `describe-instance-types` | `--filters` (capacity only) | `--instance-types`, `--max-results`, `--next-token`, `--dry-run`, other filters | **DONE** |
 | `modify-instance-attribute` | `--instance-id`, `--instance-type`, `--user-data`, `--disable-api-termination` | `--ebs-optimized`, `--source-dest-check`, `--instance-initiated-shutdown-behavior`, `--block-device-mappings`, `--groups`, `--ena-support`, `--sriov-net-support` | **DONE** |
 | `get-console-output` | `--instance-id` | `--latest`, `--dry-run` | **DONE** |
+| `get-password-data` | `--instance-id` | `--dry-run` | **DONE** |
 | `describe-instance-attribute` | `--instance-id`, `--attribute` (instanceType, userData, disableApiTermination, instanceInitiatedShutdownBehavior, disableApiStop, ebsOptimized, enaSupport, sourceDestCheck, rootDeviceName, kernel, ramdisk) | `--dry-run` | **DONE** |
 | `modify-instance-metadata-options` | `--instance-id`, `--http-put-response-hop-limit` (1–64), `--http-tokens` (`required`/`optional`), `--http-endpoint` (`enabled`), `--http-protocol-ipv6`/`--instance-metadata-tags` (`disabled`) — unmodelled values return `UnsupportedOperation` | `--dry-run` | **DONE** |
 | `describe-instance-credit-specifications` | `--instance-ids` | `--filters`, `--max-results`, `--dry-run` | **DONE** (stub — always returns `standard`) |
@@ -338,8 +341,8 @@ KV CRUD only — no OVN/OVS integration. EIGWs are stored but have no effect on
 | `delete-network-interface` | `--network-interface-id` | `--dry-run` | **DONE** |
 | `describe-network-interfaces` | `--network-interface-ids`, `--filters` (subnet-id, vpc-id, attachment.instance-id) | `--max-results`, `--dry-run` | **DONE** |
 | `modify-network-interface-attribute` | — | `--network-interface-id`, `--description`, `--groups` | **DONE** |
-| `attach-network-interface` | — | `--network-interface-id`, `--instance-id`, `--device-index` | **NOT STARTED** (internal only) |
-| `detach-network-interface` | — | `--attachment-id`, `--force` | **NOT STARTED** (internal only) |
+| `attach-network-interface` | `--network-interface-id`, `--instance-id`, `--device-index` (all required) | `--dry-run`, `--network-card-index`, `--ena-srd-specification` | **DONE** — hot-plugged into a running instance |
+| `detach-network-interface` | `--attachment-id` (required), `--force` | `--dry-run` | **DONE** |
 | `assign-private-ip-addresses` | — | `--network-interface-id`, `--private-ip-addresses`, `--secondary-private-ip-address-count` | **NOT STARTED** |
 | `unassign-private-ip-addresses` | — | `--network-interface-id`, `--private-ip-addresses` | **NOT STARTED** |
 
@@ -797,6 +800,8 @@ The default certificate cannot be added/removed via these calls — set it on th
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
 | `describe-tags` | `--resource-arns` (loadbalancer, targetgroup, listener) | — | **DONE** |
+| `add-tags` | `--resource-arns`, `--tags` | — | **DONE** |
+| `remove-tags` | `--resource-arns`, `--tag-keys` | — | **DONE** |
 
 ### ELBv2 — Not Yet Implemented
 
@@ -825,14 +830,14 @@ Spinifex both stores externally-issued certificates (`import-certificate`) and i
 | `CNAME_DELEGATION` | operator once, then Spinifex | CNAME, stable | automatic | deferred — never selected yet (northstar cannot serve public authoritative queries); an ARN-stable delegation token is minted on every managed certificate now so this lands as a non-breaking addition |
 | `PRIVATE_CA` | nobody — no validation | none | automatic | **DONE** — issues synchronously against the tenant CA, no domain outside its name constraints |
 
-`PROVIDER_API` is selected when a DNS provider credential is configured; `MANUAL_TXT` when northstar hosts the zone; otherwise `PRIVATE_CA` — the only option for a deployment with no real, publicly delegated domain. Terraform's canonical `aws_acm_certificate` → `aws_route53_record` → `aws_acm_certificate_validation` → `aws_lb_listener` flow works unmodified in every mode: where Spinifex owns the record write, no `ResourceRecord` is emitted, so `for_each` over `domain_validation_options` yields zero records and `aws_acm_certificate_validation` still blocks correctly by polling until
-`ISSUED`.
+`PROVIDER_API` is selected when a DNS provider credential is configured; `MANUAL_TXT` when northstar hosts the zone; otherwise `PRIVATE_CA` — the only option for a deployment with no real, publicly delegated domain. Terraform's canonical `aws_acm_certificate` → `aws_route53_record` → `aws_acm_certificate_validation` → `aws_lb_listener` flow works unmodified in every mode: where Spinifex owns the record write, no `ResourceRecord` is emitted, so `for_each` over `domain_validation_options` yields zero records and `aws_acm_certificate_validation` still blocks correctly by polling until `ISSUED`.
 
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
 | `import-certificate` | `--certificate`, `--private-key`, `--certificate-chain`, `--certificate-arn` (re-import) | `--tags` | **DONE** |
 | `describe-certificate` | `--certificate-arn` | — | **DONE** |
 | `list-certificates` | — | `--certificate-statuses`, `--includes`, `--max-items`, `--next-token` | **DONE** |
+| `get-certificate` | `--certificate-arn` | — | **DONE** — returns the leaf certificate and its chain |
 | `delete-certificate` | `--certificate-arn` | — | **DONE** |
 | `request-certificate` | `--domain-name`, `--subject-alternative-names`, `--tags` | `--validation-method`, `--certificate-authority-arn`, `--options`, `--idempotency-token` | **PARTIAL** — `PRIVATE_CA` issues synchronously; `PROVIDER_API`/`MANUAL_TXT` are accepted and correctly shaped but stay `PENDING_VALIDATION` until a later issuance worker lands |
 | `add-tags-to-certificate` / `list-tags-for-certificate` / `remove-tags-from-certificate` | `--certificate-arn`, `--tags`/`--tag-keys` | — | **DONE** |
@@ -840,16 +845,35 @@ Spinifex both stores externally-issued certificates (`import-certificate`) and i
 
 ---
 
-## RDS (PostgreSQL)
+## RDS (PostgreSQL and MariaDB)
 
-Each DB instance is one dedicated system-owned VM running the engine directly, launched from the `spinifex-rds-postgres` AMI, tagged `spinifex:managed-by=rds` and therefore hidden from the customer's EC2 API. The engine is reached over a customer-account ENI injected into a subnet of the DB subnet group, so **the endpoint is private — reachable from inside the VPC only**. `Endpoint.Address` is `{db-instance-identifier}.{account-id}.{region}.rds.{base-domain}` where northstar is configured, and the endpoint ENI's private IP where it is not; the IP is stable across VM replacement either way. Default port 5432.
+Each DB instance is one dedicated system-owned VM running the engine directly, launched from the `spinifex-rds-postgres` or `spinifex-rds-mariadb` AMI, tagged `spinifex:managed-by=rds` and therefore hidden from the customer's EC2 API. The engine is reached over a customer-account ENI injected into a subnet of the DB subnet group, so **the endpoint is private — reachable from inside the VPC only**, from any subnet of it rather than the endpoint ENI's own. `Endpoint.Address` is `{db-instance-identifier}.{account-id}.{region}.rds.{base-domain}` where northstar is configured, and the endpoint ENI's private IP where it is not; the IP is stable across VM replacement either way.
 
-- **Engine:** `postgres` 18 only — `EngineVersion` accepts `18` or `18.x`, and there is no in-place upgrade.
+Everything below applies to both engines unless it names one. `Engine` is fixed at create — there is no in-place engine change, no cross-engine snapshot restore and no migration between the two.
+
+| | `postgres` | `mariadb` |
+|---|---|---|
+| Version | 18 | 11.8 |
+| `EngineVersion` accepted | `18`, or omitted | `11.8`, or omitted |
+| Default port | 5432 | 3306 |
+| Parameter-group family | `postgres18` | `mariadb11.8` |
+| Implicit default group | `default.postgres18` | `default.mariadb11.8` |
+| `DBName` maximum | 63 characters | 64 characters |
+| `MasterUsername` maximum | 63 characters | 80 characters |
+| Reserved usernames | `postgres`, `rdsadmin`, `rds_superuser`, and the `pg_` prefix | `root`, `mysql`, `mariadb.sys`, `rdsadmin`, `PUBLIC`, and the `mysql.` and `mariadb.` prefixes |
+| Catalog size | 51 parameters | 54 parameters |
+
+**`mysql` is not an accepted engine and is not an alias for `mariadb`.** MariaDB is offered under its own AWS engine name, exactly as AWS RDS offers it. Naming `mysql` fails with `InvalidParameterValue`; a client — including Terraform's `aws_db_instance` — must set `engine = "mariadb"`. Aliasing would report an engine and a version the instance is not running, and the discrepancy would propagate into `DescribeDBInstances`, parameter-group families and snapshot metadata.
+
+- **Engine version:** pinned per engine, and an `EngineVersion` naming anything but the pin is rejected. A minor version such as `18.4` or `11.8.8` is rejected as well: the AMI does not promise any particular minor, so accepting one would be a promise the platform cannot keep. There is no in-place upgrade.
 - **Instance classes:** `db.t3.{micro,small,medium,large}` and `db.m5.{large,xlarge}` — a naming facade over the platform's EC2 sizing table. Any other class is rejected at create.
 - **Storage:** gp3 only, 20–65536 GiB, always encrypted with the cluster key. Grow-only, and a grow is **stop/start with downtime** — the volume cannot be resized while attached.
-- **TLS:** offered, not enforced. The engine serves a per-instance certificate signed by the cluster CA (the `ca.pem` baked into AMIs) carrying both the ENI IP and the DNS name in its SAN set, so `sslmode=verify-full` works by name or by address. A deployment with no cluster CA starts the engine without TLS.
-- **Master user:** administrative but **not a PostgreSQL superuser**, as on AWS. It gets `CREATEDB` and `CREATEROLE`, owns the initial database, and holds `rds_superuser` (`pg_monitor`, `pg_signal_backend`, `pg_checkpoint`) with `ADMIN OPTION`, so it can grant that set onward. It cannot `COPY ... FROM/TO PROGRAM`, touch server-side files, run `ALTER SYSTEM`, or install **untrusted** extensions — those need the cluster superuser, which no customer credential is. Trusted extensions install normally in a database it owns. `postgres`, `rdsadmin` and `rds_superuser` are rejected as `--master-username`.
-- **Backups:** daily COW snapshots of the data volume inside `PreferredBackupWindow`. Retention defaults to 7 days and caps at 7; `0` disables automated backups. No point-in-time recovery.
+- **`DBName`:** optional; an omitted name creates no initial database. When supplied it must begin with a letter and hold only letters, digits and underscores, within the per-engine limit above. The rule is narrower than either engine's own, because the guest interpolates the name into a `CREATE DATABASE`.
+- **TLS:** offered, not enforced, on both engines. The engine serves a per-instance certificate signed by the cluster CA (the `ca.pem` baked into AMIs) carrying both the ENI IP and the DNS name in its SAN set, so PostgreSQL's `sslmode=verify-full` verifies by name or by address, and MariaDB's `--ssl-ca` plus `--ssl-verify-server-cert` verifies against the endpoint name. A deployment with no cluster CA starts the engine without TLS.
+- **PostgreSQL master user:** administrative but **not a PostgreSQL superuser**, as on AWS. It gets `CREATEDB` and `CREATEROLE`, owns the initial database, and holds `rds_superuser` (`pg_monitor`, `pg_signal_backend`, `pg_checkpoint`) with `ADMIN OPTION`, so it can grant that set onward. It cannot `COPY ... FROM/TO PROGRAM`, touch server-side files, run `ALTER SYSTEM`, or install **untrusted** extensions — those need the cluster superuser, which no customer credential is. Trusted extensions install normally in a database it owns.
+- **MariaDB master user:** created as `'<name>'@'%'` with global `RELOAD`, `PROCESS`, `SHOW DATABASES`, `REPLICATION SLAVE` and `REPLICATION CLIENT`, while application DML, DDL, routines, events, triggers and delegation are granted only on customer databases. MariaDB has no deny or partial-revoke mechanism, so granting those object privileges `ON *.*` would also grant them on `mysql.global_priv` and let the master acquire `SUPER`, `FILE` or `SHUTDOWN`. The initial `DBName` is granted automatically. Create another customer database with `CALL _spinifex_rds.create_database('name')`; plain `CREATE DATABASE`, arbitrary account administration and access to `mysql.*` are intentionally unavailable on the stock MariaDB build.
+- **MariaDB storage engine:** `default_storage_engine` is platform-owned and pinned to **InnoDB**, and is absent from the parameter catalog — a deliberate divergence from AWS, which exposes it as modifiable. Only InnoDB keeps a redo log, so only InnoDB is covered by the snapshot guarantee. An explicit `ENGINE=Aria` or `ENGINE=MyISAM` on `CREATE TABLE` still works — the limitation is documented rather than prevented — but such a table has no redo log, so a snapshot taken without a successful quiesce can leave it inconsistent with no way back. Use InnoDB, as AWS also recommends.
+- **Backups:** daily COW snapshots of the data volume inside `PreferredBackupWindow`. Retention defaults to 7 days and caps at 7; `0` disables automated backups. No point-in-time recovery, and MariaDB's binary log is off.
 - **Availability:** single-AZ. Engine crashes restart in-guest and VM crashes on a live host restart via `ec2-health-restart`, but an instance whose host is lost is reported `failed` with a reason in `StatusInfos` and needs operator recovery.
 
 Statuses: `creating`, `available`, `modifying`, `backing-up`, `rebooting`, `stopping`, `stopped`, `starting`, `deleting`, `failed`. (`recovering` is defined in the state machine but unreachable until auto-recovery lands.)
@@ -858,7 +882,7 @@ Statuses: `creating`, `available`, `modifying`, `backing-up`, `rebooting`, `stop
 
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `create-db-instance` | `--db-instance-identifier`, `--engine` (postgres), `--engine-version` (18), `--db-instance-class`, `--allocated-storage` (20–65536 GiB), `--storage-type` (gp3), `--master-username`, `--master-user-password`, `--db-name`, `--port` (1150–65535), `--db-subnet-group-name` (unnamed → the account's default VPC subnet), `--vpc-security-group-ids`, `--db-parameter-group-name`, `--backup-retention-period` (0–7, default 7), `--preferred-backup-window`, `--preferred-maintenance-window` (unnamed → assigned from the identifier), `--deletion-protection`, `--tags`, `--storage-encrypted` (true only) | `--auto-minor-version-upgrade`, `--copy-tags-to-snapshot`, Performance Insights / Enhanced Monitoring flags (all accepted, no-op); see "Rejected Parameters" for those that fail loudly | **DONE** |
+| `create-db-instance` | `--db-instance-identifier`, `--engine` (postgres, mariadb), `--engine-version` (the engine's pin, or omitted), `--db-instance-class`, `--allocated-storage` (20–65536 GiB), `--storage-type` (gp3), `--master-username`, `--master-user-password`, `--db-name` (letters, digits and underscores, beginning with a letter, max 63 on postgres and 64 on mariadb), `--port` (1150–65535), `--db-subnet-group-name` (unnamed → the account's default VPC subnet), `--vpc-security-group-ids`, `--db-parameter-group-name`, `--backup-retention-period` (0–7, default 7), `--preferred-backup-window`, `--preferred-maintenance-window` (unnamed → assigned from the identifier), `--deletion-protection`, `--tags`, `--storage-encrypted` (true only) | `--auto-minor-version-upgrade`, `--copy-tags-to-snapshot`, Performance Insights / Enhanced Monitoring flags (all accepted, no-op); see "Rejected Parameters" for those that fail loudly | **DONE** |
 | `describe-db-instances` | `--db-instance-identifier` | `--filters`, `--max-records`, `--marker` (parsed, not applied) | **DONE** |
 | `modify-db-instance` | `--db-instance-identifier`, `--master-user-password`, `--allocated-storage` (grow only, stop/start), `--db-instance-class` (VM replace), `--db-parameter-group-name`, `--vpc-security-group-ids`, `--deletion-protection`, `--backup-retention-period`, `--preferred-backup-window`, `--preferred-maintenance-window`, `--apply-immediately` | `--new-db-instance-identifier`, `--engine-version`, `--db-port-number`, `--db-subnet-group-name`, `--max-allocated-storage`, `--ca-certificate-identifier` (all rejected, not ignored) | **DONE** |
 | `delete-db-instance` | `--db-instance-identifier`, `--skip-final-snapshot`, `--final-db-snapshot-identifier` (exactly one is required) | `--delete-automated-backups` (no-op — automated snapshots are always purged with the instance) | **DONE** — `DeletionProtection` blocks the call; a final snapshot **retains** the data volume until that snapshot is deleted |
@@ -870,7 +894,7 @@ Statuses: `creating`, `available`, `modifying`, `backing-up`, `rebooting`, `stop
 
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `create-db-snapshot` | `--db-snapshot-identifier`, `--db-instance-identifier`, `--tags` | — | **DONE** — the agent quiesces the engine first; a failed quiesce falls back to a crash-consistent snapshot and writes a `DescribeEvents` warning |
+| `create-db-snapshot` | `--db-snapshot-identifier`, `--db-instance-identifier`, `--tags` | — | **DONE** — the agent quiesces the engine first (`pg_backup_start` on postgres, `BACKUP STAGE` on mariadb); a failed quiesce falls back to a crash-consistent snapshot and writes a `DescribeEvents` warning naming what that engine does and does not recover |
 | `describe-db-snapshots` | `--db-snapshot-identifier`, `--db-instance-identifier`, `--snapshot-type` (manual, automated) | `--filters`, `--dbi-resource-id`, `--include-shared`, `--include-public` (rejected), `--max-records`, `--marker` | **DONE** |
 | `delete-db-snapshot` | `--db-snapshot-identifier` | — | **DONE** — refused while a restored volume still references the snapshot; the last snapshot released reclaims a retained data volume |
 | `restore-db-instance-from-db-snapshot` | `--db-instance-identifier`, `--db-snapshot-identifier`, `--db-instance-class`, `--allocated-storage` (≥ the snapshot's), `--storage-type` (gp3), `--port`, `--db-subnet-group-name`, `--vpc-security-group-ids`, `--db-parameter-group-name`, `--deletion-protection`, `--tags`, `--engine` (must match the snapshot) | `--db-name` (rejected when it differs from the snapshot's), plus the create rejections | **DONE** — unnamed fields are inherited from the snapshot; the master password comes from the restored datadir |
@@ -898,14 +922,16 @@ Statuses: `creating`, `available`, `modifying`, `backing-up`, `rebooting`, `stop
 
 | Command | Implemented Flags | Missing Flags | Status |
 |---------|-------------------|---------------|--------|
-| `create-db-parameter-group` | `--db-parameter-group-name`, `--db-parameter-group-family` (postgres18), `--description`, `--tags` | — | **DONE** — names beginning with `default.` are reserved; the implicit group is `default.postgres18` |
-| `describe-db-parameter-groups` | `--db-parameter-group-name` | `--filters`, `--max-records`, `--marker` | **DONE** — the implicit default group is always listed |
-| `modify-db-parameter-group` | `--db-parameter-group-name`, `--parameters` (ParameterName, ParameterValue, ApplyMethod) | — | **DONE** — the whole batch is validated before anything is written; `immediate` on a static parameter is rejected, as AWS does |
-| `describe-db-parameters` | `--db-parameter-group-name`, `--source` (user, engine-default) | `--filters`, `--max-records`, `--marker` | **DONE** — 51-parameter PostgreSQL 18 catalog; memory defaults are computed per instance class and reported as literals |
+| `create-db-parameter-group` | `--db-parameter-group-name`, `--db-parameter-group-family` (`postgres18` or `mariadb11.8`), `--description`, `--tags` | — | **DONE** — names beginning with `default.` are reserved. **`--db-parameter-group-family` defaults to `postgres18` when omitted**, so a MariaDB group must name its family explicitly or the mismatch surfaces one call later, at `create-db-instance` |
+| `describe-db-parameter-groups` | `--db-parameter-group-name` | `--filters`, `--max-records`, `--marker` | **DONE** — every engine's implicit default group is always listed, so both `default.postgres18` and `default.mariadb11.8` appear in every account |
+| `modify-db-parameter-group` | `--db-parameter-group-name`, `--parameters` (ParameterName, ParameterValue, ApplyMethod) | — | **DONE** — the whole batch is validated before anything is written and propagated to attached instances; `immediate` on a static parameter is rejected, as AWS does |
+| `describe-db-parameters` | `--db-parameter-group-name`, `--source` (user, engine-default) | `--filters`, `--max-records`, `--marker` | **DONE** — the catalog of the group's own family: 51 parameters on `postgres18`, 54 on `mariadb11.8`. Memory defaults are computed per instance class and reported as literals |
 | `delete-db-parameter-group` | `--db-parameter-group-name` | — | **DONE** — refused for a default group and while any instance references it |
 | `reset-db-parameter-group` | — | all | **NOT STARTED** (`InvalidAction`) |
 
-A group takes effect on an instance when `modify-db-instance --db-parameter-group-name` attaches it — immediately with `--apply-immediately`, otherwise at the next maintenance window. Dynamic parameters are written into the engine's config and reloaded live; static ones are recorded `pending-reboot` and applied by `reboot-db-instance`.
+A group takes effect on an instance when `modify-db-instance --db-parameter-group-name` attaches it — immediately with `--apply-immediately`, otherwise at the next maintenance window. Later group edits propagate to every attached instance. Dynamic parameters are written into the engine's config and reloaded live; static ones are recorded `pending-reboot` and applied by `reboot-db-instance`.
+
+**A group's family must match the instance's engine.** A `postgres18` group cannot be attached to a MariaDB instance and a `mariadb11.8` group cannot be attached to a PostgreSQL one, on create, on modify, on a deferred apply or on a snapshot restore; the mismatch is refused with `InvalidParameterCombination`. The catalogs are per-family too, so `modify-db-parameter-group` will not store one engine's parameter name into the other's group. Each catalog is a curated, validated subset rather than the engine's full variable set, and the platform-owned settings — the port, the datadir, the socket, the bind address, the `ssl_*` family, and on MariaDB `secure_file_priv`, `log_bin`, `default_storage_engine` and the InnoDB buffer-pool chunk settings — are absent from it rather than present-and-unmodifiable, because they are not the customer's to set.
 
 ### RDS — Tags
 
@@ -937,7 +963,8 @@ Policy: a parameter whose omission would create a false safety, security or avai
 | `DBSecurityGroups` | EC2-Classic security groups — use `VpcSecurityGroupIds` |
 | `DBClusterIdentifier`, `DBClusterSnapshotIdentifier` | Clustered engines are not offered |
 | `EnableCloudwatchLogsExports` | Log export is not implemented |
-| `EngineVersion` other than 18, `Engine` on modify | No in-place engine or version change |
+| `EngineVersion` other than the engine's pin, `Engine` on modify | No in-place engine or version change |
+| `Engine=mysql` (and Aurora engines) | Oracle MySQL is not offered; `mariadb` is a distinct engine, not an alias for it |
 | `NewDBInstanceIdentifier` | The identifier is the DNS label and the KV key |
 | `DBPortNumber`, `DBSubnetGroupName` on modify | Both would move the endpoint |
 | `MaxAllocatedStorage` | Storage autoscaling is not implemented |
@@ -956,19 +983,65 @@ Recognised actions below return `OperationNotSupported`, so a client sees "not o
 
 | Feature | Actions | Priority | Status |
 |---------|---------|----------|--------|
-| MySQL engine | — (second AMI preset + `Engine` implementation) | High | **NOT STARTED** |
-| Point-in-time recovery (WAL archiving to predastore) | `restore-db-instance-to-point-in-time` | High | **NOT STARTED** |
+| Oracle MySQL engine | — | Low | **NOT PLANNED** — MariaDB is offered instead, under its own `mariadb` engine name; see the note above on why it is not aliased onto `mysql` |
+| Point-in-time recovery (WAL archiving to predastore on postgres, binlog on mariadb) | `restore-db-instance-to-point-in-time` | High | **NOT STARTED** |
 | Auto-recovery from node loss (failure is detected, not repaired) | — | High | **NOT STARTED** |
 | Read replicas | `create-db-instance-read-replica`, `promote-read-replica` | Medium | **NOT STARTED** |
 | Aurora / DB clusters | `create-db-cluster`, `modify-db-cluster`, `delete-db-cluster`, `describe-db-clusters`, `failover-db-cluster` | Low | **NOT STARTED** |
 | Option groups | `create-option-group`, `modify-option-group`, `delete-option-group`, `describe-option-groups` | Low | **NOT STARTED** |
 | Engine-version / orderable-option data sources | `describe-db-engine-versions`, `describe-orderable-db-instance-options` | Low | **NOT STARTED** (`InvalidAction`) — not required by `aws_db_instance`; pin the class and version literally in Terraform |
-| Multi-AZ standby, online (no-downtime) storage grow, storage autoscaling, IAM database auth, Performance Insights, Enhanced Monitoring, log exports, per-tenant private DNS zones, enforced TLS | — | — | **NOT STARTED** |
+| Multi-AZ standby, online (no-downtime) storage grow, storage autoscaling, IAM database auth, Performance Insights, Enhanced Monitoring, log exports, per-tenant private DNS zones, enforced TLS, cross-engine snapshot restore, in-place migration between the two engines | — | — | **NOT STARTED** |
 
 IAM: `AmazonRDSFullAccess` and `AmazonRDSReadOnlyAccess` are available as managed policies. They grant `rds:` verb prefixes rather than `rds:*`, because `rds:*` would also appear to grant the internal agent actions the gateway reserves for a DB VM's own role.
 
 ---
 
+## ECR (Elastic Container Registry)
+
+Repository metadata is served over the AWS API on the gateway endpoint; image data moves over the OCI Distribution `/v2/` endpoint on that same host, authenticated by the bearer token `get-login-password` mints.
+
+| Command | Status |
+|---------|--------|
+| `create-repository`, `delete-repository`, `describe-repositories` | **DONE** |
+| `describe-images`, `list-images`, `batch-get-image`, `batch-delete-image`, `put-image` | **DONE** |
+| `put-image-tag-mutability` | **DONE** |
+| `get-authorization-token` | **DONE** — mints the short-lived bearer token used by `docker login` |
+| `get-repository-policy`, `set-repository-policy`, `delete-repository-policy` | **DONE** |
+| `get-lifecycle-policy`, `put-lifecycle-policy`, `delete-lifecycle-policy`, `get-lifecycle-policy-preview`, `start-lifecycle-policy-preview` | **DONE** |
+| `list-tags-for-resource` | **DONE** |
+| `tag-resource`, `untag-resource` | **NOT STARTED** (registered stub) |
+| `batch-check-layer-availability`, `initiate-layer-upload`, `upload-layer-part`, `complete-layer-upload`, `get-download-url-for-layer` | **NOT STARTED** (registered stub) — layer transfer is served by the OCI `/v2/` endpoint |
+| `describe-registry`, `get-registry-policy`, `put-registry-policy`, `put-replication-configuration`, `replicate-image` | **NOT STARTED** (registered stub) — single-registry deployment, no cross-region replication |
+| `list-repositories` | **NOT STARTED** (registered stub) — use `describe-repositories` |
+| `start-image-scan`, `describe-image-scan-findings`, `put-image-scanning-configuration`, `get-image-scanning-configuration`, `get-registry-scanning-configuration`, `put-registry-scanning-configuration`, `batch-get-repository-scanning-configuration` | **NOT STARTED** (`OperationNotSupported`) — vulnerability scanning is not offered |
+| Pull-through cache rules, repository creation templates, image replication status | **NOT STARTED** (`InvalidAction`) |
+
+---
+
+## ECS (Elastic Container Service)
+
+Clusters, services and tasks run on EC2 container instances; there is no Fargate launch type.
+
+| Command | Status |
+|---------|--------|
+| `create-cluster`, `delete-cluster`, `describe-clusters`, `list-clusters` | **DONE** |
+| `create-service`, `update-service`, `delete-service`, `describe-services`, `list-services` | **DONE** |
+| `register-task-definition`, `deregister-task-definition`, `describe-task-definition`, `list-task-definitions` | **DONE** |
+| `run-task`, `start-task`, `stop-task`, `describe-tasks`, `list-tasks` | **DONE** |
+| `register-container-instance`, `deregister-container-instance`, `describe-container-instances`, `list-container-instances`, `update-container-instances-state` | **DONE** |
+| `create-capacity-provider`, `delete-capacity-provider`, `describe-capacity-providers`, `put-cluster-capacity-providers` | **DONE** |
+| `submit-task-state-change` | **DONE** — reported by the in-guest agent |
+| `tag-resource`, `untag-resource`, `list-tags-for-resource` | **DONE** |
+| `update-cluster`, `list-account-settings`, `put-account-setting`, `list-services-by-namespace`, `list-task-definition-families` | **NOT STARTED** (registered stub) |
+| Task sets (blue/green): `create-task-set`, `update-task-set`, `delete-task-set`, `describe-task-sets`, `update-service-primary-task-set` | **NOT STARTED** (`InvalidAction`) |
+| `execute-command`, `get-task-protection`, `update-task-protection` | **NOT STARTED** (`InvalidAction`) |
+| Attributes: `put-attributes`, `delete-attributes`, `list-attributes` | **NOT STARTED** (`InvalidAction`) |
+| `update-capacity-provider`, `update-cluster-settings`, `update-container-agent` | **NOT STARTED** (`InvalidAction`) |
+| `delete-account-setting`, `put-account-setting-default`, `delete-task-definitions` | **NOT STARTED** (`InvalidAction`) |
+| `discover-poll-endpoint`, `submit-container-state-change`, `submit-attachment-state-changes` | **NOT STARTED** (`InvalidAction`) |
+
+---
+
 ## CloudWatch (Basic Monitoring)
 
 | Command | Implemented Flags | Missing Flags | Status |

docs/README.md

@@ -1,3 +1,5 @@
 # Spinifex Documentation
 
-This directory contains the Spinifex documentation. This is available in individual markdown files, and a generated website to aid navigation and browing documentation at: [https://docs.mulgadc.com](https://docs.mulgadc.com/)
+This directory contains the Spinifex documentation. This is available in individual markdown files, and a generated website to aid navigation and browsing documentation at: [https://docs.mulgadc.com](https://docs.mulgadc.com/)
+
+The [AWS model operation coverage](aws-model-operation-coverage.md) report is generated from Spinifex's dispatch tables and the pinned AWS API models. Run `make aws-model-coverage` to regenerate and print it.

docs/TAG-CONVENTIONS.md

@@ -1,32 +0,0 @@
-# Spinifex Tag Conventions
-
-Spinifex uses a small set of reserved tag keys to mark resources that are owned and managed by the platform itself (as opposed to resources created directly by customer AWS API calls). The UI uses these tags to hide system-owned resources from customer-facing listings so operators do not
-mistake them for user workloads and accidentally terminate or modify them.
-
-## `spinifex:managed-by`
-
-Identifies the Spinifex platform component that owns a resource.
-
-| Value   | Component                       | Tagged resource types                 |
-|---------|---------------------------------|---------------------------------------|
-| `elbv2` | ALB / HAProxy load balancers    | LB VM (`instance`), LB ENI, LB AMI    |
-
-The tag is applied by the backend at create/launch time:
-
-- **Instance** — set in `spinifex/spinifex/daemon/daemon_system_instance.go` when `LaunchSystemInstance` builds the `RunInstancesInput`, and mirrored onto the stored `vm.VM.ManagedBy` field for fast lookup.
-- **ENI** — set in `spinifex/spinifex/handlers/elbv2/service_impl.go` when `CreateLoadBalancer` calls `CreateNetworkInterface`.
-- **AMI** — declared on the catalog entry in `spinifex/spinifex/utils/images.go` and copied into `AMIMetadata.Tags` by the image importer in `spinifex/cmd/spinifex/cmd/admin.go`.
-
-Tag keys and values are defined once in `spinifex/spinifex/tags/tags.go`; new system components should register their value there and reuse the constants rather than hardcoding strings.
-
-## UI behaviour
-
-The admin **Nodes** page and the **Images** (AMIs) page filter out resources that carry `spinifex:managed-by`. Engineers who need to see system resources for debugging can append `?system=1` to the URL — the filter is bypassed and a hidden-count banner is replaced with the full listing. Direct navigation to a system AMI detail page without `?system=1` renders the "Image not found" fallback.
-
-The EC2 **Instances** page does not need UI-side filtering: backend SigV4 authentication scopes `DescribeInstances` by account ID, and system instances live under the global system account
-(`000000000000`), so customers never see them.
-
-## `spinifex:lb-arn`
-
-Applied to ELBv2-managed ENIs alongside `spinifex:managed-by=elbv2`. Stores the ARN of the parent load balancer so reconciliation can map an ENI back to its owner without a separate lookup. Customer-facing
-filtering keys off `spinifex:managed-by`; `spinifex:lb-arn` is informational.

docs/admin/spinifex-admin-cli/README.md

@@ -116,10 +116,7 @@ To run EKS, import the prebuilt node image from the catalog:
 spx admin images import --name spinifex-eks-node
 ```
 
-This pulls the Alpine + K3s node AMI from `iso.mulgadc.com`, verifies its
-checksum, and registers it tagged `spinifex:managed-by=eks`. `eks create-cluster`
-and `eks create-nodegroup` resolve the boot AMI by that tag, so no further
-configuration is needed.
+This pulls the Alpine + K3s node AMI from `iso.mulgadc.com`, verifies its checksum, and registers it tagged `spinifex:managed-by=eks`. `eks create-cluster` and `eks create-nodegroup` resolve the boot AMI by that tag, so no further configuration is needed.
 
 ## Cluster Shutdown
 

docs/aws-model-operation-coverage.md

@@ -0,0 +1,575 @@
+# AWS model operation coverage
+
+Generated from the cached `aws-sdk-go v1.55.8` `api-2.json` models and Spinifex's authoritative gateway dispatch tables.
+
+Implemented means a modelled operation is registered to a real handler. Stub and deliberately unsupported handlers are reported separately. Shape conformance does not imply behavioural conformance.
+
+| Service | API version | Modelled | Registered | Implemented | Stubbed | Unsupported | Missing | Extra |
+|---|---:|---:|---:|---:|---:|---:|---:|---:|
+| acm | 2015-12-08 | 15 | 9 | 9 | 0 | 0 | 6 | 0 |
+| ec2 | 2016-11-15 | 625 | 119 | 119 | 0 | 0 | 506 | 0 |
+| ecr | 2015-09-21 | 47 | 39 | 19 | 13 | 7 | 11 | 3 |
+| ecs | 2014-11-13 | 56 | 39 | 31 | 5 | 0 | 20 | 3 |
+| elasticloadbalancingv2 | 2015-12-01 | 46 | 37 | 33 | 0 | 0 | 13 | 4 |
+| iam | 2010-05-08 | 159 | 75 | 75 | 0 | 0 | 84 | 0 |
+| rds | 2014-10-31 | 162 | 40 | 24 | 0 | 12 | 126 | 4 |
+| s3 | 2006-03-01 | 99 | — | — | — | — | — | — |
+| sts | 2011-06-15 | 8 | 4 | 4 | 0 | 0 | 4 | 0 |
+
+## acm
+
+Implements **9 of 15** modelled operations (60.0%).
+
+<details><summary>Implemented (9)</summary>
+
+`AddTagsToCertificate`, `DeleteCertificate`, `DescribeCertificate`, `GetCertificate`, `ImportCertificate`, `ListCertificates`, `ListTagsForCertificate`, `RemoveTagsFromCertificate`
+
+`RequestCertificate`
+
+
+</details>
+
+<details><summary>Missing from dispatch (6)</summary>
+
+`ExportCertificate`, `GetAccountConfiguration`, `PutAccountConfiguration`, `RenewCertificate`, `ResendValidationEmail`, `UpdateCertificateOptions`
+
+
+</details>
+
+<details><summary>Registered stubs (0)</summary>
+
+None.
+
+</details>
+
+<details><summary>Deliberately unsupported (0)</summary>
+
+None.
+
+</details>
+
+<details><summary>Registered outside the pinned model (0)</summary>
+
+None.
+
+</details>
+
+## ec2
+
+Implements **119 of 625** modelled operations (19.0%).
+
+<details><summary>Implemented (119)</summary>
+
+`AllocateAddress`, `AssociateAddress`, `AssociateIamInstanceProfile`, `AssociateRouteTable`, `AttachInternetGateway`, `AttachNetworkInterface`, `AttachVolume`, `AuthorizeSecurityGroupEgress`
+
+`AuthorizeSecurityGroupIngress`, `CancelCapacityReservation`, `CancelSpotInstanceRequests`, `CopyImage`, `CopySnapshot`, `CreateCapacityReservation`, `CreateEgressOnlyInternetGateway`, `CreateImage`
+
+`CreateInternetGateway`, `CreateKeyPair`, `CreateLaunchTemplate`, `CreateLaunchTemplateVersion`, `CreateNatGateway`, `CreateNetworkInterface`, `CreatePlacementGroup`, `CreateRoute`
+
+`CreateRouteTable`, `CreateSecurityGroup`, `CreateSnapshot`, `CreateSubnet`, `CreateTags`, `CreateVolume`, `CreateVpc`, `DeleteEgressOnlyInternetGateway`
+
+`DeleteInternetGateway`, `DeleteKeyPair`, `DeleteLaunchTemplate`, `DeleteLaunchTemplateVersions`, `DeleteNatGateway`, `DeleteNetworkInterface`, `DeletePlacementGroup`, `DeleteRoute`
+
+`DeleteRouteTable`, `DeleteSecurityGroup`, `DeleteSnapshot`, `DeleteSubnet`, `DeleteTags`, `DeleteVolume`, `DeleteVpc`, `DeregisterImage`
+
+`DescribeAccountAttributes`, `DescribeAddresses`, `DescribeAddressesAttribute`, `DescribeAvailabilityZones`, `DescribeCapacityReservations`, `DescribeEgressOnlyInternetGateways`, `DescribeIamInstanceProfileAssociations`, `DescribeImageAttribute`
+
+`DescribeImages`, `DescribeInstanceAttribute`, `DescribeInstanceCreditSpecifications`, `DescribeInstanceStatus`, `DescribeInstanceTypes`, `DescribeInstances`, `DescribeInternetGateways`, `DescribeKeyPairs`
+
+`DescribeLaunchTemplateVersions`, `DescribeLaunchTemplates`, `DescribeNatGateways`, `DescribeNetworkInterfaces`, `DescribePlacementGroups`, `DescribeRegions`, `DescribeRouteTables`, `DescribeSecurityGroupRules`
+
+`DescribeSecurityGroups`, `DescribeSnapshots`, `DescribeSpotInstanceRequests`, `DescribeSubnets`, `DescribeTags`, `DescribeVolumeStatus`, `DescribeVolumes`, `DescribeVolumesModifications`
+
+`DescribeVpcAttribute`, `DescribeVpcs`, `DetachInternetGateway`, `DetachNetworkInterface`, `DetachVolume`, `DisableEbsEncryptionByDefault`, `DisableSerialConsoleAccess`, `DisassociateAddress`
+
+`DisassociateIamInstanceProfile`, `DisassociateRouteTable`, `EnableEbsEncryptionByDefault`, `EnableSerialConsoleAccess`, `GetConsoleOutput`, `GetEbsEncryptionByDefault`, `GetPasswordData`, `GetSerialConsoleAccessStatus`
+
+`ImportKeyPair`, `ModifyImageAttribute`, `ModifyInstanceAttribute`, `ModifyInstanceMetadataOptions`, `ModifyLaunchTemplate`, `ModifyNetworkInterfaceAttribute`, `ModifySubnetAttribute`, `ModifyVolume`
+
+`ModifyVpcAttribute`, `RebootInstances`, `RegisterImage`, `ReleaseAddress`, `ReplaceIamInstanceProfileAssociation`, `ReplaceRoute`, `ReplaceRouteTableAssociation`, `RequestSpotInstances`
+
+`ResetImageAttribute`, `RevokeSecurityGroupEgress`, `RevokeSecurityGroupIngress`, `RunInstances`, `StartInstances`, `StopInstances`, `TerminateInstances`
+
+
+</details>
+
+<details><summary>Missing from dispatch (506)</summary>
+
+`AcceptAddressTransfer`, `AcceptReservedInstancesExchangeQuote`, `AcceptTransitGatewayMulticastDomainAssociations`, `AcceptTransitGatewayPeeringAttachment`, `AcceptTransitGatewayVpcAttachment`, `AcceptVpcEndpointConnections`, `AcceptVpcPeeringConnection`, `AdvertiseByoipCidr`
+
+`AllocateHosts`, `AllocateIpamPoolCidr`, `ApplySecurityGroupsToClientVpnTargetNetwork`, `AssignIpv6Addresses`, `AssignPrivateIpAddresses`, `AssignPrivateNatGatewayAddress`, `AssociateClientVpnTargetNetwork`, `AssociateDhcpOptions`
+
+`AssociateEnclaveCertificateIamRole`, `AssociateInstanceEventWindow`, `AssociateIpamByoasn`, `AssociateIpamResourceDiscovery`, `AssociateNatGatewayAddress`, `AssociateSubnetCidrBlock`, `AssociateTransitGatewayMulticastDomain`, `AssociateTransitGatewayPolicyTable`
+
+`AssociateTransitGatewayRouteTable`, `AssociateTrunkInterface`, `AssociateVpcCidrBlock`, `AttachClassicLinkVpc`, `AttachVerifiedAccessTrustProvider`, `AttachVpnGateway`, `AuthorizeClientVpnIngress`, `BundleInstance`
+
+`CancelBundleTask`, `CancelCapacityReservationFleets`, `CancelConversionTask`, `CancelExportTask`, `CancelImageLaunchPermission`, `CancelImportTask`, `CancelReservedInstancesListing`, `CancelSpotFleetRequests`
+
+`ConfirmProductInstance`, `CopyFpgaImage`, `CreateCapacityReservationFleet`, `CreateCarrierGateway`, `CreateClientVpnEndpoint`, `CreateClientVpnRoute`, `CreateCoipCidr`, `CreateCoipPool`
+
+`CreateCustomerGateway`, `CreateDefaultSubnet`, `CreateDefaultVpc`, `CreateDhcpOptions`, `CreateFleet`, `CreateFlowLogs`, `CreateFpgaImage`, `CreateInstanceConnectEndpoint`
+
+`CreateInstanceEventWindow`, `CreateInstanceExportTask`, `CreateIpam`, `CreateIpamExternalResourceVerificationToken`, `CreateIpamPool`, `CreateIpamResourceDiscovery`, `CreateIpamScope`, `CreateLocalGatewayRoute`
+
+`CreateLocalGatewayRouteTable`, `CreateLocalGatewayRouteTableVirtualInterfaceGroupAssociation`, `CreateLocalGatewayRouteTableVpcAssociation`, `CreateManagedPrefixList`, `CreateNetworkAcl`, `CreateNetworkAclEntry`, `CreateNetworkInsightsAccessScope`, `CreateNetworkInsightsPath`
+
+`CreateNetworkInterfacePermission`, `CreatePublicIpv4Pool`, `CreateReplaceRootVolumeTask`, `CreateReservedInstancesListing`, `CreateRestoreImageTask`, `CreateSnapshots`, `CreateSpotDatafeedSubscription`, `CreateStoreImageTask`
+
+`CreateSubnetCidrReservation`, `CreateTrafficMirrorFilter`, `CreateTrafficMirrorFilterRule`, `CreateTrafficMirrorSession`, `CreateTrafficMirrorTarget`, `CreateTransitGateway`, `CreateTransitGatewayConnect`, `CreateTransitGatewayConnectPeer`
+
+`CreateTransitGatewayMulticastDomain`, `CreateTransitGatewayPeeringAttachment`, `CreateTransitGatewayPolicyTable`, `CreateTransitGatewayPrefixListReference`, `CreateTransitGatewayRoute`, `CreateTransitGatewayRouteTable`, `CreateTransitGatewayRouteTableAnnouncement`, `CreateTransitGatewayVpcAttachment`
+
+`CreateVerifiedAccessEndpoint`, `CreateVerifiedAccessGroup`, `CreateVerifiedAccessInstance`, `CreateVerifiedAccessTrustProvider`, `CreateVpcEndpoint`, `CreateVpcEndpointConnectionNotification`, `CreateVpcEndpointServiceConfiguration`, `CreateVpcPeeringConnection`
+
+`CreateVpnConnection`, `CreateVpnConnectionRoute`, `CreateVpnGateway`, `DeleteCarrierGateway`, `DeleteClientVpnEndpoint`, `DeleteClientVpnRoute`, `DeleteCoipCidr`, `DeleteCoipPool`
+
+`DeleteCustomerGateway`, `DeleteDhcpOptions`, `DeleteFleets`, `DeleteFlowLogs`, `DeleteFpgaImage`, `DeleteInstanceConnectEndpoint`, `DeleteInstanceEventWindow`, `DeleteIpam`
+
+`DeleteIpamExternalResourceVerificationToken`, `DeleteIpamPool`, `DeleteIpamResourceDiscovery`, `DeleteIpamScope`, `DeleteLocalGatewayRoute`, `DeleteLocalGatewayRouteTable`, `DeleteLocalGatewayRouteTableVirtualInterfaceGroupAssociation`, `DeleteLocalGatewayRouteTableVpcAssociation`
+
+`DeleteManagedPrefixList`, `DeleteNetworkAcl`, `DeleteNetworkAclEntry`, `DeleteNetworkInsightsAccessScope`, `DeleteNetworkInsightsAccessScopeAnalysis`, `DeleteNetworkInsightsAnalysis`, `DeleteNetworkInsightsPath`, `DeleteNetworkInterfacePermission`
+
+`DeletePublicIpv4Pool`, `DeleteQueuedReservedInstances`, `DeleteSpotDatafeedSubscription`, `DeleteSubnetCidrReservation`, `DeleteTrafficMirrorFilter`, `DeleteTrafficMirrorFilterRule`, `DeleteTrafficMirrorSession`, `DeleteTrafficMirrorTarget`
+
+`DeleteTransitGateway`, `DeleteTransitGatewayConnect`, `DeleteTransitGatewayConnectPeer`, `DeleteTransitGatewayMulticastDomain`, `DeleteTransitGatewayPeeringAttachment`, `DeleteTransitGatewayPolicyTable`, `DeleteTransitGatewayPrefixListReference`, `DeleteTransitGatewayRoute`
+
+`DeleteTransitGatewayRouteTable`, `DeleteTransitGatewayRouteTableAnnouncement`, `DeleteTransitGatewayVpcAttachment`, `DeleteVerifiedAccessEndpoint`, `DeleteVerifiedAccessGroup`, `DeleteVerifiedAccessInstance`, `DeleteVerifiedAccessTrustProvider`, `DeleteVpcEndpointConnectionNotifications`
+
+`DeleteVpcEndpointServiceConfigurations`, `DeleteVpcEndpoints`, `DeleteVpcPeeringConnection`, `DeleteVpnConnection`, `DeleteVpnConnectionRoute`, `DeleteVpnGateway`, `DeprovisionByoipCidr`, `DeprovisionIpamByoasn`
+
+`DeprovisionIpamPoolCidr`, `DeprovisionPublicIpv4PoolCidr`, `DeregisterInstanceEventNotificationAttributes`, `DeregisterTransitGatewayMulticastGroupMembers`, `DeregisterTransitGatewayMulticastGroupSources`, `DescribeAddressTransfers`, `DescribeAggregateIdFormat`, `DescribeAwsNetworkPerformanceMetricSubscriptions`
+
+`DescribeBundleTasks`, `DescribeByoipCidrs`, `DescribeCapacityBlockOfferings`, `DescribeCapacityReservationFleets`, `DescribeCarrierGateways`, `DescribeClassicLinkInstances`, `DescribeClientVpnAuthorizationRules`, `DescribeClientVpnConnections`
+
+`DescribeClientVpnEndpoints`, `DescribeClientVpnRoutes`, `DescribeClientVpnTargetNetworks`, `DescribeCoipPools`, `DescribeConversionTasks`, `DescribeCustomerGateways`, `DescribeDhcpOptions`, `DescribeElasticGpus`
+
+`DescribeExportImageTasks`, `DescribeExportTasks`, `DescribeFastLaunchImages`, `DescribeFastSnapshotRestores`, `DescribeFleetHistory`, `DescribeFleetInstances`, `DescribeFleets`, `DescribeFlowLogs`
+
+`DescribeFpgaImageAttribute`, `DescribeFpgaImages`, `DescribeHostReservationOfferings`, `DescribeHostReservations`, `DescribeHosts`, `DescribeIdFormat`, `DescribeIdentityIdFormat`, `DescribeImportImageTasks`
+
+`DescribeImportSnapshotTasks`, `DescribeInstanceConnectEndpoints`, `DescribeInstanceEventNotificationAttributes`, `DescribeInstanceEventWindows`, `DescribeInstanceTopology`, `DescribeInstanceTypeOfferings`, `DescribeIpamByoasn`, `DescribeIpamExternalResourceVerificationTokens`
+
+`DescribeIpamPools`, `DescribeIpamResourceDiscoveries`, `DescribeIpamResourceDiscoveryAssociations`, `DescribeIpamScopes`, `DescribeIpams`, `DescribeIpv6Pools`, `DescribeLocalGatewayRouteTableVirtualInterfaceGroupAssociations`, `DescribeLocalGatewayRouteTableVpcAssociations`
+
+`DescribeLocalGatewayRouteTables`, `DescribeLocalGatewayVirtualInterfaceGroups`, `DescribeLocalGatewayVirtualInterfaces`, `DescribeLocalGateways`, `DescribeLockedSnapshots`, `DescribeMacHosts`, `DescribeManagedPrefixLists`, `DescribeMovingAddresses`
+
+`DescribeNetworkAcls`, `DescribeNetworkInsightsAccessScopeAnalyses`, `DescribeNetworkInsightsAccessScopes`, `DescribeNetworkInsightsAnalyses`, `DescribeNetworkInsightsPaths`, `DescribeNetworkInterfaceAttribute`, `DescribeNetworkInterfacePermissions`, `DescribePrefixLists`
+
+`DescribePrincipalIdFormat`, `DescribePublicIpv4Pools`, `DescribeReplaceRootVolumeTasks`, `DescribeReservedInstances`, `DescribeReservedInstancesListings`, `DescribeReservedInstancesModifications`, `DescribeReservedInstancesOfferings`, `DescribeScheduledInstanceAvailability`
+
+`DescribeScheduledInstances`, `DescribeSecurityGroupReferences`, `DescribeSnapshotAttribute`, `DescribeSnapshotTierStatus`, `DescribeSpotDatafeedSubscription`, `DescribeSpotFleetInstances`, `DescribeSpotFleetRequestHistory`, `DescribeSpotFleetRequests`
+
+`DescribeSpotPriceHistory`, `DescribeStaleSecurityGroups`, `DescribeStoreImageTasks`, `DescribeTrafficMirrorFilterRules`, `DescribeTrafficMirrorFilters`, `DescribeTrafficMirrorSessions`, `DescribeTrafficMirrorTargets`, `DescribeTransitGatewayAttachments`
+
+`DescribeTransitGatewayConnectPeers`, `DescribeTransitGatewayConnects`, `DescribeTransitGatewayMulticastDomains`, `DescribeTransitGatewayPeeringAttachments`, `DescribeTransitGatewayPolicyTables`, `DescribeTransitGatewayRouteTableAnnouncements`, `DescribeTransitGatewayRouteTables`, `DescribeTransitGatewayVpcAttachments`
+
+`DescribeTransitGateways`, `DescribeTrunkInterfaceAssociations`, `DescribeVerifiedAccessEndpoints`, `DescribeVerifiedAccessGroups`, `DescribeVerifiedAccessInstanceLoggingConfigurations`, `DescribeVerifiedAccessInstances`, `DescribeVerifiedAccessTrustProviders`, `DescribeVolumeAttribute`
+
+`DescribeVpcClassicLink`, `DescribeVpcClassicLinkDnsSupport`, `DescribeVpcEndpointConnectionNotifications`, `DescribeVpcEndpointConnections`, `DescribeVpcEndpointServiceConfigurations`, `DescribeVpcEndpointServicePermissions`, `DescribeVpcEndpointServices`, `DescribeVpcEndpoints`
+
+`DescribeVpcPeeringConnections`, `DescribeVpnConnections`, `DescribeVpnGateways`, `DetachClassicLinkVpc`, `DetachVerifiedAccessTrustProvider`, `DetachVpnGateway`, `DisableAddressTransfer`, `DisableAwsNetworkPerformanceMetricSubscription`
+
+`DisableFastLaunch`, `DisableFastSnapshotRestores`, `DisableImage`, `DisableImageBlockPublicAccess`, `DisableImageDeprecation`, `DisableImageDeregistrationProtection`, `DisableIpamOrganizationAdminAccount`, `DisableSnapshotBlockPublicAccess`
+
+`DisableTransitGatewayRouteTablePropagation`, `DisableVgwRoutePropagation`, `DisableVpcClassicLink`, `DisableVpcClassicLinkDnsSupport`, `DisassociateClientVpnTargetNetwork`, `DisassociateEnclaveCertificateIamRole`, `DisassociateInstanceEventWindow`, `DisassociateIpamByoasn`
+
+`DisassociateIpamResourceDiscovery`, `DisassociateNatGatewayAddress`, `DisassociateSubnetCidrBlock`, `DisassociateTransitGatewayMulticastDomain`, `DisassociateTransitGatewayPolicyTable`, `DisassociateTransitGatewayRouteTable`, `DisassociateTrunkInterface`, `DisassociateVpcCidrBlock`
+
+`EnableAddressTransfer`, `EnableAwsNetworkPerformanceMetricSubscription`, `EnableFastLaunch`, `EnableFastSnapshotRestores`, `EnableImage`, `EnableImageBlockPublicAccess`, `EnableImageDeprecation`, `EnableImageDeregistrationProtection`
+
+`EnableIpamOrganizationAdminAccount`, `EnableReachabilityAnalyzerOrganizationSharing`, `EnableSnapshotBlockPublicAccess`, `EnableTransitGatewayRouteTablePropagation`, `EnableVgwRoutePropagation`, `EnableVolumeIO`, `EnableVpcClassicLink`, `EnableVpcClassicLinkDnsSupport`
+
+`ExportClientVpnClientCertificateRevocationList`, `ExportClientVpnClientConfiguration`, `ExportImage`, `ExportTransitGatewayRoutes`, `GetAssociatedEnclaveCertificateIamRoles`, `GetAssociatedIpv6PoolCidrs`, `GetAwsNetworkPerformanceData`, `GetCapacityReservationUsage`
+
+`GetCoipPoolUsage`, `GetConsoleScreenshot`, `GetDefaultCreditSpecification`, `GetEbsDefaultKmsKeyId`, `GetFlowLogsIntegrationTemplate`, `GetGroupsForCapacityReservation`, `GetHostReservationPurchasePreview`, `GetImageBlockPublicAccessState`
+
+`GetInstanceMetadataDefaults`, `GetInstanceTpmEkPub`, `GetInstanceTypesFromInstanceRequirements`, `GetInstanceUefiData`, `GetIpamAddressHistory`, `GetIpamDiscoveredAccounts`, `GetIpamDiscoveredPublicAddresses`, `GetIpamDiscoveredResourceCidrs`
+
+`GetIpamPoolAllocations`, `GetIpamPoolCidrs`, `GetIpamResourceCidrs`, `GetLaunchTemplateData`, `GetManagedPrefixListAssociations`, `GetManagedPrefixListEntries`, `GetNetworkInsightsAccessScopeAnalysisFindings`, `GetNetworkInsightsAccessScopeContent`
+
+`GetReservedInstancesExchangeQuote`, `GetSecurityGroupsForVpc`, `GetSnapshotBlockPublicAccessState`, `GetSpotPlacementScores`, `GetSubnetCidrReservations`, `GetTransitGatewayAttachmentPropagations`, `GetTransitGatewayMulticastDomainAssociations`, `GetTransitGatewayPolicyTableAssociations`
+
+`GetTransitGatewayPolicyTableEntries`, `GetTransitGatewayPrefixListReferences`, `GetTransitGatewayRouteTableAssociations`, `GetTransitGatewayRouteTablePropagations`, `GetVerifiedAccessEndpointPolicy`, `GetVerifiedAccessGroupPolicy`, `GetVpnConnectionDeviceSampleConfiguration`, `GetVpnConnectionDeviceTypes`
+
+`GetVpnTunnelReplacementStatus`, `ImportClientVpnClientCertificateRevocationList`, `ImportImage`, `ImportInstance`, `ImportSnapshot`, `ImportVolume`, `ListImagesInRecycleBin`, `ListSnapshotsInRecycleBin`
+
+`LockSnapshot`, `ModifyAddressAttribute`, `ModifyAvailabilityZoneGroup`, `ModifyCapacityReservation`, `ModifyCapacityReservationFleet`, `ModifyClientVpnEndpoint`, `ModifyDefaultCreditSpecification`, `ModifyEbsDefaultKmsKeyId`
+
+`ModifyFleet`, `ModifyFpgaImageAttribute`, `ModifyHosts`, `ModifyIdFormat`, `ModifyIdentityIdFormat`, `ModifyInstanceCapacityReservationAttributes`, `ModifyInstanceCreditSpecification`, `ModifyInstanceEventStartTime`
+
+`ModifyInstanceEventWindow`, `ModifyInstanceMaintenanceOptions`, `ModifyInstanceMetadataDefaults`, `ModifyInstancePlacement`, `ModifyIpam`, `ModifyIpamPool`, `ModifyIpamResourceCidr`, `ModifyIpamResourceDiscovery`
+
+`ModifyIpamScope`, `ModifyLocalGatewayRoute`, `ModifyManagedPrefixList`, `ModifyPrivateDnsNameOptions`, `ModifyReservedInstances`, `ModifySecurityGroupRules`, `ModifySnapshotAttribute`, `ModifySnapshotTier`
+
+`ModifySpotFleetRequest`, `ModifyTrafficMirrorFilterNetworkServices`, `ModifyTrafficMirrorFilterRule`, `ModifyTrafficMirrorSession`, `ModifyTransitGateway`, `ModifyTransitGatewayPrefixListReference`, `ModifyTransitGatewayVpcAttachment`, `ModifyVerifiedAccessEndpoint`
+
+`ModifyVerifiedAccessEndpointPolicy`, `ModifyVerifiedAccessGroup`, `ModifyVerifiedAccessGroupPolicy`, `ModifyVerifiedAccessInstance`, `ModifyVerifiedAccessInstanceLoggingConfiguration`, `ModifyVerifiedAccessTrustProvider`, `ModifyVolumeAttribute`, `ModifyVpcEndpoint`
+
+`ModifyVpcEndpointConnectionNotification`, `ModifyVpcEndpointServiceConfiguration`, `ModifyVpcEndpointServicePayerResponsibility`, `ModifyVpcEndpointServicePermissions`, `ModifyVpcPeeringConnectionOptions`, `ModifyVpcTenancy`, `ModifyVpnConnection`, `ModifyVpnConnectionOptions`
+
+`ModifyVpnTunnelCertificate`, `ModifyVpnTunnelOptions`, `MonitorInstances`, `MoveAddressToVpc`, `MoveByoipCidrToIpam`, `ProvisionByoipCidr`, `ProvisionIpamByoasn`, `ProvisionIpamPoolCidr`
+
+`ProvisionPublicIpv4PoolCidr`, `PurchaseCapacityBlock`, `PurchaseHostReservation`, `PurchaseReservedInstancesOffering`, `PurchaseScheduledInstances`, `RegisterInstanceEventNotificationAttributes`, `RegisterTransitGatewayMulticastGroupMembers`, `RegisterTransitGatewayMulticastGroupSources`
+
+`RejectTransitGatewayMulticastDomainAssociations`, `RejectTransitGatewayPeeringAttachment`, `RejectTransitGatewayVpcAttachment`, `RejectVpcEndpointConnections`, `RejectVpcPeeringConnection`, `ReleaseHosts`, `ReleaseIpamPoolAllocation`, `ReplaceNetworkAclAssociation`
+
+`ReplaceNetworkAclEntry`, `ReplaceTransitGatewayRoute`, `ReplaceVpnTunnel`, `ReportInstanceStatus`, `RequestSpotFleet`, `ResetAddressAttribute`, `ResetEbsDefaultKmsKeyId`, `ResetFpgaImageAttribute`
+
+`ResetInstanceAttribute`, `ResetNetworkInterfaceAttribute`, `ResetSnapshotAttribute`, `RestoreAddressToClassic`, `RestoreImageFromRecycleBin`, `RestoreManagedPrefixListVersion`, `RestoreSnapshotFromRecycleBin`, `RestoreSnapshotTier`
+
+`RevokeClientVpnIngress`, `RunScheduledInstances`, `SearchLocalGatewayRoutes`, `SearchTransitGatewayMulticastGroups`, `SearchTransitGatewayRoutes`, `SendDiagnosticInterrupt`, `StartNetworkInsightsAccessScopeAnalysis`, `StartNetworkInsightsAnalysis`
+
+`StartVpcEndpointServicePrivateDnsVerification`, `TerminateClientVpnConnections`, `UnassignIpv6Addresses`, `UnassignPrivateIpAddresses`, `UnassignPrivateNatGatewayAddress`, `UnlockSnapshot`, `UnmonitorInstances`, `UpdateSecurityGroupRuleDescriptionsEgress`
+
+`UpdateSecurityGroupRuleDescriptionsIngress`, `WithdrawByoipCidr`
+
+
+</details>
+
+<details><summary>Registered stubs (0)</summary>
+
+None.
+
+</details>
+
+<details><summary>Deliberately unsupported (0)</summary>
+
+None.
+
+</details>
+
+<details><summary>Registered outside the pinned model (0)</summary>
+
+None.
+
+</details>
+
+## ecr
+
+Implements **19 of 47** modelled operations (40.4%).
+
+<details><summary>Implemented (19)</summary>
+
+`BatchDeleteImage`, `BatchGetImage`, `CreateRepository`, `DeleteLifecyclePolicy`, `DeleteRepository`, `DeleteRepositoryPolicy`, `DescribeImages`, `DescribeRepositories`
+
+`GetAuthorizationToken`, `GetLifecyclePolicy`, `GetLifecyclePolicyPreview`, `GetRepositoryPolicy`, `ListImages`, `ListTagsForResource`, `PutImage`, `PutImageTagMutability`
+
+`PutLifecyclePolicy`, `SetRepositoryPolicy`, `StartLifecyclePolicyPreview`
+
+
+</details>
+
+<details><summary>Missing from dispatch (11)</summary>
+
+`CreatePullThroughCacheRule`, `CreateRepositoryCreationTemplate`, `DeletePullThroughCacheRule`, `DeleteRegistryPolicy`, `DeleteRepositoryCreationTemplate`, `DescribeImageReplicationStatus`, `DescribePullThroughCacheRules`, `DescribeRepositoryCreationTemplates`
+
+`UpdatePullThroughCacheRule`, `UpdateRepositoryCreationTemplate`, `ValidatePullThroughCacheRule`
+
+
+</details>
+
+<details><summary>Registered stubs (13)</summary>
+
+`BatchCheckLayerAvailability`, `CompleteLayerUpload`, `DescribeRegistry`, `GetDownloadUrlForLayer`, `GetRegistryPolicy`, `InitiateLayerUpload`, `ListRepositories`, `PutRegistryPolicy`
+
+`PutReplicationConfiguration`, `ReplicateImage`, `TagResource`, `UntagResource`, `UploadLayerPart`
+
+
+</details>
+
+<details><summary>Deliberately unsupported (7)</summary>
+
+`BatchGetRepositoryScanningConfiguration`, `DescribeImageScanFindings`, `GetImageScanningConfiguration`, `GetRegistryScanningConfiguration`, `PutImageScanningConfiguration`, `PutRegistryScanningConfiguration`, `StartImageScan`
+
+
+</details>
+
+<details><summary>Registered outside the pinned model (3)</summary>
+
+`GetImageScanningConfiguration`, `ListRepositories`, `ReplicateImage`
+
+
+</details>
+
+## ecs
+
+Implements **31 of 56** modelled operations (55.4%).
+
+<details><summary>Implemented (31)</summary>
+
+`CreateCapacityProvider`, `CreateCluster`, `CreateService`, `DeleteCapacityProvider`, `DeleteCluster`, `DeleteService`, `DeregisterContainerInstance`, `DeregisterTaskDefinition`
+
+`DescribeCapacityProviders`, `DescribeClusters`, `DescribeContainerInstances`, `DescribeServices`, `DescribeTaskDefinition`, `DescribeTasks`, `ListClusters`, `ListContainerInstances`
+
+`ListServices`, `ListTagsForResource`, `ListTaskDefinitions`, `ListTasks`, `PutClusterCapacityProviders`, `RegisterContainerInstance`, `RegisterTaskDefinition`, `RunTask`
+
+`StartTask`, `StopTask`, `SubmitTaskStateChange`, `TagResource`, `UntagResource`, `UpdateContainerInstancesState`, `UpdateService`
+
+
+</details>
+
+<details><summary>Missing from dispatch (20)</summary>
+
+`CreateTaskSet`, `DeleteAccountSetting`, `DeleteAttributes`, `DeleteTaskDefinitions`, `DeleteTaskSet`, `DescribeTaskSets`, `DiscoverPollEndpoint`, `ExecuteCommand`
+
+`GetTaskProtection`, `ListAttributes`, `PutAccountSettingDefault`, `PutAttributes`, `SubmitAttachmentStateChanges`, `SubmitContainerStateChange`, `UpdateCapacityProvider`, `UpdateClusterSettings`
+
+`UpdateContainerAgent`, `UpdateServicePrimaryTaskSet`, `UpdateTaskProtection`, `UpdateTaskSet`
+
+
+</details>
+
+<details><summary>Registered stubs (5)</summary>
+
+`ListAccountSettings`, `ListServicesByNamespace`, `ListTaskDefinitionFamilies`, `PutAccountSetting`, `UpdateCluster`
+
+
+</details>
+
+<details><summary>Deliberately unsupported (0)</summary>
+
+None.
+
+</details>
+
+<details><summary>Registered outside the pinned model (3)</summary>
+
+`PollAssignments`, `ProvisionCapacity`, `ReportTaskGPU`
+
+
+</details>
+
+## elasticloadbalancingv2
+
+Implements **33 of 46** modelled operations (71.7%).
+
+<details><summary>Implemented (33)</summary>
+
+`AddListenerCertificates`, `AddTags`, `CreateListener`, `CreateLoadBalancer`, `CreateRule`, `CreateTargetGroup`, `DeleteListener`, `DeleteLoadBalancer`
+
+`DeleteRule`, `DeleteTargetGroup`, `DeregisterTargets`, `DescribeListenerCertificates`, `DescribeListeners`, `DescribeLoadBalancerAttributes`, `DescribeLoadBalancers`, `DescribeRules`
+
+`DescribeSSLPolicies`, `DescribeTags`, `DescribeTargetGroupAttributes`, `DescribeTargetGroups`, `DescribeTargetHealth`, `ModifyListener`, `ModifyLoadBalancerAttributes`, `ModifyRule`
+
+`ModifyTargetGroup`, `ModifyTargetGroupAttributes`, `RegisterTargets`, `RemoveListenerCertificates`, `RemoveTags`, `SetIpAddressType`, `SetRulePriorities`, `SetSecurityGroups`
+
+`SetSubnets`
+
+
+</details>
+
+<details><summary>Missing from dispatch (13)</summary>
+
+`AddTrustStoreRevocations`, `CreateTrustStore`, `DeleteSharedTrustStoreAssociation`, `DeleteTrustStore`, `DescribeAccountLimits`, `DescribeTrustStoreAssociations`, `DescribeTrustStoreRevocations`, `DescribeTrustStores`
+
+`GetResourcePolicy`, `GetTrustStoreCaCertificatesBundle`, `GetTrustStoreRevocationContent`, `ModifyTrustStore`, `RemoveTrustStoreRevocations`
+
+
+</details>
+
+<details><summary>Registered stubs (0)</summary>
+
+None.
+
+</details>
+
+<details><summary>Deliberately unsupported (0)</summary>
+
+None.
+
+</details>
+
+<details><summary>Registered outside the pinned model (4)</summary>
+
+`DescribeListenerAttributes`, `GetLBConfig`, `LBAgentHeartbeat`, `ModifyListenerAttributes`
+
+
+</details>
+
+## iam
+
+Implements **75 of 159** modelled operations (47.2%).
+
+<details><summary>Implemented (75)</summary>
+
+`AddRoleToInstanceProfile`, `AddUserToGroup`, `AttachGroupPolicy`, `AttachRolePolicy`, `AttachUserPolicy`, `CreateAccessKey`, `CreateGroup`, `CreateInstanceProfile`
+
+`CreateOpenIDConnectProvider`, `CreatePolicy`, `CreateRole`, `CreateUser`, `DeleteAccessKey`, `DeleteGroup`, `DeleteGroupPolicy`, `DeleteInstanceProfile`
+
+`DeleteOpenIDConnectProvider`, `DeletePolicy`, `DeleteRole`, `DeleteRolePolicy`, `DeleteUser`, `DeleteUserPolicy`, `DetachGroupPolicy`, `DetachRolePolicy`
+
+`DetachUserPolicy`, `GetAccountSummary`, `GetGroup`, `GetGroupPolicy`, `GetInstanceProfile`, `GetOpenIDConnectProvider`, `GetPolicy`, `GetPolicyVersion`
+
+`GetRole`, `GetRolePolicy`, `GetUser`, `GetUserPolicy`, `ListAccessKeys`, `ListAttachedGroupPolicies`, `ListAttachedRolePolicies`, `ListAttachedUserPolicies`
+
+`ListGroupPolicies`, `ListGroups`, `ListGroupsForUser`, `ListInstanceProfileTags`, `ListInstanceProfiles`, `ListInstanceProfilesForRole`, `ListOpenIDConnectProviderTags`, `ListOpenIDConnectProviders`
+
+`ListPolicies`, `ListPolicyTags`, `ListPolicyVersions`, `ListRolePolicies`, `ListRoleTags`, `ListRoles`, `ListUserPolicies`, `ListUserTags`
+
+`ListUsers`, `PutGroupPolicy`, `PutRolePolicy`, `PutUserPolicy`, `RemoveRoleFromInstanceProfile`, `RemoveUserFromGroup`, `TagInstanceProfile`, `TagOpenIDConnectProvider`
+
+`TagPolicy`, `TagRole`, `TagUser`, `UntagInstanceProfile`, `UntagOpenIDConnectProvider`, `UntagPolicy`, `UntagRole`, `UntagUser`
+
+`UpdateAccessKey`, `UpdateAssumeRolePolicy`, `UpdateRole`
+
+
+</details>
+
+<details><summary>Missing from dispatch (84)</summary>
+
+`AddClientIDToOpenIDConnectProvider`, `ChangePassword`, `CreateAccountAlias`, `CreateLoginProfile`, `CreatePolicyVersion`, `CreateSAMLProvider`, `CreateServiceLinkedRole`, `CreateServiceSpecificCredential`
+
+`CreateVirtualMFADevice`, `DeactivateMFADevice`, `DeleteAccountAlias`, `DeleteAccountPasswordPolicy`, `DeleteLoginProfile`, `DeletePolicyVersion`, `DeleteRolePermissionsBoundary`, `DeleteSAMLProvider`
+
+`DeleteSSHPublicKey`, `DeleteServerCertificate`, `DeleteServiceLinkedRole`, `DeleteServiceSpecificCredential`, `DeleteSigningCertificate`, `DeleteUserPermissionsBoundary`, `DeleteVirtualMFADevice`, `EnableMFADevice`
+
+`GenerateCredentialReport`, `GenerateOrganizationsAccessReport`, `GenerateServiceLastAccessedDetails`, `GetAccessKeyLastUsed`, `GetAccountAuthorizationDetails`, `GetAccountPasswordPolicy`, `GetContextKeysForCustomPolicy`, `GetContextKeysForPrincipalPolicy`
+
+`GetCredentialReport`, `GetLoginProfile`, `GetMFADevice`, `GetOrganizationsAccessReport`, `GetSAMLProvider`, `GetSSHPublicKey`, `GetServerCertificate`, `GetServiceLastAccessedDetails`
+
+`GetServiceLastAccessedDetailsWithEntities`, `GetServiceLinkedRoleDeletionStatus`, `ListAccountAliases`, `ListEntitiesForPolicy`, `ListMFADeviceTags`, `ListMFADevices`, `ListPoliciesGrantingServiceAccess`, `ListSAMLProviderTags`
+
+`ListSAMLProviders`, `ListSSHPublicKeys`, `ListServerCertificateTags`, `ListServerCertificates`, `ListServiceSpecificCredentials`, `ListSigningCertificates`, `ListVirtualMFADevices`, `PutRolePermissionsBoundary`
+
+`PutUserPermissionsBoundary`, `RemoveClientIDFromOpenIDConnectProvider`, `ResetServiceSpecificCredential`, `ResyncMFADevice`, `SetDefaultPolicyVersion`, `SetSecurityTokenServicePreferences`, `SimulateCustomPolicy`, `SimulatePrincipalPolicy`
+
+`TagMFADevice`, `TagSAMLProvider`, `TagServerCertificate`, `UntagMFADevice`, `UntagSAMLProvider`, `UntagServerCertificate`, `UpdateAccountPasswordPolicy`, `UpdateGroup`
+
+`UpdateLoginProfile`, `UpdateOpenIDConnectProviderThumbprint`, `UpdateRoleDescription`, `UpdateSAMLProvider`, `UpdateSSHPublicKey`, `UpdateServerCertificate`, `UpdateServiceSpecificCredential`, `UpdateSigningCertificate`
+
+`UpdateUser`, `UploadSSHPublicKey`, `UploadServerCertificate`, `UploadSigningCertificate`
+
+
+</details>
+
+<details><summary>Registered stubs (0)</summary>
+
+None.
+
+</details>
+
+<details><summary>Deliberately unsupported (0)</summary>
+
+None.
+
+</details>
+
+<details><summary>Registered outside the pinned model (0)</summary>
+
+None.
+
+</details>
+
+## rds
+
+Implements **24 of 162** modelled operations (14.8%).
+
+<details><summary>Implemented (24)</summary>
+
+`AddTagsToResource`, `CreateDBInstance`, `CreateDBParameterGroup`, `CreateDBSnapshot`, `CreateDBSubnetGroup`, `DeleteDBInstance`, `DeleteDBParameterGroup`, `DeleteDBSnapshot`
+
+`DeleteDBSubnetGroup`, `DescribeDBInstanceAutomatedBackups`, `DescribeDBInstances`, `DescribeDBParameterGroups`, `DescribeDBParameters`, `DescribeDBSnapshots`, `DescribeDBSubnetGroups`, `DescribeEvents`
+
+`ListTagsForResource`, `ModifyDBInstance`, `ModifyDBParameterGroup`, `RebootDBInstance`, `RemoveTagsFromResource`, `RestoreDBInstanceFromDBSnapshot`, `StartDBInstance`, `StopDBInstance`
+
+
+</details>
+
+<details><summary>Missing from dispatch (126)</summary>
+
+`AddRoleToDBCluster`, `AddRoleToDBInstance`, `AddSourceIdentifierToSubscription`, `ApplyPendingMaintenanceAction`, `AuthorizeDBSecurityGroupIngress`, `BacktrackDBCluster`, `CancelExportTask`, `CopyDBClusterParameterGroup`
+
+`CopyDBClusterSnapshot`, `CopyDBParameterGroup`, `CopyDBSnapshot`, `CopyOptionGroup`, `CreateBlueGreenDeployment`, `CreateCustomDBEngineVersion`, `CreateDBClusterEndpoint`, `CreateDBClusterParameterGroup`
+
+`CreateDBClusterSnapshot`, `CreateDBProxy`, `CreateDBProxyEndpoint`, `CreateDBSecurityGroup`, `CreateDBShardGroup`, `CreateEventSubscription`, `CreateGlobalCluster`, `CreateIntegration`
+
+`CreateTenantDatabase`, `DeleteBlueGreenDeployment`, `DeleteCustomDBEngineVersion`, `DeleteDBClusterAutomatedBackup`, `DeleteDBClusterEndpoint`, `DeleteDBClusterParameterGroup`, `DeleteDBClusterSnapshot`, `DeleteDBInstanceAutomatedBackup`
+
+`DeleteDBProxy`, `DeleteDBProxyEndpoint`, `DeleteDBSecurityGroup`, `DeleteDBShardGroup`, `DeleteEventSubscription`, `DeleteGlobalCluster`, `DeleteIntegration`, `DeleteTenantDatabase`
+
+`DeregisterDBProxyTargets`, `DescribeAccountAttributes`, `DescribeBlueGreenDeployments`, `DescribeCertificates`, `DescribeDBClusterAutomatedBackups`, `DescribeDBClusterBacktracks`, `DescribeDBClusterEndpoints`, `DescribeDBClusterParameterGroups`
+
+`DescribeDBClusterParameters`, `DescribeDBClusterSnapshotAttributes`, `DescribeDBClusterSnapshots`, `DescribeDBEngineVersions`, `DescribeDBLogFiles`, `DescribeDBProxies`, `DescribeDBProxyEndpoints`, `DescribeDBProxyTargetGroups`
+
+`DescribeDBProxyTargets`, `DescribeDBRecommendations`, `DescribeDBSecurityGroups`, `DescribeDBShardGroups`, `DescribeDBSnapshotAttributes`, `DescribeDBSnapshotTenantDatabases`, `DescribeEngineDefaultClusterParameters`, `DescribeEngineDefaultParameters`
+
+`DescribeEventCategories`, `DescribeEventSubscriptions`, `DescribeExportTasks`, `DescribeGlobalClusters`, `DescribeIntegrations`, `DescribeOptionGroupOptions`, `DescribeOrderableDBInstanceOptions`, `DescribePendingMaintenanceActions`
+
+`DescribeReservedDBInstances`, `DescribeReservedDBInstancesOfferings`, `DescribeSourceRegions`, `DescribeTenantDatabases`, `DescribeValidDBInstanceModifications`, `DisableHttpEndpoint`, `DownloadDBLogFilePortion`, `EnableHttpEndpoint`
+
+`FailoverGlobalCluster`, `ModifyActivityStream`, `ModifyCertificates`, `ModifyCurrentDBClusterCapacity`, `ModifyCustomDBEngineVersion`, `ModifyDBClusterEndpoint`, `ModifyDBClusterParameterGroup`, `ModifyDBClusterSnapshotAttribute`
+
+`ModifyDBProxy`, `ModifyDBProxyEndpoint`, `ModifyDBProxyTargetGroup`, `ModifyDBRecommendation`, `ModifyDBShardGroup`, `ModifyDBSnapshot`, `ModifyDBSnapshotAttribute`, `ModifyDBSubnetGroup`
+
+`ModifyEventSubscription`, `ModifyGlobalCluster`, `ModifyIntegration`, `ModifyTenantDatabase`, `PromoteReadReplicaDBCluster`, `PurchaseReservedDBInstancesOffering`, `RebootDBCluster`, `RebootDBShardGroup`
+
+`RegisterDBProxyTargets`, `RemoveFromGlobalCluster`, `RemoveRoleFromDBCluster`, `RemoveRoleFromDBInstance`, `RemoveSourceIdentifierFromSubscription`, `ResetDBClusterParameterGroup`, `ResetDBParameterGroup`, `RestoreDBClusterFromS3`
+
+`RestoreDBClusterFromSnapshot`, `RestoreDBClusterToPointInTime`, `RestoreDBInstanceFromS3`, `RevokeDBSecurityGroupIngress`, `StartActivityStream`, `StartDBCluster`, `StartDBInstanceAutomatedBackupsReplication`, `StartExportTask`
+
+`StopActivityStream`, `StopDBCluster`, `StopDBInstanceAutomatedBackupsReplication`, `SwitchoverBlueGreenDeployment`, `SwitchoverGlobalCluster`, `SwitchoverReadReplica`
+
+
+</details>
+
+<details><summary>Registered stubs (0)</summary>
+
+None.
+
+</details>
+
+<details><summary>Deliberately unsupported (12)</summary>
+
+`CreateDBCluster`, `CreateDBInstanceReadReplica`, `CreateOptionGroup`, `DeleteDBCluster`, `DeleteOptionGroup`, `DescribeDBClusters`, `DescribeOptionGroups`, `FailoverDBCluster`
+
+`ModifyDBCluster`, `ModifyOptionGroup`, `PromoteReadReplica`, `RestoreDBInstanceToPointInTime`
+
+
+</details>
+
+<details><summary>Registered outside the pinned model (4)</summary>
+
+`GetDBBootstrapConfig`, `PollDBCommands`, `RegisterDBInstance`, `SubmitDBStateChange`
+
+
+</details>
+
+## s3
+
+Operation coverage is not enumerable: Spinifex delegates the S3 REST surface to Predastore, which has no operation-name dispatch table to compare mechanically.
+
+## sts
+
+Implements **4 of 8** modelled operations (50.0%).
+
+<details><summary>Implemented (4)</summary>
+
+`AssumeRole`, `AssumeRoleWithWebIdentity`, `GetCallerIdentity`, `GetSessionToken`
+
+
+</details>
+
+<details><summary>Missing from dispatch (4)</summary>
+
+`AssumeRoleWithSAML`, `DecodeAuthorizationMessage`, `GetAccessKeyInfo`, `GetFederationToken`
+
+
+</details>
+
+<details><summary>Registered stubs (0)</summary>
+
+None.
+
+</details>
+
+<details><summary>Deliberately unsupported (0)</summary>
+
+None.
+
+</details>
+
+<details><summary>Registered outside the pinned model (0)</summary>
+
+None.
+
+</details>

docs/compute/vpc-networking/README.md

@@ -16,10 +16,7 @@ tags:
 
 ## Overview
 
-Spinifex provides AWS-compatible VPC networking on bare-metal. Every EC2 instance
-runs inside an isolated virtual network backed by OVN (Open Virtual Network).
-Instances can operate in two modes: **private** (overlay-only, no WAN access) or
-**public** (routable from the WAN with a unique public IP).
+Spinifex provides AWS-compatible VPC networking on bare-metal. Every EC2 instance runs inside an isolated virtual network backed by OVN (Open Virtual Network). Instances can operate in two modes: **private** (overlay-only, no WAN access) or **public** (routable from the WAN with a unique public IP).
 
 ## Instructions
 
@@ -42,35 +39,25 @@ Spinifex maps AWS VPC concepts directly to OVN constructs:
   <img src="../../../.github/assets/diagrams/vpc-network-path.svg" alt="VPC logical topology — WAN, br-wan, VPC logical router, subnets, ENIs" width="900">
 </p>
 
-Cross-host traffic uses **Geneve tunnels** (UDP 6081) over the management/overlay
-NIC. Each host runs `ovn-controller` which programs OpenFlow rules on `br-int`
-(the integration bridge where all VM TAP devices connect).
+Cross-host traffic uses **Geneve tunnels** (UDP 6081) over the management/overlay NIC. Each host runs `ovn-controller` which programs OpenFlow rules on `br-int` (the integration bridge where all VM TAP devices connect).
 
 ## Private vs Public Subnets
 
-A subnet's behavior depends on three things: whether the VPC has an Internet
-Gateway, whether the subnet's route table has a default route to that IGW, and
-whether the subnet has `MapPublicIpOnLaunch` enabled.
+A subnet's behavior depends on three things: whether the VPC has an Internet Gateway, whether the subnet's route table has a default route to that IGW, and whether the subnet has `MapPublicIpOnLaunch` enabled.
 
 ## Private Subnet (Default)
 
-Instances get a private IP only. They can communicate with other instances in the
-same VPC (even across subnets and hosts via the overlay). They cannot reach the
-internet or be reached from the WAN.
+Instances get a private IP only. They can communicate with other instances in the same VPC (even across subnets and hosts via the overlay). They cannot reach the internet or be reached from the WAN.
 
 <p align="center">
   <img src="../../../.github/assets/diagrams/vpc-private-subnet-flow.svg" alt="Private subnet — instance hits router, no default route, packet dropped" width="900">
 </p>
 
-Private subnet instances reach the internet only if their route table has a
-default route to the IGW (shared SNAT, outbound only — they share the gateway
-IP) or to a NAT gateway. With no default route, egress is dropped. Either way
-they cannot be reached from the WAN because they have no public IP.
+Private subnet instances reach the internet only if their route table has a default route to the IGW (shared SNAT, outbound only — they share the gateway IP) or to a NAT gateway. With no default route, egress is dropped. Either way they cannot be reached from the WAN because they have no public IP.
 
 ## Public Subnet
 
-Instances get both a private IP and a public IP. The public IP is a 1:1 NAT
-managed by OVN — the instance OS only sees its private IP.
+Instances get both a private IP and a public IP. The public IP is a 1:1 NAT managed by OVN — the instance OS only sees its private IP.
 
 <p align="center">
   <img src="../../../.github/assets/diagrams/vpc-public-subnet-flow.svg" alt="Public subnet — outbound SNAT and inbound DNAT between private and public IPs" width="900">
@@ -83,14 +70,7 @@ managed by OVN — the instance OS only sees its private IP.
 3. Subnet has `MapPublicIpOnLaunch = true`
 4. External IP pool configured in `spinifex.toml`
 
-Spinifex follows AWS route-table semantics: a subnet is only "public" if its
-effective route table carries a default route to the IGW. A new VPC's main
-route table has the local route only — Spinifex does **not** add the IGW route
-for you. Without it, the subnet's egress is gated with a drop policy, so
-instances cannot reach the internet (and inbound connections cannot complete
-because return traffic is dropped) even with a public IP and an attached IGW.
-Add the route explicitly — either to the main route table, or to a custom route
-table associated with the subnet (see [Quick Start](#3-create-vpc-with-public-subnet)).
+Spinifex follows AWS route-table semantics: a subnet is only "public" if its effective route table carries a default route to the IGW. A new VPC's main route table has the local route only — Spinifex does **not** add the IGW route for you. Without it, the subnet's egress is gated with a drop policy, so instances cannot reach the internet (and inbound connections cannot complete because return traffic is dropped) even with a public IP and an attached IGW. Add the route explicitly — either to the main route table, or to a custom route table associated with the subnet (see [Quick Start](#3-create-vpc-with-public-subnet)).
 
 ## Comparison
 
@@ -105,14 +85,11 @@ table associated with the subnet (see [Quick Start](#3-create-vpc-with-public-su
 
 ## External Connectivity Modes
 
-The `[network]` section in `spinifex.toml` controls how VMs reach the outside
-world. There are three modes, and pool mode has two IP sources (static or DHCP).
+The `[network]` section in `spinifex.toml` controls how VMs reach the outside world. There are three modes, and pool mode has two IP sources (static or DHCP).
 
 ## `pool` — Full Public Networking (Recommended)
 
-Each VM in a public subnet gets its own public IP with bidirectional 1:1 NAT.
-Supports the full AWS feature set: public subnets, auto-assign public IPs,
-Elastic IPs, and security groups.
+Each VM in a public subnet gets its own public IP with bidirectional 1:1 NAT. Supports the full AWS feature set: public subnets, auto-assign public IPs, Elastic IPs, and security groups.
 
 Pool mode supports two ways to obtain public IPs:
 
@@ -120,11 +97,9 @@ Pool mode supports two ways to obtain public IPs:
 
 The admin defines a range of routable IPs that Spinifex manages exclusively.
 
-**Use when:** You have a block of IPs you control — datacenter ISP allocation,
-homelab range carved out of your router's DHCP scope, enterprise DMZ range.
+**Use when:** You have a block of IPs you control — datacenter ISP allocation, homelab range carved out of your router's DHCP scope, enterprise DMZ range.
 
-**Requirement:** The IP range must NOT be served by any other DHCP server. In a
-homelab, shrink your router's DHCP scope to exclude the Spinifex range.
+**Requirement:** The IP range must NOT be served by any other DHCP server. In a homelab, shrink your router's DHCP scope to exclude the Spinifex range.
 
 ```toml
 [network]
@@ -141,17 +116,11 @@ dns_servers = ["192.168.1.1", "8.8.8.8"]
 
 ### DHCP Source
 
-Instead of a static range, public IPs come from the upstream router's DHCP
-server. When a VM launches, Spinifex requests a DHCP lease from the router
-on behalf of the VM. When the VM terminates, the lease is released.
+Instead of a static range, public IPs come from the upstream router's DHCP server. When a VM launches, Spinifex requests a DHCP lease from the router on behalf of the VM. When the VM terminates, the lease is released.
 
-The VM itself never talks to the router's DHCP — it only sees its private
-VPC IP (from OVN's internal DHCP). The host-side DHCP conversation is
-invisible to the guest.
+The VM itself never talks to the router's DHCP — it only sees its private VPC IP (from OVN's internal DHCP). The host-side DHCP conversation is invisible to the guest.
 
-**Use when:** You don't control a static IP block but the router's DHCP
-server has enough leases. Homelabs where you don't want to carve out a range.
-Environments where IPs are managed centrally by the network team's DHCP.
+**Use when:** You don't control a static IP block but the router's DHCP server has enough leases. Homelabs where you don't want to carve out a range. Environments where IPs are managed centrally by the network team's DHCP.
 
 **Requirement:** `dhclient` or `dhcpcd-base` installed on the host.
 
@@ -170,8 +139,7 @@ dns_servers = ["192.168.1.1", "8.8.8.8"]
 
 ### How Pool Mode Works (Both Sources)
 
-Regardless of whether IPs come from a static range or DHCP, the OVN behavior
-is identical:
+Regardless of whether IPs come from a static range or DHCP, the OVN behavior is identical:
 
 <p align="center">
   <img src="../../../.github/assets/diagrams/vpc-dhcp-conversations.svg" alt="Two independent DHCP conversations — host-to-router and VM-to-OVN" width="900">
@@ -188,15 +156,11 @@ is identical:
 | **Best for**          | Datacenters, ISP blocks, production          | Homelabs, dev environments, shared networks      |
 | **Capacity**          | Exact: `range_end - range_start` IPs         | Limited by router's DHCP pool size               |
 
-Both support the same AWS features: public subnets, Elastic IPs, security groups,
-DescribeInstances showing public IPs.
+Both support the same AWS features: public subnets, Elastic IPs, security groups, DescribeInstances showing public IPs.
 
 ## `nat` — Shared SNAT (Simple)
 
-All VMs share a single external IP for outbound SNAT. By default there are no
-public IPs, no Elastic IPs, and no inbound from WAN — all subnets behave as
-private subnets with internet access. On routed-NAT nodes, adding a public pool
-restores full public IP parity (see below).
+All VMs share a single external IP for outbound SNAT. By default there are no public IPs, no Elastic IPs, and no inbound from WAN — all subnets behave as private subnets with internet access. On routed-NAT nodes, adding a public pool restores full public IP parity (see below).
 
 > **Limitation (routed-NAT v1):** System instances (ECS/EKS/load-balancer
 > agents) source egress from ExternalIPAM pool IPs, which do not exist in
@@ -204,12 +168,9 @@ restores full public IP parity (see below).
 > either allocate transit IPs for system instances or reject the feature at the
 > API level in nat mode.
 
-The `gateway_ip` is the IP that OVN uses for SNAT. You can set it statically or
-use `setup-ovn.sh --dhcp` to obtain one from the router. This is the router's
-DHCP — not Spinifex's internal OVN DHCP for VMs.
+The `gateway_ip` is the IP that OVN uses for SNAT. You can set it statically or use `setup-ovn.sh --dhcp` to obtain one from the router. This is the router's DHCP — not Spinifex's internal OVN DHCP for VMs.
 
-**Use when:** VMs only need outbound access (apt update, pulling images). Edge
-deployments behind ISP NAT. Single WAN IP available.
+**Use when:** VMs only need outbound access (apt update, pulling images). Edge deployments behind ISP NAT. Single WAN IP available.
 
 ```toml
 [network]
@@ -224,13 +185,9 @@ prefix_len = 24
 
 ### Host access to instances (jumpbox pattern)
 
-The spinifex host automatically reaches every instance's **private IP**: IGW
-attach installs a host route into OVN (`<vpc-cidr> via <gateway-transit-ip>
-dev spx-nat-host`) and exempts the transit net from SNAT, so replies to
-host-initiated connections come back un-NATted. No per-instance setup.
+The spinifex host automatically reaches every instance's **private IP**: IGW attach installs a host route into OVN (`<vpc-cidr> via <gateway-transit-ip> dev spx-nat-host`) and exempts the transit net from SNAT, so replies to host-initiated connections come back un-NATted. No per-instance setup.
 
-Security groups still apply and the default SG is closed to the host, same as
-AWS — open SSH/ICMP from the transit net first:
+Security groups still apply and the default SG is closed to the host, same as AWS — open SSH/ICMP from the transit net first:
 
 ```bash
 aws ec2 authorize-security-group-ingress --group-id $SG \
@@ -243,19 +200,11 @@ Then use the host as a jumpbox for remote access:
 ssh -J admin@<spinifex-host> ubuntu@<instance-private-ip>
 ```
 
-Extra networks that must reach instances without SNAT (e.g. a management LAN)
-can be added via `[network] nat_exempt_cidrs = ["192.168.50.0/24"]`.
+Extra networks that must reach instances without SNAT (e.g. a management LAN) can be added via `[network] nat_exempt_cidrs = ["192.168.50.0/24"]`.
 
 ### Public IPs in NAT mode (public pool)
 
-A routed-NAT node (`setup-ovn.sh --nat-uplink` + `spx admin init
---external-mode=nat`) can carry a public pool alongside the internal
-`nat-transit` pool. With one configured, nat mode behaves like pool mode for
-public IPs: `MapPublicIpOnLaunch` on the default subnet, auto-assigned public
-IPs, and Elastic IPs all work. Spinifex delivers each public IP at the host —
-a `/32` route steers it into OVN and a proxy-ARP neighbor entry answers for it
-on the uplink (L3 only, same MAC, so it works on WiFi and other non-bridgeable
-uplinks).
+A routed-NAT node (`setup-ovn.sh --nat-uplink` + `spx admin init --external-mode=nat`) can carry a public pool alongside the internal `nat-transit` pool. With one configured, nat mode behaves like pool mode for public IPs: `MapPublicIpOnLaunch` on the default subnet, auto-assigned public IPs, and Elastic IPs all work. Spinifex delivers each public IP at the host — a `/32` route steers it into OVN and a proxy-ARP neighbor entry answers for it on the uplink (L3 only, same MAC, so it works on WiFi and other non-bridgeable uplinks).
 
 Static range carved out of the router's DHCP scope:
 
@@ -272,11 +221,7 @@ spx admin init --external-mode=nat --external-source=dhcp \
   --external-bind-bridge wlan0
 ```
 
-On WiFi/WWAN uplinks the leases are requested with the interface's own MAC
-(`dhcp_mac = "interface"`, written automatically) and distinguished by DHCP
-client-id. Some routers key leases by MAC and ignore the client-id — Spinifex
-detects this (the router hands the same IP to two client-ids) and fails the
-allocation with advice to switch to a static range.
+On WiFi/WWAN uplinks the leases are requested with the interface's own MAC (`dhcp_mac = "interface"`, written automatically) and distinguished by DHCP client-id. Some routers key leases by MAC and ignore the client-id — Spinifex detects this (the router hands the same IP to two client-ids) and fails the allocation with advice to switch to a static range.
 
 Resulting config:
 
@@ -298,18 +243,11 @@ gateway     = "192.168.1.1"
 prefix_len  = 24
 ```
 
-**Caveat — reaching an EIP from the spinifex host itself.** Host-sourced
-traffic enters OVN from the transit net, which is exempt from NAT (that is what
-makes the jumpbox pattern work) — so a host connection to an EIP that carries
-the transit source IP would skip DNAT. Spinifex stamps the EIP route with the
-uplink's LAN IP as source to avoid this, but if no uplink address can be
-determined, connect to the instance's **private IP** from the host instead.
-Other machines on the LAN are unaffected.
+**Caveat — reaching an EIP from the spinifex host itself.** Host-sourced traffic enters OVN from the transit net, which is exempt from NAT (that is what makes the jumpbox pattern work) — so a host connection to an EIP that carries the transit source IP would skip DNAT. Spinifex stamps the EIP route with the uplink's LAN IP as source to avoid this, but if no uplink address can be determined, connect to the instance's **private IP** from the host instead. Other machines on the LAN are unaffected.
 
 ## Disabled (Empty/Omitted)
 
-VPC networking is overlay-only. No external connectivity. Instances can only
-communicate within their VPC.
+VPC networking is overlay-only. No external connectivity. Instances can only communicate within their VPC.
 
 ## Mode Comparison
 
@@ -325,17 +263,11 @@ communicate within their VPC.
 | Admin must reserve IP range       | Yes             | No            | Only static pool  | No       |
 | Needs router DHCP                 | No              | Yes           | Optional          | No       |
 
-If you start with `nat` and later need public subnets: on a bridgeable uplink
-switch to `pool` and define a range (or use `source = "dhcp"`); on a routed-NAT
-node just add a public pool alongside `nat-transit` — no data migration needed.
+If you start with `nat` and later need public subnets: on a bridgeable uplink switch to `pool` and define a range (or use `source = "dhcp"`); on a routed-NAT node just add a public pool alongside `nat-transit` — no data migration needed.
 
 ## Bridge Setup — Physical Network Wiring
 
-The WAN NIC **must** be enslaved to a Linux bridge. This is a hard requirement —
-`setup-ovn.sh` will not attach a physical NIC directly to OVS, and macvlan is
-no longer supported. The Linux bridge owns the host IP, default route, and any
-DHCP lease, so SSH and management traffic stay up while OVS/OVN are configured
-underneath.
+The WAN NIC **must** be enslaved to a Linux bridge. This is a hard requirement — `setup-ovn.sh` will not attach a physical NIC directly to OVS, and macvlan is no longer supported. The Linux bridge owns the host IP, default route, and any DHCP lease, so SSH and management traffic stay up while OVS/OVN are configured underneath.
 
 The full datapath chain looks like this:
 
@@ -350,16 +282,9 @@ physical NIC (e.g. `wan`)
                                                             └─▶ TAP devices  (VM NICs)
 ```
 
-`setup-ovn.sh` auto-detects the Linux bridge that owns the default route
-(typically `br-wan`, provisioned by cloud-init / netplan / systemd-networkd).
-You can override the detection with `--wan-bridge=<name>`. Once detected, the
-script creates the OVS bridge `br-ext` and links it to the WAN bridge with a
-veth pair. The Linux bridge keeps its IP and routes — no interruption.
-Bridge-mapping is set to `external:br-ext`.
+`setup-ovn.sh` auto-detects the Linux bridge that owns the default route (typically `br-wan`, provisioned by cloud-init / netplan / systemd-networkd). You can override the detection with `--wan-bridge=<name>`. Once detected, the script creates the OVS bridge `br-ext` and links it to the WAN bridge with a veth pair. The Linux bridge keeps its IP and routes — no interruption. Bridge-mapping is set to `external:br-ext`.
 
-If the default route is on a bare physical NIC (no bridge), `setup-ovn.sh`
-stops and prints guidance on how to convert the NIC to a bridge before
-re-running.
+If the default route is on a bare physical NIC (no bridge), `setup-ovn.sh` stops and prints guidance on how to convert the NIC to a bridge before re-running.
 
 ### Example: Required `br-wan` State
 
@@ -374,8 +299,7 @@ The host must have something resembling this before `setup-ovn.sh` is run:
        valid_lft forever preferred_lft forever
 ```
 
-The physical NIC (e.g. `wan`, `eth0`, `eno1`) is enslaved to `br-wan` and has
-no IP of its own — all L3 state lives on the bridge.
+The physical NIC (e.g. `wan`, `eth0`, `eno1`) is enslaved to `br-wan` and has no IP of its own — all L3 state lives on the bridge.
 
 Example netplan that produces this:
 
@@ -401,14 +325,9 @@ Every Spinifex node has three bridges in the datapath:
 | `br-ext` | OVS bridge   | `setup-ovn.sh`              | OVN external uplink (`localnet`)              | veth peer to `br-wan`        |
 | `br-int` | OVS bridge   | `setup-ovn.sh`              | VM overlay traffic (Geneve tunnels)           | VM TAP devices, tunnel ports |
 
-`br-wan` is provisioned by your distro's network configuration (cloud-init,
-netplan, systemd-networkd, ifupdown). The name is configurable; `br-wan` is
-the convention. `br-int` and `br-ext` are always created by `setup-ovn.sh`.
+`br-wan` is provisioned by your distro's network configuration (cloud-init, netplan, systemd-networkd, ifupdown). The name is configurable; `br-wan` is the convention. `br-int` and `br-ext` are always created by `setup-ovn.sh`.
 
-The link between `br-ext` and the VM datapath is logical, not physical: OVN's
-`localnet` port type maps the logical external switch to `br-ext` via
-`ovn-bridge-mappings`. Frames egressing a VM travel TAP → `br-int` → OVN
-pipeline → `br-ext` → veth → `br-wan` → physical NIC → wire.
+The link between `br-ext` and the VM datapath is logical, not physical: OVN's `localnet` port type maps the logical external switch to `br-ext` via `ovn-bridge-mappings`. Frames egressing a VM travel TAP → `br-int` → OVN pipeline → `br-ext` → veth → `br-wan` → physical NIC → wire.
 
 <p align="center">
   <img src="../../../.github/assets/diagrams/vpc-datapath-localnet.svg" alt="Data path — VM TAP through br-int, OVN pipeline, br-ext, veth pair, br-wan, physical NIC" width="900">
@@ -424,17 +343,13 @@ sudo setup-ovn.sh
 sudo setup-ovn.sh --wan-bridge=br-wan
 ```
 
-In environments where the WAN IP comes from a router's DHCP server (homelab,
-small office), add `--dhcp` to obtain a gateway IP from the router
-automatically:
+In environments where the WAN IP comes from a router's DHCP server (homelab, small office), add `--dhcp` to obtain a gateway IP from the router automatically:
 
 ```bash
 sudo setup-ovn.sh --dhcp
 ```
 
-This requests an IP from the **router's DHCP** (e.g., 192.168.1.1 serving
-addresses on the LAN). This is not Spinifex's internal OVN DHCP that assigns
-private IPs to VMs — it's your network's existing DHCP server.
+This requests an IP from the **router's DHCP** (e.g., 192.168.1.1 serving addresses on the LAN). This is not Spinifex's internal OVN DHCP that assigns private IPs to VMs — it's your network's existing DHCP server.
 
 | Flags                          | Result                                                                |
 | ------------------------------ | --------------------------------------------------------------------- |
@@ -442,8 +357,7 @@ private IPs to VMs — it's your network's existing DHCP server.
 | `--wan-bridge=<name>`          | Use the specified Linux bridge as the WAN uplink                      |
 | `--dhcp`                       | Obtain the OVN gateway IP from the router's DHCP                      |
 
-If no Linux bridge owns the default route, `setup-ovn.sh` exits with guidance
-rather than silently breaking host connectivity.
+If no Linux bridge owns the default route, `setup-ovn.sh` exits with guidance rather than silently breaking host connectivity.
 
 ## Per-Node Configuration
 
@@ -460,10 +374,7 @@ external_interface = "br-public"
 external_interface = "br-wan"      # br-wan enslaving bond0
 ```
 
-`external_interface` is the **Linux bridge** that owns the WAN uplink on this
-node — not the physical NIC. The physical NIC lives underneath the bridge.
-Each node runs `setup-ovn.sh` with its own WAN bridge name (or relies on
-auto-detection). OVN only requires `ovn-bridge-mappings` to point at `br-ext`.
+`external_interface` is the **Linux bridge** that owns the WAN uplink on this node — not the physical NIC. The physical NIC lives underneath the bridge. Each node runs `setup-ovn.sh` with its own WAN bridge name (or relies on auto-detection). OVN only requires `ovn-bridge-mappings` to point at `br-ext`.
 
 ## Bridge Verification
 
@@ -500,8 +411,7 @@ ip -d link show wan
 
 ## Configuration Reference
 
-All network configuration lives in `spinifex.toml`. Settings are split into three
-levels: cluster-wide mode, IP pool definitions, and per-node NIC settings.
+All network configuration lives in `spinifex.toml`. Settings are split into three levels: cluster-wide mode, IP pool definitions, and per-node NIC settings.
 
 ## Configuration Levels
 
@@ -524,8 +434,7 @@ external_mode = "pool"    # "pool", "nat", or "" (disabled)
 
 ## IP Pools: network.external_pools
 
-Each pool defines where external IPs come from. You can have one pool (homelab)
-or many (multi-region datacenter).
+Each pool defines where external IPs come from. You can have one pool (homelab) or many (multi-region datacenter).
 
 ```toml
 [[network.external_pools]]
@@ -559,19 +468,14 @@ dns_servers = ["8.8.8.8"]           # DNS for VMs (optional)
 
 ### Why range_start/range_end Instead of CIDR?
 
-Customer IP ranges rarely align to CIDR boundaries. A datacenter might have
-`203.0.113.10-203.0.113.200` from their ISP. Start/end avoids forcing admins to
-calculate CIDR blocks.
+Customer IP ranges rarely align to CIDR boundaries. A datacenter might have `203.0.113.10-203.0.113.200` from their ISP. Start/end avoids forcing admins to calculate CIDR blocks.
 
 ### Gateway vs Gateway_IP
 
 These are different things:
 
-- **`gateway`** = Your network's default gateway (e.g., 192.168.1.1). This is
-  where OVN sends packets destined for the internet. It's your router.
-- **`gateway_ip`** = The IP that OVN uses for outbound SNAT. In pool mode,
-  defaults to the first IP in the range. In NAT mode, set this explicitly.
-  Must be on the same subnet as the gateway.
+- **`gateway`** = Your network's default gateway (e.g., 192.168.1.1). This is where OVN sends packets destined for the internet. It's your router.
+- **`gateway_ip`** = The IP that OVN uses for outbound SNAT. In pool mode, defaults to the first IP in the range. In NAT mode, set this explicitly. Must be on the same subnet as the gateway.
 
 ## Per-Node: nodes.NAME.vpcd
 
@@ -598,15 +502,11 @@ When an instance needs a public IP:
 3. **Unscoped fallback**: Pool with no `region`/`az` (global, homelab configs)
 4. **Exhausted**: All pools full → `InsufficientAddressCapacity` error
 
-`AllocateAddress` accepts optional pool name to target a specific block
-(maps to AWS `PublicIpv4Pool`).
+`AllocateAddress` accepts optional pool name to target a specific block (maps to AWS `PublicIpv4Pool`).
 
 ## IPAM Storage
 
-Pool allocation state is stored durably in the cluster (NATS KV bucket
-`spinifex-external-ipam`, one entry per pool) and survives restarts. Each
-allocation records the ENI and instance holding the address. Pools are
-initialized from `spinifex.toml` on vpcd startup (idempotent).
+Pool allocation state is stored durably in the cluster (NATS KV bucket `spinifex-external-ipam`, one entry per pool) and survives restarts. Each allocation records the ENI and instance holding the address. Pools are initialized from `spinifex.toml` on vpcd startup (idempotent).
 
 ## Deployment Examples
 
@@ -633,10 +533,7 @@ prefix_len  = 24
 external_interface = "br-wan"
 ```
 
-**Setup:** Configure `br-wan` to enslave your physical WAN NIC (netplan,
-cloud-init, or systemd-networkd). Change your router's DHCP range to end
-at .149. Run `sudo setup-ovn.sh` — it auto-detects the WAN bridge from the
-default route, or specify it with `--wan-bridge=br-wan`.
+**Setup:** Configure `br-wan` to enslave your physical WAN NIC (netplan, cloud-init, or systemd-networkd). Change your router's DHCP range to end at .149. Run `sudo setup-ovn.sh` — it auto-detects the WAN bridge from the default route, or specify it with `--wan-bridge=br-wan`.
 
 ## Homelab / Dev (DHCP Pool — No Range Reservation)
 
@@ -661,9 +558,7 @@ dns_servers = ["192.168.1.1", "8.8.8.8"]
 external_interface = "br-wan"
 ```
 
-**Setup:** No router changes needed. Spinifex requests IPs from the router's
-DHCP server when VMs launch and releases them on terminate. Requires `dhclient`
-on the host (`apt install isc-dhcp-client`).
+**Setup:** No router changes needed. Spinifex requests IPs from the router's DHCP server when VMs launch and releases them on terminate. Requires `dhclient` on the host (`apt install isc-dhcp-client`).
 
 ## Host-Local Subnet (No Upstream Router)
 
@@ -673,9 +568,7 @@ Host WAN: 198.51.100.10/24 on br-wan (existing address — unchanged)
 Gateway: 192.168.10.1 — second address added to br-wan
 ```
 
-Add the VM pool gateway as a second address on `br-wan` alongside the existing WAN
-IP. The host acts as the gateway for the pool — no upstream router or DHCP server
-needed for this range.
+Add the VM pool gateway as a second address on `br-wan` alongside the existing WAN IP. The host acts as the gateway for the pool — no upstream router or DHCP server needed for this range.
 
 ```yaml
 # /etc/netplan/…
@@ -703,8 +596,7 @@ prefix_len  = 24
 dns_servers = ["8.8.8.8"]
 ```
 
-**Setup:** Apply with `sudo netplan apply`. VMs are reachable from the host at
-`192.168.10.x`. For internet access through the host's WAN interface:
+**Setup:** Apply with `sudo netplan apply`. VMs are reachable from the host at `192.168.10.x`. For internet access through the host's WAN interface:
 
 ```bash
 sysctl -w net.ipv4.ip_forward=1
@@ -744,9 +636,7 @@ external_interface = "br-public"   # br-public enslaves eno1
 
 ## Enterprise On-Prem (VLAN)
 
-The Linux WAN bridge enslaves a VLAN sub-interface (e.g. `eth1.200`) instead
-of a raw NIC. From OVN's perspective nothing changes — `external_interface`
-still points at the bridge.
+The Linux WAN bridge enslaves a VLAN sub-interface (e.g. `eth1.200`) instead of a raw NIC. From OVN's perspective nothing changes — `external_interface` still points at the bridge.
 
 ```toml
 [network]
@@ -817,21 +707,15 @@ prefix_len  = 23
 region      = "eu-west-1"
 ```
 
-Spinifex allocates from the correct pool based on where the instance launches.
-An instance in `us-east-1a` gets an IP from `us-east-1a` first; if exhausted,
-falls back to `us-east-overflow`.
+Spinifex allocates from the correct pool based on where the instance launches. An instance in `us-east-1a` gets an IP from `us-east-1a` first; if exhausted, falls back to `us-east-overflow`.
 
 ## Security Groups
 
-Security groups are stateful firewalls enforced at the OVS datapath level on
-each hypervisor. Traffic is filtered before it reaches the wire — equivalent to
-AWS Nitro card enforcement. The VM never sees dropped packets.
+Security groups are stateful firewalls enforced at the OVS datapath level on each hypervisor. Traffic is filtered before it reaches the wire — equivalent to AWS Nitro card enforcement. The VM never sees dropped packets.
 
 ## How Security Groups Work
 
-Each security group maps to an OVN **Port Group**. When an instance launches,
-its ENI port is added to the port group(s) for its security groups. ACL rules
-on the port group control traffic:
+Each security group maps to an OVN **Port Group**. When an instance launches, its ENI port is added to the port group(s) for its security groups. ACL rules on the port group control traffic:
 
 - **Default deny**: All inbound traffic dropped at priority 900
 - **Allow rules**: Specific ports/protocols allowed at priority 1000 (overrides deny)
@@ -881,9 +765,7 @@ Rule changes take effect immediately — no instance restart needed.
 
 ## Elastic IPs
 
-Elastic IPs are static public IPs that persist across instance stop/start cycles.
-Unlike auto-assigned public IPs (which change on stop/start), an Elastic IP stays
-with your instance.
+Elastic IPs are static public IPs that persist across instance stop/start cycles. Unlike auto-assigned public IPs (which change on stop/start), an Elastic IP stays with your instance.
 
 ```bash
 # Allocate
@@ -901,8 +783,7 @@ aws ec2 disassociate-address --association-id $ASSOC_ID
 aws ec2 release-address --allocation-id $EIP
 ```
 
-When you associate an Elastic IP with an instance that already has an
-auto-assigned public IP, the auto-assigned IP is released and replaced.
+When you associate an Elastic IP with an instance that already has an auto-assigned public IP, the auto-assigned IP is released and replaced.
 
 ## OVN Reference
 
@@ -945,10 +826,7 @@ ovn-nbctl lr-nat-add vpc-{vpcId} dnat_and_snat {public_ip} {private_ip} \
   port-{eniId} {vm_mac}
 ```
 
-With the WAN NIC on a Linux bridge wired to OVS via veth, OVS sees every frame
-on the wire regardless of MAC, so OVN can use distributed NAT. The DNAT is
-processed on the chassis hosting the VM rather than hairpinning through a
-single gateway chassis.
+With the WAN NIC on a Linux bridge wired to OVS via veth, OVS sees every frame on the wire regardless of MAC, so OVN can use distributed NAT. The DNAT is processed on the chassis hosting the VM rather than hairpinning through a single gateway chassis.
 
 ## Security Group
 
@@ -1004,8 +882,7 @@ sudo ovn-nbctl pg-get-ports sg-{groupId}
 
 ## 1. Set Up OVN Bridges
 
-Make sure the WAN NIC is enslaved to a Linux bridge (e.g. `br-wan`) and that
-bridge owns the default route. Then run:
+Make sure the WAN NIC is enslaved to a Linux bridge (e.g. `br-wan`) and that bridge owns the default route. Then run:
 
 ```bash
 sudo setup-ovn.sh                       # auto-detect WAN bridge
@@ -1078,8 +955,7 @@ aws ec2 describe-instances --instance-ids $INSTANCE \
 
 ### Debugging Toolkit
 
-These commands are used throughout the troubleshooting sections below. Learn
-them — they cover 90% of VPC networking issues.
+These commands are used throughout the troubleshooting sections below. Learn them — they cover 90% of VPC networking issues.
 
 ### OVN Northbound (Logical Topology)
 
@@ -1129,9 +1005,7 @@ sudo ovn-trace ext-vpc-{vpcId} \
 
 ### OVN DB RAFT cluster (NB/SB replication)
 
-The NB and SB databases run clustered across the first three nodes via native
-OVSDB RAFT. A 3-node quorum tolerates the loss of one DB node; check status when
-the control plane stalls.
+The NB and SB databases run clustered across the first three nodes via native OVSDB RAFT. A 3-node quorum tolerates the loss of one DB node; check status when the control plane stalls.
 
 ```bash
 # NB cluster status: expect 3 servers and exactly one "Role: leader"
@@ -1144,9 +1018,7 @@ sudo ovn-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound
 sudo ovn-nbctl --db=tcp:10.1.3.181:6641,tcp:10.1.3.182:6641,tcp:10.1.3.183:6641 show
 ```
 
-A node showing only itself under `Servers:` never joined the cluster — confirm
-it was bootstrapped with `--db-cluster-remote-addr` pointing at the creator and
-that RAFT ports 6643/6644 are reachable between DB nodes.
+A node showing only itself under `Servers:` never joined the cluster — confirm it was bootstrapped with `--db-cluster-remote-addr` pointing at the creator and that RAFT ports 6643/6644 are reachable between DB nodes.
 
 ### OVS (datapath / physical wiring)
 
@@ -1272,10 +1144,7 @@ ip route show
 
 ### Instance Has Public IP But No Internet
 
-The instance shows a public IP in `describe-instances` but cannot reach the
-internet, and inbound connections hang. The usual cause is a missing default
-route: the subnet's effective route table has no `0.0.0.0/0` route to the IGW,
-so Spinifex gates the subnet with a drop policy.
+The instance shows a public IP in `describe-instances` but cannot reach the internet, and inbound connections hang. The usual cause is a missing default route: the subnet's effective route table has no `0.0.0.0/0` route to the IGW, so Spinifex gates the subnet with a drop policy.
 
 ```bash
 # Find the route table that applies to the subnet and check its routes
@@ -1285,16 +1154,14 @@ aws ec2 describe-route-tables \
 # Expect a route with DestinationCidrBlock 0.0.0.0/0 and a GatewayId of igw-...
 ```
 
-If the subnet has no explicit association it falls back to the VPC's main route
-table — check that one too, then add the route to whichever table applies:
+If the subnet has no explicit association it falls back to the VPC's main route table — check that one too, then add the route to whichever table applies:
 
 ```bash
 aws ec2 create-route --route-table-id $RT \
   --destination-cidr-block 0.0.0.0/0 --gateway-id $IGW
 ```
 
-On the host, the drop policy installed when a subnet lacks an IGW route appears
-as a `Logical_Router_Policy` on the VPC router:
+On the host, the drop policy installed when a subnet lacks an IGW route appears as a `Logical_Router_Policy` on the VPC router:
 
 ```bash
 sudo ovn-nbctl lr-policy-list vpc-$VPC
@@ -1387,9 +1254,7 @@ sudo tcpdump -i br-wan -n "host {public_ip}"
 sudo tcpdump -i br-ext -n -e "host {public_ip}"
 ```
 
-If traffic arrives on the physical NIC but not br-wan, the NIC is not
-enslaved to the bridge. If it reaches br-wan but not br-ext, the veth
-pair between them is missing or down — re-run `setup-ovn.sh`.
+If traffic arrives on the physical NIC but not br-wan, the NIC is not enslaved to the bridge. If it reaches br-wan but not br-ext, the veth pair between them is missing or down — re-run `setup-ovn.sh`.
 
 ### 6. Use ovn-trace for pipeline debugging
 
@@ -1402,8 +1267,7 @@ sudo ovn-trace --ct=new ext-vpc-{vpcId} \
    arp.tpa=={public_ip}'
 ```
 
-The output shows every table the packet passes through and what action is
-taken. Look for `drop` actions or unexpected paths.
+The output shows every table the packet passes through and what action is taken. Look for `drop` actions or unexpected paths.
 
 ### OVN SB Commit Failure Loop
 
@@ -1415,9 +1279,7 @@ OVNSB commit failed, force recompute next time.
 
 Repeated millions of times. Port binding never happens (`up: false`).
 
-**Cause:** Stale entries in the OVN Southbound DB (old chassis records, port
-bindings, datapath bindings) conflict with ovn-controller's expected state,
-typically after an ungraceful shutdown.
+**Cause:** Stale entries in the OVN Southbound DB (old chassis records, port bindings, datapath bindings) conflict with ovn-controller's expected state, typically after an ungraceful shutdown.
 
 **Fix:** Delete both OVN DB files and restart:
 
@@ -1431,15 +1293,11 @@ sudo systemctl start ovn-central ovn-controller
 
 ### WAN NIC Not Enslaved to a Bridge
 
-**Symptom:** `setup-ovn.sh` exits with an error like "default route is on a
-physical NIC, not a bridge" and refuses to continue.
+**Symptom:** `setup-ovn.sh` exits with an error like "default route is on a physical NIC, not a bridge" and refuses to continue.
 
-**Cause:** Spinifex requires the WAN NIC to be enslaved to a Linux bridge
-(typically `br-wan`). Macvlan is no longer supported, and attaching the NIC
-directly to OVS would break SSH and any other host services using the NIC.
+**Cause:** Spinifex requires the WAN NIC to be enslaved to a Linux bridge (typically `br-wan`). Macvlan is no longer supported, and attaching the NIC directly to OVS would break SSH and any other host services using the NIC.
 
-**Fix:** Move the host IP and default route onto a Linux bridge. Example
-netplan:
+**Fix:** Move the host IP and default route onto a Linux bridge. Example netplan:
 
 ```yaml
 network:
@@ -1453,13 +1311,11 @@ network:
       dhcp4: true
 ```
 
-Apply (`sudo netplan apply`), confirm the host IP is now on `br-wan`
-(`ip -br addr show br-wan`), and re-run `setup-ovn.sh`.
+Apply (`sudo netplan apply`), confirm the host IP is now on `br-wan` (`ip -br addr show br-wan`), and re-run `setup-ovn.sh`.
 
 ### Stale ARP on Remote Hosts
 
-**Symptom:** Ping from a LAN host to a VM public IP fails after a reset, but
-worked before. The remote host has a stale ARP entry with the old MAC.
+**Symptom:** Ping from a LAN host to a VM public IP fails after a reset, but worked before. The remote host has a stale ARP entry with the old MAC.
 
 **Fix:** Flush the ARP entry on the remote host:
 
@@ -1469,8 +1325,7 @@ sudo ip neigh flush dev {nic} {public_ip}
 ping {public_ip}   # should work now
 ```
 
-OVN sends periodic gratuitous ARPs from the chassis hosting the VM that will
-eventually update remote ARP caches, but flushing is faster for testing.
+OVN sends periodic gratuitous ARPs from the chassis hosting the VM that will eventually update remote ARP caches, but flushing is faster for testing.
 
 ### Security Group Rules Not Taking Effect
 

docs/containers/ecr/README.md

@@ -85,9 +85,7 @@ aws ecr describe-repositories --repository-names my-app \
   --query 'repositories[0].repositoryUri'
 ```
 
-The `repositoryUri` is the value you tag and push to — the gateway `host:9999`
-with the repository path, port included (for example `192.0.2.10:9999/my-app`).
-Use it verbatim; don't hand-build a hostname.
+The `repositoryUri` is the value you tag and push to — the gateway `host:9999` with the repository path, port included (for example `192.0.2.10:9999/my-app`). Use it verbatim; don't hand-build a hostname.
 
 ### 2. Authenticate Docker
 

docs/containers/ecs/README.md

@@ -156,11 +156,7 @@ aws ecs register-task-definition \
 
 ### 3. Add capacity
 
-Launch one or more EC2 instances from the ECS node image with the `ecsInstanceRole`
-instance profile and a cloud-init that points the agent at the cluster. The
-[ECS Quickstart workbook](https://github.com/mulgadc/spinifex/tree/main/docs/terraform-workbooks/ecs-quickstart)
-produces this user-data for you; the console **Provision capacity** action is the
-one-click equivalent. Once an instance boots and its agent registers, it appears here:
+Launch one or more EC2 instances from the ECS node image with the `ecsInstanceRole` instance profile and a cloud-init that points the agent at the cluster. The [ECS Quickstart workbook](https://github.com/mulgadc/spinifex/tree/main/docs/terraform-workbooks/ecs-quickstart) produces this user-data for you; the console **Provision capacity** action is the one-click equivalent. Once an instance boots and its agent registers, it appears here:
 
 ```bash
 aws ecs list-container-instances --cluster demo

docs/install-vm-eval/README.md

@@ -228,10 +228,7 @@ Auto-detected encap IP: 192.168.0.249
 ...
 ```
 
-The `system-id` is the OVS-managed chassis UUID (read back from
-`/etc/openvswitch/system-id.conf`); this is the same value that appears as
-`Chassis.name` in `ovn-sbctl show`. Cross-reference with `Chassis.hostname`
-when you need the human-readable host:
+The `system-id` is the OVS-managed chassis UUID (read back from `/etc/openvswitch/system-id.conf`); this is the same value that appears as `Chassis.name` in `ovn-sbctl show`. Cross-reference with `Chassis.hostname` when you need the human-readable host:
 
 ```bash
 sudo ovn-sbctl --columns=name,hostname list Chassis

docs/reference-architectures/SMCI-MIG-orchestration/README.md

@@ -97,8 +97,7 @@ bridges:
         via: 198.51.100.1
 ```
 
-Then apply with `sudo netplan apply`. VMs, once provisioned, are reachable from the host at
-`192.168.10.x`. For internet access through the host's WAN interface:
+Then apply with `sudo netplan apply`. VMs, once provisioned, are reachable from the host at `192.168.10.x`. For internet access through the host's WAN interface:
 
 ```bash
 sysctl -w net.ipv4.ip_forward=1

docs/security/network-connections/README.md

@@ -163,7 +163,7 @@ Every listener and outbound destination is controlled by one of these files. Cha
 |------|------|----------|
 | `/etc/spinifex/spinifex.toml` | `nodes.<node>.{awsgw,nats,predastore,daemon}.host`, `nodes.<node>.vpcd.ovn_{nb,sb}_addr`, `nodes.<node>.daemon.dev_networking` | Per-service bind addresses/ports; dev-mode QEMU port forwarding. |
 | `/etc/spinifex/nats.conf` | `listen`, `cluster.listen`, `cluster.routes`, `http`, `tls`, `cluster.authorization` | NATS client/cluster/monitoring listeners, peer routes, TLS, cluster token. |
-| `/etc/spinifex/predastore.toml` | `host`, `port`, `[[db]].port`, `[[nodes]].port`, `tls.*` | Predastore S3 listener, Raft ports, shard ports, TLS certs. |
+| `/etc/spinifex/predastore.toml` | `[[host]].bind_addr`, `[[host]].public_addr`, `[[node]].role`, `tls.*` | Predastore intra-cluster listener (one socket per host, carrying every node pinned to it), node placement, TLS certs. The S3 listener itself is set by the service's `--host`/`--port`. |
 | OVN packages (`ovn-central`, `ovn-host`) | `ovn-nb-db`, `ovn-sb-db` (via `ovs-vsctl set open_vswitch …`) | OVN DB bind addresses. |
 | Spinifex UI service | Built-in defaults: `host = "0.0.0.0"`, `port = 3000`. No `spinifex.toml` block today. | UI listener. |
 | `spx admin init` / `spx admin join` | `--port`, `--token-ttl`, `--no-telemetry` (or `SPX_NO_TELEMETRY=1`) | Formation port, token TTL, telemetry opt-out. |

docs/terraform-workbooks/demo-app/README.md

@@ -1,19 +1,14 @@
 # Spinifex EKS demo app
 
-A small, Spinifex-themed web app shared by the EKS Terraform workbooks. It
-replaces `nginxdemos/hello` with something production-shaped that you build
-yourself and push to **ECR**, then pull onto your EKS workers.
+A small, Spinifex-themed web app shared by the EKS Terraform workbooks. It replaces `nginxdemos/hello` with something production-shaped that you build yourself and push to **ECR**, then pull onto your EKS workers.
 
-The page reports the pod, node, cluster, and region that answered the request,
-plus a hit counter — refresh and watch requests land on different pods across a
-multi-node cluster.
+The page reports the pod, node, cluster, and region that answered the request, plus a hit counter — refresh and watch requests land on different pods across a multi-node cluster.
 
 ## What it does
 
 - `GET /` — themed HTML page.
 - `GET /api/info` — JSON of the same facts.
-- `GET /healthz` — liveness/readiness probe (`200 ok`), used by the ALB target
-  group health check.
+- `GET /healthz` — liveness/readiness probe (`200 ok`), used by the ALB target group health check.
 
 All configuration is read from the environment, so one image serves every tier:
 
@@ -25,14 +20,11 @@ All configuration is read from the environment, so one image serves every tier:
 | `PORT` | listen port | default `8080` |
 | `DATA_DIR` | when set, the hit counter is persisted here and survives pod restarts | mounted **EBS-CSI PVC** in `eks-gitops-argocd` |
 
-When `DATA_DIR` points at a writable volume the page shows a **"persisted to EBS
-volume"** badge — the gitops workbook mounts a Viperblock-backed PersistentVolume
-there to demonstrate durable storage.
+When `DATA_DIR` points at a writable volume the page shows a **"persisted to EBS volume"** badge — the gitops workbook mounts a Viperblock-backed PersistentVolume there to demonstrate durable storage.
 
 ## Build and push to ECR
 
-The image is named `spinifex-demo`. Build it once and push it to a repository in
-your Spinifex ECR; every workbook references it by its ECR URI.
+The image is named `spinifex-demo`. Build it once and push it to a repository in your Spinifex ECR; every workbook references it by its ECR URI.
 
 ```bash
 export AWS_PROFILE=spinifex
@@ -57,9 +49,7 @@ docker push "${REGISTRY}:latest"
 echo "Image: ${REGISTRY}:latest"
 ```
 
-Pass the resulting `${REGISTRY}:latest` value to a workbook as the `demo_image`
-variable (or accept its default, which resolves the URI from the ECR repository
-the workbook creates).
+Pass the resulting `${REGISTRY}:latest` value to a workbook as the `demo_image` variable (or accept its default, which resolves the URI from the ECR repository the workbook creates).
 
 ## Run locally
 

docs/terraform-workbooks/eks-gitops-argocd/README.md

@@ -146,10 +146,7 @@ This registers the repo credential, creates the Argo CD `Application`, the demo
 
 ### Step 5. Watch the Sync and Open the App
 
-The demo app and the Argo CD UI share **one ALB** (an LBC IngressGroup). On
-`:443` the app is the catch-all and `argocd.eks-gitops.spinifex.local`
-host-routes to the UI; Argo CD also gets a hostless `:8443` listener so it is
-reachable on the raw ALB IP before DNS exists. Grab the shared ALB address once:
+The demo app and the Argo CD UI share **one ALB** (an LBC IngressGroup). On `:443` the app is the catch-all and `argocd.eks-gitops.spinifex.local` host-routes to the UI; Argo CD also gets a hostless `:8443` listener so it is reachable on the raw ALB IP before DNS exists. Grab the shared ALB address once:
 
 ```bash
 # Point kubectl at the cluster first (writes ~/.kube/config) — without this every
@@ -179,25 +176,15 @@ curl -k https://"$ALB_IP"/            # app — returns the Spinifex page
 curl -k https://"$ALB_IP":8443/       # Argo CD UI
 ```
 
-The bare ALB IP serves the **app** on `:443`. Hitting `https://<ALB_IP>/` and
-expecting Argo CD gives the app instead — use `:8443` for the UI. Once northstar
-(or Route 53) resolves the hostnames to the ALB, `app.eks-gitops.spinifex.local`
-and `argocd.eks-gitops.spinifex.local` both work on `:443`. Self-signed cert —
-accept the browser warning.
+The bare ALB IP serves the **app** on `:443`. Hitting `https://<ALB_IP>/` and expecting Argo CD gives the app instead — use `:8443` for the UI. Once northstar (or Route 53) resolves the hostnames to the ALB, `app.eks-gitops.spinifex.local` and `argocd.eks-gitops.spinifex.local` both work on `:443`. Self-signed cert — accept the browser warning.
 
 The page shows the **"persisted to EBS volume"** badge and a hit counter that keeps climbing — delete the pod (`kubectl delete pod -l app=spinifex-demo`) and the count survives, proving the volume is durable.
 
 ### Step 6. Open the Argo CD UI
 
-Managing deployments through the Argo CD console is the point of this workbook, so
-the UI is exposed on the **same ALB** as the app, not behind a port-forward. The
-`workloads` module adds a `NodePort` Service in front of `argocd-server` (the addon
-ships it `ClusterIP` only) and two Ingresses in the shared group — a host-routed
-one on `:443` and a hostless one on `:8443` for raw-IP access. `argocd-server`
-serves TLS on its own port, so both use `backend-protocol: HTTPS`.
+Managing deployments through the Argo CD console is the point of this workbook, so the UI is exposed on the **same ALB** as the app, not behind a port-forward. The `workloads` module adds a `NodePort` Service in front of `argocd-server` (the addon ships it `ClusterIP` only) and two Ingresses in the shared group — a host-routed one on `:443` and a hostless one on `:8443` for raw-IP access. `argocd-server` serves TLS on its own port, so both use `backend-protocol: HTTPS`.
 
-**Get the admin credentials.** Argo CD generates a one-time `admin` password into
-a Secret on install; the username is always `admin`:
+**Get the admin credentials.** Argo CD generates a one-time `admin` password into a Secret on install; the username is always `admin`:
 
 ```bash
 kubectl -n argocd get secret argocd-initial-admin-secret \
@@ -211,8 +198,7 @@ Open the UI and log in as `admin`:
 - By raw IP: `https://<ALB_IP>:8443/`
 - By DNS once wired: `https://argocd.eks-gitops.spinifex.local`
 
-The `spinifex-demo` Application shows the sync status, the resource tree, and live
-diffs against git — change a manifest in the repo and watch Argo CD reconcile it.
+The `spinifex-demo` Application shows the sync status, the resource tree, and live diffs against git — change a manifest in the repo and watch Argo CD reconcile it.
 
 If you'd rather not expose the UI at all, port-forward instead:
 
@@ -269,12 +255,7 @@ kubectl -n kube-system logs deploy/aws-load-balancer-controller --tail=50
 
 Confirm the cluster carries `spinifex.io/managed-ingress = "false"` so the LBC owns ingress.
 
-If the controller logs show `FailedBuildModel ... DescribeAvailabilityZones ...
-403 ... AccessDenied`, the node role is missing the LBC permissions. The
-controller runs with the node instance-profile credentials, so the
-`${var.cluster_name}-node-lbc` policy (`aws_iam_policy.node_lbc`) must be
-attached to the node role — re-run `tofu apply` on the parent module if it was
-provisioned before that policy existed.
+If the controller logs show `FailedBuildModel ... DescribeAvailabilityZones ... 403 ... AccessDenied`, the node role is missing the LBC permissions. The controller runs with the node instance-profile credentials, so the `${var.cluster_name}-node-lbc` policy (`aws_iam_policy.node_lbc`) must be attached to the node role — re-run `tofu apply` on the parent module if it was provisioned before that policy existed.
 
 ### Provider Connection Refused
 

docs/terraform-workbooks/rds-quickstart/README.md

@@ -1,5 +1,5 @@
 ---
-title: "RDS Quickstart"
+title: "RDS Quickstart (PostgreSQL)"
 description: "Stand up a managed PostgreSQL database on Spinifex with Terraform — a VPC, a DB subnet group, a parameter group, an aws_db_instance, and a client VM inside the VPC that connects to it with psql."
 category: "Terraform Workbooks"
 tags:
@@ -16,7 +16,7 @@ sections:
   - troubleshooting
 resources:
   - title: "RDS CLI Reference"
-    url: "https://github.com/mulgadc/spinifex/blob/main/docs/COMMANDS.md#rds-postgresql"
+    url: "https://github.com/mulgadc/spinifex/blob/main/docs/COMMANDS.md#rds-postgresql-and-mariadb"
   - title: "Terraform AWS Provider — aws_db_instance"
     url: "https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/db_instance"
   - title: "Spinifex Repository"
@@ -25,38 +25,47 @@ resources:
     url: "https://opentofu.org/"
 ---
 
-# Terraform: RDS Quickstart
+# Terraform: RDS Quickstart (PostgreSQL)
 
-> The smallest end-to-end RDS example on Spinifex: a VPC, a DB subnet group, a parameter group, a PostgreSQL DB instance, and a client VM in the same subnet with `psql` already pointed at the endpoint.
+> The smallest end-to-end RDS example on Spinifex: a VPC with a public client subnet and a private DB subnet, a DB subnet group, a parameter group, a PostgreSQL DB instance, and a client VM with `psql` already pointed at the endpoint.
+
+This workbook is **PostgreSQL throughout** — the engine, the `postgres18` parameter group, port `5432`, the client package and the `~/.pgpass` credential are all PostgreSQL's. Spinifex also serves MariaDB 11.8 under `engine = "mariadb"`; running this workbook against it means changing `engine`, `engine_version`, the parameter-group `family` to `mariadb11.8`, the security-group port to `3306` and the client to `mariadb-client`. See the [RDS command reference](https://github.com/mulgadc/spinifex/blob/main/docs/COMMANDS.md#rds-postgresql-and-mariadb) for both engines' pins, families and limits. Note that `mysql` is not an accepted engine and is not an alias for `mariadb`.
 
 ## Overview
 
 A Spinifex DB instance runs in a VM you never see. What lands in your VPC is a single **endpoint ENI** in one of the DB subnet group's subnets, so the endpoint is **always private** — there is no `publicly_accessible` mode, and a request for one is rejected. That is why this workbook builds a client VM: it is the only place from which the database can be reached.
 
-The layout is the one that shape implies:
+The layout is the one that shape implies — the tier that talks to the database in its own subnet, and the database in a subnet with no route off the VPC:
 
 ```
      IGW
       │
-   10.60.1.0/24 ──┬── client VM ──psql:5432──┐
-                  │                            ▼
-                  └─────────────────── endpoint ENI ──▶ DB VM (platform-owned)
+   client subnet 10.60.1.0/24 ──── client VM
+                                       │
+                                    psql:5432
+                                       │
+   db subnet     10.60.2.0/24 ──── endpoint ENI ──▶ DB VM (platform-owned)
 ```
 
-The client and the DB subnet group **share one subnet**, and in v1 they have to. The endpoint is reachable from clients on the endpoint ENI's own subnet only: the DB VM carries its default route on its platform-side interface, so a reply to a client in another subnet is never routed back. A subnet group spanning several subnets places the endpoint in one of them and leaves clients in the others timing out on connect. On AWS you would use two private subnets here instead.
+The endpoint is reachable from **any subnet of the VPC**, so the client does not have to share the subnet the endpoint ENI landed in — which also means a subnet group spanning several subnets works wherever the endpoint is placed within it.
 
-The shared subnet is "public" only in that it has an internet-gateway route, which the client needs to `apt-get` a psql. The endpoint ENI gets no public address and `publicly_accessible = true` is rejected, so sharing the subnet does not expose the database.
+The client subnet is public in that it has an internet-gateway route, which the client needs to `apt-get` a psql. The DB subnet has no route table association, so it stays on the main route table `create-vpc` writes: intra-VPC routing and nothing else. The endpoint ENI gets no public address and `publicly_accessible = true` is rejected either way.
 
 The database security group admits `5432` from the client security group and nothing else, which compiles to an ACL on the endpoint ENI itself — the port is deny-by-default for everything else in the VPC.
 
 ## Prerequisites
 
 - **Spinifex running**, with the AWS CLI configured for the `spinifex` profile (see [Installing Spinifex](/docs/install)) and OpenTofu (or Terraform) installed.
-- **The `spinifex-rds-postgres` image registered.** DB instances boot from it; it is built at `spx admin init` time.
+- **The `spinifex-rds-postgres` image registered.** DB instances boot from this system image. Import it once per cluster, then verify that all tags used by the RDS engine resolver are present:
 
   ```bash
-  aws ec2 describe-images \
-    --filters 'Name=tag:spinifex:managed-by,Values=rds' \
+  spx admin images import --name spinifex-rds-postgres
+
+  AWS_PROFILE=spinifex aws ec2 describe-images \
+    --filters \
+      'Name=tag:spinifex:managed-by,Values=rds' \
+      'Name=tag:engine,Values=postgres' \
+      'Name=tag:engine-version,Values=18' \
     --query 'Images[].[ImageId,Name]' --output text
   ```
 
@@ -120,7 +129,7 @@ psql "sslmode=verify-full sslrootcert=/home/ubuntu/ca.pem" \
 | `spinifex_endpoint` | `https://127.0.0.1:9999` | Gateway as seen from the host running Terraform. |
 | `instance_type` | `t3.small` | EC2 type for the **client** VM. |
 | `db_instance_class` | `db.t3.micro` | DB class. One of the curated `db.*` subset. |
-| `engine_version` | `18` | PostgreSQL major. `18` is the only version served. |
+| `engine_version` | `18` | PostgreSQL major. `18` is the only version served, and a minor such as `18.4` is rejected. |
 | `db_name` | `appdb` | Database created at bootstrap. |
 | `db_username` | `appuser` | Master user. `postgres`, `rdsadmin`, `rds_superuser` and `pg_*` are reserved. |
 | `db_password` | `QuickstartS3cret1` | Master password — override it. No `/`, `"`, `@` or spaces. |
@@ -143,12 +152,12 @@ aws rds delete-db-instance --db-instance-identifier rds-quickstart \
 
 - **`storage_encrypted = true` is set explicitly.** Storage is always encrypted, so the instance reports `true` whether or not you asked. The provider's attribute is optional but *not* computed, so leaving it out means the read-back carries a value your configuration does not — and every subsequent `plan` shows a change on an instance nothing has touched.
 - **`instance_class` and `engine_version` are literal.** `describe-db-engine-versions` and `describe-orderable-db-instance-options` are not implemented, so the `aws_rds_engine_version` and `aws_rds_orderable_db_instance` data sources are unavailable. `aws_db_instance` needs neither.
-- **One subnet, shared.** AWS requires a DB subnet group to span two AZs. Spinifex is single-AZ — every subnet reports the same zone — so the group accepts any count, and one is what the v1 reachability rule above allows.
+- **A DB subnet group over one subnet.** AWS requires a DB subnet group to span two AZs. Spinifex is single-AZ — every subnet reports the same zone — so the group accepts any count, and a second private subnet here would buy nothing but a second CIDR.
 - **The password is written to `~/.pgpass`, not to the environment.** A `PGPASSWORD` in `/etc/profile.d` is readable by every user on the client and leaks into `ps` output.
 
 ## Troubleshooting
 
-**`apply` fails resolving the DB AMI.** The `spinifex-rds-postgres` image is not registered. Re-run the `describe-images` check in Prerequisites.
+**`apply` fails resolving the DB AMI.** The `spinifex-rds-postgres` image is not registered with the required engine tags. Run `spx admin images import --name spinifex-rds-postgres`, then repeat the tagged `describe-images` check in Prerequisites before applying again.
 
 **The apply sits on `aws_db_instance.main: Still creating...`.** Several minutes is normal — a VM boot plus `initdb` plus the first healthy heartbeat. Much longer is a bootstrap that did not finish:
 
@@ -161,8 +170,8 @@ aws rds describe-events --source-type db-instance --source-identifier rds-quicks
 **`psql` hangs on the client.** Three causes, in the order worth checking:
 
 - cloud-init has not finished installing `postgresql-client` — check `cloud-init status --long`.
-- the client is not in the endpoint ENI's subnet. Off-subnet clients cannot reach the endpoint in v1; put them in the DB subnet group's subnet, as this workbook does.
 - the security group is not letting you through. Nothing outside the client security group can open `5432` — that is the point of the rule — so a psql from your workstation will always time out.
+- the client is outside the VPC. The endpoint exists only inside it: any subnet will do, but there is no path to it from anywhere else.
 
 **`InsufficientInstanceCapacity` on the DB instance.** The node admits a launch against live free memory. Free some, or drop to a smaller `db_instance_class`.
 

docs/terraform-workbooks/rds-quickstart/main.tf

@@ -10,13 +10,11 @@
 # only way to reach the database is from inside the VPC, which is what the
 # client instance here is for.
 #
-# The client and the DB subnet group deliberately share ONE subnet. In v1 the
-# endpoint is reachable from clients on the endpoint ENI's own subnet only: the
-# DB VM carries its default route on a platform-side interface and the customer
-# ENI is pure ingress, so a reply to a client in another subnet is never routed
-# back. A subnet group spanning several subnets places the endpoint in one of
-# them and leaves clients in the others unable to connect. On AWS you would use
-# two private subnets here instead.
+# Two subnets: the client sits in a public one, because it needs an
+# internet-gateway route to apt-get a psql, and the database in a private one
+# with no route off the VPC at all. The endpoint is reachable from any subnet of
+# the VPC, so the tier that talks to the database does not have to share the
+# subnet the endpoint ENI landed in.
 #
 # Usage:
 #   cd spinifex/docs/terraform-workbooks/rds-quickstart
@@ -169,7 +167,7 @@ data "aws_ami" "ubuntu" {
 }
 
 # ---------------------------------------------------------------------------
-# VPC — one subnet, shared by the client and the DB subnet group
+# VPC — a public client subnet and a private DB subnet
 # ---------------------------------------------------------------------------
 
 resource "aws_vpc" "main" {
@@ -190,18 +188,29 @@ resource "aws_internet_gateway" "igw" {
   }
 }
 
-# Public only in the sense that it has an internet-gateway route, which the
-# client needs to apt-get a psql. The endpoint ENI in it is given no public
-# address, and PubliclyAccessible=true is rejected, so the database is not
-# exposed by sharing the subnet.
-resource "aws_subnet" "main" {
+# Public: an internet-gateway route and a public address on launch, which is how
+# the client reaches apt and how you SSH to it.
+resource "aws_subnet" "client" {
   vpc_id                  = aws_vpc.main.id
   cidr_block              = "10.60.1.0/24"
   availability_zone       = data.aws_availability_zones.available.names[0]
   map_public_ip_on_launch = true
 
   tags = {
-    Name = "${var.name}-subnet"
+    Name = "${var.name}-client-subnet"
+  }
+}
+
+# Private: no association, so it stays on the main route table create-vpc writes,
+# which routes inside the VPC and nowhere else. The endpoint ENI lands here.
+resource "aws_subnet" "db" {
+  vpc_id                  = aws_vpc.main.id
+  cidr_block              = "10.60.2.0/24"
+  availability_zone       = data.aws_availability_zones.available.names[0]
+  map_public_ip_on_launch = false
+
+  tags = {
+    Name = "${var.name}-db-subnet"
   }
 }
 
@@ -218,8 +227,8 @@ resource "aws_route_table" "public" {
   }
 }
 
-resource "aws_route_table_association" "main" {
-  subnet_id      = aws_subnet.main.id
+resource "aws_route_table_association" "client" {
+  subnet_id      = aws_subnet.client.id
   route_table_id = aws_route_table.public.id
 }
 
@@ -288,7 +297,7 @@ resource "aws_security_group" "db" {
 resource "aws_db_subnet_group" "main" {
   name        = "${var.name}-subnets"
   description = "RDS quickstart DB subnets"
-  subnet_ids  = [aws_subnet.main.id]
+  subnet_ids  = [aws_subnet.db.id]
 
   tags = {
     Name = "${var.name}-subnets"
@@ -356,7 +365,8 @@ resource "aws_db_instance" "main" {
 }
 
 # ---------------------------------------------------------------------------
-# Client VM — psql inside the VPC, which is the only place the endpoint exists
+# Client VM — psql inside the VPC, which is the only place the endpoint exists,
+# from the client subnet rather than the endpoint's own
 # ---------------------------------------------------------------------------
 
 resource "tls_private_key" "client" {
@@ -377,7 +387,7 @@ resource "local_file" "client_pem" {
 resource "aws_instance" "client" {
   ami                    = data.aws_ami.ubuntu.id
   instance_type          = var.instance_type
-  subnet_id              = aws_subnet.main.id
+  subnet_id              = aws_subnet.client.id
   vpc_security_group_ids = [aws_security_group.client.id]
   key_name               = aws_key_pair.client.key_name
 

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/insomniacslk/dhcp v0.0.0-20260407060928-11b94ed970f2
 	github.com/klauspost/cpuid/v2 v2.4.0
 	github.com/miekg/dns v1.1.72
-	github.com/mulgadc/northstar v1.15.0
+	github.com/mulgadc/northstar v1.15.1-0.20260810233215-04fade67fb2c
 	github.com/mulgadc/predastore v1.15.0
 	github.com/mulgadc/viperblock v1.15.0
 	github.com/nats-io/nats-server/v2 v2.14.4
@@ -33,19 +33,19 @@ require (
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.11.1
-	go.opentelemetry.io/contrib/bridges/otelslog v0.19.0
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0
-	go.opentelemetry.io/contrib/instrumentation/runtime v0.69.0
-	go.opentelemetry.io/otel v1.44.0
-	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.20.0
-	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0
-	go.opentelemetry.io/otel/log v0.20.0
-	go.opentelemetry.io/otel/metric v1.44.0
-	go.opentelemetry.io/otel/sdk v1.44.0
-	go.opentelemetry.io/otel/sdk/log v0.20.0
-	go.opentelemetry.io/otel/sdk/metric v1.44.0
-	go.opentelemetry.io/otel/trace v1.44.0
+	go.opentelemetry.io/contrib/bridges/otelslog v0.20.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0
+	go.opentelemetry.io/contrib/instrumentation/runtime v0.70.0
+	go.opentelemetry.io/otel v1.45.0
+	go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.45.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0
+	go.opentelemetry.io/otel/log v0.21.0
+	go.opentelemetry.io/otel/metric v1.45.0
+	go.opentelemetry.io/otel/sdk v1.45.0
+	go.opentelemetry.io/otel/sdk/log v0.21.0
+	go.opentelemetry.io/otel/sdk/metric v1.45.0
+	go.opentelemetry.io/otel/trace v1.45.0
 	go.uber.org/goleak v1.3.1-0.20260602155401-5f791e3ef83b
 	go.yaml.in/yaml/v3 v3.0.5
 	golang.org/x/crypto v0.54.0
@@ -109,10 +109,10 @@ require (
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
 	github.com/fatih/color v1.19.0 // indirect
-	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/felixge/httpsnoop v1.1.0 // indirect
 	github.com/fsnotify/fsnotify v1.10.1 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.13 // indirect
-	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/logr v1.4.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
@@ -186,22 +186,22 @@ require (
 	go.etcd.io/bbolt v1.4.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.10.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.11.0 // indirect
 	go.uber.org/nilaway v0.0.0-20260721205819-4a0f7653488a // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
 	golang.org/x/exp/typeparams v0.0.0-20260611194520-c48552f49976 // indirect
 	golang.org/x/mod v0.37.0 // indirect
-	golang.org/x/net v0.56.0 // indirect
+	golang.org/x/net v0.57.0 // indirect
 	golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 // indirect
 	golang.org/x/term v0.45.0 // indirect
 	golang.org/x/text v0.40.0 // indirect
 	golang.org/x/time v0.15.0 // indirect
 	golang.org/x/tools v0.47.0 // indirect
 	golang.org/x/vuln v1.4.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20260618152121-87f3d3e198d3 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20260618152121-87f3d3e198d3 // indirect
-	google.golang.org/grpc v1.82.1 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect
+	google.golang.org/grpc v1.83.0 // indirect
 	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	sigs.k8s.io/yaml v1.6.0 // indirect

go.sum

@@ -807,6 +807,8 @@ github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w=
 github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
+github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
@@ -865,6 +867,8 @@ github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ4
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
+github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
@@ -1319,6 +1323,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
 github.com/mulgadc/northstar v1.15.0 h1:DbJE/w3FLk19PNwgaC+Rnu/gP0E5GNMK9EGwR4TPqqU=
 github.com/mulgadc/northstar v1.15.0/go.mod h1:QbP2ydcCbGHidIdp61O2HOcJANFgGetS5zw60mbjxfQ=
+github.com/mulgadc/northstar v1.15.1-0.20260810233215-04fade67fb2c h1:Ia73hhrUvRRV7iEdCczHMhzrXtOJlQRotMN7rcMGFuA=
+github.com/mulgadc/northstar v1.15.1-0.20260810233215-04fade67fb2c/go.mod h1:1tRJZAKsXvsnMaAHAt5za+sDsRpkaoDGH0s8LST2Xyg=
 github.com/mulgadc/predastore v1.15.0 h1:IGQX8bbuCrwU5mC4JyjxgdL8UXGfTA1Cu9g9Kuu8Irw=
 github.com/mulgadc/predastore v1.15.0/go.mod h1:vjad49Pql/NtQLGfAC43SSDBcPLD6ijbqMbddbt0vA4=
 github.com/mulgadc/viperblock v1.15.0 h1:nZ8d8q8lSgMucd43GYgvwpmnCWXN379z6H2MxUHGo9g=
@@ -1627,6 +1633,8 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
 go.opentelemetry.io/contrib/bridges/otelslog v0.19.0 h1:5RgvxieNq9tS3ewrV1vnODvbHPfKUIJcYtF9Cvz+6aQ=
 go.opentelemetry.io/contrib/bridges/otelslog v0.19.0/go.mod h1:iTBIdNwx/xmUhfgJs6+84S4dIK059811cO1eUBjKcHY=
+go.opentelemetry.io/contrib/bridges/otelslog v0.20.0 h1:oEl2Pw/i4OQwhAuda2pAHFAcOMivA+Xa+iTccBfab/g=
+go.opentelemetry.io/contrib/bridges/otelslog v0.20.0/go.mod h1:yMSQaiiq5dpfrSJCYLBcqFeJkFFI67seT4ngvx6jfVo=
 go.opentelemetry.io/contrib/detectors/gcp v1.28.0/go.mod h1:9BIqH22qyHWAiZxQh0whuJygro59z+nbMVuc7ciiGug=
 go.opentelemetry.io/contrib/detectors/gcp v1.29.0/go.mod h1:GW2aWZNwR2ZxDLdv8OyC2G8zkRoQBuURgV7RPQgcPoU=
 go.opentelemetry.io/contrib/detectors/gcp v1.31.0/go.mod h1:tzQL6E1l+iV44YFTkcAeNQqzXUiekSYP9jjJjXwEd00=
@@ -1664,8 +1672,12 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04=
 go.opentelemetry.io/contrib/instrumentation/runtime v0.69.0 h1:MtkMsuRo3zEXTTMALfyrszwCDZTkB6wolyPjbwFAdq0=
 go.opentelemetry.io/contrib/instrumentation/runtime v0.69.0/go.mod h1:FYTxnpsm+UPD0erZNq20GvnM8T2YQHiHtT2vokdpoac=
+go.opentelemetry.io/contrib/instrumentation/runtime v0.70.0 h1:1+WLVYezXA9tkuVzKQri8zgB1cEIVYKUSoYIRjsBiMU=
+go.opentelemetry.io/contrib/instrumentation/runtime v0.70.0/go.mod h1:rbAXUUXqQDMxpSnmof4VtcZ+7YpZQEtjXSCIfdvR0Go=
 go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=
 go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo=
 go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI=
@@ -1689,14 +1701,24 @@ go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOX
 go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
 go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
 go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
+go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU=
+go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ=
 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.20.0 h1:rydZ9sxbcFdm/oWrVyfLTjHIygMgv0bEeMd+3B/BvoM=
 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.20.0/go.mod h1:earQ25dooT0Hhspq59DZ8YCC50jWfOlFEeWoxy/P444=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0 h1:WseeVYf5dJZTsyPiyW5L14k5qsSibqXAMTSiFEDiWr0=
+go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.21.0/go.mod h1:SiLZnQS6Qk2eCpvr2CH/XMAOa64TWGXxEZJZCpD2Lmc=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0 h1:SUplec5dp06reu1zaXmOXdvqH398taqrDXqUl99jxSc=
 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.44.0/go.mod h1:ho2g4N+ane+swq5I/VBkKWnRDY4kUINH3FuqyZqX/Ug=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.45.0 h1:klTViGcsvLCd1xN3rZzfZ12NslC/OimbmR+k+A006RI=
+go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.45.0/go.mod h1:jRsK04CWmXuY8A0O+wMpSf+t90RHZ53o5Qmxn2PQPfk=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 h1:qazEJlUOQzhCpzQpFETGby7EdqjI1wsd0W+6Gg1SCTU=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0/go.mod h1:fOD2Yefuxixkx3ahVNf0O/PERb6r4OlbxfATVnYvzCo=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0 h1:fG5MCxGz8+2VtrN/WgqSpJFctVz24gpxj8CxkKmc8Ww=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.45.0/go.mod h1:BmAYTn+3ysbRe+IU2msxmf5Rx3g6DHvex+tWI3LdhYI=
 go.opentelemetry.io/otel/exporters/prometheus v0.57.0/go.mod h1:QpFWz1QxqevfjwzYdbMb4Y1NnlJvqSGwyuU0B4iuc9c=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0/go.mod h1:U2R3XyVPzn0WX7wOIypPuptulsMcPDPs/oiSVOMVnHY=
@@ -1704,6 +1726,8 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6Us
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.42.0/go.mod h1:so9ounLcuoRDu033MW/E0AD4hhUjVqswrMF5FoZlBcw=
 go.opentelemetry.io/otel/log v0.20.0 h1:/5i0vuHxCLWUfChWG41K9wkM0jafruPw9NU1/RCJirs=
 go.opentelemetry.io/otel/log v0.20.0/go.mod h1:wOcMcjsZpG8x7Bak7IhSi/lg8wscV2C1VdrKCLPlt0E=
+go.opentelemetry.io/otel/log v0.21.0 h1:SLsVDGmtyBrdw8/a2Z0bOIxou/+bN4z56GebH7T0LvA=
+go.opentelemetry.io/otel/log v0.21.0/go.mod h1:iReetQrZL9Wyg84cCkOoCmqDHS5RCFfyxC7J+r8fn8g=
 go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
 go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
 go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY=
@@ -1727,8 +1751,11 @@ go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9k
 go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
 go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
 go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
+go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M=
+go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s=
 go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA=
 go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk=
+go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns=
 go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
 go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
 go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E=
@@ -1751,10 +1778,15 @@ go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWm
 go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
 go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
 go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
+go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw=
+go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA=
 go.opentelemetry.io/otel/sdk/log v0.20.0 h1:vM3xI7TQgKPiSghe6urZtAkyFY7SodrSpC83CffDFuY=
 go.opentelemetry.io/otel/sdk/log v0.20.0/go.mod h1:Knej2nmsTUzN79T2eeXdRsjjPcoxoq2pUyUHz9TFyyU=
+go.opentelemetry.io/otel/sdk/log v0.21.0 h1:QsE7XSR0ktQdKmRKGnR+f1ObGF32WG+7MER/P9KgmYc=
+go.opentelemetry.io/otel/sdk/log v0.21.0/go.mod h1:m9mApjCoD2/1QuKCAptjv+BrG9WKOvQLVdNx+iBldTo=
 go.opentelemetry.io/otel/sdk/log/logtest v0.20.0 h1:OqdRZ1guyzamK3M6LlRsmGqRrjkHWw6WZOKKli5ELpg=
 go.opentelemetry.io/otel/sdk/log/logtest v0.20.0/go.mod h1:PuMIlm7zAt7c3z8zfOI5ox4iT1Z87We+PF6YoINux/M=
+go.opentelemetry.io/otel/sdk/log/logtest v0.21.0 h1:X+JBBgKlswCGYsmgL0CnoUUtlE//VB345c84jYAYkdQ=
 go.opentelemetry.io/otel/sdk/metric v1.28.0/go.mod h1:cWPjykihLAPvXKi4iZc1dpER3Jdq2Z0YLse3moQUCpg=
 go.opentelemetry.io/otel/sdk/metric v1.29.0/go.mod h1:6zZLdCl2fkauYoZIOn/soQIDSWFmNSRcICarHfuhNJQ=
 go.opentelemetry.io/otel/sdk/metric v1.30.0/go.mod h1:waS6P3YqFNzeP01kuo/MBBYqaoBJl7efRQHOaydhy1Y=
@@ -1771,6 +1803,8 @@ go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx
 go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
 go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI=
 go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA=
+go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o=
+go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA=
 go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
 go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ=
 go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo=
@@ -1794,6 +1828,8 @@ go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhn
 go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
 go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
 go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
+go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag=
+go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
@@ -1803,6 +1839,8 @@ go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzK
 go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
 go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g=
 go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk=
+go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
+go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
@@ -2116,6 +2154,8 @@ golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
 golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
 golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
 golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
+golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
+golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -2830,6 +2870,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260519071638-aa98bba5eb94/go.
 google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY=
 google.golang.org/genproto/googleapis/api v0.0.0-20260618152121-87f3d3e198d3 h1:ctPmKL12ZsoKAlmPUsoW70zEDiYF+/H6aLieXxgAU0k=
 google.golang.org/genproto/googleapis/api v0.0.0-20260618152121-87f3d3e198d3/go.mod h1:Z4WJ5pJOYWFWcHEQUelD5QaZDknIQkpIL/+fyJOT9+A=
+google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc=
+google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig=
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA=
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:+34luvCflYKiKylNwGJfn9cFBbcL/WrkciMmDmsTQ/A=
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20231030173426-d783a09b4405/go.mod h1:GRUCuLdzVqZte8+Dl/D4N25yLzcGqqWaYkeVOwulFqw=
@@ -3011,6 +3053,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260610212136-7ab31c22f7ad/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260618152121-87f3d3e198d3 h1:phvBWCAQMGN1945mp5fjCXP6jEF0+a0+4TjokS4sxNY=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260618152121-87f3d3e198d3/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
@@ -3106,6 +3150,8 @@ google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07
 google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I=
 google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE=
 google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=
+google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ=
+google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y=
 google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b/go.mod h1:IBqQ7wSUJ2Ep09a8rMWFsg4fmI2r38zwsq8a0GgxXpM=

images/mkosi.profiles/vllm/mkosi.conf

@@ -0,0 +1,22 @@
+# vllm profile — vLLM OpenAI-compatible serving runtime, layered on gpu-nvidia.
+#
+# Only the serving concern lives here: the GPU driver, DKMS build and
+# nvidia-container-toolkit stay in gpu-nvidia, and this profile never
+# assembles into an image without it (see image_profiles() in
+# scripts/mkosi-build.sh). No Packages= of its own: curl and ca-certificates,
+# which mkosi.postinst.chroot needs to fetch uv and CPython, already come from
+# gpu-nvidia, and duplicating them here is exactly the drift this migration
+# exists to delete.
+[Content]
+# A serving VM launches through LaunchSystemInstance, so it boots dual-NIC with
+# a fw_cfg netcfg blob and takes its user-data solely from IMDS — the same shape
+# as the EKS control plane, whose pin is reused here rather than copied.
+# The 99- prefix is load-bearing: read_conf_d() applies cloud.cfg.d in REVERSE
+# and mergemanydict() is first-source-wins, so a 90- name loses to cloud-init's
+# own 90_dpkg.cfg ('_' sorts above '-') and the pin is silently discarded.
+ExtraTrees=../../../scripts/images/eks-node/cloud-init-ec2.cfg:/etc/cloud/cloud.cfg.d/99-spinifex-ec2-imds.cfg
+
+[Build]
+# mkosi.postinst.chroot fetches the uv installer, a standalone CPython build,
+# and vLLM's own PyPI/CUDA wheels — all network, none of it in the archive.
+WithNetwork=yes

images/mkosi.profiles/vllm/mkosi.extra/etc/cloud/cloud.cfg.d/99-spinifex-serial-console.cfg

@@ -0,0 +1,7 @@
+# A serving VM is headless: the serial console get-console-output reads is the
+# only channel an operator has, and a guest that wedges before handover never
+# surrenders /var/log/cloud-init-output.log. Mirror every stage to both.
+# The 99- prefix wins the REVERSE-sorted, first-source-wins cloud.cfg.d merge
+# against cloud-init's own 05_logging.cfg, which sets output= too.
+output:
+  all: '| tee -a /var/log/cloud-init-output.log > /dev/console'

images/mkosi.profiles/vllm/mkosi.extra/etc/systemd/system/vllm-serve.service

@@ -0,0 +1,41 @@
+[Unit]
+Description=vLLM OpenAI-compatible serving runtime (Bedrock self-host)
+# mulga-ebs-byid's coldplug pass mints the by-id link the wrapper's fallback
+# device resolution depends on. Ordering after it just avoids a guaranteed
+# spurious wait on every boot — the wrapper's own bounded waits are what
+# actually cover the race, since AttachVolume runs after this VM's launch
+# already returned.
+#
+# cloud-init write_files seeds /etc/conf.d/vllm-serve during the cloud-config
+# stage (same precedent as eks-token-webhook.service), so this too is not a
+# correctness fix — await_env_file already covers a handoff that is not there
+# yet — it just avoids burning that whole wait on every cold start.
+After=mulga-ebs-byid.service cloud-config.service
+# A start failure no retry can fix restarts forever otherwise, burying its own
+# first traceback under identical copies on the only console an operator has.
+StartLimitIntervalSec=300
+StartLimitBurst=3
+
+[Service]
+Type=exec
+# ExecStart names the wrapper and takes no arguments, so this line contains no
+# $VAR at all. That is deliberate: systemd expands $VAR/${VAR} in an Exec*=
+# line from the unit's own environment before the command runs and
+# substitutes empty for a name it does not know — the wrapper waits for and
+# sources /etc/conf.d/vllm-serve itself, in a shell it owns, to sidestep the
+# whole class of failure.
+ExecStart=/usr/local/sbin/vllm-serve
+# FlashInfer JIT-compiles its sampling module on first use and needs nvcc for
+# it. vLLM's native top-k/top-p sampler is the same operation with no CUDA
+# toolchain at runtime, which a serving appliance has no reason to carry.
+Environment=VLLM_USE_FLASHINFER_SAMPLER=0
+Restart=on-failure
+RestartSec=5
+# Headless appliance: the serial console is the only view of cold start
+# (env-file wait, weights mount, vLLM's own load time) until /v1/models
+# starts answering.
+StandardOutput=journal+console
+StandardError=journal+console
+
+[Install]
+WantedBy=multi-user.target

images/mkosi.profiles/vllm/mkosi.extra/usr/local/sbin/vllm-serve

@@ -0,0 +1,293 @@
+#!/bin/sh
+# vllm-serve — wrapper for vllm-serve.service. Waits for the cloud-init
+# handoff, resolves and mounts the endpoint's cloned weights volume read-only,
+# then execs vLLM's OpenAI-compatible server against it.
+#
+# Handoff contract, written by the Bedrock launcher as a cloud-init
+# write_files entry (so it does not exist before cloud-init runs):
+#
+#   /etc/conf.d/vllm-serve   0600  KEY=value shell fragment:
+#       VLLM_MODEL_ID=<Bedrock model id, reported back via /v1/models>
+#       VLLM_ARGS=<extra vllm serve flags, space-separated>
+#       VLLM_WEIGHTS_DEVICE=<device the launcher attached the volume at>
+#       VLLM_SERVE_PORT=<listen port>
+#
+# ExecStart in vllm-serve.service takes no arguments and contains no $VAR —
+# this script sources the handoff itself, in a shell it owns, rather than
+# letting systemd expand anything from an Exec*= line.
+set -eu
+
+ENV_FILE="${VLLM_ENV_FILE:-/etc/conf.d/vllm-serve}"
+# The unit starts before cloud-init has necessarily delivered the handoff, so
+# this bounds that race rather than assuming it is already there.
+ENV_WAIT_SECS="${VLLM_ENV_WAIT:-60}"
+
+WEIGHTS_MOUNT="${VLLM_WEIGHTS_MOUNT:-/var/lib/vllm/weights}"
+VENV_BIN="${VLLM_VENV_BIN:-/opt/vllm/venv/bin}"
+
+# AttachVolume runs after the launch already returned the instance, so the
+# weights volume legitimately does not exist yet when this unit first starts.
+DEVICE_WAIT_SECS="${VLLM_DEVICE_WAIT:-120}"
+
+SYS_BLOCK="${VLLM_SYS_BLOCK:-/sys/block}"
+DEV_DIR="${VLLM_DEV_DIR:-/dev}"
+BYID_DIR="${VLLM_BYID_DIR:-/dev/disk/by-id}"
+MOUNTS_FILE="${VLLM_MOUNTS_FILE:-/proc/mounts}"
+
+log() { echo "[vllm-serve] $*"; }
+fail() { echo "[vllm-serve] ERROR: $*" >&2; exit 1; }
+
+# Tries to read a *usable* handoff once: sets VLLM_MODEL_ID/VLLM_ARGS/
+# VLLM_WEIGHTS_DEVICE/VLLM_SERVE_PORT in the caller's shell (deliberately not
+# run inside a `$(...)` — a subshell's assignments do not survive it) and
+# returns 0 on success. Returns 1 when the handoff is not there yet or is
+# still mid-write (worth retrying — cloud-init's write_files is not atomic
+# from this reader's point of view: the file can exist, created but empty
+# or half-flushed, for a brief window before its content lands), and 2 when
+# a key IS present but malformed (never becomes valid by waiting longer, so
+# not worth retrying — the same found/not-yet/bad three-way split
+# resolve_weights_device uses below).
+try_load_env() {
+    [ -f "${ENV_FILE}" ] || return 1
+
+    # VLLM_ARGS holds a space-separated list of flags, so this cannot be a
+    # plain `. "${ENV_FILE}"` the way rds-init sources its handoff: shell
+    # would parse "VLLM_ARGS=--foo --bar" as an assignment followed by a
+    # second command and fail on the second word. `read` with IFS='=' splits
+    # only the key off the front; the last variable absorbs the rest of the
+    # line verbatim, spaces and all, with no further word-splitting.
+    _raw_model="" _raw_args="" _raw_device="" _raw_port=""
+    while IFS='=' read -r _key _value || [ -n "${_key:-}" ]; do
+        case "${_key:-}" in
+            VLLM_MODEL_ID)       _raw_model="${_value:-}" ;;
+            VLLM_ARGS)           _raw_args="${_value:-}" ;;
+            VLLM_WEIGHTS_DEVICE) _raw_device="${_value:-}" ;;
+            VLLM_SERVE_PORT)     _raw_port="${_value:-}" ;;
+        esac
+    done < "${ENV_FILE}"
+
+    # Missing required keys means the write is still in flight, not that the
+    # handoff is broken — retry rather than fail.
+    [ -n "${_raw_model}" ] || return 1
+    [ -n "${_raw_device}" ] || return 1
+
+    # Unlike a merely-missing key, a port that IS present but non-numeric
+    # will never self-correct by waiting: fail now, do not retry.
+    case "${_raw_port}" in
+        '') _raw_port=8000 ;;
+        *[!0-9]*)
+            echo "[vllm-serve] ERROR: bootstrap handoff has a non-numeric VLLM_SERVE_PORT: ${_raw_port}" >&2
+            return 2
+            ;;
+    esac
+
+    VLLM_MODEL_ID="${_raw_model}"
+    VLLM_ARGS="${_raw_args}"
+    VLLM_WEIGHTS_DEVICE="${_raw_device}"
+    VLLM_SERVE_PORT="${_raw_port}"
+    return 0
+}
+
+await_env_file() {
+    _waited=0
+    while :; do
+        # `if try_load_env; then` (not `try_load_env; _rc=$?`) for the same
+        # reason as await_weights_device below: a plain assignment-less
+        # command IS a checked context under set -e too, so calling
+        # try_load_env as a bare statement and letting a 1 ("not yet")
+        # propagate would abort the script on the very first empty-file read
+        # instead of looping. The if-condition is what set -e exempts.
+        if try_load_env; then
+            log "bootstrap handoff loaded (model=${VLLM_MODEL_ID}, port=${VLLM_SERVE_PORT})"
+            return 0
+        else
+            _rc=$?
+        fi
+        [ "${_rc}" -eq 2 ] && fail "bootstrap handoff at ${ENV_FILE} is malformed (see above) — refusing to guess"
+        if [ "${_waited}" -ge "${ENV_WAIT_SECS}" ]; then
+            fail "no usable bootstrap handoff at ${ENV_FILE} after ${ENV_WAIT_SECS}s — cloud-init has not delivered it"
+        fi
+        sleep 1
+        _waited=$((_waited + 1))
+    done
+}
+
+# An already-mounted weights point makes device resolution a no-op, so a
+# Restart=on-failure re-run (vLLM itself crashed, not the mount) does not try
+# to mount the same device twice.
+weights_mounted() {
+    awk -v mnt="${WEIGHTS_MOUNT}" '$2 == mnt { found = 1 } END { exit !found }' "${MOUNTS_FILE}"
+}
+
+# The device backing "/", stripped to its parent disk (vda2 -> vda,
+# nvme0n1p2 -> nvme0n1), so the ext4 scan below never mistakes the root disk
+# for the weights volume. Read from MOUNTS_FILE rather than shelling out to
+# lsblk/findmnt against a real device node, which a test harness cannot fake.
+root_disk() {
+    # || true: a plain assignment from a command substitution is a checked
+    # command under set -e — an awk failure here (MOUNTS_FILE unreadable,
+    # never expected against the real /proc/mounts) would otherwise abort
+    # this function's caller's subshell outright instead of leaving _root_src
+    # empty, which the case below already handles safely.
+    _root_src=$(awk '$2 == "/" { print $1; exit }' "${MOUNTS_FILE}") || true
+    _name="${_root_src#/dev/}"
+    case "${_name}" in
+        *n[0-9]*p[0-9]*) echo "${_name}" | sed -E 's/p[0-9]+$//' ;;
+        *)                echo "${_name}" | sed -E 's/[0-9]+$//' ;;
+    esac
+    return 0
+}
+
+# Prints the filesystem type on dev, nothing if it holds none, and returns
+# non-zero when it could not find out at all: a blkid that is missing, errored
+# or pointed at an absent node prints exactly what a blank disk prints, so
+# only the exit status tells the two apart.
+fstype_of() {
+    if _out=$(blkid "$1" 2>/dev/null); then
+        _rc=0
+    else
+        _rc=$?
+    fi
+    # 0 is a signature found, 2 a clean probe that found none.
+    [ "${_rc}" -eq 0 ] || [ "${_rc}" -eq 2 ] || return 1
+    printf '%s' "${_out}" | sed -n 's/.*[[:space:]]TYPE="\([^"]*\)".*/\1/p'
+}
+
+# The one by-id link mulga-ebs-byid mints for a hot-attached Viperblock
+# volume. One line per candidate; the caller decides what more than one means.
+# Explicit `return 0`: this is a pure enumerator, not a signal, and the last
+# glob iteration's test can otherwise leave the function's own exit status
+# nonzero (see resolve_weights_device's callers for why that matters).
+byid_candidates() {
+    for _l in "${BYID_DIR}"/*Elastic_Block_Store*; do
+        [ -e "${_l}" ] || continue
+        printf '%s\n' "${_l}"
+    done
+    return 0
+}
+
+# The sole non-root block device carrying an ext4 filesystem — sound only
+# because a serving VM carries exactly one weights volume by construction
+# (LaunchServingVM attaches exactly one volume per launch).
+ext4_candidates() {
+    # || true: root_disk's own last command is a pipeline over MOUNTS_FILE,
+    # and an enumerator's exit status must never leak an unrelated tool's
+    # failure into the caller as if it meant something.
+    _root="$(root_disk)" || true
+    for _p in "${SYS_BLOCK}"/*; do
+        [ -d "${_p}" ] || continue
+        _name=$(basename "${_p}") || continue
+        case "${_name}" in
+            loop*|sr*|zram*) continue ;;
+        esac
+        [ "${_name}" = "${_root}" ] && continue
+        _dev="${DEV_DIR}/${_name}"
+        [ -e "${_dev}" ] || continue
+        _fstype=$(fstype_of "${_dev}") || continue
+        # Deliberately an `if`, not `[ ... ] && printf`: the latter leaves
+        # this loop iteration's (and so the function's) own exit status equal
+        # to the test's, so a last-processed disk that is not ext4 would make
+        # the enumerator itself look like it failed to its caller.
+        if [ "${_fstype}" = "ext4" ]; then
+            printf '%s\n' "${_dev}"
+        fi
+    done
+    return 0
+}
+
+# Tries the fallback chain once. Prints the device and returns 0 on a single
+# match, returns 1 when nothing is found yet (worth retrying — the volume may
+# still be attaching) and 2 when the guest's own layout is ambiguous (never
+# worth retrying: waiting longer cannot make two candidates become one).
+resolve_weights_device() {
+    # 1. The device the launcher recorded, if the guest's own enumeration
+    # happens to agree with it.
+    if [ -e "${VLLM_WEIGHTS_DEVICE}" ]; then
+        printf '%s\n' "${VLLM_WEIGHTS_DEVICE}"
+        return 0
+    fi
+
+    # 2. mulga-ebs-byid's link — a virtio/nvme guest does not necessarily
+    # present the launcher's named device, but it always mints exactly one of
+    # these for the one weights volume a serving VM carries.
+    _byid="$(byid_candidates)"
+    # grep -c exits 1 when it counts zero matches, which set -e would treat as
+    # this whole function failing rather than "no candidates yet" — || true
+    # keeps a legitimate zero from aborting the script.
+    _byid_count=$(printf '%s\n' "${_byid}" | grep -c . || true)
+    if [ "${_byid_count}" -eq 1 ]; then
+        printf '%s\n' "${_byid}"
+        return 0
+    elif [ "${_byid_count}" -gt 1 ]; then
+        echo "[vllm-serve] ERROR: found ${_byid_count} EBS by-id links; expected exactly one weights volume" >&2
+        return 2
+    fi
+
+    # 3. The last resort: whichever non-root disk holds an ext4 filesystem.
+    _ext4="$(ext4_candidates)"
+    # Same zero-matches-exits-1 guard as the by-id count above.
+    _ext4_count=$(printf '%s\n' "${_ext4}" | grep -c . || true)
+    if [ "${_ext4_count}" -eq 1 ]; then
+        printf '%s\n' "${_ext4}"
+        return 0
+    elif [ "${_ext4_count}" -gt 1 ]; then
+        echo "[vllm-serve] ERROR: found ${_ext4_count} non-root ext4 volumes; expected exactly one weights volume" >&2
+        return 2
+    fi
+
+    return 1
+}
+
+await_weights_device() {
+    _waited=0
+    while :; do
+        # A plain `_dev=$(resolve_weights_device); _rc=$?` is NOT a checked
+        # context under set -e: the assignment's own failure (resolve
+        # returning 1 while the volume is still attaching, the normal case on
+        # every cold start) would exit the whole script on the spot, and
+        # `_rc=$?` would never run to tell the difference from "ambiguous".
+        # The if/else form is the one context set -e exempts, so the "not
+        # found yet, keep waiting" path actually gets to retry.
+        if _dev=$(resolve_weights_device); then
+            printf '%s\n' "${_dev}"
+            return 0
+        else
+            _rc=$?
+        fi
+        [ "${_rc}" -eq 2 ] && fail "ambiguous weights device candidates — refusing to guess (see above)"
+        if [ "${_waited}" -ge "${DEVICE_WAIT_SECS}" ]; then
+            fail "no weights volume resolved after ${DEVICE_WAIT_SECS}s"
+        fi
+        sleep 1
+        _waited=$((_waited + 1))
+    done
+}
+
+await_env_file
+
+command -v blkid >/dev/null 2>&1 ||
+    fail "blkid is not installed in this image; the by-id and ext4-scan fallbacks cannot run without it"
+
+if weights_mounted; then
+    log "${WEIGHTS_MOUNT} is already a mount point; reusing it"
+else
+    _device=$(await_weights_device)
+    log "weights device resolved to ${_device}"
+    mkdir -p "${WEIGHTS_MOUNT}"
+    # Read-only: nothing in the serving path writes weights, and a COW clone
+    # that gets dirtied costs storage for no gain.
+    mount -o ro "${_device}" "${WEIGHTS_MOUNT}" ||
+        fail "could not mount ${_device} read-only at ${WEIGHTS_MOUNT}"
+    log "weights mounted read-only at ${WEIGHTS_MOUNT} from ${_device}"
+fi
+
+# --served-model-name reports the Bedrock model ID via /v1/models, not the
+# mount path — the gateway sends the Bedrock ID as the OpenAI `model` field.
+# VLLM_ARGS is deliberately unquoted: it is a space-separated list of extra
+# flags the operator supplied, and each one must reach vllm serve as its own
+# argument.
+# shellcheck disable=SC2086
+exec "${VENV_BIN}/vllm" serve "${WEIGHTS_MOUNT}" \
+    --port "${VLLM_SERVE_PORT}" \
+    --served-model-name "${VLLM_MODEL_ID}" \
+    ${VLLM_ARGS}

images/mkosi.profiles/vllm/mkosi.postinst.chroot

@@ -0,0 +1,126 @@
+#!/bin/bash
+set -euo pipefail
+
+# vllm postinst — bakes vLLM's OpenAI-compatible server into a uv-managed venv
+# and enables vllm-serve.service. A venv, not a container: a runtime `docker
+# pull vllm/vllm-openai` inside a guest is known-fatal on this platform (see
+# the viperblock write-plane-throughput investigation — it stalled indefinitely
+# and drove host load to ~124), and mkosi's unprivileged user-namespace build
+# has no daemon in the builder to pull into regardless. Baking the venv here
+# instead makes first boot a pure local read.
+#
+# The .chroot suffix means mkosi has already chrooted into the image root
+# before this runs, same as gpu-nvidia's postinst.
+
+VLLM_HOME=/opt/vllm
+
+# The exact CPython version vLLM's own wheels are built and tested against.
+# Ubuntu 26.04's system interpreter is newer than that range, and pinning
+# against a distro default that moves is exactly the drift the mkosi
+# migration exists to delete — uv fetches this standalone build instead, so
+# the interpreter is a build input, not a distro accident. Bumping this is a
+# deliberate edit, done alongside checking vLLM's published wheel matrix.
+readonly VLLM_PYTHON_VERSION="3.12.8"
+
+# uv's own version, not the Python or vllm it installs. Astral's installer
+# accepts a versioned URL, so this pin is the same "build input, not a distro
+# accident" guarantee as VLLM_PYTHON_VERSION above — an unpinned `install.sh`
+# would let the installer itself drift between two builds of the same commit.
+readonly UV_VERSION="0.12.1"
+
+# The vLLM release baked into the venv. Pinned for the same reason as the
+# CPython and uv versions above: two builds of the same commit must produce
+# the same image. Bumping this is a deliberate edit, done alongside checking
+# vLLM's published wheel matrix against VLLM_PYTHON_VERSION.
+readonly VLLM_VERSION="0.26.0"
+
+PYTHON_INSTALL_DIR="${VLLM_HOME}/python"
+VENV_DIR="${VLLM_HOME}/venv"
+
+mkdir -p "${VLLM_HOME}"
+
+# uv itself is not in the Ubuntu archive, so its own installer is used rather
+# than Packages=. UV_NO_MODIFY_PATH because there is no interactive shell
+# profile in a chroot for it to edit, and UV_INSTALL_DIR pins the destination.
+UV=/usr/local/bin/uv
+curl -LsSf "https://astral.sh/uv/${UV_VERSION}/install.sh" |
+    env UV_INSTALL_DIR=/usr/local/bin UV_NO_MODIFY_PATH=1 sh
+# Checked as a path, not via `command -v`: mkosi runs this postinst with a
+# minimal PATH that need not carry /usr/local/bin, so a PATH lookup reports a
+# perfectly good install as missing. Every call below names ${UV} for the
+# same reason.
+[ -x "${UV}" ] || {
+    echo "ERROR: uv installer completed but ${UV} is not executable" >&2
+    exit 1
+}
+
+echo "[vllm] installing CPython ${VLLM_PYTHON_VERSION} via uv"
+UV_PYTHON_INSTALL_DIR="${PYTHON_INSTALL_DIR}" "${UV}" python install "${VLLM_PYTHON_VERSION}"
+
+echo "[vllm] creating venv at ${VENV_DIR}"
+UV_PYTHON_INSTALL_DIR="${PYTHON_INSTALL_DIR}" "${UV}" venv --python "${VLLM_PYTHON_VERSION}" "${VENV_DIR}"
+
+echo "[vllm] installing vllm ${VLLM_VERSION} into ${VENV_DIR}"
+UV_PYTHON_INSTALL_DIR="${PYTHON_INSTALL_DIR}" "${UV}" pip install --python "${VENV_DIR}/bin/python" "vllm==${VLLM_VERSION}"
+
+# The one thing this script must not do silently: ship an image whose venv
+# does not actually hold what was asked for. Checked via package metadata and
+# not by running `vllm --version`, which initialises device config and dies
+# with "Failed to infer device type" in a build sandbox that has no GPU —
+# importing vllm itself is out for the same reason.
+if [ ! -x "${VENV_DIR}/bin/vllm" ]; then
+    echo "ERROR: no vllm entrypoint in the baked venv at ${VENV_DIR}" >&2
+    exit 1
+fi
+installed_vllm="$("${VENV_DIR}/bin/python" -c \
+    'import importlib.metadata as m; print(m.version("vllm"))')"
+if [ "${installed_vllm}" != "${VLLM_VERSION}" ]; then
+    echo "ERROR: venv holds vllm ${installed_vllm}, expected the pinned ${VLLM_VERSION}" >&2
+    exit 1
+fi
+echo "[vllm] verified vllm ${installed_vllm} in ${VENV_DIR}"
+
+# 0644 in git, and ExtraTrees= preserves source mode — without this the unit
+# dies with EACCES on first boot and nowhere earlier.
+chmod 0755 /usr/local/sbin/vllm-serve
+
+systemctl enable vllm-serve.service
+
+# Without this the failure is a runtime one, minutes into a GPU cold start:
+# FlashInfer JIT-builds its sampling module, finds no nvcc, and the endpoint
+# never reaches READY.
+UNIT=/etc/systemd/system/vllm-serve.service
+grep -q '^Environment=VLLM_USE_FLASHINFER_SAMPLER=0$' "${UNIT}" || {
+    echo "ERROR: ${UNIT} does not disable the FlashInfer sampler; this image ships no CUDA toolkit for its JIT" >&2
+    exit 1
+}
+
+# Assert the MERGED result, not the file's presence: the pin already existed at
+# a 90- prefix in the EKS image and did nothing there, shadowed by 90_dpkg.cfg.
+# read_conf_with_confd is the same function cloud-init calls at boot.
+command -v python3 >/dev/null || {
+    echo "ERROR: python3 missing — cannot verify the merged cloud-init datasource_list" >&2
+    exit 1
+}
+python3 - <<'PY'
+import sys
+
+try:
+    from cloudinit import util
+except ImportError as e:
+    sys.exit(f"ERROR: cloudinit python module unavailable ({e}) — cloud-init is expected to be installed by now")
+
+cfg = util.read_conf_with_confd("/etc/cloud/cloud.cfg")
+got = cfg.get("datasource_list")
+want = ["Ec2", "None"]
+if got != want:
+    sys.exit(
+        "ERROR: merged cloud-init datasource_list is "
+        f"{got!r}, expected {want!r} — the 99-spinifex-ec2-imds.cfg pin is "
+        "being shadowed by another cloud.cfg.d drop-in (check sort order: "
+        "read_conf_d() applies the directory in REVERSE, first source wins)"
+    )
+print(f"[vllm] cloud-init datasource_list assertion passed: {got!r}")
+PY
+
+echo "vllm postinst complete: CPython ${VLLM_PYTHON_VERSION} / uv ${UV_VERSION} / vllm ${VLLM_VERSION} venv baked at ${VENV_DIR}, vllm-serve.service enabled"

images/mkosi.profiles/vllm/vllm-serve_test.sh

@@ -0,0 +1,346 @@
+#!/bin/sh
+# Self-contained POSIX test for vllm-serve (mkosi.extra/usr/local/sbin/vllm-serve):
+# the env-file wait, the three-tier weights-device resolution (named device,
+# by-id fallback, ext4-scan fallback), the bounded waits' timeouts, and the
+# final read-only mount + exec. No root and no real block devices: blkid,
+# mount and the venv's vllm binary are stubbed on PATH, and every path the
+# script touches is redirected into a temp dir via its env knobs. Not shipped
+# into the image — it sits beside the profile, not under mkosi.extra/.
+#
+# Run: sh images/mkosi.profiles/vllm/vllm-serve_test.sh
+set -eu
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+# VLLM_TEST_SCRIPT: override to point at an alternate copy of the wrapper
+# (e.g. a saved pre-fix version), so a regression case can be run against
+# both and its before/after behaviour compared directly.
+SCRIPT="${VLLM_TEST_SCRIPT:-${SCRIPT_DIR}/mkosi.extra/usr/local/sbin/vllm-serve}"
+WORK=$(mktemp -d)
+trap 'rm -rf "${WORK}"' EXIT
+
+STUBBIN="${WORK}/bin"
+mkdir -p "${STUBBIN}"
+
+# blkid stub: prints a real blkid's line for any device listed in FS_TABLE as
+# "<dev> <fstype>", nothing (rc=2) for one that holds none.
+cat > "${STUBBIN}/blkid" <<'EOF'
+#!/bin/sh
+dev="$1"
+fstype=$(awk -v d="${dev}" '$1 == d { print $2 }' "${FS_TABLE}" 2>/dev/null)
+[ -n "${fstype}" ] || exit 2
+echo "${dev}: UUID=\"1234-5678\" TYPE=\"${fstype}\""
+EOF
+
+# mount stub: records the call and, on success, appends the mount to
+# MOUNTS_FILE so weights_mounted() sees the effect on a later invocation
+# (idempotent-restart case), the way a real kernel's /proc/mounts would.
+cat > "${STUBBIN}/mount" <<'EOF'
+#!/bin/sh
+echo "mount $*" >> "${MOUNT_CALLS}"
+[ "${MOUNT_FAIL:-0}" = "1" ] && exit 1
+dev=""; mnt=""
+for a in "$@"; do
+    case "$a" in -*) continue ;; esac
+    if [ -z "$dev" ]; then dev="$a"; else mnt="$a"; fi
+done
+echo "${dev} ${mnt} ext4 ro 0 0" >> "${MOUNTS_FILE}"
+exit 0
+EOF
+
+# vllm stub: the venv binary the wrapper execs into. Records its argv so the
+# test can assert the mount point, port and served-model-name reached it.
+cat > "${STUBBIN}/vllm" <<'EOF'
+#!/bin/sh
+echo "vllm $*" >> "${VLLM_CALLS}"
+exit 0
+EOF
+
+chmod +x "${STUBBIN}"/*
+PATH="${STUBBIN}:${PATH}"
+export PATH
+
+FAILS=0
+fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); }
+pass() { echo "ok: $*"; }
+
+ENV_FILE="${WORK}/vllm-serve.env"
+WEIGHTS_MOUNT="${WORK}/weights"
+SYS_BLOCK="${WORK}/sys/block"
+DEV_DIR="${WORK}/dev"
+BYID_DIR="${WORK}/by-id"
+MOUNTS_FILE="${WORK}/mounts"
+FS_TABLE="${WORK}/fs.table"
+MOUNT_CALLS="${WORK}/mount.calls"
+VLLM_CALLS="${WORK}/vllm.calls"
+export FS_TABLE MOUNT_CALLS VLLM_CALLS MOUNTS_FILE
+
+write_env() {
+    # write_env <device> [extra-args]: lay down the bootstrap handoff
+    # cloud-init would have written from buildServeUserData.
+    cat > "${ENV_FILE}" <<EOF
+VLLM_MODEL_ID=test-model
+VLLM_ARGS=${2:-}
+VLLM_WEIGHTS_DEVICE=$1
+VLLM_SERVE_PORT=8000
+EOF
+}
+
+# add_disk <name>: materialise a whole disk in the fake sysfs + /dev.
+add_disk() {
+    mkdir -p "${SYS_BLOCK}/$1" "${DEV_DIR}"
+    : > "${DEV_DIR}/$1"
+}
+
+reset_state() {
+    rm -rf "${WORK}/sys" "${DEV_DIR}" "${BYID_DIR}" "${WEIGHTS_MOUNT}" "${ENV_FILE}"
+    mkdir -p "${SYS_BLOCK}" "${DEV_DIR}" "${BYID_DIR}"
+    # Root is vda2, so root_disk() must resolve to "vda" and never treat it as
+    # a weights candidate.
+    printf '/dev/vda2 / ext4 rw,relatime 0 0\n' > "${MOUNTS_FILE}"
+    : > "${FS_TABLE}"
+    : > "${MOUNT_CALLS}"
+    : > "${VLLM_CALLS}"
+    add_disk vda
+}
+
+# run: invoke vllm-serve with every path knob pointed into the temp dir and
+# short bounded waits, so the "nothing ever appears" case does not stall the
+# suite.
+run() {
+    env VLLM_ENV_FILE="${ENV_FILE}" \
+        VLLM_ENV_WAIT="${VLLM_ENV_WAIT:-2}" \
+        VLLM_WEIGHTS_MOUNT="${WEIGHTS_MOUNT}" \
+        VLLM_VENV_BIN="${STUBBIN}" \
+        VLLM_DEVICE_WAIT="${VLLM_DEVICE_WAIT:-2}" \
+        VLLM_SYS_BLOCK="${SYS_BLOCK}" \
+        VLLM_DEV_DIR="${DEV_DIR}" \
+        VLLM_BYID_DIR="${BYID_DIR}" \
+        VLLM_MOUNTS_FILE="${MOUNTS_FILE}" \
+        sh "${SCRIPT}" </dev/null
+}
+
+run_ok() { run > "${WORK}/out" 2>&1 || { fail "$1: non-zero exit: $(cat "${WORK}/out")"; return 1; }; }
+run_fails() { run > "${WORK}/out" 2>&1 && fail "$1: expected a non-zero exit" || pass "$1: refused"; }
+
+# --- Case 1: named device present ---
+reset_state
+add_disk sdf
+write_env "${DEV_DIR}/sdf"
+if run_ok "named-device"; then
+    grep -q "mount -o ro ${DEV_DIR}/sdf ${WEIGHTS_MOUNT}" "${MOUNT_CALLS}" \
+        && pass "named-device: mounted read-only from the named device" || fail "named-device: not mounted from named device"
+    grep -q "vllm serve ${WEIGHTS_MOUNT} --port 8000 --served-model-name test-model" "${VLLM_CALLS}" \
+        && pass "named-device: vllm exec'd with mount point, port and model name" || fail "named-device: vllm not invoked correctly"
+fi
+
+# --- Case 2: named device absent, mulga-ebs-byid's by-id link used instead ---
+reset_state
+add_disk vdb
+: > "${BYID_DIR}/nvme-Amazon_Elastic_Block_Store_vol0123456789abcdef"
+write_env "/dev/sdf"
+if run_ok "byid-fallback"; then
+    grep -q "mount -o ro ${BYID_DIR}/nvme-Amazon_Elastic_Block_Store_vol0123456789abcdef ${WEIGHTS_MOUNT}" "${MOUNT_CALLS}" \
+        && pass "byid-fallback: mounted from the by-id link" || fail "byid-fallback: not mounted from by-id link"
+fi
+
+# --- Case 3: named device and by-id both absent, ext4-scan fallback used ---
+reset_state
+add_disk vdb
+echo "${DEV_DIR}/vdb ext4" > "${FS_TABLE}"
+write_env "/dev/sdf"
+if run_ok "ext4-fallback"; then
+    grep -q "mount -o ro ${DEV_DIR}/vdb ${WEIGHTS_MOUNT}" "${MOUNT_CALLS}" \
+        && pass "ext4-fallback: mounted from the sole non-root ext4 disk" || fail "ext4-fallback: not mounted from ext4 disk"
+fi
+
+# --- Case 3b: the root disk is never an ext4-scan candidate ---
+reset_state
+echo "${DEV_DIR}/vda ext4" > "${FS_TABLE}"
+write_env "/dev/sdf"
+run_fails "ext4-fallback-root-excluded"
+grep -q . "${MOUNT_CALLS}" \
+    && fail "ext4-fallback-root-excluded: mounted the root disk as weights" || pass "ext4-fallback-root-excluded: root disk excluded"
+
+# --- Case 4: nothing ever appears ---
+# Times the run to prove the wrapper actually spun in the retry loop for the
+# full DEVICE_WAIT_SECS bound rather than dying on the first "not found yet"
+# — an exit-status-only check cannot tell a real timeout from an instant
+# death (that was exactly how the set -e regression below slipped through).
+reset_state
+write_env "/dev/sdf"
+_t0=$(date +%s)
+VLLM_DEVICE_WAIT=3 run_fails "nothing-appears"
+_t1=$(date +%s)
+_elapsed=$((_t1 - _t0))
+[ "${_elapsed}" -ge 3 ] \
+    && pass "nothing-appears: waited the full ${_elapsed}s bound before giving up" \
+    || fail "nothing-appears: returned after only ${_elapsed}s, expected >= 3s (instant death, not a real wait)"
+grep -q . "${MOUNT_CALLS}" \
+    && fail "nothing-appears: mounted something that should not exist" || pass "nothing-appears: refused, nothing mounted"
+
+# --- Case 4b/c/d: device appears part-way through the bounded wait, for each
+# of the three resolution tiers in turn. This is the case the exit-status-only
+# "nothing-appears" case above cannot distinguish from an instant set -e
+# death: only a run that keeps retrying past the first failed probe and then
+# succeeds proves the retry loop itself works, not just that it eventually
+# gives up.
+
+# 4b: named device absent on the first probe, appears mid-wait.
+reset_state
+write_env "${DEV_DIR}/sdf"
+( sleep 1; add_disk sdf ) &
+BGPID=$!
+if VLLM_DEVICE_WAIT=5 run_ok "named-device-appears-midwait"; then
+    grep -q "mount -o ro ${DEV_DIR}/sdf ${WEIGHTS_MOUNT}" "${MOUNT_CALLS}" \
+        && pass "named-device-appears-midwait: retried until the named device showed up, then mounted it" \
+        || fail "named-device-appears-midwait: did not mount the named device once it appeared"
+fi
+wait "${BGPID}" 2>/dev/null || true
+
+# 4c: by-id link absent on the first probe, appears mid-wait.
+reset_state
+add_disk vdb
+write_env "/dev/sdf"
+( sleep 1; : > "${BYID_DIR}/nvme-Amazon_Elastic_Block_Store_vol0123456789abcdef" ) &
+BGPID=$!
+if VLLM_DEVICE_WAIT=5 run_ok "byid-appears-midwait"; then
+    grep -q "mount -o ro ${BYID_DIR}/nvme-Amazon_Elastic_Block_Store_vol0123456789abcdef ${WEIGHTS_MOUNT}" "${MOUNT_CALLS}" \
+        && pass "byid-appears-midwait: retried until the by-id link showed up, then mounted it" \
+        || fail "byid-appears-midwait: did not mount the by-id link once it appeared"
+fi
+wait "${BGPID}" 2>/dev/null || true
+
+# 4d: ext4-scan candidate absent on the first probe (neither the disk nor its
+# filesystem exists yet), appears mid-wait.
+reset_state
+write_env "/dev/sdf"
+( sleep 1; add_disk vdb; echo "${DEV_DIR}/vdb ext4" >> "${FS_TABLE}" ) &
+BGPID=$!
+if VLLM_DEVICE_WAIT=5 run_ok "ext4-appears-midwait"; then
+    grep -q "mount -o ro ${DEV_DIR}/vdb ${WEIGHTS_MOUNT}" "${MOUNT_CALLS}" \
+        && pass "ext4-appears-midwait: retried until the ext4 disk showed up, then mounted it" \
+        || fail "ext4-appears-midwait: did not mount the ext4 disk once it appeared"
+fi
+wait "${BGPID}" 2>/dev/null || true
+
+# --- Case 5: two by-id links is ambiguous, refused rather than guessed ---
+# Ambiguity is never worth retrying (more waiting cannot turn two candidates
+# into one), so this must fail fast, well inside the wait bound — timed here
+# to positively confirm the 2 (ambiguous) path is distinct from the 1 (not
+# yet) retry path, not just that both eventually return non-zero.
+reset_state
+: > "${BYID_DIR}/nvme-Amazon_Elastic_Block_Store_vol0000000000000001"
+: > "${BYID_DIR}/nvme-Amazon_Elastic_Block_Store_vol0000000000000002"
+write_env "/dev/sdf"
+_t0=$(date +%s)
+VLLM_DEVICE_WAIT=30 run_fails "byid-ambiguous"
+_t1=$(date +%s)
+_elapsed=$((_t1 - _t0))
+[ "${_elapsed}" -lt 15 ] \
+    && pass "byid-ambiguous: failed fast in ${_elapsed}s rather than waiting out the 30s bound" \
+    || fail "byid-ambiguous: took ${_elapsed}s, expected a fast refusal instead of retrying an unresolvable ambiguity"
+grep -q . "${MOUNT_CALLS}" \
+    && fail "byid-ambiguous: mounted one of two candidates" || pass "byid-ambiguous: nothing mounted"
+
+# --- Case 6: two non-root ext4 disks is likewise ambiguous ---
+reset_state
+add_disk vdb
+add_disk vdc
+echo "${DEV_DIR}/vdb ext4" > "${FS_TABLE}"
+echo "${DEV_DIR}/vdc ext4" >> "${FS_TABLE}"
+write_env "/dev/sdf"
+run_fails "ext4-ambiguous"
+grep -q . "${MOUNT_CALLS}" \
+    && fail "ext4-ambiguous: mounted one of two candidates" || pass "ext4-ambiguous: nothing mounted"
+
+# --- Case 7: env file absent, then appears mid-wait ---
+reset_state
+add_disk sdf
+VLLM_ENV_WAIT=5
+( sleep 1; write_env "${DEV_DIR}/sdf" ) &
+BGPID=$!
+if VLLM_ENV_WAIT=5 run_ok "env-appears-midwait"; then
+    pass "env-appears-midwait: wrapper picked up the handoff once cloud-init wrote it"
+fi
+wait "${BGPID}" 2>/dev/null || true
+unset VLLM_ENV_WAIT
+
+# --- Case 7b: the file exists but is empty/partial, then gets completed a
+# moment later — deterministic reproduction (not dependent on catching a real
+# `cat >file <<EOF` race window) of the bug where `[ -f "${ENV_FILE}" ]`
+# alone cleared the wait's gate: cloud-init's write_files is not atomic from
+# this reader's point of view, so a file that exists with none of its keys
+# yet must be retried, not treated as a malformed handoff.
+reset_state
+add_disk sdf
+: > "${ENV_FILE}"
+( sleep 1; write_env "${DEV_DIR}/sdf" ) &
+BGPID=$!
+if VLLM_ENV_WAIT=5 run_ok "env-partial-write-midwait"; then
+    pass "env-partial-write-midwait: retried past an empty handoff file until it was fully written"
+fi
+wait "${BGPID}" 2>/dev/null || true
+
+# --- Case 8: env file never appears ---
+# Timed for the same reason as the device-tier "nothing-appears" case: an
+# exit-status-only check cannot tell a real ENV_WAIT_SECS wait from an
+# instant death.
+reset_state
+_t0=$(date +%s)
+VLLM_ENV_WAIT=3 run_fails "env-never-appears"
+_t1=$(date +%s)
+_elapsed=$((_t1 - _t0))
+[ "${_elapsed}" -ge 3 ] \
+    && pass "env-never-appears: waited the full ${_elapsed}s bound before giving up" \
+    || fail "env-never-appears: returned after only ${_elapsed}s, expected >= 3s (instant death, not a real wait)"
+
+# --- Case 9: already-mounted weights point is a no-op restart ---
+reset_state
+add_disk sdf
+write_env "${DEV_DIR}/sdf"
+printf '%s %s ext4 ro 0 0\n' "${DEV_DIR}/sdf" "${WEIGHTS_MOUNT}" >> "${MOUNTS_FILE}"
+if run_ok "already-mounted"; then
+    grep -q . "${MOUNT_CALLS}" \
+        && fail "already-mounted: mounted over an existing mount" || pass "already-mounted: no-op, straight to exec"
+    grep -q "vllm serve" "${VLLM_CALLS}" \
+        && pass "already-mounted: still execs vllm" || fail "already-mounted: vllm never invoked"
+fi
+
+# --- Case 10: VLLM_ARGS passes each flag through as its own argument ---
+reset_state
+add_disk sdf
+write_env "${DEV_DIR}/sdf" "--dtype=bfloat16 --max-model-len=4096"
+if run_ok "extra-args"; then
+    grep -q -- "--dtype=bfloat16 --max-model-len=4096" "${VLLM_CALLS}" \
+        && pass "extra-args: VLLM_ARGS flags reached vllm serve" || fail "extra-args: flags missing"
+fi
+
+# --- Case 11: a non-numeric port is refused rather than passed through ---
+# A malformed-but-present value must fail fast rather than retry — timed here
+# against a generous ENV_WAIT to positively distinguish it from the "keys not
+# there yet" retry path above (case 7b), the same distinction
+# resolve_weights_device's ambiguous-tier cases make for the device wait.
+reset_state
+add_disk sdf
+cat > "${ENV_FILE}" <<'EOF'
+VLLM_MODEL_ID=test-model
+VLLM_ARGS=
+VLLM_WEIGHTS_DEVICE=/dev/sdf
+VLLM_SERVE_PORT=not-a-number
+EOF
+_t0=$(date +%s)
+VLLM_ENV_WAIT=30 run_fails "bad-port"
+_t1=$(date +%s)
+_elapsed=$((_t1 - _t0))
+[ "${_elapsed}" -lt 15 ] \
+    && pass "bad-port: failed fast in ${_elapsed}s rather than retrying an unfixable value" \
+    || fail "bad-port: took ${_elapsed}s, expected a fast refusal instead of waiting out the 30s bound"
+grep -q . "${VLLM_CALLS}" \
+    && fail "bad-port: vllm invoked with a bad port" || pass "bad-port: refused before exec"
+
+if [ "${FAILS}" -eq 0 ]; then
+    echo "PASS: all vllm-serve cases"
+    exit 0
+fi
+echo "FAILED: ${FAILS} case(s)"
+exit 1

internal/awsmodel/coverage.go

@@ -0,0 +1,187 @@
+package awsmodel
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// DispatchInventory is the gateway-side operation registration state supplied
+// to CompareOperations.
+type DispatchInventory struct {
+	Registered  []string
+	Stubbed     []string
+	Unsupported []string
+	Opaque      bool
+	Note        string
+}
+
+// OperationCoverage is a deterministic model-versus-dispatch comparison.
+// Implemented contains only modelled operations with a real handler; Extra
+// contains registered operations absent from the pinned model.
+type OperationCoverage struct {
+	Service     Service
+	APIVersion  string
+	Modelled    []string
+	Registered  []string
+	Implemented []string
+	Stubbed     []string
+	Unsupported []string
+	Missing     []string
+	Extra       []string
+	Opaque      bool
+	Note        string
+}
+
+// CompareOperations compares one embedded service model with an authoritative
+// gateway dispatch inventory.
+func CompareOperations(service Service, dispatch DispatchInventory) (OperationCoverage, error) {
+	model, err := Load(service)
+	if err != nil {
+		return OperationCoverage{}, err
+	}
+	coverage := OperationCoverage{
+		Service:    service,
+		APIVersion: model.Metadata().APIVersion,
+		Modelled:   model.Operations(),
+		Opaque:     dispatch.Opaque,
+		Note:       dispatch.Note,
+	}
+	if dispatch.Opaque {
+		return coverage, nil
+	}
+
+	registered, err := uniqueSorted("registered", dispatch.Registered)
+	if err != nil {
+		return OperationCoverage{}, fmt.Errorf("awsmodel: %s dispatch: %w", service, err)
+	}
+	stubbed, err := uniqueSorted("stubbed", dispatch.Stubbed)
+	if err != nil {
+		return OperationCoverage{}, fmt.Errorf("awsmodel: %s dispatch: %w", service, err)
+	}
+	unsupported, err := uniqueSorted("unsupported", dispatch.Unsupported)
+	if err != nil {
+		return OperationCoverage{}, fmt.Errorf("awsmodel: %s dispatch: %w", service, err)
+	}
+	registeredSet := toSet(registered)
+	stubbedSet := toSet(stubbed)
+	unsupportedSet := toSet(unsupported)
+	for operation := range stubbedSet {
+		if !registeredSet[operation] {
+			return OperationCoverage{}, fmt.Errorf("awsmodel: %s stubbed operation %q is not registered", service, operation)
+		}
+		if unsupportedSet[operation] {
+			return OperationCoverage{}, fmt.Errorf("awsmodel: %s operation %q is both stubbed and unsupported", service, operation)
+		}
+	}
+	for operation := range unsupportedSet {
+		if !registeredSet[operation] {
+			return OperationCoverage{}, fmt.Errorf("awsmodel: %s unsupported operation %q is not registered", service, operation)
+		}
+	}
+
+	modelledSet := toSet(coverage.Modelled)
+	for _, operation := range coverage.Modelled {
+		if !registeredSet[operation] {
+			coverage.Missing = append(coverage.Missing, operation)
+			continue
+		}
+		if !stubbedSet[operation] && !unsupportedSet[operation] {
+			coverage.Implemented = append(coverage.Implemented, operation)
+		}
+	}
+	for _, operation := range registered {
+		if !modelledSet[operation] {
+			coverage.Extra = append(coverage.Extra, operation)
+		}
+	}
+	coverage.Registered = registered
+	coverage.Stubbed = stubbed
+	coverage.Unsupported = unsupported
+	return coverage, nil
+}
+
+func uniqueSorted(label string, values []string) ([]string, error) {
+	result := append([]string(nil), values...)
+	sort.Strings(result)
+	for index, value := range result {
+		if value == "" {
+			return nil, fmt.Errorf("%s operation is empty", label)
+		}
+		if index > 0 && result[index-1] == value {
+			return nil, fmt.Errorf("%s operation %q is duplicated", label, value)
+		}
+	}
+	return result, nil
+}
+
+func toSet(values []string) map[string]bool {
+	set := make(map[string]bool, len(values))
+	for _, value := range values {
+		set[value] = true
+	}
+	return set
+}
+
+// RenderCoverageMarkdown renders a reproducible divergence report suitable for
+// publishing directly or checking as a generated artefact.
+func RenderCoverageMarkdown(coverages []OperationCoverage) string {
+	ordered := append([]OperationCoverage(nil), coverages...)
+	sort.Slice(ordered, func(i, j int) bool { return ordered[i].Service < ordered[j].Service })
+
+	var report strings.Builder
+	fmt.Fprintf(&report, "# AWS model operation coverage\n\n")
+	fmt.Fprintf(&report, "Generated from the cached `aws-sdk-go %s` `api-2.json` models and Spinifex's authoritative gateway dispatch tables.\n\n", SourceSDKVersion)
+	report.WriteString("Implemented means a modelled operation is registered to a real handler. Stub and deliberately unsupported handlers are reported separately. Shape conformance does not imply behavioural conformance.\n\n")
+	report.WriteString("| Service | API version | Modelled | Registered | Implemented | Stubbed | Unsupported | Missing | Extra |\n")
+	report.WriteString("|---|---:|---:|---:|---:|---:|---:|---:|---:|\n")
+	for _, coverage := range ordered {
+		if coverage.Opaque {
+			fmt.Fprintf(&report, "| %s | %s | %d | — | — | — | — | — | — |\n",
+				coverage.Service, coverage.APIVersion, len(coverage.Modelled))
+			continue
+		}
+		fmt.Fprintf(&report, "| %s | %s | %d | %d | %d | %d | %d | %d | %d |\n",
+			coverage.Service, coverage.APIVersion, len(coverage.Modelled), len(coverage.Registered),
+			len(coverage.Implemented), len(coverage.Stubbed), len(coverage.Unsupported), len(coverage.Missing), len(coverage.Extra))
+	}
+
+	for _, coverage := range ordered {
+		fmt.Fprintf(&report, "\n## %s\n\n", coverage.Service)
+		if coverage.Opaque {
+			fmt.Fprintf(&report, "Operation coverage is not enumerable: %s\n", coverage.Note)
+			continue
+		}
+		percentage := float64(0)
+		if len(coverage.Modelled) != 0 {
+			percentage = 100 * float64(len(coverage.Implemented)) / float64(len(coverage.Modelled))
+		}
+		fmt.Fprintf(&report, "Implements **%d of %d** modelled operations (%.1f%%).\n", len(coverage.Implemented), len(coverage.Modelled), percentage)
+		writeOperationList(&report, "Implemented", coverage.Implemented)
+		writeOperationList(&report, "Missing from dispatch", coverage.Missing)
+		writeOperationList(&report, "Registered stubs", coverage.Stubbed)
+		writeOperationList(&report, "Deliberately unsupported", coverage.Unsupported)
+		writeOperationList(&report, "Registered outside the pinned model", coverage.Extra)
+		if coverage.Note != "" {
+			fmt.Fprintf(&report, "\nNote: %s\n", coverage.Note)
+		}
+	}
+	return report.String()
+}
+
+func writeOperationList(report *strings.Builder, title string, operations []string) {
+	fmt.Fprintf(report, "\n<details><summary>%s (%d)</summary>\n\n", title, len(operations))
+	if len(operations) == 0 {
+		report.WriteString("None.\n")
+	} else {
+		for start := 0; start < len(operations); start += 8 {
+			end := min(start+8, len(operations))
+			quoted := make([]string, 0, end-start)
+			for _, operation := range operations[start:end] {
+				quoted = append(quoted, "`"+operation+"`")
+			}
+			report.WriteString(strings.Join(quoted, ", ") + "\n\n")
+		}
+	}
+	report.WriteString("\n</details>\n")
+}

internal/awsmodel/coverage_test.go

@@ -0,0 +1,51 @@
+package awsmodel
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestCompareOperations(t *testing.T) {
+	coverage, err := CompareOperations(STS, DispatchInventory{
+		Registered:  []string{"AssumeRole", "ExtraOperation", "GetCallerIdentity", "GetSessionToken"},
+		Stubbed:     []string{"GetSessionToken"},
+		Unsupported: []string{"GetCallerIdentity"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(coverage.Implemented) != 1 || coverage.Implemented[0] != "AssumeRole" {
+		t.Fatalf("implemented = %v, want [AssumeRole]", coverage.Implemented)
+	}
+	if len(coverage.Missing) != 5 {
+		t.Fatalf("missing count = %d, want 5: %v", len(coverage.Missing), coverage.Missing)
+	}
+	if len(coverage.Extra) != 1 || coverage.Extra[0] != "ExtraOperation" {
+		t.Fatalf("extra = %v, want [ExtraOperation]", coverage.Extra)
+	}
+}
+
+func TestCompareOperationsRejectsInvalidInventory(t *testing.T) {
+	_, err := CompareOperations(STS, DispatchInventory{Registered: []string{"AssumeRole"}, Stubbed: []string{"Missing"}})
+	if err == nil || !strings.Contains(err.Error(), `stubbed operation "Missing" is not registered`) {
+		t.Fatalf("error = %v", err)
+	}
+}
+
+func TestRenderCoverageMarkdown(t *testing.T) {
+	coverage, err := CompareOperations(STS, DispatchInventory{Registered: []string{"AssumeRole"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	report := RenderCoverageMarkdown([]OperationCoverage{coverage})
+	for _, want := range []string{
+		"# AWS model operation coverage",
+		"| sts | 2011-06-15 | 8 | 1 | 1 | 0 | 0 | 7 | 0 |",
+		"Implements **1 of 8** modelled operations (12.5%).",
+		"<summary>Missing from dispatch (7)</summary>",
+	} {
+		if !strings.Contains(report, want) {
+			t.Errorf("report does not contain %q:\n%s", want, report)
+		}
+	}
+}

internal/awsmodel/ec2error.go

@@ -0,0 +1,153 @@
+package awsmodel
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"slices"
+	"strings"
+	"sync"
+)
+
+const ec2ErrorCatalogPath = "models/ec2/error-codes.json"
+
+type ec2ErrorClass string
+
+const (
+	ec2ErrorClassClient ec2ErrorClass = "client"
+	ec2ErrorClassServer ec2ErrorClass = "server"
+)
+
+type ec2ErrorCatalogFile struct {
+	Source         string   `json:"source"`
+	VerifiedOn     string   `json:"verifiedOn"`
+	CommonClient   []string `json:"commonClient"`
+	SpecificClient []string `json:"specificClient"`
+	Server         []string `json:"server"`
+}
+
+type ec2ErrorCatalog struct {
+	metadata ec2ErrorCatalogFile
+	codes    map[string]ec2ErrorClass
+}
+
+var loadEC2ErrorCatalog = sync.OnceValues(func() (*ec2ErrorCatalog, error) {
+	contents, err := modelFS.ReadFile(ec2ErrorCatalogPath)
+	if err != nil {
+		return nil, fmt.Errorf("awsmodel: read EC2 error catalog: %w", err)
+	}
+
+	decoder := json.NewDecoder(bytes.NewReader(contents))
+	decoder.DisallowUnknownFields()
+	var file ec2ErrorCatalogFile
+	if err := decoder.Decode(&file); err != nil {
+		return nil, fmt.Errorf("awsmodel: parse EC2 error catalog: %w", err)
+	}
+	if err := decoder.Decode(new(any)); !errors.Is(err, io.EOF) {
+		if err == nil {
+			err = errors.New("multiple JSON values")
+		}
+		return nil, fmt.Errorf("awsmodel: parse EC2 error catalog: %w", err)
+	}
+	if file.Source == "" || file.VerifiedOn == "" {
+		return nil, errors.New("awsmodel: EC2 error catalog requires source and verifiedOn")
+	}
+
+	catalog := &ec2ErrorCatalog{metadata: file, codes: make(map[string]ec2ErrorClass)}
+	add := func(label string, values []string, class ec2ErrorClass) error {
+		if !slices.IsSorted(values) {
+			return fmt.Errorf("awsmodel: EC2 error catalog %s codes are not sorted", label)
+		}
+		for _, code := range values {
+			if code == "" {
+				return fmt.Errorf("awsmodel: EC2 error catalog %s contains an empty code", label)
+			}
+			if previous, exists := catalog.codes[code]; exists {
+				return fmt.Errorf("awsmodel: EC2 error catalog code %q is duplicated (%s and %s)", code, previous, label)
+			}
+			catalog.codes[code] = class
+		}
+		return nil
+	}
+	if err := add("commonClient", file.CommonClient, ec2ErrorClassClient); err != nil {
+		return nil, err
+	}
+	if err := add("specificClient", file.SpecificClient, ec2ErrorClassClient); err != nil {
+		return nil, err
+	}
+	if err := add("server", file.Server, ec2ErrorClassServer); err != nil {
+		return nil, err
+	}
+	return catalog, nil
+})
+
+// ValidateEC2ErrorResponse validates an EC2 Query API error envelope against
+// the checked-in catalog curated from AWS's EC2 error-code reference. The AWS
+// api-2.json model declares no operation errors, so this is intentionally a
+// separate oracle.
+func ValidateEC2ErrorResponse(status int, body []byte) ([]Violation, error) {
+	root, err := parseXML(body)
+	if err != nil {
+		return nil, fmt.Errorf("awsmodel: decode EC2 error response: %w", err)
+	}
+	if root.name != "Response" {
+		return nil, fmt.Errorf("awsmodel: decode EC2 error response: root element is %q, want %q", root.name, "Response")
+	}
+	errorsNode := directXMLChild(root, "Errors")
+	if errorsNode == nil {
+		return nil, errors.New("awsmodel: decode EC2 error response: Errors element is missing")
+	}
+	errorNode := directXMLChild(errorsNode, "Error")
+	if errorNode == nil {
+		return nil, errors.New("awsmodel: decode EC2 error response: Error element is missing")
+	}
+	code := directXMLText(errorNode, "Code")
+	if code == "" {
+		return nil, errors.New("awsmodel: decode EC2 error response: Code element is missing")
+	}
+
+	catalog, err := loadEC2ErrorCatalog()
+	if err != nil {
+		return nil, err
+	}
+	class, known := catalog.codes[code]
+	if !known {
+		return []Violation{{
+			Rule:    RuleErrorCode,
+			Path:    "$error.Code",
+			Message: fmt.Sprintf("error code %q is not in the curated EC2 catalog", code),
+		}}, nil
+	}
+
+	wantStatusClass := 4
+	if class == ec2ErrorClassServer {
+		wantStatusClass = 5
+	}
+	if status/100 != wantStatusClass {
+		return []Violation{{
+			Rule:    RuleHTTPStatus,
+			Path:    "$response.status",
+			Message: fmt.Sprintf("status %d is not in the %dxx class required for EC2 %s error %q", status, wantStatusClass, class, code),
+		}}, nil
+	}
+	return nil, nil
+}
+
+func directXMLChild(node *xmlNode, name string) *xmlNode {
+	for _, child := range node.children {
+		if child.name == name {
+			return child
+		}
+	}
+	return nil
+}
+
+func directXMLText(node *xmlNode, name string) string {
+	child := directXMLChild(node, name)
+	if child == nil {
+		return ""
+	}
+	return strings.TrimSpace(child.text.String())
+}

internal/awsmodel/ec2error_test.go

@@ -0,0 +1,93 @@
+package awsmodel
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestValidateEC2ErrorResponseAllowsCuratedCodes(t *testing.T) {
+	tests := []struct {
+		name   string
+		status int
+		code   string
+	}{
+		{name: "common client", status: 403, code: "UnauthorizedOperation"},
+		{name: "specific client", status: 404, code: "InvalidInstanceID.NotFound"},
+		{name: "server", status: 500, code: "InternalError"},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			violations, err := ValidateEC2ErrorResponse(test.status, ec2ErrorBody(test.code))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(violations) != 0 {
+				t.Fatalf("violations = %#v, want none", violations)
+			}
+		})
+	}
+}
+
+func TestValidateEC2ErrorResponseRejectsUnverifiedCode(t *testing.T) {
+	violations, err := ValidateEC2ErrorResponse(403, ec2ErrorBody("ExpiredToken"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []Violation{{
+		Rule:    RuleErrorCode,
+		Path:    "$error.Code",
+		Message: `error code "ExpiredToken" is not in the curated EC2 catalog`,
+	}}
+	if !reflect.DeepEqual(violations, want) {
+		t.Fatalf("violations = %#v, want %#v", violations, want)
+	}
+}
+
+func TestValidateEC2ErrorResponseRejectsWrongStatusClass(t *testing.T) {
+	violations, err := ValidateEC2ErrorResponse(400, ec2ErrorBody("InsufficientInstanceCapacity"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []Violation{{
+		Rule:    RuleHTTPStatus,
+		Path:    "$response.status",
+		Message: `status 400 is not in the 5xx class required for EC2 server error "InsufficientInstanceCapacity"`,
+	}}
+	if !reflect.DeepEqual(violations, want) {
+		t.Fatalf("violations = %#v, want %#v", violations, want)
+	}
+}
+
+func TestValidateEC2ErrorResponseRejectsMalformedEnvelope(t *testing.T) {
+	_, err := ValidateEC2ErrorResponse(400, []byte(`<ErrorResponse><Error><Code>InvalidAction</Code></Error></ErrorResponse>`))
+	if err == nil || !strings.Contains(err.Error(), `root element is "ErrorResponse", want "Response"`) {
+		t.Fatalf("error = %v", err)
+	}
+}
+
+func TestEC2ErrorCatalogMetadataAndClasses(t *testing.T) {
+	catalog, err := loadEC2ErrorCatalog()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if catalog.metadata.Source != "https://docs.aws.amazon.com/ec2/latest/devguide/errors-overview.html" {
+		t.Fatalf("source = %q", catalog.metadata.Source)
+	}
+	if catalog.metadata.VerifiedOn == "" {
+		t.Fatal("verifiedOn is empty")
+	}
+	if len(catalog.codes) != 112 {
+		t.Fatalf("catalog code count = %d, want 112", len(catalog.codes))
+	}
+	if catalog.codes["InvalidParameterValue"] != ec2ErrorClassClient {
+		t.Fatal("InvalidParameterValue is not classified as client")
+	}
+	if catalog.codes["ServerInternal"] != ec2ErrorClassServer {
+		t.Fatal("ServerInternal is not classified as server")
+	}
+}
+
+func ec2ErrorBody(code string) []byte {
+	return []byte(`<Response><Errors><Error><Code>` + code + `</Code><Message>test</Message></Error></Errors><RequestID>request-1</RequestID></Response>`)
+}

internal/awsmodel/embed.go

@@ -0,0 +1,22 @@
+package awsmodel
+
+import "embed"
+
+// modelFS contains only the curated EC2 error catalog. The large upstream
+// api-2.json files are resolved from the pinned aws-sdk-go module cache at
+// runtime; they are intentionally not source-controlled here.
+//
+//go:embed models/ec2/error-codes.json
+var modelFS embed.FS
+
+var modelFiles = map[Service]string{
+	ACM:                    "models/apis/acm/2015-12-08/api-2.json",
+	EC2:                    "models/apis/ec2/2016-11-15/api-2.json",
+	ECR:                    "models/apis/ecr/2015-09-21/api-2.json",
+	ECS:                    "models/apis/ecs/2014-11-13/api-2.json",
+	ElasticLoadBalancingV2: "models/apis/elasticloadbalancingv2/2015-12-01/api-2.json",
+	IAM:                    "models/apis/iam/2010-05-08/api-2.json",
+	RDS:                    "models/apis/rds/2014-10-31/api-2.json",
+	S3:                     "models/apis/s3/2006-03-01/api-2.json",
+	STS:                    "models/apis/sts/2011-06-15/api-2.json",
+}

internal/awsmodel/error.go

@@ -0,0 +1,142 @@
+package awsmodel
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"slices"
+	"sort"
+	"strings"
+)
+
+// ValidateErrorResponse decodes an AWS error envelope and checks its code
+// against the errors declared for operation. Modelled is false when AWS's
+// operation definition declares no errors, as is the case for some STS calls.
+func ValidateErrorResponse(service Service, operationName string, body []byte) (violations []Violation, modelled bool, err error) {
+	model, err := Load(service)
+	if err != nil {
+		return nil, false, err
+	}
+	operation, ok := model.Operation(operationName)
+	if !ok {
+		return nil, false, fmt.Errorf("awsmodel: %s operation %q is not modelled", service, operationName)
+	}
+
+	code, err := model.decodeErrorCode(body)
+	if err != nil {
+		return nil, false, fmt.Errorf("awsmodel: decode %s %s error response: %w", service, operationName, err)
+	}
+	if isUnmodelledCommonError(service, code) {
+		return nil, false, nil
+	}
+	codes := model.operationErrorCodes(operation)
+	if len(codes) == 0 {
+		return nil, false, nil
+	}
+	if slices.Contains(codes, code) {
+		return nil, true, nil
+	}
+	return []Violation{{
+		Rule:    RuleErrorCode,
+		Path:    "$error.Code",
+		Message: fmt.Sprintf("error code %q is not declared; allowed [%s]", code, strings.Join(codes, ", ")),
+	}}, true, nil
+}
+
+// isUnmodelledCommonError identifies service/protocol errors AWS does not put
+// in operation error lists. They remain counted as unmodelled by the harness;
+// they are not treated as proof that an operation-specific code conforms.
+func isUnmodelledCommonError(service Service, code string) bool {
+	switch service {
+	case IAM, ElasticLoadBalancingV2:
+		return isUnmodelledQueryError(code)
+	case STS:
+		// STS also uses these two runtime input errors even though api-2.json
+		// does not declare them on AssumeRole/GetSessionToken.
+		return isUnmodelledQueryError(code) || code == "ValidationError" || code == "InvalidParameterValue"
+	case ECS:
+		switch code {
+		case "AccessDeniedException", "IncompleteSignatureException", "InvalidClientTokenIdException",
+			"MissingAuthenticationTokenException", "RequestExpiredException", "SignatureDoesNotMatchException",
+			"ThrottlingException":
+			return true
+		}
+	default:
+	}
+	return false
+}
+
+func isUnmodelledQueryError(code string) bool {
+	switch code {
+	case "AccessDenied", "IncompleteSignature", "InvalidClientTokenId", "MissingAuthenticationToken",
+		"RequestExpired", "SignatureDoesNotMatch", "Throttling":
+		return true
+	default:
+		return false
+	}
+}
+
+func (m *Model) operationErrorCodes(operation *Operation) []string {
+	codes := make([]string, 0, len(operation.Errors))
+	for _, ref := range operation.Errors {
+		code := ref.Shape
+		if shape := m.shapes[ref.Shape]; shape != nil && shape.Error != nil && shape.Error.Code != "" {
+			code = shape.Error.Code
+		}
+		codes = append(codes, code)
+	}
+	sort.Strings(codes)
+	return codes
+}
+
+func (m *Model) decodeErrorCode(body []byte) (string, error) {
+	switch m.metadata.Protocol {
+	case "json", "rest-json":
+		var envelope struct {
+			Type string `json:"__type"`
+			Code string `json:"code"`
+		}
+		decoder := json.NewDecoder(bytes.NewReader(body))
+		if err := decoder.Decode(&envelope); err != nil {
+			return "", err
+		}
+		code := envelope.Type
+		if code == "" {
+			code = envelope.Code
+		}
+		if separator := strings.LastIndex(code, "#"); separator >= 0 {
+			code = code[separator+1:]
+		}
+		if separator := strings.Index(code, ":"); separator >= 0 {
+			code = code[:separator]
+		}
+		if code == "" {
+			return "", fmt.Errorf("JSON error envelope has no code")
+		}
+		return code, nil
+	case "ec2", "query":
+		root, err := parseXML(body)
+		if err != nil {
+			return "", err
+		}
+		code := findXMLText(root, "Code")
+		if code == "" {
+			return "", fmt.Errorf("XML error envelope has no Code element")
+		}
+		return code, nil
+	default:
+		return "", fmt.Errorf("error decoding for protocol %q is not implemented", m.metadata.Protocol)
+	}
+}
+
+func findXMLText(node *xmlNode, name string) string {
+	if node.name == name {
+		return strings.TrimSpace(node.text.String())
+	}
+	for _, child := range node.children {
+		if value := findXMLText(child, name); value != "" {
+			return value
+		}
+	}
+	return ""
+}

internal/awsmodel/error_test.go

@@ -0,0 +1,119 @@
+package awsmodel
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestValidateErrorResponseQueryAllowsDeclaredCode(t *testing.T) {
+	body := []byte(`<ErrorResponse><Error><Code>NoSuchEntity</Code><Message>missing</Message></Error></ErrorResponse>`)
+	violations, modelled, err := ValidateErrorResponse(IAM, "GetUser", body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !modelled {
+		t.Fatal("GetUser errors should be modelled")
+	}
+	if len(violations) != 0 {
+		t.Fatalf("violations = %#v, want none", violations)
+	}
+}
+
+func TestValidateErrorResponseRejectsUndeclaredCode(t *testing.T) {
+	body := []byte(`<ErrorResponse><Error><Code>InventedFailure</Code></Error></ErrorResponse>`)
+	violations, modelled, err := ValidateErrorResponse(IAM, "GetUser", body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !modelled {
+		t.Fatal("GetUser errors should be modelled")
+	}
+	want := []Violation{{
+		Rule:    RuleErrorCode,
+		Path:    "$error.Code",
+		Message: `error code "InventedFailure" is not declared; allowed [NoSuchEntity, ServiceFailure]`,
+	}}
+	if !reflect.DeepEqual(violations, want) {
+		t.Fatalf("violations = %#v, want %#v", violations, want)
+	}
+}
+
+func TestValidateErrorResponseJSONAllowsDeclaredShapeName(t *testing.T) {
+	body := []byte(`{"__type":"com.amazonaws.ecs#ClusterNotFoundException","message":"missing"}`)
+	violations, modelled, err := ValidateErrorResponse(ECS, "DescribeTasks", body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !modelled || len(violations) != 0 {
+		t.Fatalf("modelled = %t, violations = %#v, want modelled and conforming", modelled, violations)
+	}
+}
+
+func TestValidateErrorResponseELBAllowsDeclaredCode(t *testing.T) {
+	body := []byte(`<ErrorResponse><Error><Code>LoadBalancerNotFound</Code></Error></ErrorResponse>`)
+	violations, modelled, err := ValidateErrorResponse(ElasticLoadBalancingV2, "DescribeLoadBalancers", body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !modelled || len(violations) != 0 {
+		t.Fatalf("modelled = %t, violations = %#v, want modelled and conforming", modelled, violations)
+	}
+}
+
+func TestValidateErrorResponseRejectsMalformedEnvelope(t *testing.T) {
+	_, _, err := ValidateErrorResponse(ECS, "DescribeTasks", []byte(`{"message":"missing code"}`))
+	if err == nil {
+		t.Fatal("expected missing error code to fail decoding")
+	}
+}
+
+func TestValidateErrorResponseOperationWithoutDeclaredErrors(t *testing.T) {
+	body := []byte(`<ErrorResponse><Error><Code>AccessDenied</Code></Error></ErrorResponse>`)
+	violations, modelled, err := ValidateErrorResponse(STS, "GetCallerIdentity", body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if modelled || len(violations) != 0 {
+		t.Fatalf("modelled = %t, violations = %#v, want unmodelled and no violations", modelled, violations)
+	}
+}
+
+func TestValidateErrorResponseClassifiesCommonErrorsAsUnmodelled(t *testing.T) {
+	tests := []struct {
+		name      string
+		service   Service
+		operation string
+		body      string
+	}{
+		{
+			name:      "query authorization",
+			service:   IAM,
+			operation: "CreateUser",
+			body:      `<ErrorResponse><Error><Code>AccessDenied</Code></Error></ErrorResponse>`,
+		},
+		{
+			name:      "STS runtime validation",
+			service:   STS,
+			operation: "AssumeRole",
+			body:      `<ErrorResponse><Error><Code>ValidationError</Code></Error></ErrorResponse>`,
+		},
+		{
+			name:      "JSON authorization",
+			service:   ECS,
+			operation: "DescribeTasks",
+			body:      `{"__type":"AccessDeniedException"}`,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			violations, modelled, err := ValidateErrorResponse(test.service, test.operation, []byte(test.body))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if modelled || len(violations) != 0 {
+				t.Fatalf("modelled = %t, violations = %#v, want unmodelled and no violations", modelled, violations)
+			}
+		})
+	}
+}

internal/awsmodel/model.go

@@ -0,0 +1,297 @@
+// Package awsmodel loads the AWS api-2.json service definitions used by the
+// conformance suite.
+package awsmodel
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"os/user"
+	"path/filepath"
+	"slices"
+	"sync"
+)
+
+// SourceSDKVersion is the aws-sdk-go release whose module-cache models are
+// used by the conformance suite.
+const SourceSDKVersion = "v1.55.8"
+
+const sourceSDKModule = "github.com/aws/aws-sdk-go"
+
+// Service identifies an AWS service model loaded from the SDK module cache.
+type Service string
+
+const (
+	ACM                    Service = "acm"
+	EC2                    Service = "ec2"
+	ECR                    Service = "ecr"
+	ECS                    Service = "ecs"
+	ElasticLoadBalancingV2 Service = "elasticloadbalancingv2"
+	IAM                    Service = "iam"
+	RDS                    Service = "rds"
+	S3                     Service = "s3"
+	STS                    Service = "sts"
+)
+
+// Metadata describes the service and wire protocol represented by a model.
+type Metadata struct {
+	APIVersion          string   `json:"apiVersion"`
+	EndpointPrefix      string   `json:"endpointPrefix"`
+	Protocol            string   `json:"protocol"`
+	Protocols           []string `json:"protocols"`
+	ServiceAbbreviation string   `json:"serviceAbbreviation"`
+	ServiceFullName     string   `json:"serviceFullName"`
+	ServiceID           string   `json:"serviceId"`
+	SignatureVersion    string   `json:"signatureVersion"`
+	UID                 string   `json:"uid"`
+}
+
+// Operation describes an AWS API operation and the shapes used by its input,
+// output and declared errors.
+type Operation struct {
+	Name   string     `json:"name"`
+	HTTP   HTTP       `json:"http"`
+	Input  *ShapeRef  `json:"input"`
+	Output *ShapeRef  `json:"output"`
+	Errors []ShapeRef `json:"errors"`
+}
+
+// HTTP describes an operation's HTTP binding.
+type HTTP struct {
+	Method       string `json:"method"`
+	RequestURI   string `json:"requestUri"`
+	ResponseCode int    `json:"responseCode"`
+}
+
+// Shape describes a value in an AWS service model. Depending on Type, it can
+// refer to structure members, a list member, or map keys and values.
+type Shape struct {
+	Type            string              `json:"type"`
+	Required        []string            `json:"required"`
+	Members         map[string]ShapeRef `json:"members"`
+	Member          *ShapeRef           `json:"member"`
+	Key             *ShapeRef           `json:"key"`
+	Value           *ShapeRef           `json:"value"`
+	Enum            []string            `json:"enum"`
+	Min             *float64            `json:"min"`
+	Max             *float64            `json:"max"`
+	Pattern         string              `json:"pattern"`
+	LocationName    string              `json:"locationName"`
+	TimestampFormat string              `json:"timestampFormat"`
+	Payload         string              `json:"payload"`
+	Flattened       bool                `json:"flattened"`
+	Sensitive       bool                `json:"sensitive"`
+	Exception       bool                `json:"exception"`
+	Fault           bool                `json:"fault"`
+	Error           *ErrorInfo          `json:"error"`
+}
+
+// ErrorInfo describes an error shape's code and HTTP classification.
+type ErrorInfo struct {
+	Code           string `json:"code"`
+	HTTPStatusCode int    `json:"httpStatusCode"`
+	SenderFault    bool   `json:"senderFault"`
+}
+
+// ShapeRef names another shape and carries any wire binding specific to the
+// place where it is referenced.
+type ShapeRef struct {
+	Shape           string `json:"shape"`
+	ResultWrapper   string `json:"resultWrapper"`
+	Location        string `json:"location"`
+	LocationName    string `json:"locationName"`
+	QueryName       string `json:"queryName"`
+	TimestampFormat string `json:"timestampFormat"`
+	Flattened       bool   `json:"flattened"`
+	Streaming       bool   `json:"streaming"`
+	XMLAttribute    bool   `json:"xmlAttribute"`
+}
+
+// Model is an indexed AWS service definition.
+type Model struct {
+	service    Service
+	metadata   Metadata
+	operations map[string]*Operation
+	shapes     map[string]*Shape
+}
+
+type modelDocument struct {
+	Version    string                `json:"version"`
+	Metadata   Metadata              `json:"metadata"`
+	Operations map[string]*Operation `json:"operations"`
+	Shapes     map[string]*Shape     `json:"shapes"`
+}
+
+type loadResult struct {
+	model *Model
+	err   error
+}
+
+var modelCache sync.Map
+
+var modelRoot = sync.OnceValues(resolveModelRoot)
+
+func resolveModelRoot() (string, error) {
+	if root := os.Getenv("GOMODCACHE"); root != "" {
+		return filepath.Join(root, sourceSDKModule+"@"+SourceSDKVersion), nil
+	}
+	if output, err := exec.Command("go", "env", "GOMODCACHE").Output(); err == nil {
+		if root := string(output); root != "" {
+			return filepath.Join(stringTrimSpace(root), sourceSDKModule+"@"+SourceSDKVersion), nil
+		}
+	}
+	currentUser, err := user.Current()
+	if err != nil {
+		return "", fmt.Errorf("awsmodel: resolve Go module cache: %w", err)
+	}
+	return filepath.Join(currentUser.HomeDir, "go", "pkg", "mod", sourceSDKModule+"@"+SourceSDKVersion), nil
+}
+
+func stringTrimSpace(value string) string {
+	for len(value) > 0 && (value[len(value)-1] == '\n' || value[len(value)-1] == '\r' || value[len(value)-1] == ' ' || value[len(value)-1] == '\t') {
+		value = value[:len(value)-1]
+	}
+	return value
+}
+
+// Services returns the supported service identifiers in stable order.
+func Services() []Service {
+	services := make([]Service, 0, len(modelFiles))
+	for service := range modelFiles {
+		services = append(services, service)
+	}
+	slices.Sort(services)
+	return services
+}
+
+// Load parses and indexes the cached SDK model for service. Each service is
+// parsed at most once and the resulting Model is safe for concurrent reads.
+func Load(service Service) (*Model, error) {
+	path, ok := modelFiles[service]
+	if !ok {
+		return nil, fmt.Errorf("awsmodel: unsupported service %q", service)
+	}
+
+	loader, _ := modelCache.LoadOrStore(service, sync.OnceValue(func() loadResult {
+		root, err := modelRoot()
+		if err != nil {
+			return loadResult{err: err}
+		}
+		contents, err := os.ReadFile(filepath.Join(root, path))
+		if err != nil {
+			return loadResult{err: fmt.Errorf("awsmodel: read cached %s model at %s: %w", service, filepath.Join(root, path), err)}
+		}
+
+		var document modelDocument
+		if err := json.Unmarshal(contents, &document); err != nil {
+			return loadResult{err: fmt.Errorf("awsmodel: parse %s model: %w", service, err)}
+		}
+		if document.Version != "2.0" {
+			return loadResult{err: fmt.Errorf("awsmodel: %s model version is %q, want %q", service, document.Version, "2.0")}
+		}
+		if len(document.Operations) == 0 || len(document.Shapes) == 0 {
+			return loadResult{err: fmt.Errorf("awsmodel: %s model has no operations or shapes", service)}
+		}
+
+		model := &Model{
+			service:    service,
+			metadata:   document.Metadata,
+			operations: document.Operations,
+			shapes:     document.Shapes,
+		}
+		if err := model.validateReferences(); err != nil {
+			return loadResult{err: err}
+		}
+		return loadResult{model: model}
+	}))
+	load, ok := loader.(func() loadResult)
+	if !ok {
+		return nil, fmt.Errorf("awsmodel: invalid cache entry for service %q", service)
+	}
+	result := load()
+	return result.model, result.err
+}
+
+// Service returns the identifier used to load the model.
+func (m *Model) Service() Service { return m.service }
+
+// Metadata returns the service metadata from api-2.json.
+func (m *Model) Metadata() Metadata { return m.metadata }
+
+// Operation resolves an operation by its model key, such as DescribeInstances.
+func (m *Model) Operation(name string) (*Operation, bool) {
+	operation, ok := m.operations[name]
+	return operation, ok
+}
+
+// Shape resolves a shape by name.
+func (m *Model) Shape(name string) (*Shape, bool) {
+	shape, ok := m.shapes[name]
+	return shape, ok
+}
+
+// Operations returns all operation keys in stable order.
+func (m *Model) Operations() []string { return sortedKeys(m.operations) }
+
+// Shapes returns all shape names in stable order.
+func (m *Model) Shapes() []string { return sortedKeys(m.shapes) }
+
+func sortedKeys[T any](values map[string]T) []string {
+	keys := make([]string, 0, len(values))
+	for key := range values {
+		keys = append(keys, key)
+	}
+	slices.Sort(keys)
+	return keys
+}
+
+func (m *Model) validateReferences() error {
+	validate := func(owner string, ref *ShapeRef) error {
+		if ref == nil {
+			return nil
+		}
+		if _, ok := m.shapes[ref.Shape]; !ok {
+			return fmt.Errorf("awsmodel: %s model %s references unknown shape %q", m.service, owner, ref.Shape)
+		}
+		return nil
+	}
+
+	for name, operation := range m.operations {
+		if operation == nil {
+			return fmt.Errorf("awsmodel: %s model operation %q is null", m.service, name)
+		}
+		if err := validate("operation "+name+" input", operation.Input); err != nil {
+			return err
+		}
+		if err := validate("operation "+name+" output", operation.Output); err != nil {
+			return err
+		}
+		for i := range operation.Errors {
+			if err := validate(fmt.Sprintf("operation %s error %d", name, i), &operation.Errors[i]); err != nil {
+				return err
+			}
+		}
+	}
+
+	for name, shape := range m.shapes {
+		if shape == nil {
+			return fmt.Errorf("awsmodel: %s model shape %q is null", m.service, name)
+		}
+		for memberName, member := range shape.Members {
+			if err := validate("shape "+name+" member "+memberName, &member); err != nil {
+				return err
+			}
+		}
+		for label, ref := range map[string]*ShapeRef{
+			"member": shape.Member,
+			"key":    shape.Key,
+			"value":  shape.Value,
+		} {
+			if err := validate("shape "+name+" "+label, ref); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

internal/awsmodel/model_test.go

@@ -0,0 +1,139 @@
+package awsmodel
+
+import (
+	"reflect"
+	"strings"
+	"sync"
+	"testing"
+)
+
+func TestServices(t *testing.T) {
+	want := []Service{ACM, EC2, ECR, ECS, ElasticLoadBalancingV2, IAM, RDS, S3, STS}
+	if got := Services(); !reflect.DeepEqual(got, want) {
+		t.Fatalf("Services() = %v, want %v", got, want)
+	}
+}
+
+func TestLoadAllModels(t *testing.T) {
+	wantSizes := map[Service]struct {
+		apiVersion string
+		operations int
+		shapes     int
+	}{
+		ACM:                    {"2015-12-08", 15, 102},
+		EC2:                    {"2016-11-15", 625, 3198},
+		ECR:                    {"2015-09-21", 47, 330},
+		ECS:                    {"2014-11-13", 56, 400},
+		ElasticLoadBalancingV2: {"2015-12-01", 46, 357},
+		IAM:                    {"2010-05-08", 159, 497},
+		RDS:                    {"2014-10-31", 162, 745},
+		S3:                     {"2006-03-01", 99, 604},
+		STS:                    {"2011-06-15", 8, 76},
+	}
+
+	for service, want := range wantSizes {
+		t.Run(string(service), func(t *testing.T) {
+			model, err := Load(service)
+			if err != nil {
+				t.Fatalf("Load(%q): %v", service, err)
+			}
+			if model.Service() != service {
+				t.Errorf("Service() = %q, want %q", model.Service(), service)
+			}
+			if model.Metadata().APIVersion != want.apiVersion || model.Metadata().Protocol == "" {
+				t.Errorf("metadata is incomplete: %+v", model.Metadata())
+			}
+			if got := len(model.Operations()); got != want.operations {
+				t.Errorf("operation count = %d, want %d", got, want.operations)
+			}
+			if got := len(model.Shapes()); got != want.shapes {
+				t.Errorf("shape count = %d, want %d", got, want.shapes)
+			}
+		})
+	}
+}
+
+func TestLoadEC2IndexesOperationsAndShapes(t *testing.T) {
+	model, err := Load(EC2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	operation, ok := model.Operation("DescribeInstances")
+	if !ok {
+		t.Fatal("DescribeInstances operation was not indexed")
+	}
+	if operation.Output == nil || operation.Output.Shape != "DescribeInstancesResult" {
+		t.Fatalf("DescribeInstances output = %+v, want DescribeInstancesResult", operation.Output)
+	}
+
+	shape, ok := model.Shape(operation.Output.Shape)
+	if !ok {
+		t.Fatalf("output shape %q was not indexed", operation.Output.Shape)
+	}
+	if reservationSet, ok := shape.Members["Reservations"]; !ok || reservationSet.Shape != "ReservationList" {
+		t.Errorf("Reservations member = %+v, want ReservationList", reservationSet)
+	}
+}
+
+func TestLoadPreservesWrappersAndErrorMetadata(t *testing.T) {
+	model, err := Load(IAM)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	getUser, ok := model.Operation("GetUser")
+	if !ok || getUser.Output == nil || getUser.Output.ResultWrapper != "GetUserResult" {
+		t.Fatalf("GetUser output = %+v, want GetUserResult wrapper", getUser)
+	}
+
+	createUser, ok := model.Operation("CreateUser")
+	if !ok || len(createUser.Errors) == 0 {
+		t.Fatalf("CreateUser operation has no modelled errors: %+v", createUser)
+	}
+	errorShape, ok := model.Shape(createUser.Errors[0].Shape)
+	if !ok || errorShape.Error == nil || errorShape.Error.Code == "" || errorShape.Error.HTTPStatusCode == 0 {
+		t.Fatalf("CreateUser error metadata was not loaded: %+v", errorShape)
+	}
+}
+
+func TestLoadRejectsUnknownService(t *testing.T) {
+	_, err := Load("not-a-service")
+	if err == nil || !strings.Contains(err.Error(), `unsupported service "not-a-service"`) {
+		t.Fatalf("Load(unknown) error = %v", err)
+	}
+}
+
+func TestLoadCachesModelAcrossConcurrentCalls(t *testing.T) {
+	const callers = 16
+	models := make(chan *Model, callers)
+	errors := make(chan error, callers)
+
+	var wait sync.WaitGroup
+	for range callers {
+		wait.Go(func() {
+			model, err := Load(STS)
+			models <- model
+			errors <- err
+		})
+	}
+	wait.Wait()
+	close(models)
+	close(errors)
+
+	for err := range errors {
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	var first *Model
+	for model := range models {
+		if first == nil {
+			first = model
+			continue
+		}
+		if model != first {
+			t.Fatal("Load returned different cached model pointers")
+		}
+	}
+}

internal/awsmodel/models/README.md

@@ -0,0 +1,5 @@
+# AWS model cache
+
+The conformance suite reads `api-2.json` from the Go module cache for the pinned `github.com/aws/aws-sdk-go` version `v1.55.8`. Running Go tests or `go mod download` populates that cache; the large upstream model files are intentionally not tracked in this repository.
+
+`ec2/error-codes.json` is separate from the SDK models because EC2's `api-2.json` declares no operation errors. It is a curated subset of the official [EC2 error-code reference](https://docs.aws.amazon.com/ec2/latest/devguide/errors-overview.html): all documented common and server codes, plus the action-specific codes that Spinifex currently emits. The catalog records its verification date and must be reviewed against that source when it changes.

internal/awsmodel/models/ec2/error-codes.json

@@ -0,0 +1,122 @@
+{
+  "source": "https://docs.aws.amazon.com/ec2/latest/devguide/errors-overview.html",
+  "verifiedOn": "2026-08-05",
+  "commonClient": [
+    "AuthFailure",
+    "Blocked",
+    "DryRunOperation",
+    "IdempotentParameterMismatch",
+    "IncompleteSignature",
+    "InvalidAction",
+    "InvalidCharacter",
+    "InvalidClientTokenId",
+    "InvalidPaginationToken",
+    "InvalidParameter",
+    "InvalidParameterCombination",
+    "InvalidParameterDependency",
+    "InvalidParameterValue",
+    "InvalidQueryParameter",
+    "MalformedQueryString",
+    "MissingAction",
+    "MissingAuthenticationToken",
+    "MissingParameter",
+    "OptInRequired",
+    "PendingVerification",
+    "RequestExpired",
+    "TagPolicyViolation",
+    "UnauthorizedOperation",
+    "UnknownParameter",
+    "UnsupportedInstanceAttribute",
+    "UnsupportedOperation",
+    "UnsupportedProtocol",
+    "ValidationError"
+  ],
+  "specificClient": [
+    "CannotDelete",
+    "DependencyViolation",
+    "Gateway.NotAttached",
+    "IncorrectInstanceState",
+    "IncorrectState",
+    "InvalidAMIID.Malformed",
+    "InvalidAMIID.NotFound",
+    "InvalidAMIID.Unavailable",
+    "InvalidAMIName.Duplicate",
+    "InvalidAMIName.Malformed",
+    "InvalidAddress.Locked",
+    "InvalidAllocationID.NotFound",
+    "InvalidAssociationID.NotFound",
+    "InvalidAttachmentID.NotFound",
+    "InvalidAvailabilityZone",
+    "InvalidCapacityReservationId.Malformed",
+    "InvalidCapacityReservationId.NotFound",
+    "InvalidEgressOnlyInternetGatewayId.NotFound",
+    "InvalidGroup.Duplicate",
+    "InvalidGroup.NotFound",
+    "InvalidGroup.Reserved",
+    "InvalidInstanceAttributeValue",
+    "InvalidInstanceID.Malformed",
+    "InvalidInstanceID.NotFound",
+    "InvalidInstanceType",
+    "InvalidInternetGatewayID.NotFound",
+    "InvalidKey.Format",
+    "InvalidKeyPair.Duplicate",
+    "InvalidKeyPair.Format",
+    "InvalidKeyPair.NotFound",
+    "InvalidLaunchTemplateId.Malformed",
+    "InvalidLaunchTemplateId.NotFound",
+    "InvalidLaunchTemplateId.VersionNotFound",
+    "InvalidLaunchTemplateName.AlreadyExistsException",
+    "InvalidLaunchTemplateName.MalformedException",
+    "InvalidLaunchTemplateName.NotFoundException",
+    "InvalidNatGatewayID.NotFound",
+    "InvalidNetworkInterface.InUse",
+    "InvalidNetworkInterface.NotFound",
+    "InvalidNetworkInterfaceID.NotFound",
+    "InvalidPermission.Duplicate",
+    "InvalidPermission.NotFound",
+    "InvalidPlacementGroup.Duplicate",
+    "InvalidPlacementGroup.InUse",
+    "InvalidPlacementGroup.Unknown",
+    "InvalidRoute.NotFound",
+    "InvalidRouteTableID.NotFound",
+    "InvalidSecurityGroupRuleId.Malformed",
+    "InvalidSecurityGroupRuleId.NotFound",
+    "InvalidSnapshot.InUse",
+    "InvalidSnapshot.NotFound",
+    "InvalidSnapshotID.Malformed",
+    "InvalidSpotInstanceRequestID.NotFound",
+    "InvalidSubnet.Conflict",
+    "InvalidSubnet.Range",
+    "InvalidSubnetID.NotFound",
+    "InvalidVolume.NotFound",
+    "InvalidVolumeID.Malformed",
+    "InvalidVpcID.NotFound",
+    "InvalidVpcRange",
+    "OperationNotPermitted",
+    "ReservationCapacityExceeded",
+    "Resource.AlreadyAssociated",
+    "ResourceLimitExceeded",
+    "RouteAlreadyExists",
+    "RulesPerSecurityGroupLimitExceeded",
+    "SecurityGroupsPerInterfaceLimitExceeded",
+    "SignatureDoesNotMatch",
+    "UnknownVolumeType",
+    "Unsupported",
+    "VolumeInUse"
+  ],
+  "server": [
+    "BandwidthLimitExceeded",
+    "InsufficientAddressCapacity",
+    "InsufficientCapacity",
+    "InsufficientHostCapacity",
+    "InsufficientInstanceCapacity",
+    "InsufficientReservedInstanceCapacity",
+    "InsufficientVolumeCapacity",
+    "InternalError",
+    "InternalFailure",
+    "RequestLimitExceeded",
+    "ServerInternal",
+    "ServiceUnavailable",
+    "Unavailable"
+  ]
+}

internal/awsmodel/response.go

@@ -0,0 +1,251 @@
+package awsmodel
+
+import (
+	"bytes"
+	"encoding/json"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+)
+
+// ValidateResponse decodes an AWS wire response and validates it against the
+// operation's output shape. JSON services are decoded directly; EC2 and Query
+// XML responses are normalized from their wire names and result wrappers to
+// api-2.json member names first.
+func ValidateResponse(service Service, operationName string, body []byte) ([]Violation, error) {
+	model, err := Load(service)
+	if err != nil {
+		return nil, err
+	}
+	operation, ok := model.Operation(operationName)
+	if !ok {
+		return nil, fmt.Errorf("awsmodel: %s operation %q is not modelled", service, operationName)
+	}
+	var document any
+	switch model.metadata.Protocol {
+	case "json", "rest-json":
+		document, err = decodeJSONResponse(body)
+	case "ec2", "query":
+		if operation.Output == nil {
+			err = validateEmptyXMLResponse(operationName, body)
+		} else {
+			document, err = model.decodeXMLResponse(operationName, operation.Output, body)
+		}
+	default:
+		return nil, fmt.Errorf("awsmodel: response decoding for %s protocol %q is not implemented", service, model.metadata.Protocol)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("awsmodel: decode %s %s response: %w", service, operationName, err)
+	}
+	if operation.Output == nil {
+		// Some Query/EC2 operations deliberately have no modelled output. There
+		// is no shape to inspect, but decoding above still verifies that the
+		// response is syntactically valid and has the correct XML envelope.
+		return nil, nil
+	}
+	return Validate(service, operationName, document)
+}
+
+func decodeJSONResponse(body []byte) (any, error) {
+	decoder := json.NewDecoder(bytes.NewReader(body))
+	decoder.UseNumber()
+	var document any
+	if err := decoder.Decode(&document); err != nil {
+		return nil, err
+	}
+	if err := decoder.Decode(new(any)); !errors.Is(err, io.EOF) {
+		if err == nil {
+			return nil, errors.New("multiple JSON values")
+		}
+		return nil, err
+	}
+	return document, nil
+}
+
+type xmlNode struct {
+	name     string
+	text     strings.Builder
+	children []*xmlNode
+}
+
+func parseXML(body []byte) (*xmlNode, error) {
+	decoder := xml.NewDecoder(bytes.NewReader(body))
+	var root *xmlNode
+	var stack []*xmlNode
+	for {
+		token, err := decoder.Token()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		switch token := token.(type) {
+		case xml.StartElement:
+			node := &xmlNode{name: token.Name.Local}
+			if len(stack) == 0 {
+				if root != nil {
+					return nil, errors.New("multiple XML roots")
+				}
+				root = node
+			} else {
+				parent := stack[len(stack)-1]
+				parent.children = append(parent.children, node)
+			}
+			stack = append(stack, node)
+		case xml.CharData:
+			if len(stack) != 0 {
+				stack[len(stack)-1].text.Write([]byte(token))
+			}
+		case xml.EndElement:
+			if len(stack) == 0 {
+				return nil, errors.New("unexpected XML end element")
+			}
+			stack = stack[:len(stack)-1]
+		}
+	}
+	if root == nil {
+		return nil, errors.New("empty XML response")
+	}
+	return root, nil
+}
+
+func validateEmptyXMLResponse(operationName string, body []byte) error {
+	root, err := parseXML(body)
+	if err != nil {
+		return err
+	}
+	wantRoot := operationName + "Response"
+	if root.name != wantRoot {
+		return fmt.Errorf("root element is %q, want %q", root.name, wantRoot)
+	}
+	return nil
+}
+
+func (m *Model) decodeXMLResponse(operationName string, output *ShapeRef, body []byte) (any, error) {
+	root, err := parseXML(body)
+	if err != nil {
+		return nil, err
+	}
+	wantRoot := operationName + "Response"
+	if root.name != wantRoot {
+		return nil, fmt.Errorf("root element is %q, want %q", root.name, wantRoot)
+	}
+
+	outputNode := root
+	if output.ResultWrapper != "" {
+		outputNode = firstXMLChild(root, output.ResultWrapper)
+		if outputNode == nil {
+			return nil, fmt.Errorf("result wrapper %q is missing", output.ResultWrapper)
+		}
+	}
+	return m.normalizeXMLShape(output.Shape, []*xmlNode{outputNode}, false), nil
+}
+
+func (m *Model) normalizeXMLShape(shapeName string, nodes []*xmlNode, flattened bool) any {
+	shape := m.shapes[shapeName]
+	if shape == nil || len(nodes) == 0 {
+		return nil
+	}
+
+	switch shape.Type {
+	case "structure":
+		return m.normalizeXMLStructure(shape, nodes[0])
+	case "list":
+		itemNodes := nodes
+		if !flattened && !shape.Flattened {
+			itemName := shape.Member.LocationName
+			if itemName == "" {
+				if m.metadata.Protocol == "ec2" {
+					itemName = "item"
+				} else {
+					itemName = "member"
+				}
+			}
+			itemNodes = xmlChildren(nodes[0], itemName)
+		}
+		items := make([]any, 0, len(itemNodes))
+		for _, node := range itemNodes {
+			items = append(items, m.normalizeXMLShape(shape.Member.Shape, []*xmlNode{node}, false))
+		}
+		return items
+	case "map":
+		entries := nodes
+		if !flattened && !shape.Flattened {
+			entries = xmlChildren(nodes[0], "entry")
+		}
+		result := make(map[string]any, len(entries))
+		for _, entry := range entries {
+			keyNode := firstXMLChild(entry, xmlRefName("key", *shape.Key))
+			valueNode := firstXMLChild(entry, xmlRefName("value", *shape.Value))
+			if keyNode != nil && valueNode != nil {
+				key := strings.TrimSpace(keyNode.text.String())
+				result[key] = m.normalizeXMLShape(shape.Value.Shape, []*xmlNode{valueNode}, false)
+			}
+		}
+		return result
+	default:
+		return strings.TrimSpace(nodes[0].text.String())
+	}
+}
+
+func (m *Model) normalizeXMLStructure(shape *Shape, node *xmlNode) map[string]any {
+	document := make(map[string]any)
+	wireMembers := make(map[string]string, len(shape.Members))
+	for name, ref := range shape.Members {
+		wireMembers[xmlRefName(name, ref)] = name
+	}
+
+	grouped := make(map[string][]*xmlNode)
+	for _, child := range node.children {
+		if isTransportMetadata(child.name) {
+			continue
+		}
+		grouped[child.name] = append(grouped[child.name], child)
+	}
+	for wireName, children := range grouped {
+		name, ok := wireMembers[wireName]
+		if !ok {
+			document[wireName] = strings.TrimSpace(children[0].text.String())
+			continue
+		}
+		ref := shape.Members[name]
+		target := m.shapes[ref.Shape]
+		flattened := ref.Flattened || (target != nil && target.Flattened)
+		document[name] = m.normalizeXMLShape(ref.Shape, children, flattened)
+	}
+	return document
+}
+
+func xmlRefName(memberName string, ref ShapeRef) string {
+	if ref.LocationName != "" {
+		return ref.LocationName
+	}
+	return memberName
+}
+
+func firstXMLChild(node *xmlNode, name string) *xmlNode {
+	for _, child := range node.children {
+		if child.name == name {
+			return child
+		}
+	}
+	return nil
+}
+
+func xmlChildren(node *xmlNode, name string) []*xmlNode {
+	children := make([]*xmlNode, 0)
+	for _, child := range node.children {
+		if child.name == name {
+			children = append(children, child)
+		}
+	}
+	return children
+}
+
+func isTransportMetadata(name string) bool {
+	normalized := strings.ToLower(name)
+	return normalized == "requestid" || normalized == "responsemetadata"
+}

internal/awsmodel/response_test.go

@@ -0,0 +1,86 @@
+package awsmodel
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestValidateResponseEC2XML(t *testing.T) {
+	body := []byte(`<DescribeInstancesResponse xmlns="http://ec2.amazonaws.com/doc/2016-11-15/">
+  <requestId>req-1</requestId>
+  <reservationSet><item><instancesSet><item>
+    <instanceState><code>16</code><name>paused</name></instanceState>
+    <inventedField>surprise</inventedField>
+  </item></instancesSet></item></reservationSet>
+</DescribeInstancesResponse>`)
+
+	violations, err := ValidateResponse(EC2, "DescribeInstances", body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []Violation{
+		{Rule: RuleUnknownField, Path: "$.Reservations[0].Instances[0].inventedField", Message: "field is not present in the model shape"},
+		{Rule: RuleEnum, Path: "$.Reservations[0].Instances[0].State.Name", Message: `value "paused" is not one of [pending, running, shutting-down, terminated, stopping, stopped]`},
+	}
+	if !reflect.DeepEqual(violations, want) {
+		t.Fatalf("violations = %#v, want %#v", violations, want)
+	}
+}
+
+func TestValidateResponseQueryXML(t *testing.T) {
+	body := []byte(`<GetUserResponse><GetUserResult><User>
+  <Path>/</Path><UserId>AIDAEXAMPLE</UserId>
+  <Arn>arn:aws:iam::123456789012:user/alice</Arn>
+  <CreateDate>2026-08-05T00:00:00Z</CreateDate>
+</User></GetUserResult></GetUserResponse>`)
+
+	violations, err := ValidateResponse(IAM, "GetUser", body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []Violation{{Rule: RuleRequiredMember, Path: "$.User.UserName", Message: "required member is missing"}}
+	if !reflect.DeepEqual(violations, want) {
+		t.Fatalf("violations = %#v, want %#v", violations, want)
+	}
+}
+
+func TestValidateResponseJSON(t *testing.T) {
+	body := []byte(`{"tasks":[{"launchType":"ON_PREMISES"}]}`)
+	violations, err := ValidateResponse(ECS, "DescribeTasks", body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(violations) != 1 || violations[0].Rule != RuleEnum || violations[0].Path != "$.tasks[0].launchType" {
+		t.Fatalf("violations = %#v, want launchType enum violation", violations)
+	}
+}
+
+func TestValidateResponseConformingEmptyEC2List(t *testing.T) {
+	body := []byte(`<DescribeInstancesResponse><requestId>req-1</requestId><reservationSet></reservationSet></DescribeInstancesResponse>`)
+	violations, err := ValidateResponse(EC2, "DescribeInstances", body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(violations) != 0 {
+		t.Fatalf("violations = %#v, want none", violations)
+	}
+}
+
+func TestValidateResponseOperationWithoutOutputShape(t *testing.T) {
+	body := []byte(`<AttachInternetGatewayResponse><requestId>req-1</requestId></AttachInternetGatewayResponse>`)
+	violations, err := ValidateResponse(EC2, "AttachInternetGateway", body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(violations) != 0 {
+		t.Fatalf("violations = %#v, want none", violations)
+	}
+}
+
+func TestValidateResponseRejectsWrongWrapper(t *testing.T) {
+	_, err := ValidateResponse(IAM, "GetUser", []byte(`<GetUserResponse><WrongResult/></GetUserResponse>`))
+	if err == nil || !strings.Contains(err.Error(), `result wrapper "GetUserResult" is missing`) {
+		t.Fatalf("error = %v", err)
+	}
+}

internal/awsmodel/validate.go

@@ -0,0 +1,140 @@
+package awsmodel
+
+import (
+	"fmt"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+// Rule identifies a model constraint violated by a response document.
+type Rule string
+
+const (
+	RuleUnknownField   Rule = "unknown_field"
+	RuleRequiredMember Rule = "required_member"
+	RuleEnum           Rule = "enum"
+	RuleErrorCode      Rule = "error_code"
+	RuleHTTPStatus     Rule = "http_status"
+)
+
+// Violation describes one response value that does not conform to its output
+// shape. Path uses a JSONPath-like notation rooted at $.
+type Violation struct {
+	Rule    Rule
+	Path    string
+	Message string
+}
+
+func (v Violation) String() string {
+	return fmt.Sprintf("%s: %s (%s)", v.Path, v.Message, v.Rule)
+}
+
+// Validate checks a normalized response document against an operation's
+// output shape. Structure keys must use the member names from api-2.json;
+// protocol-specific response middleware is responsible for translating wire
+// names and wrappers to that representation.
+func Validate(service Service, operationName string, document any) ([]Violation, error) {
+	model, err := Load(service)
+	if err != nil {
+		return nil, err
+	}
+	operation, ok := model.Operation(operationName)
+	if !ok {
+		return nil, fmt.Errorf("awsmodel: %s operation %q is not modelled", service, operationName)
+	}
+	if operation.Output == nil {
+		return nil, nil
+	}
+
+	var violations []Violation
+	model.validateShape(operation.Output.Shape, "$", document, &violations)
+	return violations, nil
+}
+
+func (m *Model) validateShape(shapeName, path string, value any, violations *[]Violation) {
+	shape := m.shapes[shapeName]
+	if shape == nil {
+		// Load verifies references, so this is unreachable for a loaded model.
+		return
+	}
+
+	switch shape.Type {
+	case "structure":
+		m.validateStructure(shape, path, value, violations)
+	case "list":
+		values, ok := value.([]any)
+		if !ok {
+			return
+		}
+		for index, item := range values {
+			m.validateShape(shape.Member.Shape, fmt.Sprintf("%s[%d]", path, index), item, violations)
+		}
+	case "map":
+		values, ok := value.(map[string]any)
+		if !ok {
+			return
+		}
+		keys := sortedKeys(values)
+		for _, key := range keys {
+			keyPath := path + "[" + strconv.Quote(key) + "]"
+			m.validateShape(shape.Key.Shape, keyPath+".key", key, violations)
+			m.validateShape(shape.Value.Shape, keyPath, values[key], violations)
+		}
+	case "string":
+		stringValue, ok := value.(string)
+		if !ok || len(shape.Enum) == 0 {
+			return
+		}
+		if !slices.Contains(shape.Enum, stringValue) {
+			*violations = append(*violations, Violation{
+				Rule:    RuleEnum,
+				Path:    path,
+				Message: fmt.Sprintf("value %q is not one of [%s]", stringValue, strings.Join(shape.Enum, ", ")),
+			})
+		}
+	}
+}
+
+func (m *Model) validateStructure(shape *Shape, path string, value any, violations *[]Violation) {
+	fields, ok := value.(map[string]any)
+	if !ok {
+		fields = map[string]any{}
+	}
+
+	required := append([]string(nil), shape.Required...)
+	slices.Sort(required)
+	for _, name := range required {
+		if _, present := fields[name]; !present {
+			*violations = append(*violations, Violation{
+				Rule:    RuleRequiredMember,
+				Path:    memberPath(path, name),
+				Message: "required member is missing",
+			})
+		}
+	}
+
+	fieldNames := sortedKeys(fields)
+	for _, name := range fieldNames {
+		if _, modelled := shape.Members[name]; !modelled {
+			*violations = append(*violations, Violation{
+				Rule:    RuleUnknownField,
+				Path:    memberPath(path, name),
+				Message: "field is not present in the model shape",
+			})
+		}
+	}
+
+	memberNames := sortedKeys(shape.Members)
+	for _, name := range memberNames {
+		field, present := fields[name]
+		if !present {
+			continue
+		}
+		m.validateShape(shape.Members[name].Shape, memberPath(path, name), field, violations)
+	}
+}
+
+func memberPath(parent, name string) string {
+	return parent + "." + name
+}

internal/awsmodel/validate_test.go

@@ -0,0 +1,128 @@
+package awsmodel
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestValidateUnknownField(t *testing.T) {
+	document := validIAMGetUserDocument()
+	document["Invented"] = true
+
+	violations, err := Validate(IAM, "GetUser", document)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []Violation{{
+		Rule:    RuleUnknownField,
+		Path:    "$.Invented",
+		Message: "field is not present in the model shape",
+	}}
+	if !reflect.DeepEqual(violations, want) {
+		t.Fatalf("violations = %#v, want %#v", violations, want)
+	}
+}
+
+func TestValidateRequiredMembers(t *testing.T) {
+	document := validIAMGetUserDocument()
+	delete(document["User"].(map[string]any), "UserName")
+
+	violations, err := Validate(IAM, "GetUser", document)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []Violation{{
+		Rule:    RuleRequiredMember,
+		Path:    "$.User.UserName",
+		Message: "required member is missing",
+	}}
+	if !reflect.DeepEqual(violations, want) {
+		t.Fatalf("violations = %#v, want %#v", violations, want)
+	}
+}
+
+func TestValidateEnum(t *testing.T) {
+	document := map[string]any{
+		"tasks": []any{
+			map[string]any{"launchType": "ON_PREMISES"},
+		},
+	}
+
+	violations, err := Validate(ECS, "DescribeTasks", document)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []Violation{{
+		Rule:    RuleEnum,
+		Path:    "$.tasks[0].launchType",
+		Message: "value \"ON_PREMISES\" is not one of [EC2, FARGATE, EXTERNAL]",
+	}}
+	if !reflect.DeepEqual(violations, want) {
+		t.Fatalf("violations = %#v, want %#v", violations, want)
+	}
+}
+
+func TestValidateConformingDocuments(t *testing.T) {
+	tests := []struct {
+		name      string
+		service   Service
+		operation string
+		document  any
+	}{
+		{
+			name:      "IAM required members",
+			service:   IAM,
+			operation: "GetUser",
+			document:  validIAMGetUserDocument(),
+		},
+		{
+			name:      "ECS enum",
+			service:   ECS,
+			operation: "DescribeTasks",
+			document: map[string]any{
+				"tasks":    []any{map[string]any{"launchType": "EC2"}},
+				"failures": []any{},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			violations, err := Validate(test.service, test.operation, test.document)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(violations) != 0 {
+				t.Fatalf("violations = %#v, want none", violations)
+			}
+		})
+	}
+}
+
+func TestValidateUnknownOperation(t *testing.T) {
+	_, err := Validate(STS, "InventedOperation", map[string]any{})
+	if err == nil || !strings.Contains(err.Error(), `operation "InventedOperation" is not modelled`) {
+		t.Fatalf("Validate unknown operation error = %v", err)
+	}
+}
+
+func TestViolationString(t *testing.T) {
+	violation := Violation{Rule: RuleEnum, Path: "$.status", Message: `value "broken" is invalid`}
+	want := `$.status: value "broken" is invalid (enum)`
+	if got := violation.String(); got != want {
+		t.Fatalf("String() = %q, want %q", got, want)
+	}
+}
+
+func validIAMGetUserDocument() map[string]any {
+	return map[string]any{
+		"User": map[string]any{
+			"Path":       "/",
+			"UserName":   "alice",
+			"UserId":     "AIDAEXAMPLE",
+			"Arn":        "arn:aws:iam::123456789012:user/alice",
+			"CreateDate": "2026-08-05T00:00:00Z",
+		},
+	}
+}

internal/rdsgw/client_test.go

@@ -123,6 +123,10 @@ func TestCall_DecodesNestedListsAndOptionalFields(t *testing.T) {
 		MasterUsername:       "master",
 		MasterUserPassword:   &password,
 		Port:                 5432,
+		DataVolumeID:         "vol-data-01",
+		DataVolumeSerial:     "voldata01",
+		VMGeneration:         3,
+		FormatAuthorized:     true,
 		Parameters: []handlers_rds.Parameter{
 			{Name: "max_connections", Value: "100"},
 			{Name: "work_mem", Value: "4MB"},
@@ -150,6 +154,12 @@ func TestCall_DecodesNestedListsAndOptionalFields(t *testing.T) {
 	if got.ServingCertificate != want.ServingCertificate {
 		t.Errorf("ServingCertificate = %q, want the seeded PEM", got.ServingCertificate)
 	}
+	if got.DataVolumeID != want.DataVolumeID || got.DataVolumeSerial != want.DataVolumeSerial ||
+		got.VMGeneration != want.VMGeneration || got.FormatAuthorized != want.FormatAuthorized {
+		t.Errorf("volume contract = %q/%q/%d/%t, want %q/%q/%d/%t",
+			got.DataVolumeID, got.DataVolumeSerial, got.VMGeneration, got.FormatAuthorized,
+			want.DataVolumeID, want.DataVolumeSerial, want.VMGeneration, want.FormatAuthorized)
+	}
 }
 
 // An absent optional field must decode as nil, not as an empty value: attach
@@ -170,6 +180,9 @@ func TestCall_AbsentOptionalStaysNil(t *testing.T) {
 	if got.MasterUserPassword != nil {
 		t.Errorf("MasterUserPassword = %q, want nil in attach mode", *got.MasterUserPassword)
 	}
+	if got.DataVolumeID != "" || got.DataVolumeSerial != "" || got.VMGeneration != 0 || got.FormatAuthorized {
+		t.Errorf("absent volume contract decoded as %+v, want zero values", got)
+	}
 }
 
 // commandsResult mirrors the gateway's PollDBCommands output on both sides of

scripts/README.md

@@ -1,7 +1,6 @@
 # Spinifex Scripts
 
-Node installation, dev-environment lifecycle, guest image builds, and verification
-tooling for Spinifex.
+Node installation, dev-environment lifecycle, guest image builds, and verification tooling for Spinifex.
 
 ## Install and node setup
 
@@ -47,9 +46,7 @@ tooling for Spinifex.
 
 ### `images/` — per-image build inputs
 
-Each subdirectory holds a `manifest.conf`, init scripts, and a `setup.sh` that
-`build-system-image.sh` runs inside the libguestfs appliance after packages and files are
-placed. `*_test.sh` files are shell unit tests for the helper of the same name.
+Each subdirectory holds a `manifest.conf`, init scripts, and a `setup.sh` that `build-system-image.sh` runs inside the libguestfs appliance after packages and files are placed. `*_test.sh` files are shell unit tests for the helper of the same name.
 
 | Path | What it builds |
 |---|---|
@@ -58,12 +55,12 @@ placed. `*_test.sh` files are shell unit tests for the helper of the same name.
 | `eks-node/` | The unified Alpine EKS node AMI (K3s server *or* agent, selected at first boot). `setup.sh` installs the pinned, SHA-verified K3s binary and the role init scripts. |
 | `eks-node-gpu/` | The Ubuntu EKS GPU worker AMI. Agent role only — GPU nodes never run the control plane — plus the NVIDIA GPU stack. |
 | `rds-postgres/` | The `spinifex-rds-postgres` AMI. `setup.sh` sets exec bits on the OpenRC services and creates the root-only directory cloud-init drops the agent env file and gateway CA into. |
+| `rds-mariadb/` | The `spinifex-rds-mariadb` AMI. Same layout as `rds-postgres/`, plus the `/etc/my.cnf.d` include drop-in and the `conf.d/mariadb` dependency that keeps the packaged service off an unbootstrapped datadir. |
 | `ubuntu-gpu-{amd,nvidia}-setup.sh` | Chroot setup for GPU-capable Ubuntu guest images: ROCm CLI tools and AMD firmware, or the DKMS-built headless NVIDIA driver. Guest-side only — host VFIO setup is `spx admin gpu setup`. |
 
 ### `images/eks-node/` — in-guest helpers
 
-Installed into the EKS node AMI and run at boot or on a timer. All host communication
-goes through the AWS gateway over SigV4; the VM never speaks NATS directly.
+Installed into the EKS node AMI and run at boot or on a timer. All host communication goes through the AWS gateway over SigV4; the VM never speaks NATS directly.
 
 | Script | What it does |
 |---|---|

scripts/check-coverage.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 # Usage: scripts/check-coverage.sh <coverprofile> [quiet]
 
 PROFILE="${1:?Usage: check-coverage.sh <coverprofile> [quiet]}"
-MIN=68.0
+MIN=70.0
 QUIET="${2:-}"
 
 if [[ ! -s "$PROFILE" ]]; then

scripts/gen-asset-manifest.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+# gen-asset-manifest.sh — extracts the canonical bytes of each managed asset
+# heredoc from the provisioning scripts, sha256s them, and (re)writes
+# spinifex/preflight/manifest_gen.go.
+#
+# Run this after editing a managed heredoc in setup.sh or setup-ovn.sh.
+# manifest_gen_test.go re-extracts the same heredocs and fails if the
+# committed manifest no longer matches, catching drift in CI.
+#
+# Usage: scripts/gen-asset-manifest.sh
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+OUT="$ROOT/spinifex/preflight/manifest_gen.go"
+
+# extract_heredoc FILE ANCHOR
+# Prints the body of the first quoted 'HELPER' heredoc starting after the
+# first line containing ANCHOR — both scripts hold more than one such heredoc.
+extract_heredoc() {
+	local file=$1 anchor=$2
+	awk -v anchor="$anchor" '
+		state == 0 && index($0, anchor) { state = 1; next }
+		state == 1 && index($0, "<<") && index($0, "HELPER") { state = 2; next }
+		state == 2 && $0 == "HELPER" { exit }
+		state == 2 { print }
+	' "$file"
+}
+
+sysctl_hash=$(extract_heredoc "$ROOT/scripts/setup.sh" 'install_endpoint_sysctl_helper()' | sha256sum | cut -d' ' -f1)
+ovs_hash=$(extract_heredoc "$ROOT/scripts/setup-ovn.sh" 'Step 8: Grant the spinifex service users access to OVS/OVN' | sha256sum | cut -d' ' -f1)
+
+if [ -z "$sysctl_hash" ] || [ -z "$ovs_hash" ]; then
+	echo "gen-asset-manifest.sh: failed to extract one or both heredocs — anchors may be stale" >&2
+	exit 1
+fi
+
+mkdir -p "$(dirname "$OUT")"
+cat >"$OUT" <<EOF
+// Code generated by scripts/gen-asset-manifest.sh; DO NOT EDIT.
+
+package preflight
+
+var canonicalHashes = map[string]string{
+	"/usr/local/lib/spinifex/spinifex-set-endpoint-sysctl": "$sysctl_hash",
+	"/usr/local/lib/spinifex/ovs-socket-perms.sh":          "$ovs_hash",
+}
+EOF
+
+if command -v gofmt >/dev/null 2>&1; then
+	gofmt -w "$OUT"
+fi
+
+echo "wrote $OUT"
+echo "  spinifex-set-endpoint-sysctl: $sysctl_hash"
+echo "  ovs-socket-perms.sh:          $ovs_hash"

scripts/images/common/mulga-mgmt-net.sh

@@ -8,7 +8,9 @@
 #   - NIC<n>_DHCP=1: the primary data ENI. OVN serves DHCP; we lease it (retrying
 #     one-shots on a budget until the cross-host datapath is up) so the Ec2 IMDS
 #     datasource can reach 169.254.169.254, and pin a /32 to it so a link-local
-#     169.254.0.0/16 route on another NIC cannot hijack IMDS.
+#     169.254.0.0/16 route on another NIC cannot hijack IMDS. A DHCP NIC with
+#     NIC<n>_DEFAULT=0 (an RDS DB VM's customer ENI) instead loses its leased
+#     default route and gets a source-based policy table for its replies.
 #   - NIC<n>_CIDR: a static NIC (mgmt0 on br-mgmt, which has no DHCP). Applied
 #     address-only; NIC<n>_DEFAULT is 0, so mgmt0 is never the default route.
 #
@@ -23,8 +25,9 @@
 #     cloud-init's network stage re-renders every ENI from IMDS and re-DHCPs
 #     them, which puts back the default route the setup pass deleted from a
 #     NIC<n>_DEFAULT=0 interface — leaving the guest with two default routes and
-#     the customer ENI able to capture egress and IMDS. Nothing re-leases or
-#     re-addresses an interface in this pass.
+#     the customer ENI able to capture egress and IMDS. It also rewrites that
+#     NIC's policy table, whose routes went down with the bounced interface.
+#     Nothing re-leases or re-addresses an interface in this pass.
 set -eu
 
 NETCFG="${MULGA_NETCFG:-/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw}"
@@ -55,7 +58,10 @@ dhcp_oneshot() {
     if command -v udhcpc >/dev/null 2>&1 && [ -x /usr/share/udhcpc/default.script ]; then
         udhcpc -i "$iface" -f -q -n -t 8 -T 2 >/dev/null 2>&1
     elif command -v dhcpcd >/dev/null 2>&1; then
-        dhcpcd -q -t 20 "$iface" >/dev/null 2>&1
+        # -q is only quiet; -1 is what exits, and -p keeps the address on the
+        # way out. A surviving daemon owns the control socket, so cloud-init's
+        # own dhcpcd is forwarded to it and never writes the pid file it waits on.
+        dhcpcd -q -1 -p -t 20 "$iface" >/dev/null 2>&1
     else
         echo "[mulga-mgmt-net] no DHCP client (udhcpc/dhcpcd) for $iface" >&2
         return 1
@@ -96,6 +102,65 @@ drop_default_route() {
     done
 }
 
+# Network address of a leased CIDR, computed rather than read back from
+# `ip route show`, so the policy table's contents do not depend on how a given
+# `ip` renders a connected route.
+cidr_network() {
+    echo "$1" | awk -F/ '{
+        split($1, o, ".")
+        v = o[1] * 16777216 + o[2] * 65536 + o[3] * 256 + o[4]
+        blk = 2 ^ (32 - $2)
+        n = int(v / blk) * blk
+        printf "%d.%d.%d.%d/%d\n", int(n / 16777216) % 256, int(n / 65536) % 256, \
+            int(n / 256) % 256, n % 256, $2
+    }'
+}
+
+# A NIC<n>_DEFAULT=0 DHCP NIC (an RDS DB VM's customer ENI) has its leased
+# default route deleted, so a reply to a client outside its own subnet leaves by
+# the primary ENI with this NIC's source address and is dropped. Give it a
+# private table and steer its replies at it by source address.
+#
+# Matching on source needs no knowledge of the customer VPC's CIDR: the engine
+# binds its socket to this address, so every reply carries it whoever the client
+# is. Every command reports its own failure — a `set -e` trip here is silent and
+# would leave every later NIC in the blob unconfigured.
+install_return_policy() {
+    iface="$1"
+    tid=$(( 100 + $2 ))
+
+    addr=$(ip -4 addr show dev "$iface" 2>/dev/null | awk '$1 == "inet" { print $2; exit }')
+    if [ -z "$addr" ]; then
+        echo "[mulga-mgmt-net] ERROR: no address on $iface; no return-path policy" >&2
+        return 0
+    fi
+    ip4="${addr%%/*}"
+    net=$(cidr_network "$addr")
+    # The lease's gateway is OVN's per-subnet router IP, which reaches every
+    # subnet of the VPC. Read it before the default route is dropped from main.
+    gw=$(ip -4 route show default dev "$iface" 2>/dev/null | awk '{print $3; exit}')
+
+    # Both passes rewrite the table, not just the rule: a rule survives an
+    # interface bounce but the routes go down with the device, and a rule whose
+    # table is empty falls through to main — the asymmetry this exists to fix.
+    ip route replace "$net" dev "$iface" scope link src "$ip4" table "$tid" || \
+        echo "[mulga-mgmt-net] ERROR: failed to add $net to table $tid on $iface" >&2
+    if [ -n "$gw" ]; then
+        ip route replace default via "$gw" dev "$iface" table "$tid" || \
+            echo "[mulga-mgmt-net] ERROR: failed to add default via $gw to table $tid on $iface" >&2
+    else
+        echo "[mulga-mgmt-net] no lease gateway on $iface; leaving table $tid's default as-is"
+    fi
+
+    # A rule added twice is listed twice, so drop it by priority first. The
+    # delete is expected to fail on the first pass of a boot.
+    ip rule del pref "$tid" 2>/dev/null || true
+    ip rule add from "$ip4" table "$tid" pref "$tid" || \
+        echo "[mulga-mgmt-net] ERROR: failed to add rule from $ip4 to table $tid" >&2
+
+    echo "[mulga-mgmt-net] return-path policy on $iface: from $ip4 lookup $tid"
+}
+
 for n in 0 1 2 3 4 5; do
     # ${VAR:-} keeps the unset NIC<n>_* slots safe under `set -u`.
     eval "mac=\${NIC${n}_MAC:-}"
@@ -138,6 +203,7 @@ for n in 0 1 2 3 4 5; do
         # its own default route and, left alone, would both blackhole egress and
         # steal IMDS from the primary ENI.
         if [ "$isdefault" != "1" ]; then
+            install_return_policy "$iface" "$n"
             drop_default_route "$iface"
             if [ "$MODE" = "enforce" ]; then
                 echo "[mulga-mgmt-net] re-asserted no default route on $iface ($mac)"