release: update go.mod for v1.17.0

A user who signs up for the public sandbox and logs in at console.spx3.com
lands on the normal Dashboard, which reads "No instances", "No volumes" and
"No VPCs" for a brand new account. Nothing says this is a sandbox and nothing
suggests a first action.

Show a one-time dialog on the Dashboard when the console is reached at the
sandbox hostname. It names the sandbox, sets expectations about the cluster
being shared and reclaimable, and links to the areas worth visiting first.
Dismissal is remembered in localStorage, and storage being unavailable only
means it shows again next visit.

Sandbox-ness is decided from the hostname rather than a server flag: it is a
property of how the user reached the console, not of the node serving it, and
the same nodes answer on a LAN address and on the public name. This follows
isDirectHostAccess, which already gates the certificate banner on the login
page the same way.

The console SPA is embedded in the binary, so the rebuilt bundle is included.

.github/resolve-sibling-refs.sh

@@ -1,20 +1,21 @@
 #!/usr/bin/env bash
-# Resolve which branch of each sibling repo (viperblock, predastore) the CI
-# workspace should build against. A cross-repo change is developed on a branch
+# Resolve which branch of each sibling repo (viperblock, predastore, northstar,
+# bluebottle) the CI workspace should build against. A cross-repo change is developed on a branch
 # of the SAME name in each repo; build against that branch when it exists so the
 # set can be tested together, and fall back to dev otherwise. This mirrors the
 # resolution the e2e path already does in
 # mulga/scripts/tofu-cluster/build-install-artifacts.sh, so unit CI and e2e CI
 # agree on how sibling branches are selected.
 #
-# Reads $BRANCH (the triggering head/ref name) and writes viperblock=<ref> and
-# predastore=<ref> to $GITHUB_OUTPUT for the checkout steps to consume.
+# Reads $BRANCH (the triggering head/ref name) and writes viperblock=<ref>,
+# predastore=<ref>, northstar=<ref> and bluebottle=<ref> to $GITHUB_OUTPUT for
+# the checkout steps to consume.
 set -euo pipefail
 
 : "${BRANCH:?BRANCH must be set to the triggering branch name}"
 : "${GITHUB_OUTPUT:?GITHUB_OUTPUT must be set}"
 
-for repo in viperblock predastore; do
+for repo in viperblock predastore northstar bluebottle; do
   if git ls-remote --exit-code --heads \
       "https://github.com/mulgadc/${repo}.git" "$BRANCH" >/dev/null 2>&1; then
     ref="$BRANCH"

.github/workflows/e2e-nightly.yml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
     inputs:
       cells:
-        description: "Comma-separated cell numbers to run (blank = all)"
+        description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-19 tofu matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection"
         required: false
 
 permissions:
@@ -379,8 +379,11 @@ jobs:
           if-no-files-found: warn
           retention-days: 14
 
+      # Gated like every other step, not just always(): a cell the dispatch
+      # deselected never checked out or ran tofu init, so destroying would fail
+      # on missing plugins and report the cell red for correctly doing nothing.
       - name: Teardown
-        if: always()
+        if: always() && steps.gate.outputs.selected == 'true'
         working-directory: mulga/scripts/tofu-cluster
         env:
           MATRIX_HOST: ${{ matrix.host }}
@@ -397,7 +400,7 @@ jobs:
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
 
       - name: Upload tofu log
-        if: always()
+        if: always() && steps.gate.outputs.selected == 'true'
         uses: actions/upload-artifact@v7
         with:
           name: tofu-log-cell-${{ matrix.cell }}-${{ github.run_id }}
@@ -426,3 +429,613 @@ jobs:
       # There's no matrix leg here, just the one fixed baremetal cell.
       otel-attrs: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=20,ci.matrix=baremetal/pxe/real-hw
     secrets: inherit
+
+  # ISO cells. Unlike 1-19 these need no tofu: the harnesses run QEMU directly
+  # on the runner. The nodes are therefore L1, which keeps the instances they
+  # launch at L2 and their benchmark numbers comparable between the two cells.
+  #
+  # Serialised against each other because the guest MACs are fixed
+  # (52:54:00:12:...): stable MACs are what make DHCP discovery work, and two
+  # concurrent runs would put duplicates on the rack LAN, which is a confusing
+  # failure for everyone on it, not just CI.
+  iso-single:
+    name: cell-21 (iso/dhcp/single)
+    if: >-
+      github.event_name == 'schedule' ||
+      inputs.cells == '' ||
+      contains(format(',{0},', inputs.cells), ',21,')
+    runs-on: [self-hosted, ci-single]
+    # ISO build, install, boot and pool re-form is ~35 of these on its own,
+    # before the workload step imports an AMI and launches an instance.
+    timeout-minutes: 90
+    concurrency:
+      group: iso-qemu-rack
+      cancel-in-progress: false
+    env:
+      CI_WAN_MODE: dhcp
+      CI_PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
+      # What a real node gets. The harness default of 4G is a dev-box size, and
+      # importing an 820 MiB AMI on top of the running stack does not fit in it.
+      MEM: 8G
+      SMP: 8
+      # .230 is consumed by the OVN router's SNAT, leaving .231-.234 for
+      # instances. Disjoint from cell-22's pool so a run that dies holding its
+      # SNAT address cannot collide with the next cell.
+      EXT_POOL_START: 192.168.0.230
+      EXT_POOL_END: 192.168.0.234
+      EXT_GATEWAY: 192.168.1.1
+      EXT_PREFIX_LEN: 23
+      # Pins where the benchmark writes, so collecting and removing it later are
+      # both exact. Left to itself the harness picks a timestamped name.
+      BENCH_OUT: /tmp/cell21-bench-${{ github.run_id }}
+    steps:
+      - name: Clean Workspace
+        run: |
+          sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          path: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          path: mulga/spinifex
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: mulga/spinifex
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: mulga/viperblock
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: mulga/predastore
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: mulga/northstar
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: mulga/bluebottle
+
+      # Assert the host can do what the cell assumes, before spending 40
+      # minutes finding out it cannot. A checklist in a plan doc goes stale;
+      # this does not.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          ip link show "${BRIDGE:-br-wan}" >/dev/null ||
+            { echo "::error::bridge br-wan not found on this runner"; exit 1; }
+          grep -qE "^allow +${BRIDGE:-br-wan}$" /etc/qemu/bridge.conf ||
+            { echo "::error::br-wan not permitted in /etc/qemu/bridge.conf"; exit 1; }
+          command -v sshpass >/dev/null ||
+            { echo "::error::sshpass not installed"; exit 1; }
+          free_mb=$(free -m | awk '/^Mem:/ {print $7}')
+          [ "$free_mb" -ge 8192 ] ||
+            { echo "::error::need 8 GiB available, have ${free_mb} MiB"; exit 1; }
+          # A 40G image on tmpfs competes with the 8 GiB the VM itself needs.
+          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
+          [ "$fstype" != tmpfs ] ||
+            { echo "::error::$TMPDIR is tmpfs — the node disk would be in RAM"; exit 1; }
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          [ "$avail_gb" -ge 60 ] ||
+            { echo "::error::need 60 GiB on $TMPDIR, have ${avail_gb} GiB"; exit 1; }
+
+      - name: Build artefacts
+        working-directory: mulga/scripts/iso-builder
+        run: make build-rootfs build-squashfs build-initramfs
+
+      # One key for both halves of the cell. test-workload.sh SSHes to the node
+      # AND registers the same key as the EC2 key pair it uses to reach the
+      # instances, so the key test-smoke.sh installs has to be that one — its
+      # default ~/.ssh/id_*.pub would leave the workload step locked out.
+      - name: Prepare SSH key
+        run: |
+          set -euo pipefail
+          mkdir -p "$HOME/.ssh" && chmod 700 "$HOME/.ssh"
+          # Two keys, because the guest key must not share a name with the one
+          # smoke-test.sh registers from the node: the node holds that private
+          # half, and IMDS caches material by key name for five minutes, so
+          # re-importing over it is invisible to the guests.
+          for k in spinifex-key spinifex-workload-key; do
+            [ -f "$HOME/.ssh/$k" ] ||
+              ssh-keygen -t ed25519 -N "" -f "$HOME/.ssh/$k" -C "$k-ci"
+          done
+
+      # Boots the ISO, installs unattended, reads the lease off the serial
+      # console, runs the in-guest smoke test, and installs the key for the
+      # next step.
+      - name: Install and smoke test
+        id: smoke
+        working-directory: mulga/scripts/iso-builder
+        run: |
+          set -euo pipefail
+          # KEEP_NODE: the harness owns the only VM there is and tears it down
+          # on exit, so without this the workload step below has no node left.
+          KEEP_NODE=1 SSH_IDENTITY="$HOME/.ssh/spinifex-key.pub" \
+            ./test/test-smoke.sh | tee /tmp/smoke-$GITHUB_RUN_ID.log
+          ip=$(sed -n 's/^\[smoke\] node address: //p' /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
+          [ -n "$ip" ] || { echo "::error::smoke test did not report a node address"; exit 1; }
+          echo "node_ip=$ip" >> "$GITHUB_OUTPUT"
+
+      # The customer path: AMI, key, VPC, instance, SSH in. --benchmark records
+      # host and guest disk numbers so a regression shows up as a number rather
+      # than a story.
+      - name: Workload and benchmark
+        working-directory: mulga/scripts/iso-builder
+        run: |
+          set -euo pipefail
+          cat > /tmp/cell21.env <<EOF
+          NODES=(${{ steps.smoke.outputs.node_ip }})
+          NODE_NAMES=(spinifex-ci)
+          SSH_USER=spinifex
+          SSH_KEY_PATH=$HOME/.ssh/spinifex-workload-key
+          # SSH_KEY_PATH reaches the guests, SSH_IDENTITY reaches the node, and
+          # they are deliberately different keys. Without the second, node SSH
+          # falls back to the runner's default identity, which the node has
+          # never seen.
+          SSH_IDENTITY=$HOME/.ssh/spinifex-key
+          EXT_POOL_START=$EXT_POOL_START
+          EXT_POOL_END=$EXT_POOL_END
+          EXT_GATEWAY=$EXT_GATEWAY
+          EXT_PREFIX_LEN=$EXT_PREFIX_LEN
+          EOF
+          ./test/test-workload.sh --env /tmp/cell21.env --benchmark
+
+      - name: Collect logs
+        if: always()
+        run: |
+          mkdir -p /tmp/cell21-artifacts-$GITHUB_RUN_ID
+          cp /tmp/smoke-$GITHUB_RUN_ID.log /tmp/cell21-artifacts-$GITHUB_RUN_ID/ ||
+            echo "::warning::no smoke log to collect"
+          # fio JSON and the summary table. Warned about rather than swallowed:
+          # a benchmark that ran and produced nothing is worth knowing about.
+          if [ -d "$BENCH_OUT" ]; then
+            cp -r "$BENCH_OUT" /tmp/cell21-artifacts-$GITHUB_RUN_ID/benchmark
+          else
+            echo "::warning::no benchmark results at $BENCH_OUT"
+          fi
+          # The guest's own view. Without it a node that dies mid-test is only
+          # visible as the harness command that stopped producing output.
+          serial=$(sed -n 's/^\[smoke\] serial log preserved: //p' \
+                   /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
+          [ -n "$serial" ] && cp "$serial" /tmp/cell21-artifacts-$GITHUB_RUN_ID/serial.log || true
+          node=$(sed -n 's/^\[smoke\] node logs preserved: //p' \
+                 /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
+          [ -n "$node" ] && cp "$node" /tmp/cell21-artifacts-$GITHUB_RUN_ID/node.log || true
+          # QEMU's stderr no longer reaches the step log: it is redirected so a
+          # kept-alive VM cannot hold the step's stdout open.
+          qemu=$(sed -n 's/^\[smoke\] qemu log preserved: //p' \
+                 /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
+          [ -n "$qemu" ] && cp "$qemu" /tmp/cell21-artifacts-$GITHUB_RUN_ID/qemu.log || true
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: e2e-artifacts-cell-21-${{ github.run_id }}
+          path: /tmp/cell21-artifacts-${{ github.run_id }}
+          if-no-files-found: warn
+          # Matches the predastore benchmark in cell-23: a night's numbers are
+          # only useful against the nights before them.
+          retention-days: 90
+
+      - name: Teardown
+        if: always()
+        working-directory: mulga/scripts/iso-builder
+        run: |
+          # No env file means the workload step never ran, so there is nothing
+          # of its to clean up.
+          if [ -f /tmp/cell21.env ]; then
+            ./test/test-workload.sh --env /tmp/cell21.env --cleanup || true
+          fi
+          sudo pkill -f spinifex-smoke-test || true
+          rm -rf /tmp/cell21-artifacts-$GITHUB_RUN_ID /tmp/cell21.env \
+                 /tmp/smoke-$GITHUB_RUN_ID.log "$BENCH_OUT"
+          # Runs before BENCH_OUT existed left a timestamped directory behind
+          # every night. Sweep anything older than a day.
+          find /tmp -maxdepth 1 -name 'spinifex-benchmark-*' -type d -mmin +1440 \
+               -exec rm -rf {} + 2>/dev/null || true
+          # KEEP_NODE leaves the node's disk and serial log behind for us.
+          sleep 2
+          rm -f "$TMPDIR"/tmp.*.qcow2 "$TMPDIR"/tmp.*.log "$TMPDIR"/tmp.*-node.log
+
+  iso-multi:
+    name: cell-22 (iso/dhcp/three-node)
+    # After cell-21 rather than beside it: same concurrency group, and running
+    # them together would need 32 GiB on one box.
+    needs: [iso-single]
+    if: >-
+      always() && (
+      github.event_name == 'schedule' ||
+      inputs.cells == '' ||
+      contains(format(',{0},', inputs.cells), ',22,') )
+    runs-on: [self-hosted, ci-multi]
+    timeout-minutes: 90
+    concurrency:
+      group: iso-qemu-rack
+      cancel-in-progress: false
+    env:
+      MEM: 8G
+      SMP: 8
+      PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
+      EXT_POOL_START: 192.168.0.235
+      EXT_POOL_END: 192.168.0.239
+      EXT_GATEWAY: 192.168.1.1
+      EXT_PREFIX_LEN: 23
+    steps:
+      - name: Clean Workspace
+        run: |
+          sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+          # Pinned so the per-node serial logs can be collected afterwards, and
+          # kept beside TMPDIR rather than in /tmp: it also holds nine 40G node
+          # disks, which is what the tmpfs assertion below is about.
+          echo "WORK_DIR=$E2E_ROOT/work/cell22-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          path: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          path: mulga/spinifex
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: mulga/spinifex
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: mulga/viperblock
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: mulga/predastore
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: mulga/northstar
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: mulga/bluebottle
+
+      # 3 x 8 GiB plus 9 sparse 40G disks, before the instances the cluster
+      # launches on top.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          free_mb=$(free -m | awk '/^Mem:/ {print $7}')
+          [ "$free_mb" -ge 26624 ] ||
+            { echo "::error::need 26 GiB available, have ${free_mb} MiB"; exit 1; }
+          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
+          [ "$fstype" != tmpfs ] ||
+            { echo "::error::$TMPDIR is tmpfs — nine node disks would be in RAM"; exit 1; }
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          [ "$avail_gb" -ge 120 ] ||
+            { echo "::error::need 120 GiB on $TMPDIR ($fstype), have ${avail_gb} GiB"; exit 1; }
+
+      - name: Build artefacts
+        working-directory: mulga/scripts/iso-builder
+        run: make build-rootfs build-squashfs build-initramfs
+
+      # Four phases: install three nodes, convert to one cluster, assert it
+      # formed (including the external pool), then pull node 1's power and
+      # assert the survivors hold quorum.
+      #
+      # Phase 5 (test-workload.sh, as cell-21 runs it) is not wired yet: the
+      # nodes are still on user-mode NAT and reachable only through the
+      # harness's own port forwards, and test-cluster.sh does not open the
+      # formation port. Both land with mulga-yxtda and the bridged WAN mode;
+      # until then this cell proves the cluster forms and survives, not that a
+      # customer can use it.
+      - name: Install, form and verify the cluster
+        working-directory: mulga/scripts/iso-builder
+        run: |
+          set -euo pipefail
+          ./test/test-cluster.sh 2>&1 | tee /tmp/cell22-$GITHUB_RUN_ID.log
+
+      - name: Collect logs
+        if: always()
+        run: |
+          mkdir -p /tmp/cell22-artifacts-$GITHUB_RUN_ID
+          cp /tmp/cell22-$GITHUB_RUN_ID.log /tmp/cell22-artifacts-$GITHUB_RUN_ID/ ||
+            echo "::warning::no cluster log to collect"
+          # Serial only. The same directory holds the nodes' disk images, which
+          # are gigabytes and must not go into an artifact.
+          cp "$WORK_DIR"/*-serial.log /tmp/cell22-artifacts-$GITHUB_RUN_ID/ 2>/dev/null ||
+            echo "no per-node serial logs — the run passed and cleaned up after itself"
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: e2e-artifacts-cell-22-${{ github.run_id }}
+          path: /tmp/cell22-artifacts-${{ github.run_id }}
+          if-no-files-found: warn
+          retention-days: 90
+
+      - name: Teardown
+        if: always()
+        run: |
+          sudo pkill -f spinifex-cluster-test || true
+          rm -rf /tmp/cell22-artifacts-$GITHUB_RUN_ID /tmp/cell22-$GITHUB_RUN_ID.log \
+                 "$WORK_DIR"
+
+  # Predastore throughput ("cell 23"). Predastore has no runners of its own and
+  # spinifex depends on it, so the benchmark lives here and checks predastore
+  # out as a sibling.
+  #
+  # needs: is the whole point. A throughput benchmark sharing a host with a cell
+  # measures the neighbour, and ci-single and ci-multi both resolve to
+  # bottlebrush, so "a different label" would not have separated them. needs: on
+  # a matrix job waits for every leg, so this starts on a quiet machine.
+  #
+  # always() because a failed cell must not silently skip the benchmark and
+  # leave a green-looking night with no number in it.
+  performance:
+    name: cell-23 (predastore/throughput)
+    needs: [matrix, baremetal, iso-single, iso-multi]
+    if: >-
+      always() && (
+      github.event_name == 'schedule' ||
+      inputs.cells == '' ||
+      contains(format(',{0},', inputs.cells), ',23,') )
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 60
+    env:
+      # Two-minute samples. The 30s smoke preset is too short to compare nights.
+      PERF_PRESET: compare
+    steps:
+      - name: Clean Workspace
+        run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+          echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
+
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      # Which predastore branch to measure: the same-named branch when one
+      # exists, dev otherwise. Matches how the e2e path already picks siblings,
+      # so a cross-repo change is benchmarked as a set.
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      # The benchmark works in $TMPDIR and predastore refuses writes below a 5%
+      # free-space watermark. Without this the symptom is several hundred
+      # "storage pool is at capacity" lines from warp and no statement of how
+      # much space there actually was.
+      #
+      # TMPDIR is pointed at the runner's disk above, and the tmpfs assertion is
+      # the one that matters: a storage benchmark on a RAM-backed filesystem
+      # measures memcpy. It would report a number that looks excellent, sit in
+      # the baseline, and never notice a disk regression.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
+          case "$fstype" in
+            tmpfs|ramfs)
+              echo "::error::$TMPDIR is $fstype — a storage benchmark there measures RAM"
+              exit 1 ;;
+          esac
+          # Work dirs from runs that were killed before their cleanup trap ran.
+          # -mtime +0 so a concurrent run's directory is never in scope.
+          find "$TMPDIR" -maxdepth 1 -name 'predastore-e2e-performance.*' \
+               -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          pct_used=$(df --output=pcent "$TMPDIR" | tail -1 | tr -dc '0-9')
+          if [ "$pct_used" -ge 90 ] || [ "$avail_gb" -lt 20 ]; then
+            echo "::error::$TMPDIR ($fstype) is ${pct_used}% full with ${avail_gb} GiB free."
+            echo "::error::The benchmark needs headroom above predastore's 5% watermark."
+            exit 1
+          fi
+
+      # PERF_PORT_OFFSET defaults to 10000, so the benchmark's own nodes do not
+      # collide with a cluster left on the default ports.
+      - name: Run throughput benchmark
+        working-directory: predastore
+        run: |
+          set -euo pipefail
+          # Into the results directory, so the artifact reads on its own rather
+          # than needing the job log it was printed into. The benchmark only
+          # ever creates this directory, so a log placed here first survives.
+          mkdir -p "$PERF_RESULTS_ROOT"
+          make e2e-performance 2>&1 | tee "$PERF_RESULTS_ROOT/e2e-performance.log"
+
+      # Report-only. The baseline comparison and the -10% gate are mulga-wpjsx;
+      # until the run-to-run spread is known from real nights, a threshold would
+      # be a guess, and a gate that cries wolf gets muted.
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: predastore-perf-${{ github.run_id }}
+          path: ${{ env.PERF_RESULTS_ROOT }}
+          if-no-files-found: warn
+          retention-days: 90
+
+      - name: Clean up results
+        if: always()
+        run: rm -rf "$PERF_RESULTS_ROOT"
+
+  # Fault injection, not throughput. Cell-23 asks how fast predastore is when
+  # every node is healthy; this asks whether it survives one that is not.
+  #
+  # After cell-23 rather than beside it: both want a quiet single runner, and a
+  # benchmark sharing a machine with a fault injection run measures the wrong
+  # thing. The port offset keeps them apart even so.
+  stress:
+    name: cell-24 (predastore/fault-injection)
+    needs: [matrix, baremetal, iso-single, iso-multi, performance]
+    if: >-
+      always() && (
+      github.event_name == 'schedule' ||
+      inputs.cells == '' ||
+      contains(format(',{0},', inputs.cells), ',24,') )
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 30
+    env:
+      # Freezes a follower. STRESS_HOST=leader is a known failure: a gate keeps
+      # the frozen node first in its meta read order and pays the full client
+      # timeout per key, so listing slows in proportion to object count.
+      STRESS_HOST: follower
+      # Clear of cell-23's default 10000, so the two never collide on a runner
+      # that lets them overlap.
+      STRESS_PORT_OFFSET: "20000"
+    steps:
+      - name: Clean Workspace
+        run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+          echo "STRESS_RESULTS_ROOT=$E2E_ROOT/predastore-stress-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
+
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      # A frozen node is indistinguishable from a slow one, so a run on a
+      # loaded or nearly full disk fails as a timeout and reads as a real
+      # fault. The checks are the cheap way to tell those apart afterwards.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          find "$TMPDIR" -maxdepth 1 -name 'predastore-e2e-stress.*' \
+               -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          if [ "$avail_gb" -lt 10 ]; then
+            echo "::error::$TMPDIR has ${avail_gb} GiB free; the run needs headroom above predastore's 5% watermark."
+            exit 1
+          fi
+
+      # Leftovers from a run killed before its cleanup trap: a surviving node
+      # holds its ports and fails this run somewhere unrelated.
+      - name: Clear stale nodes
+        run: pkill -KILL -f '/bin/s3d -config' || true
+
+      - name: Run fault injection
+        working-directory: predastore
+        run: |
+          set -euo pipefail
+          # events.txt carries the timeline and the FAIL line, but not the build
+          # output or anything Warp wrote before the harness took over. The
+          # harness only creates this directory, so a log placed here survives.
+          mkdir -p "$STRESS_RESULTS_ROOT"
+          make e2e-stress 2>&1 | tee "$STRESS_RESULTS_ROOT/e2e-stress.log"
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: predastore-stress-${{ github.run_id }}
+          path: ${{ env.STRESS_RESULTS_ROOT }}
+          if-no-files-found: warn
+          retention-days: 90
+
+      - name: Clean up
+        if: always()
+        run: |
+          pkill -KILL -f '/bin/s3d -config' || true
+          rm -rf "$STRESS_RESULTS_ROOT"

.github/workflows/e2e.yml

@@ -52,7 +52,7 @@ jobs:
     # inputs.suites.
     if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-iam') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-ecs') || contains(github.event.inputs.suites, 'e2e-storagegrowth') || contains(github.event.inputs.suites, 'e2e-partialblock') || contains(github.event.inputs.suites, 'e2e-rds')
     runs-on: [self-hosted, ci-single]
-    timeout-minutes: 50
+    timeout-minutes: 60
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
@@ -250,53 +250,58 @@ jobs:
           EOF
           ok "kubectl installed on VM"
 
-      # CreateDBInstance boots the DB VM from the rds-postgres AMI, so that image
+      # CreateDBInstance boots the DB VM from an RDS AMI, so both engines' images
       # must be in the VM's image store before the suite runs — same split as the
       # eks-node pair above: build on the runner (needs source + network), import
-      # on the VM (needs predastore/NATS). The suite's psql client needs no step
-      # here: it runs inside a client VM that installs postgresql-client from its
-      # own user-data, because a private-only endpoint is unreachable from the
-      # runner and from the cluster host alike.
-      - name: Build rds-postgres AMI (libguestfs, runner)
+      # on the VM (needs predastore/NATS). The suite's psql and mariadb clients
+      # need no step here: they run inside a client VM that installs them from
+      # its own user-data, because a private-only endpoint is unreachable from
+      # the runner and from the cluster host alike.
+      - name: Build RDS AMIs (libguestfs, runner)
         if: contains(env.SUITES, 'e2e-rds')
         working-directory: spinifex
         run: |
           set -euo pipefail
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          banner "build rds-postgres AMI (libguestfs — no host devices)"
           sudo apt-get update -qq
           sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
           export LIBGUESTFS_BACKEND=direct
           sudo chmod 0644 /boot/vmlinuz-* || true
-          group_open "make build-rds-postgres-image"
-          make build-rds-postgres-image
-          group_close
-          ok "raw: $(du -h /tmp/rds-postgres-image-build/rds-postgres-alpine.raw | cut -f1)"
+          for IMAGE in rds-postgres rds-mariadb; do
+            banner "build ${IMAGE} AMI (libguestfs — no host devices)"
+            group_open "make build-${IMAGE}-image"
+            make "build-${IMAGE}-image"
+            group_close
+            ok "raw: $(du -h "/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" | cut -f1)"
+          done
 
-      - name: Import rds-postgres AMI on VM
+      - name: Import RDS AMIs on VM
         if: contains(env.SUITES, 'e2e-rds')
         run: |
           set -euo pipefail
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
-            spinifex/scripts/images/rds-postgres/manifest.conf)"
-          # SYSTEM_TAG carries three tags, not one: managed-by marks it a system
-          # image and engine/engine-version are what an EngineVersion request
-          # resolves against. --tag is repeatable, so expand rather than pass the
-          # whole string as a single value.
-          TAG_ARGS=""
-          for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
           SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
           VM=tf-user@${{ steps.vm.outputs.wan_ip }}
-          banner "import rds-postgres AMI to VM image store"
-          scp "${SSH_OPTS[@]}" \
-            /tmp/rds-postgres-image-build/rds-postgres-alpine.raw \
-            "$VM":/tmp/rds-postgres-alpine.raw
-          ssh "${SSH_OPTS[@]}" "$VM" \
-            "spx admin images import --file /tmp/rds-postgres-alpine.raw \
-               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
-               --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
-          ok "rds-postgres AMI imported"
+          for IMAGE in rds-postgres rds-mariadb; do
+            eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
+              "spinifex/scripts/images/${IMAGE}/manifest.conf")"
+            # SYSTEM_TAG carries four tags, not one: managed-by marks it a system
+            # image, engine/engine-version are what an EngineVersion request
+            # resolves against, and the contract tag gates the data-volume format
+            # grant. --tag is repeatable, so expand rather than pass the whole
+            # string as a single value.
+            TAG_ARGS=""
+            for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
+            banner "import ${IMAGE} AMI to VM image store"
+            scp "${SSH_OPTS[@]}" \
+              "/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" \
+              "$VM":"/tmp/${IMAGE}-alpine.raw"
+            ssh "${SSH_OPTS[@]}" "$VM" \
+              "spx admin images import --file /tmp/${IMAGE}-alpine.raw \
+                 --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
+                 --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
+            ok "${IMAGE} AMI imported"
+          done
 
       - name: Build E2E test binaries on runner
         working-directory: spinifex/tests/e2e
@@ -383,9 +388,10 @@ jobs:
                echo \"::group::suite \$SUITE\"; \
                printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
+               case \$SUITE in rds) TMO=50m ;; *) TMO=30m ;; esac; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
-                 ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
+                 ./\$SUITE.test -test.v -test.timeout=\$TMO ${TEST_RUN_FLAG} \
                  2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \
                       | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
                rc=\${PIPESTATUS[0]}; \

.github/workflows/publish-system-ami.yml

@@ -2,23 +2,26 @@ name: Publish System AMIs
 
 # Builds the system AMIs (libguestfs, on the runner) and publishes each
 # compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket that backs
-# iso.mulgadc.com/system-ami/. Five images ship:
+# iso.mulgadc.com/system-ami/. Six images ship:
 #   spinifex-eks-node       Alpine EKS K3s node (server|agent, role at boot)
 #   spinifex-ecs-node       Alpine ECS container instance
 #   spinifex-eks-node-gpu   Ubuntu EKS K3s GPU worker (NVIDIA driver + toolkit)
 #   spinifex-ecs-node-gpu   Ubuntu ECS GPU container instance
 #   spinifex-rds-postgres   Alpine RDS PostgreSQL instance
+#   spinifex-rds-mariadb    Alpine RDS MariaDB instance
 # Operators then install them on any cluster with:
 #   spx admin images import --name spinifex-eks-node
 #   spx admin images import --name spinifex-ecs-node
 #   spx admin images import --name spinifex-eks-node-gpu
 #   spx admin images import --name spinifex-ecs-node-gpu
 #   spx admin images import --name spinifex-rds-postgres
+#   spx admin images import --name spinifex-rds-mariadb
 #
-# Manual-only: publishing overwrites the mirror artifact, so it is a deliberate
-# action, not something every push should trigger. The `variant` input scopes
-# which images build — the Ubuntu GPU legs are 16G and much slower (NVIDIA
-# driver download + DKMS build), so publishing only the CPU AMIs stays cheap.
+# Publishing overwrites the mirror artifact, so it is a deliberate action, not
+# something every push should trigger: it runs manually, or from release.yml
+# once a tag has passed its gates. The `variant` input scopes which images
+# build — the Ubuntu GPU legs are 16G and much slower (NVIDIA driver download +
+# DKMS build), so publishing only the CPU AMIs stays cheap.
 
 on:
   workflow_dispatch:
@@ -36,6 +39,16 @@ on:
         description: "Build + convert only; skip the R2 upload"
         type: boolean
         default: false
+  workflow_call:
+    inputs:
+      variant:
+        description: "Which images to build (all|cpu|gpu|rds)"
+        type: string
+        default: all
+      dry_run:
+        description: "Build + convert only; skip the R2 upload"
+        type: boolean
+        default: false
 
 permissions:
   contents: read
@@ -59,7 +72,7 @@ jobs:
           set -euo pipefail
           cpu='{"name":"spinifex-eks-node","manifest":"scripts/images/eks-node/manifest.conf"},{"name":"spinifex-ecs-node","manifest":"scripts/images/ecs-agent/manifest.conf"}'
           gpu='{"name":"spinifex-eks-node-gpu","manifest":"scripts/images/eks-node-gpu/manifest.conf"},{"name":"spinifex-ecs-node-gpu","manifest":"scripts/images/ecs-agent-gpu/manifest.conf"}'
-          rds='{"name":"spinifex-rds-postgres","manifest":"scripts/images/rds-postgres/manifest.conf"}'
+          rds='{"name":"spinifex-rds-postgres","manifest":"scripts/images/rds-postgres/manifest.conf"},{"name":"spinifex-rds-mariadb","manifest":"scripts/images/rds-mariadb/manifest.conf"}'
           case "${VARIANT}" in
             cpu) items="${cpu}" ;;
             gpu) items="${gpu}" ;;
@@ -99,6 +112,20 @@ jobs:
           ref: dev
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: dev
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: dev
+          path: bluebottle
+
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
@@ -116,7 +143,7 @@ jobs:
         # "go.work already exists".
         run: |
           rm -f go.work go.work.sum
-          go work init ./spinifex ./viperblock ./predastore
+          go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Install build + upload tooling
         run: |

.github/workflows/release.yml

@@ -28,13 +28,25 @@ jobs:
           repository: mulgadc/viperblock
           path: viperblock
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          path: bluebottle
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
 
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
-          go-version: "1.26.5"
+          go-version: "1.26.6"
 
       - name: Build microVM artifacts (lb-agent + kernel + initramfs)
         run: cd spinifex && make build-microvm-image
@@ -73,6 +85,18 @@ jobs:
           repository: mulgadc/viperblock
           path: viperblock
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          path: bluebottle
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
 
@@ -207,6 +231,17 @@ jobs:
             echo "Release ${TAG} promoted to latest"
           fi
 
+  # Refresh the system AMIs on the R2 mirror only once the release is out of
+  # draft. Shares the ci-single runner with `iso`, so it queues behind it.
+  system-images:
+    needs: promote
+    permissions:
+      contents: read
+    uses: ./.github/workflows/publish-system-ami.yml
+    with:
+      variant: all
+    secrets: inherit
+
   # Annotate the (still-draft) release if either gate failed.
   mark-failed:
     needs: [ verify, baremetal ]
@@ -268,6 +303,18 @@ jobs:
           repository: mulgadc/viperblock
           path: viperblock
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          path: bluebottle
+
       - name: Checkout Mulga iso-builder
         uses: actions/checkout@v7
         with:
@@ -286,7 +333,7 @@ jobs:
           cache: false
 
       - name: Configure Go workspace
-        run: go work init ./spinifex ./predastore ./viperblock
+        run: go work init ./spinifex ./predastore ./viperblock ./northstar ./bluebottle
 
       - name: Install ISO builder dependencies
         run: |

.github/workflows/segscan-oracle.yml

@@ -51,6 +51,20 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
       - uses: actions/create-github-app-token@v3
         id: app-token
         with:
@@ -73,7 +87,7 @@ jobs:
           go-version-file: "spinifex/go.mod"
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Run Segscan Oracle Test
         env:

.github/workflows/spinifex.yml

@@ -53,14 +53,65 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
+      # the last go.sum change and never refreshed it. Both caches are managed
+      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
-          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
+          cache: false
+
+      # Module contents follow the five go.sum files, so a content key is exact.
+      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
+      # dependencies out of the key and re-downloading on every run.
+      - name: Cache Go modules
+        uses: actions/cache@v6
+        with:
+          path: ~/go/pkg/mod
+          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
+          restore-keys: gomod-
+
+      # Build artifacts follow source, so the key rolls per run and falls back to
+      # this job's newest entry. Keyed per job because -race, -covermode=atomic
+      # and plain builds produce disjoint entries and would evict each other.
+      - name: Cache Go build
+        uses: actions/cache@v6
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
+
+      # The EBS provider contract suite needs a real implementation on each
+      # side of the boundary: qemu-img/qemu-nbd/qemu-io back qemunbdd, which
+      # shares no code with viperblock, and nbdkit plus viperblock's plugin
+      # back viperblockd, the provider production actually uses. Without them
+      # the suite runs against MemoryProvider alone. nbdkit-plugin-dev carries
+      # the headers and nbdkit.pc that cgo needs to build that plugin, and
+      # libnbd-bin supplies the third-party NBD client (nbdinfo/nbdcopy) that
+      # judges the published export by something outside this codebase.
+      - name: Install conformance tools
+        run: sudo apt-get update && sudo apt-get install -y --no-install-recommends qemu-utils nbdkit nbdkit-plugin-dev libnbd-bin
+
+      - name: Build viperblock nbdkit plugin
+        run: make -C viperblock go_build_nbd
 
       - name: Build Spinifex
         run: make -C spinifex build
@@ -69,10 +120,13 @@ jobs:
         run: make -C spinifex manifest-check manifest-lint
 
       - name: Run Tests with Coverage
-        run: make -C spinifex test-cover
+        env:
+          SPINIFEX_REQUIRE_CONFORMANCE_TOOLS: "1"
+          VIPERBLOCK_NBDKIT_PLUGIN: ${{ github.workspace }}/viperblock/lib/nbdkit-viperblock-plugin.so
+        run: make -C spinifex test-cover GOTESTFLAGS=-count=1
 
       - name: Run Action Tests
-        run: make -C spinifex test-actions
+        run: make -C spinifex test-actions GOTESTFLAGS=-count=1
 
       - name: Check Diff Coverage
         run: cd spinifex && scripts/diff-coverage.sh coverage.out --base HEAD~1
@@ -114,14 +168,51 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
+      # the last go.sum change and never refreshed it. Both caches are managed
+      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
-          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
+          cache: false
+
+      # Module contents follow the five go.sum files, so a content key is exact.
+      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
+      # dependencies out of the key and re-downloading on every run.
+      - name: Cache Go modules
+        uses: actions/cache@v6
+        with:
+          path: ~/go/pkg/mod
+          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
+          restore-keys: gomod-
+
+      # Build artifacts follow source, so the key rolls per run and falls back to
+      # this job's newest entry. Keyed per job because -race, -covermode=atomic
+      # and plain builds produce disjoint entries and would evict each other.
+      - name: Cache Go build
+        uses: actions/cache@v6
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Run Integration Tests
         run: make -C spinifex test-integration
@@ -157,17 +248,54 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
+      # the last go.sum change and never refreshed it. Both caches are managed
+      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
-          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
+          cache: false
+
+      # Module contents follow the five go.sum files, so a content key is exact.
+      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
+      # dependencies out of the key and re-downloading on every run.
+      - name: Cache Go modules
+        uses: actions/cache@v6
+        with:
+          path: ~/go/pkg/mod
+          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
+          restore-keys: gomod-
+
+      # Build artifacts follow source, so the key rolls per run and falls back to
+      # this job's newest entry. Keyed per job because -race, -covermode=atomic
+      # and plain builds produce disjoint entries and would evict each other.
+      - name: Cache Go build
+        uses: actions/cache@v6
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Run Tests with Race Detector
-        run: make -C spinifex test-race
+        run: make -C spinifex test-race GOTESTFLAGS=-count=1
 
   lint_and_security:
     name: Lint and Security
@@ -200,14 +328,51 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
+      # the last go.sum change and never refreshed it. Both caches are managed
+      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
-          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
+          cache: false
+
+      # Module contents follow the five go.sum files, so a content key is exact.
+      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
+      # dependencies out of the key and re-downloading on every run.
+      - name: Cache Go modules
+        uses: actions/cache@v6
+        with:
+          path: ~/go/pkg/mod
+          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
+          restore-keys: gomod-
+
+      # Build artifacts follow source, so the key rolls per run and falls back to
+      # this job's newest entry. Keyed per job because -race, -covermode=atomic
+      # and plain builds produce disjoint entries and would evict each other.
+      - name: Cache Go build
+        uses: actions/cache@v6
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Lint
         uses: golangci/golangci-lint-action@v9

.golangci.yml

@@ -69,6 +69,18 @@ linters:
             - pkg: math/rand$
               desc: Use math/rand/v2 instead, see https://go.dev/blog/randv2
 
+        # The EC2 control plane reaches block storage through ebsprovider, not
+        # by importing the engine. viperblock.New starts a chunk uploader, so
+        # every construction is a writer against a volume viperblockd serves.
+        # Nothing under handlers/ may import viperblock, in any form, tests
+        # included. A types-only import counts: one became an engine import.
+        ec2-control-plane:
+          files:
+            - '**/spinifex/handlers/**'
+          deny:
+            - pkg: github.com/mulgadc/viperblock
+              desc: Use the ebsprovider.EBSProvider interface; the control plane must not construct or name viperblock types.
+
     embeddedstructfieldcheck:
       forbid-mutex: true
 

Makefile

@@ -6,6 +6,11 @@ SHELL := /bin/bash
 # surface each other's issues. Scope it per worktree.
 export GOLANGCI_LINT_CACHE := $(CURDIR)/.cache/golangci-lint
 
+# GOFIPS140 is a global build-config change: it alters every package's action ID,
+# stdlib included, so a target that omits it shares no build-cache entry with one
+# that sets it. Exported once here rather than per-recipe so the two can't drift.
+export GOFIPS140 := v1.0.0
+
 # Detect architecture for cross-platform support
 ARCH := $(shell uname -m)
 ifeq ($(ARCH),x86_64)
@@ -63,19 +68,23 @@ LDFLAGS := -s -w -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Version=$(VERSI
 
 go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
-	GOFIPS140=v1.0.0 go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
+	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
 
 build-installer:
 	@echo -e "\n....Building spinifex-installer"
-	GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
+	go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
 
 build-lb-agent:
 	@echo -e "\n....Building lb-agent (static)"
-	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
+	CGO_ENABLED=0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
 
 build-ecs-agent: ## Build the ecs-agent (ships in the ECS-AMI guest; not in host `build`)
 	@echo -e "\n....Building ecs-agent (static)"
-	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w -X main.version=$(VERSION)" -o ./bin/ecs-agent ./cmd/ecs-agent
+	CGO_ENABLED=0 go build -ldflags "-s -w -X main.version=$(VERSION)" -o ./bin/ecs-agent ./cmd/ecs-agent
+
+build-loadgen: ## Build spx-loadgen (drives the stress harness's measured load; not shipped to nodes)
+	@echo -e "\n....Building spx-loadgen"
+	GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/spx-loadgen ./cmd/spx-loadgen
 
 build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-node)
 ifndef IMAGE
@@ -111,6 +120,12 @@ build-rds-postgres-image: ## Build the spinifex-rds-postgres AMI (Alpine + Postg
 import-rds-postgres-image: ## Build + register the rds-postgres AMI (requires a running cluster)
 	$(MAKE) build-system-image IMAGE=rds-postgres IMPORT=1
 
+build-rds-mariadb-image: ## Build the spinifex-rds-mariadb AMI (Alpine + MariaDB 11.8 + rds-init; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=rds-mariadb
+
+import-rds-mariadb-image: ## Build + register the rds-mariadb AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=rds-mariadb IMPORT=1
+
 MICROVM_OUT_DIR := build/microvm
 MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
 MICROVM_INPUTS := scripts/build-microvm-image.sh $(MICROVM_OUT_DIR)/init.sh $(MICROVM_OUT_DIR)/inittab bin/lb-agent
@@ -134,9 +149,22 @@ install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share
 # the pre-commit gate: manifest checks, lint, vuln, and the unit and e2e-harness tiers.
 # integration and race tests skipped to keep quick, they run in CI.
 preflight:
-	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-harness
+	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-package-check test-harness test-build-scripts
 	@echo -e "\n ✅ Preflight passed — safe to commit."
 
+# Shell suites + shellcheck for build/scripts/, the systemd-unit helpers that
+# ship on every node (unlike scripts/images/, kept in preflight: a wrong
+# restart decision here is an outage, not asset churn).
+test-build-scripts:
+	@echo -e "\n....Running build/scripts/**/*_test.sh...."
+	@for t in $$(find build/scripts -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		sh "$$t" || exit 1; \
+	done
+	@echo -e "\n....Running shellcheck over build/scripts/**/*.sh...."
+	shellcheck -S warning $$(find build/scripts -name '*.sh' | sort)
+	@echo "  test-build-scripts ok"
+
 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
 # default `go test ./spinifex/...`. Runs with mocked AWS clients — no
 # infrastructure required, safe to run in CI without a cluster.
@@ -149,9 +177,13 @@ test-harness:
 AWS_MODEL_CONFORMANCE_REPORT ?= $(CURDIR)/.cache/aws-model-conformance-report.txt
 AWS_MODEL_CONFORMANCE_MODE ?= fail
 AWS_MODEL_OPERATION_COVERAGE_REPORT ?= $(CURDIR)/docs/aws-model-operation-coverage.md
+# -count=1 is load-bearing: the conformance report is written by the test binary,
+# so a cached pass skips the run, leaves no report, and the cat below fails the
+# target. It also keeps the conformance gate honest — a cached result would mean
+# the check never ran against this commit.
 test-integration:
 	@echo -e "\n....Running in-process integration tests...."
-	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -count=1 -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
 	@cat $(AWS_MODEL_CONFORMANCE_REPORT)
 	@$(MAKE) --no-print-directory generate-aws-model-coverage
 	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_REPORT)"
@@ -192,25 +224,30 @@ manifest-lint-update:
 # Run unit tests
 test:
 	@echo -e "\n....Running tests for $(GO_PROJECT_NAME)...."
-	GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/...
+	LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/...
+
+# Empty locally, where reusing a cached result between runs is the point.
+# CI passes -count=1 so a green run means the suite executed against that commit
+# rather than replaying results the persisted build cache carried over.
+GOTESTFLAGS ?=
 
 # Run unit tests with coverage profile
 COVERPROFILE ?= coverage.out
 test-cover:
 	@echo -e "\n....Running tests with coverage for $(GO_PROJECT_NAME)...."
-	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
 	@scripts/check-coverage.sh $(COVERPROFILE) $(QUIET)
 
 # Run unit tests with race detector
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
-	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... $(_RACEQ)
 
 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
 test-actions:
 	@echo -e "\n....Running action tests...."
-	LOG_IGNORE=1 go test -timeout 60s ./.github/actions/...
+	LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 60s ./.github/actions/...
 
 # Shell suites + shellcheck for scripts/images/ and images/mkosi.profiles/
 # helpers baked into system images. Kept out of `preflight` (a dedicated CI
@@ -249,6 +286,10 @@ test-images:
 diff-coverage: test-cover
 	@QUIET=$(QUIET) scripts/diff-coverage.sh $(COVERPROFILE)
 
+# Check that newly added test files use an external test package
+test-package-check:
+	@QUIET=$(QUIET) scripts/check-test-package.sh
+
 bench:
 	@echo -e "\n....Running benchmarks for $(GO_PROJECT_NAME)...."
 	LOG_IGNORE=1 go test -benchmem -run=. -bench=. ./...
@@ -303,9 +344,9 @@ install-system:
 		ovn-central ovn-host openvswitch-switch dhcpcd-base
 
 install-go:
-	@echo -e "\n....Installing Go 1.26.5 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.26.6 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.26.5.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.26.6.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi
@@ -388,7 +429,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
+.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

README.md

@@ -12,7 +12,7 @@
   <a href="#what-is-spinifex">What is Spinifex?</a> ·
   <a href="#the-platform">Platform</a> ·
   <a href="#three-ways-to-deploy">Deploy</a> ·
-  <a href="#aws-compatibility">AWS services</a> ·
+  <a href="#aws-compatibility">AWS Compatibility</a> ·
   <a href="#core-components">Components</a> ·
   <a href="#architecture-at-a-glance">Architecture</a> ·
   <a href="#installation">Installation</a> ·
@@ -48,7 +48,7 @@ You change *where* your software runs, not *what* it is.
 
 From commodity hardware up to unmodified AWS tooling, every layer is replaceable and yours to own.
 
-## Three ways to deploy
+## Three Ways To Deploy
 
 One AWS-compatible surface, three deployment shapes. Pick the one that matches your reality; the platform on top is the same.
 
@@ -64,21 +64,21 @@ Bring your own hardware and host Spinifex in your own data centre. A real multi-
 
 Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels, factories, and clinics. Compute next to the data, running through disconnection, on hardware you own. The same AWS APIs your software already uses.
 
-## AWS compatibility
+## AWS Compatibility
 
 Speak the AWS API surface, natively. The AWS SDKs, AWS CLI, and Terraform: everything you deploy on AWS deploys on Spinifex unchanged. At the edge, on-premise, or on a partner Neocloud.
 
 | Service | What it is | Status |
 | --- | --- | --- |
 | **EC2** | Compute | Available |
-| **EBS** | Block storage | Available |
-| **S3** | Object storage | Available |
+| **EBS** | Block Storage | Available |
+| **S3** | Object Storage | Available |
 | **VPC** | Networking | Available |
-| **IAM** | Identity & auth | Available |
-| **ALB / NLB** | Load balancers | Available |
+| **IAM** | Identity & Auth | Available |
+| **ALB / NLB** | Load Balancers | Available |
 | **EKS** | Kubernetes | Available |
-| **ECR** | Container registry | Available |
-| **ECS** | Container service | Available |
+| **ECR** | Container Registry | Available |
+| **ECS** | Container Service | Available |
 | **RDS** | Databases | Available |
 | **Bedrock** | AI Deployment | Q3 2026 |
 
@@ -112,6 +112,14 @@ Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs
 - Multipart uploads, streaming reads/writes
 - Data redundancy with Reed-Solomon encoding
 
+### Northstar (Authoritative DNS)
+
+[Northstar](https://github.com/mulgadc/northstar) is a lightweight authoritative DNS server that handles both internal service discovery and public-facing DNS for a Spinifex cluster. Zones are human-readable TOML files, served from the local filesystem or synced from Predastore, keeping DNS configuration in the same object storage as the rest of your infrastructure.
+
+- Authoritative DNS over UDP, TCP, and DNS-over-TLS
+- Service discovery for cluster components via SRV records
+- Zone files stored locally or in S3-compatible storage
+
 ## Architecture at a Glance
 
 <p align="center">
@@ -131,7 +139,7 @@ Every AWS API call is authenticated at the gateway, published to a NATS subject,
 
 ## Installation
 
-Installation requires an Ubuntu / Debian system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for maintaining and installing Spinifex.
+Installation requires an Ubuntu 26.04 or Debian 13 system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for installing Spinifex.
 
 ### Bare Metal ISO
 
@@ -141,16 +149,11 @@ The recommended installation is a [bootable x86 installer](https://iso.mulgadc.c
 curl -fLO https://iso.mulgadc.com/spinifex.iso
 ```
 
-Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware. The install guide walks through the full process.
-
+Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware.
 
 ### Single Node Install
 
-The installation is straightforward to set up and running on a single node for testing purposes. Debian 13 is currently supported, additional Linux distributions are on the immediate roadmap.
-
->*Prerequisite:* Linux bridge for networking.
-
-Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) prerequisites for setup details.
+>*Prerequisite:* Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) for setup details.
 
 ```bash
 curl -fsSL https://install.mulgadc.com | bash
@@ -168,7 +171,7 @@ aws ec2 describe-instance-types
 
 ### Development Setup
 
-For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation
+For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation.
 
 ### Component Repositories
 
@@ -176,6 +179,7 @@ Spinifex coordinates these independent components:
 
 - **[Predastore](https://github.com/mulgadc/predastore)** - S3-compatible object storage
 - **[Viperblock](https://github.com/mulgadc/viperblock)** - EBS-compatible block storage
+- **[Northstar](https://github.com/mulgadc/northstar)** - Authoritative DNS server
 
 Each component can be developed independently. See component-specific documentation for focused development guides.
 
@@ -189,10 +193,6 @@ Spinifex ships with a built-in web console — an optional alternative to the AW
 
 The console is served by each node on port `3000` over TLS, and becomes available as soon as `spinifex.target` is up:
 
-```bash
-open https://YOUR_NODE_IP:3000
-```
-
 - **Same API, different surface.** Every action in the UI is the same AWS SigV4 call the CLI makes — so RBAC, audit trails, and IAM policies apply uniformly.
 - **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed — no separate user database.
 - **Self-hosted, works offline.** The UI is embedded in the Spinifex binary and served from the node itself. No external CDN, no analytics calls, no cloud dependency.

build/Dockerfile.distro

@@ -16,9 +16,11 @@ COPY predastore/go.mod predastore/go.sum ./predastore/
 COPY viperblock/go.mod viperblock/go.sum ./viperblock/
 COPY viperblock/nbd/go.mod viperblock/nbd/go.sum ./viperblock/nbd/
 COPY viperblock/nbd/libguestfs.org/nbdkit/go.mod ./viperblock/nbd/libguestfs.org/nbdkit/
+COPY northstar/go.mod northstar/go.sum ./northstar/
+COPY bluebottle/go.mod bluebottle/go.sum ./bluebottle/
 
 # Generate go.work (gitignored, so not in checkout)
-RUN cd spinifex && go work init . ../predastore ../viperblock ../viperblock/nbd ../viperblock/nbd/libguestfs.org/nbdkit
+RUN cd spinifex && go work init . ../predastore ../viperblock ../viperblock/nbd ../viperblock/nbd/libguestfs.org/nbdkit ../northstar ../bluebottle
 
 # Download dependencies (cached layer)
 RUN cd spinifex && go mod download
@@ -27,6 +29,8 @@ RUN cd spinifex && go mod download
 COPY spinifex/ ./spinifex/
 COPY predastore/ ./predastore/
 COPY viperblock/ ./viperblock/
+COPY northstar/ ./northstar/
+COPY bluebottle/ ./bluebottle/
 
 # Build spinifex binary (spx)
 RUN COMMIT=$(cd spinifex && git rev-parse --short HEAD 2>/dev/null || echo "unknown") && \

build/scripts/nats-js-watchdog.sh

@@ -18,22 +18,97 @@
 # So a health check alone misses case 2 entirely. Both probes run here, and a
 # sustained failure of either restarts the service. The multi-sample window
 # avoids restarting during transient startup/quorum 503s.
+#
+# Three more guards sit around that decision, because "restart when the
+# probes fail" is not safe on its own:
+#
+#   - Age gate: spinifex-nats.service is Type=simple, so it reports active the
+#     instant the process forks, long before JetStream finishes replaying its
+#     state. Acting before a recovery budget has elapsed restarts a server
+#     that is still coming up, not one that has failed.
+#   - Cooldown: a restart that did not fix things must not be retried by the
+#     very next run, seconds later.
+#   - Escalation: a restart that has not fixed things after several attempts
+#     is not going to fix it on the next one either, and an unrunnable spx
+#     must never be read as proof of a JetStream failure at all.
 
 set -eu
 
 HEALTH_URL="http://127.0.0.1:8222/healthz?js-enabled-only=true"
 SERVICE="spinifex-nats.service"
 SPX="${SPX_BIN:-/usr/local/bin/spx}"
-SAMPLES=3        # consecutive failing samples required before acting
-SAMPLE_GAP=2     # seconds between samples
-PROBE_TIMEOUT=5  # seconds for one canary round-trip
+SAMPLES="${SAMPLES:-3}"           # consecutive failing samples required before acting
+SAMPLE_GAP="${SAMPLE_GAP:-2}"     # seconds between samples
+PROBE_TIMEOUT="${PROBE_TIMEOUT:-5}"  # seconds for one canary round-trip
 
-# Only act on a running service: a stopped/activating unit is systemd's to manage,
-# and try-restart on it would either no-op or fight an in-flight (re)start.
-if ! systemctl is-active --quiet "$SERVICE"; then
+# Recovery budget: on casuarina NATS needed 3m49s between "Started" and
+# JetStream actually accepting writes, and the bead recorded ~6min elsewhere.
+# 600s clears both with margin.
+NATS_MIN_AGE="${NATS_MIN_AGE:-600}"
+
+# Minimum time between restarts this watchdog performs, independent of the
+# age gate: closes the case where a restart did not fix things and the very
+# next run pulls the trigger again before the new process could recover.
+RESTART_COOLDOWN="${RESTART_COOLDOWN:-600}"
+
+# A restart that has not fixed things after this many attempts is not going
+# to fix it on the next one either, and looping costs more than stopping.
+RESTART_ESCALATE_AT="${RESTART_ESCALATE_AT:-3}"
+
+STATE_DIR="${STATE_DIR:-/run/spinifex}"
+COOLDOWN_STAMP="$STATE_DIR/nats-js-watchdog.last-restart"
+RESTART_COUNT_FILE="$STATE_DIR/nats-js-watchdog.restart-count"
+# Marks that the escalation message has already been printed for the current
+# latch, so a node that stays broken says so once instead of once per tick.
+ESCALATED_STAMP="$STATE_DIR/nats-js-watchdog.escalated"
+
+# Monotonic clock source, so a wall-clock step during boot (NTP correcting an
+# unset RTC) cannot manufacture a false age or cooldown. Overridable so tests
+# can stub a clock without faking /proc.
+UPTIME_FILE="${UPTIME_FILE:-/proc/uptime}"
+
+uptime_seconds() {
+    awk '{printf "%d", $1}' "$UPTIME_FILE"
+}
+
+# nats_age_seconds prints how long $SERVICE has been running. It reads
+# ExecMainStartTimestampMonotonic rather than is-active: that property goes
+# true the instant Type=simple forks, long before JetStream has replayed its
+# state, so it cannot express the recovery window this gate exists to
+# protect. A unit systemd has never started prints 0, which also covers a
+# stopped/activating unit the way the old is-active guard did.
+nats_age_seconds() {
+    start_usec=$(systemctl show --property=ExecMainStartTimestampMonotonic --value "$SERVICE" 2>/dev/null) || start_usec=""
+    case "$start_usec" in
+        ''|*[!0-9]*) echo 0; return ;;
+    esac
+    if [ "$start_usec" -eq 0 ]; then
+        echo 0
+        return
+    fi
+    echo $(( $(uptime_seconds) - start_usec / 1000000 ))
+}
+
+age=$(nats_age_seconds)
+if [ "$age" -lt "$NATS_MIN_AGE" ]; then
     exit 0
 fi
 
+if [ -f "$COOLDOWN_STAMP" ]; then
+    last=$(cat "$COOLDOWN_STAMP" 2>/dev/null) || last=""
+    case "$last" in ''|*[!0-9]*) last=0 ;; esac
+    if [ $(( $(uptime_seconds) - last )) -lt "$RESTART_COOLDOWN" ]; then
+        exit 0
+    fi
+fi
+
+restart_count=$(cat "$RESTART_COUNT_FILE" 2>/dev/null) || restart_count=""
+case "$restart_count" in ''|*[!0-9]*) restart_count=0 ;; esac
+
+# The escalation count is read here but acted on only after the probes, below.
+# Checking it up front would return before the one line that clears it, so a
+# latched watchdog could never observe recovery and never re-arm itself.
+
 # curl is the probe transport; if it is unavailable, do nothing rather than
 # false-restart a healthy server.
 if ! command -v curl >/dev/null 2>&1; then
@@ -45,13 +120,22 @@ js_healthy() {
     curl -fsS -o /dev/null --max-time 5 "$HEALTH_URL" 2>/dev/null
 }
 
-# Round-trips a canary key through the cluster-state bucket. Absent spx this
-# degrades to the health check alone rather than restarting on a missing tool.
+# Round-trips a canary key through the cluster-state bucket. Exit 0 means the
+# write succeeded and exit 1 means JetStream refused it; anything else
+# (spx's own exit 2, a missing binary, or one that cannot run at all) means
+# the probe observed nothing about JetStream and must never be read as
+# evidence of a failure.
 js_writable() {
     if [ ! -x "$SPX" ]; then
-        return 0
+        return 2
     fi
     "$SPX" admin node js-probe --timeout "${PROBE_TIMEOUT}s" >/dev/null 2>&1
+    rc=$?
+    case "$rc" in
+        0) return 0 ;;
+        1) return 1 ;;
+        *) return 2 ;;
+    esac
 }
 
 failure=""
@@ -59,6 +143,18 @@ i=1
 while [ "$i" -le "$SAMPLES" ]; do
     if js_healthy; then
         if js_writable; then
+            wrc=0
+        else
+            wrc=$?
+        fi
+        if [ "$wrc" -eq 0 ]; then
+            # Recovery: drop the history so a later failure gets the full
+            # restart budget again, and re-arm the escalation message.
+            rm -f "$RESTART_COUNT_FILE" "$ESCALATED_STAMP"
+            exit 0
+        fi
+        if [ "$wrc" -eq 2 ]; then
+            echo "nats-js-watchdog: js-probe inconclusive (config, connect, bucket-open, or spx itself failed); not evidence of a JetStream failure, taking no action" >&2
             exit 0
         fi
         failure="JetStream accepts no writes (canary round-trip failed) while $HEALTH_URL reports healthy"
@@ -71,5 +167,24 @@ while [ "$i" -le "$SAMPLES" ]; do
     i=$((i + 1))
 done
 
+# The failure is real and current, so the restart history now means what it
+# says. Above the threshold another restart will not help, so stop acting —
+# but keep probing on later runs, because that is what notices a recovery.
+if [ "$restart_count" -ge "$RESTART_ESCALATE_AT" ]; then
+    if [ ! -f "$ESCALATED_STAMP" ]; then
+        mkdir -p "$STATE_DIR"
+        : > "$ESCALATED_STAMP"
+        echo "nats-js-watchdog: already restarted $SERVICE $restart_count time(s) without recovery; giving up rather than looping (clear $RESTART_COUNT_FILE to re-arm)" >&2
+    fi
+    exit 0
+fi
+
 echo "nats-js-watchdog: $failure on $SAMPLES consecutive probes; restarting $SERVICE" >&2
+mkdir -p "$STATE_DIR"
+uptime_seconds > "$COOLDOWN_STAMP"
+restart_count=$((restart_count + 1))
+echo "$restart_count" > "$RESTART_COUNT_FILE"
+if [ "$restart_count" -ge "$RESTART_ESCALATE_AT" ]; then
+    echo "nats-js-watchdog: restart $restart_count of $RESTART_ESCALATE_AT before escalation; if $SERVICE is still failing after this, the watchdog will stop acting until $RESTART_COUNT_FILE is cleared" >&2
+fi
 systemctl try-restart "$SERVICE"

build/scripts/nats-js-watchdog_test.sh

@@ -0,0 +1,197 @@
+#!/bin/sh
+# Self-contained POSIX test for nats-js-watchdog.sh's restart-decision logic:
+# the age gate, cooldown, escalation, and js-probe exit-2 handling. No real
+# NATS or spx: systemctl, curl, and spx are all stubbed on PATH ahead of the
+# real ones, driven entirely by env vars this test controls.
+#
+# Run: sh build/scripts/nats-js-watchdog_test.sh
+set -eu
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+SCRIPT="${SCRIPT_DIR}/nats-js-watchdog.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "${WORK}"' EXIT
+
+FAKE_BIN="${WORK}/bin"
+mkdir -p "${FAKE_BIN}"
+
+FAILS=0
+fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); }
+pass() { echo "ok: $*"; }
+
+# Fake systemctl: `show` reports a controllable ExecMainStartTimestampMonotonic,
+# and `try-restart` just records that it was called rather than touching a
+# real unit.
+cat > "${FAKE_BIN}/systemctl" <<'EOF'
+#!/bin/sh
+case "$1" in
+    show)
+        echo "${FAKE_NATS_START_USEC:-0}"
+        ;;
+    try-restart)
+        echo "try-restart $2" >> "${FAKE_RESTART_LOG:-/dev/null}"
+        ;;
+esac
+EOF
+chmod +x "${FAKE_BIN}/systemctl"
+
+# Fake curl: js_healthy() only cares about the exit status.
+cat > "${FAKE_BIN}/curl" <<'EOF'
+#!/bin/sh
+[ "${FAKE_HEALTHY:-1}" = "1" ]
+EOF
+chmod +x "${FAKE_BIN}/curl"
+
+# Fake spx: js-probe's whole contract from the watchdog's side is its exit
+# code (0 ok, 1 JetStream refused, 2 inconclusive).
+cat > "${FAKE_BIN}/spx" <<'EOF'
+#!/bin/sh
+exit "${FAKE_SPX_EXIT:-0}"
+EOF
+chmod +x "${FAKE_BIN}/spx"
+
+PATH="${FAKE_BIN}:${PATH}"
+export PATH
+
+# reset_env <name>: fresh per-case dirs and a default env that already clears
+# the age, cooldown, and escalation gates (old enough NATS, no cooldown
+# stamp, no restart history) and a healthy, single-sample probe, so a case
+# only has to override what it is actually testing.
+reset_env() {
+    CASE="${WORK}/$1"
+    mkdir -p "${CASE}/state"
+    UPTIME_FILE="${CASE}/uptime"
+    printf '700.00 0.00\n' > "${UPTIME_FILE}"
+    STATE_DIR="${CASE}/state"
+    FAKE_RESTART_LOG="${CASE}/restart.log"
+    FAKE_NATS_START_USEC=1000000   # 1s: 700s uptime - 1s start = 699s age, clears the 600s default gate
+    FAKE_HEALTHY=1
+    FAKE_SPX_EXIT=0
+    SAMPLES=1
+    SAMPLE_GAP=0
+    NATS_MIN_AGE=600
+    RESTART_COOLDOWN=600
+    RESTART_ESCALATE_AT=3
+    SPX_BIN="${FAKE_BIN}/spx"
+    STDOUT="${CASE}/stdout"
+    STDERR="${CASE}/stderr"
+    export UPTIME_FILE STATE_DIR FAKE_RESTART_LOG FAKE_NATS_START_USEC \
+        FAKE_HEALTHY FAKE_SPX_EXIT SAMPLES SAMPLE_GAP NATS_MIN_AGE \
+        RESTART_COOLDOWN RESTART_ESCALATE_AT SPX_BIN
+}
+
+# Run the watchdog, capturing its exit code (without tripping set -e) + streams.
+invoke() { rc=0; sh "${SCRIPT}" >"${STDOUT}" 2>"${STDERR}" || rc=$?; }
+
+restarted() { [ -s "${FAKE_RESTART_LOG}" ]; }
+
+# --- Case 1: NATS younger than the age gate -> no restart ---
+reset_env young
+FAKE_NATS_START_USEC=690000000  # 690s: 700s uptime - 690s start = 10s age, well under 600s
+export FAKE_NATS_START_USEC
+invoke
+[ "${rc}" -eq 0 ] && pass "young: exit 0" || fail "young: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "young: must not restart while still in the recovery window" || pass "young: no restart"
+
+# --- Case 2: cooldown stamp fresh -> no restart ---
+reset_env cooldown
+echo 650 > "${STATE_DIR}/nats-js-watchdog.last-restart"
+invoke
+[ "${rc}" -eq 0 ] && pass "cooldown: exit 0" || fail "cooldown: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "cooldown: must not restart again inside the cooldown window" || pass "cooldown: no restart"
+
+# --- Case 3: escalation threshold reached while still failing -> no restart,
+# loud log. The probe must fail: a healthy probe at the threshold is case 3b,
+# and it re-arms rather than giving up. ---
+reset_env escalated
+FAKE_SPX_EXIT=1
+export FAKE_SPX_EXIT
+echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count"
+invoke
+[ "${rc}" -eq 0 ] && pass "escalated: exit 0" || fail "escalated: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "escalated: must not restart past the escalation threshold" || pass "escalated: no restart"
+grep -q 'giving up rather than looping' "${STDERR}" \
+    && pass "escalated: loud log emitted" || fail "escalated: escalation log missing: $(cat "${STDERR}")"
+
+# Second identical run: the node is still broken, so the decision stands, but
+# repeating the message every timer tick is what buried the real signal for 42
+# hours on node1.
+invoke
+grep -q 'giving up rather than looping' "${STDERR}" \
+    && fail "escalated: message repeated on every run instead of once per latch" \
+    || pass "escalated: message not repeated"
+
+# --- Case 3b: escalation threshold reached but JetStream has recovered ->
+# the watchdog must re-arm itself. This is the production bug: the escalation
+# check used to run before the probe, so the clear below was unreachable and
+# the node stayed disarmed until reboot. ---
+reset_env escalated_recovered
+echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count"
+: > "${STATE_DIR}/nats-js-watchdog.escalated"
+invoke
+[ "${rc}" -eq 0 ] && pass "escalated-recovered: exit 0" || fail "escalated-recovered: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "escalated-recovered: must not restart a recovered server" || pass "escalated-recovered: no restart"
+[ -f "${STATE_DIR}/nats-js-watchdog.restart-count" ] \
+    && fail "escalated-recovered: restart count must clear once JetStream is writable again" \
+    || pass "escalated-recovered: restart count cleared"
+[ -f "${STATE_DIR}/nats-js-watchdog.escalated" ] \
+    && fail "escalated-recovered: escalation marker must clear so a later failure is reported" \
+    || pass "escalated-recovered: escalation marker cleared"
+[ -s "${STDERR}" ] \
+    && fail "escalated-recovered: a recovered node must be silent: $(cat "${STDERR}")" \
+    || pass "escalated-recovered: silent"
+
+# --- Case 3c: re-armed watchdog acts again. One healthy run clears the latch,
+# and the next genuine failure must restart -- proving it is armed, not just
+# quiet. ---
+reset_env escalated_rearm
+echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count"
+invoke
+FAKE_SPX_EXIT=1
+export FAKE_SPX_EXIT
+invoke
+restarted && pass "rearm: restarts again after recovery cleared the latch" \
+    || fail "rearm: expected a restart once re-armed: $(cat "${STDERR}")"
+[ "$(cat "${STATE_DIR}/nats-js-watchdog.restart-count" 2>/dev/null)" = "1" ] \
+    && pass "rearm: restart count restarts from 1" \
+    || fail "rearm: expected count 1, got $(cat "${STATE_DIR}/nats-js-watchdog.restart-count" 2>/dev/null)"
+
+# --- Case 4: js-probe exits 2 (inconclusive) -> no restart ---
+reset_env inconclusive
+FAKE_SPX_EXIT=2
+export FAKE_SPX_EXIT
+invoke
+[ "${rc}" -eq 0 ] && pass "inconclusive: exit 0" || fail "inconclusive: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "inconclusive: exit 2 must never be read as a JetStream failure" || pass "inconclusive: no restart"
+grep -q 'inconclusive' "${STDERR}" \
+    && pass "inconclusive: reason logged" || fail "inconclusive: log missing: $(cat "${STDERR}")"
+
+# --- Case 5: genuine JetStream write failure -> restart happens ---
+reset_env genuine_failure
+FAKE_SPX_EXIT=1
+export FAKE_SPX_EXIT
+echo 2 > "${STATE_DIR}/nats-js-watchdog.restart-count"
+invoke
+[ "${rc}" -eq 0 ] && pass "genuine-failure: exit 0" || fail "genuine-failure: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && pass "genuine-failure: restart triggered" || fail "genuine-failure: expected a restart: $(cat "${STDERR}")"
+[ "$(cat "${STATE_DIR}/nats-js-watchdog.restart-count" 2>/dev/null)" = "3" ] \
+    && pass "genuine-failure: restart count incremented" || fail "genuine-failure: restart count not incremented"
+[ -f "${STATE_DIR}/nats-js-watchdog.last-restart" ] \
+    && pass "genuine-failure: cooldown stamp written" || fail "genuine-failure: cooldown stamp missing"
+
+# --- Case 6: healthy NATS -> no restart, and prior restart history clears ---
+reset_env healthy
+echo 2 > "${STATE_DIR}/nats-js-watchdog.restart-count"
+invoke
+[ "${rc}" -eq 0 ] && pass "healthy: exit 0" || fail "healthy: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "healthy: must not restart a healthy server" || pass "healthy: no restart"
+[ -f "${STATE_DIR}/nats-js-watchdog.restart-count" ] \
+    && fail "healthy: restart count must clear once JetStream is confirmed writable" \
+    || pass "healthy: restart count cleared"
+
+if [ "${FAILS}" -eq 0 ]; then
+    echo "PASS: all nats-js-watchdog cases"
+    exit 0
+fi
+echo "FAILED: ${FAILS} case(s)"
+exit 1

build/scripts/wait-for-nats.sh

@@ -26,6 +26,7 @@ fi
 # Final fallback
 NATS_HOST="${NATS_HOST:-127.0.0.1}"
 
+# shellcheck disable=SC2034  # loop count only, index unused
 for i in $(seq 1 "$TIMEOUT"); do
     if nc -z "$NATS_HOST" "$NATS_PORT" 2>/dev/null; then
         exit 0

build/systemd/regenerate-ssh-host-keys.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Regenerate SSH host keys
 ConditionPathExists=!/etc/ssh/ssh_host_ed25519_key

build/systemd/spinifex-awsgw.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex AWS Gateway
 PartOf=spinifex.target

build/systemd/spinifex-daemon.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 3
 [Unit]
 Description=Spinifex Daemon (VM Orchestration)
 PartOf=spinifex.target
@@ -38,12 +39,13 @@ PrivateDevices=no
 DevicePolicy=closed
 DeviceAllow=/dev/kvm rw
 DeviceAllow=/dev/net/tun rw
+DeviceAllow=/dev/vhost-net rw
 DeviceAllow=char-vfio rw
 DeviceAllow=/dev/vfio/vfio rw
 ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
-# /proc/sys/net/ipv4/conf: per-tap IMDS endpoints set rp_filter/accept_local, which
-# ProtectKernelTunables=yes would otherwise keep read-only.
-ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf
+# /proc/sys/net/ipv4/conf: ProtectKernelTunables would keep the per-tap IMDS
+# rp_filter/accept_local read-only; /etc/spinifex/firewall: the root helper writes peers.nft.
+ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf -/etc/spinifex/firewall
 
 # RG-9: hardening baseline
 ProtectHome=yes

build/systemd/spinifex-firewall.service

@@ -0,0 +1,25 @@
+# spinifex-unit-version: 1
+[Unit]
+Description=Spinifex host firewall (nftables input policy)
+Documentation=file:/etc/spinifex/firewall/spinifex.nft
+Wants=network-pre.target
+Before=network-pre.target shutdown.target
+DefaultDependencies=no
+ConditionPathExists=/etc/spinifex/firewall/peers.nft
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+StandardInput=null
+ProtectSystem=full
+ProtectHome=true
+ExecStart=/usr/local/lib/spinifex/spinifex-firewall-apply
+ExecReload=/usr/local/lib/spinifex/spinifex-firewall-apply
+TimeoutStopSec=10s
+
+# No ExecStop: flushing on stop would take vpcd's MASQUERADE and per-EIP FORWARD
+# rules with it. Remove the policy deliberately with
+# `nft delete table inet spinifex_filter`.
+
+[Install]
+WantedBy=sysinit.target

build/systemd/spinifex-guests.slice

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex guest VM tier
 Before=slices.target

build/systemd/spinifex-nats-watchdog.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 2
 [Unit]
 Description=Spinifex NATS JetStream liveness watchdog
 PartOf=spinifex.target
@@ -15,6 +16,10 @@ ExecStart=/var/lib/spinifex/nats-js-watchdog.sh
 # cutting it off. Only the failing path runs that long; a healthy node exits on
 # the first sample.
 TimeoutStopSec=45
+# Same 34s worst case bounds the start side: without this the unit inherits
+# oneshot's infinite default, so a genuinely hung run would sit in
+# activating(start) forever and could not be retriggered by its timer.
+TimeoutStartSec=120
 
 # Hardening (kept compatible with talking to PID1 over /run/systemd).
 NoNewPrivileges=yes

build/systemd/spinifex-nats-watchdog.timer

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex NATS JetStream liveness watchdog timer
 PartOf=spinifex.target

build/systemd/spinifex-nats.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex NATS Message Bus
 PartOf=spinifex.target

build/systemd/spinifex-northstar.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Northstar (Authoritative DNS)
 PartOf=spinifex.target

build/systemd/spinifex-predastore.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 2
 [Unit]
 Description=Spinifex Predastore (S3-compatible Object Storage)
 PartOf=spinifex.target
@@ -17,9 +18,9 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service predastore start
 Restart=on-failure
 RestartSec=5
-# Matches viperblock's drain budget: viperblock's WAL flush writes into
-# predastore, and After= already stops viperblock first, but this is the
-# backstop if that ordering is ever bypassed (direct stop, future edit).
+# Bounds predastore's own graceful stop. viperblock's WAL flush writes into
+# predastore, but After= already stops viperblock first (within its own 150s
+# budget); this is the backstop if that ordering is ever bypassed.
 TimeoutStopSec=30
 OOMScoreAdjust=-500
 # Legitimately the most memory-hungry infra service (S3 buffering/caching);

build/systemd/spinifex-qmp-collector.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex QMP Telemetry Collector (guest metrics)
 PartOf=spinifex.target

build/systemd/spinifex-shutdown.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex graceful guest drain
 PartOf=spinifex.target
@@ -11,11 +12,11 @@ RemainAfterExit=yes
 ExecStart=/bin/true
 # ExecStop fires on ANY target stop, including the stop half of `systemctl
 # restart spinifex.target` (a rolling deploy) -- PartOf propagates regardless
-# of whether the host is actually going down. --only-if-host-stopping drains
-# only when systemd itself is unwinding into shutdown.target; a plain restart
-# or `systemctl stop spinifex.target` falls through so QEMU/nbdkit survive
-# (KillMode=process) and the daemon reconnects to them on the way back up.
-ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --only-if-host-stopping
+# of whether the host is actually going down. --unless-restarting drains on a
+# real shutdown/reboot and on a plain `systemctl stop spinifex.target`; it
+# skips only when a restart of the stack is already queued, so QEMU/nbdkit
+# survive (KillMode=process) and the daemon reconnects on the way back up.
+ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --unless-restarting
 TimeoutStopSec=180
 
 [Install]

build/systemd/spinifex-system.slice

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex control/infra tier (NATS, predastore, viperblock, daemon, awsgw, vpcd, ui)
 Before=slices.target

build/systemd/spinifex-ui.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Web UI
 PartOf=spinifex.target

build/systemd/spinifex-viperblock.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 2
 [Unit]
 Description=Spinifex Viperblock (Block Storage)
 PartOf=spinifex.target
@@ -23,8 +24,10 @@ MemoryMax=25%
 # way guest QEMU survives a daemon restart. Tearing one out mid-write corrupts
 # that guest's filesystem; idle nbdkit is reaped explicitly on shutdown.
 KillMode=process
-# Bounds this process's own WAL drain, which the plugin caps at 20s.
-TimeoutStopSec=30
+# One worst-case nbdkit reap: killProcessGracePeriod (120s, utils.KillProcess)
+# plus its own shutdownDrainTimeout (20s, viperblock/nbd) and headroom — must
+# strictly exceed 120s now shutdownVolumes reaps concurrently, not serially.
+TimeoutStopSec=150
 EnvironmentFile=/etc/spinifex/systemd.env
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/
@@ -48,7 +51,8 @@ ProtectKernelTunables=yes
 ProtectKernelModules=yes
 ProtectKernelLogs=yes
 ProtectControlGroups=yes
-ProtectProc=invisible
+# ProtectProc must stay default: hidepid=invisible hides spinifex-daemon's
+# QEMU /proc entries (different uid) orphan nbdkit reaping must see.
 SystemCallArchitectures=native
 MemoryDenyWriteExecute=yes
 RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK

build/systemd/spinifex-vpcd.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex VPC Daemon (OVN Networking)
 PartOf=spinifex.target

build/systemd/spinifex.slice

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex resource-bounded slice (host ceiling)
 Before=slices.target

build/systemd/spinifex.target

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Infrastructure Platform
 After=network-online.target

cmd/ecs-agent/ecrresolver.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"net/http"
+	"net/url"
 	"strings"
 	"sync"
 
@@ -60,12 +61,25 @@ func (e *ecrResolver) client() (*http.Client, error) {
 	return c, nil
 }
 
-// isECRRef reports whether ref targets an ECR registry, i.e. its registry host
-// is "<acct>.dkr.ecr.<region>.<domain>". Only ECR pulls need a minted token;
-// public registries (docker.io and friends) pull anonymously, so refs without
-// an ECR host must not drag in IMDS host credentials.
-func isECRRef(ref string) bool {
+// gatewayRegistryHost returns the host[:port] of the co-located ECR registry
+// from the gateway URL; empty/unparseable yields "" (matches no ref). The
+// registry shares the gateway listener, so its port is the gateway port.
+func gatewayRegistryHost(gatewayURL string) string {
+	u, err := url.Parse(gatewayURL)
+	if err != nil {
+		return ""
+	}
+	return u.Host
+}
+
+// isECRRef reports whether ref targets an ECR registry: an "<acct>.dkr.ecr."
+// host, or one equal to gatewayHost (the agent's own co-located registry
+// advertised as a bare host/IP). Other refs pull anonymously, no token.
+func isECRRef(ref, gatewayHost string) bool {
 	host, _, _ := strings.Cut(ref, "/")
+	if gatewayHost != "" && host == gatewayHost {
+		return true
+	}
 	if !strings.Contains(host, ".") && !strings.Contains(host, ":") {
 		return false // no registry host -> docker.io default
 	}
@@ -75,7 +89,7 @@ func isECRRef(ref string) bool {
 // Authorize returns the ECR token's user/password and proxy endpoint for ref.
 // Non-ECR refs resolve to anonymous pull (empty credentials).
 func (e *ecrResolver) Authorize(ctx context.Context, ref string) (user, pass, endpoint string, err error) {
-	if !isECRRef(ref) {
+	if !isECRRef(ref, gatewayRegistryHost(e.gatewayURL)) {
 		return "", "", "", nil
 	}
 	c, err := e.creds.Retrieve(ctx)

cmd/ecs-agent/ecrresolver_test.go

@@ -115,17 +115,44 @@ func TestECRResolver_NonECRAnonymous(t *testing.T) {
 }
 
 func TestIsECRRef(t *testing.T) {
-	cases := map[string]bool{
-		"nginx:alpine":            false,
-		"docker.io/library/nginx": false,
-		"ghcr.io/org/app:1":       false,
-		"registry:5000/app:1":     false,
-		"123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1": true,
-		"123456789012.dkr.ecr.ap-southeast-2.amazonaws.com/x":    true,
+	cases := []struct {
+		ref, gatewayHost string
+		want             bool
+	}{
+		{"nginx:alpine", "192.168.1.13:9999", false},
+		{"docker.io/library/nginx", "192.168.1.13:9999", false},
+		{"ghcr.io/org/app:1", "192.168.1.13:9999", false},
+		{"registry:5000/app:1", "192.168.1.13:9999", false},
+		{"123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1", "192.168.1.13:9999", true},
+		{"123456789012.dkr.ecr.ap-southeast-2.amazonaws.com/x", "192.168.1.13:9999", true},
+		// Co-located gateway registry advertised as a bare host/IP.
+		{"192.168.1.13:9999/bunyip:latest", "192.168.1.13:9999", true},
+		{"192.168.1.13:9999/bunyip:latest", "", false},
 	}
-	for ref, want := range cases {
-		if got := isECRRef(ref); got != want {
-			t.Errorf("isECRRef(%q) = %v, want %v", ref, got, want)
+	for _, c := range cases {
+		if got := isECRRef(c.ref, c.gatewayHost); got != c.want {
+			t.Errorf("isECRRef(%q, %q) = %v, want %v", c.ref, c.gatewayHost, got, c.want)
 		}
 	}
 }
+
+// TestECRResolver_Authorize_BareHostGateway asserts that a ref whose host
+// equals the agent's own gateway host authenticates via the co-located ECR
+// registry, even though it lacks a ".dkr.ecr." name.
+func TestECRResolver_Authorize_BareHostGateway(t *testing.T) {
+	srv := gatewayStub(t, "AWS:jwt", "https://192.168.1.13:9999", http.StatusOK)
+	creds := stubCreds{c: credentials.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "sess",
+		Expiration: time.Now().Add(time.Hour),
+	}}
+	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
+
+	host := strings.TrimPrefix(srv.URL, "https://")
+	user, pass, _, err := r.Authorize(context.Background(), host+"/bunyip:latest")
+	if err != nil {
+		t.Fatalf("Authorize: %v", err)
+	}
+	if user != "AWS" || pass != "jwt" {
+		t.Errorf("got %q:%q, want AWS:jwt", user, pass)
+	}
+}

cmd/installer/firstboot/firstboot.go

@@ -146,6 +146,21 @@ hostnamectl set-hostname %s
 printf "nameserver 1.1.1.1\nnameserver 8.8.8.8\n" > /etc/resolvconf/resolv.conf.d/base
 resolvconf -u
 
+# The RTC on a fresh machine can be badly wrong — a flat CMOS battery is routine
+# on refurbished hardware. Credentials and TLS certificates are minted further
+# down, and a certificate issued against a wrong clock carries a NotBefore that
+# outlives the correction, so the clock has to be right before that happens.
+# chronyd is already running; makestep forces an immediate correction instead of
+# waiting out the slew. Bounded and non-fatal: an air-gapped install must still
+# complete, and rtcsync writes the corrected time back so later boots start close.
+echo "[firstboot] syncing clock before credentials are minted"
+chronyc -a makestep >/dev/null 2>&1 || true
+if chronyc waitsync 12 0.1 0 5 >/dev/null 2>&1; then
+    echo "[firstboot] clock synced: $(date -uIs)"
+else
+    echo "[firstboot] WARNING: clock not synced, continuing with $(date -uIs)"
+fi
+
 # Configure OVN networking.
 # br-wan (and br-lan if present) are Linux bridges managed by systemd-networkd
 # (declared in /etc/systemd/network/ by the installer). setup-ovn.sh
@@ -207,6 +222,12 @@ fi
 
 # Enable services to start, on reboot
 systemctl enable spinifex.target spinifex-banner.service
+# setup.sh enables this on the package install path, but the ISO chroot returns
+# before that line (no systemd to talk to), so the ISO needs it here. An nft
+# ruleset is runtime state and does not survive a reboot; without this the unit
+# never reapplies the policy and the node comes back up open. Enabled, not
+# started: ConditionPathExists holds it inert until the daemon writes a peer file.
+systemctl enable spinifex-firewall.service
 # WantedBy=timers.target only self-activates once enabled — without this the
 # JetStream ENOSPC-latch watchdog never runs and a full disk needs a manual restart.
 systemctl enable --now spinifex-nats-watchdog.timer

cmd/installer/install/blockdev.go

@@ -45,6 +45,7 @@ var (
 	sysClassBlockDir = "/sys/class/block"
 	devByIDDir       = "/dev/disk/by-id"
 	procMountsPath   = "/proc/mounts"
+	procSwapsPath    = "/proc/swaps"
 )
 
 // ListDisks enumerates install-candidate disks from /sys/block. It reads sysfs
@@ -132,6 +133,144 @@ func partitionPath(disk string, n int) string {
 	return fmt.Sprintf("%s%d", disk, n)
 }
 
+// readKernelPartitions returns the partition device names the kernel currently
+// publishes for a disk, sorted.
+//
+// It reads sysfs rather than /dev deliberately. A partition device node
+// outlives the table that created it, so a stale node is indistinguishable
+// from a fresh one by existence alone — which is how a disk whose old table
+// the kernel never dropped passes every check and then gets formatted at the
+// previous layout's offsets.
+func readKernelPartitions(disk string) ([]string, error) {
+	name := filepath.Base(disk)
+	entries, err := os.ReadDir(filepath.Join(sysBlockDir, name))
+	if err != nil {
+		return nil, fmt.Errorf("read the kernel's partition list for %s: %w", disk, err)
+	}
+	var out []string
+	for _, e := range entries {
+		// Partitions are the child directories carrying a "partition" attribute.
+		// "queue", "device" and the plain attribute files sit alongside them.
+		if _, err := os.Stat(filepath.Join(sysBlockDir, name, e.Name(), "partition")); err != nil {
+			continue
+		}
+		out = append(out, e.Name())
+	}
+	slices.Sort(out)
+	return out, nil
+}
+
+// kernelPartitions is indirected so tests can drive the partition-table
+// assertions without a real block device.
+var kernelPartitions = readKernelPartitions
+
+// partitionDevice maps a kernel partition name back to its device node,
+// resolved alongside the disk it belongs to rather than assuming /dev.
+func partitionDevice(disk, part string) string {
+	return filepath.Join(filepath.Dir(disk), part)
+}
+
+// diskHolders names the md, LVM or device-mapper devices claiming this disk or
+// one of its partitions. A holder is the usual reason the kernel refuses to
+// re-read a partition table, so it is what the operator needs told when the
+// installer gives up on a disk.
+func diskHolders(disk string) []string {
+	name := filepath.Base(disk)
+	dirs := []string{filepath.Join(sysBlockDir, name, "holders")}
+	parts, _ := kernelPartitions(disk)
+	for _, p := range parts {
+		dirs = append(dirs, filepath.Join(sysBlockDir, name, p, "holders"))
+	}
+
+	seen := map[string]bool{}
+	var out []string
+	for _, dir := range dirs {
+		entries, err := os.ReadDir(dir)
+		if err != nil {
+			continue
+		}
+		for _, e := range entries {
+			if seen[e.Name()] {
+				continue
+			}
+			seen[e.Name()] = true
+			out = append(out, e.Name())
+		}
+	}
+	slices.Sort(out)
+	return out
+}
+
+// diskMount is one filesystem mounted from a disk the installer is about to
+// erase.
+type diskMount struct {
+	Device     string
+	Mountpoint string
+}
+
+// devicesOnDisk is the set of device paths belonging to a disk: the whole disk
+// and every partition the kernel currently publishes. Matching /proc tables
+// against a set rather than a string prefix keeps /dev/nvme0n1 from claiming
+// /dev/nvme0n11.
+func devicesOnDisk(disk string) map[string]bool {
+	out := map[string]bool{disk: true}
+	parts, _ := kernelPartitions(disk)
+	for _, p := range parts {
+		out[partitionDevice(disk, p)] = true
+	}
+	return out
+}
+
+// mountsOnDisk returns the filesystems mounted from a disk, deepest mountpoint
+// first so a nested mount is released before the one it sits inside.
+func mountsOnDisk(disk string) []diskMount {
+	devs := devicesOnDisk(disk)
+	var out []diskMount
+	for _, fields := range procTable(procMountsPath, 0) {
+		if len(fields) >= 2 && devs[fields[0]] {
+			out = append(out, diskMount{Device: fields[0], Mountpoint: fields[1]})
+		}
+	}
+	slices.SortFunc(out, func(a, b diskMount) int {
+		return strings.Count(b.Mountpoint, "/") - strings.Count(a.Mountpoint, "/")
+	})
+	return out
+}
+
+// swapsOnDisk returns the active swap areas backed by a disk.
+func swapsOnDisk(disk string) []string {
+	devs := devicesOnDisk(disk)
+	var out []string
+	// The first line of /proc/swaps is a column header, not an entry.
+	for _, fields := range procTable(procSwapsPath, 1) {
+		if len(fields) >= 1 && devs[fields[0]] {
+			out = append(out, fields[0])
+		}
+	}
+	return out
+}
+
+// procTable splits a whitespace-delimited /proc table into fields per line,
+// dropping the first skip lines and any blank ones.
+func procTable(path string, skip int) [][]string {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil
+	}
+	var out [][]string
+	n := 0
+	for line := range strings.SplitSeq(string(data), "\n") {
+		n++
+		if n <= skip {
+			continue
+		}
+		if fields := strings.Fields(line); len(fields) > 0 {
+			out = append(out, fields)
+		}
+	}
+	return out
+}
+
 // byIDPreference orders the /dev/disk/by-id/ namespaces we prefer to name a
 // pool member by. wwn- ids are last: they are stable but opaque, so a degraded
 // pool reports a number nobody can map to a drive bay.

cmd/installer/install/blockdev_test.go

@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 	"slices"
 	"strconv"
+	"strings"
 	"testing"
 )
 
@@ -40,6 +41,7 @@ func newFakeSysfs(t *testing.T, disks ...fakeDisk) *fakeSysfs {
 	swap(&sysClassBlockDir, filepath.Join(f.root, "sys/class/block"))
 	swap(&devByIDDir, filepath.Join(f.root, "dev/disk/by-id"))
 	swap(&procMountsPath, filepath.Join(f.root, "proc/mounts"))
+	swap(&procSwapsPath, filepath.Join(f.root, "proc/swaps"))
 
 	f.mkdirAll(sysBlockDir)
 	f.mkdirAll(sysClassBlockDir)
@@ -113,6 +115,18 @@ func (f *fakeSysfs) setMounts(content string) {
 	f.write(procMountsPath, content)
 }
 
+func (f *fakeSysfs) setSwaps(content string) {
+	f.t.Helper()
+	f.write(procSwapsPath, content)
+}
+
+// addHolder marks a partition as claimed by md, LVM or device-mapper — the
+// state that makes the kernel refuse to re-read the disk's partition table.
+func (f *fakeSysfs) addHolder(disk, part, holder string) {
+	f.t.Helper()
+	f.mkdirAll(filepath.Join(sysBlockDir, disk, part, "holders", holder))
+}
+
 func boolAttr(b bool) string {
 	if b {
 		return "1\n"
@@ -352,3 +366,119 @@ func TestHasAnyPrefix(t *testing.T) {
 		t.Error("sda should not match")
 	}
 }
+
+func TestReadKernelPartitionsListsOnlyPartitions(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{
+		name: "nvme0n1", sectors: 1 << 22,
+		partitions: []string{"nvme0n1p3", "nvme0n1p1", "nvme0n1p2"},
+	})
+
+	got, err := readKernelPartitions("/dev/nvme0n1")
+	if err != nil {
+		t.Fatalf("readKernelPartitions: %v", err)
+	}
+	// Sorted, and "queue" and "device" alongside them are not partitions.
+	want := []string{"nvme0n1p1", "nvme0n1p2", "nvme0n1p3"}
+	if !slices.Equal(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+}
+
+func TestReadKernelPartitionsIsEmptyForAnUnpartitionedDisk(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{name: "nvme2n1", sectors: 1 << 22})
+
+	got, err := readKernelPartitions("/dev/nvme2n1")
+	if err != nil {
+		t.Fatalf("readKernelPartitions: %v", err)
+	}
+	if len(got) != 0 {
+		t.Errorf("got %v, want none", got)
+	}
+}
+
+func TestReadKernelPartitionsFailsOnAnUnknownDisk(t *testing.T) {
+	newFakeSysfs(t)
+
+	if _, err := readKernelPartitions("/dev/nvme9n1"); err == nil {
+		t.Fatal("expected an error for a disk with no sysfs entry")
+	}
+}
+
+func TestDiskHoldersNamesEveryClaimant(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{
+		name: "sda", sectors: 1 << 22, partitions: []string{"sda1", "sda2"},
+	})
+	f.addHolder("sda", "sda1", "md0")
+	f.addHolder("sda", "sda2", "dm-0")
+
+	if got, want := diskHolders("/dev/sda"), []string{"dm-0", "md0"}; !slices.Equal(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+}
+
+func TestDiskHoldersIsEmptyWhenNothingClaimsTheDisk(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1"}})
+
+	if got := diskHolders("/dev/sda"); len(got) != 0 {
+		t.Errorf("got %v, want none", got)
+	}
+}
+
+func TestMountsOnDiskFindsOnlyThisDiskDeepestFirst(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1", "sda2"}},
+		fakeDisk{name: "sdb", sectors: 1 << 22, partitions: []string{"sdb1"}},
+	)
+	f.setMounts(strings.Join([]string{
+		"/dev/sda1 /oldroot ext4 rw 0 0",
+		"/dev/sda2 /oldroot/boot/efi vfat rw 0 0",
+		"/dev/sdb1 /other ext4 rw 0 0",
+		"proc /proc proc rw 0 0",
+	}, "\n") + "\n")
+
+	got := mountsOnDisk("/dev/sda")
+	want := []diskMount{
+		{Device: "/dev/sda2", Mountpoint: "/oldroot/boot/efi"},
+		{Device: "/dev/sda1", Mountpoint: "/oldroot"},
+	}
+	if !slices.Equal(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+}
+
+// /dev/nvme0n1 must not claim /dev/nvme0n11, which a string-prefix match would.
+func TestMountsOnDiskDoesNotMatchASiblingNamespace(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "nvme0n1", sectors: 1 << 22, partitions: []string{"nvme0n1p1"}},
+		fakeDisk{name: "nvme0n11", sectors: 1 << 22, partitions: []string{"nvme0n11p1"}},
+	)
+	f.setMounts("/dev/nvme0n11p1 /elsewhere ext4 rw 0 0\n")
+
+	if got := mountsOnDisk("/dev/nvme0n1"); len(got) != 0 {
+		t.Errorf("got %v, want none", got)
+	}
+}
+
+func TestSwapsOnDiskSkipsTheHeaderAndOtherDisks(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1"}},
+		fakeDisk{name: "sdb", sectors: 1 << 22, partitions: []string{"sdb1"}},
+	)
+	f.setSwaps(strings.Join([]string{
+		"Filename\t\t\t\tType\t\tSize\tUsed\tPriority",
+		"/dev/sda1 partition 8388604 0 -2",
+		"/dev/sdb1 partition 8388604 0 -3",
+	}, "\n") + "\n")
+
+	if got, want := swapsOnDisk("/dev/sda"), []string{"/dev/sda1"}; !slices.Equal(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+}
+
+func TestSwapsOnDiskIsEmptyWithoutASwapsTable(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1"}})
+
+	if got := swapsOnDisk("/dev/sda"); len(got) != 0 {
+		t.Errorf("got %v, want none", got)
+	}
+}

cmd/installer/install/disk.go

@@ -4,6 +4,9 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	"path/filepath"
+	"slices"
+	"strings"
 	"time"
 )
 
@@ -47,6 +50,95 @@ const (
 	tailReserveMiB = 1024
 )
 
+// partitionSettleTimeout bounds the wait for the kernel and udev to publish a
+// disk's new partition table. A var so tests do not pay it.
+var partitionSettleTimeout = 15 * time.Second
+
+// releaseDisk detaches whatever the live environment has attached to a disk
+// before it is erased. A mounted filesystem or an active swap area makes
+// BLKRRPART return EBUSY, and the kernel then keeps serving the disk's old
+// partition table while sgdisk writes the new one and reports success.
+func releaseDisk(d Disk) {
+	for _, m := range mountsOnDisk(d.Path) {
+		err := runQuiet("umount", m.Device)
+		if err != nil {
+			// Lazy detach: a busy mount must not stall an install that is about to
+			// erase the filesystem underneath it anyway.
+			err = runQuiet("umount", "-l", m.Device)
+		}
+		if err != nil {
+			slog.Warn("could not unmount before erase", "device", m.Device, "mountpoint", m.Mountpoint, "err", err)
+			continue
+		}
+		slog.Info("released mount before erase", "device", m.Device, "mountpoint", m.Mountpoint)
+	}
+	for _, s := range swapsOnDisk(d.Path) {
+		if err := runQuiet("swapoff", s); err != nil {
+			slog.Warn("could not disable swap before erase", "device", s, "err", err)
+			continue
+		}
+		slog.Info("disabled swap before erase", "device", s)
+	}
+}
+
+// settlePartitions forces the kernel to re-read a disk's partition table and
+// blocks until its view matches want, naming the partitions by number.
+//
+// An empty want asserts the disk has no partitions at all. That is the only
+// state no leftover can satisfy: a disk whose previous table had the same
+// partition numbers as the one being written — an old OS install is p1, p2, p3,
+// exactly what the boot layout writes — passes an "are the expected partitions
+// there" check on stale nodes alone.
+func settlePartitions(d Disk, want ...int) error {
+	if err := runQuiet("partprobe", d.Path); err != nil {
+		// partprobe rescans the whole disk and fails if it cannot open any part of
+		// it; the ioctl on its own sometimes still gets through.
+		if err := runQuiet("blockdev", "--rereadpt", d.Path); err != nil {
+			slog.Warn("could not force a partition table re-read", "disk", d.Path, "err", err)
+		}
+	}
+	_ = runQuiet("udevadm", "settle", "--timeout=10")
+
+	wanted := make([]string, 0, len(want))
+	for _, n := range want {
+		wanted = append(wanted, filepath.Base(d.PartitionPath(n)))
+	}
+	slices.Sort(wanted)
+
+	deadline := time.Now().Add(partitionSettleTimeout)
+	for {
+		got, err := kernelPartitions(d.Path)
+		if err != nil {
+			return err
+		}
+		if slices.Equal(got, wanted) {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			return staleTableError(d, wanted, got)
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+// staleTableError explains a kernel partition view that never caught up. The
+// erase case gets its own wording because it is the dangerous one — proceeding
+// would format the previous layout — and because the holder is usually the
+// whole diagnosis.
+func staleTableError(d Disk, wanted, got []string) error {
+	if len(wanted) == 0 {
+		msg := fmt.Sprintf("%s still shows partitions [%s] after its table was erased — the kernel is using the old table, so formatting would write at the previous layout's offsets",
+			d.Path, strings.Join(got, ", "))
+		if h := diskHolders(d.Path); len(h) > 0 {
+			return fmt.Errorf("%s; it is held by %s, which must be stopped before this disk can be installed to",
+				msg, strings.Join(h, ", "))
+		}
+		return fmt.Errorf("%s; something in the live environment is holding it open", msg)
+	}
+	return fmt.Errorf("%s: the kernel shows partitions [%s] but the new table has [%s] — the table was written but never picked up",
+		d.Path, strings.Join(got, ", "), strings.Join(wanted, ", "))
+}
+
 // espSizeMiB picks the ESP size for a disk.
 func espSizeMiB(d Disk) int {
 	if d.Bytes >= espLargeDiskGiB<<30 {
@@ -98,7 +190,7 @@ func partitionDataDisk(rm RoleMount) error {
 	if err := run("sgdisk", "-G", rm.Disk.Path); err != nil {
 		slog.Warn("could not randomise GPT GUIDs", "disk", rm.Disk.Path, "err", err)
 	}
-	return nil
+	return settlePartitions(rm.Disk, dataPartNum)
 }
 
 // partitionOne writes the GPT for a single disk.
@@ -129,25 +221,15 @@ func partitionOne(d Disk, rootType string) error {
 	if err := run("sgdisk", "-G", d.Path); err != nil {
 		slog.Warn("could not randomise GPT GUIDs", "disk", d.Path, "err", err)
 	}
-	return nil
+	return settlePartitions(d, biosPartNum, espPartNum, rootPartNum)
 }
 
 // waitForPartitions ensures every partition device node the installer is about
-// to use exists. Trixie's udev is slow enough after a table rewrite that mkfs
-// races it and fails with ENOENT on a device the kernel has already accepted.
+// to use exists. The kernel's own view was already asserted per disk as each
+// table was written; udev publishes the /dev nodes and by-id symlinks after
+// that, and Trixie is slow enough about it that mkfs races them.
 func waitForPartitions(cfg DiskConfig) error {
-	for _, d := range cfg.Disks {
-		// Best-effort: partprobe failing is not fatal, since udev may still act
-		// on the BLKRRPART ioctl sgdisk issued itself.
-		if err := run("partprobe", d.Path); err != nil {
-			slog.Warn("partprobe failed, continuing", "disk", d.Path, "err", err)
-		}
-	}
-	if err := run("udevadm", "settle", "--timeout=10"); err != nil {
-		slog.Warn("udevadm settle failed, continuing", "err", err)
-	}
-
-	deadline := time.Now().Add(15 * time.Second)
+	deadline := time.Now().Add(partitionSettleTimeout)
 	for _, d := range cfg.bootDisks() {
 		wanted := []string{d.PartitionPath(espPartNum), d.PartitionPath(rootPartNum)}
 		// The pool is built from by-id paths, so their symlinks have to exist

cmd/installer/install/steps_test.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"slices"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -16,17 +18,69 @@ import (
 type fakeExec struct {
 	calls []string
 	fail  func(name string, args []string) error
+
+	// parts is the kernel's partition view per disk. Seeded by the test and then
+	// moved by the sgdisk calls the installer makes, so the partition-table
+	// assertions can run without a block device.
+	parts map[string][]string
+
+	// stale marks disks whose view never moves however many tables are written
+	// to them: something holds the disk open, BLKRRPART returns EBUSY, and the
+	// kernel keeps serving the old table while sgdisk reports success.
+	stale map[string]bool
+}
+
+// holdOpen makes a disk keep its current partition view forever.
+func (f *fakeExec) holdOpen(disk string) { f.stale[disk] = true }
+
+// seedPartitions gives a disk a pre-existing partition table — the state a
+// drive is in when the node has been installed on before.
+func (f *fakeExec) seedPartitions(disk string, nums ...int) {
+	for _, n := range nums {
+		f.parts[disk] = append(f.parts[disk], filepath.Base(partitionPath(disk, n)))
+	}
+}
+
+// applySgdisk moves the fake kernel view in step with a table rewrite, the way
+// a kernel that honoured every BLKRRPART would: -Z empties it, each -n adds the
+// partition it creates.
+func (f *fakeExec) applySgdisk(args []string) {
+	if len(args) == 0 {
+		return
+	}
+	disk := args[len(args)-1]
+	if f.stale[disk] {
+		return
+	}
+	for i, a := range args {
+		switch {
+		case a == "-Z":
+			f.parts[disk] = nil
+		case a == "-n" && i+1 < len(args):
+			num, _, _ := strings.Cut(args[i+1], ":")
+			n, err := strconv.Atoi(num)
+			if err != nil {
+				continue
+			}
+			f.parts[disk] = append(f.parts[disk], filepath.Base(partitionPath(disk, n)))
+		}
+	}
 }
 
 func fakeCommands(t *testing.T) *fakeExec {
 	t.Helper()
-	f := &fakeExec{}
-	prevRun, prevQuiet, prevEnv := run, runQuiet, runEnv
+	f := &fakeExec{parts: map[string][]string{}, stale: map[string]bool{}}
+	prevRun, prevQuiet, prevEnv, prevParts := run, runQuiet, runEnv, kernelPartitions
 
 	record := func(name string, args ...string) error {
 		f.calls = append(f.calls, strings.TrimSpace(name+" "+strings.Join(args, " ")))
 		if f.fail != nil {
-			return f.fail(name, args)
+			if err := f.fail(name, args); err != nil {
+				return err
+			}
+		}
+		if name == "sgdisk" {
+			f.applySgdisk(args)
 		}
 		return nil
 	}
@@ -35,8 +89,21 @@ func fakeCommands(t *testing.T) *fakeExec {
 	runEnv = func(env []string, name string, args ...string) error {
 		return record(name, append(env, args...)...)
 	}
+	kernelPartitions = func(disk string) ([]string, error) {
+		got := slices.Clone(f.parts[disk])
+		slices.Sort(got)
+		return got, nil
+	}
 
-	t.Cleanup(func() { run, runQuiet, runEnv = prevRun, prevQuiet, prevEnv })
+	// A short settle so a test asserting the stale-table failure does not wait
+	// out the install's real budget.
+	prevTimeout := partitionSettleTimeout
+	partitionSettleTimeout = 10 * time.Millisecond
+
+	t.Cleanup(func() {
+		run, runQuiet, runEnv, kernelPartitions = prevRun, prevQuiet, prevEnv, prevParts
+		partitionSettleTimeout = prevTimeout
+	})
 	return f
 }
 
@@ -114,6 +181,24 @@ func targetSkeleton(t *testing.T, root string) {
 
 // tempDisks builds disks whose device nodes are real files, so the steps that
 // wait for partition nodes to appear can complete.
+// fakeProcTables points the mount and swap readers at fixture tables for the
+// duration of a test.
+func fakeProcTables(t *testing.T, mounts, swaps string) {
+	t.Helper()
+	dir := t.TempDir()
+	point := func(dst *string, name, content string) {
+		prev := *dst
+		p := filepath.Join(dir, name)
+		if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
+			t.Fatalf("write %s: %v", p, err)
+		}
+		*dst = p
+		t.Cleanup(func() { *dst = prev })
+	}
+	point(&procMountsPath, "mounts", mounts)
+	point(&procSwapsPath, "swaps", swaps)
+}
+
 func tempDisks(t *testing.T, n int, gib int64) []Disk {
 	t.Helper()
 	dir := t.TempDir()
@@ -237,21 +322,154 @@ func TestEspSizeMiB(t *testing.T) {
 	}
 }
 
-func TestClearDiskOnlyLabelclearsAnExistingPartition(t *testing.T) {
+// A drive arriving with a previous install's layout must have every partition
+// cleared, not just the one the root filesystem used to be on: a data drive's
+// partition is p1, so a p3-only labelclear leaves a former pool member's ZFS
+// label in place at both ends of the device.
+func TestClearDiskWipesEveryPartitionItFinds(t *testing.T) {
 	f := fakeCommands(t)
-	disks := tempDisks(t, 1, 40)
-	if err := clearDisk(disks[0]); err != nil {
+	d := tempDisks(t, 1, 40)[0]
+	f.seedPartitions(d.Path, biosPartNum, espPartNum, rootPartNum)
+
+	if err := clearDisk(d); err != nil {
 		t.Fatalf("clearDisk: %v", err)
 	}
-	f.mustRun(t, "zpool labelclear -f "+disks[0].PartitionPath(rootPartNum))
+	for _, n := range []int{biosPartNum, espPartNum, rootPartNum} {
+		f.mustRun(t, "zpool labelclear -f "+d.PartitionPath(n))
+		f.mustRun(t, "wipefs -a "+d.PartitionPath(n))
+	}
+	f.mustRun(t, "wipefs -a "+d.Path)
+	f.mustRun(t, "sgdisk -Z "+d.Path)
+}
 
-	f2 := fakeCommands(t)
+func TestClearDiskProbesNoPartitionsOnABlankDrive(t *testing.T) {
+	f := fakeCommands(t)
 	blank := Disk{Path: filepath.Join(t.TempDir(), "sdb")}
+
 	if err := clearDisk(blank); err != nil {
 		t.Fatalf("clearDisk: %v", err)
 	}
-	f2.mustNotRun(t, "labelclear")
-	f2.mustRun(t, "wipefs -a "+blank.Path)
+	f.mustNotRun(t, "labelclear")
+	f.mustRun(t, "wipefs -a "+blank.Path)
+}
+
+// The reported field failure: a drive whose old table the kernel never dropped.
+// Proceeding would format the previous layout's offsets, so the install must
+// stop, and the message must say the table is stale rather than blaming the
+// device node that did not appear.
+func TestClearDiskFailsWhenTheKernelKeepsTheOldTable(t *testing.T) {
+	f := fakeCommands(t)
+	d := tempDisks(t, 1, 40)[0]
+	f.seedPartitions(d.Path, biosPartNum, espPartNum, rootPartNum)
+	// sgdisk writes to the platters and exits 0, but BLKRRPART returned EBUSY,
+	// so the kernel's view never changes.
+	f.holdOpen(d.Path)
+
+	err := clearDisk(d)
+	if err == nil {
+		t.Fatal("clearDisk succeeded against a stale partition table")
+	}
+	for _, want := range []string{d.Path, "still shows partitions", "previous layout"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("err = %q, want it to mention %q", err, want)
+		}
+	}
+}
+
+func TestClearDiskReleasesMountsAndSwapFirst(t *testing.T) {
+	f := fakeCommands(t)
+	d := tempDisks(t, 1, 40)[0]
+	f.seedPartitions(d.Path, biosPartNum, espPartNum, rootPartNum)
+
+	root, esp := d.PartitionPath(rootPartNum), d.PartitionPath(espPartNum)
+	swapDev := d.PartitionPath(biosPartNum)
+	fakeProcTables(t,
+		fmt.Sprintf("%s /oldroot ext4 rw 0 0\n%s /oldroot/boot/efi vfat rw 0 0\n", root, esp),
+		fmt.Sprintf("Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n%s partition 1000 0 -2\n", swapDev),
+	)
+
+	if err := clearDisk(d); err != nil {
+		t.Fatalf("clearDisk: %v", err)
+	}
+	f.mustRun(t, "swapoff "+swapDev)
+
+	// Deepest mountpoint first: unmounting /oldroot while /oldroot/boot/efi is
+	// still attached fails with EBUSY and leaves the disk held.
+	var order []string
+	for _, c := range f.calls {
+		if c == "umount "+root || c == "umount "+esp {
+			order = append(order, c)
+		}
+	}
+	if want := []string{"umount " + esp, "umount " + root}; !slices.Equal(order, want) {
+		t.Errorf("unmount order:\n got %v\nwant %v", order, want)
+	}
+}
+
+// A drive held by md or LVM cannot be released — the ISO ships neither tool —
+// so the operator has to be told what is holding it.
+func TestStaleTableErrorNamesTheHolder(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 20, partitions: []string{"sda1"}})
+	f.addHolder("sda", "sda1", "md0")
+
+	prev := kernelPartitions
+	kernelPartitions = readKernelPartitions
+	t.Cleanup(func() { kernelPartitions = prev })
+
+	err := staleTableError(Disk{Path: "/dev/sda"}, nil, []string{"sda1"})
+	if err == nil || !strings.Contains(err.Error(), "held by md0") {
+		t.Fatalf("err = %v, want it to name the md0 holder", err)
+	}
+}
+
+// The reported field failure as a unit test: two drives arrive carrying a
+// previous install's partitions and the third has never been partitioned. All
+// three must be taken over, and the blank one must not be the only one to get
+// a new table.
+func TestPartitionDisksTakesOverDrivesThatAlreadyHavePartitions(t *testing.T) {
+	f := fakeCommands(t)
+	nodes := tempDisks(t, 3, 200)
+	f.seedPartitions(nodes[0].Path, biosPartNum, espPartNum, rootPartNum) // held an OS
+	f.seedPartitions(nodes[1].Path, dataPartNum)                          // held data
+	// nodes[2] arrives blank.
+
+	cfg := DiskConfig{FS: FSExt4}.WithRoles([]RoleMount{
+		{Role: RoleOS, Disk: nodes[0]},
+		{Role: RoleSpinifex, Disk: nodes[1]},
+		{Role: RolePredastore, Disk: nodes[2]},
+	})
+	if err := partitionDisks(cfg); err != nil {
+		t.Fatalf("partitionDisks: %v", err)
+	}
+	for _, d := range nodes {
+		f.mustRun(t, "sgdisk -Z "+d.Path)
+	}
+	f.mustRun(t, "sgdisk", nodes[0].Path, "-n 3:0:-1024M")
+	for _, d := range nodes[1:] {
+		f.mustRun(t, "sgdisk", d.Path, "-n 1:1M:-1024M", "-t 1:8300")
+	}
+}
+
+// A previous OS install leaves p1, p2, p3 — exactly the numbering the boot
+// layout writes. Asserting the expected partitions exist would pass on those
+// stale nodes and the install would format the old offsets, so the erase-stage
+// assertion on an empty table is what has to catch it.
+func TestPartitionOneStopsWhenAnOldTableHasTheSameLayout(t *testing.T) {
+	f := fakeCommands(t)
+	d := tempDisks(t, 1, 200)[0]
+	f.seedPartitions(d.Path, biosPartNum, espPartNum, rootPartNum)
+	f.holdOpen(d.Path)
+
+	err := partitionOne(d, typeLinuxFS)
+	if err == nil {
+		t.Fatal("partitionOne succeeded against a stale partition table")
+	}
+	// Specifically the erase-stage message: reaching the post-write check would
+	// mean the stale nodes had already been accepted.
+	if !strings.Contains(err.Error(), "still shows partitions") {
+		t.Errorf("err = %q, want the erase-stage stale-table message", err)
+	}
+	f.mustNotRun(t, "sgdisk -n")
 }
 
 func TestFormatESPsLabelsEveryMember(t *testing.T) {

cmd/installer/install/zfs.go

@@ -340,23 +340,35 @@ func exportPool() {
 	}
 }
 
-// clearDisk removes any prior filesystem, partition table and ZFS label from a
-// disk so a retry after a partial install is not rejected for stale metadata —
-// a failure that otherwise reads as mysterious rather than as leftover debris.
+// clearDisk takes a disk over unconditionally, whatever it holds: everything
+// attached to it is released, every signature on it is wiped, its partition
+// table is erased, and the kernel is made to agree the disk is now empty.
+//
+// The last part is the point. sgdisk writes to the platters and exits 0 even
+// when the kernel refuses to drop the old table, so without the assertion an
+// install proceeds against partition nodes describing the previous layout.
 //
 // The probes are quiet because a blank disk makes all of them fail, and their
 // stderr on the install console reads like the install is going wrong.
 func clearDisk(d Disk) error {
-	// ZFS labels live at both ends of the device and wipefs does not know about
-	// them, so an old pool member has to be cleared before its partition goes.
-	if old := d.PartitionPath(rootPartNum); exists(old) {
-		_ = runQuiet("zpool", "labelclear", "-f", old)
-	}
-	_ = runQuiet("wipefs", "-a", d.Path)
-	return runQuiet("sgdisk", "-Z", d.Path)
-}
+	releaseDisk(d)
 
-func exists(path string) bool {
-	_, err := os.Stat(path)
-	return err == nil
+	// Every partition, not just the root slice. ZFS labels live at both ends of
+	// a device and wipefs does not know about them, so a data drive that was
+	// once a pool member keeps its label and fails a later zpool create.
+	parts, err := kernelPartitions(d.Path)
+	if err != nil {
+		return err
+	}
+	for _, p := range parts {
+		dev := partitionDevice(d.Path, p)
+		_ = runQuiet("zpool", "labelclear", "-f", dev)
+		_ = runQuiet("wipefs", "-a", dev)
+	}
+
+	_ = runQuiet("wipefs", "-a", d.Path)
+	if err := runQuiet("sgdisk", "-Z", d.Path); err != nil {
+		return fmt.Errorf("erase partition table: %w", err)
+	}
+	return settlePartitions(d)
 }

cmd/lb-agent/main.go

@@ -18,8 +18,8 @@ import (
 	"os/signal"
 	"syscall"
 
+	"github.com/mulgadc/bluebottle/pkg/otelsetup"
 	"github.com/mulgadc/spinifex/spinifex/lbagent"
-	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 
 	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
@@ -28,7 +28,7 @@ func main() {
 	// Without this the agent falls back to slog's default text handler, which folds
 	// attributes into the message so the console scraper recovers a level and nothing
 	// else. No OTLP export: the agent's only egress is the gateway it is watching.
-	otelsetup.SetDefaultJSONLogger(slog.LevelInfo)
+	otelsetup.SetDefaultJSONLogger("lb-agent", slog.LevelInfo)
 
 	var (
 		lbID       string

cmd/rds-agent/agent.go

@@ -180,7 +180,7 @@ type Agent struct {
 	id              identity
 	cp              controlPlane
 	probe           *engineProbe
-	engine          *postgresEngine
+	engine          engine
 	handoffWriter   func(string, *handlers_rds.GetDBBootstrapConfigOutput) error
 	dataMountWaiter func(context.Context, string, string) error
 
@@ -195,10 +195,7 @@ type Agent struct {
 
 // Assembles from already-built parts, so tests can pass fakes; New builds the
 // production control plane and delegates here.
-func newAgent(cfg config, cp controlPlane, probe *engineProbe) *Agent {
-	if cfg.DataMount == "" {
-		cfg.DataMount = defaultDataMount
-	}
+func newAgent(cfg config, cp controlPlane, probe *engineProbe) (*Agent, error) {
 	if cfg.MountsFile == "" {
 		cfg.MountsFile = defaultMountsFile
 	}
@@ -211,11 +208,15 @@ func newAgent(cfg config, cp controlPlane, probe *engineProbe) *Agent {
 		cfg: cfg, id: id, cp: cp, probe: probe,
 		handoffWriter: writeHandoff, dataMountWaiter: waitForDataMount,
 	}
-	a.engine = newPostgresEngine(cfg, execCommandRunner, execSessionRunner, probe)
+	eng, err := newEngine(cfg, execCommandRunner, execSessionRunner, probe)
+	if err != nil {
+		return nil, err
+	}
+	a.engine = eng
 	a.hb = newHeartbeater(cp, probe, a.engine, handlers_rds.HeartbeatInterval)
 	a.cmd = newCommander(cp, newCommandRegistry(a.engine, newGuestStorage(cfg, execCommandRunner)), cfg.PollWait)
 	a.guard = newParamGuard(a.engine, probe, cp)
-	return a
+	return a, nil
 }
 
 // Does not wait for IMDS: the register loop rides out a datapath still coming up.
@@ -227,14 +228,17 @@ func New(cfg config) (*Agent, error) {
 	if err != nil {
 		return nil, err
 	}
-	return newAgent(cfg, cp, newEngineProbe(cfg, execProbeRunner)), nil
+	probe, err := newProbe(cfg, execProbeRunner)
+	if err != nil {
+		return nil, err
+	}
+	return newAgent(cfg, cp, probe)
 }
 
 func (a *Agent) Run(ctx context.Context) error {
-	// Named before the first dial. When register cannot reach the gateway the
-	// retry loop is all that survives, and the address it is dialing is what
-	// separates a mgmt NIC that came up without an address from a control plane
-	// that is genuinely down.
+	// Named before the first dial: when register cannot reach the gateway, the
+	// address it is dialing is what separates a mgmt NIC that came up without one
+	// from a control plane that is genuinely down.
 	slog.Info("rds-agent: starting",
 		"gatewayURL", a.cfg.GatewayURL, "dbInstanceIdentifier", a.id.DBInstanceIdentifier,
 		"agentVersion", a.id.AgentVersion)
@@ -245,6 +249,13 @@ func (a *Agent) Run(ctx context.Context) error {
 	}
 	a.hb.id, a.cmd.id, a.guard.id = a.id, a.id, a.id
 
+	// Ahead of the handoff, so rds-init never unblocks and no datadir is touched.
+	// An agent running on the wrong image would initialise one engine over
+	// another's data, coming up healthy and empty beside data nothing references.
+	if err := a.refuseEngineMismatch(ctx); err != nil {
+		return err
+	}
+
 	// Beating before the bootstrap keeps a stuck boot visible as a live VM with
 	// a down engine rather than as silence.
 	go a.hb.Run(ctx)
@@ -279,6 +290,26 @@ func (a *Agent) Run(ctx context.Context) error {
 	return nil
 }
 
+// The engine cloud-init says this VM was launched as, against the engine the
+// image bakes. An empty expectation is no assertion rather than a disagreement:
+// a VM launched before the control plane emitted RDS_ENGINE carries none.
+func (a *Agent) refuseEngineMismatch(ctx context.Context) error {
+	expected := strings.ToLower(strings.TrimSpace(a.cfg.Engine))
+	if expected == "" || expected == a.cfg.BakedEngine {
+		return nil
+	}
+
+	message := fmt.Sprintf("this VM was launched as engine %q but its image bakes %q; refusing to bootstrap",
+		expected, a.cfg.BakedEngine)
+	slog.ErrorContext(ctx, "rds-agent: "+message)
+	// Reported before returning, so the control plane learns why the instance
+	// never came up rather than only that it never did.
+	if _, err := a.cp.SubmitState(ctx, a.id, handlers_rds.EngineHealthUnhealthy, message); err != nil {
+		slog.ErrorContext(ctx, "rds-agent: reporting the engine mismatch failed", "err", err)
+	}
+	return errors.New(message)
+}
+
 const dataMountPollInterval = time.Second
 
 // waitForDataMount blocks the stateful agent loops until the kernel reports the

cmd/rds-agent/agent_test.go

@@ -168,23 +168,31 @@ func (f *fakeControlPlane) snapshotReplies() [][]handlers_rds.CommandReply {
 	return out
 }
 
-// testConfig points the agent's file output at a temp dir and its probe at a
-// name no real binary answers to.
+// testConfig stamps the engine a real rds-postgres image bakes, so the agent
+// resolves the same guest layout it would in a guest, and points its file output
+// at a temp dir.
 func testConfig(t *testing.T) config {
 	t.Helper()
-	return config{
-		GatewayURL: "https://gw.test",
-		HandoffDir: filepath.Join(t.TempDir(), "spinifex-rds"),
-		EngineHost: defaultEngineHost,
-		EnginePort: defaultEnginePort,
-		PGIsReady:  "pg_isready",
-		PollWait:   time.Second,
+	cfg := testLoadConfig(t, enginePostgres)
+	cfg.GatewayURL = "https://gw.test"
+	cfg.HandoffDir = filepath.Join(t.TempDir(), "spinifex-rds")
+	cfg.PollWait = time.Second
+	return cfg
+}
+
+// newTestAgent builds the agent over a probe the test drives directly.
+func newTestAgent(t *testing.T, cfg config, cp controlPlane, run probeRunner) *Agent {
+	t.Helper()
+	a, err := newAgent(cfg, cp, newPostgresProbe(cfg, run))
+	if err != nil {
+		t.Fatalf("newAgent: %v", err)
 	}
+	return a
 }
 
 // staticProbe is a probeRunner that always reports the same exit code.
 func staticProbe(code int) probeRunner {
-	return func(context.Context, string, ...string) (int, error) { return code, nil }
+	return func(context.Context, string, ...string) (int, string, error) { return code, "", nil }
 }
 
 func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput {
@@ -272,7 +280,7 @@ func TestRun_RegistersThenDeliversBootstrap(t *testing.T) {
 	cp.bootstrapOut = bootstrapOutput(&password)
 
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	if err := runAgent(t, a); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
@@ -303,7 +311,7 @@ func TestRun_AttachModeNeedsNoPassword(t *testing.T) {
 	cp.bootstrapOut = bootstrapOutput(nil)
 
 	cfg := testConfig(t)
-	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
 
@@ -325,7 +333,7 @@ func TestRun_RetriesRegisterAndBootstrap(t *testing.T) {
 	cp.bootstrapErrs = []error{errors.New("ServerInternal")}
 
 	cfg := testConfig(t)
-	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
 
@@ -344,7 +352,7 @@ func TestBootstrap_RetriesHandoffWithoutRefetching(t *testing.T) {
 	password := "staged-secret"
 	cp.bootstrapOut = bootstrapOutput(&password)
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 
 	writes := 0
 	a.handoffWriter = func(_ string, got *handlers_rds.GetDBBootstrapConfigOutput) error {
@@ -385,7 +393,7 @@ func TestBootstrap_RetriesInitializeWithNoPassword(t *testing.T) {
 	}}
 	cp.bootstrapOut = bootstrapOutput(&password)
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 
 	var handed []*handlers_rds.GetDBBootstrapConfigOutput
 	a.handoffWriter = func(_ string, got *handlers_rds.GetDBBootstrapConfigOutput) error {
@@ -408,7 +416,7 @@ func TestAcknowledgeBootstrap_WaitsForTheReceiptThenRetiresIt(t *testing.T) {
 	cp := newFakeControlPlane()
 	cfg := testConfig(t)
 	cfg.DataMount = t.TempDir()
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	a.id = identity{DBInstanceIdentifier: "db-resolved"}
 	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1, dataVolumeID: "vol-data-01"}
 
@@ -447,7 +455,7 @@ func TestAcknowledgeBootstrap_IgnoresAForeignReceipt(t *testing.T) {
 			cp := newFakeControlPlane()
 			cfg := testConfig(t)
 			cfg.DataMount = t.TempDir()
-			a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+			a := newTestAgent(t, cfg, cp, staticProbe(0))
 			a.id = identity{DBInstanceIdentifier: "db-resolved"}
 			a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
 			writeReceipt(t, a, tc.payloadID, tc.dbID)
@@ -470,7 +478,7 @@ func TestAcknowledgeBootstrap_IsANoOpOnAttach(t *testing.T) {
 	cp := newFakeControlPlane()
 	cfg := testConfig(t)
 	cfg.DataMount = t.TempDir()
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 
 	if err := a.acknowledgeBootstrap(t.Context()); err != nil {
 		t.Fatalf("acknowledgeBootstrap: %v", err)
@@ -480,6 +488,52 @@ func TestAcknowledgeBootstrap_IsANoOpOnAttach(t *testing.T) {
 	}
 }
 
+// The image is the authority on which engine the agent runs. A VM launched as
+// another engine has to stop ahead of the handoff: rds-init would otherwise
+// initialise one engine over the other's datadir and come up healthy and empty
+// beside data nothing references.
+func TestRun_RefusesAnEngineTheImageDoesNotBake(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+
+	cfg := testConfig(t)
+	cfg.Engine = "mariadb"
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+
+	err := a.Run(t.Context())
+	if err == nil {
+		t.Fatal("Run returned nil for a VM launched as an engine this image does not bake")
+	}
+	if !strings.Contains(err.Error(), "mariadb") {
+		t.Errorf("error = %v, want it to name the engine the VM was launched as", err)
+	}
+	if _, statErr := os.Stat(filepath.Join(cfg.HandoffDir, handoffEnvFile)); !os.IsNotExist(statErr) {
+		t.Error("the agent wrote a bootstrap handoff for an engine it refused to run")
+	}
+	if n := len(cp.bootstrapReqs); n != 0 {
+		t.Errorf("bootstrap fetches = %d, want none before the refusal", n)
+	}
+	// Reported rather than only logged, so the control plane learns why the
+	// instance never came up rather than only that it never did.
+	states := cp.snapshotStates()
+	if len(states) != 1 || states[0].health != handlers_rds.EngineHealthUnhealthy {
+		t.Fatalf("submitted states = %+v, want a single unhealthy report", states)
+	}
+}
+
+// The same assertion when it agrees is not a gate: the boot goes on exactly as
+// it does for a VM launched before the control plane sent one at all.
+func TestRun_ProceedsWhenTheDeliveredEngineMatchesTheImage(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+
+	cfg := testConfig(t)
+	cfg.Engine = enginePostgres
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+}
+
 // A configured identifier is asserted on the wire so a mis-provisioned VM is
 // rejected by the gateway instead of adopting whatever it resolves to.
 func TestRun_SendsConfiguredIdentifier(t *testing.T) {
@@ -489,7 +543,7 @@ func TestRun_SendsConfiguredIdentifier(t *testing.T) {
 	cfg := testConfig(t)
 	cfg.DBInstanceIdentifier = "db-configured"
 	cfg.EngineVersion = "18.1"
-	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
 
@@ -503,12 +557,12 @@ func TestRun_WaitsForDataMountBeforePollingCommands(t *testing.T) {
 	cp := newFakeControlPlane()
 	cp.bootstrapOut = bootstrapOutput(nil)
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(2)))
+	a := newTestAgent(t, cfg, cp, staticProbe(2))
 
 	waiting := make(chan struct{})
 	release := make(chan struct{})
 	a.dataMountWaiter = func(ctx context.Context, mountsFile, target string) error {
-		if mountsFile != defaultMountsFile || target != defaultDataMount {
+		if mountsFile != defaultMountsFile || target != cfg.DataMount {
 			t.Errorf("mount waiter got %q/%q, want configured defaults", mountsFile, target)
 		}
 		close(waiting)
@@ -580,7 +634,7 @@ func TestRun_HeartbeatIncludesBootstrapFailure(t *testing.T) {
 	cp.registerOut.HeartbeatIntervalSeconds = 0
 
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(2)))
+	a := newTestAgent(t, cfg, cp, staticProbe(2))
 	a.hb.interval = 10 * time.Millisecond
 
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
@@ -609,7 +663,7 @@ func TestRun_HeartbeatsEngineHealth(t *testing.T) {
 	cp.registerOut.HeartbeatIntervalSeconds = 0
 
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	a.hb.interval = 10 * time.Millisecond
 
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
@@ -639,7 +693,7 @@ func TestRegister_AdoptsControlPlaneCadence(t *testing.T) {
 	cp.registerOut.HeartbeatIntervalSeconds = 45
 
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	if err := runAgent(t, a); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
@@ -656,12 +710,15 @@ func TestRun_HeartbeatReportsDownEngine(t *testing.T) {
 	cp.registerOut.HeartbeatIntervalSeconds = 0
 
 	cfg := testConfig(t)
-	probe := newEngineProbe(cfg, staticProbe(2))
+	probe := newPostgresProbe(cfg, staticProbe(2))
 	// The engine has served before, so a silent one now is a failure rather
 	// than a boot still in progress.
 	probe.seenHealthy = true
 
-	a := newAgent(cfg, cp, probe)
+	a, err := newAgent(cfg, cp, probe)
+	if err != nil {
+		t.Fatalf("newAgent: %v", err)
+	}
 	a.hb.interval = 10 * time.Millisecond
 
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
@@ -752,7 +809,7 @@ func TestAcknowledgeBootstrap_StopsOnATerminalDenial(t *testing.T) {
 	}}
 	cfg := testConfig(t)
 	cfg.DataMount = t.TempDir()
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	a.id = identity{DBInstanceIdentifier: "db-resolved"}
 	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
 	writeReceipt(t, a, testPayloadID, "db-resolved")
@@ -780,7 +837,7 @@ func TestAcknowledgeBootstrap_DoesNotReportAnAbsentReceiptAsAFailure(t *testing.
 	cp := newFakeControlPlane()
 	cfg := testConfig(t)
 	cfg.DataMount = t.TempDir()
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	a.id = identity{DBInstanceIdentifier: "db-resolved"}
 	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
 

cmd/rds-agent/bootstrap.go

@@ -32,10 +32,9 @@ func (a *Agent) bootstrap(ctx context.Context) error {
 		if fetched == nil {
 			return fmt.Errorf("bootstrap fetch returned no config")
 		}
-		// A control plane that predates the encrypted payload answers initialize
-		// with nothing to initialise with. Retrying re-dials the worker queue
-		// group until an upgraded node answers, so a mixed-version fleet heals
-		// within one boot rather than needing a reboot.
+		// A control plane predating the encrypted payload answers initialize with
+		// nothing to initialise with. Retrying re-dials the queue group until an
+		// upgraded node answers, so a mixed-version fleet heals within one boot.
 		if fetched.Mode == handlers_rds.BootstrapModeInitialize &&
 			(fetched.MasterUserPassword == nil || *fetched.MasterUserPassword == "") {
 			return fmt.Errorf("bootstrap fetch returned mode=%s with no master password", fetched.Mode)
@@ -61,7 +60,6 @@ func (a *Agent) bootstrap(ctx context.Context) error {
 	a.hb.clearBootstrapFailure()
 	if cfg.Port > 0 {
 		a.probe.setPort(int(cfg.Port))
-		a.engine.setPort(int(cfg.Port))
 	}
 	if cfg.BootstrapPending && cfg.PayloadID != "" {
 		a.pending = &pendingBootstrap{
@@ -107,7 +105,11 @@ func writeHandoff(dir string, cfg *handlers_rds.GetDBBootstrapConfigOutput) erro
 		return fmt.Errorf("secure handoff dir %s: %w", dir, err)
 	}
 
-	if err := writeHandoffFile(dir, handoffParamsFile, renderParameters(cfg.Parameters)); err != nil {
+	params, err := renderParameters(cfg.Engine, cfg.Parameters)
+	if err != nil {
+		return err
+	}
+	if err := writeHandoffFile(dir, handoffParamsFile, params); err != nil {
 		return err
 	}
 	// Cert and key are written as a pair; half of one would start the engine
@@ -177,18 +179,23 @@ func shellQuote(s string) string {
 	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
 }
 
-// postgresql.conf syntax. Values are quoted so a setting with a space or unit
-// suffix survives; the engine accepts quoted numerics and booleans too.
-func renderParameters(params []handlers_rds.Parameter) string {
+// postgresql.conf syntax, which MariaDB's option files also parse. Values are
+// quoted so a space or unit suffix survives. Names are the engine's startup
+// spellings, not the customer's: the wrong one leaves the engine unable to boot.
+func renderParameters(engineName string, params []handlers_rds.Parameter) (string, error) {
+	engine, err := handlers_rds.LookupEngine(engineName)
+	if err != nil {
+		return "", fmt.Errorf("render the resolved parameters: %w", err)
+	}
 	var b strings.Builder
 	b.WriteString("# Resolved parameter group, written by rds-agent.\n")
 	for _, p := range params {
 		if p.Name == "" {
 			continue
 		}
-		fmt.Fprintf(&b, "%s = '%s'\n", p.Name, strings.ReplaceAll(p.Value, "'", "''"))
+		fmt.Fprintf(&b, "%s = '%s'\n", engine.OptionFileName(p.Name), strings.ReplaceAll(p.Value, "'", "''"))
 	}
-	return b.String()
+	return b.String(), nil
 }
 
 // Atomic: a temp file in the same directory, renamed over the target. The mode

cmd/rds-agent/bootstrap_test.go

@@ -16,6 +16,7 @@ func TestWriteHandoff_WritesEveryFileRootOnly(t *testing.T) {
 
 	err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:               handlers_rds.BootstrapModeInitialize,
+		Engine:             "postgres",
 		MasterUsername:     "master",
 		MasterUserPassword: &password,
 		DBName:             "appdb",
@@ -82,6 +83,7 @@ func TestWriteHandoff_AttachOmitsPassword(t *testing.T) {
 	dir := filepath.Join(t.TempDir(), "handoff")
 	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:           handlers_rds.BootstrapModeAttach,
+		Engine:         "postgres",
 		MasterUsername: "master",
 		Port:           5432,
 	}); err != nil {
@@ -112,6 +114,7 @@ func TestWriteHandoff_QuotesShellMetacharacters(t *testing.T) {
 
 	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:               handlers_rds.BootstrapModeInitialize,
+		Engine:             "postgres",
 		MasterUsername:     "master",
 		MasterUserPassword: &password,
 		Port:               5432,
@@ -132,6 +135,7 @@ func TestWriteHandoff_SkipsHalfATLSPair(t *testing.T) {
 	dir := filepath.Join(t.TempDir(), "handoff")
 	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:               handlers_rds.BootstrapModeAttach,
+		Engine:             "postgres",
 		MasterUsername:     "master",
 		Port:               5432,
 		ServingCertificate: "CERT",
@@ -149,8 +153,9 @@ func TestWriteHandoff_SkipsHalfATLSPair(t *testing.T) {
 func TestWriteHandoff_RejectsConfigWithNoMasterUser(t *testing.T) {
 	dir := filepath.Join(t.TempDir(), "handoff")
 	err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
-		Mode: handlers_rds.BootstrapModeAttach,
-		Port: 5432,
+		Mode:   handlers_rds.BootstrapModeAttach,
+		Engine: "postgres",
+		Port:   5432,
 	})
 	if err == nil {
 		t.Fatal("writeHandoff accepted a config with no master username, want an error")
@@ -170,6 +175,7 @@ func TestWriteHandoff_TightensExistingDirectory(t *testing.T) {
 
 	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:           handlers_rds.BootstrapModeAttach,
+		Engine:         "postgres",
 		MasterUsername: "master",
 		Port:           5432,
 	}); err != nil {
@@ -219,3 +225,46 @@ func findLine(t *testing.T, body, prefix string) string {
 	t.Fatalf("no line starting %q in:\n%s", prefix, body)
 	return ""
 }
+
+// The include rds-init installs is what the engine starts on, so the spelling
+// written here is the one that has to boot.
+func TestRenderParameters_UsesTheEngineStartupSpelling(t *testing.T) {
+	params := []handlers_rds.Parameter{
+		{Name: "time_zone", Value: "SYSTEM"},
+		{Name: "max_connections", Value: "85"},
+	}
+
+	rendered, err := renderParameters("mariadb", params)
+	if err != nil {
+		t.Fatalf("renderParameters: %v", err)
+	}
+	if !strings.Contains(rendered, "default_time_zone = 'SYSTEM'\n") {
+		t.Errorf("time_zone was not written under its startup spelling:\n%s", rendered)
+	}
+	if strings.Contains(rendered, "\ntime_zone = ") {
+		t.Errorf("time_zone reached the option file, which mariadbd refuses:\n%s", rendered)
+	}
+	if !strings.Contains(rendered, "max_connections = '85'\n") {
+		t.Errorf("an unmapped parameter was not written unchanged:\n%s", rendered)
+	}
+}
+
+func TestRenderParameters_PostgresKeepsEveryName(t *testing.T) {
+	rendered, err := renderParameters("postgres", []handlers_rds.Parameter{
+		{Name: "shared_buffers", Value: "262144"},
+	})
+	if err != nil {
+		t.Fatalf("renderParameters: %v", err)
+	}
+	if !strings.Contains(rendered, "shared_buffers = '262144'\n") {
+		t.Errorf("postgres name was rewritten:\n%s", rendered)
+	}
+}
+
+// Failing closed beats writing a file the engine may refuse: the bad include
+// would already be on the data volume by the time the server rejected it.
+func TestRenderParameters_RefusesAnUnknownEngine(t *testing.T) {
+	if _, err := renderParameters("", nil); err == nil {
+		t.Fatal("renderParameters accepted a config naming no engine, want an error")
+	}
+}

cmd/rds-agent/config.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/mulgadc/spinifex/internal/guestenv"
@@ -12,17 +14,11 @@ const (
 	defaultGatewayCA  = "/etc/spinifex-rds/gateway-ca.pem"
 	defaultHandoffDir = "/run/spinifex-rds"
 	defaultEngineHost = "127.0.0.1"
-	defaultEnginePort = 5432
-	defaultPGIsReady  = "pg_isready"
-
-	// The guest layout rds-init lays down. Overridable for the same reason it is
-	// there: a sibling engine preset points at its own paths.
-	defaultPGBin         = "/usr/libexec/postgresql18"
-	defaultPGData        = "/var/lib/postgresql/18/data"
-	defaultSocketDir     = "/run/postgresql"
-	defaultPGUser        = "postgres"
-	defaultRCService     = "rc-service"
-	defaultEngineSvcName = "postgresql"
+	// Where setup.sh stamps the engine the image bakes. The agent builds its
+	// engine implementation from this file rather than from anything delivered.
+	defaultEngineFile         = "/etc/spinifex-rds/engine"
+	defaultRCService          = "rc-service"
+	defaultEngineProbeTimeout = 5 * time.Second
 	// The long-poll window the agent asks the gateway to hold a request open
 	// for. The gateway caps it at 20s.
 	defaultPollWait = 20 * time.Second
@@ -44,19 +40,33 @@ type config struct {
 	HandoffDir    string
 	EngineHost    string
 	EnginePort    int
-	// Overridable so a test or a sibling engine preset can point at its own.
-	PGIsReady string
-	PollWait  time.Duration
+	PollWait      time.Duration
+
+	// The engine the control plane launched this VM as, delivered by cloud-init,
+	// and the engine the image itself bakes. Two independent statements of the
+	// same fact: the agent refuses to bootstrap when they disagree.
+	Engine      string
+	EngineFile  string
+	BakedEngine string
 
 	// Where the command handlers reach the engine: the client binaries, the
 	// datadir the parameter file is installed into, the socket they connect
 	// over, the OS user they drop to, and the service the stop goes through.
-	PGBin         string
-	PGData        string
+	EngineBinDir  string
+	EngineDataDir string
 	SocketDir     string
-	PGUser        string
+	EngineUser    string
 	RCService     string
 	EngineService string
+	// Where the engine records the pid its probe checks for liveness, for an
+	// engine whose own client cannot tell a server still recovering from one that
+	// is not running at all.
+	EnginePidFile      string
+	EngineProbeTimeout time.Duration
+	// Where the engine records why it would not start, which the probe quotes
+	// when the server is up but not serving. Overridable so a test can point it
+	// at a fixture.
+	EngineErrorLog string
 
 	// Where the data volume is mounted, and the two kernel surfaces a storage
 	// grow resolves its device from. Overridable so a test can point them at
@@ -75,19 +85,22 @@ func loadConfig(envFile string) config {
 		GatewayCA:            get("RDS_GATEWAY_CA"),
 		Region:               get("RDS_REGION"),
 		DBInstanceIdentifier: get("RDS_DB_INSTANCE_IDENTIFIER"),
+		Engine:               get("RDS_ENGINE"),
+		EngineFile:           get("RDS_ENGINE_FILE"),
 		EngineVersion:        get("RDS_ENGINE_VERSION"),
 		HandoffDir:           get("RDS_HANDOFF_DIR"),
 		EngineHost:           get("RDS_ENGINE_HOST"),
-		PGIsReady:            get("RDS_PG_ISREADY"),
-		PGBin:                get("RDS_PG_BIN"),
-		PGData:               get("RDS_PGDATA"),
+		EngineBinDir:         get("RDS_ENGINE_BIN"),
+		EngineDataDir:        get("RDS_ENGINE_DATA"),
 		SocketDir:            get("RDS_SOCKET_DIR"),
-		PGUser:               get("RDS_PG_USER"),
+		EngineUser:           get("RDS_ENGINE_USER"),
 		RCService:            get("RDS_RC_SERVICE"),
 		EngineService:        get("RDS_ENGINE_SERVICE"),
+		EnginePidFile:        get("RDS_ENGINE_PIDFILE"),
+		EngineErrorLog:       get("RDS_ENGINE_LOG"),
 		DataMount:            get("RDS_DATA_MOUNT"),
-		EnginePort:           defaultEnginePort,
 		PollWait:             defaultPollWait,
+		EngineProbeTimeout:   defaultEngineProbeTimeout,
 	}
 	if cfg.GatewayCA == "" {
 		cfg.GatewayCA = defaultGatewayCA
@@ -98,32 +111,20 @@ func loadConfig(envFile string) config {
 	if cfg.EngineHost == "" {
 		cfg.EngineHost = defaultEngineHost
 	}
-	if cfg.PGIsReady == "" {
-		cfg.PGIsReady = defaultPGIsReady
-	}
-	if cfg.PGBin == "" {
-		cfg.PGBin = defaultPGBin
-	}
-	if cfg.PGData == "" {
-		cfg.PGData = defaultPGData
-	}
-	if cfg.SocketDir == "" {
-		cfg.SocketDir = defaultSocketDir
-	}
-	if cfg.PGUser == "" {
-		cfg.PGUser = defaultPGUser
+	if cfg.EngineFile == "" {
+		cfg.EngineFile = defaultEngineFile
 	}
 	if cfg.RCService == "" {
 		cfg.RCService = defaultRCService
 	}
-	if cfg.EngineService == "" {
-		cfg.EngineService = defaultEngineSvcName
-	}
+	// The image's own stamp rather than the delivered engine: an agent that took
+	// its layout from the payload could run one engine's implementation against
+	// another engine's datadir, which is the failure the stamp exists to prevent.
+	cfg.BakedEngine = readBakedEngine(cfg.EngineFile)
+	cfg.applyLayout(engineLayouts[cfg.BakedEngine])
+
 	// rds-datadir reads the same RDS_DATA_MOUNT, so the two agree on where the
 	// volume landed without either asserting it to the other.
-	if cfg.DataMount == "" {
-		cfg.DataMount = defaultDataMount
-	}
 	cfg.MountsFile = defaultMountsFile
 	cfg.SysBlock = defaultSysBlock
 	// The authoritative port comes from the bootstrap config; this is only what
@@ -138,5 +139,53 @@ func loadConfig(envFile string) config {
 			cfg.PollWait = d
 		}
 	}
+	if v := get("RDS_ENGINE_PROBE_TIMEOUT"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.EngineProbeTimeout = d
+		}
+	}
 	return cfg
 }
+
+// Fills in whatever the delivered configuration left unset. An override always
+// wins, so a test can point any of these at a fixture. An unrecognised engine
+// leaves the layout empty, and New refuses rather than guessing at one.
+func (c *config) applyLayout(layout engineLayout) {
+	if c.EngineBinDir == "" {
+		c.EngineBinDir = layout.binDir
+	}
+	if c.EngineDataDir == "" {
+		c.EngineDataDir = layout.dataDir
+	}
+	if c.SocketDir == "" {
+		c.SocketDir = layout.socketDir
+	}
+	if c.EngineUser == "" {
+		c.EngineUser = layout.osUser
+	}
+	if c.EngineService == "" {
+		c.EngineService = layout.service
+	}
+	if c.EnginePidFile == "" {
+		c.EnginePidFile = layout.pidFile
+	}
+	if c.EngineErrorLog == "" {
+		c.EngineErrorLog = layout.errorLog
+	}
+	if c.DataMount == "" {
+		c.DataMount = layout.dataMount
+	}
+	if c.EnginePort == 0 {
+		c.EnginePort = layout.port
+	}
+}
+
+// The engine the image bakes. An absent or unreadable stamp reads as empty,
+// which New refuses: an agent that guessed would defeat the stamp.
+func readBakedEngine(path string) string {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return ""
+	}
+	return strings.ToLower(strings.TrimSpace(string(raw)))
+}

cmd/rds-agent/config_test.go

@@ -3,21 +3,34 @@ package main
 import (
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 )
 
+// Stamps the engine a real image bakes and loads the configuration against it,
+// which is what selects the guest layout the agent's defaults come from.
+func testLoadConfig(t *testing.T, engine string) config {
+	t.Helper()
+	stamp := filepath.Join(t.TempDir(), "engine")
+	if err := os.WriteFile(stamp, []byte(engine+"\n"), 0o444); err != nil {
+		t.Fatalf("write the engine stamp: %v", err)
+	}
+	t.Setenv("RDS_ENGINE_FILE", stamp)
+	return loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+}
+
 func TestLoadConfig_Defaults(t *testing.T) {
-	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+	cfg := testLoadConfig(t, enginePostgres)
 
 	if cfg.GatewayCA != defaultGatewayCA || cfg.HandoffDir != defaultHandoffDir {
 		t.Errorf("CA/handoff = %q/%q, want the built-in defaults", cfg.GatewayCA, cfg.HandoffDir)
 	}
-	if cfg.EngineHost != defaultEngineHost || cfg.EnginePort != defaultEnginePort {
-		t.Errorf("engine endpoint = %s:%d, want the local default", cfg.EngineHost, cfg.EnginePort)
+	if cfg.EngineHost != defaultEngineHost || cfg.PollWait != defaultPollWait {
+		t.Errorf("engine host/pollWait = %q/%v, want the built-in defaults", cfg.EngineHost, cfg.PollWait)
 	}
-	if cfg.PGIsReady != defaultPGIsReady || cfg.PollWait != defaultPollWait {
-		t.Errorf("probe/pollWait = %q/%v, want the built-in defaults", cfg.PGIsReady, cfg.PollWait)
+	if cfg.EngineProbeTimeout != defaultEngineProbeTimeout {
+		t.Errorf("engine probe timeout = %v, want %v", cfg.EngineProbeTimeout, defaultEngineProbeTimeout)
 	}
 	// Identity is optional: the gateway resolves it from the caller's
 	// credentials, so an agent with no configured identifier is normal.
@@ -26,11 +39,94 @@ func TestLoadConfig_Defaults(t *testing.T) {
 	}
 }
 
+// The guest layout is the image's to state, not the control plane's: everything
+// the agent reaches the engine through comes from the stamp it was built with.
+func TestLoadConfig_TakesTheLayoutFromTheBakedEngine(t *testing.T) {
+	cfg := testLoadConfig(t, enginePostgres)
+
+	if cfg.BakedEngine != enginePostgres {
+		t.Fatalf("baked engine = %q, want %q", cfg.BakedEngine, enginePostgres)
+	}
+	layout := engineLayouts[enginePostgres]
+	if cfg.EngineBinDir != layout.binDir || cfg.EngineDataDir != layout.dataDir {
+		t.Errorf("bin/data = %q/%q, want the PostgreSQL layout", cfg.EngineBinDir, cfg.EngineDataDir)
+	}
+	if cfg.EngineUser != layout.osUser || cfg.EngineService != layout.service {
+		t.Errorf("user/service = %q/%q, want the PostgreSQL layout", cfg.EngineUser, cfg.EngineService)
+	}
+	if cfg.SocketDir != layout.socketDir || cfg.DataMount != layout.dataMount {
+		t.Errorf("socket/mount = %q/%q, want the PostgreSQL layout", cfg.SocketDir, cfg.DataMount)
+	}
+	if cfg.EnginePort != layout.port {
+		t.Errorf("port = %d, want the PostgreSQL default %d", cfg.EnginePort, layout.port)
+	}
+}
+
+// An image with no stamp cannot say which engine it is, and an agent that
+// guessed would be the failure the stamp exists to prevent.
+func TestNewEngine_RefusesAnImageWithNoEngineStamp(t *testing.T) {
+	t.Setenv("RDS_ENGINE_FILE", filepath.Join(t.TempDir(), "absent-engine"))
+	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+
+	if _, err := newProbe(cfg, staticProbe(0)); err == nil {
+		t.Error("newProbe built a probe for an image carrying no engine stamp")
+	}
+	if _, err := newEngine(cfg, nil, nil, nil); err == nil {
+		t.Error("newEngine built an engine for an image carrying no engine stamp")
+	}
+}
+
+func TestNewEngine_RefusesAnEngineItDoesNotImplement(t *testing.T) {
+	cfg := testLoadConfig(t, "oracle")
+
+	_, err := newEngine(cfg, nil, nil, nil)
+	if err == nil {
+		t.Fatal("newEngine built an engine this agent does not implement")
+	}
+	if !strings.Contains(err.Error(), "oracle") {
+		t.Errorf("error = %v, want it to name the engine the image bakes", err)
+	}
+}
+
+// The implementation is the agent's, but the definition it validates names and
+// classifies parameters against is the control plane's, which now offers MariaDB.
+func TestNewEngine_BuildsMariaDBForAMariaDBImage(t *testing.T) {
+	cfg := testLoadConfig(t, engineMariaDB)
+	probe, err := newProbe(cfg, staticProbe(0))
+	if err != nil {
+		t.Fatalf("newProbe: %v", err)
+	}
+
+	built, err := newEngine(cfg, nil, nil, probe)
+	if err != nil {
+		t.Fatalf("newEngine: %v", err)
+	}
+	if _, ok := built.(*mariadbEngine); !ok {
+		t.Errorf("newEngine built %T for a MariaDB image", built)
+	}
+}
+
+// The probe reads the pidfile Alpine's packaged service starts mariadbd with,
+// because nothing in the generated configuration can move it: the service passes
+// --pid-file on the command line, which beats an option file.
+func TestLoadConfig_TakesTheMariaDBPidfileFromTheLayout(t *testing.T) {
+	cfg := testLoadConfig(t, engineMariaDB)
+
+	if cfg.EnginePidFile != engineLayouts[engineMariaDB].pidFile {
+		t.Errorf("pidfile = %q, want the MariaDB layout's", cfg.EnginePidFile)
+	}
+	if cfg.EnginePidFile == "" {
+		t.Error("an engine whose probe needs a pidfile was given none")
+	}
+}
+
 func TestLoadConfig_ReadsEnvFile(t *testing.T) {
+	t.Setenv("RDS_ENGINE_FILE", filepath.Join(t.TempDir(), "absent-engine"))
 	path := filepath.Join(t.TempDir(), "agent.env")
 	body := "RDS_GATEWAY_URL=https://gw.internal:9999\n" +
 		"RDS_REGION=ap-southeast-2\n" +
 		"RDS_DB_INSTANCE_IDENTIFIER=db-1\n" +
+		"RDS_ENGINE=postgres\n" +
 		"RDS_ENGINE_VERSION=18.1\n" +
 		"RDS_ENGINE_PORT=6543\n" +
 		"RDS_POLL_WAIT=5s\n"
@@ -45,19 +141,22 @@ func TestLoadConfig_ReadsEnvFile(t *testing.T) {
 	if cfg.DBInstanceIdentifier != "db-1" || cfg.EngineVersion != "18.1" {
 		t.Errorf("identity = %q/%q, want db-1/18.1", cfg.DBInstanceIdentifier, cfg.EngineVersion)
 	}
+	if cfg.Engine != enginePostgres {
+		t.Errorf("delivered engine = %q, want the one cloud-init carried", cfg.Engine)
+	}
 	if cfg.EnginePort != 6543 || cfg.PollWait != 5*time.Second {
 		t.Errorf("port/pollWait = %d/%v, want 6543/5s", cfg.EnginePort, cfg.PollWait)
 	}
 }
 
-// An unparseable override falls back to the default rather than leaving the
-// agent with a zero port or a zero-length long poll.
+// An unparseable override falls back to the layout's own default rather than
+// leaving the agent with a zero port or a zero-length long poll.
 func TestLoadConfig_IgnoresUnusableOverrides(t *testing.T) {
 	t.Setenv("RDS_ENGINE_PORT", "not-a-port")
 	t.Setenv("RDS_POLL_WAIT", "forever")
 
-	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
-	if cfg.EnginePort != defaultEnginePort || cfg.PollWait != defaultPollWait {
+	cfg := testLoadConfig(t, enginePostgres)
+	if cfg.EnginePort != engineLayouts[enginePostgres].port || cfg.PollWait != defaultPollWait {
 		t.Errorf("port/pollWait = %d/%v, want the defaults", cfg.EnginePort, cfg.PollWait)
 	}
 }

cmd/rds-agent/defaultparams_test.go

@@ -0,0 +1,73 @@
+package main
+
+//test:in-package — the agent is a main package, which has no external test
+// package to import it from, and this covers the unexported default parameter
+// set the image bakes.
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The include the image build validates against a real mariadbd. It is checked
+// in rather than generated during the build because the build runs inside the
+// libguestfs appliance, where there is no Go toolchain to resolve the catalog.
+const mariadbDefaultParametersFixture = "../../scripts/images/rds-mariadb/default-parameters.cnf"
+
+// Byte-for-byte what rds-init installs on a smallest-class instance that
+// overrides nothing. Regenerate with RDS_UPDATE_FIXTURES=1 go test ./cmd/rds-agent/.
+//
+// Keeping it in step matters because setup.sh feeds exactly this file to
+// mariadbd: a catalog that grew a name the server refuses at startup would
+// otherwise reach a customer as an instance that never boots.
+func TestMariaDBDefaultParametersFixture_MatchesTheCatalog(t *testing.T) {
+	engine, err := handlers_rds.LookupEngine("mariadb")
+	if err != nil {
+		t.Fatalf("LookupEngine: %v", err)
+	}
+	class := handlers_rds.SmallestInstanceClass()
+	params, err := engine.ResolveEffectiveParameters(class, nil)
+	if err != nil {
+		t.Fatalf("ResolveEffectiveParameters(%s): %v", class, err)
+	}
+	body, err := renderParameters("mariadb", params)
+	if err != nil {
+		t.Fatalf("renderParameters: %v", err)
+	}
+	want := mariadbParametersHeader + body
+
+	path := filepath.FromSlash(mariadbDefaultParametersFixture)
+	if os.Getenv("RDS_UPDATE_FIXTURES") == "1" {
+		if err := os.WriteFile(path, []byte(want), 0o644); err != nil {
+			t.Fatalf("update %s: %v", path, err)
+		}
+		return
+	}
+
+	got, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read %s: %v", path, err)
+	}
+	if string(got) != want {
+		t.Errorf("%s is stale for class %s.\nRegenerate with RDS_UPDATE_FIXTURES=1 go test ./cmd/rds-agent/\n\ngot:\n%s\nwant:\n%s",
+			path, class, got, want)
+	}
+}
+
+// The one thing the fixture exists to prove cannot regress without the image
+// build catching it, asserted here too so a plain preflight names it.
+func TestMariaDBDefaultParametersFixture_CarriesNoSetOnlyName(t *testing.T) {
+	got, err := os.ReadFile(filepath.FromSlash(mariadbDefaultParametersFixture))
+	if err != nil {
+		t.Fatalf("read the fixture: %v", err)
+	}
+	for _, name := range []string{"\ntime_zone ", "\ntx_isolation ", "\ninnodb_buffer_pool_instances "} {
+		if strings.Contains(string(got), name) {
+			t.Errorf("the default include carries %q, which mariadbd refuses at startup", name)
+		}
+	}
+}

cmd/rds-agent/engine.go

@@ -3,7 +3,6 @@ package main
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"os"
 	"os/exec"
@@ -11,17 +10,15 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-	"sync"
-	"sync/atomic"
 	"syscall"
 	"time"
 
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
 
-// The in-guest half of the engine seam. Everything PostgreSQL-shaped that a
-// control-plane command needs lives behind this, so the command registry above
-// stays engine-agnostic.
+// The in-guest half of the engine seam. Everything engine-shaped that a
+// control-plane command needs lives behind this, so the command registry stays
+// engine-agnostic.
 type engineOps interface {
 	// Rotates the master role's password live. Never persisted anywhere in the
 	// guest: it exists only for the length of this call.
@@ -44,18 +41,129 @@ type engineOps interface {
 // a control-plane directive: nothing issues it, the agent runs it when the
 // engine fails to come up after a parameter change.
 type parameterRecovery interface {
-	// Reports whether the installed set differed and was replaced.
+	// Reports whether the installed set differed and was replaced. True with a
+	// non-nil error is a set already on disk whose re-derived TLS enforcement did
+	// not land: the caller still has to restart and report.
 	RestoreLastKnownGoodParameters(ctx context.Context) (bool, error)
 	Restart(ctx context.Context) error
 }
 
-// Records the parameter file only after the probe observes the postmaster
-// serving it. This is separate from apply, where static values are installed
-// but not adopted until a later restart.
+// Records the parameter file only after the probe observes the engine serving
+// it. This is separate from apply, where static values are installed but not
+// adopted until a later restart.
 type servingParameterRecorder interface {
 	RecordServingParameters(ctx context.Context) error
 }
 
+// Everything one engine implementation owns. The narrow interfaces above are
+// what the heartbeat, the commander and the rollback guard each hold; this is
+// only what the agent has to construct.
+type engine interface {
+	engineOps
+	parameterRecovery
+	servingParameterRecorder
+}
+
+// The guest layout an engine preset lays down, keyed by the engine the image
+// bakes. Every default that belongs to one engine lives here, so nothing
+// outside an implementation names its paths, service, port or probe.
+type engineLayout struct {
+	binDir    string
+	dataDir   string
+	socketDir string
+	osUser    string
+	service   string
+	dataMount string
+	port      int
+	// Where the engine records the pid the probe checks for liveness. Empty for
+	// an engine whose probe answers without one.
+	pidFile string
+	// Where the engine records why it would not start. Empty for an engine whose
+	// image does not direct one, which leaves the probe reporting its own reason
+	// alone rather than guessing at a path.
+	errorLog string
+	// The engine's own liveness signal, which is the only part of engineProbe
+	// that is not shared.
+	newProbe func(cfg config, run probeRunner) *engineProbe
+	// The implementation every control-plane directive is served by. It fails
+	// when the control plane this agent was built against does not offer the
+	// engine the image bakes, rather than serving one it has no definition for.
+	newEngine func(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error)
+}
+
+// The names the control plane knows these implementations by, which are also
+// what setup.sh stamps into each image.
+const (
+	enginePostgres = "postgres"
+	engineMariaDB  = "mariadb"
+)
+
+var engineLayouts = map[string]engineLayout{
+	enginePostgres: {
+		binDir:    "/usr/libexec/postgresql18",
+		dataDir:   "/var/lib/postgresql/18/data",
+		socketDir: "/run/postgresql",
+		osUser:    "postgres",
+		service:   "postgresql",
+		dataMount: "/var/lib/postgresql",
+		port:      5432,
+		newProbe:  newPostgresProbe,
+		newEngine: func(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+			return newPostgresEngine(cfg, run, startSess, probe), nil
+		},
+	},
+	engineMariaDB: {
+		binDir:    "/usr/bin",
+		dataDir:   "/var/lib/mysql/data",
+		socketDir: "/run/mysqld",
+		osUser:    "mysql",
+		service:   "mariadb",
+		dataMount: "/var/lib/mysql",
+		port:      3306,
+		// Alpine's service passes --pid-file=/run/mysqld/$RC_SVCNAME.pid, and the
+		// command line beats the option file, so this is the path rather than
+		// anything the drop-in names. setup.sh asserts it at build time.
+		pidFile: "/run/mysqld/mariadb.pid",
+		// log_error in the platform drop-in rds-init writes. Without it
+		// mysqld_safe would send the server's errors to syslog, which nothing
+		// collects off a guest that has no SSH.
+		errorLog:  "/var/log/mysql/error.log",
+		newProbe:  newMariaDBProbe,
+		newEngine: newMariaDBEngineFromCatalog,
+	},
+}
+
+// Builds the health probe the image's engine supplies.
+func newProbe(cfg config, run probeRunner) (*engineProbe, error) {
+	layout, err := layoutFor(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return layout.newProbe(cfg, run), nil
+}
+
+// Builds the implementation the image bakes. Nothing here consults the control
+// plane for which engine to be: the agent runs the engine it is made of, and the
+// delivered engine is only ever checked against it.
+func newEngine(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+	layout, err := layoutFor(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return layout.newEngine(cfg, run, startSess, probe)
+}
+
+func layoutFor(cfg config) (engineLayout, error) {
+	if cfg.BakedEngine == "" {
+		return engineLayout{}, fmt.Errorf("this image carries no engine stamp at %s", cfg.EngineFile)
+	}
+	layout, ok := engineLayouts[cfg.BakedEngine]
+	if !ok {
+		return engineLayout{}, fmt.Errorf("this image bakes engine %q, which this agent does not implement", cfg.BakedEngine)
+	}
+	return layout, nil
+}
+
 // One child process. Env replaces the agent's own environment rather than
 // extending it, so a secret placed here reaches only the process that needs it.
 type command struct {
@@ -65,10 +173,14 @@ type command struct {
 	Stdin string
 	// The OS user to drop to before exec. Empty runs as the agent itself.
 	User string
+	// The statement that makes this client print the session sentinel. Read only
+	// when the command is started as a held session, whose reader has to know
+	// when the engine has finished each script it was fed.
+	SentinelStatement string
 }
 
-// Returns stdout. stderr is folded into the error, because psql reports the
-// actual SQL failure there while exiting with a bare status.
+// Returns stdout. stderr is folded into the error, because both clients report
+// the actual SQL failure there while exiting with a bare status.
 type commandRunner func(ctx context.Context, c command) (string, error)
 
 func execCommandRunner(ctx context.Context, c command) (string, error) {
@@ -113,417 +225,51 @@ func lookupCredential(username string) (*syscall.Credential, error) {
 	return &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)}, nil
 }
 
-// The engine is reached over its unix socket under peer authentication, so the
-// agent drops to the postgres OS user rather than holding a password of its own.
-type postgresEngine struct {
-	run       commandRunner
-	startSess sessionRunner
-	psql      string
-	rcService string
-	service   string
-	pgData    string
-	socketDir string
-	osUser    string
-	probe     *engineProbe
-	// Set from the bootstrap config, on a different goroutine than the commands
-	// that read it.
-	port atomic.Int64
-
-	// Serializes parameter installs, serving snapshots and rollback restores so
-	// none can copy or replace an intermediate configuration.
-	paramMu       sync.Mutex
-	repairTimeout time.Duration
-	repairPoll    time.Duration
-
-	// The backup mode currently held, if any. Guarded because the expiry timer
-	// releases it on its own goroutine.
-	mu   sync.Mutex
-	held *quiesceHold
-}
-
-var (
-	_ engineOps                = (*postgresEngine)(nil)
-	_ parameterRecovery        = (*postgresEngine)(nil)
-	_ servingParameterRecorder = (*postgresEngine)(nil)
-)
-
-const (
-	parameterRepairTimeout = 90 * time.Second
-	parameterRepairPoll    = time.Second
-)
-
-func newPostgresEngine(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) *postgresEngine {
-	e := &postgresEngine{
-		run:           run,
-		startSess:     startSess,
-		psql:          filepath.Join(cfg.PGBin, "psql"),
-		rcService:     cfg.RCService,
-		service:       cfg.EngineService,
-		pgData:        cfg.PGData,
-		socketDir:     cfg.SocketDir,
-		osUser:        cfg.PGUser,
-		probe:         probe,
-		repairTimeout: parameterRepairTimeout,
-		repairPoll:    parameterRepairPoll,
-	}
-	e.port.Store(int64(cfg.EnginePort))
-	return e
-}
-
-func (e *postgresEngine) setPort(port int) {
-	e.port.Store(int64(port))
-}
-
-// The role name and the password ride the environment and are re-quoted by
-// psql, so neither reaches a shell word or an argv another process can read.
-func (e *postgresEngine) SetPassword(ctx context.Context, username, password string) error {
-	if username == "" || password == "" {
-		return fmt.Errorf("set-password requires both %s and %s",
-			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
-	}
-	// The statement below runs as the cluster superuser over the socket under
-	// peer auth, so a reserved name in the command payload would hand the
-	// customer the bootstrap superuser rather than rotate their own role.
-	if err := handlers_rds.EnginePostgres().ValidateUsernameNotReserved(username); err != nil {
-		return fmt.Errorf("refusing to set the password of a role the engine reserves: %w", err)
-	}
-	// psql interpolates the password into the ALTER ROLE before the server sees
-	// it, and these three are what would write it to the log — the last on any
-	// failure at its own default. SUSET, so the parameter group cannot win.
-	const sql = `SET log_statement = 'none';
-SET log_min_duration_statement = -1;
-SET log_min_error_statement = 'panic';
-\getenv master RDS_MASTER_USERNAME
-\getenv password RDS_MASTER_PASSWORD
-ALTER ROLE :"master" WITH LOGIN PASSWORD :'password';
-`
-	_, err := e.psqlRun(ctx, sql,
-		"RDS_MASTER_USERNAME="+username,
-		"RDS_MASTER_PASSWORD="+password,
-	)
-	if err != nil {
-		// psql echoes the failing statement, which here would carry the new
-		// password back to the control plane and into the event ring.
-		return fmt.Errorf("apply the master password: %s", redact(err.Error(), password))
+// Through the service manager rather than the engine's own control tool, so the
+// supervisor records the state and does not restart the engine underneath a VM
+// that is going down.
+func serviceAction(ctx context.Context, run commandRunner, rcService, service, action string) error {
+	if _, err := run(ctx, command{
+		Name: rcService,
+		Args: []string{service, action},
+		Env:  []string{"PATH=" + defaultGuestPath},
+	}); err != nil {
+		return fmt.Errorf("%s the %s service: %w", action, service, err)
 	}
 	return nil
 }
 
-// The include the resolved parameter set is rendered to, and the copy of the
-// last one the engine accepted. Both live beside the data rather than in /etc,
-// matching rds-init: a class change boots a fresh root volume, which would
-// otherwise revert them when a fresh root volume is used.
-const (
-	parametersFileName = "10-rds-parameters.conf"
-	// Deliberately not a .conf name: include_dir globs *.conf, so the rollback
-	// copy must not be read as a second set of settings.
-	lastGoodFileName = "10-rds-parameters.last-good"
-)
-
-// Installs the resolved set, validates it with the engine's own config parser
-// before the engine ever adopts it, and reloads. A value the engine refuses is
-// rolled back here rather than left on the data volume, where it would survive
-// every VM replace and turn the next restart into a boot loop.
-func (e *postgresEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
-	e.paramMu.Lock()
-	defer e.paramMu.Unlock()
-
-	initialState, _ := e.probe.state(ctx)
-	if initialState == engineRecovering {
-		return nil, errors.New("apply parameters while the engine is still starting or recovering")
-	}
-	if initialState == engineServing {
-		// A command can arrive before the first heartbeat. Seed the configuration
-		// currently being served before replacing its file in that window.
-		if err := e.recordServingParametersLocked(ctx); err != nil {
-			return nil, fmt.Errorf("record the parameters serving before the apply: %w", err)
-		}
-	}
-
-	// The check has to run against the file in place, because the engine parses
-	// the datadir's own include_dir. The window that leaves is closed from both
-	// ends: the rollback below, and the last-known-good restore the agent runs at
-	// boot when the engine does not come up.
-	_, restore, err := e.installParameters(params)
-	if err != nil {
-		return nil, err
-	}
-	if err := e.checkConfig(ctx); err != nil {
-		return nil, errors.Join(fmt.Errorf("the engine rejected the parameter set: %w", err), restore())
-	}
-
-	state, _ := e.probe.state(ctx)
-	switch state {
-	case engineAbsent:
-		return e.restartOnRepairSetLocked(ctx)
-	case engineRecovering:
-		return nil, errors.Join(errors.New("the engine entered startup or recovery during the parameter apply"), restore())
-	}
-
-	if _, err := e.psqlRun(ctx, "SELECT pg_reload_conf();\n"); err != nil {
-		// The engine may have gone down after the first probe. In that case start
-		// it on the new, parser-checked repair set rather than restoring the set
-		// that already left it unable to serve.
-		if state, _ := e.probe.state(ctx); state == engineAbsent {
-			return e.restartOnRepairSetLocked(ctx)
-		}
-		return nil, errors.Join(fmt.Errorf("reload the engine configuration: %w", err), restore())
-	}
-	return e.pendingRestartParameters(ctx)
-}
-
-func (e *postgresEngine) restartOnRepairSetLocked(ctx context.Context) ([]string, error) {
-	repairCtx, cancel := context.WithTimeout(ctx, e.repairTimeout)
-	defer cancel()
-
-	if err := e.Restart(repairCtx); err != nil {
-		return nil, fmt.Errorf("start the engine on the repaired parameter set: %w", err)
-	}
-
-	timer := time.NewTimer(0)
-	defer timer.Stop()
-	lastMessage := "engine did not respond"
-	for {
-		select {
-		case <-repairCtx.Done():
-			if ctx.Err() != nil {
-				return nil, fmt.Errorf("wait for the engine on the repaired parameter set: %w", ctx.Err())
-			}
-			return nil, fmt.Errorf("wait for the engine on the repaired parameter set: %s: %w", lastMessage, repairCtx.Err())
-		case <-timer.C:
-		}
-
-		state, message := e.probe.state(repairCtx)
-		if state == engineServing {
-			pending, err := e.pendingRestartParameters(repairCtx)
-			if err == nil {
-				return pending, nil
-			}
-			lastMessage = err.Error()
-		} else if message != "" {
-			lastMessage = message
-		}
-		timer.Reset(e.repairPoll)
-	}
-}
-
-func (e *postgresEngine) pendingRestartParameters(ctx context.Context) ([]string, error) {
-	out, err := e.psqlRun(ctx, "SELECT name FROM pg_settings WHERE pending_restart ORDER BY name;\n")
-	if err != nil {
-		return nil, fmt.Errorf("read the settings pending a restart: %w", err)
-	}
-	var pending []string
-	for line := range strings.SplitSeq(out, "\n") {
-		if name := strings.TrimSpace(line); name != "" {
-			pending = append(pending, name)
-		}
-	}
-	return pending, nil
-}
-
-func (e *postgresEngine) parametersPath() string {
-	return filepath.Join(e.pgData, "conf.d", parametersFileName)
-}
-
-func (e *postgresEngine) lastGoodPath() string {
-	return filepath.Join(e.pgData, "conf.d", lastGoodFileName)
-}
-
-// Written as root and handed to the engine's user, at the same path and mode
-// rds-init installs it at, so a later boot overwrites rather than shadows it.
-// Returns the content that was there and a func putting it back, so the caller
-// can undo the install without re-rendering anything.
-func (e *postgresEngine) installParameters(params []handlers_rds.Parameter) (previous []byte, restore func() error, err error) {
-	path := e.parametersPath()
-	// A missing file is the first-ever apply, and its rollback is a removal.
-	previous, readErr := os.ReadFile(path)
-	if readErr != nil && !os.IsNotExist(readErr) {
-		return nil, nil, fmt.Errorf("read the installed parameters at %s: %w", path, readErr)
-	}
-	existed := readErr == nil
-
-	if err := e.writeEngineFile(path, []byte(renderParameters(params))); err != nil {
-		return nil, nil, err
-	}
-	return previous, func() error {
-		if !existed {
-			if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
-				return fmt.Errorf("withdraw the rejected parameters at %s: %w", path, err)
-			}
-			return nil
-		}
-		return e.writeEngineFile(path, previous)
-	}, nil
-}
-
-// Snapshots the include only when the running postmaster has no settings still
-// pending a restart. The parameter mutex keeps an apply from replacing the file
-// between that check and the copy.
-func (e *postgresEngine) RecordServingParameters(ctx context.Context) error {
-	e.paramMu.Lock()
-	defer e.paramMu.Unlock()
-	return e.recordServingParametersLocked(ctx)
-}
-
-func (e *postgresEngine) recordServingParametersLocked(ctx context.Context) error {
-	if _, err := os.Stat(e.parametersPath()); err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return fmt.Errorf("inspect the serving parameters: %w", err)
-	}
-	pending, err := e.pendingRestartParameters(ctx)
-	if err != nil {
-		return err
-	}
-	if len(pending) > 0 {
-		return nil
-	}
-
-	content, err := os.ReadFile(e.parametersPath())
-	if err != nil {
-		return fmt.Errorf("read the serving parameters: %w", err)
-	}
-	lastGood, err := os.ReadFile(e.lastGoodPath())
-	if err == nil && bytes.Equal(content, lastGood) {
-		return nil
-	}
-	if err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("read the last known good parameters: %w", err)
-	}
-	return e.writeEngineFile(e.lastGoodPath(), content)
-}
-
-// Puts the last set the engine accepted back in place, for a restart that failed
-// after a parameter change. The probe is checked again under the parameter lock
-// so a repair that just brought the engine back cannot be reversed.
-func (e *postgresEngine) RestoreLastKnownGoodParameters(ctx context.Context) (bool, error) {
-	e.paramMu.Lock()
-	defer e.paramMu.Unlock()
-
-	if state, _ := e.probe.state(ctx); state != engineAbsent {
-		return false, nil
-	}
-	lastGood, err := os.ReadFile(e.lastGoodPath())
-	if err != nil {
-		if os.IsNotExist(err) {
-			return false, nil
-		}
-		return false, fmt.Errorf("read the last known good parameters: %w", err)
-	}
-	current, err := os.ReadFile(e.parametersPath())
+// Whether the installed set requires TLS of client connections. Read back from
+// the file rather than taken from the set being applied, because the restore and
+// repair paths put a file in place without holding one. Both engines' generated
+// parameter files are written in the same syntax, so one reader serves them.
+func installedTLSEnforcement(name, installedPath string) (bool, error) {
+	values, err := readOptionFile(installedPath)
 	if err != nil && !os.IsNotExist(err) {
 		return false, fmt.Errorf("read the installed parameters: %w", err)
 	}
-	if bytes.Equal(current, lastGood) {
+	value, ok := values[name]
+	if !ok {
+		// The ordinary state of an instance whose resolved set predates the
+		// parameter, and the whole of the migration story: it begins enforcing at
+		// its next boot with no control-plane work at all.
+		return true, nil
+	}
+	switch value {
+	case "1":
+		return true, nil
+	case "0":
 		return false, nil
 	}
-	if err := e.writeEngineFile(e.parametersPath(), lastGood); err != nil {
-		return false, err
-	}
-	return true, nil
-}
-
-// Atomic: a temp file in the same directory, renamed over the target. The temp
-// name deliberately does not end in .conf, so a crash between write and rename
-// cannot leave the engine reading a half-written include.
-func (e *postgresEngine) writeEngineFile(path string, content []byte) error {
-	tmp := path + ".new"
-	if err := os.WriteFile(tmp, content, 0o600); err != nil {
-		return fmt.Errorf("write %s: %w", tmp, err)
-	}
-	if err := e.chownToEngine(tmp); err != nil {
-		os.Remove(tmp)
-		return err
-	}
-	if err := os.Rename(tmp, path); err != nil {
-		os.Remove(tmp)
-		return fmt.Errorf("install %s: %w", path, err)
-	}
-	return nil
-}
-
-// The engine's own parser, run offline against the datadir. Reading one setting
-// back is enough: postgres parses postgresql.conf and every include first, and
-// exits non-zero naming the file and line of an unknown parameter or a value
-// outside its range.
-func (e *postgresEngine) checkConfig(ctx context.Context) error {
-	if _, err := e.run(ctx, command{
-		Name: filepath.Join(filepath.Dir(e.psql), "postgres"),
-		Args: []string{"-D", e.pgData, "-C", "shared_buffers"},
-		Env:  []string{"PATH=" + defaultGuestPath},
-		User: e.osUser,
-	}); err != nil {
-		return err
-	}
-	return nil
-}
-
-// The engine reads its own config, so a root-owned file would be unreadable to
-// it. A guest with no such user is a broken image, not something to work around.
-func (e *postgresEngine) chownToEngine(path string) error {
-	credential, err := lookupCredential(e.osUser)
-	if err != nil {
-		return err
-	}
-	if err := os.Chown(path, int(credential.Uid), int(credential.Gid)); err != nil {
-		return fmt.Errorf("hand %s to %s: %w", path, e.osUser, err)
-	}
-	return nil
-}
-
-// Through the service manager rather than pg_ctl, so the supervisor records the
-// engine as stopped and does not restart it underneath a VM that is going down.
-func (e *postgresEngine) Stop(ctx context.Context) error {
-	return e.serviceAction(ctx, "stop")
-}
-
-// Only the parameter rollback calls this: a restart the control plane wants goes
-// through RebootDBInstance, which cycles the VM.
-func (e *postgresEngine) Restart(ctx context.Context) error {
-	return e.serviceAction(ctx, "restart")
-}
-
-func (e *postgresEngine) serviceAction(ctx context.Context, action string) error {
-	if _, err := e.run(ctx, command{
-		Name: e.rcService,
-		Args: []string{e.service, action},
-		Env:  []string{"PATH=" + defaultGuestPath},
-	}); err != nil {
-		return fmt.Errorf("%s the %s service: %w", action, e.service, err)
-	}
-	return nil
+	// The resolver canonicalises every boolean, so anything else means the file
+	// was written by something other than the platform. The permissive reading is
+	// the one that must not be chosen for a setting that turns TLS off.
+	return false, fmt.Errorf("the installed parameters set %s to %q, which is neither 1 nor 0", name, value)
 }
 
 // A minimal environment for a child that is not carrying secrets.
 const defaultGuestPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 
-// Feeds sql on stdin rather than as an argument, so a statement is never
-// visible in the process table.
-func (e *postgresEngine) psqlRun(ctx context.Context, sql string, env ...string) (string, error) {
-	return e.run(ctx, command{
-		Name:  e.psql,
-		Args:  e.psqlArgs(),
-		Env:   append([]string{"PATH=" + defaultGuestPath}, env...),
-		Stdin: sql,
-		User:  e.osUser,
-	})
-}
-
-// Reading the script from stdin is what lets one invocation serve both a
-// one-shot run and a session held open across several statements.
-func (e *postgresEngine) psqlArgs() []string {
-	return []string{
-		"--no-psqlrc", "--quiet", "--no-align", "--tuples-only",
-		"-v", "ON_ERROR_STOP=1",
-		"-h", e.socketDir,
-		"-p", strconv.FormatInt(e.port.Load(), 10),
-		"-U", e.osUser,
-		"-d", "postgres",
-		"-f", "-",
-	}
-}
-
 func redact(s, secret string) string {
 	if secret == "" {
 		return s

cmd/rds-agent/engine_contract_test.go

@@ -0,0 +1,333 @@
+package main
+
+//test:in-package — the agent is a main package, which has no external test
+// package to import it from, and these cases are built on the unexported
+// engine interface and each implementation's own constructor.
+
+import (
+	"context"
+	"errors"
+	"os"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// A probe the test drives directly, in place of whichever binaries the engine's
+// own state function would have run. Each implementation's state function is
+// covered on its own; what these cases exercise is what the engine does with the
+// answer.
+func drivenProbe(state *engineState) *engineProbe {
+	return newEngineProbe(0, func(context.Context, int64) (engineState, string) {
+		return *state, "driven by the test"
+	})
+}
+
+// One engine behind the interfaces the heartbeat, the commander and the rollback
+// guard hold, plus the few facts a shared case needs to say what it means.
+type engineHarness struct {
+	engine  engine
+	runner  *recordingRunner
+	params  parameterStore
+	service string
+	// The state the probe reports, moved by the test rather than by the engine.
+	state *engineState
+	// Points the next quiesce at a session the test can inspect, and returns the
+	// commands those sessions were opened with.
+	useSession func(*fakeSession) *[]command
+	// A setting this engine's catalog really carries, so a shared case installs
+	// something the implementation would genuinely be handed.
+	parameter handlers_rds.Parameter
+	// What the engine's own include directive globs. Neither the rollback copy
+	// nor the serving copy may end in it.
+	includeSuffix string
+}
+
+func engineHarnesses() []struct {
+	name  string
+	build func(*testing.T) *engineHarness
+} {
+	return []struct {
+		name  string
+		build func(*testing.T) *engineHarness
+	}{
+		{name: enginePostgres, build: buildPostgresHarness},
+		{name: engineMariaDB, build: buildMariaDBHarness},
+	}
+}
+
+func forEachEngine(t *testing.T, run func(*testing.T, *engineHarness)) {
+	t.Helper()
+	for _, fixture := range engineHarnesses() {
+		t.Run(fixture.name, func(t *testing.T) { run(t, fixture.build(t)) })
+	}
+}
+
+func buildPostgresHarness(t *testing.T) *engineHarness {
+	t.Helper()
+	runner := &recordingRunner{}
+	e := newTestEngine(t, runner.run)
+	state := engineServing
+	e.probe = drivenProbe(&state)
+
+	return &engineHarness{
+		engine: e, runner: runner, params: e.params, service: e.service, state: &state,
+		useSession: func(session *fakeSession) *[]command {
+			started := &[]command{}
+			e.startSess = func(_ context.Context, c command) (engineSession, error) {
+				*started = append(*started, c)
+				return session, nil
+			}
+			return started
+		},
+		parameter:     handlers_rds.Parameter{Name: "work_mem", Value: "4096"},
+		includeSuffix: ".conf",
+	}
+}
+
+func buildMariaDBHarness(t *testing.T) *engineHarness {
+	t.Helper()
+	runner := &recordingRunner{}
+	e := newTestMariaDBEngine(t, runner.run)
+	state := engineServing
+	e.probe = drivenProbe(&state)
+
+	return &engineHarness{
+		engine: e, runner: runner, params: e.params, service: e.service, state: &state,
+		useSession: func(session *fakeSession) *[]command {
+			started := &[]command{}
+			e.startSess = func(_ context.Context, c command) (engineSession, error) {
+				*started = append(*started, c)
+				return session, nil
+			}
+			return started
+		},
+		parameter:     handlers_rds.Parameter{Name: "max_connections", Value: "200"},
+		includeSuffix: ".cnf",
+	}
+}
+
+// The parameters land where rds-init installs them, so the next boot overwrites
+// the file rather than shadowing it with a second copy.
+func TestEngineContract_ApplyParametersInstallsWhereRdsInitDoes(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		if _, err := h.engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{h.parameter}); err != nil {
+			t.Fatalf("ApplyParameters: %v", err)
+		}
+
+		installed, err := os.ReadFile(h.params.installedPath())
+		if err != nil {
+			t.Fatalf("read the installed parameters: %v", err)
+		}
+		if !strings.Contains(string(installed), h.parameter.Name+" = '"+h.parameter.Value+"'") {
+			t.Errorf("installed parameters = %q, want the resolved value", installed)
+		}
+		if !strings.HasSuffix(h.params.installedPath(), h.includeSuffix) {
+			t.Errorf("the installed set at %s is not a file the engine includes", h.params.installedPath())
+		}
+		// Comparison material, not a second set of settings: an include glob that
+		// read either of these would apply a withdrawn configuration.
+		copies := []string{h.params.lastGoodPath()}
+		if h.params.serving != "" {
+			copies = append(copies, h.params.servingPath())
+		}
+		for _, path := range copies {
+			if strings.HasSuffix(path, h.includeSuffix) {
+				t.Errorf("the copy at %s would be included by the engine", path)
+			}
+		}
+	})
+}
+
+// An engine still starting or replaying has not finished adopting the set it is
+// on, so replacing that set underneath it would be installing over a moving
+// target.
+func TestEngineContract_ApplyParametersRefusesWhileTheEngineIsRecovering(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		*h.state = engineRecovering
+
+		if _, err := h.engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{h.parameter}); err == nil {
+			t.Fatal("ApplyParameters succeeded against an engine that was still coming up")
+		}
+		if _, err := os.Stat(h.params.installedPath()); !os.IsNotExist(err) {
+			t.Errorf("the refused set was installed anyway (stat err = %v)", err)
+		}
+	})
+}
+
+// The boot-time half of the parameter safety net: the last set the engine
+// accepted goes back only when the engine is genuinely down, and only once.
+func TestEngineContract_RestoreLastKnownGoodOnlyRollsBackADownEngine(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		accepted := h.parameter
+		if _, err := h.engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{accepted}); err != nil {
+			t.Fatalf("ApplyParameters: %v", err)
+		}
+		if err := h.engine.RecordServingParameters(t.Context()); err != nil {
+			t.Fatalf("RecordServingParameters: %v", err)
+		}
+		if _, err := h.engine.ApplyParameters(t.Context(),
+			[]handlers_rds.Parameter{{Name: accepted.Name, Value: accepted.Value + "0"}}); err != nil {
+			t.Fatalf("second ApplyParameters: %v", err)
+		}
+
+		// A guard whose deadline expired while an apply brought the engine back must
+		// not reverse that repair.
+		restored, err := h.engine.RestoreLastKnownGoodParameters(t.Context())
+		if err != nil {
+			t.Fatalf("RestoreLastKnownGoodParameters against a serving engine: %v", err)
+		}
+		if restored {
+			t.Fatal("rolled the parameters back while the engine was serving")
+		}
+
+		*h.state = engineAbsent
+		restored, err = h.engine.RestoreLastKnownGoodParameters(t.Context())
+		if err != nil {
+			t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
+		}
+		if !restored {
+			t.Fatal("did not roll back to a set that differs from the last accepted one")
+		}
+		installed, err := os.ReadFile(h.params.installedPath())
+		if err != nil {
+			t.Fatalf("read the installed parameters: %v", err)
+		}
+		if !strings.Contains(string(installed), accepted.Name+" = '"+accepted.Value+"'") {
+			t.Errorf("installed = %q, want the last accepted set", installed)
+		}
+
+		// Repeating a rollback would only churn a cluster that is failing for
+		// another reason.
+		restored, err = h.engine.RestoreLastKnownGoodParameters(t.Context())
+		if err != nil {
+			t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
+		}
+		if restored {
+			t.Error("rolled back again when the installed set already matched")
+		}
+	})
+}
+
+// Through the service manager rather than the engine's own control tool, so the
+// supervisor records the engine as stopped and does not restart it underneath a
+// VM that is going down.
+func TestEngineContract_StopGoesThroughTheServiceManager(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		if err := h.engine.Stop(t.Context()); err != nil {
+			t.Fatalf("Stop: %v", err)
+		}
+		if len(h.runner.calls) != 1 {
+			t.Fatalf("ran %d commands, want the service stop", len(h.runner.calls))
+		}
+		call := h.runner.calls[0]
+		if !slices.Equal(call.Args, []string{h.service, "stop"}) {
+			t.Errorf("stop ran %s %v, want the service manager", call.Name, call.Args)
+		}
+	})
+}
+
+// One backup at a time, released by Unquiesce or by its own deadline, whichever
+// comes first. The deadline is what keeps a control plane that dies mid-snapshot
+// from leaving the engine held.
+func TestEngineContract_QuiesceHoldsUntilReleased(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		session := &fakeSession{}
+		started := h.useSession(session)
+
+		if err := h.engine.Quiesce(t.Context(), "orders-db-pre-upgrade", time.Minute); err != nil {
+			t.Fatalf("Quiesce: %v", err)
+		}
+		if len(*started) != 1 {
+			t.Fatalf("started %d sessions, want 1", len(*started))
+		}
+		if session.closes() != 0 {
+			t.Fatal("the session was closed while the backup was meant to be held")
+		}
+
+		err := h.engine.Quiesce(t.Context(), "second", time.Minute)
+		if err == nil {
+			t.Fatal("a second quiesce was accepted while the first was held")
+		}
+		if !strings.Contains(err.Error(), "orders-db-pre-upgrade") {
+			t.Errorf("error %q does not name the backup already holding the engine", err)
+		}
+		if len(*started) != 1 {
+			t.Errorf("started %d sessions, want the second refused before opening one", len(*started))
+		}
+
+		if err := h.engine.Unquiesce(t.Context()); err != nil {
+			t.Fatalf("Unquiesce: %v", err)
+		}
+		if session.closes() != 1 {
+			t.Errorf("closed %d times, want the session ended exactly once", session.closes())
+		}
+	})
+}
+
+// A hold that expired mid-snapshot means the snapshot was not taken against a
+// held checkpoint, so the release reports it rather than succeeding silently.
+func TestEngineContract_QuiesceReleasesItselfOnItsDeadline(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		session := &fakeSession{}
+		h.useSession(session)
+
+		if err := h.engine.Quiesce(t.Context(), "orders-db-pre-upgrade", 10*time.Millisecond); err != nil {
+			t.Fatalf("Quiesce: %v", err)
+		}
+		deadline := time.Now().Add(2 * time.Second)
+		for session.closes() == 0 && time.Now().Before(deadline) {
+			time.Sleep(time.Millisecond)
+		}
+		if session.closes() != 1 {
+			t.Fatal("the backup session outlived its deadline")
+		}
+
+		err := h.engine.Unquiesce(t.Context())
+		if err == nil {
+			t.Fatal("Unquiesce succeeded after the hold expired")
+		}
+		if !strings.Contains(err.Error(), "expired") {
+			t.Errorf("error %q does not say the hold had expired", err)
+		}
+	})
+}
+
+// A failed start leaves nothing held, so a retry is not refused by a hold that
+// never took effect.
+func TestEngineContract_QuiesceEndsTheSessionWhenTheBackupWillNotStart(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		session := &fakeSession{execErr: errors.New("the engine refused the backup")}
+		h.useSession(session)
+
+		if err := h.engine.Quiesce(t.Context(), "orders-db-pre-upgrade", time.Minute); err == nil {
+			t.Fatal("Quiesce succeeded against an engine that refused the backup")
+		}
+		if session.closes() != 1 {
+			t.Errorf("closed %d times, want the failed session ended", session.closes())
+		}
+		if err := h.engine.Unquiesce(t.Context()); err == nil {
+			t.Error("a failed quiesce left a hold behind")
+		}
+	})
+}
+
+// Every command the control plane can issue is served by both implementations,
+// so a directive cannot be answered as unsupported on one engine and honoured on
+// the other.
+func TestEngineContract_ServesEveryRegisteredCommand(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		registry := newCommandRegistry(h.engine, &fakeStorage{})
+		for _, name := range []string{
+			handlers_rds.CommandSetPassword, handlers_rds.CommandApplyParams, handlers_rds.CommandStopEngine,
+			handlers_rds.CommandGrowFilesystem, handlers_rds.CommandQuiesce, handlers_rds.CommandUnquiesce,
+		} {
+			if _, ok := registry[name]; !ok {
+				t.Errorf("command %q has no handler", name)
+			}
+		}
+	})
+}

cmd/rds-agent/engine_mariadb.go

@@ -0,0 +1,814 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The control plane's own statements about this engine, resolved once at
+// startup. Held as functions rather than as the Engine value so nothing here
+// re-derives a rule the API already owns.
+type controlPlaneRules struct {
+	// The whole master-username rule rather than only the reserved set. MariaDB's
+	// client has no identifier quoting, so the name is interpolated into SQL by
+	// this process and every part of the rule is load-bearing here.
+	validateUsername func(username string) error
+	// Whether a setting takes effect only at a restart. Already customer-facing
+	// and authoritative, since the API refuses ApplyMethod=immediate on one.
+	isStatic func(name string) bool
+	// The catalog name behind a spelling read back out of an option file. Only a
+	// setting whose startup spelling differs from the customer's moves.
+	catalogName func(optionFileName string) string
+	// The catalog's data type for a setting, which decides how its value is
+	// rendered into SQL. Empty for a name the catalog does not carry.
+	dataType func(name string) string
+	// The engine's own name for the setting that requires TLS of a client
+	// connection. Held rather than spelled out here, so the guest derives
+	// enforcement from the same key the control plane resolves it under.
+	tlsEnforcementParameter string
+}
+
+func controlPlaneRulesFrom(meta handlers_rds.Engine) controlPlaneRules {
+	// Built once per engine: everything read back out of a generated file has to
+	// return to the catalog's namespace before it is classified or reported, or a
+	// startup spelling would read as an unknown name.
+	catalogNames := map[string]string{}
+	for _, name := range meta.CatalogParameterNames() {
+		if optionFileName := meta.OptionFileName(name); optionFileName != name {
+			catalogNames[optionFileName] = name
+		}
+	}
+
+	return controlPlaneRules{
+		validateUsername: meta.ValidateMasterUsername,
+		isStatic: func(name string) bool {
+			spec, ok := meta.LookupParameter(name)
+			// A name the catalog does not carry cannot be shown to have been adopted
+			// without a restart, and is never issued as a live SET GLOBAL either.
+			return !ok || spec.ApplyType == handlers_rds.ApplyTypeStatic
+		},
+		catalogName: func(optionFileName string) string {
+			if name, ok := catalogNames[optionFileName]; ok {
+				return name
+			}
+			return optionFileName
+		},
+		dataType: func(name string) string {
+			spec, ok := meta.LookupParameter(name)
+			if !ok {
+				return ""
+			}
+			return spec.DataType
+		},
+		tlsEnforcementParameter: meta.TLSEnforcementParameter(),
+	}
+}
+
+// Reached over its unix socket as root, which the datadir's unix_socket plugin
+// authenticates from the connecting process' own uid. That is why the agent
+// holds no password of its own.
+type mariadbEngine struct {
+	quiesceState
+	parameterManager
+
+	rules     controlPlaneRules
+	run       commandRunner
+	startSess sessionRunner
+	client    string
+	socket    string
+	rcService string
+	service   string
+}
+
+var _ engine = (*mariadbEngine)(nil)
+
+const (
+	mariadbClientBinary = "mariadb"
+	mariadbAdminBinary  = "mariadb-admin"
+	// The account mariadb-install-db creates for the unix_socket plugin, which
+	// rds-init keeps for the platform and never hands to the customer.
+	mariadbSuperuser                  = "root"
+	mariadbProbeConnectTimeoutSeconds = 3
+	// How much of the engine's error log a probe reason carries. The server names
+	// what it refused on in its last few lines, and the reason reaches a customer
+	// through StatusInfos, so this quotes the tail rather than the whole file.
+	mariadbErrorLogTailBytes = 4096
+	mariadbErrorLogTailLines = 4
+	// How much of the probe client's stderr a reason carries, on the same grounds
+	// as the log tail above.
+	mariadbProbeStderrMaxBytes = 512
+	// Named by the platform drop-in rds-init writes, so both halves reach the
+	// same socket without either asserting it to the other.
+	mariadbSocketFile = "mysqld.sock"
+	// The customer reaches the instance over its customer ENI, so the master is
+	// created and rotated on the wildcard host rather than on localhost.
+	mariadbMasterHost = "%"
+)
+
+// The generated drop-ins on the data volume. The rollback and serving copies
+// deliberately do not end in .cnf: !includedir reads *.cnf, and neither is a
+// second set of settings.
+const (
+	mariadbParametersFile = "10-rds-parameters.cnf"
+	mariadbLastGoodFile   = "10-rds-parameters.last-good"
+	mariadbServingFile    = "10-rds-parameters.serving"
+	// MariaDB treats a setting before any group as a fatal parsing error, so
+	// every generated file carries the header rds-init writes.
+	mariadbParametersHeader = "[mysqld]\n"
+)
+
+// The layout's factory. The rules come from the control plane's own definition
+// of this engine, so a build whose control plane does not offer MariaDB refuses
+// to run this implementation rather than inventing a definition for it.
+func newMariaDBEngineFromCatalog(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+	meta, err := handlers_rds.LookupEngine(engineMariaDB)
+	if err != nil {
+		return nil, fmt.Errorf("this image bakes %s, which this build's control plane does not offer: %w", engineMariaDB, err)
+	}
+	return newMariaDBEngine(cfg, controlPlaneRulesFrom(meta), run, startSess, probe), nil
+}
+
+func newMariaDBEngine(cfg config, rules controlPlaneRules, run commandRunner, startSess sessionRunner, probe *engineProbe) *mariadbEngine {
+	return &mariadbEngine{
+		rules:     rules,
+		run:       run,
+		startSess: startSess,
+		client:    filepath.Join(cfg.EngineBinDir, mariadbClientBinary),
+		socket:    mariadbSocketPath(cfg),
+		rcService: cfg.RCService,
+		service:   cfg.EngineService,
+		parameterManager: parameterManager{
+			probe: probe,
+			params: parameterStore{
+				// The mount point rather than the datadir one level inside it: the
+				// include directory has to outlive the sweep a failed bootstrap runs.
+				dir:       filepath.Join(cfg.DataMount, "conf.d"),
+				installed: mariadbParametersFile,
+				lastGood:  mariadbLastGoodFile,
+				serving:   mariadbServingFile,
+				header:    mariadbParametersHeader,
+				osUser:    cfg.EngineUser,
+				engine:    engineMariaDB,
+			},
+			repairTimeout: parameterRepairTimeout,
+			repairPoll:    parameterRepairPoll,
+		},
+	}
+}
+
+func mariadbSocketPath(cfg config) string {
+	return filepath.Join(cfg.SocketDir, mariadbSocketFile)
+}
+
+// Every local connection, from the probe's ping to a held quiesce session.
+// --no-defaults so no option file can move it; --skip-ssl because the serving
+// cert names the endpoint rather than this socket, so a verifying client fails.
+func mariadbSocketConnectArgs(socket string) []string {
+	return []string{
+		"--no-defaults", "--protocol=socket", "--socket=" + socket,
+		"--user=" + mariadbSuperuser, "--skip-ssl",
+	}
+}
+
+// Reports whether a pid is a live process. EPERM is a process this agent may not
+// signal rather than one that is gone, which matters because a stale pidfile
+// read as "alive" is far less dangerous than a live engine read as "absent".
+type processLivenessFn func(pid int) bool
+
+func processAlive(pid int) bool {
+	if pid <= 0 {
+		return false
+	}
+	err := syscall.Kill(pid, 0)
+	return err == nil || errors.Is(err, syscall.EPERM)
+}
+
+func newMariaDBProbe(cfg config, run probeRunner) *engineProbe {
+	return newEngineProbe(cfg.EnginePort, mariadbProbeState(cfg, run, processAlive))
+}
+
+// Three stages: during InnoDB crash recovery mariadbd opens neither socket nor
+// port, so a ping cannot tell a recovering engine from an absent one — and
+// reading recovery as absent has the rollback guard restart a server mid-replay.
+func mariadbProbeState(cfg config, run probeRunner, alive processLivenessFn) probeStateFn {
+	pidFile, socket := cfg.EnginePidFile, mariadbSocketPath(cfg)
+	admin, client := filepath.Join(cfg.EngineBinDir, mariadbAdminBinary), filepath.Join(cfg.EngineBinDir, mariadbClientBinary)
+	// The probe alone bounds the connect: it must come back and report, where a
+	// statement the agent issues is allowed to take as long as the engine needs.
+	connect := append(mariadbSocketConnectArgs(socket),
+		fmt.Sprintf("--connect-timeout=%d", mariadbProbeConnectTimeoutSeconds))
+	probeTimeout := cfg.EngineProbeTimeout
+	if probeTimeout <= 0 {
+		probeTimeout = defaultEngineProbeTimeout
+	}
+	runBounded := func(ctx context.Context, name string, args ...string) (int, string, error) {
+		probeCtx, cancel := context.WithTimeout(ctx, probeTimeout)
+		defer cancel()
+		return run(probeCtx, name, args...)
+	}
+
+	return func(ctx context.Context, _ int64) (engineState, string) {
+		pid, err := readPidFile(pidFile)
+		switch {
+		case os.IsNotExist(err):
+			return engineAbsent, fmt.Sprintf("the engine has written no pidfile at %s", pidFile)
+		case err != nil:
+			return engineAbsent, fmt.Sprintf("the engine pidfile %s could not be read: %v", pidFile, err)
+		case !alive(pid):
+			return engineAbsent, fmt.Sprintf("the engine pidfile %s names pid %d, which is not running", pidFile, pid)
+		}
+
+		// The process is up from here on, so every remaining failure is an engine
+		// not serving yet rather than one that is gone. Reporting absent against a
+		// live server would have the rollback guard restart one making progress.
+		switch code, stderr, err := runBounded(ctx, admin, append(slices.Clone(connect), "ping")...); {
+		case err != nil:
+			return engineRecovering, fmt.Sprintf("engine probe could not run: %v", err)
+		case code != 0:
+			// A server that is genuinely recovering and one that refused the client
+			// are indistinguishable from the socket. The server states its side in
+			// its own log, and the client states its side on stderr.
+			return engineRecovering, withProbeStderr(withEngineLogTail(
+				"engine is not answering on its socket yet (startup or crash recovery)",
+				cfg.EngineErrorLog), stderr)
+		}
+
+		// ping answers even on ER_ACCESS_DENIED, so on its own it certifies a server
+		// that may be unable to execute anything. The statement rides argv rather
+		// than stdin: it carries nothing secret and a probe reads no result back.
+		query := append(slices.Clone(connect), "--batch", "--skip-column-names", "--execute=SELECT 1")
+		switch code, stderr, err := runBounded(ctx, client, query...); {
+		case err != nil:
+			return engineRecovering, fmt.Sprintf("engine probe could not run: %v", err)
+		case code != 0:
+			return engineRecovering, withProbeStderr(
+				"engine answered its socket but would not execute a statement", stderr)
+		}
+		return engineServing, ""
+	}
+}
+
+// Appends what the probe client said on stderr. A client that fails during the
+// handshake is refused by the server without the server ever logging why, so
+// this is the only account of that half of the exchange.
+func withProbeStderr(reason, stderr string) string {
+	stderr = strings.TrimSpace(stderr)
+	if stderr == "" {
+		return reason
+	}
+	return reason + "; the probe client reported: " + collapseProbeStderr(stderr)
+}
+
+// One line, bounded: the reason reaches a customer through StatusInfos, and a
+// client that repeats itself per attempt would otherwise crowd out the rest.
+func collapseProbeStderr(stderr string) string {
+	line := strings.Join(strings.Fields(strings.ReplaceAll(stderr, "\n", " ")), " ")
+	if len(line) > mariadbProbeStderrMaxBytes {
+		return line[:mariadbProbeStderrMaxBytes] + "..."
+	}
+	return line
+}
+
+// Appends the last few lines of the engine's error log to reason. An unset path,
+// an unreadable file or an empty one leaves reason alone: the probe's own
+// statement is still true, and a guessed cause would be worse than none.
+func withEngineLogTail(reason, path string) string {
+	tail := engineLogTail(path)
+	if tail == "" {
+		return reason
+	}
+	return reason + "; the engine's log ends: " + tail
+}
+
+func engineLogTail(path string) string {
+	if path == "" {
+		return ""
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return ""
+	}
+	defer func() { _ = f.Close() }()
+
+	info, err := f.Stat()
+	if err != nil {
+		return ""
+	}
+	offset := max(info.Size()-mariadbErrorLogTailBytes, 0)
+	buf := make([]byte, info.Size()-offset)
+	if _, err := f.ReadAt(buf, offset); err != nil {
+		return ""
+	}
+
+	lines := strings.Split(string(buf), "\n")
+	// A read that started mid-file almost certainly started mid-line, and half a
+	// line is more misleading than one fewer.
+	if offset > 0 && len(lines) > 0 {
+		lines = lines[1:]
+	}
+	kept := make([]string, 0, mariadbErrorLogTailLines)
+	for i := len(lines) - 1; i >= 0 && len(kept) < mariadbErrorLogTailLines; i-- {
+		if line := strings.TrimSpace(lines[i]); line != "" {
+			kept = append(kept, line)
+		}
+	}
+	slices.Reverse(kept)
+	return strings.Join(kept, " | ")
+}
+
+func readPidFile(path string) (int, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return 0, err
+	}
+	pid, err := strconv.Atoi(strings.TrimSpace(string(raw)))
+	if err != nil {
+		return 0, fmt.Errorf("%s does not hold a pid: %w", path, err)
+	}
+	return pid, nil
+}
+
+func (e *mariadbEngine) Stop(ctx context.Context) error {
+	return serviceAction(ctx, e.run, e.rcService, e.service, "stop")
+}
+
+// Only the parameter rollback and a failed apply call this: a restart the
+// control plane wants goes through RebootDBInstance, which cycles the VM. The
+// serving copy is refreshed first, since the server starts on what is installed.
+func (e *mariadbEngine) Restart(ctx context.Context) error {
+	if err := e.recordServingCopy(); err != nil {
+		return err
+	}
+	return serviceAction(ctx, e.run, e.rcService, e.service, "restart")
+}
+
+// The installed set copied to the name the pending-restart comparison reads. A
+// live SET GLOBAL apply deliberately does not touch it: that value was adopted
+// without a restart, which is what "not pending" means.
+func (e *mariadbEngine) recordServingCopy() error {
+	content, err := os.ReadFile(e.params.installedPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("read the installed parameters: %w", err)
+	}
+	return e.params.write(e.params.servingPath(), content)
+}
+
+// A value safe to interpolate into a single-quoted SQL literal. The quote is
+// doubled, which holds in every sql_mode, and the backslash is doubled, which is
+// why every statement built here pins a mode with backslash escapes on.
+func sqlLiteral(value string) string {
+	return strings.ReplaceAll(strings.ReplaceAll(value, `\`, `\\`), `'`, `''`)
+}
+
+// The spellings MariaDB parses for a boolean system variable, and the only ones
+// the catalog's own enum narrows a boolean to.
+var mariadbBooleanValues = map[string]string{
+	"on": "ON", "off": "OFF", "true": "ON", "false": "OFF", "1": "ON", "0": "OFF",
+}
+
+// One SET GLOBAL right-hand side. MariaDB refuses a quoted literal for a numeric
+// or boolean with ER_WRONG_TYPE_FOR_VAR, so the catalog's data type picks the
+// form; re-rendering from the parsed value is what keeps it safe to interpolate.
+func mariadbSetValue(dataType, value string) (string, error) {
+	switch dataType {
+	case handlers_rds.ParamTypeInteger:
+		n, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64)
+		if err != nil {
+			return "", fmt.Errorf("value %q is not an integer", value)
+		}
+		return strconv.FormatInt(n, 10), nil
+	case handlers_rds.ParamTypeReal:
+		f, err := strconv.ParseFloat(strings.TrimSpace(value), 64)
+		if err != nil {
+			return "", fmt.Errorf("value %q is not a real number", value)
+		}
+		return strconv.FormatFloat(f, 'g', -1, 64), nil
+	case handlers_rds.ParamTypeBoolean:
+		if literal, ok := mariadbBooleanValues[strings.ToLower(strings.TrimSpace(value))]; ok {
+			return literal, nil
+		}
+		return "", fmt.Errorf("value %q is not a boolean", value)
+	default:
+		return "'" + sqlLiteral(value) + "'", nil
+	}
+}
+
+// Turns off the two logs that would otherwise copy a statement's text, and pins
+// the mode the escaping above assumes. All three are session-scoped: a rotation
+// must not leave the customer's own general log switched off behind it.
+const mariadbSessionGuard = `SET SESSION sql_log_off = 1;
+SET SESSION slow_query_log = 0;
+SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';
+`
+
+// Rotates the master's password. The statement is built here rather than by the
+// client, which has neither psql's identifier interpolation nor its parameter
+// quoting, and it rides stdin so it is never visible in the process table.
+func (e *mariadbEngine) SetPassword(ctx context.Context, username, password string) error {
+	if username == "" || password == "" {
+		return fmt.Errorf("set-password requires both %s and %s",
+			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
+	}
+	// This runs as the server's own superuser, so a reserved name in the payload
+	// would rotate the platform's account. The whole rule matters here, not just
+	// the reserved set: the name is interpolated below rather than client-quoted.
+	if err := e.rules.validateUsername(username); err != nil {
+		return fmt.Errorf("refusing to set the password of role %q: %w", username, err)
+	}
+
+	escaped := sqlLiteral(password)
+	sql := fmt.Sprintf("%sALTER USER '%s'@'%s' IDENTIFIED BY '%s';\n",
+		mariadbSessionGuard, username, mariadbMasterHost, escaped)
+	if _, err := e.clientRun(ctx, sql); err != nil {
+		// The client echoes the failing statement, which carries the escaped form
+		// of the password rather than the raw one — so redacting only the raw one
+		// would leak exactly the passwords that needed escaping.
+		return fmt.Errorf("apply the master password: %s", redact(redact(err.Error(), password), escaped))
+	}
+	return nil
+}
+
+// Installs the resolved set and applies the half a running server can adopt.
+// MariaDB re-reads no file while up, so SET GLOBAL is the only immediate path.
+// It has no offline parser, so the safety net is the boot-time rollback.
+func (e *mariadbEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+
+	initialState, _ := e.probe.state(ctx)
+	if initialState == engineRecovering {
+		return nil, errors.New("apply parameters while the engine is still starting or recovering")
+	}
+	if initialState == engineServing {
+		// A command can arrive before the first heartbeat. Seed the configuration
+		// currently being served before replacing its file in that window.
+		if err := e.recordServingParametersLocked(ctx); err != nil {
+			return nil, fmt.Errorf("record the parameters serving before the apply: %w", err)
+		}
+	}
+
+	restore, err := e.params.install(params)
+	if err != nil {
+		return nil, err
+	}
+
+	state, _ := e.probe.state(ctx)
+	switch state {
+	case engineAbsent:
+		return e.restartOnRepairSetLocked(ctx)
+	case engineRecovering:
+		return nil, errors.Join(errors.New("the engine entered startup or recovery during the parameter apply"), restore())
+	}
+
+	// Before the set reaches the server: enforcement is one of the values below,
+	// and a server told to require a transport it cannot offer would refuse every
+	// client the statement after.
+	if err := e.requireServingTLSForEnforcement(ctx); err != nil {
+		return nil, errors.Join(err, restore())
+	}
+
+	if err := e.applyDynamicParameters(ctx, params); err != nil {
+		// The engine may have gone down after the first probe. In that case start it
+		// on the newly installed set rather than restoring one that already left it
+		// unable to serve.
+		if state, _ := e.probe.state(ctx); state == engineAbsent {
+			return e.restartOnRepairSetLocked(ctx)
+		}
+		return nil, errors.Join(err, restore())
+	}
+	// Last, and on its own: the batch above is not a transaction, so moving the
+	// TLS posture inside it would leave enforcement changed on a set the server
+	// went on to refuse, with only the parameter file rolled back.
+	if err := e.applyTLSEnforcement(ctx); err != nil {
+		if state, _ := e.probe.state(ctx); state == engineAbsent {
+			return e.restartOnRepairSetLocked(ctx)
+		}
+		return nil, errors.Join(err, restore())
+	}
+	return e.pendingRestartParameters(ctx)
+}
+
+// The dynamic half, in one invocation that stops at the first refusal so the
+// server names the setting it would not take. Only catalog names are emitted,
+// which is what makes the identifier safe to build into the statement.
+//
+// TLS enforcement is excluded: applyTLSEnforcement issues it after this batch
+// has succeeded.
+func (e *mariadbEngine) applyDynamicParameters(ctx context.Context, params []handlers_rds.Parameter) error {
+	var b strings.Builder
+	b.WriteString("SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';\n")
+	applied := 0
+	for _, p := range params {
+		if p.Name == "" || p.Name == e.rules.tlsEnforcementParameter || e.rules.isStatic(p.Name) {
+			continue
+		}
+		value, err := mariadbSetValue(e.rules.dataType(p.Name), p.Value)
+		if err != nil {
+			return fmt.Errorf("apply the dynamic parameters: %s: %w", p.Name, err)
+		}
+		fmt.Fprintf(&b, "SET GLOBAL %s = %s;\n", p.Name, value)
+		applied++
+	}
+	if applied == 0 {
+		return nil
+	}
+	if _, err := e.clientRun(ctx, b.String()); err != nil {
+		return fmt.Errorf("apply the dynamic parameters: %w", err)
+	}
+	return nil
+}
+
+// Puts the installed set's enforcement on the running server. The value comes
+// from the installed file rather than the set in hand, so a set that predates
+// the parameter enforces here rather than leaving the server on mariadbd's own
+// default of off while the API reports enforcement.
+func (e *mariadbEngine) applyTLSEnforcement(ctx context.Context) error {
+	enforce, err := installedTLSEnforcement(e.rules.tlsEnforcementParameter, e.params.installedPath())
+	if err != nil {
+		return err
+	}
+	previous, err := e.readTLSEnforcement(ctx)
+	if err != nil {
+		return err
+	}
+	if previous == enforce {
+		return nil
+	}
+	applyErr := e.setTLSEnforcement(ctx, enforce)
+	if applyErr == nil {
+		return nil
+	}
+	// The client can fail without saying whether the server ran the statement, so
+	// the posture it left behind is read rather than assumed unchanged: the one
+	// outcome that must not survive a failed apply is plaintext nobody asked for.
+	live, readErr := e.readTLSEnforcement(ctx)
+	if readErr != nil {
+		return errors.Join(applyErr, readErr)
+	}
+	if live == previous {
+		return applyErr
+	}
+	return errors.Join(applyErr, e.setTLSEnforcement(ctx, previous))
+}
+
+func (e *mariadbEngine) readTLSEnforcement(ctx context.Context) (bool, error) {
+	name := e.rules.tlsEnforcementParameter
+	out, err := e.clientRun(ctx, "SELECT @@global."+name+";\n")
+	if err != nil {
+		return false, fmt.Errorf("read whether the engine requires TLS of clients: %w", err)
+	}
+	switch value := strings.TrimSpace(out); value {
+	case "1":
+		return true, nil
+	case "0":
+		return false, nil
+	default:
+		return false, fmt.Errorf("the engine reports %s as %q, which is neither 1 nor 0", name, value)
+	}
+}
+
+func (e *mariadbEngine) setTLSEnforcement(ctx context.Context, enforce bool) error {
+	name, value := e.rules.tlsEnforcementParameter, "0"
+	if enforce {
+		value = "1"
+	}
+	// Rendered the way the batch renders every other boolean, so the statement the
+	// server sees does not depend on which path issued it.
+	literal, err := mariadbSetValue(handlers_rds.ParamTypeBoolean, value)
+	if err != nil {
+		return fmt.Errorf("render %s: %w", name, err)
+	}
+	sql := fmt.Sprintf("SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';\nSET GLOBAL %s = %s;\n", name, literal)
+	if _, err := e.clientRun(ctx, sql); err != nil {
+		return fmt.Errorf("set %s to %s: %w", name, literal, err)
+	}
+	return nil
+}
+
+// Refuses to require TLS of clients from a server that is not serving it, which
+// would reject every TCP connection while the parameter path reported success.
+// Read back from the installed file rather than taken from the set in hand, so
+// what is checked is also what the server reads at its next start.
+func (e *mariadbEngine) requireServingTLSForEnforcement(ctx context.Context) error {
+	enforce, err := installedTLSEnforcement(e.rules.tlsEnforcementParameter, e.params.installedPath())
+	if err != nil {
+		return err
+	}
+	if !enforce {
+		return nil
+	}
+	// DISABLED is a server built with TLS and started without a certificate,
+	// which is what a boot that was handed none leaves behind. The agent's own
+	// connections are unaffected either way: MariaDB counts a unix socket as a
+	// secure transport.
+	out, err := e.clientRun(ctx, "SELECT @@global.have_ssl;\n")
+	if err != nil {
+		return fmt.Errorf("read whether the engine is serving TLS: %w", err)
+	}
+	if serving := strings.TrimSpace(out); serving != "YES" {
+		return fmt.Errorf("the engine is serving without TLS (have_ssl = %q), so requiring it of clients would reject every connection", serving)
+	}
+	return nil
+}
+
+func (e *mariadbEngine) restartOnRepairSetLocked(ctx context.Context) ([]string, error) {
+	return awaitRepairedEngine(ctx, e.probe, e.Restart, e.pendingRestartParameters, e.repairTimeout, e.repairPoll)
+}
+
+// MariaDB has no pending_restart, so this is computed from the files: the
+// catalog-static keys whose installed value differs from the one the server
+// started on. Live values are not compared — mysqld silently rewrites them.
+func (e *mariadbEngine) pendingRestartParameters(context.Context) ([]string, error) {
+	installed, err := readOptionFile(e.params.installedPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			// Nothing is installed, so nothing is waiting to be adopted.
+			return nil, nil
+		}
+		return nil, fmt.Errorf("read the installed parameters: %w", err)
+	}
+	serving, err := readOptionFile(e.params.servingPath())
+	if err != nil && !os.IsNotExist(err) {
+		return nil, fmt.Errorf("read the parameters the engine started on: %w", err)
+	}
+	// Back to the catalog's names before anything is classified or reported: the
+	// files are written in the engine's startup spellings, and the answer is a
+	// customer-facing list the API refuses ApplyMethod=immediate against.
+	installed = e.catalogKeyed(installed)
+	serving = e.catalogKeyed(serving)
+
+	// An absent serving copy is a set the engine has not started on: rds-init
+	// writes the two together on every boot, so the whole static half counts as
+	// pending rather than being promoted on the strength of a missing file.
+	var pending []string
+	for name, value := range installed {
+		if !e.rules.isStatic(name) {
+			continue
+		}
+		if served, ok := serving[name]; !ok || served != value {
+			pending = append(pending, name)
+		}
+	}
+	// A static setting the group stopped naming reverts to its default at the next
+	// start, which the running server has not adopted either.
+	for name := range serving {
+		if _, ok := installed[name]; !ok && e.rules.isStatic(name) {
+			pending = append(pending, name)
+		}
+	}
+	slices.Sort(pending)
+	return pending, nil
+}
+
+// Snapshots the installed set as the rollback target, but only when nothing in
+// it is still waiting for a restart. The parameter mutex keeps an apply from
+// replacing the file between that check and the copy.
+func (e *mariadbEngine) RecordServingParameters(ctx context.Context) error {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return e.recordServingParametersLocked(ctx)
+}
+
+func (e *mariadbEngine) recordServingParametersLocked(ctx context.Context) error {
+	return recordLastGood(ctx, e.params, e.pendingRestartParameters)
+}
+
+// Puts the last set the engine accepted back in place, for a start that failed
+// after a parameter change.
+func (e *mariadbEngine) RestoreLastKnownGoodParameters(ctx context.Context) (bool, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return restoreLastGood(ctx, e.params, e.probe)
+}
+
+// The same settings under the names the catalog and the API know them by, since
+// a file is written in the engine's startup spellings and those are not always
+// the customer's.
+func (e *mariadbEngine) catalogKeyed(values map[string]string) map[string]string {
+	out := make(map[string]string, len(values))
+	for name, value := range values {
+		out[e.rules.catalogName(name)] = value
+	}
+	return out
+}
+
+// The subset of MariaDB's option-file syntax the generated drop-ins are written
+// in: a group header, comments, and one `name = value` per line. Nothing else
+// reaches these two files, because only rds-init and this agent write them.
+func readOptionFile(path string) (map[string]string, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	values := make(map[string]string)
+	for line := range strings.SplitSeq(string(raw), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") || strings.HasPrefix(line, "[") {
+			continue
+		}
+		// A bare option such as skip-log-bin carries no value; it is still a setting
+		// whose presence or absence differs.
+		name, value, _ := strings.Cut(line, "=")
+		values[normaliseOptionName(name)] = strings.Trim(strings.TrimSpace(value), `"'`)
+	}
+	return values, nil
+}
+
+// MariaDB reads - and _ as the same character in an option name, so the two
+// spellings of one setting must not compare as two settings.
+func normaliseOptionName(name string) string {
+	return strings.ReplaceAll(strings.TrimSpace(name), "-", "_")
+}
+
+// How long any one BACKUP STAGE waits for its metadata locks. Well under the
+// control plane's quiesce timeout, so a stage that cannot take its lock is
+// reported rather than left queued in front of live traffic.
+const mariadbQuiesceLockWait = 20 * time.Second
+
+// MariaDB's own backup API rather than FLUSH TABLES WITH READ LOCK: FTWRL makes
+// the whole database read-only for the full hold, and its acquisition phase is
+// unbounded. BACKUP STAGE blocks commits instead, and covers Aria and MyISAM.
+func (e *mariadbEngine) Quiesce(ctx context.Context, label string, hold time.Duration) error {
+	if err := validateQuiesceRequest(label, hold); err != nil {
+		return err
+	}
+
+	// Held across the whole start, so a second quiesce waits and then finds the
+	// first one's hold rather than opening a concurrent backup alongside it.
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.held != nil {
+		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)
+	}
+
+	// Deliberately not the command's context: the session has to outlive the call
+	// that started it, and its own deadline is what bounds it instead.
+	session, err := e.startSess(context.WithoutCancel(ctx), command{
+		Name:              e.client,
+		Args:              e.clientArgs(),
+		Env:               []string{"PATH=" + defaultGuestPath},
+		SentinelStatement: "SELECT '" + sessionSentinel + "';\n",
+	})
+	if err != nil {
+		return fmt.Errorf("open a backup session: %w", err)
+	}
+
+	// In order and on one connection: MariaDB releases the hold with the session
+	// that took it, which bounds a control plane that dies mid-snapshot. Fed one
+	// at a time so a stage that cannot take its locks is named.
+	stages := []string{
+		fmt.Sprintf("SET SESSION lock_wait_timeout = %d;\n", int(mariadbQuiesceLockWait.Seconds())),
+		"BACKUP STAGE START;\n",
+		"BACKUP STAGE FLUSH;\n",
+		"BACKUP STAGE BLOCK_DDL;\n",
+		"BACKUP STAGE BLOCK_COMMIT;\n",
+	}
+	for _, sql := range stages {
+		if err := session.Exec(ctx, sql); err != nil {
+			if closeErr := session.Close(); closeErr != nil {
+				slog.Warn("rds-agent: closing a failed backup session", "err", closeErr)
+			}
+			return fmt.Errorf("put the engine into backup mode at %q: %w", strings.TrimSpace(sql), err)
+		}
+	}
+
+	e.beginHoldLocked(label, session, hold)
+	return nil
+}
+
+func (e *mariadbEngine) Unquiesce(ctx context.Context) error {
+	return e.releaseHold(ctx, "BACKUP STAGE END;\n")
+}
+
+// Feeds sql on stdin rather than as an argument, so a statement is never visible
+// in the process table.
+func (e *mariadbEngine) clientRun(ctx context.Context, sql string) (string, error) {
+	return e.run(ctx, command{
+		Name:  e.client,
+		Args:  e.clientArgs(),
+		Env:   []string{"PATH=" + defaultGuestPath},
+		Stdin: sql,
+	})
+}
+
+// --unbuffered so a held session's output reaches the reader as each statement
+// finishes rather than when a pipe buffer fills. No User on the command: the
+// connection args carry it, and the agent runs as the account they name.
+func (e *mariadbEngine) clientArgs() []string {
+	return append(mariadbSocketConnectArgs(e.socket),
+		"--batch", "--skip-column-names", "--unbuffered")
+}

cmd/rds-agent/engine_mariadb_log_test.go

@@ -0,0 +1,211 @@
+package main
+
+//test:in-package — the agent is a main package, which has no external test
+// package to import it from, and this covers the unexported engineLogTail and
+// probe-reason helpers directly.
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+)
+
+const mariadbRDSInitScript = "../../scripts/images/rds-mariadb/rds-init"
+
+func writeEngineLog(t *testing.T, body string) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "error.log")
+	if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
+		t.Fatalf("write engine log: %v", err)
+	}
+	return path
+}
+
+// The whole point of the log: "not answering on its socket" is the same message
+// for a server still recovering and for one that refused to start, and only the
+// engine's own log separates them.
+func TestMariaDBProbe_QuotesTheEngineLogWhenTheSocketIsSilent(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	cfg.EngineErrorLog = writeEngineLog(t, strings.Join([]string{
+		"2026-08-12  1:49:26 0 [Note] Starting MariaDB 11.8.0",
+		"2026-08-12  1:49:26 0 [ERROR] InnoDB: Cannot allocate memory for the buffer pool",
+		"2026-08-12  1:49:26 0 [ERROR] Plugin 'InnoDB' registration as a STORAGE ENGINE failed.",
+		"",
+	}, "\n"))
+	writePid(t, cfg.EnginePidFile, os.Getpid())
+
+	state, message := mariadbProbeState(cfg,
+		func(context.Context, string, ...string) (int, string, error) { return 1, "", nil },
+		func(int) bool { return true })(t.Context(), int64(cfg.EnginePort))
+
+	if state != engineRecovering {
+		t.Errorf("state = %v, want recovering", state)
+	}
+	if !strings.Contains(message, "Cannot allocate memory for the buffer pool") {
+		t.Errorf("message = %q, want the engine's own refusal quoted", message)
+	}
+}
+
+func TestMariaDBProbe_SurvivesAnAbsentEngineLog(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	cfg.EngineErrorLog = filepath.Join(t.TempDir(), "does-not-exist.log")
+	writePid(t, cfg.EnginePidFile, os.Getpid())
+
+	state, message := mariadbProbeState(cfg,
+		func(context.Context, string, ...string) (int, string, error) { return 1, "", nil },
+		func(int) bool { return true })(t.Context(), int64(cfg.EnginePort))
+
+	if state != engineRecovering {
+		t.Errorf("state = %v, want recovering", state)
+	}
+	if message != "engine is not answering on its socket yet (startup or crash recovery)" {
+		t.Errorf("message = %q, want the bare reason when there is no log to quote", message)
+	}
+}
+
+// A client refused during the handshake is closed by the server without the
+// server ever logging a cause, so the engine log alone leaves the failure
+// unexplained and only the client's own stderr names it.
+func TestMariaDBProbe_QuotesTheProbeClientWhenItIsRefused(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	cfg.EngineErrorLog = writeEngineLog(t, "2026-08-12  4:32:39 148 [Warning] Aborted connection 148\n")
+	writePid(t, cfg.EnginePidFile, os.Getpid())
+
+	state, message := mariadbProbeState(cfg,
+		func(context.Context, string, ...string) (int, string, error) {
+			return 1, "ERROR 2026 (HY000): TLS/SSL error: self-signed certificate\n", nil
+		},
+		func(int) bool { return true })(t.Context(), int64(cfg.EnginePort))
+
+	if state != engineRecovering {
+		t.Errorf("state = %v, want recovering", state)
+	}
+	if !strings.Contains(message, "self-signed certificate") {
+		t.Errorf("message = %q, want the probe client's own error quoted", message)
+	}
+	if !strings.Contains(message, "Aborted connection") {
+		t.Errorf("message = %q, want the engine log kept alongside the client's error", message)
+	}
+}
+
+func TestCollapseProbeStderr(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{name: "folds to one line", in: "ERROR 2026:\n  TLS error\n", want: "ERROR 2026: TLS error"},
+		{name: "empty stays empty", in: "   \n", want: ""},
+		{
+			name: "bounded",
+			in:   strings.Repeat("y", mariadbProbeStderrMaxBytes+50),
+			want: strings.Repeat("y", mariadbProbeStderrMaxBytes) + "...",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := collapseProbeStderr(tt.in); got != tt.want {
+				t.Errorf("collapseProbeStderr() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+// A probe that says nothing must not append an empty clause: the reason reaches
+// a customer, and a dangling "reported:" would read as truncated output.
+func TestWithProbeStderrLeavesReasonAloneWhenTheClientIsSilent(t *testing.T) {
+	const reason = "engine is not answering on its socket yet"
+	if got := withProbeStderr(reason, "  \n "); got != reason {
+		t.Errorf("withProbeStderr() = %q, want the reason unchanged", got)
+	}
+}
+
+func TestEngineLogTail(t *testing.T) {
+	tests := []struct {
+		name        string
+		body        string
+		want        string
+		wantMissing string
+	}{
+		{
+			name: "keeps only the last lines",
+			body: "one\ntwo\nthree\nfour\nfive\nsix\n",
+			want: "three | four | five | six",
+		},
+		{
+			name: "skips blank lines",
+			body: "alpha\n\n\nbeta\n\n",
+			want: "alpha | beta",
+		},
+		{
+			name: "empty file yields nothing",
+			body: "",
+			want: "",
+		},
+		{
+			name:        "drops a line the byte cap cut in half",
+			body:        strings.Repeat("x", mariadbErrorLogTailBytes) + "\nlast\n",
+			want:        "last",
+			wantMissing: "x",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := engineLogTail(writeEngineLog(t, tt.body))
+			if got != tt.want {
+				t.Errorf("engineLogTail() = %q, want %q", got, tt.want)
+			}
+			if tt.wantMissing != "" && strings.Contains(got, tt.wantMissing) {
+				t.Errorf("engineLogTail() = %q, want no partial line", got)
+			}
+		})
+	}
+}
+
+func TestEngineLogTailUnsetPath(t *testing.T) {
+	if got := engineLogTail(""); got != "" {
+		t.Errorf("engineLogTail(\"\") = %q, want empty", got)
+	}
+}
+
+// The agent reads a path the image writes, and nothing at runtime would notice
+// the two drifting apart: the log would simply always be missing and every
+// refusal would go back unexplained again.
+func TestMariaDBErrorLogPathMatchesTheImage(t *testing.T) {
+	raw, err := os.ReadFile(mariadbRDSInitScript)
+	if err != nil {
+		t.Fatalf("read rds-init: %v", err)
+	}
+	script := string(raw)
+
+	logDir := shellDefault(t, script, "LOG_DIR", "RDS_LOG_DIR")
+	engineLog := shellDefault(t, script, "ENGINE_LOG", "RDS_ENGINE_LOG")
+	want := strings.Replace(engineLog, "${LOG_DIR}", logDir, 1)
+
+	if got := engineLayouts[engineMariaDB].errorLog; got != want {
+		t.Errorf("layout errorLog = %q, but rds-init writes log_error = %q", got, want)
+	}
+}
+
+// Pulls the fallback out of a `NAME="${OVERRIDE:-fallback}"` assignment.
+func shellDefault(t *testing.T, script, name, override string) string {
+	t.Helper()
+	re := regexp.MustCompile(`(?m)^` + name + `="\$\{` + override + `:-(.*)\}"$`)
+	m := re.FindStringSubmatch(script)
+	if m == nil {
+		t.Fatalf("rds-init has no %s assignment defaulting from %s", name, override)
+	}
+	return m[1]
+}
+
+func TestWithEngineLogTailLeavesReasonAloneWithoutALog(t *testing.T) {
+	const reason = "engine is not answering on its socket yet"
+	if got := withEngineLogTail(reason, ""); got != reason {
+		t.Errorf("withEngineLogTail() = %q, want the reason unchanged", got)
+	}
+}

cmd/rds-agent/engine_postgres.go

@@ -0,0 +1,547 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The engine is reached over its unix socket under peer authentication, so the
+// agent drops to the postgres OS user rather than holding a password of its own.
+type postgresEngine struct {
+	quiesceState
+	parameterManager
+
+	// The control plane's own metadata for this engine, resolved once at
+	// startup: the live password apply runs as the cluster superuser, so it
+	// re-checks the role name against the same reserved set the API validates.
+	meta      handlers_rds.Engine
+	run       commandRunner
+	startSess sessionRunner
+	psql      string
+	rcService string
+	service   string
+	pgData    string
+	socketDir string
+	osUser    string
+}
+
+var _ engine = (*postgresEngine)(nil)
+
+// pg_isready gives PostgreSQL three states for free: exit 0 is serving, exit 1
+// is a postmaster up and rejecting connections, anything else is an engine that
+// is not there. Resolved on PATH, where the client package puts it.
+const postgresProbeBinary = "pg_isready"
+
+func newPostgresProbe(cfg config, run probeRunner) *engineProbe {
+	return newEngineProbe(cfg.EnginePort, postgresProbeState(cfg.EngineHost, run))
+}
+
+func postgresProbeState(host string, run probeRunner) probeStateFn {
+	return func(ctx context.Context, port int64) (engineState, string) {
+		portArg := strconv.FormatInt(port, 10)
+		code, _, err := run(ctx, postgresProbeBinary, "-h", host, "-p", portArg, "-q")
+		switch {
+		case err != nil:
+			// A missing binary or broken image. Reporting healthy on the strength of
+			// nothing would hide it, so this reads as absent like an engine that did
+			// not answer.
+			return engineAbsent, fmt.Sprintf("engine probe could not run: %v", err)
+		case code == 0:
+			return engineServing, ""
+		case code == 1:
+			return engineRecovering, "engine is rejecting connections (startup or recovery)"
+		default:
+			return engineAbsent, fmt.Sprintf("engine did not respond on %s:%s", host, portArg)
+		}
+	}
+}
+
+// Resolved once, under the name the control plane knows this implementation by.
+// A lookup failure is a mismatch between the agent and the control plane it was
+// built against, so it fails at startup rather than at the first rotation.
+var postgresEngineMeta = mustLookupEngine(enginePostgres)
+
+func mustLookupEngine(name string) handlers_rds.Engine {
+	engine, err := handlers_rds.LookupEngine(name)
+	if err != nil {
+		panic("rds-agent: " + err.Error())
+	}
+	return engine
+}
+
+// The include the resolved set is rendered to, and the copy of the last one the
+// engine accepted. Both live beside the data rather than in /etc, matching
+// rds-init: a class change boots a fresh root volume and would revert them.
+const (
+	postgresParametersFile = "10-rds-parameters.conf"
+	// Deliberately not a .conf name: include_dir globs *.conf, so the rollback
+	// copy must not be read as a second set of settings.
+	postgresLastGoodFile = "10-rds-parameters.last-good"
+)
+
+// The pg_hba rule that implements TLS enforcement, and the directory the
+// generated pg_hba includes it from. Its content is a constant, so the copy
+// rds-init writes in shell and the one written here have no template to drift
+// apart on: enforcement is the presence of the file and nothing more.
+const (
+	postgresHBADir           = "hba.d"
+	postgresForceSSLRuleFile = "20-rds-force-ssl.conf"
+	// An explicit reject rather than an omitted permit: both refuse the
+	// connection, and only this one puts the reason in the server log and in the
+	// client's error.
+	postgresForceSSLRules = `# Managed by the platform: the rule that requires TLS of client connections.
+# Written by rds-init at boot and by rds-agent on a parameter apply.
+hostnossl all all 0.0.0.0/0 reject
+hostnossl all all ::/0 reject
+`
+	// What the read-back must count once the file is in place. Asserting the shape
+	// rather than only the absence of parse errors is what catches the failure
+	// that will actually happen: the derivation above writing the wrong thing.
+	postgresForceSSLRuleCount = 2
+)
+
+func newPostgresEngine(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) *postgresEngine {
+	return &postgresEngine{
+		meta:      postgresEngineMeta,
+		run:       run,
+		startSess: startSess,
+		psql:      filepath.Join(cfg.EngineBinDir, "psql"),
+		rcService: cfg.RCService,
+		service:   cfg.EngineService,
+		pgData:    cfg.EngineDataDir,
+		socketDir: cfg.SocketDir,
+		osUser:    cfg.EngineUser,
+		parameterManager: parameterManager{
+			probe: probe,
+			params: parameterStore{
+				dir:       filepath.Join(cfg.EngineDataDir, "conf.d"),
+				installed: postgresParametersFile,
+				lastGood:  postgresLastGoodFile,
+				osUser:    cfg.EngineUser,
+				engine:    enginePostgres,
+			},
+			repairTimeout: parameterRepairTimeout,
+			repairPoll:    parameterRepairPoll,
+		},
+	}
+}
+
+// The role name and the password ride the environment and are re-quoted by
+// psql, so neither reaches a shell word or an argv another process can read.
+func (e *postgresEngine) SetPassword(ctx context.Context, username, password string) error {
+	if username == "" || password == "" {
+		return fmt.Errorf("set-password requires both %s and %s",
+			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
+	}
+	// The statement below runs as the cluster superuser over the socket under
+	// peer auth, so a reserved name in the command payload would hand the
+	// customer the bootstrap superuser rather than rotate their own role.
+	if err := e.meta.ValidateUsernameNotReserved(username); err != nil {
+		return fmt.Errorf("refusing to set the password of a role the engine reserves: %w", err)
+	}
+	// psql interpolates the password into the ALTER ROLE before the server sees
+	// it, and these three are what would write it to the log — the last on any
+	// failure at its own default. SUSET, so the parameter group cannot win.
+	const sql = `SET log_statement = 'none';
+SET log_min_duration_statement = -1;
+SET log_min_error_statement = 'panic';
+\getenv master RDS_MASTER_USERNAME
+\getenv password RDS_MASTER_PASSWORD
+ALTER ROLE :"master" WITH LOGIN PASSWORD :'password';
+`
+	_, err := e.psqlRun(ctx, sql,
+		"RDS_MASTER_USERNAME="+username,
+		"RDS_MASTER_PASSWORD="+password,
+	)
+	if err != nil {
+		// psql echoes the failing statement, which here would carry the new
+		// password back to the control plane and into the event ring.
+		return fmt.Errorf("apply the master password: %s", redact(err.Error(), password))
+	}
+	return nil
+}
+
+// Installs the resolved set, validates it with the engine's own parser before
+// the engine adopts it, and reloads. A refused value is rolled back here rather
+// than left on the data volume, where it would survive every VM replace.
+func (e *postgresEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+
+	initialState, _ := e.probe.state(ctx)
+	if initialState == engineRecovering {
+		return nil, errors.New("apply parameters while the engine is still starting or recovering")
+	}
+	if initialState == engineServing {
+		// A command can arrive before the first heartbeat. Seed the configuration
+		// currently being served before replacing its file in that window.
+		if err := e.recordServingParametersLocked(ctx); err != nil {
+			return nil, fmt.Errorf("record the parameters serving before the apply: %w", err)
+		}
+	}
+
+	// The check runs against the file in place, because the engine parses the
+	// datadir's own include_dir. The window that leaves is closed by the rollback
+	// below and the last-known-good restore the agent runs at boot.
+	restoreParameters, err := e.params.install(params)
+	if err != nil {
+		return nil, err
+	}
+	if err := e.checkConfig(ctx); err != nil {
+		return nil, errors.Join(fmt.Errorf("the engine rejected the parameter set: %w", err), restoreParameters())
+	}
+
+	state, _ := e.probe.state(ctx)
+	switch state {
+	case engineAbsent:
+		return e.restartOnRepairSetLocked(ctx)
+	case engineRecovering:
+		return nil, errors.Join(errors.New("the engine entered startup or recovery during the parameter apply"), restoreParameters())
+	}
+
+	// After the engine's own parser has passed the set and before the reload that
+	// adopts both files at once.
+	enforce, restoreEnforcement, err := e.syncTLSEnforcement(ctx, state)
+	if err != nil {
+		return nil, errors.Join(fmt.Errorf("apply the TLS enforcement of the parameter set: %w", err), restoreParameters())
+	}
+	// The two files carry one apply from here: a rejected set withdrawn without
+	// its enforcement rule would leave the instance serving the old parameters
+	// under the new set's TLS posture.
+	restore := func() error { return errors.Join(restoreEnforcement(), restoreParameters()) }
+
+	if err := e.reload(ctx); err != nil {
+		// The engine may have gone down after the first probe. In that case start
+		// it on the new, parser-checked repair set rather than restoring the set
+		// that already left it unable to serve.
+		if state, _ := e.probe.state(ctx); state == engineAbsent {
+			return e.restartOnRepairSetLocked(ctx)
+		}
+		return nil, errors.Join(err, restore())
+	}
+	if err := e.verifyTLSEnforcement(ctx, enforce); err != nil {
+		// The reload already happened, so putting the files back is not enough: a
+		// pg_hba that parsed but says the wrong thing has been adopted, and only a
+		// second reload takes the engine off it.
+		return nil, errors.Join(err, restore(), e.reload(ctx))
+	}
+	return e.pendingRestartParameters(ctx)
+}
+
+// Re-derived before the restart: the engine reads pg_hba when it starts, so a
+// repair set started under the previous set's enforcement would serve one TLS
+// posture while the API reported the other.
+func (e *postgresEngine) restartOnRepairSetLocked(ctx context.Context) ([]string, error) {
+	if _, _, err := e.syncTLSEnforcement(ctx, engineAbsent); err != nil {
+		return nil, fmt.Errorf("apply the TLS enforcement of the repair set: %w", err)
+	}
+	return awaitRepairedEngine(ctx, e.probe, e.Restart, e.pendingRestartParameters, e.repairTimeout, e.repairPoll)
+}
+
+// The function's own answer is the reload, not psql's exit status: it returns f
+// when it could not signal the postmaster, and the read-back that follows parses
+// pg_hba from disk, so an unsignalled reload would otherwise verify clean.
+func (e *postgresEngine) reload(ctx context.Context) error {
+	out, err := e.psqlRun(ctx, "SELECT pg_reload_conf();\n")
+	if err != nil {
+		return fmt.Errorf("reload the engine configuration: %w", err)
+	}
+	if result := strings.TrimSpace(out); result != "t" {
+		return fmt.Errorf("reload the engine configuration: pg_reload_conf() returned %q", result)
+	}
+	return nil
+}
+
+// The postmaster's own answer: a static setting it has stored but not adopted is
+// what it reports here, so nothing in the guest has to classify one.
+func (e *postgresEngine) pendingRestartParameters(ctx context.Context) ([]string, error) {
+	out, err := e.psqlRun(ctx, "SELECT name FROM pg_settings WHERE pending_restart ORDER BY name;\n")
+	if err != nil {
+		return nil, fmt.Errorf("read the settings pending a restart: %w", err)
+	}
+	var pending []string
+	for line := range strings.SplitSeq(out, "\n") {
+		if name := strings.TrimSpace(line); name != "" {
+			pending = append(pending, name)
+		}
+	}
+	return pending, nil
+}
+
+// Snapshots the include only when the running postmaster has no settings still
+// pending a restart. The parameter mutex keeps an apply from replacing the file
+// between that check and the copy.
+func (e *postgresEngine) RecordServingParameters(ctx context.Context) error {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return e.recordServingParametersLocked(ctx)
+}
+
+func (e *postgresEngine) recordServingParametersLocked(ctx context.Context) error {
+	return recordLastGood(ctx, e.params, e.pendingRestartParameters)
+}
+
+// Puts the last set the engine accepted back in place, for a restart that failed
+// after a parameter change.
+func (e *postgresEngine) RestoreLastKnownGoodParameters(ctx context.Context) (bool, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	restored, err := restoreLastGood(ctx, e.params, e.probe)
+	if err != nil || !restored {
+		return restored, err
+	}
+	// A service restart does not re-run rds-init, so a restore that stopped at the
+	// parameter file would bring the last known good set up under the enforcement
+	// of the set that was just rejected. restoreLastGood only replaces the file of
+	// an engine that is down.
+	if _, _, err := e.syncTLSEnforcement(ctx, engineAbsent); err != nil {
+		return true, fmt.Errorf("apply the TLS enforcement of the restored parameter set: %w", err)
+	}
+	return true, nil
+}
+
+// Turns the installed enforcement value into the pg_hba fact that implements it.
+// PostgreSQL has no server setting for this, so the parameter file on its own
+// would report the change applied and enforce nothing.
+//
+// The serving-TLS guard runs only against an engine that is up. The restore and
+// repair paths reach this with the engine down, where the rule is read by the
+// start that follows rather than by a reload.
+func (e *postgresEngine) syncTLSEnforcement(ctx context.Context, state engineState) (enforce bool, restore func() error, err error) {
+	enforce, err = installedTLSEnforcement(e.meta.TLSEnforcementParameter(), e.params.installedPath())
+	if err != nil {
+		return false, nil, err
+	}
+	if enforce && state == engineServing {
+		if err := e.requireServingTLS(ctx); err != nil {
+			return false, nil, err
+		}
+	}
+	restore, err = e.writeTLSEnforcement(enforce)
+	if err != nil {
+		return false, nil, err
+	}
+	return enforce, restore, nil
+}
+
+// Writes or removes the enforcement rule, returning a func putting back whatever
+// was there. Removing matters as much as writing: a restored data volume carries
+// hba.d with it, so a stale rule would keep enforcing under a parameter group
+// that turns enforcement off.
+func (e *postgresEngine) writeTLSEnforcement(enforce bool) (func() error, error) {
+	path := e.forceSSLRulePath()
+	previous, readErr := os.ReadFile(path)
+	if readErr != nil && !os.IsNotExist(readErr) {
+		return nil, fmt.Errorf("read the TLS enforcement rule at %s: %w", path, readErr)
+	}
+	existed := readErr == nil
+	restore := func() error {
+		if !existed {
+			return e.removeTLSEnforcement()
+		}
+		return e.params.write(path, previous)
+	}
+
+	if !enforce {
+		if err := e.removeTLSEnforcement(); err != nil {
+			return nil, err
+		}
+		return restore, nil
+	}
+	// rds-init lays the directory down on every boot, so this covers an agent that
+	// reached an apply before one ran rather than the ordinary case.
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, 0o700); err != nil {
+		return nil, fmt.Errorf("create %s: %w", dir, err)
+	}
+	if err := e.params.chownToEngine(dir); err != nil {
+		return nil, err
+	}
+	if err := e.params.write(path, []byte(postgresForceSSLRules)); err != nil {
+		return nil, err
+	}
+	return restore, nil
+}
+
+func (e *postgresEngine) removeTLSEnforcement() error {
+	path := e.forceSSLRulePath()
+	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("withdraw the TLS enforcement rule at %s: %w", path, err)
+	}
+	return nil
+}
+
+func (e *postgresEngine) forceSSLRulePath() string {
+	return filepath.Join(e.pgData, postgresHBADir, postgresForceSSLRuleFile)
+}
+
+// Refuses to require TLS of clients from an engine that is not serving it, which
+// would reject every connection while the parameter path reported success.
+func (e *postgresEngine) requireServingTLS(ctx context.Context) error {
+	out, err := e.psqlRun(ctx, "SHOW ssl;\n")
+	if err != nil {
+		return fmt.Errorf("read whether the engine is serving TLS: %w", err)
+	}
+	if serving := strings.TrimSpace(out); serving != "on" {
+		return fmt.Errorf("the engine is serving without TLS (ssl = %q), so requiring it of clients would reject every connection", serving)
+	}
+	return nil
+}
+
+// PostgreSQL discards a pg_hba it cannot parse on reload and keeps the rules it
+// is already serving, which presents as an apply that succeeded and enforces
+// nothing. So the file is read back.
+//
+// pg_hba_file_rules re-parses from disk, so this proves the file would be
+// adopted rather than that the postmaster has adopted it. The inference holds
+// because a reloaded pg_hba is discarded only on a parse error.
+const postgresHBAReadBack = `SELECT count(*) FILTER (WHERE type = 'hostnossl' AND auth_method = 'reject'),
+       count(*) FILTER (WHERE error IS NOT NULL) FROM pg_hba_file_rules;
+`
+
+func (e *postgresEngine) verifyTLSEnforcement(ctx context.Context, enforce bool) error {
+	out, err := e.psqlRun(ctx, postgresHBAReadBack)
+	if err != nil {
+		return fmt.Errorf("read back the client authentication rules: %w", err)
+	}
+	rules, unparsable, err := parseHBAReadBack(out)
+	if err != nil {
+		return err
+	}
+	if unparsable > 0 {
+		return fmt.Errorf("the client authentication file carries %d rules the engine could not parse, so it would keep the rules it is already serving", unparsable)
+	}
+	want := 0
+	if enforce {
+		want = postgresForceSSLRuleCount
+	}
+	if rules != want {
+		return fmt.Errorf("the client authentication file carries %d TLS enforcement rules, want %d", rules, want)
+	}
+	return nil
+}
+
+func parseHBAReadBack(out string) (rules, unparsable int, err error) {
+	answer := strings.TrimSpace(out)
+	malformed := fmt.Errorf("the client authentication read-back returned %q, want two counts", answer)
+	counts := strings.Split(answer, "|")
+	if len(counts) != 2 {
+		return 0, 0, malformed
+	}
+	parsed := make([]int, len(counts))
+	for i, count := range counts {
+		if parsed[i], err = strconv.Atoi(strings.TrimSpace(count)); err != nil {
+			return 0, 0, malformed
+		}
+	}
+	return parsed[0], parsed[1], nil
+}
+
+// The engine's own parser, run offline against the datadir. Reading one setting
+// back is enough: postgres parses every include first and exits non-zero naming
+// the file and line of an unknown parameter or an out-of-range value.
+func (e *postgresEngine) checkConfig(ctx context.Context) error {
+	if _, err := e.run(ctx, command{
+		Name: filepath.Join(filepath.Dir(e.psql), "postgres"),
+		Args: []string{"-D", e.pgData, "-C", "shared_buffers"},
+		Env:  []string{"PATH=" + defaultGuestPath},
+		User: e.osUser,
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (e *postgresEngine) Stop(ctx context.Context) error {
+	return serviceAction(ctx, e.run, e.rcService, e.service, "stop")
+}
+
+// Only the parameter rollback calls this: a restart the control plane wants goes
+// through RebootDBInstance, which cycles the VM.
+func (e *postgresEngine) Restart(ctx context.Context) error {
+	return serviceAction(ctx, e.run, e.rcService, e.service, "restart")
+}
+
+// Feeds sql on stdin rather than as an argument, so a statement is never
+// visible in the process table.
+func (e *postgresEngine) psqlRun(ctx context.Context, sql string, env ...string) (string, error) {
+	return e.run(ctx, command{
+		Name:  e.psql,
+		Args:  e.psqlArgs(),
+		Env:   append([]string{"PATH=" + defaultGuestPath}, env...),
+		Stdin: sql,
+		User:  e.osUser,
+	})
+}
+
+// Reading the script from stdin is what lets one invocation serve both a
+// one-shot run and a session held open across several statements.
+func (e *postgresEngine) psqlArgs() []string {
+	return []string{
+		"--no-psqlrc", "--quiet", "--no-align", "--tuples-only",
+		"-v", "ON_ERROR_STOP=1",
+		"-h", e.socketDir,
+		"-p", strconv.FormatInt(e.probe.port.Load(), 10),
+		"-U", e.osUser,
+		"-d", "postgres",
+		"-f", "-",
+	}
+}
+
+// Puts the engine into backup mode: the datadir is checkpointed and the engine
+// stops writing over the pages a snapshot is about to read. The hold is released
+// by Unquiesce, or by its own deadline, whichever comes first.
+func (e *postgresEngine) Quiesce(ctx context.Context, label string, hold time.Duration) error {
+	if err := validateQuiesceRequest(label, hold); err != nil {
+		return err
+	}
+
+	// Held across the whole start, so a second quiesce waits and then finds the
+	// first one's hold rather than opening a concurrent backup alongside it.
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.held != nil {
+		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)
+	}
+
+	// Deliberately not the command's context: the session has to outlive the call
+	// that started it, and its own deadline is what bounds it instead.
+	session, err := e.startSess(context.WithoutCancel(ctx), command{
+		Name:              e.psql,
+		Args:              e.psqlArgs(),
+		Env:               []string{"PATH=" + defaultGuestPath, "RDS_BACKUP_LABEL=" + label},
+		User:              e.osUser,
+		SentinelStatement: `\echo ` + sessionSentinel + "\n",
+	})
+	if err != nil {
+		return fmt.Errorf("open a backup session: %w", err)
+	}
+
+	// fast forces an immediate checkpoint rather than spreading it over the
+	// checkpoint interval, which would hold the snapshot open for minutes.
+	const sql = `\getenv label RDS_BACKUP_LABEL
+SELECT pg_backup_start(:'label', fast => true);
+`
+	if err := session.Exec(ctx, sql); err != nil {
+		if closeErr := session.Close(); closeErr != nil {
+			slog.Warn("rds-agent: closing a failed backup session", "err", closeErr)
+		}
+		return fmt.Errorf("put the engine into backup mode: %w", err)
+	}
+
+	e.beginHoldLocked(label, session, hold)
+	return nil
+}
+
+func (e *postgresEngine) Unquiesce(ctx context.Context) error {
+	return e.releaseHold(ctx, "SELECT pg_backup_stop();\n")
+}

cmd/rds-agent/engine_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"errors"
+	"fmt"
 	"os"
 	"os/user"
 	"path/filepath"
@@ -141,9 +142,24 @@ func (r *recordingRunner) run(_ context.Context, c command) (string, error) {
 
 func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
 	t.Helper()
-	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
-	cfg.PGData = t.TempDir()
-	if err := os.MkdirAll(filepath.Join(cfg.PGData, "conf.d"), 0o700); err != nil {
+	cfg := testEngineConfig(t)
+	return newPostgresEngine(cfg, withPostgresReadBacks(cfg.EngineDataDir, run), nil, newPostgresProbe(cfg, staticProbe(0)))
+}
+
+// The same engine with the read-backs left to the runner, for the cases that are
+// about what the engine does with each answer rather than about the rest of an
+// apply.
+func newScriptedTestEngine(t *testing.T, run commandRunner) *postgresEngine {
+	t.Helper()
+	cfg := testEngineConfig(t)
+	return newPostgresEngine(cfg, run, nil, newPostgresProbe(cfg, staticProbe(0)))
+}
+
+func testEngineConfig(t *testing.T) config {
+	t.Helper()
+	cfg := testLoadConfig(t, enginePostgres)
+	cfg.EngineDataDir = t.TempDir()
+	if err := os.MkdirAll(filepath.Join(cfg.EngineDataDir, "conf.d"), 0o700); err != nil {
 		t.Fatalf("create conf.d: %v", err)
 	}
 	// The guest's postgres user does not exist here, and the chown of the
@@ -152,8 +168,36 @@ func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
 	if err != nil {
 		t.Fatalf("resolve the current user: %v", err)
 	}
-	cfg.PGUser = current.Username
-	return newPostgresEngine(cfg, run, nil, newEngineProbe(cfg, staticProbe(0)))
+	cfg.EngineUser = current.Username
+	return cfg
+}
+
+// Answers the read-backs every apply runs against a live postmaster, from the
+// datadir the case is working in: an engine serving TLS, a reload the postmaster
+// took, and a pg_hba carrying whatever the agent last wrote. The call still
+// reaches the runner, so a case counting commands sees them; one that is about
+// the read-backs themselves builds its own engine.
+func withPostgresReadBacks(dataDir string, run commandRunner) commandRunner {
+	rule := filepath.Join(dataDir, postgresHBADir, postgresForceSSLRuleFile)
+	return func(ctx context.Context, c command) (string, error) {
+		out, err := run(ctx, c)
+		if err != nil {
+			return out, err
+		}
+		switch {
+		case strings.Contains(c.Stdin, "SHOW ssl"):
+			return "on\n", nil
+		case strings.Contains(c.Stdin, "pg_reload_conf"):
+			return "t\n", nil
+		case strings.Contains(c.Stdin, "pg_hba_file_rules"):
+			rules := 0
+			if _, statErr := os.Stat(rule); statErr == nil {
+				rules = postgresForceSSLRuleCount
+			}
+			return fmt.Sprintf("%d|0\n", rules), nil
+		}
+		return out, err
+	}
 }
 
 // The password reaches psql through the environment and is re-quoted there,
@@ -288,19 +332,27 @@ func TestPostgresEngine_ApplyParametersInstallsAndReloads(t *testing.T) {
 	if !strings.Contains(string(installed), "shared_buffers = '256MB'") {
 		t.Errorf("installed parameters = %q, want the resolved value", installed)
 	}
-	if len(runner.calls) != 3 {
-		t.Fatalf("ran %d commands, want a config check, a reload and a pending-restart read", len(runner.calls))
+	if len(runner.calls) != 5 {
+		t.Fatalf("ran %d commands, want a config check, the TLS guard, a reload, the rule read-back and a pending-restart read", len(runner.calls))
 	}
-	// The check runs before the reload, so a value the engine would refuse never
-	// reaches a running cluster.
+	// The check and the TLS guard both run before the reload, so neither a value
+	// the engine would refuse nor a rule rejecting every client reaches a running
+	// cluster. The read-back of the rules comes after, since the reload is what
+	// puts them in force.
 	if !slices.Contains(runner.calls[0].Args, "-C") {
 		t.Errorf("first command = %v, want the offline config check", runner.calls[0].Args)
 	}
-	if !strings.Contains(runner.calls[1].Stdin, "pg_reload_conf") {
-		t.Errorf("second statement = %q, want the reload", runner.calls[1].Stdin)
+	if !strings.Contains(runner.calls[1].Stdin, "SHOW ssl") {
+		t.Errorf("second statement = %q, want the serving-TLS guard", runner.calls[1].Stdin)
 	}
-	if !strings.Contains(runner.calls[2].Stdin, "pending_restart") {
-		t.Errorf("third statement = %q, want the pending-restart read", runner.calls[2].Stdin)
+	if !strings.Contains(runner.calls[2].Stdin, "pg_reload_conf") {
+		t.Errorf("third statement = %q, want the reload", runner.calls[2].Stdin)
+	}
+	if !strings.Contains(runner.calls[3].Stdin, "pg_hba_file_rules") {
+		t.Errorf("fourth statement = %q, want the client authentication read-back", runner.calls[3].Stdin)
+	}
+	if !strings.Contains(runner.calls[4].Stdin, "pending_restart") {
+		t.Errorf("fifth statement = %q, want the pending-restart read", runner.calls[4].Stdin)
 	}
 	if len(pending) != 2 || pending[0] != "shared_buffers" || pending[1] != "max_connections" {
 		t.Errorf("pending = %v, want both settings the engine reported", pending)
@@ -319,6 +371,21 @@ func TestPostgresEngine_ApplyParametersSurfacesAFailedReload(t *testing.T) {
 	}
 }
 
+// pg_reload_conf() returns f when it could not signal the postmaster, and psql
+// exits 0 all the same. Every read-back after it parses files from disk, so an
+// apply that trusted the exit status would verify clean against a server still
+// serving the old rules.
+func TestPostgresEngine_ApplyParametersSurfacesAnUnsignalledReload(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount, reload: "f"}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	_, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "work_mem", Value: "8MB"}})
+	if err == nil || !strings.Contains(err.Error(), `pg_reload_conf() returned "f"`) {
+		t.Fatalf("ApplyParameters error = %v, want a refusal naming the answer the postmaster gave", err)
+	}
+}
+
 type reloadFailingRunner struct {
 	recordingRunner
 
@@ -341,10 +408,10 @@ func TestPostgresEngine_ApplyParametersRestartsOnARepairSetWhileEngineIsDown(t *
 	codes := []int{2, 2, 0}
 	probeCall := 0
 	cfg := testProbeConfig()
-	engine.probe = newEngineProbe(cfg, func(context.Context, string, ...string) (int, error) {
+	engine.probe = newPostgresProbe(cfg, func(context.Context, string, ...string) (int, string, error) {
 		code := codes[min(probeCall, len(codes)-1)]
 		probeCall++
-		return code, nil
+		return code, "", nil
 	})
 	engine.repairTimeout = 100 * time.Millisecond
 	engine.repairPoll = time.Millisecond
@@ -372,7 +439,7 @@ func TestPostgresEngine_ApplyParametersRestartsOnARepairSetWhileEngineIsDown(t *
 	if !strings.Contains(runner.calls[2].Stdin, "pending_restart") {
 		t.Errorf("third call = %+v, want the pending-restart read", runner.calls[2])
 	}
-	installed, err := os.ReadFile(engine.parametersPath())
+	installed, err := os.ReadFile(engine.params.installedPath())
 	if err != nil {
 		t.Fatalf("read installed parameters: %v", err)
 	}
@@ -381,6 +448,224 @@ func TestPostgresEngine_ApplyParametersRestartsOnARepairSetWhileEngineIsDown(t *
 	}
 }
 
+// Answers the two read-backs an apply runs from fields the case sets, so what
+// the engine does with each answer is what is under test.
+type tlsReadBackRunner struct {
+	recordingRunner
+
+	ssl        string
+	ruleRows   int
+	brokenRows int
+	// What pg_reload_conf() answered. Empty is the postmaster having taken the
+	// signal, which is every case but the one about it not having.
+	reload string
+}
+
+func (r *tlsReadBackRunner) run(ctx context.Context, c command) (string, error) {
+	switch {
+	case strings.Contains(c.Stdin, "SHOW ssl"):
+		r.calls = append(r.calls, c)
+		return r.ssl + "\n", nil
+	case strings.Contains(c.Stdin, "pg_reload_conf"):
+		r.calls = append(r.calls, c)
+		if r.reload == "" {
+			return "t\n", nil
+		}
+		return r.reload + "\n", nil
+	case strings.Contains(c.Stdin, "pg_hba_file_rules"):
+		r.calls = append(r.calls, c)
+		return fmt.Sprintf("%d|%d\n", r.ruleRows, r.brokenRows), nil
+	}
+	return r.recordingRunner.run(ctx, c)
+}
+
+func forceSSLParameter(value string) []handlers_rds.Parameter {
+	return []handlers_rds.Parameter{
+		{Name: "work_mem", Value: "4096"},
+		{Name: postgresEngineMeta.TLSEnforcementParameter(), Value: value},
+	}
+}
+
+// PostgreSQL has no server setting for this, so the value in the parameter file
+// is inert: the rule the generated pg_hba includes is the whole of it.
+func TestPostgresEngine_ApplyParametersWritesTheEnforcementRule(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("1")); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	rule, err := os.ReadFile(engine.forceSSLRulePath())
+	if err != nil {
+		t.Fatalf("read the enforcement rule: %v", err)
+	}
+	if string(rule) != postgresForceSSLRules {
+		t.Errorf("enforcement rule = %q, want the constant rds-init also writes", rule)
+	}
+}
+
+// A snapshot carries hba.d with it, so restoring one taken while enforcing into
+// a group that turns enforcement off has to take the rule with it.
+func TestPostgresEngine_ApplyParametersRemovesAStaleEnforcementRule(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on"}
+	engine := newScriptedTestEngine(t, runner.run)
+	if err := os.MkdirAll(filepath.Dir(engine.forceSSLRulePath()), 0o700); err != nil {
+		t.Fatalf("create hba.d: %v", err)
+	}
+	if err := os.WriteFile(engine.forceSSLRulePath(), []byte(postgresForceSSLRules), 0o600); err != nil {
+		t.Fatalf("write the rule the restored volume carried: %v", err)
+	}
+
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("0")); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if _, err := os.Stat(engine.forceSSLRulePath()); !os.IsNotExist(err) {
+		t.Errorf("the stale enforcement rule is still in place (stat err = %v)", err)
+	}
+}
+
+// The ordinary state of an instance whose resolved set predates the parameter,
+// and the whole of the migration: it begins requiring TLS with no control-plane
+// work at all.
+func TestPostgresEngine_ApplyParametersEnforcesOnAnAbsentKey(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(t.Context(),
+		[]handlers_rds.Parameter{{Name: "work_mem", Value: "4096"}}); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if _, err := os.Stat(engine.forceSSLRulePath()); err != nil {
+		t.Errorf("a set predating the parameter did not enforce (stat err = %v)", err)
+	}
+}
+
+// The resolver canonicalises every boolean, so a value that is neither means the
+// file was written by something other than the platform. Reading an unparsable
+// security setting as off is the one choice not open here.
+func TestPostgresEngine_ApplyParametersRefusesAnUnparsableEnforcementValue(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on"}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	_, err := engine.ApplyParameters(t.Context(), forceSSLParameter("yes"))
+	if err == nil || !strings.Contains(err.Error(), "neither 1 nor 0") {
+		t.Fatalf("ApplyParameters error = %v, want a refusal naming the unreadable value", err)
+	}
+	if _, statErr := os.Stat(engine.params.installedPath()); !os.IsNotExist(statErr) {
+		t.Errorf("the refused set is still installed (stat err = %v)", statErr)
+	}
+}
+
+// Without this, a live enable against an engine started with no certificate
+// would reject every subsequent connection and report the parameter applied.
+func TestPostgresEngine_ApplyParametersRefusesToEnforceWithoutTLS(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "off"}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	_, err := engine.ApplyParameters(t.Context(), forceSSLParameter("1"))
+	if err == nil || !strings.Contains(err.Error(), "serving without TLS") {
+		t.Fatalf("ApplyParameters error = %v, want a refusal naming the engine's own TLS state", err)
+	}
+	if _, statErr := os.Stat(engine.forceSSLRulePath()); !os.IsNotExist(statErr) {
+		t.Errorf("the enforcement rule was written anyway (stat err = %v)", statErr)
+	}
+}
+
+// PostgreSQL discards a pg_hba it cannot parse and keeps the rules it is already
+// serving, which presents as an apply that succeeded and enforces nothing. The
+// shape is asserted as well as the errors, because our own derivation writing
+// the wrong thing is the failure that will actually happen.
+func TestPostgresEngine_ApplyParametersFailsAnUnverifiableEnforcement(t *testing.T) {
+	tests := []struct {
+		name       string
+		value      string
+		ruleRows   int
+		brokenRows int
+		want       string
+	}{
+		{name: "rule missing", value: "1", ruleRows: 0, want: "carries 0 TLS enforcement rules, want 2"},
+		{name: "rule present when it should not be", value: "0", ruleRows: 2, want: "carries 2 TLS enforcement rules, want 0"},
+		{name: "a rule the engine could not parse", value: "1", ruleRows: 2, brokenRows: 1, want: "could not parse"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			runner := &tlsReadBackRunner{ssl: "on", ruleRows: tc.ruleRows, brokenRows: tc.brokenRows}
+			engine := newScriptedTestEngine(t, runner.run)
+
+			_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(tc.value))
+			if err == nil || !strings.Contains(err.Error(), tc.want) {
+				t.Fatalf("ApplyParameters error = %v, want %q", err, tc.want)
+			}
+		})
+	}
+}
+
+// A rejected set has to take its enforcement with it. Withdrawing one without
+// the other would leave the instance serving the previous parameters under the
+// rejected set's TLS posture.
+func TestPostgresEngine_ApplyParametersWithdrawsBothFilesTogether(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
+	engine := newScriptedTestEngine(t, runner.run)
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("1")); err != nil {
+		t.Fatalf("first ApplyParameters: %v", err)
+	}
+
+	// Turning enforcement off, verified against a pg_hba that still carries the
+	// rule — so the apply fails after both files have already been replaced.
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("0")); err == nil {
+		t.Fatal("ApplyParameters succeeded against an unverifiable enforcement")
+	}
+	installed, err := os.ReadFile(engine.params.installedPath())
+	if err != nil {
+		t.Fatalf("read the installed parameters: %v", err)
+	}
+	if !strings.Contains(string(installed), "rds.force_ssl = '1'") {
+		t.Errorf("installed parameters = %q, want the previous accepted set restored", installed)
+	}
+	if _, err := os.Stat(engine.forceSSLRulePath()); err != nil {
+		t.Errorf("the enforcement of the previous set was not restored with it (stat err = %v)", err)
+	}
+	// The reload already adopted the rejected files, so putting them back is not
+	// enough on its own.
+	if !strings.Contains(runner.calls[len(runner.calls)-1].Stdin, "pg_reload_conf") {
+		t.Errorf("last statement = %q, want a reload taking the engine off the withdrawn set", runner.calls[len(runner.calls)-1].Stdin)
+	}
+}
+
+// A service restart does not re-run rds-init, so a restore stopping at the
+// parameter file would bring the last known good set up under the enforcement of
+// the set that was just rejected.
+func TestPostgresEngine_RestoreLastKnownGoodReDerivesEnforcement(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("1")); err != nil {
+		t.Fatalf("ApplyParameters(enforcing): %v", err)
+	}
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("RecordServingParameters: %v", err)
+	}
+	runner.ruleRows = 0
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("0")); err != nil {
+		t.Fatalf("ApplyParameters(not enforcing): %v", err)
+	}
+	if _, err := os.Stat(engine.forceSSLRulePath()); !os.IsNotExist(err) {
+		t.Fatalf("the enforcement rule survived the set that turned it off (stat err = %v)", err)
+	}
+
+	engine.probe = stubProbe(t, 2, nil)
+	restored, err := engine.RestoreLastKnownGoodParameters(t.Context())
+	if err != nil {
+		t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
+	}
+	if !restored {
+		t.Fatal("did not restore a set that differs from the last known good")
+	}
+	if _, err := os.Stat(engine.forceSSLRulePath()); err != nil {
+		t.Errorf("the restored set's enforcement was not re-derived (stat err = %v)", err)
+	}
+}
+
 func TestPostgresEngine_ApplyParametersKeepsRepairSetWhenRestartTimesOut(t *testing.T) {
 	runner := &recordingRunner{}
 	engine := newTestEngine(t, runner.run)
@@ -392,7 +677,7 @@ func TestPostgresEngine_ApplyParametersKeepsRepairSetWhenRestartTimesOut(t *test
 	if err == nil || !strings.Contains(err.Error(), "wait for the engine") {
 		t.Fatalf("ApplyParameters error = %v, want the repair wait failure", err)
 	}
-	installed, readErr := os.ReadFile(engine.parametersPath())
+	installed, readErr := os.ReadFile(engine.params.installedPath())
 	if readErr != nil {
 		t.Fatalf("read installed parameters: %v", readErr)
 	}
@@ -400,24 +685,3 @@ func TestPostgresEngine_ApplyParametersKeepsRepairSetWhenRestartTimesOut(t *test
 		t.Errorf("installed parameters = %q, want the checked repair set retained", installed)
 	}
 }
-
-// Through the service manager, so the supervisor records the engine as stopped
-// and does not restart it underneath a VM that is going down.
-func TestPostgresEngine_StopGoesThroughTheServiceManager(t *testing.T) {
-	runner := &recordingRunner{}
-	engine := newTestEngine(t, runner.run)
-
-	if err := engine.Stop(context.Background()); err != nil {
-		t.Fatalf("Stop: %v", err)
-	}
-	if len(runner.calls) != 1 {
-		t.Fatalf("ran %d commands, want 1", len(runner.calls))
-	}
-	call := runner.calls[0]
-	if filepath.Base(call.Name) != "rc-service" {
-		t.Errorf("ran %q, want rc-service", call.Name)
-	}
-	if len(call.Args) != 2 || call.Args[0] != "postgresql" || call.Args[1] != "stop" {
-		t.Errorf("args = %v, want [postgresql stop]", call.Args)
-	}
-}

cmd/rds-agent/health.go

@@ -1,38 +1,50 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"errors"
-	"fmt"
 	"os/exec"
-	"strconv"
+	"strings"
 	"sync/atomic"
 
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
 
 // A non-zero exit is a result, not a fault, so it comes back as a code; err is
-// reserved for the probe failing to run at all.
-type probeRunner func(ctx context.Context, name string, args ...string) (int, error)
+// reserved for the probe failing to run at all. The client's stderr comes back
+// with it: on a refusal it is the only place the reason is ever stated.
+type probeRunner func(ctx context.Context, name string, args ...string) (int, string, error)
 
-func execProbeRunner(ctx context.Context, name string, args ...string) (int, error) {
-	err := exec.CommandContext(ctx, name, args...).Run()
+func execProbeRunner(ctx context.Context, name string, args ...string) (int, string, error) {
+	var stderr bytes.Buffer
+	cmd := exec.CommandContext(ctx, name, args...)
+	cmd.Stderr = &stderr
+	err := cmd.Run()
+	out := strings.TrimSpace(stderr.String())
 	if err == nil {
-		return 0, nil
+		return 0, out, nil
+	}
+	if ctxErr := ctx.Err(); ctxErr != nil {
+		return -1, out, ctxErr
 	}
 	var exitErr *exec.ExitError
 	if errors.As(err, &exitErr) {
-		return exitErr.ExitCode(), nil
+		return exitErr.ExitCode(), out, nil
 	}
-	return -1, err
+	return -1, out, err
 }
 
-// Shells out to pg_isready rather than dialling the port, since only a startup
-// exchange separates an engine that is serving from one still in recovery.
+// What the engine reports about itself, which is the only part of the probe
+// that differs between engines. The port is passed in rather than read from the
+// engine, so the atomic below stays the one place it is followed from.
+type probeStateFn func(ctx context.Context, port int64) (engineState, string)
+
+// Engine-neutral: the latch, the port, the degraded mapping and the translation
+// to a reported health are shared, and only the state function is supplied by
+// the engine the image bakes.
 type engineProbe struct {
-	run    probeRunner
-	binary string
-	host   string
+	stateFn probeStateFn
 	// Set from the bootstrap config once it lands, from a different goroutine
 	// than the heartbeat that reads it.
 	port atomic.Int64
@@ -41,9 +53,9 @@ type engineProbe struct {
 	seenHealthy bool
 }
 
-func newEngineProbe(cfg config, run probeRunner) *engineProbe {
-	p := &engineProbe{run: run, binary: cfg.PGIsReady, host: cfg.EngineHost}
-	p.port.Store(int64(cfg.EnginePort))
+func newEngineProbe(port int, stateFn probeStateFn) *engineProbe {
+	p := &engineProbe{stateFn: stateFn}
+	p.port.Store(int64(port))
 	return p
 }
 
@@ -52,15 +64,14 @@ func (p *engineProbe) setPort(port int) {
 }
 
 // What the engine is doing, before the seenHealthy latch collapses "never up"
-// and "was up and went away" into one health. The parameter rollback needs them
-// apart: a postmaster that is up and replaying WAL will come back on its own, one
-// that is not running at all after a parameter change will not.
+// and "went away" into one health. The rollback needs them apart: an engine
+// replaying WAL comes back on its own, one that is not running will not.
 type engineState int
 
 const (
 	// Not answering at all, or the probe could not run.
 	engineAbsent engineState = iota
-	// The postmaster is up and rejecting connections: startup or crash recovery.
+	// The process is up but not yet serving: startup or crash recovery.
 	engineRecovering
 	engineServing
 )
@@ -83,21 +94,7 @@ func (p *engineProbe) Check(ctx context.Context) (handlers_rds.EngineHealth, str
 // The raw probe result. Safe to call from a goroutine other than the
 // heartbeat's: it touches no latched state.
 func (p *engineProbe) state(ctx context.Context) (engineState, string) {
-	port := strconv.FormatInt(p.port.Load(), 10)
-	code, err := p.run(ctx, p.binary, "-h", p.host, "-p", port, "-q")
-	switch {
-	case err != nil:
-		// A missing binary or broken image. Reporting healthy on the strength of
-		// nothing would hide it, so this reads as absent like an engine that did
-		// not answer.
-		return engineAbsent, fmt.Sprintf("engine probe could not run: %v", err)
-	case code == 0:
-		return engineServing, ""
-	case code == 1:
-		return engineRecovering, "engine is rejecting connections (startup or recovery)"
-	default:
-		return engineAbsent, fmt.Sprintf("engine did not respond on %s:%s", p.host, port)
-	}
+	return p.stateFn(ctx, p.port.Load())
 }
 
 // A non-answering engine: starting until it has answered once, unhealthy after.

cmd/rds-agent/health_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"strings"
 	"testing"
+	"time"
 
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
@@ -28,8 +29,8 @@ func TestEngineProbe_MapsExitCodesToHealth(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			probe := newEngineProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
-				return tt.code, tt.runErr
+			probe := newPostgresProbe(testProbeConfig(), func(context.Context, string, ...string) (int, string, error) {
+				return tt.code, "", tt.runErr
 			})
 			probe.seenHealthy = tt.seenHealthy
 
@@ -51,8 +52,8 @@ func TestEngineProbe_MapsExitCodesToHealth(t *testing.T) {
 // has to flip the first time the engine answers.
 func TestEngineProbe_LatchesAfterFirstHealthy(t *testing.T) {
 	code := 0
-	probe := newEngineProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
-		return code, nil
+	probe := newPostgresProbe(testProbeConfig(), func(context.Context, string, ...string) (int, string, error) {
+		return code, "", nil
 	})
 
 	if got, _ := probe.Check(context.Background()); got != handlers_rds.EngineHealthHealthy {
@@ -68,9 +69,9 @@ func TestEngineProbe_LatchesAfterFirstHealthy(t *testing.T) {
 // must follow it rather than keep asking the default.
 func TestEngineProbe_ProbesTheAssignedPort(t *testing.T) {
 	var gotArgs []string
-	probe := newEngineProbe(testProbeConfig(), func(_ context.Context, _ string, args ...string) (int, error) {
+	probe := newPostgresProbe(testProbeConfig(), func(_ context.Context, _ string, args ...string) (int, string, error) {
 		gotArgs = args
-		return 0, nil
+		return 0, "", nil
 	})
 
 	probe.setPort(6543)
@@ -81,6 +82,34 @@ func TestEngineProbe_ProbesTheAssignedPort(t *testing.T) {
 	}
 }
 
-func testProbeConfig() config {
-	return config{EngineHost: defaultEngineHost, EnginePort: defaultEnginePort, PGIsReady: defaultPGIsReady}
+func TestExecProbeRunner_PreservesTheProbeDeadline(t *testing.T) {
+	ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
+	defer cancel()
+
+	code, _, err := execProbeRunner(ctx, "sh", "-c", "while :; do :; done")
+	if code != -1 {
+		t.Errorf("exit code = %d, want -1 for a timed-out probe", code)
+	}
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Errorf("error = %v, want the context deadline", err)
+	}
+}
+
+// The client's account of a refusal is the reason a probe failure is legible at
+// all, so a runner that dropped it would leave the caller nothing to report.
+func TestExecProbeRunner_CapturesTheClientsStderr(t *testing.T) {
+	code, stderr, err := execProbeRunner(t.Context(), "sh", "-c", "echo 'TLS handshake failed' >&2; exit 1")
+	if err != nil {
+		t.Fatalf("execProbeRunner: %v", err)
+	}
+	if code != 1 {
+		t.Errorf("exit code = %d, want 1", code)
+	}
+	if stderr != "TLS handshake failed" {
+		t.Errorf("stderr = %q, want the client's own message", stderr)
+	}
+}
+
+func testProbeConfig() config {
+	return config{EngineHost: defaultEngineHost, EnginePort: engineLayouts[enginePostgres].port}
 }

cmd/rds-agent/heartbeat_test.go

@@ -17,7 +17,7 @@ func TestHeartbeater_FirstServingProbeSeedsLastKnownGood(t *testing.T) {
 	runner := &recordingRunner{}
 	engine := newTestEngine(t, runner.run)
 	content := []byte("work_mem = '4096'\n")
-	if err := os.WriteFile(engine.parametersPath(), content, 0o600); err != nil {
+	if err := os.WriteFile(engine.params.installedPath(), content, 0o600); err != nil {
 		t.Fatalf("write installed parameters: %v", err)
 	}
 
@@ -25,7 +25,7 @@ func TestHeartbeater_FirstServingProbeSeedsLastKnownGood(t *testing.T) {
 	h := newHeartbeater(cp, engine.probe, engine, 0)
 	h.beat(context.Background())
 
-	lastGood, err := os.ReadFile(engine.lastGoodPath())
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
 	if err != nil {
 		t.Fatalf("read last known good: %v", err)
 	}
@@ -39,7 +39,7 @@ func TestHeartbeater_FirstServingProbeSeedsLastKnownGood(t *testing.T) {
 }
 
 func TestHeartbeater_BoundsBootstrapFailure(t *testing.T) {
-	h := newHeartbeater(newFakeControlPlane(), newEngineProbe(testProbeConfig(), staticProbe(2)), nil, 0)
+	h := newHeartbeater(newFakeControlPlane(), newPostgresProbe(testProbeConfig(), staticProbe(2)), nil, 0)
 	h.setBootstrapFailure("bootstrap fetch", errors.New(strings.Repeat("界", 1000)))
 
 	failure := h.bootstrapFailure.Load()
@@ -60,8 +60,8 @@ func TestHeartbeater_BoundsBootstrapFailure(t *testing.T) {
 func TestHeartbeater_ChecksServingParametersOnEveryHealthyProbe(t *testing.T) {
 	code := 0
 	cfg := testProbeConfig()
-	probe := newEngineProbe(cfg, func(context.Context, string, ...string) (int, error) {
-		return code, nil
+	probe := newPostgresProbe(cfg, func(context.Context, string, ...string) (int, string, error) {
+		return code, "", nil
 	})
 	recorder := &countingServingRecorder{}
 	h := newHeartbeater(newFakeControlPlane(), probe, recorder, 0)

cmd/rds-agent/parameters.go

@@ -0,0 +1,233 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The generated parameter files and what it takes to install one. They live on
+// the data volume at the paths rds-init writes: a class change replaces the VM,
+// so a set under /etc would revert to defaults while the API reported otherwise.
+type parameterStore struct {
+	dir string
+	// The include the resolved set is rendered to.
+	installed string
+	// The copy of the last set the engine accepted, which a boot loop is rolled
+	// back to. Deliberately not a name the engine's own include glob reads, so it
+	// is never parsed as a second set of settings.
+	lastGood string
+	// What the engine actually started on, for an engine that cannot be asked
+	// which of its settings are still pending a restart. Empty for one that can.
+	serving string
+	// Prepended to every file written here, for a configuration format that has
+	// no setting outside a group.
+	header string
+	// The engine parses its own configuration, so a root-owned file would be
+	// unreadable to it.
+	osUser string
+	// Names the engine whose option-file spellings these files are written in.
+	engine string
+}
+
+func (s parameterStore) installedPath() string { return filepath.Join(s.dir, s.installed) }
+func (s parameterStore) lastGoodPath() string  { return filepath.Join(s.dir, s.lastGood) }
+func (s parameterStore) servingPath() string   { return filepath.Join(s.dir, s.serving) }
+
+// Byte-for-byte what rds-init writes for the same set, so the copies compared
+// against it are comparing values rather than formatting.
+func (s parameterStore) render(params []handlers_rds.Parameter) ([]byte, error) {
+	body, err := renderParameters(s.engine, params)
+	if err != nil {
+		return nil, err
+	}
+	return []byte(s.header + body), nil
+}
+
+// Installs the resolved set at the path rds-init uses, so a later boot
+// overwrites rather than shadows it. Returns a func putting back whatever was
+// there, so the caller can undo the install without re-rendering anything.
+func (s parameterStore) install(params []handlers_rds.Parameter) (restore func() error, err error) {
+	path := s.installedPath()
+	// A missing file is the first-ever apply, and its rollback is a removal.
+	previous, readErr := os.ReadFile(path)
+	if readErr != nil && !os.IsNotExist(readErr) {
+		return nil, fmt.Errorf("read the installed parameters at %s: %w", path, readErr)
+	}
+	existed := readErr == nil
+
+	rendered, err := s.render(params)
+	if err != nil {
+		return nil, err
+	}
+	if err := s.write(path, rendered); err != nil {
+		return nil, err
+	}
+	return func() error {
+		if !existed {
+			if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+				return fmt.Errorf("withdraw the rejected parameters at %s: %w", path, err)
+			}
+			return nil
+		}
+		return s.write(path, previous)
+	}, nil
+}
+
+// Atomic: a temp file in the same directory, renamed over the target. The temp
+// name deliberately does not end in the engine's include suffix, so a crash
+// between write and rename cannot leave the engine reading a half-written file.
+func (s parameterStore) write(path string, content []byte) error {
+	tmp := path + ".new"
+	if err := os.WriteFile(tmp, content, 0o600); err != nil {
+		return fmt.Errorf("write %s: %w", tmp, err)
+	}
+	if err := s.chownToEngine(tmp); err != nil {
+		os.Remove(tmp)
+		return err
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("install %s: %w", path, err)
+	}
+	return nil
+}
+
+// A guest with no engine user is a broken image, not something to work around.
+func (s parameterStore) chownToEngine(path string) error {
+	credential, err := lookupCredential(s.osUser)
+	if err != nil {
+		return err
+	}
+	if err := os.Chown(path, int(credential.Uid), int(credential.Gid)); err != nil {
+		return fmt.Errorf("hand %s to %s: %w", path, s.osUser, err)
+	}
+	return nil
+}
+
+// Snapshots the installed set as the rollback target, but only once the engine
+// is running all of it. A set still pending a restart has not been shown to
+// serve, so promoting it would point the rollback at a config about to fail.
+func recordLastGood(ctx context.Context, store parameterStore, pending pendingRestartFn) error {
+	if _, err := os.Stat(store.installedPath()); err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("inspect the serving parameters: %w", err)
+	}
+	names, err := pending(ctx)
+	if err != nil {
+		return err
+	}
+	if len(names) > 0 {
+		return nil
+	}
+
+	content, err := os.ReadFile(store.installedPath())
+	if err != nil {
+		return fmt.Errorf("read the serving parameters: %w", err)
+	}
+	lastGood, err := os.ReadFile(store.lastGoodPath())
+	if err == nil && bytes.Equal(content, lastGood) {
+		return nil
+	}
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("read the last known good parameters: %w", err)
+	}
+	return store.write(store.lastGoodPath(), content)
+}
+
+// Puts the last set the engine accepted back in place, for a restart that failed
+// after a parameter change. The probe is checked again under the caller's
+// parameter lock, so a repair that just brought the engine back is not reversed.
+func restoreLastGood(ctx context.Context, store parameterStore, probe *engineProbe) (bool, error) {
+	if state, _ := probe.state(ctx); state != engineAbsent {
+		return false, nil
+	}
+	lastGood, err := os.ReadFile(store.lastGoodPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, fmt.Errorf("read the last known good parameters: %w", err)
+	}
+	current, err := os.ReadFile(store.installedPath())
+	if err != nil && !os.IsNotExist(err) {
+		return false, fmt.Errorf("read the installed parameters: %w", err)
+	}
+	if bytes.Equal(current, lastGood) {
+		return false, nil
+	}
+	if err := store.write(store.installedPath(), lastGood); err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+// The settings the engine has accepted but will not honour until it restarts.
+// Each implementation answers it its own way, and both the apply and the serving
+// snapshot are gated on the same answer.
+type pendingRestartFn func(ctx context.Context) ([]string, error)
+
+// The parameter-file state every engine keeps. The operations over it are free
+// functions above rather than methods here, so each implementation names its own
+// restart and pending answers at the call site where the compiler checks them.
+type parameterManager struct {
+	params parameterStore
+	probe  *engineProbe
+	// Serializes parameter installs, serving snapshots and rollback restores so
+	// none can copy or replace an intermediate configuration.
+	paramMu       sync.Mutex
+	repairTimeout time.Duration
+	repairPoll    time.Duration
+}
+
+const (
+	parameterRepairTimeout = 90 * time.Second
+	parameterRepairPoll    = time.Second
+)
+
+// Starts a down engine on the set just installed and waits for it to serve.
+// Reached when the engine went away during an apply: the installed set is the
+// newer, so starting on it beats restoring one that already failed.
+func awaitRepairedEngine(ctx context.Context, probe *engineProbe, restart func(context.Context) error,
+	pending pendingRestartFn, timeout, poll time.Duration) ([]string, error) {
+	repairCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	if err := restart(repairCtx); err != nil {
+		return nil, fmt.Errorf("start the engine on the repaired parameter set: %w", err)
+	}
+
+	timer := time.NewTimer(0)
+	defer timer.Stop()
+	lastMessage := "engine did not respond"
+	for {
+		select {
+		case <-repairCtx.Done():
+			if ctx.Err() != nil {
+				return nil, fmt.Errorf("wait for the engine on the repaired parameter set: %w", ctx.Err())
+			}
+			return nil, fmt.Errorf("wait for the engine on the repaired parameter set: %s: %w", lastMessage, repairCtx.Err())
+		case <-timer.C:
+		}
+
+		state, message := probe.state(repairCtx)
+		if state == engineServing {
+			names, err := pending(repairCtx)
+			if err == nil {
+				return names, nil
+			}
+			lastMessage = err.Error()
+		} else if message != "" {
+			lastMessage = message
+		}
+		timer.Reset(poll)
+	}
+}

cmd/rds-agent/paramrecovery.go

@@ -49,10 +49,16 @@ func (g *paramGuard) Run(ctx context.Context) {
 		return
 	}
 
+	// A restore that replaced the file and then failed has still moved the
+	// instance onto the last accepted set, so the restart and the report below
+	// are what it needs; returning here would leave it down and unaccounted for.
 	restored, err := g.engine.RestoreLastKnownGoodParameters(ctx)
 	if err != nil {
-		slog.Error("rds-agent: could not restore the last known good parameters", "err", err)
-		return
+		slog.Error("rds-agent: restoring the last known good parameters failed",
+			"err", err, "parametersReplaced", restored)
+		if !restored {
+			return
+		}
 	}
 	if !restored {
 		slog.Warn("rds-agent: engine is down but no different last accepted parameter set is available; not rolling back")

cmd/rds-agent/paramrecovery_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"os"
-	"path/filepath"
 	"slices"
 	"strings"
 	"testing"
@@ -50,7 +49,7 @@ func TestPostgresEngine_ApplyParametersRollsBackAValueTheEngineRejects(t *testin
 		t.Fatal("ApplyParameters succeeded against a config the engine rejected")
 	}
 
-	installed, err := os.ReadFile(engine.parametersPath())
+	installed, err := os.ReadFile(engine.params.installedPath())
 	if err != nil {
 		t.Fatalf("read the installed parameters: %v", err)
 	}
@@ -69,7 +68,7 @@ func TestPostgresEngine_ApplyParametersWithdrawsTheFirstRejectedSet(t *testing.T
 		[]handlers_rds.Parameter{{Name: "work_mem", Value: "0"}}); err == nil {
 		t.Fatal("ApplyParameters succeeded against a config the engine rejected")
 	}
-	if _, err := os.Stat(engine.parametersPath()); !os.IsNotExist(err) {
+	if _, err := os.Stat(engine.params.installedPath()); !os.IsNotExist(err) {
 		t.Errorf("the rejected include is still installed (stat err = %v)", err)
 	}
 }
@@ -97,7 +96,7 @@ func TestPostgresEngine_ApplyBeforeFirstHeartbeatSeedsLastKnownGood(t *testing.T
 	runner := &pendingAfterReloadRunner{}
 	engine := newTestEngine(t, runner.run)
 	first := []byte("shared_buffers = '32768'\n")
-	if err := os.WriteFile(engine.parametersPath(), first, 0o600); err != nil {
+	if err := os.WriteFile(engine.params.installedPath(), first, 0o600); err != nil {
 		t.Fatalf("write serving parameter set: %v", err)
 	}
 
@@ -108,7 +107,7 @@ func TestPostgresEngine_ApplyBeforeFirstHeartbeatSeedsLastKnownGood(t *testing.T
 	if !slices.Equal(pending, []string{"shared_buffers"}) {
 		t.Errorf("pending = %v, want shared_buffers", pending)
 	}
-	lastGood, err := os.ReadFile(engine.lastGoodPath())
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
 	if err != nil {
 		t.Fatalf("read last known good: %v", err)
 	}
@@ -153,10 +152,10 @@ func TestPostgresEngine_ServingSnapshotWaitsForParameterApply(t *testing.T) {
 	}
 	engine := newTestEngine(t, runner.run)
 	first := []byte("shared_buffers = '32768'\n")
-	if err := os.WriteFile(engine.parametersPath(), first, 0o600); err != nil {
+	if err := os.WriteFile(engine.params.installedPath(), first, 0o600); err != nil {
 		t.Fatalf("write serving parameter set: %v", err)
 	}
-	if err := os.WriteFile(engine.lastGoodPath(), first, 0o600); err != nil {
+	if err := os.WriteFile(engine.params.lastGoodPath(), first, 0o600); err != nil {
 		t.Fatalf("write last known good: %v", err)
 	}
 
@@ -192,7 +191,7 @@ func TestPostgresEngine_ServingSnapshotWaitsForParameterApply(t *testing.T) {
 		t.Error("serving snapshot completed while the parameter apply held its transaction")
 	}
 
-	lastGood, err := os.ReadFile(engine.lastGoodPath())
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
 	if err != nil {
 		t.Fatalf("read last known good: %v", err)
 	}
@@ -216,7 +215,7 @@ func TestPostgresEngine_LastKnownGoodTracksTheServingSet(t *testing.T) {
 	}
 	assertLastGood := func(value string) {
 		t.Helper()
-		lastGood, err := os.ReadFile(engine.lastGoodPath())
+		lastGood, err := os.ReadFile(engine.params.lastGoodPath())
 		if err != nil {
 			t.Fatalf("read the last known good parameters: %v", err)
 		}
@@ -240,8 +239,8 @@ func TestPostgresEngine_LastKnownGoodTracksTheServingSet(t *testing.T) {
 	apply("16384")
 	assertLastGood("8192")
 
-	if strings.HasSuffix(engine.lastGoodPath(), ".conf") {
-		t.Errorf("the rollback copy at %s would be included by the engine", engine.lastGoodPath())
+	if strings.HasSuffix(engine.params.lastGoodPath(), ".conf") {
+		t.Errorf("the rollback copy at %s would be included by the engine", engine.params.lastGoodPath())
 	}
 }
 
@@ -249,7 +248,7 @@ func TestPostgresEngine_RecordServingParametersSkipsPendingRestart(t *testing.T)
 	runner := &recordingRunner{}
 	engine := newTestEngine(t, runner.run)
 	first := []byte("shared_buffers = '32768'\n")
-	if err := os.WriteFile(engine.parametersPath(), first, 0o600); err != nil {
+	if err := os.WriteFile(engine.params.installedPath(), first, 0o600); err != nil {
 		t.Fatalf("write first parameter set: %v", err)
 	}
 	if err := engine.RecordServingParameters(t.Context()); err != nil {
@@ -257,14 +256,14 @@ func TestPostgresEngine_RecordServingParametersSkipsPendingRestart(t *testing.T)
 	}
 
 	second := []byte("shared_buffers = '65536'\n")
-	if err := os.WriteFile(engine.parametersPath(), second, 0o600); err != nil {
+	if err := os.WriteFile(engine.params.installedPath(), second, 0o600); err != nil {
 		t.Fatalf("write pending parameter set: %v", err)
 	}
 	runner.out = "shared_buffers\n"
 	if err := engine.RecordServingParameters(t.Context()); err != nil {
 		t.Fatalf("record with a pending restart: %v", err)
 	}
-	lastGood, err := os.ReadFile(engine.lastGoodPath())
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
 	if err != nil {
 		t.Fatalf("read last known good: %v", err)
 	}
@@ -278,7 +277,7 @@ func TestPostgresEngine_RecordServingParametersSkipsPendingRestart(t *testing.T)
 	if err := engine.RecordServingParameters(t.Context()); err != nil {
 		t.Fatalf("record the restarted parameter set: %v", err)
 	}
-	lastGood, err = os.ReadFile(engine.lastGoodPath())
+	lastGood, err = os.ReadFile(engine.params.lastGoodPath())
 	if err != nil {
 		t.Fatalf("read advanced last known good: %v", err)
 	}
@@ -330,7 +329,7 @@ func TestPostgresEngine_RestoreLastKnownGoodParameters(t *testing.T) {
 	if !restored {
 		t.Fatal("did not restore a set that differs from the last known good")
 	}
-	installed, err := os.ReadFile(engine.parametersPath())
+	installed, err := os.ReadFile(engine.params.installedPath())
 	if err != nil {
 		t.Fatalf("read the installed parameters: %v", err)
 	}
@@ -369,9 +368,9 @@ func (f *fakeRecovery) Restart(context.Context) error {
 // A probe whose result the test drives directly, standing in for pg_isready.
 func stubProbe(t *testing.T, code int, err error) *engineProbe {
 	t.Helper()
-	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
-	return newEngineProbe(cfg, func(context.Context, string, ...string) (int, error) {
-		return code, err
+	cfg := testLoadConfig(t, enginePostgres)
+	return newPostgresProbe(cfg, func(context.Context, string, ...string) (int, string, error) {
+		return code, "", err
 	})
 }
 
@@ -405,6 +404,47 @@ func TestParamGuard_RollsBackReportsAndRestartsWhenTheEngineNeverComesUp(t *test
 	}
 }
 
+// PostgreSQL re-derives TLS enforcement from the set it has just put back, so
+// the file can be replaced and the call still fail. The rollback has happened
+// either way, and the worse outcome is an instance left down and unreported.
+func TestParamGuard_ReportsAndRestartsWhenOnlyTheEnforcementFailed(t *testing.T) {
+	engine := &fakeRecovery{restored: true, restoreErr: errors.New("write the TLS enforcement rule: read-only file system")}
+	cp := newFakeControlPlane()
+	guard := newParamGuard(engine, stubProbe(t, 2, nil), cp)
+	guard.id = identity{DBInstanceIdentifier: "db-1"}
+	guard.after, guard.poll = 20*time.Millisecond, time.Millisecond
+	guard.Run(t.Context())
+
+	if engine.restarts != 1 {
+		t.Errorf("restarts = %d, want the engine restarted on the set already put back", engine.restarts)
+	}
+	states := cp.snapshotStates()
+	if len(states) != 1 {
+		t.Fatalf("submitted states = %d, want the rollback still reported", len(states))
+	}
+	if states[0].health != handlers_rds.EngineHealthUnhealthy || states[0].message != handlers_rds.ParameterRollbackMessage {
+		t.Errorf("rollback state = %+v, want the unhealthy rollback report", states[0])
+	}
+}
+
+// The other half of the same contract: a restore that replaced nothing leaves
+// the instance on the set it is already failing under, so restarting it would
+// only churn a cluster that is down for a reason the rollback cannot fix.
+func TestParamGuard_DoesNotRestartWhenTheRestoreReplacedNothing(t *testing.T) {
+	engine := &fakeRecovery{restored: false, restoreErr: errors.New("read the last known good parameters: input/output error")}
+	cp := newFakeControlPlane()
+	guard := newParamGuard(engine, stubProbe(t, 2, nil), cp)
+	guard.after, guard.poll = 20*time.Millisecond, time.Millisecond
+	guard.Run(t.Context())
+
+	if engine.restarts != 0 {
+		t.Errorf("restarts = %d, want none when nothing was rolled back", engine.restarts)
+	}
+	if states := cp.snapshotStates(); len(states) != 0 {
+		t.Errorf("submitted states = %d, want no rollback report", len(states))
+	}
+}
+
 func TestParamGuard_DoesNothingWhenTheEngineIsServing(t *testing.T) {
 	engine := &fakeRecovery{restored: true}
 	newTestGuard(engine, stubProbe(t, 0, nil)).Run(t.Context())

cmd/rds-agent/quiesce.go

@@ -15,10 +15,9 @@ import (
 	"time"
 )
 
-// PostgreSQL's non-exclusive backup API ties the backup to the session that
-// started it: when that session ends, for any reason, the backup is aborted.
-// That is what makes the hold self-expiring — a control plane that dies mid
-// snapshot cannot leave the engine in backup mode forever.
+// Both engines tie a backup hold to the session that took it: PostgreSQL aborts
+// a non-exclusive backup when its session ends, MariaDB releases a BACKUP STAGE
+// with its connection. That is what makes the hold self-expiring.
 type engineSession interface {
 	// Runs sql and waits for the engine to finish it.
 	Exec(ctx context.Context, sql string) error
@@ -31,7 +30,8 @@ type engineSession interface {
 type sessionRunner func(ctx context.Context, c command) (engineSession, error)
 
 // Written after every statement so the reader knows the engine is done with it.
-// The statements this carries return LSNs, so it cannot appear in a result.
+// The statements this carries return LSNs and backup stages, so it cannot appear
+// in a result.
 const sessionSentinel = "--rds-agent-ready--"
 
 // The backup mode currently held, and the timer that ends it regardless of what
@@ -42,76 +42,66 @@ type quiesceHold struct {
 	expiry  *time.Timer
 }
 
-// Puts the engine into backup mode: the datadir is checkpointed and the engine
-// stops writing over the pages a snapshot is about to read. The hold is released
-// by Unquiesce, or by its own deadline, whichever comes first.
-func (e *postgresEngine) Quiesce(ctx context.Context, label string, hold time.Duration) error {
+// The hold bookkeeping, which is the same for both engines: only the statements
+// that take and release it differ, not the single-hold rule or the deadline that
+// releases it whatever happens to the caller.
+type quiesceState struct {
+	// Guarded because the expiry timer releases the hold on its own goroutine.
+	mu   sync.Mutex
+	held *quiesceHold
+}
+
+// The two things every quiesce needs, checked before a session is opened so a
+// malformed command costs the engine nothing.
+func validateQuiesceRequest(label string, hold time.Duration) error {
 	if label == "" {
 		return errors.New("quiesce requires a backup label")
 	}
 	if hold <= 0 {
 		return errors.New("quiesce requires a positive hold deadline")
 	}
-
-	// Held across the whole start, so a second quiesce waits and then finds the
-	// first one's hold rather than opening a concurrent backup alongside it.
-	e.mu.Lock()
-	defer e.mu.Unlock()
-	if e.held != nil {
-		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)
-	}
-
-	// Deliberately not the command's context: the session has to outlive the call
-	// that started it, and its own deadline is what bounds it instead.
-	session, err := e.startSess(context.WithoutCancel(ctx), command{
-		Name:  e.psql,
-		Args:  e.psqlArgs(),
-		Env:   []string{"PATH=" + defaultGuestPath, "RDS_BACKUP_LABEL=" + label},
-		Stdin: "",
-		User:  e.osUser,
-	})
-	if err != nil {
-		return fmt.Errorf("open a backup session: %w", err)
-	}
-
-	// fast forces an immediate checkpoint rather than spreading it over the
-	// checkpoint interval, which would hold the snapshot open for minutes.
-	const sql = `\getenv label RDS_BACKUP_LABEL
-SELECT pg_backup_start(:'label', fast => true);
-`
-	if err := session.Exec(ctx, sql); err != nil {
-		if closeErr := session.Close(); closeErr != nil {
-			slog.Warn("rds-agent: closing a failed backup session", "err", closeErr)
-		}
-		return fmt.Errorf("put the engine into backup mode: %w", err)
-	}
-
-	e.held = &quiesceHold{
-		label:   label,
-		session: session,
-		expiry:  time.AfterFunc(hold, func() { e.expireQuiesce(label) }),
-	}
-	slog.Info("rds-agent: engine quiesced for backup", "label", label, "hold", hold)
 	return nil
 }
 
-// Ends the backup cleanly. A missing hold is an error rather than a silent
-// success: it means the deadline fired first, so the snapshot the control plane
-// just took was not taken against a held checkpoint.
-func (e *postgresEngine) Unquiesce(ctx context.Context) error {
-	e.mu.Lock()
-	held := e.held
-	e.held = nil
-	e.mu.Unlock()
+// Records the hold and arms its deadline. Called with mu held, so a second
+// quiesce waits and then finds this hold rather than opening a concurrent
+// backup alongside it.
+func (q *quiesceState) beginHoldLocked(label string, session engineSession, hold time.Duration) {
+	q.held = &quiesceHold{
+		label:   label,
+		session: session,
+		expiry:  time.AfterFunc(hold, func() { q.expire(label) }),
+	}
+	slog.Info("rds-agent: engine quiesced for backup", "label", label, "hold", hold)
+}
 
+// Takes the hold off the engine, if one is still there, and disarms its
+// deadline. The caller ends the session, because the statement that releases the
+// backup has to run on it first.
+func (q *quiesceState) takeHold() *quiesceHold {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	held := q.held
+	q.held = nil
+	if held != nil {
+		held.expiry.Stop()
+	}
+	return held
+}
+
+// Ends the backup cleanly, running releaseSQL on the session that took the hold.
+// A missing hold is an error rather than a silent success: it means the deadline
+// fired first, so what the control plane just snapshotted was not held.
+func (q *quiesceState) releaseHold(ctx context.Context, releaseSQL string) error {
+	held := q.takeHold()
 	if held == nil {
 		return errors.New("the engine is not quiesced; the backup hold had already expired")
 	}
-	held.expiry.Stop()
 
-	// The stop has to run on the session that started the backup, and the session
-	// is closed either way — the engine aborts an unstopped backup with it.
-	execErr := held.session.Exec(ctx, "SELECT pg_backup_stop();\n")
+	// The release has to run on the session that took the hold, and the session is
+	// closed either way — both engines end an unreleased backup with it.
+	execErr := held.session.Exec(ctx, releaseSQL)
 	closeErr := held.session.Close()
 	if execErr != nil {
 		return fmt.Errorf("take the engine out of backup mode: %w", errors.Join(execErr, closeErr))
@@ -123,17 +113,17 @@ func (e *postgresEngine) Unquiesce(ctx context.Context) error {
 	return nil
 }
 
-// The deadline. Ending the session is enough — the engine aborts the backup it
-// was holding — so this never has to reach the engine itself.
-func (e *postgresEngine) expireQuiesce(label string) {
-	e.mu.Lock()
-	held := e.held
+// The deadline. Ending the session is enough — both engines release the hold
+// with it — so this never has to reach the engine itself.
+func (q *quiesceState) expire(label string) {
+	q.mu.Lock()
+	held := q.held
 	if held == nil || held.label != label {
-		e.mu.Unlock()
+		q.mu.Unlock()
 		return
 	}
-	e.held = nil
-	e.mu.Unlock()
+	q.held = nil
+	q.mu.Unlock()
 
 	slog.Warn("rds-agent: backup hold expired; releasing the engine", "label", label)
 	if err := held.session.Close(); err != nil {
@@ -141,21 +131,27 @@ func (e *postgresEngine) expireQuiesce(label string) {
 	}
 }
 
-// A psql child with its stdin held open, so statements can be fed to it one at a
-// time and the engine keeps seeing a single session.
-type psqlSession struct {
+// A client child with its stdin held open, so statements can be fed to it one at
+// a time and the engine keeps seeing a single session.
+type clientSession struct {
 	cmd    *exec.Cmd
 	stdin  io.WriteCloser
 	stdout *bufio.Reader
 	stderr *bytes.Buffer
+	// The statement that makes this client print the sentinel, which is the only
+	// part of a held session that differs between engines.
+	sentinel string
 
 	mu     sync.Mutex
 	closed bool
 }
 
-var _ engineSession = (*psqlSession)(nil)
+var _ engineSession = (*clientSession)(nil)
 
 func execSessionRunner(ctx context.Context, c command) (engineSession, error) {
+	if c.SentinelStatement == "" {
+		return nil, fmt.Errorf("%s was started as a session with no sentinel statement", c.Name)
+	}
 	cmd := exec.CommandContext(ctx, c.Name, c.Args...)
 	cmd.Env = c.Env
 	if c.User != "" {
@@ -180,24 +176,30 @@ func execSessionRunner(ctx context.Context, c command) (engineSession, error) {
 	if err := cmd.Start(); err != nil {
 		return nil, fmt.Errorf("start %s: %w", c.Name, err)
 	}
-	return &psqlSession{cmd: cmd, stdin: stdin, stdout: bufio.NewReader(stdout), stderr: &stderr}, nil
+	return &clientSession{
+		cmd:      cmd,
+		stdin:    stdin,
+		stdout:   bufio.NewReader(stdout),
+		stderr:   &stderr,
+		sentinel: c.SentinelStatement,
+	}, nil
 }
 
-// Feeds sql and waits for the sentinel that follows it. ON_ERROR_STOP makes psql
-// exit on a failed statement, so a sentinel that never arrives is the failure
-// itself and the engine's own message is on stderr.
-func (s *psqlSession) Exec(ctx context.Context, sql string) error {
+// Feeds sql and waits for the sentinel that follows it. Both clients are run in
+// a mode that exits on a failed statement, so a sentinel that never arrives is
+// the failure itself and the engine's own message is on stderr.
+func (s *clientSession) Exec(ctx context.Context, sql string) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.closed {
 		return errors.New("the backup session is closed")
 	}
-	if _, err := io.WriteString(s.stdin, sql+"\\echo "+sessionSentinel+"\n"); err != nil {
+	if _, err := io.WriteString(s.stdin, sql+s.sentinel); err != nil {
 		return fmt.Errorf("%w: %s", err, strings.TrimSpace(s.stderr.String()))
 	}
 
-	// Read on its own goroutine so a psql that never answers costs the caller its
-	// deadline rather than blocking it forever.
+	// Read on its own goroutine so a client that never answers costs the caller
+	// its deadline rather than blocking it forever.
 	done := make(chan error, 1)
 	go func() { done <- s.awaitSentinel() }()
 	select {
@@ -208,7 +210,7 @@ func (s *psqlSession) Exec(ctx context.Context, sql string) error {
 	}
 }
 
-func (s *psqlSession) awaitSentinel() error {
+func (s *clientSession) awaitSentinel() error {
 	for {
 		line, err := s.stdout.ReadString('\n')
 		if strings.TrimSpace(line) == sessionSentinel {
@@ -226,7 +228,7 @@ func (s *psqlSession) awaitSentinel() error {
 // Closing stdin ends the script, and the wait reaps the child. A session that
 // will not end is killed, because leaving it is what would keep the engine in
 // backup mode.
-func (s *psqlSession) Close() error {
+func (s *clientSession) Close() error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.closed {

cmd/rds-agent/quiesce_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"errors"
 	"slices"
 	"strings"
 	"sync"
@@ -113,81 +112,6 @@ func TestPostgresEngine_UnquiesceStopsTheBackupAndEndsTheSession(t *testing.T) {
 	}
 }
 
-// A hold that expired mid-snapshot means the snapshot was not taken against a
-// held checkpoint, so the release reports it rather than succeeding silently.
-func TestPostgresEngine_UnquiesceWithoutAHoldIsAnError(t *testing.T) {
-	engine, _ := newQuiesceEngine(t, &fakeSession{})
-
-	err := engine.Unquiesce(context.Background())
-	if err == nil {
-		t.Fatal("Unquiesce succeeded with no hold to release")
-	}
-	if !strings.Contains(err.Error(), "expired") {
-		t.Errorf("error %q does not say the hold had expired", err)
-	}
-}
-
-// The engine aborts the backup when the session ends, which is what makes the
-// hold self-expiring: a control plane that dies cannot leave it held forever.
-func TestPostgresEngine_QuiesceReleasesItselfOnItsDeadline(t *testing.T) {
-	session := &fakeSession{}
-	engine, _ := newQuiesceEngine(t, session)
-
-	if err := engine.Quiesce(context.Background(), "orders-db-pre-upgrade", 10*time.Millisecond); err != nil {
-		t.Fatalf("Quiesce: %v", err)
-	}
-	deadline := time.Now().Add(2 * time.Second)
-	for session.closes() == 0 && time.Now().Before(deadline) {
-		time.Sleep(time.Millisecond)
-	}
-
-	if session.closes() != 1 {
-		t.Fatal("the backup session outlived its deadline")
-	}
-	// The hold is gone with it, so a later release is honest about that.
-	if err := engine.Unquiesce(context.Background()); err == nil {
-		t.Error("Unquiesce succeeded after the hold expired")
-	}
-}
-
-// One backup at a time: a second start would open a concurrent backup alongside
-// the first rather than extending it.
-func TestPostgresEngine_QuiesceRefusesASecondHold(t *testing.T) {
-	session := &fakeSession{}
-	engine, started := newQuiesceEngine(t, session)
-	if err := engine.Quiesce(context.Background(), "first", time.Minute); err != nil {
-		t.Fatalf("Quiesce: %v", err)
-	}
-
-	err := engine.Quiesce(context.Background(), "second", time.Minute)
-	if err == nil {
-		t.Fatal("a second quiesce was accepted while the first was held")
-	}
-	if !strings.Contains(err.Error(), "first") {
-		t.Errorf("error %q does not name the backup already holding the engine", err)
-	}
-	if len(*started) != 1 {
-		t.Errorf("started %d sessions, want the second refused before opening one", len(*started))
-	}
-}
-
-// A failed start leaves nothing held, so a retry is not refused by a hold that
-// never took effect.
-func TestPostgresEngine_QuiesceEndsTheSessionWhenTheBackupWillNotStart(t *testing.T) {
-	session := &fakeSession{execErr: errors.New(`ERROR:  a backup is already in progress`)}
-	engine, _ := newQuiesceEngine(t, session)
-
-	if err := engine.Quiesce(context.Background(), "orders-db-pre-upgrade", time.Minute); err == nil {
-		t.Fatal("Quiesce succeeded against an engine that refused the backup")
-	}
-	if session.closes() != 1 {
-		t.Errorf("closed %d times, want the failed session ended", session.closes())
-	}
-	if engine.held != nil {
-		t.Error("a failed quiesce left a hold behind")
-	}
-}
-
 func TestCommandRegistry_QuiesceCarriesTheLabelAndDeadline(t *testing.T) {
 	engine := &fakeEngine{}
 	reply := newCommander(nil, newCommandRegistry(engine, &fakeStorage{}), 0).execute(context.Background(),

cmd/rds-agent/storage.go

@@ -10,10 +10,9 @@ import (
 	"strings"
 )
 
-// The in-guest half of a storage grow. The control plane has already
-// grown the block device by the time this runs; all that is left is to put the
-// filesystem on the capacity that is now there. Both ext4 and XFS grow while
-// mounted, so this needs no ordering against the engine start.
+// The in-guest half of a storage grow: the control plane has already grown the
+// block device, so all that is left is the filesystem. Both ext4 and XFS grow
+// while mounted, so this needs no ordering against the engine start.
 type storageOps interface {
 	// Extends the filesystem at the data mount onto its whole device, and
 	// reports what it did for the command reply.
@@ -28,8 +27,6 @@ const (
 	defaultXFSGrowfs  = "xfs_growfs"
 	defaultMountsFile = "/proc/mounts"
 	defaultSysBlock   = "/sys/class/block"
-	// Where rds-datadir mounts the data volume; the datadir sits one level in.
-	defaultDataMount = "/var/lib/postgresql"
 	// growpart says this when the partition already fills the disk, which a
 	// resumed grow is the normal way to reach.
 	growpartNoChange = "NOCHANGE"

cmd/rds-agent/storage_test.go

@@ -61,12 +61,12 @@ func newTestStorage(t *testing.T, device, fstype string, run commandRunner) *gue
 	mounts := filepath.Join(dir, "mounts")
 	content := "proc /proc proc rw 0 0\n"
 	if device != "" {
-		content += device + " " + defaultDataMount + " " + fstype + " rw,relatime 0 0\n"
+		content += device + " " + testDataMount + " " + fstype + " rw,relatime 0 0\n"
 	}
 	if err := os.WriteFile(mounts, []byte(content), 0o600); err != nil {
 		t.Fatalf("write the mount table: %v", err)
 	}
-	cfg := config{DataMount: defaultDataMount, MountsFile: mounts, SysBlock: filepath.Join(dir, "block")}
+	cfg := config{DataMount: testDataMount, MountsFile: mounts, SysBlock: filepath.Join(dir, "block")}
 	if err := os.MkdirAll(cfg.SysBlock, 0o755); err != nil {
 		t.Fatalf("create the sysfs tree: %v", err)
 	}
@@ -93,6 +93,9 @@ func seedPartition(t *testing.T, g *guestStorage, disk, name, number string) {
 
 // The data volume is formatted whole, so the usual grow is resize2fs against
 // the device with no partition to push out first.
+// Where rds-datadir mounts the data volume in the PostgreSQL preset.
+const testDataMount = "/var/lib/postgresql"
+
 func TestGrowFilesystem_ResizesAnExt4DeviceOnline(t *testing.T) {
 	log := &runLog{}
 	g := newTestStorage(t, "/dev/vdb", "ext4", log.run)
@@ -108,7 +111,7 @@ func TestGrowFilesystem_ResizesAnExt4DeviceOnline(t *testing.T) {
 	if args := log.calls[0].Args; len(args) != 1 || args[0] != "/dev/vdb" {
 		t.Errorf("resize2fs args = %v, want the device", args)
 	}
-	if !strings.Contains(message, "/dev/vdb") || !strings.Contains(message, defaultDataMount) {
+	if !strings.Contains(message, "/dev/vdb") || !strings.Contains(message, testDataMount) {
 		t.Errorf("reply message = %q, want the device and mount it grew", message)
 	}
 }
@@ -125,7 +128,7 @@ func TestGrowFilesystem_GrowsXFSByMountPoint(t *testing.T) {
 	if got := log.names(); len(got) != 1 || got[0] != defaultXFSGrowfs {
 		t.Fatalf("ran %v, want just %s", got, defaultXFSGrowfs)
 	}
-	if args := log.calls[0].Args; len(args) != 1 || args[0] != defaultDataMount {
+	if args := log.calls[0].Args; len(args) != 1 || args[0] != testDataMount {
 		t.Errorf("xfs_growfs args = %v, want the mount point", args)
 	}
 }

cmd/spinifex/cmd/admin.go

@@ -26,22 +26,22 @@ import (
 	"sync"
 	"time"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/google/uuid"
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/ebsmetadata"
+	"github.com/mulgadc/spinifex/spinifex/ebsprovider"
 	"github.com/mulgadc/spinifex/spinifex/formation"
 	"github.com/mulgadc/spinifex/spinifex/gpu"
+	handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot"
 	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
 	"github.com/mulgadc/spinifex/spinifex/hostdns"
 	"github.com/mulgadc/spinifex/spinifex/network/host"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/utils"
-	"github.com/mulgadc/viperblock/viperblock"
-	"github.com/mulgadc/viperblock/viperblock/backends/s3"
-	"github.com/mulgadc/viperblock/viperblock/v_utils"
 	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/pterm/pterm"
 	"github.com/spf13/cobra"
 )
@@ -118,8 +118,9 @@ var nodeDrainCmd = &cobra.Command{
 its guests via QMP and unmount their volumes (flushing the viperblock WAL) while
 every service is still running. STORAGE/PERSIST/INFRA are left to systemd's
 ordered unit teardown. This is what spinifex-shutdown.service's ExecStop runs,
-with --only-if-host-stopping so it only drains on a genuine host
-shutdown/reboot -- not on a plain "systemctl restart/stop spinifex.target".
+with --unless-restarting so it drains on a genuine host shutdown/reboot and on
+a plain "systemctl stop spinifex.target", but skips the drain when a restart
+of the stack (e.g. "systemctl restart spinifex.target") is already queued.
 Run by hand without that flag, it always drains unconditionally.`,
 	Run: runNodeDrainLocal,
 }
@@ -197,6 +198,25 @@ takes effect immediately. Prompts for confirmation unless --yes is passed.`,
 	Run: runimagesPromoteCmd,
 }
 
+var volumesCmd = &cobra.Command{
+	Use:   "volumes",
+	Short: "Inspect block storage volumes",
+	Long:  `Inspect the volumes the storage provider holds, independently of the EC2 API.`,
+}
+
+var volumesOrphansCmd = &cobra.Command{
+	Use:   "orphans",
+	Short: "Report volumes the provider holds with no control-plane record",
+	Long: `List volumes the storage provider holds that the control plane has no record
+of. Their blocks are consuming space but they have no API handle, so they
+cannot be described or deleted through the EC2 API.
+
+This command only reports. It never deletes: the evidence that an orphan is
+still wanted is the record that went missing, so removal is an operator
+decision made per volume.`,
+	Run: runVolumesOrphansCmd,
+}
+
 var accountCmd = &cobra.Command{
 	Use:   "account",
 	Short: "Manage Spinifex accounts",
@@ -283,7 +303,9 @@ func init() {
 	nodeCmd.AddCommand(nodeDrainCmd)
 	nodeDrainCmd.Flags().Bool("local", false, "Drain the local node only (required)")
 	nodeDrainCmd.Flags().Duration("timeout", 120*time.Second, "Maximum time to wait per phase")
-	nodeDrainCmd.Flags().Bool("only-if-host-stopping", false, "Skip the drain unless systemd is unwinding into shutdown.target (real reboot/poweroff); unset, always drains")
+	nodeDrainCmd.Flags().Bool("unless-restarting", false, "Drain on a real host shutdown or a plain target stop; skip only when a restart of the stack is already queued. Unset, always drains")
+	nodeDrainCmd.Flags().Bool("only-if-host-stopping", false, "Deprecated: use --unless-restarting")
+	_ = nodeDrainCmd.Flags().MarkDeprecated("only-if-host-stopping", "use --unless-restarting instead")
 	nodeCmd.AddCommand(nodeJSProbeCmd)
 	nodeJSProbeCmd.Flags().Duration("timeout", 10*time.Second, "Maximum time to wait for the canary round-trip")
 
@@ -293,6 +315,9 @@ func init() {
 	imagesCmd.AddCommand(imagesRemoveCmd)
 	imagesCmd.AddCommand(imagesPromoteCmd)
 
+	adminCmd.AddCommand(volumesCmd)
+	volumesCmd.AddCommand(volumesOrphansCmd)
+
 	adminCmd.AddCommand(accountCmd)
 	accountCmd.AddCommand(accountCreateCmd)
 	accountCmd.AddCommand(accountListCmd)
@@ -305,6 +330,9 @@ func init() {
 
 	adminCmd.AddCommand(upgradeCmd)
 	upgradeCmd.Flags().Bool("yes", false, "Apply migrations without prompting")
+	upgradeCmd.Flags().Bool("dry-run", false, "Report pending config and unit changes without applying them")
+	upgradeCmd.Flags().Bool("units-only", false, "Reconcile systemd units only, skip config migrations")
+	upgradeCmd.Flags().Bool("skip-units", false, "Apply config migrations only, skip systemd unit reconciliation")
 	certRenewCmd.Flags().StringSlice("extra-ip", nil, "Additional IP addresses to include in SANs")
 	certRenewCmd.Flags().StringSlice("extra-dns", nil, "Additional DNS names to include in SANs")
 	accountCreateCmd.Flags().String("name", "", "Account name (required)")
@@ -400,6 +428,10 @@ func init() {
 
 const bytesPerGiB = 1024 * 1024 * 1024
 
+// imageImportTimeout bounds each provider call the import makes. Generous
+// because the snapshot at the end runs over the whole imported volume.
+const imageImportTimeout = 30 * time.Minute
+
 // amiVolumeSizeGiB returns the smallest whole GiB that still holds sizeBytes.
 //
 // Rounding up is load-bearing. The image is copied into a root volume of
@@ -427,16 +459,10 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	var imageStat os.FileInfo
 	var err error
 
-	cfgFile, _ := cmd.Flags().GetString("config")
 	forceCmd, _ := cmd.Flags().GetBool("force")
 	skipVerify, _ := cmd.Flags().GetBool("skip-verify")
 	ostmpDir, _ := cmd.Flags().GetString("tmp-dir")
 
-	// Use default config path
-	if cfgFile == "" {
-		cfgFile = DefaultConfigFile()
-	}
-
 	//configDir, _ := cmd.Flags().GetString("config-dir")
 	baseDir, _ := cmd.Flags().GetString("spinifex-dir")
 
@@ -543,9 +569,11 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	// Create the specified image directory
 	imagePath := fmt.Sprintf("%s/%s/%s/%s", imageDir, image.Distro, image.Version, image.Arch)
 
-	// Create config directory
-	if err := os.MkdirAll(imagePath, 0700); err != nil {
-		fmt.Fprintf(os.Stderr, "Error creating config directory: %v\n", err)
+	// 0770, matching the root:spinifex convention setup.sh gives the shared data
+	// dirs. At 0700 the first sudo import leaves a directory no group member can
+	// descend into, so every later import without sudo fails on mkdir.
+	if err := os.MkdirAll(imagePath, 0770); err != nil {
+		fmt.Fprintf(os.Stderr, "Error creating image directory %s: %v\n", imagePath, err)
 		os.Exit(1)
 	}
 
@@ -607,57 +635,46 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	// Create the specified manifest file to describe the image/AMI
-	manifest := viperblock.VolumeConfig{}
-
-	// Calculate the size
-
-	manifest.AMIMetadata.Name = fmt.Sprintf("ami-%s-%s-%s", image.Distro, image.Version, image.Arch)
-	if amiNameOverride != "" {
-		manifest.AMIMetadata.Name = amiNameOverride
-	}
+	// Describe the image/AMI as the control-plane document it will become.
 	volumeId := utils.GenerateResourceID("ami")
-	manifest.AMIMetadata.ImageID = volumeId
+	amiName := fmt.Sprintf("ami-%s-%s-%s", image.Distro, image.Version, image.Arch)
+	if amiNameOverride != "" {
+		amiName = amiNameOverride
+	}
 
-	manifest.AMIMetadata.Description = fmt.Sprintf("%s cloud image prepared for Spinifex", manifest.AMIMetadata.Name)
-	manifest.AMIMetadata.Architecture = image.Arch
-	manifest.AMIMetadata.PlatformDetails = image.Platform
-	manifest.AMIMetadata.CreationDate = time.Now()
-	manifest.AMIMetadata.RootDeviceType = "ebs"
-	manifest.AMIMetadata.Virtualization = "hvm"
-	manifest.AMIMetadata.ImageOwnerAlias = "system"
-	manifest.AMIMetadata.VolumeSizeGiB = amiVolumeSizeGiB(imageStat.Size())
-	manifest.AMIMetadata.BootMode = image.BootMode
-	manifest.AMIMetadata.Distro = image.Distro
-	manifest.AMIMetadata.DistroFamily = utils.DistroFamily(image.Distro)
+	ami := ebsmetadata.AMI{
+		ImageID:         volumeId,
+		Name:            amiName,
+		Description:     fmt.Sprintf("%s cloud image prepared for Spinifex", amiName),
+		Architecture:    image.Arch,
+		PlatformDetails: image.Platform,
+		CreationDate:    time.Now(),
+		RootDeviceType:  "ebs",
+		Virtualization:  "hvm",
+		ImageOwnerAlias: "system",
+		VolumeSizeGiB:   amiVolumeSizeGiB(imageStat.Size()),
+		SnapshotID:      admin.SnapPrefix(volumeId),
+		BootMode:        image.BootMode,
+		Distro:          image.Distro,
+		DistroFamily:    utils.DistroFamily(image.Distro),
+	}
 
 	// Copy catalog-provided tags (e.g. spinifex:managed-by for system AMIs)
 	// onto the imported AMI so the UI can filter them out.
 	if len(image.Tags) > 0 {
-		manifest.AMIMetadata.Tags = make(map[string]string, len(image.Tags))
-		maps.Copy(manifest.AMIMetadata.Tags, image.Tags)
+		ami.Tags = make(map[string]string, len(image.Tags))
+		maps.Copy(ami.Tags, image.Tags)
 	}
 
-	// Volume Data
-	manifest.VolumeMetadata.VolumeID = volumeId // TODO: Confirm if unique, e.g vol-, if ami- used
-	manifest.VolumeMetadata.VolumeName = manifest.AMIMetadata.Name
-	manifest.VolumeMetadata.TenantID = "system"
-	manifest.VolumeMetadata.SizeGiB = manifest.AMIMetadata.VolumeSizeGiB
-	manifest.VolumeMetadata.State = "available"
-	manifest.VolumeMetadata.AvailabilityZone = "" // TODO: Confirm
-	manifest.VolumeMetadata.CreatedAt = time.Now()
-	manifest.VolumeMetadata.VolumeType = "gp3"
-	manifest.VolumeMetadata.IOPS = 1000
-
 	// Write the manifest to disk
 	// Save as JSON
-	jsonData, err := json.Marshal(manifest)
+	jsonData, err := json.Marshal(ami)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Could not marshal manifest: %v\n", err)
 		os.Exit(1)
 	}
 
-	manifestFilename := fmt.Sprintf("%s/%s.json", imagePath, manifest.AMIMetadata.Name)
+	manifestFilename := fmt.Sprintf("%s/%s.json", imagePath, ami.Name)
 	// Write to file
 	err = os.WriteFile(manifestFilename, jsonData, 0600)
 	if err != nil {
@@ -665,145 +682,118 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	// Upload the image to S3 (predastore)
-
-	appConfig, err := config.LoadConfig(cfgFile)
-
-	if err != nil {
-		fmt.Println("Error loading config file:", err)
-		return
-	}
-
-	s3Config := s3.S3Config{
-		VolumeName: volumeId,
-		VolumeSize: utils.SafeInt64ToUint64(imageStat.Size()),
-		Bucket:     appConfig.Nodes[appConfig.Node].Predastore.Bucket,
-		Region:     appConfig.Nodes[appConfig.Node].Predastore.Region,
-		AccessKey:  appConfig.Nodes[appConfig.Node].Predastore.AccessKey,
-		SecretKey:  appConfig.Nodes[appConfig.Node].Predastore.SecretKey,
-		Host:       appConfig.Nodes[appConfig.Node].Predastore.Host,
-	}
-
-	mkey, err := utils.LoadViperblockMasterKey(appConfig.Nodes[appConfig.Node].Viperblock.EncryptionKeyFile)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not load viperblock encryption key: %v\n", err)
-		os.Exit(1)
-	}
-
-	// Import is an interactive command whose output is a progress bar, so the
-	// volume's routine mount chatter is noise here: a fresh import has no prior
-	// state, making the "no state found" / 404 lines expected rather than
-	// notable. Scoping the logger to this VB (New copies it onto the backend
-	// too) keeps it off the process-wide default. Errors still surface, both
-	// through this logger and as the returned error the caller prints.
-	vbConfig := viperblock.VB{
-		VolumeName: volumeId,
-		VolumeSize: utils.SafeInt64ToUint64(imageStat.Size()),
-		BaseDir:    tmpDir,
-		Cache: viperblock.Cache{
-			Config: viperblock.CacheConfig{
-				Size: 0,
-			},
-		},
-		VolumeConfig:      manifest,
-		MasterKey:         mkey,
-		EncryptionEnabled: mkey != nil,
-		Logger:            slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})),
-	}
-
-	// Bake the deployment CA into the image trust store so a stock cloud image
-	// trusts the gateway from first boot. Best-effort: never fails the import.
-	// The CA path derives from the --spinifex-dir data root (its config/ symlink
-	// holds ca.pem), not NodeBaseDir(): node.BaseDir is unset in the import
-	// command, which would collapse the path to a relative config/ca.pem.
-	bakeCACertIntoImage(extractedImagePath, filepath.Join(baseDir, "config", "ca.pem"))
-
-	// Render the flush bar here rather than inside viperblock, which stays a
-	// pure storage library. The bar is built lazily on the first update so it is
-	// never drawn if the import fails before any bytes flush; viperblock
-	// throttles the callback to ≤101 invocations, so the human-readable title
-	// renders without the per-block render-frequency regression.
-	var flushBar *pterm.ProgressbarPrinter
-	var flushUpdate func(current uint64)
-	progress := func(current, total uint64) {
-		if flushBar == nil {
-			flushBar, flushUpdate = utils.NewByteProgressBar("Flushing image to storage", total)
-		}
-		flushUpdate(current)
-	}
-
-	err = v_utils.ImportDiskImage(&s3Config, &vbConfig, extractedImagePath, progress)
-
-	if flushBar != nil {
-		_, _ = flushBar.Stop()
-	}
-
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not import image to predastore: %v\n", err)
-		os.Exit(1)
-	}
-
 	defer os.RemoveAll(tmpDir)
 
+	appConfig, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not connect to the cluster: %v\n", err)
+		os.Exit(1)
+	}
+	defer nc.Close()
+	node := appConfig.Nodes[appConfig.Node]
+
+	fmt.Println("Writing image to storage ...")
+	provider := ebsprovider.NewNATSProvider(nc, imageImportTimeout)
+	err = admin.ImportImage(context.Background(), provider, admin.ImportOpts{
+		VolumeID:         volumeId,
+		NodeID:           appConfig.Node,
+		SizeBytes:        utils.SafeUint64ToInt64(ami.VolumeSizeGiB * bytesPerGiB),
+		AvailabilityZone: node.AZ,
+		SourcePath:       extractedImagePath,
+		Snapshot:         true,
+		Progress:         os.Stdout,
+	})
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not import image: %v\n", err)
+		os.Exit(1)
+	}
+
+	// admin.ImportImage only wrote the provider's half of the snapshot (its
+	// blocks); this writes the EC2 control plane's, without which
+	// DescribeSnapshots and GetAMISourceVolumeID cannot resolve the import.
+	metaStore := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
+	if err := registerImportedAMISnapshot(metaStore, node.Predastore.Bucket, ami, node.AZ, node.Viperblock.EncryptionKeyFile != ""); err != nil {
+		fmt.Fprintf(os.Stderr, "Imported %s but could not register its snapshot: %v\n", volumeId, err)
+		os.Exit(1)
+	}
+
+	// The document is what DescribeImages enumerates, so it is written last:
+	// until it exists the AMI is not launchable, and an import that failed
+	// partway leaves no half-registered image behind.
+	ami.State = "available"
+	if err := ebsmetadata.NewStore(metaStore, node.Predastore.Bucket).PutAMI(context.Background(), ami); err != nil {
+		fmt.Fprintf(os.Stderr, "Imported %s but could not register it: %v\n", volumeId, err)
+		os.Exit(1)
+	}
+
 	fmt.Printf("✅ Image import complete. Image-ID (AMI): %s\n", volumeId)
 }
 
-// caBakeRunCommand installs the uploaded CA into the guest trust store across
-// the distro families, then removes the staged copy. Each branch is gated on its
-// updater existing so only the matching distro's branch writes anything (install
-// -D creates the anchor dir on minimal images). Debian/Ubuntu use
-// update-ca-certificates, RHEL/Rocky use update-ca-trust, and stock Alpine ships
-// only ca-certificates-bundle (no updater, no anchor dir) so it falls back to
-// appending the PEM to the static bundle. The install chain's exit code is
-// preserved past the cleanup rm so a failure surfaces instead of a false success.
-// The RHEL branch restorecon's only the trust-store paths it touched so the cert
-// is correctly labelled without the image-wide --no-selinux-relabel (below)
-// leaving an unlabelled bundle; restorecon is best-effort and never fails the bake.
-const caBakeRunCommand = `( { command -v update-ca-certificates >/dev/null && install -D -m644 /tmp/spinifex-ca.pem /usr/local/share/ca-certificates/spinifex.crt && update-ca-certificates; } || ` +
-	`{ command -v update-ca-trust >/dev/null && install -D -m644 /tmp/spinifex-ca.pem /etc/pki/ca-trust/source/anchors/spinifex.crt && update-ca-trust && ` +
-	`{ command -v restorecon >/dev/null 2>&1 && restorecon -RF /etc/pki/ca-trust/source/anchors/spinifex.crt /etc/pki/ca-trust/extracted >/dev/null 2>&1; true; }; } || ` +
-	`{ cat /tmp/spinifex-ca.pem >> /etc/ssl/certs/ca-certificates.crt; } ); ` +
-	`rc=$?; rm -f /tmp/spinifex-ca.pem; exit $rc`
-
-// caBakeTimeout bounds the virt-customize run so a stalled libguestfs appliance
-// cannot hang the import indefinitely; on timeout the bake degrades to a skip.
-const caBakeTimeout = 5 * time.Minute
-
-// caBakeCmd builds the virt-customize invocation that uploads the deployment CA
-// into the disk image at imagePath and installs it into the guest trust store.
-// --no-selinux-relabel stops virt-customize flagging the image for a first-boot
-// SELinux autorelabel: that relabel+reboot corrupts XFS roots (RHEL/Rocky) on the
-// reboot, so the run-command relabels only the touched trust paths instead.
-func caBakeCmd(ctx context.Context, imagePath, caCertPath string) *exec.Cmd {
-	return exec.CommandContext(ctx, "virt-customize", "-a", imagePath,
-		"--no-selinux-relabel",
-		"--upload", caCertPath+":/tmp/spinifex-ca.pem",
-		"--run-command", caBakeRunCommand)
+// registerImportedAMISnapshot writes the EC2 control plane's snapshot document
+// for a catalog-imported AMI. The provider-backed import path only writes the
+// storage backend's blocks under ami.SnapshotID, so without this a launch or
+// DescribeSnapshots call cannot resolve it. The volume ID matches ami.ImageID:
+// admin.ImportImage creates the volume under the AMI's own ID.
+func registerImportedAMISnapshot(store objectstore.ObjectStore, bucket string, ami ebsmetadata.AMI, az string, encrypted bool) error {
+	cfg := &handlers_ec2_snapshot.SnapshotConfig{
+		SnapshotID:       ami.SnapshotID,
+		VolumeID:         ami.ImageID,
+		VolumeSize:       utils.SafeUint64ToInt64(ami.VolumeSizeGiB),
+		State:            "completed",
+		Progress:         "100%",
+		StartTime:        time.Now(),
+		Description:      fmt.Sprintf("Imported AMI volume for %s", ami.Name),
+		Encrypted:        encrypted,
+		OwnerID:          utils.GlobalAccountID,
+		AvailabilityZone: az,
+	}
+	if err := handlers_ec2_snapshot.WriteSnapshotConfig(store, bucket, ami.SnapshotID, cfg); err != nil {
+		return fmt.Errorf("register snapshot metadata: %w", err)
+	}
+	return nil
 }
 
-// caBakeRunner resolves virt-customize and runs the CA bake; overridable in tests.
-var caBakeRunner = func(imagePath, caCertPath string) ([]byte, error) {
-	if _, err := exec.LookPath("virt-customize"); err != nil {
-		return nil, fmt.Errorf("virt-customize not found: %w", err)
+func runVolumesOrphansCmd(_ *cobra.Command, _ []string) {
+	appConfig, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not connect to the cluster: %v\n", err)
+		os.Exit(1)
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), caBakeTimeout)
-	defer cancel()
-	return caBakeCmd(ctx, imagePath, caCertPath).CombinedOutput()
-}
+	defer nc.Close()
 
-// bakeCACertIntoImage uploads the deployment CA into the image's trust store via
-// virt-customize so an imported stock image trusts the gateway from first boot.
-// Best-effort: a missing CA, an absent virt-customize, or an image libguestfs
-// cannot inspect logs and continues — image import never fails on the CA.
-func bakeCACertIntoImage(imagePath, caCertPath string) {
-	if _, err := os.Stat(caCertPath); err != nil {
-		slog.Warn("CA bake skipped: deployment CA not found; imported image will not auto-trust the gateway", "ca", caCertPath, "err", err)
+	node := appConfig.Nodes[appConfig.Node]
+	store := objectstore.NewS3ObjectStoreFromConfig(
+		node.Predastore.Host,
+		node.Predastore.Region,
+		node.Predastore.AccessKey,
+		node.Predastore.SecretKey,
+	)
+
+	orphans, err := admin.FindOrphanVolumes(
+		context.Background(),
+		ebsprovider.NewNATSProvider(nc, imageImportTimeout),
+		ebsmetadata.NewStore(store, node.Predastore.Bucket),
+	)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Orphan scan failed:", err)
+		os.Exit(1)
+	}
+	if len(orphans) == 0 {
+		fmt.Println("No orphaned volumes: every volume the provider holds has a control-plane record.")
 		return
 	}
-	if out, err := caBakeRunner(imagePath, caCertPath); err != nil {
-		slog.Warn("CA bake skipped: virt-customize could not customize image; imported image will not auto-trust the gateway", "err", err, "output", string(out))
-		return
+
+	fmt.Printf("%d orphaned volume(s) — held by the provider, unknown to the control plane:\n\n", len(orphans))
+	for _, orphan := range orphans {
+		fmt.Printf("  %s\n", orphan.VolumeID)
+		fmt.Printf("    handle:  %s\n", orphan.Handle)
+		if orphan.Derived {
+			fmt.Println("    note:    derived volume; its base volume is also unknown")
+		}
 	}
+	fmt.Println()
+	fmt.Println("Nothing was deleted. Each of these holds data that cannot be reached")
+	fmt.Println("through the EC2 API; removing one is a per-volume operator decision.")
 }
 
 func runimagesRemoveCmd(cmd *cobra.Command, args []string) {
@@ -1342,18 +1332,32 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	if masterKeyExisted {
 		// Preserve path: reuse the existing identity. The system credentials must
 		// match what seeded the NATS KV `system` secret, so load them rather than
-		// mint new ones. The admin credentials are not recovered: bootstrap.json is
-		// consumed and deleted by awsgw after first boot, and the operator's copy
-		// already lives in ~/.aws/credentials. Leaving them empty makes
-		// finalizeNodeSetup refresh only ~/.aws/config (endpoint/CA for a changed
-		// bind IP), not the credentials.
+		// mint new ones.
 		accessKey, secretKey, err = loadSystemCredentials(configDir)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error loading preserved system credentials: %v\n", err)
 			os.Exit(1)
 		}
-		fmt.Println("\n🔐 Preserved existing identity (master key, credentials and CA unchanged)")
+
+		// The admin pair is minted afresh rather than preserved, because it cannot
+		// be read back: bootstrap.json is consumed and deleted by awsgw on first
+		// start. The half of the identity that survives a re-init is on local disk,
+		// but the IAM store lives in NATS and predastore — so forming a cluster out
+		// of already-initialized nodes rebuilds it and the old access key stops
+		// existing. Carrying an empty pair through instead is worse than useless:
+		// it reaches every joiner's bootstrap.json, where seeding rejects the empty
+		// key ID and awsgw fails to start on every restart, forever.
+		bootstrapResult, err := writeBootstrapFiles(configDir, bootstrapDir, masterKey, accessKey, secretKey, accountID)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error writing bootstrap files: %v\n", err)
+			os.Exit(1)
+		}
+		adminAccessKey = bootstrapResult.AdminAccessKey
+		adminSecretKey = bootstrapResult.AdminSecretKey
+
+		fmt.Println("\n🔐 Preserved existing identity (master key and CA unchanged)")
 		fmt.Printf("   Master key: %s\n", filepath.Join(configDir, "master.key"))
+		fmt.Printf("   Admin credentials reissued in %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
 	} else {
 		// Fresh install: mint system + admin credentials and seed the bootstrap files.
 		accessKey, err = admin.GenerateAWSAccessKey()
@@ -1524,8 +1528,10 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	// is node-local, so it is derived here from this node's own config dirs.
 	northstarConfigPath := filepath.Join(dirs.Northstar, "northstar.toml")
 
-	// Parse multi-node predastore configuration (legacy flag-based approach for single-node)
-	var predastoreHostID int
+	// Parse multi-node predastore configuration. A single-node install is host
+	// 1, the only [[host]] its predastore.toml declares; the multi-node path
+	// below replaces this with the host matching this machine.
+	predastoreHostID := 1
 	if predastoreNodesStr != "" {
 		ips := strings.Split(predastoreNodesStr, ",")
 		if len(ips) < 2 {
@@ -1806,6 +1812,10 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		os.Exit(1)
 	}
 
+	// The host firewall scopes this port to known peers, and a node joining is
+	// not one yet. Closed again on every exit below.
+	closeFormationPort := openFormationPort(port)
+
 	fmt.Printf("\n📡 Formation server started on %s\n", formationAddr)
 	fmt.Printf("   Waiting for %d more node(s) to join...\n", expectedNodes-1)
 	fmt.Printf("   Token expires in %s\n\n", tokenTTL)
@@ -1816,6 +1826,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	if err := fs.WaitForCompletion(formationTimeout); err != nil {
 		fmt.Fprintf(os.Stderr, "❌ %v\n", err)
 		fs.Shutdown(context.Background())
+		closeFormationPort()
 		os.Remove(tokenPath)
 		os.Exit(1)
 	}
@@ -1842,8 +1853,10 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	// is node-local, so it is derived here from this node's own config dirs.
 	northstarConfigPath := filepath.Join(dirs.Northstar, "northstar.toml")
 
-	// Generate multi-node predastore config
-	var predastoreHostID int
+	// Generate multi-node predastore config. A single-node install is host 1,
+	// the only [[host]] its predastore.toml declares; the multi-node path below
+	// replaces this with the host matching this machine.
+	predastoreHostID := 1
 	hasPredastoreConfig := len(predastoreNodes) >= 2
 	if hasPredastoreConfig {
 		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, spxRoot, bindIP, compactionInterval, northstarCreds)
@@ -1858,7 +1871,14 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 			os.Exit(1)
 		}
 
+		// One machine is one predastore host, so the node list entry matching
+		// this bind IP is this node's host ID. Without one the service has no
+		// [[host]] to run and refuses to start.
 		predastoreHostID = admin.FindNodeIDByIP(predastoreNodes, bindIP)
+		if predastoreHostID == 0 {
+			fmt.Fprintf(os.Stderr, "❌ Error: bind IP %s not found in the predastore node list\n", bindIP)
+			os.Exit(1)
+		}
 		fmt.Printf("✅ Created: multi-node predastore.toml (host ID: %d)\n", predastoreHostID)
 	}
 
@@ -1925,6 +1945,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 
 	// Shutdown formation server
 	fs.Shutdown(context.Background())
+	closeFormationPort()
 	if err := os.Remove(tokenPath); err != nil && !errors.Is(err, os.ErrNotExist) {
 		slog.Warn("Failed to remove join token file", "path", tokenPath, "error", err)
 	}
@@ -2392,8 +2413,10 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 
 	northstarCreds, northstarConfigPath := northstarFromFormation(creds, dirs)
 
-	// Generate multi-node predastore config
-	var predastoreHostID int
+	// Generate multi-node predastore config. A single-node install is host 1,
+	// the only [[host]] its predastore.toml declares; the multi-node path below
+	// replaces this with the host matching this machine.
+	predastoreHostID := 1
 	hasPredastoreConfig := len(predastoreNodes) >= 2
 
 	if hasPredastoreConfig {
@@ -2576,13 +2599,24 @@ func initIAMServiceFromConfig() (*handlers_iam.IAMServiceImpl, *config.ClusterCo
 	return svc, cfg, nc, func() { nc.Close() }, nil
 }
 
-// adminAccessPolicyDocument is the AdministratorAccess policy document that
-// grants full access to all actions and resources.
-const adminAccessPolicyDocument = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"*","Resource":"*"}]}`
+// openAccountNameIndex opens the account-name reservation index over an
+// existing NATS connection.
+func openAccountNameIndex(ctx context.Context, nc *nats.Conn) (*handlers_iam.AccountNameIndex, error) {
+	js, err := jetstream.New(nc)
+	if err != nil {
+		return nil, fmt.Errorf("get JetStream context: %w", err)
+	}
+	return handlers_iam.NewAccountNameIndex(ctx, js)
+}
 
 func runAccountCreate(cmd *cobra.Command, args []string) {
 	name, _ := cmd.Flags().GetString("name")
 
+	if remote, _ := cmd.Flags().GetBool("remote"); remote {
+		runAccountCreateRemote(cmd, name)
+		return
+	}
+
 	svc, cfg, nc, cleanup, err := initIAMServiceFromConfig()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
@@ -2590,60 +2624,73 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 	}
 	defer cleanup()
 
-	// 1. Create the account
-	account, err := svc.CreateAccount(name)
+	ctx := context.Background()
+
+	// Reject a duplicate before allocating an account ID. CreateAccount does not
+	// check names, so without this two runs of the same command silently produce
+	// two accounts that differ only by ID.
+	existing, err := handlers_iam.FindAccountByName(svc, name)
 	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error checking existing accounts: %v\n", err)
+		os.Exit(1)
+	}
+	if existing != nil {
+		fmt.Fprintf(os.Stderr, "Error: account %q already exists (account ID %s)\n",
+			existing.AccountName, existing.AccountID)
+		os.Exit(1)
+	}
+
+	// Reserve the name so a concurrent create — including one through
+	// /admin/CreateAccount — cannot take it between the check above and the
+	// create below. The CLI has no client token, so it owns the reservation by
+	// a per-invocation identifier.
+	names, err := openAccountNameIndex(ctx, nc)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	reservationOwner := "spx-cli-" + uuid.NewString()
+	switch err := names.Reserve(ctx, name, reservationOwner); {
+	case errors.Is(err, handlers_iam.ErrNameTaken):
+		fmt.Fprintf(os.Stderr, "Error: account %q already exists\n", name)
+		os.Exit(1)
+	case errors.Is(err, handlers_iam.ErrNameInFlight):
+		fmt.Fprintf(os.Stderr, "Error: another account creation for %q is in progress\n", name)
+		os.Exit(1)
+	case err != nil:
+		fmt.Fprintf(os.Stderr, "Error reserving account name: %v\n", err)
+		os.Exit(1)
+	}
+
+	account, err := svc.CreateAccount(handlers_iam.NormalizeAccountName(name))
+	if err != nil {
+		if relErr := names.Release(ctx, name, reservationOwner); relErr != nil {
+			fmt.Fprintf(os.Stderr, "Warning: could not release name reservation: %v\n", relErr)
+		}
 		fmt.Fprintf(os.Stderr, "Error creating account: %v\n", err)
 		os.Exit(1)
 	}
 	accountID := account.AccountID
 
+	if err := names.Commit(ctx, name, accountID, reservationOwner); err != nil {
+		fmt.Fprintf(os.Stderr, "Warning: could not index account name: %v\n", err)
+	}
+
 	// Create default VPC for the new account (belt-and-suspenders: daemon also
 	// does this via iam.account.created event, but daemon may not be running).
 	nodeConfig := cfg.Nodes[cfg.Node]
-	vpcSvc, vpcErr := handlers_ec2_vpc.NewVPCServiceImplWithNATS(context.Background(), &nodeConfig, nc)
+	vpcSvc, vpcErr := handlers_ec2_vpc.NewVPCServiceImplWithNATS(ctx, &nodeConfig, nc)
 	if vpcErr != nil {
 		fmt.Fprintf(os.Stderr, "Warning: could not create default VPC service: %v\n", vpcErr)
 	} else if _, vpcErr = vpcSvc.EnsureDefaultVPC(accountID); vpcErr != nil {
 		fmt.Fprintf(os.Stderr, "Warning: could not create default VPC: %v\n", vpcErr)
 	}
 
-	// 2. Create admin user
-	_, err = svc.CreateUser(accountID, &iam.CreateUserInput{
-		UserName: aws.String("admin"),
-	})
+	// Shared with /admin/CreateAccount so the CLI and the remote endpoint cannot
+	// drift in what an account is made of.
+	provisioned, err := handlers_iam.ProvisionAccount(svc, accountID, account.AccountName)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error creating admin user: %v\n", err)
-		os.Exit(1)
-	}
-
-	// 3. Create access key for admin user
-	akOut, err := svc.CreateAccessKey(accountID, &iam.CreateAccessKeyInput{
-		UserName: aws.String("admin"),
-	})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error creating access key: %v\n", err)
-		os.Exit(1)
-	}
-
-	// 4. Create AdministratorAccess policy scoped to this account
-	policyARN := fmt.Sprintf("arn:aws:iam::%s:policy/AdministratorAccess", accountID)
-	_, err = svc.CreatePolicy(accountID, &iam.CreatePolicyInput{
-		PolicyName:     aws.String("AdministratorAccess"),
-		PolicyDocument: aws.String(adminAccessPolicyDocument),
-	})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error creating admin policy: %v\n", err)
-		os.Exit(1)
-	}
-
-	// 5. Attach policy to admin user
-	_, err = svc.AttachUserPolicy(accountID, &iam.AttachUserPolicyInput{
-		UserName:  aws.String("admin"),
-		PolicyArn: aws.String(policyARN),
-	})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error attaching policy: %v\n", err)
+		fmt.Fprintf(os.Stderr, "Error provisioning account: %v\n", err)
 		os.Exit(1)
 	}
 
@@ -2665,8 +2712,8 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 	configPath := filepath.Join(homeDir, ".aws", "config")
 
 	if err := admin.UpdateAWSINIFile(credPath, profileName, map[string]string{
-		"aws_access_key_id":     *akOut.AccessKey.AccessKeyId,
-		"aws_secret_access_key": *akOut.AccessKey.SecretAccessKey,
+		"aws_access_key_id":     provisioned.AccessKeyID,
+		"aws_secret_access_key": provisioned.SecretAccessKey,
 	}); err != nil {
 		fmt.Fprintf(os.Stderr, "Warning: could not update AWS credentials: %v\n", err)
 	}
@@ -2686,16 +2733,24 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 	// Print credentials
 	fmt.Println("\nAccount created successfully!")
 	fmt.Printf("  Account ID:        %s\n", accountID)
-	fmt.Printf("  Account Name:      %s\n", name)
-	fmt.Printf("  Admin User:        admin\n")
-	fmt.Printf("  Access Key ID:     %s\n", *akOut.AccessKey.AccessKeyId)
-	fmt.Printf("  Secret Access Key: %s\n", *akOut.AccessKey.SecretAccessKey)
+	fmt.Printf("  Account Name:      %s\n", provisioned.AccountName)
+	fmt.Printf("  Admin User:        %s\n", provisioned.AdminUser)
+	fmt.Printf("  Access Key ID:     %s\n", provisioned.AccessKeyID)
+	fmt.Printf("  Secret Access Key: %s\n", provisioned.SecretAccessKey)
 	fmt.Printf("  AWS Profile:       %s\n", profileName)
 	fmt.Println("\nUse with:")
 	fmt.Printf("  AWS_PROFILE=%s aws ec2 describe-instances\n", profileName)
 }
 
 func runAccountList(cmd *cobra.Command, args []string) {
+	if remote, _ := cmd.Flags().GetBool("remote"); remote {
+		if err := runAccountListRemote(cmd); err != nil {
+			fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+			os.Exit(1)
+		}
+		return
+	}
+
 	svc, _, _, cleanup, err := initIAMServiceFromConfig()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
@@ -2709,20 +2764,7 @@ func runAccountList(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	if len(accounts) == 0 {
-		fmt.Println("No accounts found.")
-		return
-	}
-
-	fmt.Printf("%-14s %-20s %-10s %s\n", "ACCOUNT ID", "NAME", "STATUS", "CREATED")
-	fmt.Printf("%-14s %-20s %-10s %s\n", "----------", "----", "------", "-------")
-	for _, a := range accounts {
-		created := a.CreatedAt
-		if t, err := time.Parse(time.RFC3339, a.CreatedAt); err == nil {
-			created = t.Format("2006-01-02 15:04")
-		}
-		fmt.Printf("%-14s %-20s %-10s %s\n", a.AccountID, a.AccountName, a.Status, created)
-	}
+	printAccountTable(accountSummaries(accounts))
 }
 
 func runCertRenew(cmd *cobra.Command, _ []string) {

cmd/spinifex/cmd/admin_account_delete.go

@@ -0,0 +1,200 @@
+package cmd
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/accountteardown"
+	"github.com/mulgadc/spinifex/spinifex/admin"
+	"github.com/mulgadc/spinifex/spinifex/objectstore"
+	"github.com/spf13/cobra"
+)
+
+// accountDeleteTimeout is the outer bound on the whole teardown. It must stay
+// well above the sum of the per-stage drain budgets, which are what actually
+// decide when to give up on a resource; cutting in before them would abandon a
+// large account midway with no stuck report to show for it.
+const accountDeleteTimeout = 2 * time.Hour
+
+var accountDeleteCmd = &cobra.Command{
+	Use:   "delete <account-id>",
+	Short: "Tear down an account's infrastructure and remove the account",
+	Long: `Remove an account and everything it owns.
+
+The account is marked TERMINATING first, which blocks its credentials, and then
+its resources are deleted in dependency order: compute, attachments, storage,
+network, platform, identity, account. Nothing is removed until the account can
+no longer create more.
+
+The account name must be given to confirm, so a mistyped account id fails
+rather than emptying a live tenant. The system account (000000000000) and the
+super admin (000000000001) can never be deleted.
+
+  spx admin account delete 000000000042 --dry-run
+  spx admin account delete 000000000042 --name tenant@example.com
+
+--force is for resources the ordinary API refuses to delete: a volume attached
+to an instance that will not terminate leaves both stranded and the account can
+never be emptied. It clears the attachment in the control plane without the
+guest's cooperation, so use it only once the guest is being destroyed.`,
+	Args: cobra.ExactArgs(1),
+	RunE: runAccountDelete,
+
+	SilenceUsage: true,
+}
+
+func init() {
+	accountCmd.AddCommand(accountDeleteCmd)
+
+	accountDeleteCmd.Flags().Bool("dry-run", false, "Report what would be deleted and change nothing")
+	accountDeleteCmd.Flags().Bool("force", false, "Escalate past state guards for resources that will not delete")
+	accountDeleteCmd.Flags().Bool("yes", false, "Skip the interactive confirmation (requires --name)")
+	accountDeleteCmd.Flags().String("name", "", "Account name, which must match the stored record")
+	accountDeleteCmd.Flags().Bool("remote", false, "Delete over POST /admin/DeleteAccount instead of connecting to NATS")
+	accountDeleteCmd.Flags().String("endpoint", "", "Gateway endpoint for --remote (default: this node's AWS gateway)")
+	accountDeleteCmd.Flags().String("region", "", "SigV4 region for --remote (default: this node's region)")
+	accountDeleteCmd.Flags().String("ca-bundle", "", "CA certificate for --remote (default: this node's CA)")
+	accountDeleteCmd.Flags().String("client-token", "", "Idempotency token for --remote (default: generated; reuse it to follow the same teardown)")
+}
+
+func runAccountDelete(cmd *cobra.Command, args []string) error {
+	accountID := args[0]
+	if remote, _ := cmd.Flags().GetBool("remote"); remote {
+		return runAccountDeleteRemote(cmd, accountID)
+	}
+
+	dryRun, _ := cmd.Flags().GetBool("dry-run")
+	force, _ := cmd.Flags().GetBool("force")
+	assumeYes, _ := cmd.Flags().GetBool("yes")
+	name, _ := cmd.Flags().GetString("name")
+
+	svc, cfg, nc, cleanup, err := initIAMServiceFromConfig()
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), accountDeleteTimeout)
+	defer cancel()
+
+	node := cfg.Nodes[cfg.Node]
+	buckets := objectstore.NewS3ObjectStoreFromConfig(
+		admin.DialTarget(node.Predastore.Host),
+		node.Predastore.Region,
+		node.Predastore.AccessKey,
+		node.Predastore.SecretKey,
+	)
+
+	engine, err := accountteardown.NewClusterEngine(ctx, nc, len(cfg.Nodes), svc, buckets)
+	if err != nil {
+		return err
+	}
+
+	if dryRun {
+		result, err := engine.Inventory(ctx, accountID)
+		if err != nil {
+			return err
+		}
+		printTeardownPlan(result)
+		fmt.Printf("\nDry run: nothing was deleted. %d resources would be removed.\n", result.DeletedCount())
+		return nil
+	}
+
+	// Always show the inventory before destroying it. An operator who has not
+	// seen what is about to go has not really confirmed anything.
+	plan, err := engine.Inventory(ctx, accountID)
+	if err != nil {
+		return err
+	}
+	printTeardownPlan(plan)
+
+	if name == "" {
+		if assumeYes {
+			return errors.New("--yes requires --name so the account is still confirmed")
+		}
+		name, err = promptAccountName(accountID)
+		if err != nil {
+			return err
+		}
+	}
+
+	result, err := engine.Teardown(ctx, accountteardown.Request{
+		AccountID: accountID, AccountName: name, Force: force,
+	})
+	if result != nil {
+		printTeardownResult(result)
+	}
+	if errors.Is(err, accountteardown.ErrResourcesStuck) {
+		fmt.Fprintf(os.Stderr, "\nThe account is still TERMINATING so its leftovers keep an owner.\n")
+		if !force {
+			fmt.Fprintf(os.Stderr, "Re-run with --force to clear resources the ordinary API refuses to delete.\n")
+		}
+		return err
+	}
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("\nAccount %s deleted. %d resources removed.\n", accountID, result.DeletedCount())
+	return nil
+}
+
+// promptAccountName asks for the confirmation name on a terminal.
+func promptAccountName(accountID string) (string, error) {
+	fmt.Printf("\nThis permanently deletes account %s and everything above.\n", accountID)
+	fmt.Print("Type the account name to confirm: ")
+
+	reader := bufio.NewReader(os.Stdin)
+	entered, err := reader.ReadString('\n')
+	if err != nil {
+		return "", fmt.Errorf("read confirmation: %w", err)
+	}
+	entered = strings.TrimSpace(entered)
+	if entered == "" {
+		return "", errors.New("cancelled: no account name given")
+	}
+	return entered, nil
+}
+
+func printTeardownPlan(result *accountteardown.Result) {
+	fmt.Printf("Account %s (%s)\n\n", result.AccountID, result.AccountName)
+	if result.DeletedCount() == 0 {
+		fmt.Println("The account holds no resources.")
+		return
+	}
+	for _, stage := range result.Stages {
+		if len(stage.Deleted) == 0 {
+			continue
+		}
+		fmt.Printf("  %s\n", stage.Stage)
+		for _, resource := range stage.Deleted {
+			fmt.Printf("    %s\n", resource)
+		}
+	}
+}
+
+func printTeardownResult(result *accountteardown.Result) {
+	fmt.Println()
+	for _, stage := range result.Stages {
+		printTeardownStage(stage)
+	}
+}
+
+// printTeardownStage reports one finished stage. A stage that removed nothing
+// and left nothing behind is not printed: an account with no ECS clusters
+// should not read as seven lines of activity.
+func printTeardownStage(stage accountteardown.StageResult) {
+	if len(stage.Deleted) == 0 && len(stage.Stuck) == 0 {
+		return
+	}
+	fmt.Printf("  %-12s %d deleted, %d stuck (%s)\n",
+		stage.Stage, len(stage.Deleted), len(stage.Stuck), stage.Elapsed)
+	for _, stuck := range stage.Stuck {
+		fmt.Printf("    STUCK %s: %s\n", stuck.Resource, stuck.Reason)
+	}
+}

cmd/spinifex/cmd/admin_account_delete_output_test.go

@@ -0,0 +1,162 @@
+package cmd_test
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+	"github.com/mulgadc/spinifex/spinifex/accountteardown"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// captureOutput redirects os.Stdout for the duration of fn and returns what was
+// written. The teardown printers are the operator's only view of what is about
+// to be destroyed, so what they print is the thing under test.
+func captureOutput(t *testing.T, fn func()) string {
+	t.Helper()
+	r, w, err := os.Pipe()
+	require.NoError(t, err)
+	orig := os.Stdout
+	os.Stdout = w //nolint:reassign // test-local stdout capture, restored below
+
+	fn()
+
+	require.NoError(t, w.Close())
+	os.Stdout = orig //nolint:reassign // restoring the real os.Stdout captured above
+
+	var buf bytes.Buffer
+	_, err = io.Copy(&buf, r)
+	require.NoError(t, err)
+	return buf.String()
+}
+
+// feedStdin swaps os.Stdin for a pipe carrying content, mirroring the stdout
+// capture above.
+func feedStdin(t *testing.T, content string, fn func()) {
+	t.Helper()
+	r, w, err := os.Pipe()
+	require.NoError(t, err)
+	orig := os.Stdin
+	os.Stdin = r                          //nolint:reassign // test-local stdin swap, restored below
+	t.Cleanup(func() { os.Stdin = orig }) //nolint:reassign // restoring the real os.Stdin captured above
+
+	go func() {
+		_, _ = io.WriteString(w, content)
+		_ = w.Close()
+	}()
+
+	fn()
+}
+
+func teardownPlan() *accountteardown.Result {
+	return &accountteardown.Result{
+		AccountID:   "000000000042",
+		AccountName: "tenant@example.com",
+		DryRun:      true,
+		Stages: []accountteardown.StageResult{
+			{
+				Stage:   accountteardown.StageCompute,
+				Deleted: []accountteardown.Resource{{Kind: "instance", ID: "i-1", Detail: "running"}},
+				Elapsed: "0s",
+			},
+			{Stage: accountteardown.StageAttachments, Elapsed: "0s"},
+			{
+				Stage:   accountteardown.StageStorage,
+				Deleted: []accountteardown.Resource{{Kind: "volume", ID: "vol-1"}},
+				Elapsed: "0s",
+			},
+		},
+	}
+}
+
+// The plan is what the operator confirms against, so every resource has to
+// appear under its stage — and an empty stage must not pad the list.
+func TestPrintTeardownPlanListsEveryResource(t *testing.T) {
+	output := captureOutput(t, func() { cmd.PrintTeardownPlan(teardownPlan()) })
+
+	assert.Contains(t, output, "000000000042")
+	assert.Contains(t, output, "tenant@example.com")
+	assert.Contains(t, output, "i-1")
+	assert.Contains(t, output, "vol-1")
+	assert.Contains(t, output, "compute")
+	assert.Contains(t, output, "storage")
+	assert.NotContains(t, output, "attachments", "an empty stage is noise, not information")
+}
+
+// An account with nothing in it must say so. A bare header reads as truncated
+// output, which is exactly when an operator stops and asks.
+func TestPrintTeardownPlanSaysWhenTheAccountIsEmpty(t *testing.T) {
+	output := captureOutput(t, func() {
+		cmd.PrintTeardownPlan(&accountteardown.Result{
+			AccountID:   "000000000042",
+			AccountName: "tenant@example.com",
+			Stages:      []accountteardown.StageResult{{Stage: accountteardown.StageCompute, Elapsed: "0s"}},
+		})
+	})
+
+	assert.Contains(t, output, "holds no resources")
+}
+
+// A stuck resource is the most valuable thing a teardown reports: each one is a
+// delete path that does not work. It must be named, with the reason.
+func TestPrintTeardownResultNamesStuckResources(t *testing.T) {
+	result := &accountteardown.Result{
+		AccountID: "000000000042",
+		Stages: []accountteardown.StageResult{
+			{
+				Stage:   accountteardown.StageCompute,
+				Deleted: []accountteardown.Resource{{Kind: "instance", ID: "i-1"}},
+				Elapsed: "5s",
+			},
+			{Stage: accountteardown.StageAttachments, Elapsed: "0s"},
+			{
+				Stage: accountteardown.StageStorage,
+				Stuck: []accountteardown.Stuck{{
+					Resource: accountteardown.Resource{Kind: "volume", ID: "vol-1"},
+					Reason:   "still attached to i-1",
+				}},
+				Elapsed: "5m0s",
+			},
+		},
+	}
+
+	output := captureOutput(t, func() { cmd.PrintTeardownResult(result) })
+
+	assert.Contains(t, output, "1 deleted, 0 stuck")
+	assert.Contains(t, output, "STUCK")
+	assert.Contains(t, output, "vol-1")
+	assert.Contains(t, output, "still attached to i-1")
+	assert.NotContains(t, output, "attachments")
+}
+
+// The typed name is the confirmation. Surrounding whitespace is a paste
+// artefact rather than a mismatch, but an empty line is a cancellation.
+func TestPromptAccountNameReadsTheConfirmation(t *testing.T) {
+	var name string
+	var err error
+
+	captureOutput(t, func() {
+		feedStdin(t, "  tenant@example.com  \n", func() {
+			name, err = cmd.PromptAccountName("000000000042")
+		})
+	})
+
+	require.NoError(t, err)
+	assert.Equal(t, "tenant@example.com", name)
+}
+
+func TestPromptAccountNameTreatsAnEmptyReplyAsCancelled(t *testing.T) {
+	var err error
+
+	captureOutput(t, func() {
+		feedStdin(t, "\n", func() {
+			_, err = cmd.PromptAccountName("000000000042")
+		})
+	})
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "cancelled")
+}

cmd/spinifex/cmd/admin_account_delete_remote.go

@@ -0,0 +1,142 @@
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/accountteardown"
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	"github.com/spf13/cobra"
+)
+
+// accountDeletePollInterval is how often --remote asks the gateway how far the
+// teardown has got. The work is minutes long, so a tighter poll would only add
+// load without telling the operator anything new.
+var accountDeletePollInterval = 10 * time.Second
+
+// runAccountDeleteRemote drives POST /admin/DeleteAccount and then follows the
+// job to completion, so the remote path reports the same outcome the local one
+// prints rather than only that the teardown started.
+func runAccountDeleteRemote(cmd *cobra.Command, accountID string) error {
+	dryRun, _ := cmd.Flags().GetBool("dry-run")
+	force, _ := cmd.Flags().GetBool("force")
+	assumeYes, _ := cmd.Flags().GetBool("yes")
+	name, _ := cmd.Flags().GetString("name")
+	clientToken, _ := cmd.Flags().GetString("client-token")
+
+	target, err := resolveAdminTarget(cmd)
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), accountDeleteTimeout)
+	defer cancel()
+
+	// The inventory is fetched first even for a real deletion: an operator who
+	// has not seen what is about to go has not really confirmed anything.
+	plan, err := deleteAccountRemote(ctx, target, gateway.DeleteAccountRequest{
+		AccountID: accountID, DryRun: true,
+	})
+	if err != nil {
+		return err
+	}
+	if plan.Inventory != nil {
+		printTeardownPlan(plan.Inventory)
+	}
+	if dryRun {
+		fmt.Printf("\nDry run: nothing was deleted. %d resources would be removed.\n", plan.Inventory.DeletedCount())
+		return nil
+	}
+
+	if name == "" {
+		if assumeYes {
+			return errors.New("--yes requires --name so the account is still confirmed")
+		}
+		if name, err = promptAccountName(accountID); err != nil {
+			return err
+		}
+	}
+	if clientToken == "" {
+		clientToken = newClientToken()
+	}
+
+	started, err := deleteAccountRemote(ctx, target, gateway.DeleteAccountRequest{
+		AccountID: accountID, AccountName: name, ClientToken: clientToken, Force: force,
+	})
+	if err != nil {
+		var adminErr *adminError
+		if errors.As(err, &adminErr) && retryableAdminErrors[adminErr.Code] {
+			fmt.Fprintf(os.Stderr, "Retry with --client-token %s to follow the same teardown.\n", clientToken)
+		}
+		return err
+	}
+	fmt.Printf("\nTeardown started: %s\n", started.DeletionID)
+
+	return followAccountDeletion(ctx, target, accountID)
+}
+
+// followAccountDeletion polls until the job reaches a terminal state, printing
+// each stage as it lands.
+func followAccountDeletion(ctx context.Context, target adminTarget, accountID string) error {
+	reported := 0
+	for {
+		job, err := describeAccountDeletionRemote(ctx, target, accountID)
+		if err != nil {
+			return err
+		}
+
+		for ; reported < len(job.Stages); reported++ {
+			printTeardownStage(job.Stages[reported])
+		}
+
+		switch job.State {
+		case gateway.DeletionStateCompleted:
+			fmt.Printf("\nAccount %s deleted.\n", accountID)
+			return nil
+		case gateway.DeletionStateFailed:
+			fmt.Fprintf(os.Stderr, "\nThe account is still TERMINATING so its leftovers keep an owner.\n")
+			return fmt.Errorf("teardown failed: %s", job.Error)
+		}
+
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("stopped following teardown of %s: %w", accountID, ctx.Err())
+		case <-time.After(accountDeletePollInterval):
+		}
+	}
+}
+
+func deleteAccountRemote(
+	ctx context.Context,
+	target adminTarget,
+	req gateway.DeleteAccountRequest,
+) (*gateway.DeleteAccountResponse, error) {
+	var out gateway.DeleteAccountResponse
+	if err := callAdmin(ctx, target, "DeleteAccount", req, &out); err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func describeAccountDeletionRemote(ctx context.Context, target adminTarget, accountID string) (*accountDeletionStatus, error) {
+	var out accountDeletionStatus
+	err := callAdmin(ctx, target, "DescribeAccountDeletion",
+		gateway.DescribeAccountDeletionRequest{AccountID: accountID}, &out)
+	if err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+// accountDeletionStatus is the subset of the stored job this command reports.
+// The gateway's own record type is unexported, and the CLI needs no more than
+// the state, the stages and why it stopped.
+type accountDeletionStatus struct {
+	DeletionID string                        `json:"deletionId"`
+	State      string                        `json:"state"`
+	Stages     []accountteardown.StageResult `json:"stages"`
+	Error      string                        `json:"error"`
+}

cmd/spinifex/cmd/admin_account_delete_remote_test.go

@@ -0,0 +1,199 @@
+package cmd_test
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// adminTestCredentials puts a static key in the environment so the SigV4 signer
+// resolves without reaching for IMDS.
+func adminTestCredentials(t *testing.T) {
+	t.Helper()
+	t.Setenv("AWS_ACCESS_KEY_ID", "AKIAEXAMPLE")
+	t.Setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+}
+
+// newScriptedAdminServer answers each successive request with the next body,
+// repeating the last one, which is how a job that changes state between polls
+// is reproduced.
+func newScriptedAdminServer(t *testing.T, bodies ...string) (*httptest.Server, *atomic.Int32) {
+	t.Helper()
+	var calls atomic.Int32
+
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		index := int(calls.Add(1)) - 1
+		if index >= len(bodies) {
+			index = len(bodies) - 1
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(bodies[index]))
+	}))
+	t.Cleanup(srv.Close)
+	return srv, &calls
+}
+
+// The delete request must reach /admin/DeleteAccount signed for the spinifex
+// service: a key signed for another service is treated as probing and denied.
+func TestDeleteAccountRemoteSignsAndPosts(t *testing.T) {
+	adminTestCredentials(t)
+
+	var got http.Request
+	srv := newAdminTestServer(t, http.StatusOK,
+		`{"deletionId":"d-1","accountId":"000000000042","state":"RUNNING"}`, &got)
+
+	out, err := cmd.DeleteAccountRemote(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)),
+		gateway.DeleteAccountRequest{
+			AccountID:   "000000000042",
+			AccountName: "tenant@example.com",
+			ClientToken: strings.Repeat("a", 32),
+		})
+	require.NoError(t, err)
+
+	assert.Equal(t, "d-1", out.DeletionID)
+	assert.Equal(t, gateway.DeletionStateRunning, out.State)
+
+	assert.Equal(t, http.MethodPost, got.Method)
+	assert.Equal(t, "/admin/DeleteAccount", got.URL.Path)
+	assert.Contains(t, got.Header.Get("Authorization"), "/us-west-1/spinifex/aws4_request")
+}
+
+// A dry run answers inline with the inventory, which is what the operator
+// confirms against before the real call.
+func TestDeleteAccountRemoteDryRunCarriesTheInventory(t *testing.T) {
+	adminTestCredentials(t)
+
+	var got http.Request
+	srv := newAdminTestServer(t, http.StatusOK,
+		`{"accountId":"000000000042","state":"DRY_RUN","dryRun":true,"inventory":{"accountId":"000000000042",`+
+			`"dryRun":true,"stages":[{"stage":1,"deleted":[{"kind":"instance","id":"i-1"}],"elapsed":"0s"}]}}`, &got)
+
+	out, err := cmd.DeleteAccountRemote(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)),
+		gateway.DeleteAccountRequest{AccountID: "000000000042", DryRun: true})
+	require.NoError(t, err)
+
+	assert.True(t, out.DryRun)
+	require.NotNil(t, out.Inventory)
+	assert.Equal(t, 1, out.Inventory.DeletedCount())
+}
+
+// A refusal must surface as the gateway's code, not as a decode failure, so the
+// operator can tell a protected account from an unreachable cluster.
+func TestDeleteAccountRemoteSurfacesGatewayError(t *testing.T) {
+	adminTestCredentials(t)
+
+	var got http.Request
+	srv := newAdminTestServer(t, http.StatusForbidden,
+		`{"error":{"code":"AccessDenied","message":"denied"},"requestId":"req-9"}`, &got)
+
+	_, err := cmd.DeleteAccountRemote(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)),
+		gateway.DeleteAccountRequest{
+			AccountID:   "000000000001",
+			AccountName: "spinifex",
+			ClientToken: strings.Repeat("a", 32),
+		})
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "AccessDenied")
+	assert.Contains(t, err.Error(), "req-9")
+	assert.False(t, cmd.RetryableAdminError(err), "a protected account is not fixed by retrying")
+}
+
+func TestDescribeAccountDeletionRemoteReportsProgress(t *testing.T) {
+	adminTestCredentials(t)
+
+	var got http.Request
+	srv := newAdminTestServer(t, http.StatusOK,
+		`{"deletionId":"d-1","state":"RUNNING","stages":[{"stage":1,"elapsed":"5s"}]}`, &got)
+
+	out, err := cmd.DescribeAccountDeletionRemote(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042")
+	require.NoError(t, err)
+
+	assert.Equal(t, "/admin/DescribeAccountDeletion", got.URL.Path)
+	assert.Equal(t, gateway.DeletionStateRunning, out.State)
+	assert.Len(t, out.Stages, 1)
+}
+
+// The follow loop is what makes --remote report the same outcome the local
+// path prints. It must keep polling while the job runs and stop on the first
+// terminal state.
+func TestFollowAccountDeletionPollsUntilComplete(t *testing.T) {
+	adminTestCredentials(t)
+	defer cmd.SetAccountDeletePollInterval(time.Millisecond)()
+
+	srv, calls := newScriptedAdminServer(t,
+		`{"deletionId":"d-1","state":"RUNNING","stages":[{"stage":1,"elapsed":"5s"}]}`,
+		`{"deletionId":"d-1","state":"RUNNING","stages":[{"stage":1,"elapsed":"5s"},{"stage":2,"elapsed":"1s"}]}`,
+		`{"deletionId":"d-1","state":"COMPLETED","stages":[{"stage":1,"elapsed":"5s"},{"stage":2,"elapsed":"1s"}]}`,
+	)
+
+	err := cmd.FollowAccountDeletion(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042")
+
+	require.NoError(t, err)
+	assert.Equal(t, int32(3), calls.Load(), "the loop must keep polling while the job runs")
+}
+
+// A failed teardown must exit non-zero with the stored reason: the account is
+// left TERMINATING, and an operator who reads "done" would not go looking.
+func TestFollowAccountDeletionReportsFailure(t *testing.T) {
+	adminTestCredentials(t)
+	defer cmd.SetAccountDeletePollInterval(time.Millisecond)()
+
+	srv, _ := newScriptedAdminServer(t,
+		`{"deletionId":"d-1","state":"FAILED","stages":[],"error":"2 resources left"}`)
+
+	err := cmd.FollowAccountDeletion(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042")
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "2 resources left")
+}
+
+// Giving up on following is not the same as the teardown stopping, so the
+// message has to name the account the caller must go back to.
+func TestFollowAccountDeletionStopsWhenTheContextEnds(t *testing.T) {
+	adminTestCredentials(t)
+	defer cmd.SetAccountDeletePollInterval(50 * time.Millisecond)()
+
+	srv, _ := newScriptedAdminServer(t, `{"deletionId":"d-1","state":"RUNNING","stages":[]}`)
+
+	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+	defer cancel()
+
+	err := cmd.FollowAccountDeletion(ctx,
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042")
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "000000000042")
+}
+
+// An unreadable job is not a finished one. Reporting it as complete would hide
+// a teardown still holding an account's resources.
+func TestFollowAccountDeletionFailsOnAnUnreadableJob(t *testing.T) {
+	adminTestCredentials(t)
+	defer cmd.SetAccountDeletePollInterval(time.Millisecond)()
+
+	var got http.Request
+	srv := newAdminTestServer(t, http.StatusNotFound,
+		`{"error":{"code":"NoSuchEntity","message":"no job"},"requestId":"req-2"}`, &got)
+
+	err := cmd.FollowAccountDeletion(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042")
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "NoSuchEntity")
+}

cmd/spinifex/cmd/admin_account_list_remote.go

@@ -0,0 +1,67 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	"github.com/spf13/cobra"
+)
+
+// runAccountListRemote reads the listing over POST /admin/ListAccounts, which
+// is how an off-cluster operator or the load-test harness sees the tenants it
+// created without SSH access to a node.
+func runAccountListRemote(cmd *cobra.Command) error {
+	target, err := resolveAdminTarget(cmd)
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), adminRequestTimeout)
+	defer cancel()
+
+	var out gateway.ListAccountsResponse
+	if err := callAdmin(ctx, target, "ListAccounts", struct{}{}, &out); err != nil {
+		return err
+	}
+
+	printAccountTable(out.Accounts)
+	return nil
+}
+
+// accountSummaries adapts the local listing to the shape the remote one
+// returns, so both paths print through one function and cannot drift.
+func accountSummaries(accounts []*handlers_iam.Account) []gateway.AccountSummary {
+	summaries := make([]gateway.AccountSummary, 0, len(accounts))
+	for _, account := range accounts {
+		if account == nil {
+			continue
+		}
+		summaries = append(summaries, gateway.AccountSummary{
+			AccountID:   account.AccountID,
+			AccountName: account.AccountName,
+			Status:      account.Status,
+			CreatedAt:   account.CreatedAt,
+		})
+	}
+	return summaries
+}
+
+func printAccountTable(accounts []gateway.AccountSummary) {
+	if len(accounts) == 0 {
+		fmt.Println("No accounts found.")
+		return
+	}
+
+	fmt.Printf("%-14s %-30s %-12s %s\n", "ACCOUNT ID", "NAME", "STATUS", "CREATED")
+	fmt.Printf("%-14s %-30s %-12s %s\n", "----------", "----", "------", "-------")
+	for _, account := range accounts {
+		created := account.CreatedAt
+		if parsed, err := time.Parse(time.RFC3339, account.CreatedAt); err == nil {
+			created = parsed.Format("2006-01-02 15:04")
+		}
+		fmt.Printf("%-14s %-30s %-12s %s\n", account.AccountID, account.AccountName, account.Status, created)
+	}
+}

cmd/spinifex/cmd/admin_cabake_test.go

@@ -1,67 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"errors"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestCABakeCmd_Args(t *testing.T) {
-	cmd := caBakeCmd(context.Background(), "/img/disk.raw", "/data/config/ca.pem")
-	args := cmd.Args
-
-	assert.Equal(t, "virt-customize", filepath.Base(args[0]))
-	assert.Contains(t, args, "-a")
-	assert.Contains(t, args, "/img/disk.raw")
-	assert.Contains(t, args, "--upload")
-	assert.Contains(t, args, "/data/config/ca.pem:/tmp/spinifex-ca.pem")
-	assert.Contains(t, args, "--run-command")
-
-	// --no-selinux-relabel stops virt-customize forcing a first-boot autorelabel,
-	// whose relabel+reboot corrupts XFS roots on RHEL/Rocky.
-	assert.Contains(t, args, "--no-selinux-relabel")
-
-	// The run-command covers Debian/Ubuntu (update-ca-certificates), RHEL/Rocky
-	// (update-ca-trust), and the stock-Alpine fallback that appends to the static
-	// ca-certificates bundle when no updater is present. The RHEL branch relabels
-	// only the touched trust paths so the cert is labelled without a full relabel.
-	joined := strings.Join(args, " ")
-	assert.Contains(t, joined, "update-ca-certificates")
-	assert.Contains(t, joined, "update-ca-trust")
-	assert.Contains(t, joined, "restorecon")
-	assert.Contains(t, joined, "/etc/ssl/certs/ca-certificates.crt")
-}
-
-func TestBakeCACertIntoImage_SkipsWhenCAMissing(t *testing.T) {
-	called := false
-	orig := caBakeRunner
-	caBakeRunner = func(string, string) ([]byte, error) { called = true; return nil, nil }
-	defer func() { caBakeRunner = orig }()
-
-	// A missing CA must not invoke virt-customize and must not panic.
-	bakeCACertIntoImage("/img/disk.raw", filepath.Join(t.TempDir(), "absent-ca.pem"))
-	assert.False(t, called, "runner must not be called when the CA is absent")
-}
-
-func TestBakeCACertIntoImage_ContinuesOnRunnerError(t *testing.T) {
-	caPath := filepath.Join(t.TempDir(), "ca.pem")
-	require.NoError(t, os.WriteFile(caPath, []byte("ca"), 0600))
-
-	called := false
-	orig := caBakeRunner
-	caBakeRunner = func(string, string) ([]byte, error) {
-		called = true
-		return []byte("libguestfs: cannot inspect image"), errors.New("exit status 1")
-	}
-	defer func() { caBakeRunner = orig }()
-
-	// An image libguestfs cannot inspect must skip-and-continue, never panic/fail.
-	bakeCACertIntoImage("/img/disk.raw", caPath)
-	assert.True(t, called, "runner must be invoked when the CA is present")
-}

cmd/spinifex/cmd/admin_firewall.go

@@ -0,0 +1,54 @@
+package cmd
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+	"os/exec"
+	"strconv"
+)
+
+// firewallApplyHelper is the fixed-verb root helper setup.sh installs. Absent on
+// a developer machine and on any node installed with --firewall=off, where there
+// is no policy to punch a hole in.
+var firewallApplyHelper = "/usr/local/lib/spinifex/spinifex-firewall-apply"
+
+// openFormationPort opens the formation port to any source and returns the
+// function that closes it again.
+//
+// The port is peer-scoped in the steady-state policy, where it carries the
+// daemon cluster manager. A node dialling in to join is not a peer yet, so
+// without this the join handshake is dropped rather than refused and the joiner
+// retries in silence until it times out. The window is bounded by formation
+// itself, and the handshake behind it is TLS 1.3 with a bearer token.
+//
+// Best effort: a node with no policy installed needs nothing done, and a
+// failure to open must not stop an operator forming a cluster.
+func openFormationPort(port int) func() {
+	if _, err := os.Stat(firewallApplyHelper); err != nil {
+		return func() {}
+	}
+
+	if err := runFirewallHelper("open-port", port); err != nil {
+		fmt.Printf("⚠️  Could not open port %d in the host firewall: %v\n", port, err)
+		fmt.Printf("   Joining nodes may be unable to reach the formation server.\n")
+		return func() {}
+	}
+
+	return func() {
+		if err := runFirewallHelper("close-port", port); err != nil {
+			// Loud, not fatal: formation has already succeeded by here, and the
+			// port stays open until the next reconcile or setup.sh run.
+			slog.Error("Failed to close the formation port in the host firewall",
+				"port", port, "helper", firewallApplyHelper, "err", err)
+		}
+	}
+}
+
+func runFirewallHelper(verb string, port int) error {
+	cmd := exec.Command(firewallApplyHelper, verb, strconv.Itoa(port))
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("%s: %w: %s", verb, err, out)
+	}
+	return nil
+}

cmd/spinifex/cmd/admin_firewall_test.go

@@ -0,0 +1,70 @@
+package cmd_test
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// stubFirewallHelper points the firewall helper at a script that logs its
+// arguments, and returns the path of that log.
+func stubFirewallHelper(t *testing.T, body string) string {
+	t.Helper()
+	dir := t.TempDir()
+	log := filepath.Join(dir, "calls")
+	helper := filepath.Join(dir, "spinifex-firewall-apply")
+	script := "#!/bin/sh\necho \"$*\" >> " + log + "\n" + body
+	require.NoError(t, os.WriteFile(helper, []byte(script), 0o755))
+	t.Cleanup(cmd.SetFirewallApplyHelper(helper))
+	return log
+}
+
+func helperCalls(t *testing.T, log string) []string {
+	t.Helper()
+	data, err := os.ReadFile(log)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	require.NoError(t, err)
+	return strings.Fields(strings.TrimSpace(strings.ReplaceAll(string(data), "\n", " ")))
+}
+
+func TestOpenFormationPort(t *testing.T) {
+	log := stubFirewallHelper(t, "exit 0\n")
+
+	closePort := cmd.OpenFormationPort(4432)
+	assert.Equal(t, []string{"open-port", "4432"}, helperCalls(t, log))
+
+	closePort()
+	assert.Equal(t, []string{"open-port", "4432", "close-port", "4432"}, helperCalls(t, log),
+		"the window must be closed again once formation ends")
+}
+
+// A machine with no policy installed has nothing to open, and formation must not
+// be held up by looking for a helper that was never installed.
+func TestOpenFormationPort_NoHelperInstalled(t *testing.T) {
+	t.Cleanup(cmd.SetFirewallApplyHelper(filepath.Join(t.TempDir(), "absent")))
+
+	closePort := cmd.OpenFormationPort(4432)
+	require.NotNil(t, closePort)
+	closePort()
+}
+
+// Opening is best effort: an operator forming a cluster must not be blocked by a
+// firewall helper that fails. The close must then not run either, since there is
+// nothing open to close.
+func TestOpenFormationPort_FailureDoesNotBlockFormation(t *testing.T) {
+	log := stubFirewallHelper(t, "exit 1\n")
+
+	closePort := cmd.OpenFormationPort(4432)
+	require.NotNil(t, closePort)
+	closePort()
+
+	assert.Equal(t, []string{"open-port", "4432"}, helperCalls(t, log),
+		"a failed open must not be followed by a close")
+}

cmd/spinifex/cmd/admin_jsprobe.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"time"
@@ -17,6 +18,12 @@ import (
 // other consumer reads.
 const canaryKey = "_watchdog.canary"
 
+// errProbeInconclusive marks a failure that happened before the probe ever
+// reached JetStream: config load, connect, or bucket open. The watchdog must
+// treat this as "learned nothing", not as JetStream refusing a write — during
+// recovery the bucket may simply not exist yet.
+var errProbeInconclusive = errors.New("probe did not reach JetStream")
+
 var nodeJSProbeCmd = &cobra.Command{
 	Use:   "js-probe",
 	Short: "Verify JetStream still accepts writes on this node",
@@ -42,6 +49,9 @@ func runNodeJSProbe(cmd *cobra.Command, _ []string) {
 
 	if err := probeJetStreamWrite(ctx); err != nil {
 		fmt.Fprintf(os.Stderr, "js-probe: %v\n", err)
+		if errors.Is(err, errProbeInconclusive) {
+			os.Exit(2)
+		}
 		os.Exit(1)
 	}
 	fmt.Println("js-probe: ok")
@@ -52,21 +62,26 @@ func runNodeJSProbe(cmd *cobra.Command, _ []string) {
 // than a bucket of its own: a dedicated bucket is a separate stream that can be
 // placed on a different node, so it could stay writable while the bucket the
 // control plane actually depends on is wedged.
+//
+// Every stage before canaryRoundTrip is wrapped in errProbeInconclusive: the
+// probe has observed nothing about JetStream's write path yet, so a caller
+// must not read config, connect, or bucket-open failures as a JetStream
+// failure.
 func probeJetStreamWrite(ctx context.Context) error {
 	_, nc, err := loadConfigAndConnect()
 	if err != nil {
-		return fmt.Errorf("connect: %w", err)
+		return fmt.Errorf("%w: connect: %w", errProbeInconclusive, err)
 	}
 	defer nc.Close()
 
 	js, err := jetstream.New(nc)
 	if err != nil {
-		return fmt.Errorf("jetstream: %w", err)
+		return fmt.Errorf("%w: jetstream: %w", errProbeInconclusive, err)
 	}
 
 	kv, err := js.KeyValue(ctx, daemon.ClusterStateBucket)
 	if err != nil {
-		return fmt.Errorf("open bucket %s: %w", daemon.ClusterStateBucket, err)
+		return fmt.Errorf("%w: open bucket %s: %w", errProbeInconclusive, daemon.ClusterStateBucket, err)
 	}
 	return canaryRoundTrip(ctx, kv)
 }

cmd/spinifex/cmd/admin_jsprobe_test.go

@@ -2,12 +2,17 @@ package cmd
 
 import (
 	"context"
+	"fmt"
+	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
 	"github.com/nats-io/nats.go/jetstream"
+	"github.com/spf13/viper"
 	"github.com/stretchr/testify/require"
 
+	"github.com/mulgadc/spinifex/spinifex/daemon"
 	"github.com/mulgadc/spinifex/spinifex/testutil"
 )
 
@@ -73,3 +78,82 @@ func TestCanaryRoundTrip_FailsWhenBucketIsGone(t *testing.T) {
 func TestCanaryKeyIsNamespaced(t *testing.T) {
 	require.True(t, len(canaryKey) > 0 && canaryKey[0] == '_')
 }
+
+// nodeTomlWithNATSHost writes a minimal single-node spinifex.toml pointing at
+// the given NATS host, the shape probeJetStreamWrite's loadConfigAndConnect
+// call needs to resolve node1's NATS endpoint.
+func nodeTomlWithNATSHost(t *testing.T, natsHost string) string {
+	t.Helper()
+	return writeSpinifexToml(t, fmt.Sprintf(`
+version = "1.0"
+epoch = 1
+node = "node1"
+
+[nodes.node1]
+node = "node1"
+region = "us-east-1"
+az = "us-east-1a"
+
+[nodes.node1.nats]
+host = %q
+`, natsHost))
+}
+
+// A config file viper cannot parse must never read as a JetStream failure:
+// the probe has not connected to anything yet.
+func TestProbeJetStreamWrite_ConfigLoadFailureIsInconclusive(t *testing.T) {
+	resetGlobalViper(t)
+
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	require.NoError(t, os.WriteFile(path, []byte("not [[[ valid toml"), 0o600))
+	viper.Set("config", path)
+
+	err := probeJetStreamWrite(context.Background())
+	require.Error(t, err)
+	require.ErrorIs(t, err, errProbeInconclusive, "config-load failure must be inconclusive")
+}
+
+// A NATS endpoint that refuses the connection must also read as inconclusive,
+// not as JetStream refusing a write — this is the class of failure that
+// turned a broken spx binary into a cluster-wide NATS restart.
+func TestProbeJetStreamWrite_ConnectFailureIsInconclusive(t *testing.T) {
+	resetGlobalViper(t)
+
+	viper.Set("config", nodeTomlWithNATSHost(t, "nats://127.0.0.1:1"))
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	err := probeJetStreamWrite(ctx)
+	require.Error(t, err)
+	require.ErrorIs(t, err, errProbeInconclusive, "connect failure must be inconclusive")
+}
+
+// A reachable NATS with no cluster-state bucket yet (recovery, or a bucket
+// that has not been created) must not read as a JetStream write failure.
+func TestProbeJetStreamWrite_BucketOpenFailureIsInconclusive(t *testing.T) {
+	resetGlobalViper(t)
+
+	ns, _, _ := testutil.StartTestJetStream(t)
+	viper.Set("config", nodeTomlWithNATSHost(t, ns.ClientURL()))
+
+	err := probeJetStreamWrite(context.Background())
+	require.Error(t, err)
+	require.ErrorIs(t, err, errProbeInconclusive, "missing bucket must be inconclusive")
+}
+
+// The positive end-to-end path: once the bucket exists and NATS accepts
+// writes, the probe succeeds through every stage rather than just at the
+// canaryRoundTrip unit level covered above.
+func TestProbeJetStreamWrite_SucceedsAgainstRealClusterStateBucket(t *testing.T) {
+	resetGlobalViper(t)
+
+	ns, _, js := testutil.StartTestJetStream(t)
+	_, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: daemon.ClusterStateBucket})
+	require.NoError(t, err)
+
+	viper.Set("config", nodeTomlWithNATSHost(t, ns.ClientURL()))
+
+	require.NoError(t, probeJetStreamWrite(context.Background()))
+}

cmd/spinifex/cmd/admin_ochre.go

@@ -1,10 +1,11 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
-	"log/slog"
 	"os"
 	"os/exec"
 	"path"
@@ -15,16 +16,16 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/mulgadc/predastore/pkg/masterkey"
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/ebsprovider"
 	gateway_bedrock "github.com/mulgadc/spinifex/spinifex/gateway/bedrock"
+	"github.com/mulgadc/spinifex/spinifex/gateway/bedrock/hfhub"
 	handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/utils"
-	"github.com/mulgadc/viperblock/viperblock"
-	vbs3 "github.com/mulgadc/viperblock/viperblock/backends/s3"
-	"github.com/mulgadc/viperblock/viperblock/v_utils"
+	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/pterm/pterm"
 	"github.com/spf13/cobra"
@@ -77,12 +78,51 @@ explicit act.`,
 	Run: runOchreWeightsRemove,
 }
 
+var ochreWeightsPullCmd = &cobra.Command{
+	Use:   "pull",
+	Short: "Fetch a self-host model's weights from Hugging Face into predastore",
+	Long: `pull resolves --revision to an immutable commit SHA, lists the repo's file
+tree, downloads only *.safetensors shards plus the fixed config/tokenizer
+set, and streams each into predastore under --s3-uri (or a keyed default
+when omitted). It never touches the offline 'stage' path or the weights KV
+store itself; it prints the resulting s3:// URI for 'weights stage --s3-uri'.
+
+Refuses before any object lands if --model-id is not a self-host catalog
+entry, the repo has no safetensors, or the hub returns 401/403 (a gated repo
+with no usable token). A failure partway through removes every object
+already written so 'stage' never sees a half-model.`,
+	Run: runOchreWeightsPull,
+}
+
+// ochreCredentialsCmd groups vendor credential management for pull's
+// optional stored-token fallback (D2); it holds no Run of its own.
+var ochreCredentialsCmd = &cobra.Command{
+	Use:   "credentials",
+	Short: "Manage stored provider credentials for self-host model pulls",
+}
+
+var ochreCredentialsSetCmd = &cobra.Command{
+	Use:   "set",
+	Short: "Store a vendor API token/licence credential",
+	Long: `set stores --vendor's token, encrypted at rest under the cluster's IAM master
+key, in the bedrock-credentials KV bucket. --account defaults to the
+platform account, since self-host models are platform-level, not per-tenant.
+
+The token itself is never a command-line argument: --token must be '-', and
+the secret is read from stdin, so it never appears in shell history or a
+process listing.`,
+	Run: runOchreCredentialsSet,
+}
+
 func init() {
 	adminCmd.AddCommand(ochreCmd)
 	ochreCmd.AddCommand(ochreWeightsCmd)
+	ochreCmd.AddCommand(ochreCredentialsCmd)
 	ochreWeightsCmd.AddCommand(ochreWeightsStageCmd)
 	ochreWeightsCmd.AddCommand(ochreWeightsListCmd)
 	ochreWeightsCmd.AddCommand(ochreWeightsRemoveCmd)
+	ochreWeightsCmd.AddCommand(ochreWeightsPullCmd)
+	ochreCredentialsCmd.AddCommand(ochreCredentialsSetCmd)
 
 	ochreWeightsStageCmd.Flags().String("model-id", "", "Catalog model ID to stage weights for (required)")
 	ochreWeightsStageCmd.Flags().String("s3-uri", "", "predastore S3 URI holding the model's Hugging Face files, e.g. s3://bucket/prefix/ (required)")
@@ -92,6 +132,20 @@ func init() {
 
 	ochreWeightsRemoveCmd.Flags().String("model-id", "", "Model ID to remove from staged weights (required)")
 	_ = ochreWeightsRemoveCmd.MarkFlagRequired("model-id")
+
+	ochreWeightsPullCmd.Flags().String("model-id", "", "Self-host catalog model ID this pull is for (required)")
+	ochreWeightsPullCmd.Flags().String("hf-repo", "", "Hugging Face repo, e.g. meta-llama/Llama-3.2-1B-Instruct (required)")
+	ochreWeightsPullCmd.Flags().String("revision", "main", "Hugging Face branch, tag or commit SHA to resolve and pin")
+	ochreWeightsPullCmd.Flags().String("s3-uri", "", "predastore destination prefix, e.g. s3://bucket/prefix/ (default: s3://ochre-weights/<repo>/<sha>/)")
+	ochreWeightsPullCmd.Flags().String("hf-token", "", "Hugging Face token for a gated repo (falls back to HF_TOKEN, then a stored platform credential)")
+	_ = ochreWeightsPullCmd.MarkFlagRequired("model-id")
+	_ = ochreWeightsPullCmd.MarkFlagRequired("hf-repo")
+
+	ochreCredentialsSetCmd.Flags().String("vendor", "", "Vendor to store a credential for, e.g. huggingface (required)")
+	ochreCredentialsSetCmd.Flags().String("account", "", "Account ID to store the credential under (default: platform account)")
+	ochreCredentialsSetCmd.Flags().String("token", "", "Must be '-': reads the secret from stdin (required)")
+	_ = ochreCredentialsSetCmd.MarkFlagRequired("vendor")
+	_ = ochreCredentialsSetCmd.MarkFlagRequired("token")
 }
 
 // requiredWeightsFiles are the fixed-name Hugging Face artefacts stage
@@ -109,6 +163,233 @@ var requiredWeightsFiles = []string{
 // tokenizer.model under original/, not at the prefix root. Either is enough.
 var tokenizerFileNames = []string{"tokenizer.json", "tokenizer.model"}
 
+// vendorHuggingFace is the CredentialStore vendor key 'ochre credentials
+// set' and pull's stored-credential fallback share for the Hugging Face
+// licence token (D2).
+const vendorHuggingFace = "huggingface"
+
+// ochrePullManifestFile is the fixed name 'weights pull' writes into the
+// destination prefix and 'weights stage' reads back (D3), recording which
+// upstream commit a staged model's weights came from.
+const ochrePullManifestFile = "ochre-pull.json"
+
+// ochrePullManifest is the JSON body of ochrePullManifestFile.
+type ochrePullManifest struct {
+	HFRepo      string    `json:"hf_repo"`
+	RevisionSHA string    `json:"revision_sha"`
+	PulledAt    time.Time `json:"pulled_at"`
+}
+
+// allowedPullFileNames are the fixed-name Hugging Face artefacts pull fetches
+// alongside safetensors shards -- config and tokenizer files needed to serve,
+// never weights themselves. Matched by basename so a nested path (e.g.
+// Llama's original/tokenizer.model) still qualifies.
+var allowedPullFileNames = map[string]bool{
+	"config.json":                  true,
+	"tokenizer_config.json":        true,
+	"tokenizer.json":               true,
+	"tokenizer.model":              true,
+	"generation_config.json":       true,
+	"special_tokens_map.json":      true,
+	"model.safetensors.index.json": true,
+}
+
+// selectPullFiles filters a Hugging Face tree listing down to the
+// safetensors-only set pull downloads (D4): every *.safetensors file plus
+// the fixed config/tokenizer set. Anything else -- notably .bin/.pt pickle
+// checkpoints, which can execute code on load -- is dropped silently rather
+// than aborting the whole pull.
+func selectPullFiles(entries []hfhub.TreeEntry) []hfhub.TreeEntry {
+	var selected []hfhub.TreeEntry
+	for _, e := range entries {
+		if e.Type != "file" {
+			continue
+		}
+		if strings.HasSuffix(e.Path, ".safetensors") || allowedPullFileNames[path.Base(e.Path)] {
+			selected = append(selected, e)
+		}
+	}
+	return selected
+}
+
+// anySafetensors reports whether entries contains at least one *.safetensors
+// file. selectPullFiles's config/tokenizer set alone is never enough --
+// D4 aborts a pull with no actual weights to serve.
+func anySafetensors(entries []hfhub.TreeEntry) bool {
+	for _, e := range entries {
+		if strings.HasSuffix(e.Path, ".safetensors") {
+			return true
+		}
+	}
+	return false
+}
+
+// defaultPullPrefix derives D7's fallback destination when --s3-uri is
+// omitted: keyed by the exact repo and resolved commit SHA, so re-pulling
+// the same commit lands at the same, self-deduping URI.
+func defaultPullPrefix(hfRepo, sha string) string {
+	return fmt.Sprintf("s3://ochre-weights/%s/%s/", strings.Trim(hfRepo, "/"), sha)
+}
+
+// putPullManifest writes ochrePullManifestFile into bucket/prefix, the last
+// step of a successful pull (D3).
+func putPullManifest(ctx context.Context, store objectstore.ObjectStore, bucket, prefix string, manifest ochrePullManifest) error {
+	data, err := json.Marshal(manifest)
+	if err != nil {
+		return fmt.Errorf("encode pull manifest: %w", err)
+	}
+	key := prefix + ochrePullManifestFile
+	if _, err := store.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String(key), Body: bytes.NewReader(data)}); err != nil {
+		return fmt.Errorf("put pull manifest s3://%s/%s: %w", bucket, key, err)
+	}
+	return nil
+}
+
+// readPullManifest reads and decodes bucket/prefix's ochre-pull.json, if
+// present. A missing manifest is not an error: stage's offline path (an
+// operator-supplied prefix with no pull manifest) must keep working
+// unchanged, so ok is simply false.
+func readPullManifest(ctx context.Context, store objectstore.ObjectStore, bucket, prefix string) (ochrePullManifest, bool, error) {
+	key := prefix + ochrePullManifestFile
+	out, err := store.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
+	if err != nil {
+		if objectstore.IsNoSuchKeyError(err) {
+			return ochrePullManifest{}, false, nil
+		}
+		return ochrePullManifest{}, false, fmt.Errorf("get pull manifest s3://%s/%s: %w", bucket, key, err)
+	}
+	defer out.Body.Close()
+
+	var manifest ochrePullManifest
+	if err := json.NewDecoder(out.Body).Decode(&manifest); err != nil {
+		return ochrePullManifest{}, false, fmt.Errorf("decode pull manifest s3://%s/%s: %w", bucket, key, err)
+	}
+	return manifest, true, nil
+}
+
+// cleanupPulledObjects best-effort deletes every key already written by a
+// pull that failed partway through (D5): a half-uploaded prefix must never
+// be left for 'stage' to mistake for a complete model. A delete failure is
+// logged, not returned -- the original pull error is what the operator needs.
+func cleanupPulledObjects(ctx context.Context, store objectstore.ObjectStore, bucket string, keys []string) {
+	for _, key := range keys {
+		if _, err := store.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)}); err != nil {
+			fmt.Fprintf(os.Stderr, "warning: could not clean up s3://%s/%s after a failed pull: %v\n", bucket, key, err)
+		}
+	}
+}
+
+// pullOneFile downloads hfPath from the hub to a local temp file and PUTs it
+// to bucket/key. A temp file is required, not a direct HTTP-to-S3 pipe:
+// predastore's PutObject signs the request over an io.ReadSeeker, which an
+// HTTP response body is not. tmpDir is cleaned up by the caller.
+func pullOneFile(ctx context.Context, hf *hfhub.Client, store objectstore.ObjectStore, hfRepo, sha, hfPath, bucket, key, tmpDir string) error {
+	body, _, err := hf.DownloadFile(ctx, hfRepo, sha, hfPath)
+	if err != nil {
+		return err
+	}
+	defer body.Close()
+
+	tmpFile, err := os.CreateTemp(tmpDir, "part-*")
+	if err != nil {
+		return fmt.Errorf("create temp file: %w", err)
+	}
+	defer os.Remove(tmpFile.Name())
+	defer tmpFile.Close()
+
+	if _, err := io.Copy(tmpFile, body); err != nil {
+		return fmt.Errorf("download to temp file: %w", err)
+	}
+	if _, err := tmpFile.Seek(0, io.SeekStart); err != nil {
+		return fmt.Errorf("seek temp file: %w", err)
+	}
+
+	if _, err := store.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String(key), Body: tmpFile}); err != nil {
+		return fmt.Errorf("put s3://%s/%s: %w", bucket, key, err)
+	}
+	return nil
+}
+
+// pullFilesToPrefix downloads and uploads every selected file, one at a time
+// so at most one file is ever held on local disk (D6). Keys are flattened to
+// each file's basename to match the flat layout 'stage' validates -- e.g.
+// Llama's original/tokenizer.model lands at <prefix>tokenizer.model. It
+// returns the keys written so far even on error, so the caller can clean up
+// a partial prefix (D5).
+func pullFilesToPrefix(ctx context.Context, hf *hfhub.Client, store objectstore.ObjectStore, hfRepo, sha, bucket, prefix string, files []hfhub.TreeEntry) ([]string, error) {
+	tmpDir, err := os.MkdirTemp("", "spinifex-ochre-pull-*")
+	if err != nil {
+		return nil, fmt.Errorf("create temp dir: %w", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	var uploaded []string
+	for _, f := range files {
+		key := prefix + path.Base(f.Path)
+		if err := pullOneFile(ctx, hf, store, hfRepo, sha, f.Path, bucket, key, tmpDir); err != nil {
+			return uploaded, fmt.Errorf("pull %s: %w", f.Path, err)
+		}
+		uploaded = append(uploaded, key)
+	}
+	return uploaded, nil
+}
+
+// runPullWeights holds 'ochre weights pull' decision and side-effect logic:
+// catalog validation, ref resolution to an immutable commit SHA (D3), tree
+// listing and safetensors-only filtering (D4), streaming each selected file
+// into predastore (D6), and finally the ochre-pull.json manifest. It never
+// touches the weights KV store; 'stage' consumes the printed s3:// URI
+// separately. A failure removes every object already written (D5).
+func runPullWeights(ctx context.Context, hf *hfhub.Client, store objectstore.ObjectStore, modelID, hfRepo, revision, s3URIFlag string) (string, error) {
+	if _, found, selfHost := gateway_bedrock.LookupServingSpec(modelID); !found || !selfHost {
+		if !found {
+			return "", fmt.Errorf("unknown model ID %q: not present in the Ochre catalog", modelID)
+		}
+		return "", fmt.Errorf("%q is a provider-served model, not self-host; weights pull does not apply", modelID)
+	}
+
+	fmt.Printf("Resolving %s@%s ...\n", hfRepo, revision)
+	sha, err := hf.ResolveRevision(ctx, hfRepo, revision)
+	if err != nil {
+		return "", err
+	}
+
+	fmt.Printf("Listing files at %s@%s ...\n", hfRepo, sha)
+	tree, err := hf.ListTree(ctx, hfRepo, sha)
+	if err != nil {
+		return "", err
+	}
+
+	selected := selectPullFiles(tree)
+	if !anySafetensors(selected) {
+		return "", fmt.Errorf("%s@%s has no *.safetensors files; refusing to pull pickle-format (.bin) weights", hfRepo, sha)
+	}
+
+	s3URI := s3URIFlag
+	if s3URI == "" {
+		s3URI = defaultPullPrefix(hfRepo, sha)
+	}
+	bucket, prefix, err := parseWeightsS3URI(s3URI)
+	if err != nil {
+		return "", err
+	}
+
+	fmt.Printf("Pulling %d file(s) into s3://%s/%s ...\n", len(selected), bucket, prefix)
+	uploaded, err := pullFilesToPrefix(ctx, hf, store, hfRepo, sha, bucket, prefix, selected)
+	if err != nil {
+		cleanupPulledObjects(ctx, store, bucket, uploaded)
+		return "", err
+	}
+
+	manifest := ochrePullManifest{HFRepo: hfRepo, RevisionSHA: sha, PulledAt: time.Now().UTC()}
+	if err := putPullManifest(ctx, store, bucket, prefix, manifest); err != nil {
+		cleanupPulledObjects(ctx, store, bucket, uploaded)
+		return "", err
+	}
+
+	return fmt.Sprintf("s3://%s/%s", bucket, prefix), nil
+}
+
 // parseWeightsS3URI splits an s3://bucket/prefix URI into its bucket and
 // prefix. The prefix is normalised to end with '/' so downstream listing and
 // validation scope to the directory rather than any key merely sharing the
@@ -307,63 +588,6 @@ func buildWeightsImage(srcDir, imagePath string, contentBytes int64) error {
 	return mkfsExt4Runner(srcDir, imagePath)
 }
 
-// snapshotImportedWeightsVolume snapshots a viperblock volume that
-// v_utils.ImportDiskImage just wrote and closed -- never attached, never
-// nbdkit-served, so there is no live checkpoint to load. This mirrors the
-// offline snapshot sequence handlers/ec2/image/service_impl.go's
-// snapshotStoppedVolume uses for a stopped instance's root volume: reopen
-// read-only, load the numbered checkpoint Close() wrote, then CreateSnapshot.
-//
-// az names the zone recorded on the snapshot for DescribeSnapshots; it is not
-// used to resolve the clone.
-func snapshotImportedWeightsVolume(s3Config *vbs3.S3Config, volumeID string, volumeSize uint64, walDir, az string, mkey *masterkey.Key) (string, error) {
-	vbConfig := viperblock.VB{
-		VolumeName:        volumeID,
-		VolumeSize:        volumeSize,
-		BaseDir:           walDir,
-		Cache:             viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}},
-		MasterKey:         mkey,
-		EncryptionEnabled: mkey != nil,
-		Logger:            slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})),
-	}
-
-	vb, err := viperblock.New(&vbConfig, "s3", *s3Config)
-	if err != nil {
-		return "", fmt.Errorf("new viperblock: %w", err)
-	}
-	defer func() {
-		vb.StopChunkUploader()
-		vb.StopWALSyncer()
-	}()
-
-	if err := vb.Backend.Init(); err != nil {
-		return "", fmt.Errorf("backend init: %w", err)
-	}
-	if err := vb.LoadState(); err != nil {
-		return "", fmt.Errorf("load state: %w", err)
-	}
-	if err := vb.LoadBlockState(); err != nil {
-		return "", fmt.Errorf("load block state: %w", err)
-	}
-	defer func() {
-		if err := vb.RemoveLocalFiles(); err != nil {
-			slog.Warn("snapshotImportedWeightsVolume: failed to remove local files", "volumeId", volumeID, "err", err)
-		}
-	}()
-
-	snapshotID := admin.SnapPrefix(volumeID)
-	if _, err := vb.CreateSnapshot(snapshotID); err != nil {
-		return "", fmt.Errorf("create snapshot: %w", err)
-	}
-
-	store := objectstore.NewS3ObjectStoreFromConfig(s3Config.Host, s3Config.Region, s3Config.AccessKey, s3Config.SecretKey)
-	if err := registerWeightsSnapshot(store, s3Config.Bucket, snapshotID, volumeID, volumeSize, az, mkey != nil); err != nil {
-		return "", err
-	}
-
-	return snapshotID, nil
-}
-
 // registerWeightsSnapshot writes the EC2 control plane's half of the snapshot
 // prefix. CreateSnapshot above writes only viperblock's half -- the block
 // checkpoint and config.json -- but CreateVolume resolves a SnapshotId through
@@ -447,6 +671,16 @@ func runStageWeights(ctx context.Context, store objectstore.ObjectStore, weights
 		return "", err
 	}
 
+	// A pull manifest is optional: absent means an operator-supplied prefix
+	// from the offline path, which stage has always supported and must keep
+	// supporting unchanged.
+	var sourceRevision string
+	if manifest, ok, err := readPullManifest(ctx, store, bucket, prefix); err != nil {
+		return "", err
+	} else if ok {
+		sourceRevision = manifest.RevisionSHA
+	}
+
 	tmpDir, err := os.MkdirTemp(tmpDirFlag, "spinifex-weights-tmp-*")
 	if err != nil {
 		return "", fmt.Errorf("could not create temp dir: %w", err)
@@ -469,7 +703,7 @@ func runStageWeights(ctx context.Context, store objectstore.ObjectStore, weights
 		return "", err
 	}
 
-	if err := weightsStore.PutWeights(ctx, modelID, sourceURI, snapshotID); err != nil {
+	if err := weightsStore.PutWeightsWithRevision(ctx, modelID, sourceURI, snapshotID, sourceRevision); err != nil {
 		return "", err
 	}
 
@@ -485,12 +719,11 @@ func runStageWeights(ctx context.Context, store objectstore.ObjectStore, weights
 }
 
 // materializeWeightsVolume is the real weightsMaterializer: it packages
-// downloadDir into a filesystem image, imports it into a new viperblock
-// volume backed by node's predastore, and snapshots it. This is the
-// expensive, environment-dependent half of stage -- it shells out to
-// mkfs.ext4 and talks to a live S3-shaped backend -- so it is exercised live
-// (see the plan's Testing section) rather than in unit tests.
-func materializeWeightsVolume(node config.Config, downloadDir string, contentBytes int64, mkey *masterkey.Key) (string, error) {
+// downloadDir into a filesystem image, imports it into a new volume through
+// the EBS provider, and snapshots it. This is the expensive,
+// environment-dependent half of stage -- it shells out to mkfs.ext4 and needs
+// a live provider -- so it is exercised live rather than in unit tests.
+func materializeWeightsVolume(ctx context.Context, provider ebsprovider.EBSProvider, nodeID string, node config.Config, downloadDir string, contentBytes int64) (string, error) {
 	tmpDir := filepath.Dir(downloadDir)
 	volumeId := utils.GenerateResourceID("vol")
 	imagePath := filepath.Join(tmpDir, volumeId+".img")
@@ -505,74 +738,32 @@ func materializeWeightsVolume(node config.Config, downloadDir string, contentByt
 		return "", fmt.Errorf("could not stat image: %w", err)
 	}
 
-	// Round the volume up to a whole GiB rather than using the raw image size.
-	// viperblock rejects a tail write whose block overruns VolumeSize, so an
-	// unaligned size fails at the last block. This also keeps the byte size and
-	// the manifest's SizeGiB describing the same volume.
+	// Round the volume up to a whole GiB rather than using the raw image size,
+	// so the byte size and the recorded SizeGiB describe the same volume.
 	sizeGiB := amiVolumeSizeGiB(imageStat.Size())
 	volumeBytes := sizeGiB * bytesPerGiB
 
-	s3Config := vbs3.S3Config{
-		VolumeName: volumeId,
-		VolumeSize: volumeBytes,
-		Bucket:     node.Predastore.Bucket,
-		Region:     node.Predastore.Region,
-		AccessKey:  node.Predastore.AccessKey,
-		SecretKey:  node.Predastore.SecretKey,
-		Host:       node.Predastore.Host,
-	}
-
-	// AMIMetadata is deliberately left zero-valued: a weights volume is not
-	// bootable and must never be registered as a launchable AMI. Leaving it
-	// unset also means ImportDiskImage does not attempt its own automatic
-	// snapshot -- that happens explicitly below, after the volume is closed.
-	manifest := viperblock.VolumeConfig{}
-	manifest.VolumeMetadata.VolumeID = volumeId
-	manifest.VolumeMetadata.VolumeName = volumeId
-	manifest.VolumeMetadata.TenantID = "system"
-	manifest.VolumeMetadata.SizeGiB = sizeGiB
-	manifest.VolumeMetadata.State = "available"
-	manifest.VolumeMetadata.AvailabilityZone = node.Predastore.Region
-	manifest.VolumeMetadata.CreatedAt = time.Now()
-	manifest.VolumeMetadata.VolumeType = "gp3"
-	manifest.VolumeMetadata.IOPS = 1000
-
-	vbConfig := viperblock.VB{
-		VolumeName:        volumeId,
-		VolumeSize:        volumeBytes,
-		BaseDir:           tmpDir,
-		Cache:             viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}},
-		VolumeConfig:      manifest,
-		MasterKey:         mkey,
-		EncryptionEnabled: mkey != nil,
-		Logger:            slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})),
-	}
-
-	var flushBar *pterm.ProgressbarPrinter
-	var flushUpdate func(current uint64)
-	progress := func(current, total uint64) {
-		if flushBar == nil {
-			flushBar, flushUpdate = utils.NewByteProgressBar("Flushing weights to storage", total)
-		}
-		flushUpdate(current)
-	}
-
 	// Name the volume in the failure path: a part-written import leaves blocks
 	// in predastore, and without the ID an operator cannot find them to reclaim.
-	if err := v_utils.ImportDiskImage(&s3Config, &vbConfig, imagePath, progress); err != nil {
-		if flushBar != nil {
-			_, _ = flushBar.Stop()
-		}
+	fmt.Println("Writing weights to storage ...")
+	if err := admin.ImportImage(ctx, provider, admin.ImportOpts{
+		VolumeID:         volumeId,
+		NodeID:           nodeID,
+		SizeBytes:        utils.SafeUint64ToInt64(volumeBytes),
+		AvailabilityZone: node.AZ,
+		SourcePath:       imagePath,
+		Snapshot:         true,
+		Progress:         os.Stdout,
+	}); err != nil {
 		return "", fmt.Errorf("could not import weights volume %s: %w", volumeId, err)
 	}
-	if flushBar != nil {
-		_, _ = flushBar.Stop()
-	}
 
-	fmt.Println("Snapshotting weights volume ...")
-	snapshotID, err := snapshotImportedWeightsVolume(&s3Config, volumeId, volumeBytes, tmpDir, node.AZ, mkey)
-	if err != nil {
-		return "", fmt.Errorf("could not snapshot weights volume: %w", err)
+	// The provider wrote viperblock's half of the snapshot prefix; this writes
+	// the EC2 control plane's, without which a launch cannot resolve it.
+	snapshotID := admin.SnapPrefix(volumeId)
+	store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
+	if err := registerWeightsSnapshot(store, node.Predastore.Bucket, snapshotID, volumeId, volumeBytes, node.AZ, node.Viperblock.EncryptionKeyFile != ""); err != nil {
+		return "", err
 	}
 	return snapshotID, nil
 }
@@ -583,8 +774,8 @@ func materializeWeightsVolume(node config.Config, downloadDir string, contentByt
 // stage time -- and treats a missing object as gone. Any other read failure
 // is a real error, not a signal to guess the snapshot's existence.
 func newWeightsSnapshotChecker(store objectstore.ObjectStore, bucket string) weightsSnapshotChecker {
-	return func(_ context.Context, snapshotID string) (bool, error) {
-		if _, err := handlers_ec2_snapshot.ReadSnapshotConfig(store, bucket, snapshotID); err != nil {
+	return func(ctx context.Context, snapshotID string) (bool, error) {
+		if _, err := handlers_ec2_snapshot.ReadSnapshotConfig(ctx, store, bucket, snapshotID); err != nil {
 			if objectstore.IsNoSuchKeyError(err) {
 				return false, nil
 			}
@@ -629,15 +820,9 @@ func runOchreWeightsStage(cmd *cobra.Command, _ []string) {
 	weightsStore := gateway_bedrock.NewWeightsStore(js, len(appConfig.Nodes))
 	store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
 
-	mkey, err := utils.LoadViperblockMasterKey(node.Viperblock.EncryptionKeyFile)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not load viperblock encryption key: %v\n", err)
-		ochreExit(1)
-		return
-	}
-
-	materialize := func(_ context.Context, downloadDir string, contentBytes int64) (string, error) {
-		return materializeWeightsVolume(node, downloadDir, contentBytes, mkey)
+	provider := ebsprovider.NewNATSProvider(nc, imageImportTimeout)
+	materialize := func(ctx context.Context, downloadDir string, contentBytes int64) (string, error) {
+		return materializeWeightsVolume(ctx, provider, appConfig.Node, node, downloadDir, contentBytes)
 	}
 	checkSnapshotLive := newWeightsSnapshotChecker(store, node.Predastore.Bucket)
 
@@ -673,7 +858,7 @@ func listWeightsOutput(ctx context.Context, weightsStore *gateway_bedrock.Weight
 		return "No models staged.", nil
 	}
 
-	tableData := pterm.TableData{{"MODEL ID", "SOURCE URI", "SNAPSHOT ID", "STATUS"}}
+	tableData := pterm.TableData{{"MODEL ID", "SOURCE URI", "REVISION", "SNAPSHOT ID", "STATUS"}}
 	for _, e := range entries {
 		live, err := checkSnapshotLive(ctx, e.SnapshotID)
 		if err != nil {
@@ -683,7 +868,11 @@ func listWeightsOutput(ctx context.Context, weightsStore *gateway_bedrock.Weight
 		if !live {
 			status = weightsStatusDangling
 		}
-		tableData = append(tableData, []string{e.ModelID, e.SourceURI, e.SnapshotID, status})
+		revision := e.SourceRevision
+		if revision == "" {
+			revision = "-"
+		}
+		tableData = append(tableData, []string{e.ModelID, e.SourceURI, revision, e.SnapshotID, status})
 	}
 	return pterm.DefaultTable.WithHasHeader().WithLeftAlignment().WithData(tableData).Srender()
 }
@@ -766,6 +955,141 @@ func runOchreWeightsRemove(cmd *cobra.Command, _ []string) {
 	fmt.Printf("✅ Removed staged-weights entry for %s (snapshot %s and source objects untouched).\n", modelID, entry.SnapshotID)
 }
 
+// resolveHFToken applies pull's token resolution order (D2): the --hf-token
+// flag, then HF_TOKEN env (the must-have path, self-sufficient on its own),
+// then a stored platform credential for vendor "huggingface". The stored
+// credential step is a best-effort fallback: a cluster with no IAM master
+// key provisioned yet simply yields no token rather than failing the pull.
+func resolveHFToken(ctx context.Context, cmd *cobra.Command, cfg *config.ClusterConfig, nc *nats.Conn) string {
+	if token, _ := cmd.Flags().GetString("hf-token"); token != "" {
+		return token
+	}
+	if token := os.Getenv("HF_TOKEN"); token != "" {
+		return token
+	}
+
+	masterKeyPath := filepath.Join(cfg.NodeBaseDir(), "config", "master.key")
+	masterKey, err := handlers_iam.LoadMasterKey(masterKeyPath)
+	if err != nil {
+		return ""
+	}
+	js, err := jetstream.New(nc)
+	if err != nil {
+		return ""
+	}
+	credStore := gateway_bedrock.NewCredentialStore(js, masterKey, len(cfg.Nodes), nil)
+	token, ok, err := credStore.Resolve(ctx, utils.GlobalAccountID, vendorHuggingFace)
+	if err != nil || !ok {
+		return ""
+	}
+	return token
+}
+
+func runOchreWeightsPull(cmd *cobra.Command, _ []string) {
+	modelID, _ := cmd.Flags().GetString("model-id")
+	hfRepo, _ := cmd.Flags().GetString("hf-repo")
+	revision, _ := cmd.Flags().GetString("revision")
+	s3URI, _ := cmd.Flags().GetString("s3-uri")
+
+	appConfig, nc, err := loadConfigAndConnectFn()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+	defer nc.Close()
+	node := appConfig.Nodes[appConfig.Node]
+
+	ctx := context.Background()
+	token := resolveHFToken(ctx, cmd, appConfig, nc)
+
+	hf := hfhub.NewClient(token)
+	store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
+
+	finalURI, err := runPullWeights(ctx, hf, store, modelID, hfRepo, revision, s3URI)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		ochreExit(1)
+		return
+	}
+	fmt.Printf("✅ Pulled %s@%s into %s\n", hfRepo, revision, finalURI)
+	fmt.Println(finalURI)
+}
+
+// readTokenFromStdin reads a credential from r, trimming exactly one
+// trailing newline (matching 'printf "%s" "$TOKEN" | cmd --token -' or a
+// piped 'echo'). The token is never logged.
+func readTokenFromStdin(r io.Reader) (string, error) {
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return "", fmt.Errorf("read token from stdin: %w", err)
+	}
+	token := strings.TrimSuffix(strings.TrimSuffix(string(data), "\n"), "\r")
+	if token == "" {
+		return "", fmt.Errorf("no token read from stdin")
+	}
+	return token, nil
+}
+
+// ochreCredentialsStore connects to the cluster, loads the node's IAM master
+// key from disk -- the same on-disk key initIAMServiceFromConfig uses -- and
+// returns a CredentialStore ready for PutCredential/Resolve.
+func ochreCredentialsStore() (*gateway_bedrock.CredentialStore, func(), error) {
+	cfg, nc, err := loadConfigAndConnectFn()
+	if err != nil {
+		return nil, nil, fmt.Errorf("connect to cluster: %w", err)
+	}
+	masterKeyPath := filepath.Join(cfg.NodeBaseDir(), "config", "master.key")
+	masterKey, err := handlers_iam.LoadMasterKey(masterKeyPath)
+	if err != nil {
+		nc.Close()
+		return nil, nil, fmt.Errorf("load master key: %w", err)
+	}
+	js, err := jetstream.New(nc)
+	if err != nil {
+		nc.Close()
+		return nil, nil, fmt.Errorf("jetstream context: %w", err)
+	}
+	return gateway_bedrock.NewCredentialStore(js, masterKey, len(cfg.Nodes), nil), func() { nc.Close() }, nil
+}
+
+func runOchreCredentialsSet(cmd *cobra.Command, _ []string) {
+	vendor, _ := cmd.Flags().GetString("vendor")
+	accountID, _ := cmd.Flags().GetString("account")
+	tokenFlag, _ := cmd.Flags().GetString("token")
+
+	if tokenFlag != "-" {
+		fmt.Fprintln(os.Stderr, "Error: --token must be '-' (the secret is read from stdin, never a command-line argument)")
+		ochreExit(1)
+		return
+	}
+	if accountID == "" {
+		accountID = utils.GlobalAccountID
+	}
+
+	token, err := readTokenFromStdin(os.Stdin)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+
+	store, cleanup, err := ochreCredentialsStore()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+	defer cleanup()
+
+	if err := store.PutCredential(context.Background(), accountID, vendor, token); err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+	fmt.Printf("✅ Stored %s credential for account %s\n", vendor, accountID)
+}
+
 var adminOchreAccessCmd = &cobra.Command{
 	Use:   "access",
 	Short: "Manage per-account model access grants",

cmd/spinifex/cmd/admin_ochre_appliance.go

@@ -0,0 +1,87 @@
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	handlers_ochrevector "github.com/mulgadc/spinifex/spinifex/handlers/ochrevector"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/spf13/cobra"
+)
+
+// applianceTeardownTimeout bounds the teardown NATS round trip: it covers an
+// RDS DeleteDBInstance call plus a KV delete, not a VM boot, so it is far
+// shorter than the appliance's own launch timeout.
+const applianceTeardownTimeout = 5 * time.Minute
+
+var ochreApplianceCmd = &cobra.Command{
+	Use:   "appliance",
+	Short: "Manage the shared platform Postgres appliance backing every Ochre vector index",
+}
+
+var ochreApplianceTeardownCmd = &cobra.Command{
+	Use:   "teardown",
+	Short: "Destroy the shared platform Postgres appliance",
+	Long: `teardown destroys the platform appliance singleton: its backing RDS DB
+instance, this node's VPC host port, and its KV record.
+
+This is a destructive, cluster-wide operator action -- every account's vector
+index data stored in the appliance is lost with it, and it is not
+automatically rebuilt. A future daemon startup re-provisions a fresh
+appliance (a new instance, a new password) only if Ochre remains enabled;
+existing index registry records are not restored.
+
+Requires --confirm.`,
+	Run: runOchreApplianceTeardown,
+}
+
+func init() {
+	ochreCmd.AddCommand(ochreApplianceCmd)
+	ochreApplianceCmd.AddCommand(ochreApplianceTeardownCmd)
+
+	ochreApplianceTeardownCmd.Flags().Bool("confirm", false,
+		"Required: confirms you intend to destroy the shared platform appliance and every account's vector index data with it")
+}
+
+// applianceTeardownFn indirects the NATS call so runApplianceTeardownGuarded
+// is testable without a live daemon, mirroring vectorServiceFn/
+// endpointServiceFn.
+var applianceTeardownFn = func(ctx context.Context) error {
+	_, nc, err := loadConfigAndConnectFn()
+	if err != nil {
+		return err
+	}
+	defer nc.Close()
+	_, err = utils.NATSRequest[handlers_ochrevector.TeardownApplianceResponse](ctx, nc,
+		handlers_ochrevector.SubjectTeardownAppliance, &handlers_ochrevector.TeardownApplianceRequest{}, applianceTeardownTimeout, utils.GlobalAccountID)
+	return err
+}
+
+// runApplianceTeardownGuarded is the testable core of 'ochre appliance
+// teardown': it refuses without --confirm before making any NATS call, so a
+// missing flag never even opens a connection to the cluster.
+func runApplianceTeardownGuarded(ctx context.Context, confirmed bool, teardown func(context.Context) error) (string, error) {
+	if !confirmed {
+		return "", errors.New("refusing to tear down the shared platform appliance without --confirm: " +
+			"this destroys every account's vector index data and is not automatically rebuilt")
+	}
+	if err := teardown(ctx); err != nil {
+		return "", err
+	}
+	return "✅ Platform appliance torn down.", nil
+}
+
+func runOchreApplianceTeardown(cmd *cobra.Command, _ []string) {
+	confirmed, _ := cmd.Flags().GetBool("confirm")
+
+	msg, err := runApplianceTeardownGuarded(context.Background(), confirmed, applianceTeardownFn)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		ochreExit(1)
+		return
+	}
+	fmt.Println(msg)
+}

cmd/spinifex/cmd/admin_ochre_appliance_test.go

@@ -0,0 +1,104 @@
+// Exercises unexported ochre appliance CLI command internals with no
+// exported surface to drive them through.
+//
+//test:in-package
+package cmd
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestRunApplianceTeardownGuarded_RefusesWithoutConfirm proves the guard
+// refuses before ever invoking teardown -- a missing --confirm must never
+// reach a NATS call, let alone destroy the appliance.
+func TestRunApplianceTeardownGuarded_RefusesWithoutConfirm(t *testing.T) {
+	called := false
+	teardown := func(context.Context) error {
+		called = true
+		return nil
+	}
+
+	_, err := runApplianceTeardownGuarded(context.Background(), false, teardown)
+	require.Error(t, err)
+	require.ErrorContains(t, err, "--confirm")
+	require.False(t, called, "teardown must not run without --confirm")
+}
+
+// TestRunApplianceTeardownGuarded_ConfirmedInvokesTeardown proves --confirm
+// drives the teardown call through and reports success.
+func TestRunApplianceTeardownGuarded_ConfirmedInvokesTeardown(t *testing.T) {
+	called := false
+	teardown := func(context.Context) error {
+		called = true
+		return nil
+	}
+
+	msg, err := runApplianceTeardownGuarded(context.Background(), true, teardown)
+	require.NoError(t, err)
+	require.True(t, called)
+	require.Contains(t, msg, "torn down")
+}
+
+// TestRunApplianceTeardownGuarded_TeardownErrorSurfaces proves a teardown
+// failure (e.g. the daemon reporting no appliance up) surfaces to the caller
+// rather than being swallowed behind a generic success message.
+func TestRunApplianceTeardownGuarded_TeardownErrorSurfaces(t *testing.T) {
+	teardown := func(context.Context) error {
+		return errors.New("ochrevector: platform appliance is not enabled or not up on this node")
+	}
+
+	_, err := runApplianceTeardownGuarded(context.Background(), true, teardown)
+	require.Error(t, err)
+	require.ErrorContains(t, err, "not enabled or not up")
+}
+
+// withApplianceTeardown swaps applianceTeardownFn for one that never dials
+// NATS, so the Run wrapper's real flag/exit control flow is exercised
+// without a live daemon.
+func withApplianceTeardown(t *testing.T, fn func(context.Context) error) {
+	t.Helper()
+	orig := applianceTeardownFn
+	t.Cleanup(func() { applianceTeardownFn = orig })
+	applianceTeardownFn = fn
+}
+
+func TestRunOchreApplianceTeardown_MissingConfirmExits1(t *testing.T) {
+	withApplianceTeardown(t, func(context.Context) error {
+		t.Fatal("teardown must not be called without --confirm")
+		return nil
+	})
+
+	cmd := *ochreApplianceTeardownCmd
+	code := withOchreExitCapture(t, func() { runOchreApplianceTeardown(&cmd, nil) })
+	require.Equal(t, 1, code)
+}
+
+func TestRunOchreApplianceTeardown_ConfirmedPrintsSuccess(t *testing.T) {
+	withApplianceTeardown(t, func(context.Context) error { return nil })
+
+	cmd := *ochreApplianceTeardownCmd
+	require.NoError(t, cmd.Flags().Set("confirm", "true"))
+
+	var out string
+	code := withOchreExitCapture(t, func() {
+		out = captureStdout(t, func() { runOchreApplianceTeardown(&cmd, nil) })
+	})
+	require.Equal(t, -1, code)
+	require.Contains(t, out, "torn down")
+}
+
+func TestRunOchreApplianceTeardown_TeardownErrorExits1(t *testing.T) {
+	withApplianceTeardown(t, func(context.Context) error {
+		return errors.New("ochrevector: platform appliance is not enabled or not up on this node")
+	})
+
+	cmd := *ochreApplianceTeardownCmd
+	require.NoError(t, cmd.Flags().Set("confirm", "true"))
+
+	code := withOchreExitCapture(t, func() { runOchreApplianceTeardown(&cmd, nil) })
+	require.Equal(t, 1, code)
+}

cmd/spinifex/cmd/admin_ochre_pull_test.go

@@ -0,0 +1,578 @@
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/mulgadc/spinifex/spinifex/config"
+	gateway_bedrock "github.com/mulgadc/spinifex/spinifex/gateway/bedrock"
+	"github.com/mulgadc/spinifex/spinifex/gateway/bedrock/hfhub"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	"github.com/mulgadc/spinifex/spinifex/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/testutil"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/nats-io/nats.go"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// --- selectPullFiles / anySafetensors (D4) ---
+
+// TestSelectPullFiles_SafetensorsAndConfigOnly covers D4's filter: every
+// *.safetensors file, the fixed config/tokenizer set, and the shard index are
+// kept (matched by basename, so a nested path like Llama's
+// original/tokenizer.model still qualifies); a pickle .bin checkpoint and
+// directory entries are dropped.
+func TestSelectPullFiles_SafetensorsAndConfigOnly(t *testing.T) {
+	entries := []hfhub.TreeEntry{
+		{Type: "file", Path: "config.json"},
+		{Type: "file", Path: "model-00001-of-00002.safetensors"},
+		{Type: "file", Path: "model-00002-of-00002.safetensors"},
+		{Type: "file", Path: "tokenizer_config.json"},
+		{Type: "file", Path: "tokenizer.json"},
+		{Type: "file", Path: "pytorch_model.bin"},
+		{Type: "file", Path: "model.safetensors.index.json"},
+		{Type: "directory", Path: "original"},
+		{Type: "file", Path: "original/tokenizer.model"},
+	}
+
+	selected := selectPullFiles(entries)
+
+	var paths []string
+	for _, e := range selected {
+		paths = append(paths, e.Path)
+	}
+	assert.ElementsMatch(t, []string{
+		"config.json",
+		"model-00001-of-00002.safetensors",
+		"model-00002-of-00002.safetensors",
+		"tokenizer_config.json",
+		"tokenizer.json",
+		"model.safetensors.index.json",
+		"original/tokenizer.model",
+	}, paths)
+}
+
+// TestAnySafetensors_TrueWhenPresent covers the pass case: at least one
+// selected file is a safetensors shard.
+func TestAnySafetensors_TrueWhenPresent(t *testing.T) {
+	assert.True(t, anySafetensors([]hfhub.TreeEntry{{Type: "file", Path: "model.safetensors"}}))
+}
+
+// TestAnySafetensors_FalseWhenOnlyPickleAndConfig covers D4's abort trigger:
+// a repo whose only weights are pickle .bin, even with config/tokenizer
+// files also selected, has nothing pull will actually download as weights.
+func TestAnySafetensors_FalseWhenOnlyPickleAndConfig(t *testing.T) {
+	selected := selectPullFiles([]hfhub.TreeEntry{
+		{Type: "file", Path: "config.json"},
+		{Type: "file", Path: "pytorch_model.bin"},
+	})
+	assert.False(t, anySafetensors(selected))
+}
+
+// --- defaultPullPrefix (D7) ---
+
+// TestDefaultPullPrefix_KeyedByRepoAndSHA covers the fallback destination
+// when --s3-uri is omitted: immutable and self-deduping, keyed by the exact
+// repo and resolved commit SHA.
+func TestDefaultPullPrefix_KeyedByRepoAndSHA(t *testing.T) {
+	got := defaultPullPrefix("meta-llama/Llama-3.2-1B-Instruct", "abc123def456")
+	assert.Equal(t, "s3://ochre-weights/meta-llama/Llama-3.2-1B-Instruct/abc123def456/", got)
+}
+
+// --- ochre-pull.json manifest (D3) ---
+
+// TestPullManifest_RoundTrips covers put then read of the manifest 'pull'
+// writes and 'stage' reads back.
+func TestPullManifest_RoundTrips(t *testing.T) {
+	store := objectstore.NewMemoryObjectStore()
+	ctx := context.Background()
+	want := ochrePullManifest{HFRepo: "meta-llama/Llama-3.2-1B-Instruct", RevisionSHA: "abc123", PulledAt: time.Now().UTC().Truncate(time.Second)}
+
+	require.NoError(t, putPullManifest(ctx, store, "ochre-weights", "meta-llama/Llama-3.2-1B-Instruct/abc123/", want))
+
+	got, ok, err := readPullManifest(ctx, store, "ochre-weights", "meta-llama/Llama-3.2-1B-Instruct/abc123/")
+	require.NoError(t, err)
+	require.True(t, ok)
+	assert.Equal(t, want.HFRepo, got.HFRepo)
+	assert.Equal(t, want.RevisionSHA, got.RevisionSHA)
+	assert.WithinDuration(t, want.PulledAt, got.PulledAt, time.Second)
+}
+
+// TestReadPullManifest_MissingIsNotError covers stage's offline path: a
+// prefix with no manifest must report ok=false, not an error.
+func TestReadPullManifest_MissingIsNotError(t *testing.T) {
+	store := objectstore.NewMemoryObjectStore()
+	_, ok, err := readPullManifest(context.Background(), store, "ochre-weights", "some/prefix/")
+	require.NoError(t, err)
+	assert.False(t, ok)
+}
+
+// --- runStageWeights x manifest integration ---
+
+// TestRunStageWeights_RecordsSourceRevisionFromManifest covers D3's stage
+// side: when the source prefix carries a pull manifest, stage must record
+// its commit SHA against the model.
+func TestRunStageWeights_RecordsSourceRevisionFromManifest(t *testing.T) {
+	weightsStore := newWeightsStoreForTest(t)
+	ctx := context.Background()
+
+	store := objectstore.NewMemoryObjectStore()
+	seedCompleteWeightsPrefix(t, store, "models", "llama-3.2-1b/")
+	require.NoError(t, putPullManifest(ctx, store, "models", "llama-3.2-1b/", ochrePullManifest{
+		HFRepo: "meta-llama/Llama-3.2-1B-Instruct", RevisionSHA: "abc123def456", PulledAt: time.Now().UTC(),
+	}))
+
+	materialize := func(context.Context, string, int64) (string, error) { return "snap-0001", nil }
+	_, err := runStageWeights(ctx, store, weightsStore, t.TempDir(), selfHostModelID, "s3://models/llama-3.2-1b/", materialize, explodingSnapshotChecker(t))
+	require.NoError(t, err)
+
+	entry, ok, err := weightsStore.GetWeights(ctx, selfHostModelID)
+	require.NoError(t, err)
+	require.True(t, ok)
+	assert.Equal(t, "abc123def456", entry.SourceRevision)
+}
+
+// TestRunStageWeights_NoManifestLeavesSourceRevisionEmpty covers the offline
+// path explicitly: an operator-supplied prefix with no pull manifest stages
+// fine and records an empty SourceRevision, not an error or a fabricated one.
+func TestRunStageWeights_NoManifestLeavesSourceRevisionEmpty(t *testing.T) {
+	weightsStore := newWeightsStoreForTest(t)
+	ctx := context.Background()
+
+	store := objectstore.NewMemoryObjectStore()
+	seedCompleteWeightsPrefix(t, store, "models", "llama-3.2-1b/")
+
+	materialize := func(context.Context, string, int64) (string, error) { return "snap-0001", nil }
+	_, err := runStageWeights(ctx, store, weightsStore, t.TempDir(), selfHostModelID, "s3://models/llama-3.2-1b/", materialize, explodingSnapshotChecker(t))
+	require.NoError(t, err)
+
+	entry, ok, err := weightsStore.GetWeights(ctx, selfHostModelID)
+	require.NoError(t, err)
+	require.True(t, ok)
+	assert.Empty(t, entry.SourceRevision)
+}
+
+// --- fake Hugging Face server ---
+
+type fakeHFFile struct {
+	path    string
+	content string
+}
+
+// newFakeHFServer serves a Hugging Face-shaped API for repo@revision
+// resolving to sha: revision resolution, a recursive tree listing built from
+// files, and each file's content at /resolve/<sha>/<path>.
+func newFakeHFServer(t *testing.T, repo, revision, sha string, files []fakeHFFile) *httptest.Server {
+	t.Helper()
+	mux := http.NewServeMux()
+	mux.HandleFunc(fmt.Sprintf("/api/models/%s/revision/%s", repo, revision), func(w http.ResponseWriter, _ *http.Request) {
+		_ = json.NewEncoder(w).Encode(map[string]string{"sha": sha})
+	})
+	mux.HandleFunc(fmt.Sprintf("/api/models/%s/tree/%s", repo, sha), func(w http.ResponseWriter, _ *http.Request) {
+		entries := make([]hfhub.TreeEntry, len(files))
+		for i, f := range files {
+			entries[i] = hfhub.TreeEntry{Type: "file", Path: f.path, Size: int64(len(f.content))}
+		}
+		_ = json.NewEncoder(w).Encode(entries)
+	})
+	for _, f := range files {
+		mux.HandleFunc(fmt.Sprintf("/%s/resolve/%s/%s", repo, sha, f.path), func(w http.ResponseWriter, _ *http.Request) {
+			_, _ = io.WriteString(w, f.content)
+		})
+	}
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+// gatedHFServer answers every request with status, simulating a gated repo
+// with no usable token (D5).
+func gatedHFServer(t *testing.T, status int) *httptest.Server {
+	t.Helper()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(status)
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+// deadHFClient is a Client pointed at an address nothing listens on, so a
+// path that must never touch the network fails fast instead of reaching the
+// real Hugging Face hub.
+func deadHFClient() *hfhub.Client {
+	return &hfhub.Client{BaseURL: "http://127.0.0.1:1", HTTP: &http.Client{Timeout: time.Second}}
+}
+
+const (
+	testHFRepo = "meta-llama/Llama-3.2-1B-Instruct"
+	testHFSHA  = "abc123def456"
+)
+
+// --- runPullWeights (core logic) ---
+
+// TestRunPullWeights_UnknownModelRefused covers the catalog gate: an
+// unrecognised model ID must refuse before any Hugging Face or S3 work.
+func TestRunPullWeights_UnknownModelRefused(t *testing.T) {
+	_, err := runPullWeights(context.Background(), deadHFClient(), explodingObjectStore{t: t}, "not-a-real-model", testHFRepo, "main", "")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "not present in the Ochre catalog")
+}
+
+// TestRunPullWeights_ProviderModelRefused covers the self-host gate: a
+// provider-served model must refuse before any network work, same as stage.
+func TestRunPullWeights_ProviderModelRefused(t *testing.T) {
+	_, err := runPullWeights(context.Background(), deadHFClient(), explodingObjectStore{t: t}, providerModelID, testHFRepo, "main", "")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "not self-host")
+}
+
+// TestRunPullWeights_ResolvesUploadsAndWritesManifest is the happy path: it
+// resolves main to a pinned SHA, uploads only the safetensors + config set
+// flattened under the default prefix (D7), skips the pickle checkpoint, and
+// writes the ochre-pull.json manifest (D3).
+func TestRunPullWeights_ResolvesUploadsAndWritesManifest(t *testing.T) {
+	files := []fakeHFFile{
+		{"config.json", "{}"},
+		{"tokenizer_config.json", "{}"},
+		{"tokenizer.json", "{}"},
+		{"model.safetensors", "fake-weights-bytes"},
+		{"pytorch_model.bin", "must-not-be-uploaded"},
+	}
+	srv := newFakeHFServer(t, testHFRepo, "main", testHFSHA, files)
+	hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
+	store := objectstore.NewMemoryObjectStore()
+
+	finalURI, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", "")
+	require.NoError(t, err)
+	assert.Equal(t, defaultPullPrefix(testHFRepo, testHFSHA), finalURI)
+
+	bucket, prefix, err := parseWeightsS3URI(finalURI)
+	require.NoError(t, err)
+
+	out, err := store.GetObject(context.Background(), &s3.GetObjectInput{Bucket: strPtr(bucket), Key: strPtr(prefix + "model.safetensors")})
+	require.NoError(t, err)
+	got, _ := io.ReadAll(out.Body)
+	assert.Equal(t, "fake-weights-bytes", string(got))
+
+	_, err = store.GetObject(context.Background(), &s3.GetObjectInput{Bucket: strPtr(bucket), Key: strPtr(prefix + "pytorch_model.bin")})
+	assert.True(t, objectstore.IsNoSuchKeyError(err), "pickle checkpoint must never be uploaded")
+
+	manifest, ok, err := readPullManifest(context.Background(), store, bucket, prefix)
+	require.NoError(t, err)
+	require.True(t, ok)
+	assert.Equal(t, testHFRepo, manifest.HFRepo)
+	assert.Equal(t, testHFSHA, manifest.RevisionSHA)
+}
+
+// TestRunPullWeights_ExplicitS3URIOverridesDefault covers D7's primary path:
+// an operator-named --s3-uri is used verbatim (normalised) instead of the
+// keyed default.
+func TestRunPullWeights_ExplicitS3URIOverridesDefault(t *testing.T) {
+	files := []fakeHFFile{{"config.json", "{}"}, {"tokenizer_config.json", "{}"}, {"tokenizer.json", "{}"}, {"model.safetensors", "x"}}
+	srv := newFakeHFServer(t, testHFRepo, "main", testHFSHA, files)
+	hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
+	store := objectstore.NewMemoryObjectStore()
+
+	finalURI, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", "s3://custom-bucket/my-prefix")
+	require.NoError(t, err)
+	assert.Equal(t, "s3://custom-bucket/my-prefix/", finalURI)
+}
+
+// TestRunPullWeights_NoSafetensorsAborts covers D4's abort case: a repo with
+// only pickle checkpoints must be refused before any object lands.
+func TestRunPullWeights_NoSafetensorsAborts(t *testing.T) {
+	files := []fakeHFFile{{"config.json", "{}"}, {"pytorch_model.bin", "x"}}
+	srv := newFakeHFServer(t, testHFRepo, "main", testHFSHA, files)
+	hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
+	store := objectstore.NewMemoryObjectStore()
+
+	_, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", "")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "no *.safetensors")
+	assert.Equal(t, 0, store.Count(), "a refused pull must land nothing")
+}
+
+// TestRunPullWeights_GatedRepoAbortsCleanBeforeAnyUpload covers D5: a
+// 401/403 from the hub must fail before any object lands, with a clear
+// awserrors-coded licence/credential error.
+func TestRunPullWeights_GatedRepoAbortsCleanBeforeAnyUpload(t *testing.T) {
+	for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden} {
+		srv := gatedHFServer(t, status)
+		hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
+		store := objectstore.NewMemoryObjectStore()
+
+		_, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", "")
+		require.Error(t, err)
+		assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorAccessDeniedException), "status %d: got %v", status, err)
+		assert.Equal(t, 0, store.Count(), "a gated pull must land nothing")
+	}
+}
+
+// failAfterNPutsStore wraps a MemoryObjectStore, succeeding the first n
+// PutObject calls and failing every one after, so a mid-pull failure and its
+// cleanup (D5) are exercised deterministically.
+type failAfterNPutsStore struct {
+	*objectstore.MemoryObjectStore
+
+	n     int
+	calls int
+}
+
+func (f *failAfterNPutsStore) PutObject(ctx context.Context, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
+	f.calls++
+	if f.calls > f.n {
+		return nil, errors.New("simulated upload failure")
+	}
+	return f.MemoryObjectStore.PutObject(ctx, input)
+}
+
+var _ objectstore.ObjectStore = (*failAfterNPutsStore)(nil)
+
+// TestRunPullWeights_MidStreamFailureCleansUpPartialPrefix covers D5's core
+// promise: a failure partway through the upload loop must remove every
+// object already written, so 'stage' never sees a half-model.
+func TestRunPullWeights_MidStreamFailureCleansUpPartialPrefix(t *testing.T) {
+	files := []fakeHFFile{
+		{"config.json", "{}"},
+		{"tokenizer_config.json", "{}"},
+		{"tokenizer.json", "{}"},
+		{"model.safetensors", "weights"},
+	}
+	srv := newFakeHFServer(t, testHFRepo, "main", testHFSHA, files)
+	hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
+	store := &failAfterNPutsStore{MemoryObjectStore: objectstore.NewMemoryObjectStore(), n: 2}
+
+	_, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", "")
+	require.Error(t, err)
+	assert.Equal(t, 0, store.Count(), "objects uploaded before the failure must be cleaned up")
+}
+
+// --- resolveHFToken (D2) ---
+
+func newOchreWeightsPullTestCmd(t *testing.T, modelID, hfRepo, revision, s3URI, hfToken string) *cobra.Command {
+	t.Helper()
+	require.NoError(t, ochreWeightsPullCmd.Flags().Set("model-id", modelID))
+	require.NoError(t, ochreWeightsPullCmd.Flags().Set("hf-repo", hfRepo))
+	require.NoError(t, ochreWeightsPullCmd.Flags().Set("revision", revision))
+	require.NoError(t, ochreWeightsPullCmd.Flags().Set("s3-uri", s3URI))
+	require.NoError(t, ochreWeightsPullCmd.Flags().Set("hf-token", hfToken))
+	return ochreWeightsPullCmd
+}
+
+// TestResolveHFToken_PrefersFlagOverEnvAndStored covers the top of D2's
+// resolution order.
+func TestResolveHFToken_PrefersFlagOverEnvAndStored(t *testing.T) {
+	t.Setenv("HF_TOKEN", "hf_env_token")
+	cmd := newOchreWeightsPullTestCmd(t, selfHostModelID, testHFRepo, "main", "", "hf_flag_token")
+	cfg := &config.ClusterConfig{Node: "node1", Nodes: map[string]config.Config{"node1": {BaseDir: t.TempDir()}}}
+
+	got := resolveHFToken(context.Background(), cmd, cfg, nil)
+	assert.Equal(t, "hf_flag_token", got)
+}
+
+// TestResolveHFToken_FallsBackToEnv covers the second link: no flag, HF_TOKEN set.
+func TestResolveHFToken_FallsBackToEnv(t *testing.T) {
+	t.Setenv("HF_TOKEN", "hf_env_token")
+	cmd := newOchreWeightsPullTestCmd(t, selfHostModelID, testHFRepo, "main", "", "")
+	cfg := &config.ClusterConfig{Node: "node1", Nodes: map[string]config.Config{"node1": {BaseDir: t.TempDir()}}}
+
+	got := resolveHFToken(context.Background(), cmd, cfg, nil)
+	assert.Equal(t, "hf_env_token", got)
+}
+
+// TestResolveHFToken_NoTokenAnywhereReturnsEmpty covers the terminal case:
+// no flag, no env, and no master key on disk to even attempt the stored
+// credential -- must return empty, not error or panic, so a public repo
+// pull still works standalone.
+func TestResolveHFToken_NoTokenAnywhereReturnsEmpty(t *testing.T) {
+	t.Setenv("HF_TOKEN", "")
+	cmd := newOchreWeightsPullTestCmd(t, selfHostModelID, testHFRepo, "main", "", "")
+	cfg := &config.ClusterConfig{Node: "node1", Nodes: map[string]config.Config{"node1": {BaseDir: t.TempDir()}}}
+
+	got := resolveHFToken(context.Background(), cmd, cfg, nil)
+	assert.Empty(t, got)
+}
+
+// TestResolveHFToken_FallsBackToStoredPlatformCredential covers D2's
+// third link: with no flag or env, a credential stored via 'ochre
+// credentials set' (platform account, vendor "huggingface") is used.
+func TestResolveHFToken_FallsBackToStoredPlatformCredential(t *testing.T) {
+	t.Setenv("HF_TOKEN", "")
+	_, nc, js := testutil.StartTestJetStream(t)
+
+	baseDir := t.TempDir()
+	masterKey := bytes32Key()
+	require.NoError(t, os.MkdirAll(filepath.Join(baseDir, "config"), 0750))
+	require.NoError(t, handlers_iam.SaveMasterKey(filepath.Join(baseDir, "config", "master.key"), masterKey))
+
+	credStore := gateway_bedrock.NewCredentialStore(js, masterKey, 1, nil)
+	require.NoError(t, credStore.PutCredential(context.Background(), utils.GlobalAccountID, vendorHuggingFace, "hf_stored_token"))
+
+	cmd := newOchreWeightsPullTestCmd(t, selfHostModelID, testHFRepo, "main", "", "")
+	cfg := &config.ClusterConfig{Node: "node1", Nodes: map[string]config.Config{"node1": {BaseDir: baseDir}}}
+
+	got := resolveHFToken(context.Background(), cmd, cfg, nc)
+	assert.Equal(t, "hf_stored_token", got)
+}
+
+func bytes32Key() []byte {
+	key := make([]byte, 32)
+	for i := range key {
+		key[i] = byte(i + 1)
+	}
+	return key
+}
+
+func strPtr(s string) *string { return &s }
+
+// --- runOchreWeightsPull wrapper ---
+
+// TestRunOchreWeightsPull_ConnectFailureExits1 mirrors stage's own coverage:
+// pull cannot even reach the catalog check without a cluster connection.
+func TestRunOchreWeightsPull_ConnectFailureExits1(t *testing.T) {
+	stubConnect(t, nil, nil, errors.New("dial failed"))
+
+	cmd := newOchreWeightsPullTestCmd(t, selfHostModelID, testHFRepo, "main", "", "")
+	code := withOchreExitCapture(t, func() { runOchreWeightsPull(cmd, nil) })
+	assert.Equal(t, 1, code)
+}
+
+// TestRunOchreWeightsPull_UnknownModelExits1 drives the real cobra Run
+// function end to end, confirming the wrapper propagates runPullWeights's
+// catalog refusal into a process exit.
+func TestRunOchreWeightsPull_UnknownModelExits1(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	stubConnect(t, fakeClusterConfig(), nc, nil)
+
+	cmd := newOchreWeightsPullTestCmd(t, "not-a-real-model", testHFRepo, "main", "", "")
+	code := withOchreExitCapture(t, func() { runOchreWeightsPull(cmd, nil) })
+	assert.Equal(t, 1, code)
+}
+
+// --- ochre credentials set ---
+
+// withStdin swaps os.Stdin for a pipe fed with content, restoring the real
+// os.Stdin afterwards, mirroring captureStdout's os.Stdout swap.
+func withStdin(t *testing.T, content string, fn func()) {
+	t.Helper()
+	r, w, err := os.Pipe()
+	require.NoError(t, err)
+	orig := os.Stdin
+	os.Stdin = r                          //nolint:reassign // test-local stdin swap, restored below
+	t.Cleanup(func() { os.Stdin = orig }) //nolint:reassign // restoring the real os.Stdin captured above
+
+	go func() {
+		_, _ = io.WriteString(w, content)
+		_ = w.Close()
+	}()
+
+	fn()
+}
+
+func newOchreCredentialsSetTestCmd(t *testing.T, vendor, account, token string) *cobra.Command {
+	t.Helper()
+	require.NoError(t, ochreCredentialsSetCmd.Flags().Set("vendor", vendor))
+	require.NoError(t, ochreCredentialsSetCmd.Flags().Set("account", account))
+	require.NoError(t, ochreCredentialsSetCmd.Flags().Set("token", token))
+	return ochreCredentialsSetCmd
+}
+
+// TestReadTokenFromStdin_TrimsTrailingNewline covers the common 'echo token
+// | cmd --token -' shape.
+func TestReadTokenFromStdin_TrimsTrailingNewline(t *testing.T) {
+	token, err := readTokenFromStdin(strings.NewReader("hf_mytoken\n"))
+	require.NoError(t, err)
+	assert.Equal(t, "hf_mytoken", token)
+}
+
+// TestReadTokenFromStdin_EmptyIsError covers a piped-in nothing (e.g. an
+// empty file) refusing rather than silently storing an empty credential.
+func TestReadTokenFromStdin_EmptyIsError(t *testing.T) {
+	_, err := readTokenFromStdin(strings.NewReader(""))
+	require.Error(t, err)
+}
+
+// TestRunOchreCredentialsSet_RejectsNonDashToken covers the no-secrets-as-args
+// guard: --token must be literally '-', never the secret itself.
+func TestRunOchreCredentialsSet_RejectsNonDashToken(t *testing.T) {
+	cmd := newOchreCredentialsSetTestCmd(t, vendorHuggingFace, "", "hf_literal_secret")
+	code := withOchreExitCapture(t, func() { runOchreCredentialsSet(cmd, nil) })
+	assert.Equal(t, 1, code)
+}
+
+// TestRunOchreCredentialsSet_ConnectFailureExits1 covers the connect
+// failure path, same shape as every other ochre wrapper.
+func TestRunOchreCredentialsSet_ConnectFailureExits1(t *testing.T) {
+	stubConnect(t, nil, nil, errors.New("dial failed"))
+
+	cmd := newOchreCredentialsSetTestCmd(t, vendorHuggingFace, "", "-")
+	withStdin(t, "hf_mytoken\n", func() {
+		code := withOchreExitCapture(t, func() { runOchreCredentialsSet(cmd, nil) })
+		assert.Equal(t, 1, code)
+	})
+}
+
+// TestRunOchreCredentialsSet_MasterKeyLoadFailureExits1 covers the store's
+// own setup failing (no master.key provisioned yet) before PutCredential is
+// ever reached.
+func TestRunOchreCredentialsSet_MasterKeyLoadFailureExits1(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	cfg := &config.ClusterConfig{Node: "node1", Nodes: map[string]config.Config{"node1": {BaseDir: t.TempDir()}}}
+	stubConnect(t, cfg, nc, nil)
+
+	cmd := newOchreCredentialsSetTestCmd(t, vendorHuggingFace, "", "-")
+	withStdin(t, "hf_mytoken\n", func() {
+		code := withOchreExitCapture(t, func() { runOchreCredentialsSet(cmd, nil) })
+		assert.Equal(t, 1, code)
+	})
+}
+
+// TestRunOchreCredentialsSet_RoundTripsAndDefaultsToPlatformAccount drives
+// the real Run function end to end: stores a credential read from stdin
+// under the platform account (D2's default), then confirms a separately
+// constructed CredentialStore resolves it, and that the token never appears
+// on stdout.
+func TestRunOchreCredentialsSet_RoundTripsAndDefaultsToPlatformAccount(t *testing.T) {
+	ns, nc, _ := testutil.StartTestJetStream(t)
+	baseDir := t.TempDir()
+	masterKey := bytes32Key()
+	require.NoError(t, os.MkdirAll(filepath.Join(baseDir, "config"), 0750))
+	require.NoError(t, handlers_iam.SaveMasterKey(filepath.Join(baseDir, "config", "master.key"), masterKey))
+	cfg := &config.ClusterConfig{Node: "node1", Nodes: map[string]config.Config{"node1": {BaseDir: baseDir}}}
+	stubConnect(t, cfg, nc, nil)
+
+	cmd := newOchreCredentialsSetTestCmd(t, vendorHuggingFace, "", "-")
+	var out string
+	withStdin(t, "hf_supersecret\n", func() {
+		out = captureStdout(t, func() { runOchreCredentialsSet(cmd, nil) })
+	})
+
+	assert.NotContains(t, out, "hf_supersecret", "the token must never be printed")
+
+	// runOchreCredentialsSet closes its own NATS connection on return, so
+	// verification dials a second connection to the same embedded server.
+	verifyConn, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	defer verifyConn.Close()
+	verifyJS := testutil.NewJetStream(t, verifyConn)
+
+	credStore := gateway_bedrock.NewCredentialStore(verifyJS, masterKey, 1, nil)
+	token, ok, err := credStore.Resolve(context.Background(), utils.GlobalAccountID, vendorHuggingFace)
+	require.NoError(t, err)
+	require.True(t, ok)
+	assert.Equal(t, "hf_supersecret", token)
+}

cmd/spinifex/cmd/admin_ochre_test.go

@@ -1100,7 +1100,7 @@ func TestRegisterWeightsSnapshot_WritesEC2ReadableMetadata(t *testing.T) {
 	require.NoError(t, registerWeightsSnapshot(store, bucket,
 		"snap-vol-abc", "vol-abc", 12*bytesPerGiB, "ap-southeast-2a", true))
 
-	cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(store, bucket, "snap-vol-abc")
+	cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(context.Background(), store, bucket, "snap-vol-abc")
 	require.NoError(t, err)
 	assert.Equal(t, "snap-vol-abc", cfg.SnapshotID)
 	assert.Equal(t, "vol-abc", cfg.VolumeID)
@@ -1117,7 +1117,7 @@ func TestRegisterWeightsSnapshot_UnencryptedVolume(t *testing.T) {
 	require.NoError(t, registerWeightsSnapshot(store, "predastore",
 		"snap-vol-plain", "vol-plain", bytesPerGiB, "ap-southeast-2a", false))
 
-	cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(store, "predastore", "snap-vol-plain")
+	cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(context.Background(), store, "predastore", "snap-vol-plain")
 	require.NoError(t, err)
 	assert.False(t, cfg.Encrypted)
 }

cmd/spinifex/cmd/admin_ochre_vector.go

@@ -0,0 +1,450 @@
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strconv"
+	"time"
+
+	handlers_ochrevector "github.com/mulgadc/spinifex/spinifex/handlers/ochrevector"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/pterm/pterm"
+	"github.com/spf13/cobra"
+)
+
+// defaultVectorEmbeddingModel is the platform's default embedding model for
+// a new index when --model is not given.
+const defaultVectorEmbeddingModel = "nomic-embed-text-v1.5"
+
+// cliChunkPreviewChars bounds a chunk's preview in the query results table
+// (D10 payload-size discipline extended to terminal readability): the full,
+// server-truncated chunk text is always available via --json.
+const cliChunkPreviewChars = 120
+
+var ochreVectorCmd = &cobra.Command{
+	Use:   "vector",
+	Short: "Manage Ochre tenant vector-store indexes, ingestion and queries",
+	Long: `Operator surface over the daemon's tenant vector store: create/delete/list
+indexes, start an ingestion job from a predastore bucket, check a job's
+status, and run a similarity query -- the live-verify surface for the
+internal ochre.vector.* NATS API.`,
+}
+
+var ochreVectorIndexCmd = &cobra.Command{
+	Use:   "index",
+	Short: "Manage vector index lifecycle",
+}
+
+var ochreVectorIndexCreateCmd = &cobra.Command{
+	Use:   "create",
+	Short: "Create a new vector index",
+	Long: `create reserves a new index under the platform account, provisions its
+account schema and backing table, and returns once it is READY. The index's
+embedding model and dimension are fixed for its lifetime (D6): every ingest
+and query against it uses exactly these values, regardless of what a caller
+sends.`,
+	Run: runOchreVectorIndexCreate,
+}
+
+var ochreVectorIndexDeleteCmd = &cobra.Command{
+	Use:   "delete <index-id>",
+	Short: "Delete a vector index",
+	Long: `delete drops the index's backing table and its registry record. Idempotent:
+deleting an already-absent index reports success.`,
+	Args: cobra.ExactArgs(1),
+	Run:  runOchreVectorIndexDelete,
+}
+
+var ochreVectorIndexListCmd = &cobra.Command{
+	Use:   "list",
+	Short: "List every vector index",
+	Run:   runOchreVectorIndexList,
+}
+
+var ochreVectorIngestCmd = &cobra.Command{
+	Use:   "ingest",
+	Short: "Start an ingestion job for a vector index from a predastore bucket",
+	Long: `ingest lists --bucket/--prefix, chunks and embeds each object, and upserts
+its rows into --index. The embedding model and dimension used are always the
+index's own pinned values, not any --model flag -- an index's embedding
+model is a property of the index, not of one ingest call. Returns as soon as
+the job is claimed (PENDING); use 'job describe' to follow its progress.`,
+	Run: runOchreVectorIngest,
+}
+
+var ochreVectorJobCmd = &cobra.Command{
+	Use:   "job",
+	Short: "Inspect vector-store ingestion jobs",
+}
+
+var ochreVectorJobDescribeCmd = &cobra.Command{
+	Use:   "describe <job-id>",
+	Short: "Show an ingestion job's current record",
+	Args:  cobra.ExactArgs(1),
+	Run:   runOchreVectorJobDescribe,
+}
+
+var ochreVectorQueryCmd = &cobra.Command{
+	Use:   "query",
+	Short: "Run a similarity query against a vector index",
+	Long: `query embeds --text server-side against --index's own pinned embedding model
+(D8) and returns the nearest chunks, nearest first. --filter takes a
+JSON-encoded metadata Filter AST (Op/Key/Value/Children). Long chunk text is
+truncated in the default table; pass --json for the full response.`,
+	Run: runOchreVectorQuery,
+}
+
+func init() {
+	ochreCmd.AddCommand(ochreVectorCmd)
+	ochreVectorCmd.AddCommand(ochreVectorIndexCmd)
+	ochreVectorIndexCmd.AddCommand(ochreVectorIndexCreateCmd)
+	ochreVectorIndexCmd.AddCommand(ochreVectorIndexDeleteCmd)
+	ochreVectorIndexCmd.AddCommand(ochreVectorIndexListCmd)
+	ochreVectorCmd.AddCommand(ochreVectorIngestCmd)
+	ochreVectorCmd.AddCommand(ochreVectorJobCmd)
+	ochreVectorJobCmd.AddCommand(ochreVectorJobDescribeCmd)
+	ochreVectorCmd.AddCommand(ochreVectorQueryCmd)
+
+	ochreVectorIndexCreateCmd.Flags().String("name", "", "Human-readable index name (required)")
+	ochreVectorIndexCreateCmd.Flags().Int("dimension", 0, "Embedding vector dimension (required)")
+	ochreVectorIndexCreateCmd.Flags().String("model", defaultVectorEmbeddingModel, "Embedding model this index is pinned to")
+	_ = ochreVectorIndexCreateCmd.MarkFlagRequired("name")
+	_ = ochreVectorIndexCreateCmd.MarkFlagRequired("dimension")
+
+	ochreVectorIngestCmd.Flags().String("index", "", "Target index ID (required)")
+	ochreVectorIngestCmd.Flags().String("bucket", "", "Source predastore bucket (required)")
+	ochreVectorIngestCmd.Flags().String("prefix", "", "Source object key prefix")
+	ochreVectorIngestCmd.Flags().Int("chunk-size", 0, "Chunk size in runes (0 takes the package default)")
+	ochreVectorIngestCmd.Flags().Int("chunk-overlap", 0, "Chunk overlap in runes (0 takes the package default)")
+	ochreVectorIngestCmd.Flags().StringToString("meta", nil, "Static metadata tag stamped on every ingested row, k=v (repeatable)")
+	_ = ochreVectorIngestCmd.MarkFlagRequired("index")
+	_ = ochreVectorIngestCmd.MarkFlagRequired("bucket")
+
+	ochreVectorQueryCmd.Flags().String("index", "", "Index ID to query (required)")
+	ochreVectorQueryCmd.Flags().String("text", "", "Query text, embedded server-side against the index's pinned model (required)")
+	ochreVectorQueryCmd.Flags().Int("k", 0, "Number of results (0 takes the service default; hard-capped at 100)")
+	ochreVectorQueryCmd.Flags().String("filter", "", "JSON-encoded metadata Filter AST, e.g. {\"Op\":\"equals\",\"Key\":\"category\",\"Value\":\"faq\"}")
+	ochreVectorQueryCmd.Flags().Bool("json", false, "Print full, untruncated results as JSON")
+	_ = ochreVectorQueryCmd.MarkFlagRequired("index")
+	_ = ochreVectorQueryCmd.MarkFlagRequired("text")
+}
+
+// vectorServiceFn indirects the NATS-backed client so the Run functions'
+// connect/exit control flow can be tested without a live daemon, mirroring
+// endpointServiceFn.
+var vectorServiceFn = func() (handlers_ochrevector.VectorService, func(), error) {
+	_, nc, err := loadConfigAndConnectFn()
+	if err != nil {
+		return nil, nil, err
+	}
+	return handlers_ochrevector.NewNATSVectorService(nc), nc.Close, nil
+}
+
+// formatIndexRecord renders one index record as aligned key/value lines.
+func formatIndexRecord(rec handlers_ochrevector.Record) string {
+	rows := [][2]string{
+		{"Index ID", rec.ID},
+		{"Name", rec.Name},
+		{"State", rec.State},
+		{"Dimension", strconv.Itoa(rec.Dimension)},
+		{"Embedding model", rec.EmbeddingModel},
+	}
+	if !rec.CreatedAt.IsZero() {
+		rows = append(rows, [2]string{"Created at", rec.CreatedAt.Format(time.RFC3339)})
+	}
+	out := ""
+	for _, row := range rows {
+		out += fmt.Sprintf("%-16s %s\n", row[0]+":", row[1])
+	}
+	return out
+}
+
+// formatJobRecord renders one ingestion job record as aligned key/value lines.
+func formatJobRecord(job handlers_ochrevector.JobRecord) string {
+	rows := [][2]string{
+		{"Job ID", job.ID},
+		{"Index ID", job.IndexID},
+		{"State", job.State},
+		{"Documents", fmt.Sprintf("%d/%d", job.DocumentsDone, job.DocumentsTotal)},
+	}
+	if len(job.FailedDocuments) > 0 {
+		rows = append(rows, [2]string{"Failed docs", strconv.Itoa(len(job.FailedDocuments))})
+	}
+	if job.Error != "" {
+		rows = append(rows, [2]string{"Error", job.Error})
+	}
+	if !job.CreatedAt.IsZero() {
+		rows = append(rows, [2]string{"Created at", job.CreatedAt.Format(time.RFC3339)})
+	}
+	if !job.UpdatedAt.IsZero() {
+		rows = append(rows, [2]string{"Updated at", job.UpdatedAt.Format(time.RFC3339)})
+	}
+	out := ""
+	for _, row := range rows {
+		out += fmt.Sprintf("%-14s %s\n", row[0]+":", row[1])
+	}
+	return out
+}
+
+// truncateForTable bounds s to maxRunes runes for a table cell, so one
+// oversized chunk cannot blow out terminal width; the untruncated text is
+// always available via --json.
+func truncateForTable(s string, maxRunes int) string {
+	runes := []rune(s)
+	if len(runes) <= maxRunes {
+		return s
+	}
+	return string(runes[:maxRunes]) + "…"
+}
+
+// formatQueryResults renders 'ochre vector query' output: a truncated table
+// by default, or the full response as indented JSON when asJSON is set
+// (D10's "full text via source ref" -- here, via --json).
+func formatQueryResults(results []handlers_ochrevector.QueryResult, asJSON bool) (string, error) {
+	if asJSON {
+		//nolint:musttag // QueryResult is an internal, non-wire Go type (predates this CLI); it round-trips fine on its default field names.
+		data, err := json.MarshalIndent(results, "", "  ")
+		if err != nil {
+			return "", fmt.Errorf("encode results as JSON: %w", err)
+		}
+		return string(data), nil
+	}
+	if len(results) == 0 {
+		return "No results.", nil
+	}
+	tableData := pterm.TableData{{"SCORE", "SOURCE", "OFFSET", "CHUNK"}}
+	for _, r := range results {
+		tableData = append(tableData, []string{
+			strconv.FormatFloat(float64(r.Score), 'f', 4, 32),
+			r.SourceKey,
+			strconv.Itoa(r.SourceOffset),
+			truncateForTable(r.Chunk, cliChunkPreviewChars),
+		})
+	}
+	return pterm.DefaultTable.WithHasHeader().WithLeftAlignment().WithData(tableData).Srender()
+}
+
+// parseFilterFlag decodes --filter's JSON-encoded Filter AST. An empty raw
+// string is not an error: it means no filter, matching QueryRequest.Filter's
+// nil-is-unfiltered contract.
+func parseFilterFlag(raw string) (*handlers_ochrevector.Filter, error) {
+	if raw == "" {
+		return nil, nil
+	}
+	var f handlers_ochrevector.Filter
+	//nolint:musttag // Filter is the existing filter-AST type (D9); it round-trips fine on its default field names.
+	if err := json.Unmarshal([]byte(raw), &f); err != nil {
+		return nil, fmt.Errorf("decode --filter: %w", err)
+	}
+	return &f, nil
+}
+
+// runIndexCreate is the testable core of 'ochre vector index create': it
+// mints a fresh index ID (indexes are named, not caller-numbered) and
+// creates it. Returns the message to print.
+func runIndexCreate(ctx context.Context, svc handlers_ochrevector.VectorService, name string, dimension int, model string) (string, error) {
+	out, err := svc.CreateIndex(ctx, &handlers_ochrevector.CreateIndexRequest{
+		IndexID:        utils.GenerateResourceID("idx"),
+		Name:           name,
+		Dimension:      dimension,
+		EmbeddingModel: model,
+	}, utils.GlobalAccountID)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("✅ Created index %s.\n\n%s", out.Index.ID, formatIndexRecord(out.Index)), nil
+}
+
+// runIndexDelete is the testable core of 'ochre vector index delete'.
+func runIndexDelete(ctx context.Context, svc handlers_ochrevector.VectorService, indexID string) (string, error) {
+	if _, err := svc.DeleteIndex(ctx, &handlers_ochrevector.DeleteIndexRequest{IndexID: indexID}, utils.GlobalAccountID); err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("✅ Index %s deleted.", indexID), nil
+}
+
+// listIndexesOutput renders 'ochre vector index list'. Split from its Run
+// function so it is testable against a fake service with no NATS connection.
+func listIndexesOutput(ctx context.Context, svc handlers_ochrevector.VectorService) (string, error) {
+	out, err := svc.ListIndexes(ctx, &handlers_ochrevector.ListIndexesRequest{}, utils.GlobalAccountID)
+	if err != nil {
+		return "", err
+	}
+	if len(out.Indexes) == 0 {
+		return "No vector indexes.", nil
+	}
+	tableData := pterm.TableData{{"INDEX ID", "NAME", "STATE", "DIMENSION", "MODEL"}}
+	for _, r := range out.Indexes {
+		tableData = append(tableData, []string{r.ID, r.Name, r.State, strconv.Itoa(r.Dimension), r.EmbeddingModel})
+	}
+	return pterm.DefaultTable.WithHasHeader().WithLeftAlignment().WithData(tableData).Srender()
+}
+
+// runIngest is the testable core of 'ochre vector ingest'. The daemon stamps
+// the index's own embedding model/dimension onto the source spec server-side
+// (see handlers_ochrevector.vectorService.Ingest), so none is sent here.
+func runIngest(ctx context.Context, svc handlers_ochrevector.VectorService, indexID, bucket, prefix string, chunkSize, chunkOverlap int, meta map[string]string) (string, error) {
+	out, err := svc.Ingest(ctx, &handlers_ochrevector.IngestRequest{
+		IndexID: indexID,
+		Source: handlers_ochrevector.SourceSpec{
+			Bucket:       bucket,
+			Prefix:       prefix,
+			ChunkSize:    chunkSize,
+			ChunkOverlap: chunkOverlap,
+			Metadata:     meta,
+		},
+	}, utils.GlobalAccountID)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("✅ Started ingestion job %s for index %s (state %s).", out.Job.ID, indexID, out.Job.State), nil
+}
+
+// runJobDescribe is the testable core of 'ochre vector job describe'.
+func runJobDescribe(ctx context.Context, svc handlers_ochrevector.VectorService, jobID string) (string, error) {
+	out, err := svc.DescribeJob(ctx, &handlers_ochrevector.DescribeJobRequest{JobID: jobID}, utils.GlobalAccountID)
+	if err != nil {
+		return "", err
+	}
+	return formatJobRecord(out.Job), nil
+}
+
+// runQuery is the testable core of 'ochre vector query'.
+func runQuery(ctx context.Context, svc handlers_ochrevector.VectorService, indexID, text string, k int, filterJSON string, asJSON bool) (string, error) {
+	filter, err := parseFilterFlag(filterJSON)
+	if err != nil {
+		return "", err
+	}
+	out, err := svc.Query(ctx, &handlers_ochrevector.QueryRequest{IndexID: indexID, Text: text, K: k, Filter: filter}, utils.GlobalAccountID)
+	if err != nil {
+		return "", err
+	}
+	return formatQueryResults(out.Results, asJSON)
+}
+
+func runOchreVectorIndexCreate(cmd *cobra.Command, _ []string) {
+	name, _ := cmd.Flags().GetString("name")
+	dimension, _ := cmd.Flags().GetInt("dimension")
+	model, _ := cmd.Flags().GetString("model")
+
+	svc, closeFn, err := vectorServiceFn()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+	defer closeFn()
+
+	msg, err := runIndexCreate(context.Background(), svc, name, dimension, model)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		ochreExit(1)
+		return
+	}
+	fmt.Println(msg)
+}
+
+func runOchreVectorIndexDelete(_ *cobra.Command, args []string) {
+	svc, closeFn, err := vectorServiceFn()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+	defer closeFn()
+
+	msg, err := runIndexDelete(context.Background(), svc, args[0])
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		ochreExit(1)
+		return
+	}
+	fmt.Println(msg)
+}
+
+func runOchreVectorIndexList(_ *cobra.Command, _ []string) {
+	svc, closeFn, err := vectorServiceFn()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+	defer closeFn()
+
+	out, err := listIndexesOutput(context.Background(), svc)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		ochreExit(1)
+		return
+	}
+	fmt.Println(out)
+}
+
+func runOchreVectorIngest(cmd *cobra.Command, _ []string) {
+	indexID, _ := cmd.Flags().GetString("index")
+	bucket, _ := cmd.Flags().GetString("bucket")
+	prefix, _ := cmd.Flags().GetString("prefix")
+	chunkSize, _ := cmd.Flags().GetInt("chunk-size")
+	chunkOverlap, _ := cmd.Flags().GetInt("chunk-overlap")
+	meta, _ := cmd.Flags().GetStringToString("meta")
+
+	svc, closeFn, err := vectorServiceFn()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+	defer closeFn()
+
+	msg, err := runIngest(context.Background(), svc, indexID, bucket, prefix, chunkSize, chunkOverlap, meta)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		ochreExit(1)
+		return
+	}
+	fmt.Println(msg)
+}
+
+func runOchreVectorJobDescribe(_ *cobra.Command, args []string) {
+	svc, closeFn, err := vectorServiceFn()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+	defer closeFn()
+
+	out, err := runJobDescribe(context.Background(), svc, args[0])
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		ochreExit(1)
+		return
+	}
+	fmt.Print(out)
+}
+
+func runOchreVectorQuery(cmd *cobra.Command, _ []string) {
+	indexID, _ := cmd.Flags().GetString("index")
+	text, _ := cmd.Flags().GetString("text")
+	k, _ := cmd.Flags().GetInt("k")
+	filterJSON, _ := cmd.Flags().GetString("filter")
+	asJSON, _ := cmd.Flags().GetBool("json")
+
+	svc, closeFn, err := vectorServiceFn()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+	defer closeFn()
+
+	out, err := runQuery(context.Background(), svc, indexID, text, k, filterJSON, asJSON)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		ochreExit(1)
+		return
+	}
+	fmt.Println(out)
+}

cmd/spinifex/cmd/admin_ochre_vector_test.go

@@ -0,0 +1,384 @@
+// Exercises unexported ochre vector CLI command internals with no
+// exported surface to drive them through.
+//
+//test:in-package
+package cmd
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	handlers_ochrevector "github.com/mulgadc/spinifex/spinifex/handlers/ochrevector"
+	"github.com/stretchr/testify/require"
+)
+
+// fakeVectorService serves scripted responses so the CLI's testable cores
+// can be exercised without a daemon or a live NATS connection.
+type fakeVectorService struct {
+	createIndex    handlers_ochrevector.CreateIndexResponse
+	createIndexErr error
+	deleteIndexErr error
+	list           []handlers_ochrevector.Record
+	listErr        error
+	ingest         handlers_ochrevector.IngestResponse
+	ingestErr      error
+	describeJob    handlers_ochrevector.JobRecord
+	describeErr    error
+	query          []handlers_ochrevector.QueryResult
+	queryErr       error
+	listJobs       []handlers_ochrevector.JobRecord
+	listJobsErr    error
+
+	createIndexInputs []*handlers_ochrevector.CreateIndexRequest
+	deleteIndexInputs []*handlers_ochrevector.DeleteIndexRequest
+	ingestInputs      []*handlers_ochrevector.IngestRequest
+	queryInputs       []*handlers_ochrevector.QueryRequest
+}
+
+func (f *fakeVectorService) CreateIndex(_ context.Context, in *handlers_ochrevector.CreateIndexRequest, _ string) (*handlers_ochrevector.CreateIndexResponse, error) {
+	f.createIndexInputs = append(f.createIndexInputs, in)
+	if f.createIndexErr != nil {
+		return nil, f.createIndexErr
+	}
+	out := f.createIndex
+	if out.Index.ID == "" {
+		out.Index.ID = in.IndexID
+	}
+	return &out, nil
+}
+
+func (f *fakeVectorService) DeleteIndex(_ context.Context, in *handlers_ochrevector.DeleteIndexRequest, _ string) (*handlers_ochrevector.DeleteIndexResponse, error) {
+	f.deleteIndexInputs = append(f.deleteIndexInputs, in)
+	if f.deleteIndexErr != nil {
+		return nil, f.deleteIndexErr
+	}
+	return &handlers_ochrevector.DeleteIndexResponse{}, nil
+}
+
+func (f *fakeVectorService) ListIndexes(_ context.Context, _ *handlers_ochrevector.ListIndexesRequest, _ string) (*handlers_ochrevector.ListIndexesResponse, error) {
+	if f.listErr != nil {
+		return nil, f.listErr
+	}
+	return &handlers_ochrevector.ListIndexesResponse{Indexes: f.list}, nil
+}
+
+func (f *fakeVectorService) Ingest(_ context.Context, in *handlers_ochrevector.IngestRequest, _ string) (*handlers_ochrevector.IngestResponse, error) {
+	f.ingestInputs = append(f.ingestInputs, in)
+	if f.ingestErr != nil {
+		return nil, f.ingestErr
+	}
+	return &f.ingest, nil
+}
+
+func (f *fakeVectorService) DescribeJob(_ context.Context, _ *handlers_ochrevector.DescribeJobRequest, _ string) (*handlers_ochrevector.DescribeJobResponse, error) {
+	if f.describeErr != nil {
+		return nil, f.describeErr
+	}
+	return &handlers_ochrevector.DescribeJobResponse{Job: f.describeJob}, nil
+}
+
+func (f *fakeVectorService) Query(_ context.Context, in *handlers_ochrevector.QueryRequest, _ string) (*handlers_ochrevector.QueryResponse, error) {
+	f.queryInputs = append(f.queryInputs, in)
+	if f.queryErr != nil {
+		return nil, f.queryErr
+	}
+	return &handlers_ochrevector.QueryResponse{Results: f.query}, nil
+}
+
+func (f *fakeVectorService) ListJobs(_ context.Context, _ *handlers_ochrevector.ListJobsRequest, _ string) (*handlers_ochrevector.ListJobsResponse, error) {
+	if f.listJobsErr != nil {
+		return nil, f.listJobsErr
+	}
+	return &handlers_ochrevector.ListJobsResponse{Jobs: f.listJobs}, nil
+}
+
+var _ handlers_ochrevector.VectorService = (*fakeVectorService)(nil)
+
+const testIndexID = "idx-0123456789abcdef0"
+
+func TestRunIndexCreate_MintsIDAndPrintsRecord(t *testing.T) {
+	svc := &fakeVectorService{createIndex: handlers_ochrevector.CreateIndexResponse{
+		Index: handlers_ochrevector.Record{Name: "kb1", State: handlers_ochrevector.StateReady, Dimension: 768, EmbeddingModel: "nomic-embed-text-v1.5"},
+	}}
+
+	msg, err := runIndexCreate(context.Background(), svc, "kb1", 768, "nomic-embed-text-v1.5")
+	require.NoError(t, err)
+	require.Contains(t, msg, "READY")
+	require.Contains(t, msg, "nomic-embed-text-v1.5")
+
+	require.Len(t, svc.createIndexInputs, 1)
+	require.NotEmpty(t, svc.createIndexInputs[0].IndexID, "the CLI must mint an index ID, not leave it empty")
+	require.Equal(t, 768, svc.createIndexInputs[0].Dimension)
+}
+
+func TestRunIndexCreate_ErrorSurfaces(t *testing.T) {
+	svc := &fakeVectorService{createIndexErr: errors.New("ValidationException")}
+	_, err := runIndexCreate(context.Background(), svc, "kb1", 768, "m")
+	require.ErrorContains(t, err, "ValidationException")
+}
+
+func TestRunIndexDelete_ReportsDeletion(t *testing.T) {
+	svc := &fakeVectorService{}
+	msg, err := runIndexDelete(context.Background(), svc, testIndexID)
+	require.NoError(t, err)
+	require.Contains(t, msg, testIndexID)
+	require.Len(t, svc.deleteIndexInputs, 1)
+	require.Equal(t, testIndexID, svc.deleteIndexInputs[0].IndexID)
+}
+
+func TestRunIndexDelete_ErrorSurfaces(t *testing.T) {
+	svc := &fakeVectorService{deleteIndexErr: errors.New("nats: timeout")}
+	_, err := runIndexDelete(context.Background(), svc, testIndexID)
+	require.ErrorContains(t, err, "timeout")
+}
+
+func TestListIndexesOutput_NoIndexes(t *testing.T) {
+	msg, err := listIndexesOutput(context.Background(), &fakeVectorService{})
+	require.NoError(t, err)
+	require.Equal(t, "No vector indexes.", msg)
+}
+
+func TestListIndexesOutput_ListsIndexes(t *testing.T) {
+	svc := &fakeVectorService{list: []handlers_ochrevector.Record{
+		{ID: testIndexID, Name: "kb1", State: handlers_ochrevector.StateReady, Dimension: 768, EmbeddingModel: "nomic-embed-text-v1.5"},
+	}}
+	msg, err := listIndexesOutput(context.Background(), svc)
+	require.NoError(t, err)
+	require.Contains(t, msg, testIndexID)
+	require.Contains(t, msg, "READY")
+}
+
+func TestListIndexesOutput_ErrorSurfaces(t *testing.T) {
+	svc := &fakeVectorService{listErr: errors.New("nats: no responders")}
+	_, err := listIndexesOutput(context.Background(), svc)
+	require.ErrorContains(t, err, "no responders")
+}
+
+func TestRunIngest_NoModelFlagSentByCLI(t *testing.T) {
+	svc := &fakeVectorService{ingest: handlers_ochrevector.IngestResponse{
+		Job: handlers_ochrevector.JobRecord{ID: "job-abc", IndexID: testIndexID, State: handlers_ochrevector.JobStatePending},
+	}}
+
+	msg, err := runIngest(context.Background(), svc, testIndexID, "docs", "kb/", 100, 10, map[string]string{"team": "eng"})
+	require.NoError(t, err)
+	require.Contains(t, msg, "job-abc")
+	require.Contains(t, msg, "PENDING")
+
+	require.Len(t, svc.ingestInputs, 1)
+	in := svc.ingestInputs[0]
+	require.Equal(t, testIndexID, in.IndexID)
+	require.Equal(t, "docs", in.Source.Bucket)
+	require.Equal(t, "kb/", in.Source.Prefix)
+	require.Equal(t, 100, in.Source.ChunkSize)
+	require.Equal(t, "eng", in.Source.Metadata["team"])
+	// The CLI has no --model flag: the daemon stamps the index's pinned
+	// model server-side, so the request's EmbeddingModel is left zero.
+	require.Empty(t, in.Source.EmbeddingModel)
+}
+
+func TestRunIngest_ErrorSurfaces(t *testing.T) {
+	svc := &fakeVectorService{ingestErr: errors.New("ResourceNotFoundException")}
+	_, err := runIngest(context.Background(), svc, "no-such-index", "docs", "", 0, 0, nil)
+	require.ErrorContains(t, err, "ResourceNotFoundException")
+}
+
+func TestRunJobDescribe_PrintsRecord(t *testing.T) {
+	svc := &fakeVectorService{describeJob: handlers_ochrevector.JobRecord{
+		ID: "job-abc", IndexID: testIndexID, State: handlers_ochrevector.JobStateReady, DocumentsDone: 3, DocumentsTotal: 3,
+	}}
+	out, err := runJobDescribe(context.Background(), svc, "job-abc")
+	require.NoError(t, err)
+	require.Contains(t, out, "job-abc")
+	require.Contains(t, out, "3/3")
+}
+
+func TestRunJobDescribe_ErrorSurfaces(t *testing.T) {
+	svc := &fakeVectorService{describeErr: errors.New("ochrevector: job not found")}
+	_, err := runJobDescribe(context.Background(), svc, "no-such-job")
+	require.ErrorContains(t, err, "job not found")
+}
+
+func TestRunQuery_TableTruncatesLongChunk(t *testing.T) {
+	longChunk := strings.Repeat("x", cliChunkPreviewChars+50)
+	svc := &fakeVectorService{query: []handlers_ochrevector.QueryResult{
+		{Chunk: longChunk, SourceKey: "docs/a.txt", SourceOffset: 0, Score: 0.9},
+	}}
+
+	out, err := runQuery(context.Background(), svc, testIndexID, "hello", 5, "", false)
+	require.NoError(t, err)
+	require.Contains(t, out, "docs/a.txt")
+	require.NotContains(t, out, longChunk, "the table must not print the full untruncated chunk")
+
+	require.Len(t, svc.queryInputs, 1)
+	require.Equal(t, testIndexID, svc.queryInputs[0].IndexID)
+	require.Equal(t, "hello", svc.queryInputs[0].Text)
+	require.Equal(t, 5, svc.queryInputs[0].K)
+}
+
+func TestRunQuery_JSONFlagPrintsFullChunk(t *testing.T) {
+	longChunk := strings.Repeat("x", cliChunkPreviewChars+50)
+	svc := &fakeVectorService{query: []handlers_ochrevector.QueryResult{
+		{Chunk: longChunk, SourceKey: "docs/a.txt", Score: 0.9},
+	}}
+
+	out, err := runQuery(context.Background(), svc, testIndexID, "hello", 0, "", true)
+	require.NoError(t, err)
+	require.Contains(t, out, longChunk, "--json must print the full, untruncated chunk")
+}
+
+func TestRunQuery_NoResults(t *testing.T) {
+	out, err := runQuery(context.Background(), &fakeVectorService{}, testIndexID, "hello", 0, "", false)
+	require.NoError(t, err)
+	require.Equal(t, "No results.", out)
+}
+
+func TestRunQuery_FilterFlagReachesRequest(t *testing.T) {
+	svc := &fakeVectorService{}
+	filterJSON := `{"Op":"equals","Key":"category","Value":"faq"}`
+
+	_, err := runQuery(context.Background(), svc, testIndexID, "hello", 0, filterJSON, false)
+	require.NoError(t, err)
+	require.Len(t, svc.queryInputs, 1)
+	require.NotNil(t, svc.queryInputs[0].Filter)
+	require.Equal(t, handlers_ochrevector.FilterEquals, svc.queryInputs[0].Filter.Op)
+	require.Equal(t, "category", svc.queryInputs[0].Filter.Key)
+}
+
+func TestRunQuery_InvalidFilterJSONErrors(t *testing.T) {
+	_, err := runQuery(context.Background(), &fakeVectorService{}, testIndexID, "hello", 0, "{not json", false)
+	require.Error(t, err)
+}
+
+func TestRunQuery_ErrorSurfaces(t *testing.T) {
+	svc := &fakeVectorService{queryErr: errors.New("nats: timeout")}
+	_, err := runQuery(context.Background(), svc, testIndexID, "hello", 0, "", false)
+	require.ErrorContains(t, err, "timeout")
+}
+
+// withVectorService swaps vectorServiceFn for one returning svc, so a Run
+// wrapper's real connect/validate/exit control flow runs without a daemon.
+func withVectorService(t *testing.T, svc handlers_ochrevector.VectorService, connErr error) {
+	t.Helper()
+	orig := vectorServiceFn
+	t.Cleanup(func() { vectorServiceFn = orig })
+	vectorServiceFn = func() (handlers_ochrevector.VectorService, func(), error) {
+		if connErr != nil {
+			return nil, nil, connErr
+		}
+		return svc, func() {}, nil
+	}
+}
+
+func TestRunOchreVectorIndexCreate_PrintsRecord(t *testing.T) {
+	withVectorService(t, &fakeVectorService{createIndex: handlers_ochrevector.CreateIndexResponse{
+		Index: handlers_ochrevector.Record{ID: testIndexID, State: handlers_ochrevector.StateReady},
+	}}, nil)
+
+	cmd := *ochreVectorIndexCreateCmd
+	require.NoError(t, cmd.Flags().Set("name", "kb1"))
+	require.NoError(t, cmd.Flags().Set("dimension", "768"))
+
+	var out string
+	code := withOchreExitCapture(t, func() {
+		out = captureStdout(t, func() { runOchreVectorIndexCreate(&cmd, nil) })
+	})
+	require.Equal(t, -1, code)
+	require.Contains(t, out, testIndexID)
+}
+
+func TestRunOchreVectorIndexCreate_ConnectFailureExits1(t *testing.T) {
+	withVectorService(t, nil, errors.New("dial nats: connection refused"))
+
+	cmd := *ochreVectorIndexCreateCmd
+	require.NoError(t, cmd.Flags().Set("name", "kb1"))
+	require.NoError(t, cmd.Flags().Set("dimension", "768"))
+
+	code := withOchreExitCapture(t, func() { runOchreVectorIndexCreate(&cmd, nil) })
+	require.Equal(t, 1, code)
+}
+
+func TestRunOchreVectorIndexDelete_ReportsDeletion(t *testing.T) {
+	svc := &fakeVectorService{}
+	withVectorService(t, svc, nil)
+
+	var out string
+	code := withOchreExitCapture(t, func() {
+		out = captureStdout(t, func() { runOchreVectorIndexDelete(ochreVectorIndexDeleteCmd, []string{testIndexID}) })
+	})
+	require.Equal(t, -1, code)
+	require.Contains(t, out, testIndexID)
+	require.Len(t, svc.deleteIndexInputs, 1)
+}
+
+func TestRunOchreVectorIndexList_PrintsTable(t *testing.T) {
+	withVectorService(t, &fakeVectorService{list: []handlers_ochrevector.Record{
+		{ID: testIndexID, Name: "kb1", State: handlers_ochrevector.StateReady},
+	}}, nil)
+
+	var out string
+	code := withOchreExitCapture(t, func() {
+		out = captureStdout(t, func() { runOchreVectorIndexList(ochreVectorIndexListCmd, nil) })
+	})
+	require.Equal(t, -1, code)
+	require.Contains(t, out, testIndexID)
+}
+
+func TestRunOchreVectorIngest_PrintsJobID(t *testing.T) {
+	withVectorService(t, &fakeVectorService{ingest: handlers_ochrevector.IngestResponse{
+		Job: handlers_ochrevector.JobRecord{ID: "job-abc", IndexID: testIndexID, State: handlers_ochrevector.JobStatePending},
+	}}, nil)
+
+	cmd := *ochreVectorIngestCmd
+	require.NoError(t, cmd.Flags().Set("index", testIndexID))
+	require.NoError(t, cmd.Flags().Set("bucket", "docs"))
+
+	var out string
+	code := withOchreExitCapture(t, func() {
+		out = captureStdout(t, func() { runOchreVectorIngest(&cmd, nil) })
+	})
+	require.Equal(t, -1, code)
+	require.Contains(t, out, "job-abc")
+}
+
+func TestRunOchreVectorJobDescribe_PrintsRecord(t *testing.T) {
+	withVectorService(t, &fakeVectorService{describeJob: handlers_ochrevector.JobRecord{
+		ID: "job-abc", IndexID: testIndexID, State: handlers_ochrevector.JobStateReady,
+	}}, nil)
+
+	var out string
+	code := withOchreExitCapture(t, func() {
+		out = captureStdout(t, func() { runOchreVectorJobDescribe(ochreVectorJobDescribeCmd, []string{"job-abc"}) })
+	})
+	require.Equal(t, -1, code)
+	require.Contains(t, out, "job-abc")
+}
+
+func TestRunOchreVectorQuery_PrintsTable(t *testing.T) {
+	withVectorService(t, &fakeVectorService{query: []handlers_ochrevector.QueryResult{
+		{Chunk: "hello world", SourceKey: "docs/a.txt", Score: 0.9},
+	}}, nil)
+
+	cmd := *ochreVectorQueryCmd
+	require.NoError(t, cmd.Flags().Set("index", testIndexID))
+	require.NoError(t, cmd.Flags().Set("text", "hello"))
+
+	var out string
+	code := withOchreExitCapture(t, func() {
+		out = captureStdout(t, func() { runOchreVectorQuery(&cmd, nil) })
+	})
+	require.Equal(t, -1, code)
+	require.Contains(t, out, "docs/a.txt")
+}
+
+func TestRunOchreVectorQuery_ConnectFailureExits1(t *testing.T) {
+	withVectorService(t, nil, errors.New("dial nats: connection refused"))
+
+	cmd := *ochreVectorQueryCmd
+	require.NoError(t, cmd.Flags().Set("index", testIndexID))
+	require.NoError(t, cmd.Flags().Set("text", "hello"))
+
+	code := withOchreExitCapture(t, func() { runOchreVectorQuery(&cmd, nil) })
+	require.Equal(t, 1, code)
+}

cmd/spinifex/cmd/admin_preflight.go

@@ -19,6 +19,11 @@ has the privileged helper scripts and sudoers grants that build depends on —
 setup.sh and setup-ovn.sh install them, and a binary-swap deploy (make
 deploy or a manual binary copy) skips those steps.
 
+It also reports any running service still executing a replaced binary.
+Overwriting /usr/local/bin/spx while spinifex.target is up does not move the
+services onto it: each keeps executing the replaced inode until it restarts,
+so a node can serve a mixture of two builds indefinitely.
+
 Checks compare content, not mere existence, so a present-but-stale asset is
 caught too. Exits non-zero if any asset is Missing, Stale, or Ungranted.`,
 	RunE:         runAdminPreflight,

cmd/spinifex/cmd/admin_principal.go

@@ -0,0 +1,628 @@
+package cmd
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/mulgadc/spinifex/spinifex/admin"
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	"github.com/spf13/cobra"
+)
+
+// adminPrincipalPolicyName is the inline policy every admin principal carries.
+// One name for every principal is what makes the listing readable: a user with
+// this policy is an admin-API caller and its actions are the whole grant.
+const adminPrincipalPolicyName = "spx-admin-methods"
+
+// adminPrincipalService is the SigV4 credential scope the admin surface
+// requires, and the prefix each granted action carries.
+const adminPrincipalService = "spinifex"
+
+// runAdminPrincipalCreate provisions a named principal in the super-admin
+// account. Every step is create-if-absent except the access key, which is
+// replaced so exactly one key is ever live.
+func runAdminPrincipalCreate(cmd *cobra.Command, args []string) error {
+	userName := args[0]
+	if err := validatePrincipalName(userName); err != nil {
+		return err
+	}
+
+	requested, _ := cmd.Flags().GetStringSlice("grant")
+	grants, err := resolvePrincipalGrants(requested)
+	if err != nil {
+		return err
+	}
+
+	svc, _, _, cleanup, err := initIAMServiceFromConfig()
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+
+	accountID := admin.DefaultAccountID()
+
+	if _, err := svc.CreateUser(accountID, &iam.CreateUserInput{
+		UserName: aws.String(userName),
+	}); err != nil && !strings.Contains(err.Error(), "EntityAlreadyExists") {
+		return fmt.Errorf("create user %s: %w", userName, err)
+	}
+
+	if _, err := svc.PutUserPolicy(accountID, &iam.PutUserPolicyInput{
+		UserName:       aws.String(userName),
+		PolicyName:     aws.String(adminPrincipalPolicyName),
+		PolicyDocument: aws.String(principalPolicyDocument(grants)),
+	}); err != nil {
+		return fmt.Errorf("attach policy to %s: %w", userName, err)
+	}
+
+	dropped, err := dropStalePrincipalPolicies(svc, accountID, userName)
+	if err != nil {
+		return err
+	}
+	for _, name := range dropped {
+		fmt.Printf("Removed stale inline policy %s\n", name)
+	}
+
+	revoked, err := revokePrincipalKeys(svc, accountID, userName)
+	if err != nil {
+		return err
+	}
+	for _, keyID := range revoked {
+		fmt.Printf("Revoked previous access key %s\n", keyID)
+	}
+
+	akOut, err := svc.CreateAccessKey(accountID, &iam.CreateAccessKeyInput{
+		UserName: aws.String(userName),
+	})
+	if err != nil {
+		return fmt.Errorf("create access key for %s: %w", userName, err)
+	}
+	if akOut == nil || akOut.AccessKey == nil {
+		return fmt.Errorf("create access key for %s: no key returned", userName)
+	}
+
+	printPrincipalCredential(accountID, userName, grants,
+		aws.StringValue(akOut.AccessKey.AccessKeyId),
+		aws.StringValue(akOut.AccessKey.SecretAccessKey))
+	return nil
+}
+
+// printPrincipalCredential writes the one and only view of the secret, then the
+// two things a caller still needs: the SigV4 scope and the region the key is
+// useless without.
+func printPrincipalCredential(accountID, userName string, grants []string, keyID, secret string) {
+	fmt.Printf("\nPrincipal %q ready.\n", userName)
+	fmt.Printf("  Account ID:        %s\n", accountID)
+	fmt.Printf("  User:              %s\n", userName)
+	fmt.Printf("  Permitted actions: %s\n", strings.Join(grants, ", "))
+	fmt.Printf("  Access Key ID:     %s\n", keyID)
+	fmt.Printf("  Secret Access Key: %s\n", secret)
+	fmt.Println("\nThe secret is shown once and is not recoverable. Store it in a secret manager")
+	fmt.Println("or an AWS profile; never commit it and never place it in a Spinifex config file.")
+	fmt.Printf("\nRequests must be SigV4-signed with service=%q and this cluster's region.\n", adminPrincipalService)
+}
+
+// runAdminPrincipalList reports every principal in the super-admin account and
+// what it may call, so an operator can answer "who can delete an account" from
+// the cluster rather than from memory.
+func runAdminPrincipalList(_ *cobra.Command, _ []string) error {
+	svc, _, _, cleanup, err := initIAMServiceFromConfig()
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+
+	accountID := admin.DefaultAccountID()
+	users, err := svc.ListUsers(accountID, &iam.ListUsersInput{})
+	if err != nil {
+		return fmt.Errorf("list users: %w", err)
+	}
+
+	rows := make([]principalRow, 0, len(users.Users))
+	for _, user := range users.Users {
+		if user == nil || user.UserName == nil {
+			continue
+		}
+		row, err := describePrincipal(svc, accountID, aws.StringValue(user.UserName))
+		if err != nil {
+			return err
+		}
+		rows = append(rows, row)
+	}
+
+	printPrincipalTable(accountID, rows)
+	return nil
+}
+
+// principalRow is one line of the principal listing.
+type principalRow struct {
+	UserName string
+	Grants   []string
+	Keys     int
+}
+
+// describePrincipal resolves what a user in the super-admin account may call.
+// An attached wildcard policy is reported as such rather than expanded: the
+// point of the listing is to show which principals are unscoped.
+func describePrincipal(svc handlers_iam.IAMService, accountID, userName string) (principalRow, error) {
+	row := principalRow{UserName: userName}
+
+	attached, err := svc.ListAttachedUserPolicies(accountID, &iam.ListAttachedUserPoliciesInput{
+		UserName: aws.String(userName),
+	})
+	if err != nil {
+		return row, fmt.Errorf("list attached policies for %s: %w", userName, err)
+	}
+	if attached != nil {
+		for _, policy := range attached.AttachedPolicies {
+			if policy == nil {
+				continue
+			}
+			row.Grants = append(row.Grants, aws.StringValue(policy.PolicyName)+" (attached)")
+		}
+	}
+
+	inline, err := svc.GetUserPolicy(accountID, &iam.GetUserPolicyInput{
+		UserName:   aws.String(userName),
+		PolicyName: aws.String(adminPrincipalPolicyName),
+	})
+	if err == nil && inline != nil {
+		row.Grants = append(row.Grants, principalPolicyActions(aws.StringValue(inline.PolicyDocument))...)
+	}
+
+	keys, err := svc.ListAccessKeys(accountID, &iam.ListAccessKeysInput{UserName: aws.String(userName)})
+	if err != nil {
+		return row, fmt.Errorf("list access keys for %s: %w", userName, err)
+	}
+	if keys != nil {
+		row.Keys = len(keys.AccessKeyMetadata)
+	}
+	return row, nil
+}
+
+// principalPolicyActions returns the actions an inline policy allows, shortened
+// to the method name. An undecodable document reports nothing rather than
+// guessing, and the listing shows the user with no grants.
+func principalPolicyActions(document string) []string {
+	var doc struct {
+		Statement []struct {
+			Effect string                   `json:"Effect"`
+			Action handlers_iam.StringOrArr `json:"Action"`
+		} `json:"Statement"`
+	}
+	if err := json.Unmarshal([]byte(document), &doc); err != nil {
+		return nil
+	}
+
+	var actions []string
+	for _, stmt := range doc.Statement {
+		if stmt.Effect != "Allow" {
+			continue
+		}
+		for _, action := range stmt.Action {
+			actions = append(actions, strings.TrimPrefix(action, adminPrincipalService+":"))
+		}
+	}
+	sort.Strings(actions)
+	return actions
+}
+
+// printPrincipalTable renders the listing.
+func printPrincipalTable(accountID string, rows []principalRow) {
+	fmt.Printf("\nPrincipals in the super-admin account (%s):\n\n", accountID)
+	fmt.Printf("%-24s %-6s %s\n", "USER", "KEYS", "GRANTS")
+	fmt.Println(strings.Repeat("-", 96))
+	for _, row := range rows {
+		grants := strings.Join(row.Grants, ", ")
+		if grants == "" {
+			grants = "-"
+		}
+		fmt.Printf("%-24s %-6d %s\n", row.UserName, row.Keys, grants)
+	}
+	fmt.Printf("\n%d principal(s)\n", len(rows))
+}
+
+// runAdminPrincipalRevoke removes every access key a principal holds, leaving
+// the user and its policy in place. This is the response to a leaked secret:
+// it takes effect cluster-wide immediately and needs no restart.
+func runAdminPrincipalRevoke(_ *cobra.Command, args []string) error {
+	userName := args[0]
+
+	svc, _, _, cleanup, err := initIAMServiceFromConfig()
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+
+	accountID := admin.DefaultAccountID()
+	if _, err := svc.GetUser(accountID, &iam.GetUserInput{UserName: aws.String(userName)}); err != nil {
+		return fmt.Errorf("no principal %q in account %s", userName, accountID)
+	}
+
+	revoked, err := revokePrincipalKeys(svc, accountID, userName)
+	if err != nil {
+		return err
+	}
+	if len(revoked) == 0 {
+		fmt.Printf("Principal %q holds no access keys.\n", userName)
+		return nil
+	}
+	for _, keyID := range revoked {
+		fmt.Printf("Revoked access key %s\n", keyID)
+	}
+	fmt.Printf("\nPrincipal %q can no longer sign requests. Re-issue with:\n", userName)
+	fmt.Printf("  spx admin principal create %s\n", userName)
+	return nil
+}
+
+// dropStalePrincipalPolicies removes every inline policy on the principal that
+// this command did not write. A principal's grants are exactly one policy, so
+// re-creating one with narrower grants cannot leave a wider policy behind.
+func dropStalePrincipalPolicies(svc handlers_iam.IAMService, accountID, userName string) ([]string, error) {
+	listed, err := svc.ListUserPolicies(accountID, &iam.ListUserPoliciesInput{
+		UserName: aws.String(userName),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("list inline policies for %s: %w", userName, err)
+	}
+	if listed == nil {
+		return nil, nil
+	}
+
+	var dropped []string
+	for _, name := range listed.PolicyNames {
+		policyName := aws.StringValue(name)
+		if policyName == "" || policyName == adminPrincipalPolicyName {
+			continue
+		}
+		if _, err := svc.DeleteUserPolicy(accountID, &iam.DeleteUserPolicyInput{
+			UserName:   aws.String(userName),
+			PolicyName: name,
+		}); err != nil {
+			return nil, fmt.Errorf("remove inline policy %s from %s: %w", policyName, userName, err)
+		}
+		dropped = append(dropped, policyName)
+	}
+	return dropped, nil
+}
+
+// runAdminPrincipalDelete removes a principal entirely: its keys, its policies
+// and the user. Revoking leaves a principal that can be re-issued; this is for
+// one that should not exist at all.
+func runAdminPrincipalDelete(_ *cobra.Command, args []string) error {
+	userName := args[0]
+	if err := validatePrincipalName(userName); err != nil {
+		return err
+	}
+
+	svc, _, _, cleanup, err := initIAMServiceFromConfig()
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+
+	accountID := admin.DefaultAccountID()
+	if _, err := svc.GetUser(accountID, &iam.GetUserInput{UserName: aws.String(userName)}); err != nil {
+		return fmt.Errorf("no principal %q in account %s", userName, accountID)
+	}
+
+	revoked, err := revokePrincipalKeys(svc, accountID, userName)
+	if err != nil {
+		return err
+	}
+	for _, keyID := range revoked {
+		fmt.Printf("Revoked access key %s\n", keyID)
+	}
+
+	policies, err := svc.ListUserPolicies(accountID, &iam.ListUserPoliciesInput{
+		UserName: aws.String(userName),
+	})
+	if err != nil {
+		return fmt.Errorf("list inline policies for %s: %w", userName, err)
+	}
+	if policies != nil {
+		for _, name := range policies.PolicyNames {
+			if _, err := svc.DeleteUserPolicy(accountID, &iam.DeleteUserPolicyInput{
+				UserName:   aws.String(userName),
+				PolicyName: name,
+			}); err != nil {
+				return fmt.Errorf("remove inline policy %s from %s: %w",
+					aws.StringValue(name), userName, err)
+			}
+			fmt.Printf("Removed inline policy %s\n", aws.StringValue(name))
+		}
+	}
+
+	if _, err := svc.DeleteUser(accountID, &iam.DeleteUserInput{
+		UserName: aws.String(userName),
+	}); err != nil {
+		return fmt.Errorf("delete user %s: %w", userName, err)
+	}
+
+	fmt.Printf("\nPrincipal %q removed from account %s.\n", userName, accountID)
+	return nil
+}
+
+// revokePrincipalKeys deletes every access key the user holds and returns the
+// IDs. A key from an earlier run is unrecoverable to its holder but still
+// authenticates, so it is removed rather than left live alongside a new one.
+func revokePrincipalKeys(svc handlers_iam.IAMService, accountID, userName string) ([]string, error) {
+	listed, err := svc.ListAccessKeys(accountID, &iam.ListAccessKeysInput{
+		UserName: aws.String(userName),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("list access keys for %s: %w", userName, err)
+	}
+	if listed == nil {
+		return nil, nil
+	}
+
+	var revoked []string
+	for _, meta := range listed.AccessKeyMetadata {
+		if meta == nil || meta.AccessKeyId == nil {
+			continue
+		}
+		if _, err := svc.DeleteAccessKey(accountID, &iam.DeleteAccessKeyInput{
+			UserName:    aws.String(userName),
+			AccessKeyId: meta.AccessKeyId,
+		}); err != nil {
+			return nil, fmt.Errorf("remove access key for %s: %w", userName, err)
+		}
+		revoked = append(revoked, aws.StringValue(meta.AccessKeyId))
+	}
+	return revoked, nil
+}
+
+// resolvePrincipalGrants turns the requested method names into the actions the
+// policy allows. An empty request means every admin method: a principal minted
+// without a --grant is the operator credential.
+func resolvePrincipalGrants(requested []string) ([]string, error) {
+	known := gateway.AdminMethodNames()
+	if len(requested) == 0 {
+		return prefixGrants(known), nil
+	}
+
+	byLower := make(map[string]string, len(known))
+	for _, method := range known {
+		byLower[strings.ToLower(method)] = method
+	}
+
+	seen := make(map[string]bool, len(requested))
+	methods := make([]string, 0, len(requested))
+	for _, entry := range requested {
+		name := strings.TrimSpace(entry)
+		name = strings.TrimPrefix(name, adminPrincipalService+":")
+		method, ok := byLower[strings.ToLower(name)]
+		if !ok {
+			return nil, fmt.Errorf("unknown admin method %q — the surface serves: %s",
+				entry, strings.Join(known, ", "))
+		}
+		if seen[method] {
+			continue
+		}
+		seen[method] = true
+		methods = append(methods, method)
+	}
+
+	sort.Strings(methods)
+	return prefixGrants(methods), nil
+}
+
+// prefixGrants qualifies method names with the service scope the gateway
+// evaluates them under.
+func prefixGrants(methods []string) []string {
+	grants := make([]string, 0, len(methods))
+	for _, method := range methods {
+		grants = append(grants, adminPrincipalService+":"+method)
+	}
+	return grants
+}
+
+// principalPolicyDocument builds the inline policy. Every method is named:
+// granting spinifex:* would authorise whatever is added to the admin surface
+// next with a key minted before it existed.
+func principalPolicyDocument(grants []string) string {
+	quoted := make([]string, 0, len(grants))
+	for _, grant := range grants {
+		quoted = append(quoted, `"`+grant+`"`)
+	}
+	return `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":[` +
+		strings.Join(quoted, ",") + `],"Resource":"*"}]}`
+}
+
+// validatePrincipalName rejects names that would collide with the account's own
+// administrator or that IAM would not accept.
+func validatePrincipalName(name string) error {
+	if name == "" {
+		return errors.New("principal name is required")
+	}
+	if name == handlers_iam.AdminUserName {
+		return fmt.Errorf("%q is the super-admin account's own administrator — "+
+			"rotating its key would lock out the bootstrap credential; choose another name",
+			handlers_iam.AdminUserName)
+	}
+	if len(name) > 64 {
+		return errors.New("principal name must be 64 characters or fewer")
+	}
+	for _, r := range name {
+		switch {
+		case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
+		case r == '-', r == '_', r == '.', r == '+', r == '=', r == ',', r == '@':
+		default:
+			return fmt.Errorf("principal name %q contains an unsupported character %q", name, r)
+		}
+	}
+	return nil
+}
+
+// runAdminPrincipalAudit reports roles in the super-admin account that any
+// principal in the account can assume, which includes every admin principal.
+func runAdminPrincipalAudit(_ *cobra.Command, _ []string) error {
+	svc, _, _, cleanup, err := initIAMServiceFromConfig()
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+
+	accountID := admin.DefaultAccountID()
+	roles, err := svc.ListRoles(accountID, &iam.ListRolesInput{})
+	if err != nil {
+		return fmt.Errorf("list roles: %w", err)
+	}
+
+	var flagged []string
+	for _, role := range roles.Roles {
+		if role == nil || role.AssumeRolePolicyDocument == nil {
+			continue
+		}
+		if trustsWholeAccount(aws.StringValue(role.AssumeRolePolicyDocument), accountID) {
+			flagged = append(flagged, aws.StringValue(role.RoleName))
+		}
+	}
+
+	if len(flagged) == 0 {
+		fmt.Printf("No roles in %s trust the account as a whole.\n", accountID)
+		return nil
+	}
+
+	fmt.Fprintf(os.Stderr, "Roles in %s assumable by any principal in the account, including every admin principal:\n", accountID)
+	for _, name := range flagged {
+		fmt.Fprintf(os.Stderr, "  %s\n", name)
+	}
+	return fmt.Errorf("%d role(s) trust the account as a whole — scope their trust policies to a specific principal", len(flagged))
+}
+
+// trustsWholeAccount reports whether an AssumeRolePolicyDocument admits every
+// principal in accountID. A malformed document is reported as trusting, since
+// a document nobody can parse is not a document anybody has verified.
+func trustsWholeAccount(document, accountID string) bool {
+	doc, err := handlers_iam.ValidateTrustPolicyDocument(document)
+	if err != nil {
+		return true
+	}
+	for _, stmt := range doc.Statement {
+		if stmt.Effect != "Allow" {
+			continue
+		}
+		var principal struct {
+			AWS handlers_iam.StringOrArr `json:"AWS"`
+		}
+		if err := json.Unmarshal(stmt.Principal, &principal); err != nil {
+			continue
+		}
+		for _, entry := range principal.AWS {
+			switch entry {
+			case "*", accountID, "arn:aws:iam::" + accountID + ":root":
+				return true
+			}
+		}
+	}
+	return false
+}
+
+var adminPrincipalCmd = &cobra.Command{
+	Use:   "principal",
+	Short: "Manage the credentials that call the private admin API",
+	Long: `Manage the IAM users in the super-admin account that sign POST /admin/<Method>
+requests.
+
+The private admin API creates, lists and deletes tenant accounts. It is not an
+AWS API and is unreachable until a principal exists — there is no config toggle
+and no default credential for it.`,
+}
+
+var adminPrincipalCreateCmd = &cobra.Command{
+	Use:   "create <name>",
+	Short: "Create an admin principal and print its access key once",
+	Long: `Create an IAM user in the super-admin account with an inline policy allowing the
+requested admin methods, then mint an access key.
+
+Without --grant the principal may call every admin method, which is the
+credential an operator or a test harness uses. Pass --grant to scope it: a
+self-service signup form, for example, needs only CreateAccount, and a key with
+that grant cannot delete anything if the form's host is compromised.
+
+Each method is granted by name rather than with a wildcard, so a later addition
+to the admin surface is not authorised by an existing key.
+
+The principal's grants are exactly one inline policy: re-running with fewer
+--grant values narrows it, and any other inline policy on the user is removed.
+
+The secret is printed once and never recoverable. Re-running replaces the access
+key, which revokes the previous one.
+
+  spx admin principal create operator
+  spx admin principal create signup --grant CreateAccount
+  spx admin principal create auditor --grant ListAccounts,DescribeAccountDeletion`,
+	Args:         cobra.ExactArgs(1),
+	RunE:         runAdminPrincipalCreate,
+	SilenceUsage: true,
+}
+
+var adminPrincipalListCmd = &cobra.Command{
+	Use:          "list",
+	Short:        "List the principals in the super-admin account and what they may call",
+	Args:         cobra.NoArgs,
+	RunE:         runAdminPrincipalList,
+	SilenceUsage: true,
+}
+
+var adminPrincipalRevokeCmd = &cobra.Command{
+	Use:   "revoke <name>",
+	Short: "Revoke a principal's access keys without removing the principal",
+	Long: `Delete every access key the named principal holds. The user and its policy stay,
+so re-issuing is one "principal create" away.
+
+Revocation is immediate and cluster-wide.`,
+	Args:         cobra.ExactArgs(1),
+	RunE:         runAdminPrincipalRevoke,
+	SilenceUsage: true,
+}
+
+var adminPrincipalDeleteCmd = &cobra.Command{
+	Use:   "delete <name>",
+	Short: "Remove a principal, its policies and its access keys",
+	Long: `Delete the named principal from the super-admin account entirely.
+
+Use "principal revoke" instead to stop a credential working while keeping the
+principal, which is what a rotation after a leak needs.`,
+	Args:         cobra.ExactArgs(1),
+	RunE:         runAdminPrincipalDelete,
+	SilenceUsage: true,
+}
+
+var adminPrincipalAuditCmd = &cobra.Command{
+	Use:   "audit",
+	Short: "Flag roles an admin principal could assume",
+	Long: `Report roles in the super-admin account whose trust policy names the account,
+its root ARN, or a wildcard.
+
+STS does not evaluate the caller's identity policy on AssumeRole, so any such
+role is assumable by a scoped admin principal, which then inherits its
+permissions. Roles trusting a service principal are unaffected.
+
+Exits non-zero if any role is flagged.`,
+	Args:         cobra.NoArgs,
+	RunE:         runAdminPrincipalAudit,
+	SilenceUsage: true,
+}
+
+func init() {
+	adminCmd.AddCommand(adminPrincipalCmd)
+	adminPrincipalCmd.AddCommand(adminPrincipalCreateCmd)
+	adminPrincipalCmd.AddCommand(adminPrincipalListCmd)
+	adminPrincipalCmd.AddCommand(adminPrincipalRevokeCmd)
+	adminPrincipalCmd.AddCommand(adminPrincipalDeleteCmd)
+	adminPrincipalCmd.AddCommand(adminPrincipalAuditCmd)
+
+	adminPrincipalCreateCmd.Flags().StringSlice("grant", nil,
+		"Admin methods this principal may call (default: all of "+
+			strings.Join(gateway.AdminMethodNames(), ", ")+")")
+}

cmd/spinifex/cmd/admin_principal_iam_test.go

@@ -0,0 +1,143 @@
+package cmd_test
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const superAdminAccountID = "000000000001"
+
+// principalIAMStub embeds the IAMService interface so only the calls the
+// principal commands make are implemented. Anything else nil-panics, which is
+// the point: an unexpected call is a change in behaviour, not a passing test.
+type principalIAMStub struct {
+	handlers_iam.IAMService
+
+	inlinePolicies   []string
+	inlineDocument   string
+	attachedPolicies []string
+	accessKeys       []string
+
+	deletedPolicies []string
+	deletedKeys     []string
+	listKeysErr     error
+}
+
+func (s *principalIAMStub) ListUserPolicies(_ string, _ *iam.ListUserPoliciesInput) (*iam.ListUserPoliciesOutput, error) {
+	return &iam.ListUserPoliciesOutput{PolicyNames: aws.StringSlice(s.inlinePolicies)}, nil
+}
+
+func (s *principalIAMStub) DeleteUserPolicy(_ string, input *iam.DeleteUserPolicyInput) (*iam.DeleteUserPolicyOutput, error) {
+	s.deletedPolicies = append(s.deletedPolicies, aws.StringValue(input.PolicyName))
+	return &iam.DeleteUserPolicyOutput{}, nil
+}
+
+func (s *principalIAMStub) GetUserPolicy(_ string, input *iam.GetUserPolicyInput) (*iam.GetUserPolicyOutput, error) {
+	if s.inlineDocument == "" {
+		return nil, errors.New("NoSuchEntity")
+	}
+	return &iam.GetUserPolicyOutput{
+		PolicyName:     input.PolicyName,
+		PolicyDocument: aws.String(s.inlineDocument),
+	}, nil
+}
+
+func (s *principalIAMStub) ListAttachedUserPolicies(_ string, _ *iam.ListAttachedUserPoliciesInput) (*iam.ListAttachedUserPoliciesOutput, error) {
+	out := &iam.ListAttachedUserPoliciesOutput{}
+	for _, name := range s.attachedPolicies {
+		out.AttachedPolicies = append(out.AttachedPolicies,
+			&iam.AttachedPolicy{PolicyName: aws.String(name)})
+	}
+	return out, nil
+}
+
+func (s *principalIAMStub) ListAccessKeys(_ string, _ *iam.ListAccessKeysInput) (*iam.ListAccessKeysOutput, error) {
+	if s.listKeysErr != nil {
+		return nil, s.listKeysErr
+	}
+	out := &iam.ListAccessKeysOutput{}
+	for _, id := range s.accessKeys {
+		out.AccessKeyMetadata = append(out.AccessKeyMetadata,
+			&iam.AccessKeyMetadata{AccessKeyId: aws.String(id)})
+	}
+	return out, nil
+}
+
+func (s *principalIAMStub) DeleteAccessKey(_ string, input *iam.DeleteAccessKeyInput) (*iam.DeleteAccessKeyOutput, error) {
+	s.deletedKeys = append(s.deletedKeys, aws.StringValue(input.AccessKeyId))
+	return &iam.DeleteAccessKeyOutput{}, nil
+}
+
+// Re-creating a principal with fewer grants has to narrow it. A policy left
+// from an earlier name would keep authorising what the operator just removed.
+func TestCreateDropsAnInlinePolicyItDidNotWrite(t *testing.T) {
+	svc := &principalIAMStub{inlinePolicies: []string{
+		"operator-admin-methods", cmd.AdminPrincipalPolicyName, "signup-createaccount",
+	}}
+
+	dropped, err := cmd.DropStalePrincipalPolicies(svc, superAdminAccountID, "operator")
+	require.NoError(t, err)
+
+	assert.Equal(t, []string{"operator-admin-methods", "signup-createaccount"}, dropped)
+	assert.NotContains(t, svc.deletedPolicies, cmd.AdminPrincipalPolicyName,
+		"the policy the command owns must survive its own rewrite")
+}
+
+// Every key goes, not just the first: two live keys means two holders, and a
+// rotation that leaves one behind has revoked nothing.
+func TestRevokeRemovesEveryAccessKey(t *testing.T) {
+	svc := &principalIAMStub{accessKeys: []string{"AKIAONE", "AKIATWO"}}
+
+	revoked, err := cmd.RevokePrincipalKeys(svc, superAdminAccountID, "operator")
+	require.NoError(t, err)
+
+	assert.Equal(t, []string{"AKIAONE", "AKIATWO"}, revoked)
+	assert.Equal(t, []string{"AKIAONE", "AKIATWO"}, svc.deletedKeys)
+}
+
+// A listing failure must stop the rotation. Minting a second key while an
+// unknown number are live is the opposite of what create promises.
+func TestRevokeFailsRatherThanGuessing(t *testing.T) {
+	svc := &principalIAMStub{listKeysErr: errors.New("kv unavailable")}
+
+	_, err := cmd.RevokePrincipalKeys(svc, superAdminAccountID, "operator")
+
+	require.Error(t, err)
+	assert.Empty(t, svc.deletedKeys)
+}
+
+// The listing is how an operator answers "who can delete an account", so it
+// has to report the scoped grants by method name.
+func TestDescribePrincipalReportsScopedGrants(t *testing.T) {
+	svc := &principalIAMStub{
+		inlineDocument: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow",` +
+			`"Action":["spinifex:ListAccounts","spinifex:CreateAccount"],"Resource":"*"}]}`,
+		accessKeys: []string{"AKIAONE"},
+	}
+
+	row, err := cmd.DescribePrincipal(svc, superAdminAccountID, "signup")
+	require.NoError(t, err)
+
+	assert.Equal(t, "signup", row.UserName)
+	assert.Equal(t, []string{"CreateAccount", "ListAccounts"}, row.Grants)
+	assert.Equal(t, 1, row.Keys)
+}
+
+// An attached wildcard policy is reported rather than expanded: an unscoped
+// principal is exactly what the listing exists to surface.
+func TestDescribePrincipalMarksAnAttachedPolicy(t *testing.T) {
+	svc := &principalIAMStub{attachedPolicies: []string{handlers_iam.AdminPolicyName}}
+
+	row, err := cmd.DescribePrincipal(svc, superAdminAccountID, handlers_iam.AdminUserName)
+	require.NoError(t, err)
+
+	assert.Equal(t, []string{handlers_iam.AdminPolicyName + " (attached)"}, row.Grants)
+	assert.Zero(t, row.Keys)
+}

cmd/spinifex/cmd/admin_principal_test.go

@@ -0,0 +1,193 @@
+package cmd_test
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// policyActions returns the actions an inline policy document allows.
+func policyActions(t *testing.T, document string) []string {
+	t.Helper()
+
+	var doc struct {
+		Statement []struct {
+			Effect string                   `json:"Effect"`
+			Action handlers_iam.StringOrArr `json:"Action"`
+		} `json:"Statement"`
+	}
+	require.NoError(t, json.Unmarshal([]byte(document), &doc))
+
+	var actions []string
+	for _, statement := range doc.Statement {
+		if statement.Effect != "Allow" {
+			continue
+		}
+		actions = append(actions, statement.Action...)
+	}
+	return actions
+}
+
+// A principal minted without --grant is the operator credential, so it has to
+// cover the surface the gateway actually serves rather than a list that can
+// fall behind it.
+func TestPrincipalGrantsDefaultToEveryAdminMethod(t *testing.T) {
+	grants, err := cmd.ResolvePrincipalGrants(nil)
+	require.NoError(t, err)
+
+	var want []string
+	for _, method := range gateway.AdminMethodNames() {
+		want = append(want, "spinifex:"+method)
+	}
+	assert.ElementsMatch(t, want, grants)
+}
+
+// Scoping is the whole point of the flag: a signup credential holds exactly
+// CreateAccount, so a leaked key cannot remove a tenant.
+func TestPrincipalGrantsScopeToTheRequestedMethods(t *testing.T) {
+	grants, err := cmd.ResolvePrincipalGrants([]string{"CreateAccount"})
+	require.NoError(t, err)
+
+	assert.Equal(t, []string{"spinifex:CreateAccount"}, grants)
+}
+
+// Operators write the method either way, and a duplicate is a typo rather than
+// a reason to refuse.
+func TestPrincipalGrantsAcceptQualifiedAndRepeatedNames(t *testing.T) {
+	grants, err := cmd.ResolvePrincipalGrants([]string{
+		"spinifex:ListAccounts", "listaccounts", " DeleteAccount ",
+	})
+	require.NoError(t, err)
+
+	assert.Equal(t, []string{"spinifex:DeleteAccount", "spinifex:ListAccounts"}, grants)
+}
+
+// A misspelt method must fail rather than mint a key that silently authorises
+// nothing — the caller would only find out at the first AccessDenied.
+func TestPrincipalGrantsRejectAnUnknownMethod(t *testing.T) {
+	_, err := cmd.ResolvePrincipalGrants([]string{"DeleteEverything"})
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "DeleteEverything")
+	assert.Contains(t, err.Error(), "CreateAccount")
+}
+
+// A wildcard here would authorise whatever is added to the surface next with a
+// key minted before it existed.
+func TestPrincipalPolicyNamesEveryMethod(t *testing.T) {
+	grants, err := cmd.ResolvePrincipalGrants(nil)
+	require.NoError(t, err)
+
+	actions := policyActions(t, cmd.PrincipalPolicyDocument(grants))
+
+	assert.ElementsMatch(t, grants, actions)
+	for _, action := range actions {
+		assert.NotContains(t, action, "*", "an admin grant must name its method")
+	}
+}
+
+// The listing reads back what the create path wrote, so a principal's grants
+// can be audited without decoding the document by hand.
+func TestPrincipalPolicyActionsRoundTrip(t *testing.T) {
+	grants, err := cmd.ResolvePrincipalGrants([]string{"CreateAccount", "ListAccounts"})
+	require.NoError(t, err)
+
+	assert.Equal(t, []string{"CreateAccount", "ListAccounts"},
+		cmd.PrincipalPolicyActions(cmd.PrincipalPolicyDocument(grants)))
+}
+
+// A document nobody can parse is reported as no grants rather than as a guess.
+func TestPrincipalPolicyActionsIgnoreAnUndecodableDocument(t *testing.T) {
+	assert.Empty(t, cmd.PrincipalPolicyActions("{not json"))
+}
+
+// Rotating the super-admin account's own administrator would revoke the
+// bootstrap credential the operator is holding.
+func TestPrincipalNameRefusesTheAccountAdministrator(t *testing.T) {
+	err := cmd.ValidatePrincipalName(handlers_iam.AdminUserName)
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), handlers_iam.AdminUserName)
+}
+
+func TestPrincipalNameRejectsUnusableNames(t *testing.T) {
+	assert.Error(t, cmd.ValidatePrincipalName(""))
+	assert.Error(t, cmd.ValidatePrincipalName("has space"))
+	assert.Error(t, cmd.ValidatePrincipalName(strings.Repeat("a", 65)))
+
+	assert.NoError(t, cmd.ValidatePrincipalName("signup"))
+	assert.NoError(t, cmd.ValidatePrincipalName("ops-team_1@example.com"))
+}
+
+// The listing prints one line per principal, including a revoked credential
+// that holds no key — an operator has to see it to know it is still there.
+func TestPrintPrincipalTableRendersEveryPrincipal(t *testing.T) {
+	output := captureOutput(t, func() {
+		cmd.PrintPrincipalTable("000000000001", []cmd.PrincipalRowSpec{
+			{UserName: "admin", Grants: []string{"AdministratorAccess (attached)"}, Keys: 1},
+			{UserName: "signup", Grants: []string{"CreateAccount"}, Keys: 1},
+			{UserName: "retired", Keys: 0},
+		})
+	})
+
+	assert.Contains(t, output, "000000000001")
+	assert.Contains(t, output, "AdministratorAccess (attached)")
+	assert.Contains(t, output, "signup")
+	assert.Contains(t, output, "retired")
+	assert.Contains(t, output, "3 principal(s)")
+}
+
+// A principal with nothing granted must read as a dash. A blank cell reads as
+// a rendering fault rather than as an answer.
+func TestPrintPrincipalTableMarksAnEmptyGrant(t *testing.T) {
+	output := captureOutput(t, func() {
+		cmd.PrintPrincipalTable("000000000001", []cmd.PrincipalRowSpec{{UserName: "retired"}})
+	})
+
+	assert.Contains(t, output, "-")
+}
+
+// The local and remote listings print through one function, so a tenant that is
+// TERMINATING reads the same either way — which is how a stuck teardown is
+// noticed.
+func TestAccountSummariesCarryStatus(t *testing.T) {
+	summaries := cmd.AccountSummaries([]*handlers_iam.Account{
+		{AccountID: "000000000042", AccountName: "t@example.com", Status: handlers_iam.AccountStatusActive},
+		nil,
+		{AccountID: "000000000043", AccountName: "u@example.com", Status: handlers_iam.AccountStatusTerminating},
+	})
+
+	require.Len(t, summaries, 2)
+	assert.Equal(t, handlers_iam.AccountStatusTerminating, summaries[1].Status)
+}
+
+func TestPrintAccountTableRendersEveryAccount(t *testing.T) {
+	output := captureOutput(t, func() {
+		cmd.PrintAccountTable([]gateway.AccountSummary{
+			{
+				AccountID:   "000000000042",
+				AccountName: "tenant@example.com",
+				Status:      handlers_iam.AccountStatusTerminating,
+				CreatedAt:   "2026-08-16T07:00:00Z",
+			},
+		})
+	})
+
+	assert.Contains(t, output, "000000000042")
+	assert.Contains(t, output, "tenant@example.com")
+	assert.Contains(t, output, handlers_iam.AccountStatusTerminating)
+	assert.Contains(t, output, "2026-08-16 07:00")
+}
+
+// An empty listing must say so. A bare header reads as truncated output.
+func TestPrintAccountTableSaysWhenThereAreNone(t *testing.T) {
+	output := captureOutput(t, func() { cmd.PrintAccountTable(nil) })
+
+	assert.Contains(t, output, "No accounts found")
+}

cmd/spinifex/cmd/admin_remote.go

@@ -0,0 +1,299 @@
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/gwsign"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+// adminRequestTimeout bounds a remote /admin call. Provisioning waits on the
+// default VPC, so it is comfortably longer than the gateway's own wait.
+const adminRequestTimeout = 90 * time.Second
+
+// adminMaxResponseBytes caps the response read. Most answers are a few hundred
+// bytes; a teardown inventory is one line per resource, so the cap is sized for
+// a large account rather than for the common case.
+const adminMaxResponseBytes = 1 << 20
+
+// retryableAdminErrors are the only codes worth retrying, and only with the
+// same client token. Suggesting a retry for the rest invites a caller to send
+// a fresh token, which is what produces a duplicate account.
+var retryableAdminErrors = map[string]bool{
+	awserrors.ErrorOperationInProgress: true,
+	awserrors.ErrorServiceUnavailable:  true,
+	awserrors.ErrorInternalError:       true,
+}
+
+func init() {
+	accountCreateCmd.Flags().Bool("remote", false, "Create the account over POST /admin/CreateAccount instead of connecting to NATS")
+	accountCreateCmd.Flags().String("endpoint", "", "Gateway endpoint for --remote (default: this node's AWS gateway)")
+	accountCreateCmd.Flags().String("region", "", "SigV4 region for --remote (default: this node's region)")
+	accountCreateCmd.Flags().String("ca-bundle", "", "CA certificate for --remote (default: this node's CA)")
+	accountCreateCmd.Flags().String("client-token", "", "Idempotency token for --remote (default: generated; reuse it to retry)")
+	accountCreateCmd.Flags().String("source", "spx-cli", "Provenance tag recorded in the gateway log for --remote")
+
+	accountListCmd.Flags().Bool("remote", false, "List over POST /admin/ListAccounts instead of connecting to NATS")
+	accountListCmd.Flags().String("endpoint", "", "Gateway endpoint for --remote (default: this node's AWS gateway)")
+	accountListCmd.Flags().String("region", "", "SigV4 region for --remote (default: this node's region)")
+	accountListCmd.Flags().String("ca-bundle", "", "CA certificate for --remote (default: this node's CA)")
+}
+
+// runAccountCreateRemote drives POST /admin/CreateAccount, exercising the same
+// path a self-service signup form uses. Credentials come from the standard AWS
+// chain (env vars or AWS_PROFILE), so this never reads the cluster master key.
+func runAccountCreateRemote(cmd *cobra.Command, name string) {
+	ctx, cancel := context.WithTimeout(context.Background(), adminRequestTimeout)
+	defer cancel()
+
+	clientToken, _ := cmd.Flags().GetString("client-token")
+	source, _ := cmd.Flags().GetString("source")
+
+	target, err := resolveAdminTarget(cmd)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	if clientToken == "" {
+		clientToken = newClientToken()
+	}
+
+	out, err := createAccountRemote(ctx, target,
+		gateway.CreateAccountRequest{Name: name, ClientToken: clientToken, Source: source})
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		var adminErr *adminError
+		if errors.As(err, &adminErr) && retryableAdminErrors[adminErr.Code] {
+			fmt.Fprintf(os.Stderr, "Retry with --client-token %s to resume; a new token would create a second account.\n", clientToken)
+		}
+		os.Exit(1)
+	}
+
+	fmt.Println("\nAccount created successfully!")
+	fmt.Printf("  Account ID:        %s\n", out.AccountID)
+	fmt.Printf("  Account Name:      %s\n", out.AccountName)
+	fmt.Printf("  Admin User:        %s\n", out.AdminUser)
+	fmt.Printf("  Access Key ID:     %s\n", out.AccessKeyID)
+	fmt.Printf("  Secret Access Key: %s\n", out.SecretAccessKey)
+	fmt.Printf("  Default VPC:       %s\n", out.DefaultVpcID)
+	fmt.Printf("  Client Token:      %s\n", clientToken)
+}
+
+// adminTarget is where a remote /admin call goes and how it is signed.
+type adminTarget struct {
+	endpoint string
+	region   string
+	caBundle string
+}
+
+// resolveAdminTarget fills the endpoint, region and CA from this node's config
+// when the command runs on a cluster member. Off-cluster callers pass flags,
+// which always win over what the local config says.
+func resolveAdminTarget(cmd *cobra.Command) (adminTarget, error) {
+	endpoint, _ := cmd.Flags().GetString("endpoint")
+	region, _ := cmd.Flags().GetString("region")
+	caBundle, _ := cmd.Flags().GetString("ca-bundle")
+
+	if endpoint == "" || region == "" || caBundle == "" {
+		if cfg, err := loadLocalConfig(); err == nil {
+			node := cfg.Nodes[cfg.Node]
+			if endpoint == "" {
+				endpoint = localGatewayEndpoint(node)
+			}
+			if region == "" {
+				region = node.Region
+			}
+			if caBundle == "" {
+				caBundle = filepath.Join(cfg.NodeBaseDir(), "config", "ca.pem")
+			}
+		}
+	}
+	if endpoint == "" || region == "" {
+		return adminTarget{}, errors.New("--endpoint and --region are required when no local node config is available")
+	}
+	return adminTarget{endpoint: endpoint, region: region, caBundle: caBundle}, nil
+}
+
+// adminError is a structured error from the /admin surface. The code is
+// carried separately from the message so the caller can decide whether a
+// retry is safe without matching on text.
+type adminError struct {
+	Code       string
+	Message    string
+	RequestID  string
+	StatusCode int
+}
+
+func (e *adminError) Error() string {
+	return fmt.Sprintf("%s: %s (HTTP %d, requestId %s)", e.Code, e.Message, e.StatusCode, e.RequestID)
+}
+
+// createAccountRemote signs and sends POST /admin/CreateAccount. Credentials
+// come from the standard AWS chain, so this never reads the cluster master
+// key — it is the same request a self-service signup form makes.
+func createAccountRemote(ctx context.Context, target adminTarget, req gateway.CreateAccountRequest) (*gateway.CreateAccountResponse, error) {
+	var out gateway.CreateAccountResponse
+	if err := callAdmin(ctx, target, "CreateAccount", req, &out); err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+// callAdmin signs and sends one POST /admin/<method>, decoding the success body
+// into out. Credentials come from the standard AWS chain, so this never reads
+// the cluster master key.
+func callAdmin(ctx context.Context, target adminTarget, method string, req, out any) error {
+	body, err := json.Marshal(req)
+	if err != nil {
+		return fmt.Errorf("marshal request: %w", err)
+	}
+
+	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost,
+		strings.TrimRight(target.endpoint, "/")+"/admin/"+method, bytes.NewReader(body))
+	if err != nil {
+		return err
+	}
+	httpReq.Header.Set("Content-Type", "application/json")
+
+	signer, err := gwsign.NewIMDS(ctx, target.region)
+	if err != nil {
+		return fmt.Errorf("resolve AWS credentials: %w", err)
+	}
+	sum := sha256.Sum256(body)
+	if err := signer.Sign(httpReq, hex.EncodeToString(sum[:]), "spinifex", target.region); err != nil {
+		return fmt.Errorf("sign request: %w", err)
+	}
+
+	client, err := adminHTTPClient(target.caBundle)
+	if err != nil {
+		return err
+	}
+	resp, err := client.Do(httpReq)
+	if err != nil {
+		return fmt.Errorf("call %s: %w", target.endpoint, err)
+	}
+	defer resp.Body.Close()
+
+	payload, err := io.ReadAll(io.LimitReader(resp.Body, adminMaxResponseBytes))
+	if err != nil {
+		return fmt.Errorf("gateway returned HTTP %d with an unreadable body: %w", resp.StatusCode, err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return decodeAdminError(resp.StatusCode, payload)
+	}
+	if err := json.Unmarshal(payload, out); err != nil {
+		return fmt.Errorf("decode response: %w", err)
+	}
+	return nil
+}
+
+// decodeAdminError turns a non-200 into an adminError. A body that is not the
+// JSON envelope is reported verbatim rather than as a decode failure, since
+// then the response came from something other than the gateway.
+func decodeAdminError(statusCode int, payload []byte) error {
+	var body struct {
+		Error struct {
+			Code    string `json:"code"`
+			Message string `json:"message"`
+		} `json:"error"`
+		RequestID string `json:"requestId"`
+	}
+	if err := json.Unmarshal(payload, &body); err != nil || body.Error.Code == "" {
+		return fmt.Errorf("gateway returned HTTP %d: %s", statusCode, payload)
+	}
+	return &adminError{
+		Code:       body.Error.Code,
+		Message:    body.Error.Message,
+		RequestID:  body.RequestID,
+		StatusCode: statusCode,
+	}
+}
+
+// newClientToken returns a fresh idempotency token in the character set the
+// endpoint accepts.
+func newClientToken() string {
+	var buf [24]byte
+	if _, err := rand.Read(buf[:]); err != nil {
+		// crypto/rand does not fail on any supported platform; if it does, a
+		// weaker token would silently break idempotency.
+		fmt.Fprintf(os.Stderr, "Error generating client token: %v\n", err)
+		os.Exit(1)
+	}
+	return hex.EncodeToString(buf[:])
+}
+
+// adminHTTPClient trusts caBundle in addition to the system roots so the same
+// command works against a cluster's self-signed CA and against a public
+// certificate. An unreadable bundle is an error, never a silent downgrade.
+func adminHTTPClient(caBundle string) (*http.Client, error) {
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	if caBundle != "" {
+		pem, err := os.ReadFile(caBundle)
+		if err != nil {
+			return nil, fmt.Errorf("read CA bundle %s: %w", caBundle, err)
+		}
+		pool, err := x509.SystemCertPool()
+		if err != nil || pool == nil {
+			pool = x509.NewCertPool()
+		}
+		if !pool.AppendCertsFromPEM(pem) {
+			return nil, fmt.Errorf("CA bundle %s contains no certificates", caBundle)
+		}
+		transport.TLSClientConfig = &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}
+	}
+	return &http.Client{Transport: transport}, nil
+}
+
+// loadLocalConfig reads this node's cluster config without connecting to NATS.
+func loadLocalConfig() (*config.ClusterConfig, error) {
+	cfgPath := viper.GetString("config")
+	if cfgPath == "" {
+		cfgPath = DefaultConfigFile()
+	}
+	cfg, err := config.LoadConfig(cfgPath)
+	if err != nil {
+		return nil, fmt.Errorf("load config: %w", err)
+	}
+	if nodeConfig, ok := cfg.Nodes[cfg.Node]; ok && nodeConfig.BaseDir == "" {
+		if isProductionLayout() {
+			nodeConfig.BaseDir = DefaultDataDir()
+		} else {
+			nodeConfig.BaseDir = filepath.Dir(filepath.Dir(cfgPath))
+		}
+		cfg.Nodes[cfg.Node] = nodeConfig
+	}
+	return cfg, nil
+}
+
+// localGatewayEndpoint is this node's AWS gateway URL. A wildcard bind address
+// is not dialable, so it resolves to localhost.
+func localGatewayEndpoint(node config.Config) string {
+	host, port, err := net.SplitHostPort(node.AWSGW.Host)
+	if err != nil {
+		return ""
+	}
+	if host == "" || host == "0.0.0.0" || host == "::" {
+		host = "localhost"
+	}
+	return "https://" + net.JoinHostPort(host, port)
+}

cmd/spinifex/cmd/admin_remote_test.go

@@ -0,0 +1,285 @@
+package cmd_test
+
+import (
+	"encoding/pem"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const auditAccountID = "000000000001"
+
+// STS gates AssumeRole on the trust policy alone, so an account-wide trust
+// policy is assumable by every principal in the account — including the
+// single-action signup credential.
+func TestTrustsWholeAccount(t *testing.T) {
+	tests := []struct {
+		name     string
+		document string
+		want     bool
+	}{
+		{
+			name:     "root ARN of the account",
+			document: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"arn:aws:iam::000000000001:root"},"Action":"sts:AssumeRole"}]}`,
+			want:     true,
+		},
+		{
+			name:     "bare account ID",
+			document: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"000000000001"},"Action":"sts:AssumeRole"}]}`,
+			want:     true,
+		},
+		{
+			name:     "wildcard",
+			document: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"*"},"Action":"sts:AssumeRole"}]}`,
+			want:     true,
+		},
+		{
+			name:     "account-wide among several entries",
+			document: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam::000000000001:user/alice","000000000001"]},"Action":"sts:AssumeRole"}]}`,
+			want:     true,
+		},
+		{
+			name:     "specific user",
+			document: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"arn:aws:iam::000000000001:user/alice"},"Action":"sts:AssumeRole"}]}`,
+			want:     false,
+		},
+		{
+			name:     "another account",
+			document: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"arn:aws:iam::123456789012:root"},"Action":"sts:AssumeRole"}]}`,
+			want:     false,
+		},
+		{
+			name:     "service principal",
+			document: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"ec2.amazonaws.com"},"Action":"sts:AssumeRole"}]}`,
+			want:     false,
+		},
+		{
+			name:     "deny statement is not a grant",
+			document: `{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Principal":{"AWS":"*"},"Action":"sts:AssumeRole"}]}`,
+			want:     false,
+		},
+		{
+			name:     "unparseable document is reported rather than passed over",
+			document: `not json`,
+			want:     true,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.want, cmd.TrustsWholeAccount(tc.document, auditAccountID))
+		})
+	}
+}
+
+func TestNewClientTokenIsAccepted(t *testing.T) {
+	valid := regexp.MustCompile(`^[A-Za-z0-9_-]{32,128}$`)
+
+	first := cmd.NewClientToken()
+	second := cmd.NewClientToken()
+
+	assert.Regexp(t, valid, first)
+	assert.NotEqual(t, first, second)
+}
+
+func TestLocalGatewayEndpoint(t *testing.T) {
+	tests := []struct {
+		name string
+		host string
+		want string
+	}{
+		{"explicit host", "10.0.0.1:9999", "https://10.0.0.1:9999"},
+		{"wildcard bind is not dialable", "0.0.0.0:9999", "https://localhost:9999"},
+		{"empty host", ":9999", "https://localhost:9999"},
+		{"no port", "10.0.0.1", ""},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			node := config.Config{AWSGW: config.AWSGWConfig{Host: tc.host}}
+			assert.Equal(t, tc.want, cmd.LocalGatewayEndpoint(node))
+		})
+	}
+}
+
+// An unreadable or empty CA bundle must be an error, never a silent fallback
+// to an unverified connection.
+func TestAdminHTTPClientRejectsBadCABundle(t *testing.T) {
+	_, err := cmd.AdminHTTPClient(filepath.Join(t.TempDir(), "absent.pem"))
+	assert.Error(t, err)
+
+	empty := filepath.Join(t.TempDir(), "empty.pem")
+	require.NoError(t, os.WriteFile(empty, []byte("not a certificate"), 0o600))
+	_, err = cmd.AdminHTTPClient(empty)
+	assert.Error(t, err)
+}
+
+// writeServerCA writes a test server's certificate where adminHTTPClient can
+// trust it, standing in for a cluster's self-signed CA.
+func writeServerCA(t *testing.T, srv *httptest.Server) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "ca.pem")
+	pemBytes := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: srv.Certificate().Raw})
+	require.NoError(t, os.WriteFile(path, pemBytes, 0o600))
+	return path
+}
+
+// newAdminTestServer serves one canned response over TLS and records the
+// request the client actually sent.
+func newAdminTestServer(t *testing.T, status int, body string, got *http.Request) *httptest.Server {
+	t.Helper()
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		*got = *r
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(status)
+		_, _ = w.Write([]byte(body))
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+const createAccountOK = `{"accountId":"000000000042","accountName":"ben@example.com","adminUser":"admin",` +
+	`"accessKeyId":"AKIATEST","secretAccessKey":"secret","defaultVpcId":"vpc-1"}`
+
+// The request must be SigV4-signed for service spinifex at the configured
+// region, and must reach /admin/CreateAccount as a POST.
+func TestCreateAccountRemoteSignsAndPosts(t *testing.T) {
+	t.Setenv("AWS_ACCESS_KEY_ID", "AKIAEXAMPLE")
+	t.Setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+
+	var got http.Request
+	srv := newAdminTestServer(t, http.StatusOK, createAccountOK, &got)
+
+	out, err := cmd.CreateAccountRemote(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)),
+		gateway.CreateAccountRequest{Name: "ben@example.com", ClientToken: strings.Repeat("a", 32)})
+	require.NoError(t, err)
+
+	assert.Equal(t, "000000000042", out.AccountID)
+	assert.Equal(t, "secret", out.SecretAccessKey)
+
+	assert.Equal(t, http.MethodPost, got.Method)
+	assert.Equal(t, "/admin/CreateAccount", got.URL.Path)
+	assert.Equal(t, "application/json", got.Header.Get("Content-Type"))
+	assert.NotEmpty(t, got.Header.Get("X-Amz-Content-Sha256"))
+	assert.Contains(t, got.Header.Get("Authorization"), "AWS4-HMAC-SHA256")
+	assert.Contains(t, got.Header.Get("Authorization"), "/us-west-1/spinifex/aws4_request")
+}
+
+// A trailing slash on the endpoint must not produce a double slash in the path,
+// which SigV4 would sign and the gateway would not route.
+func TestCreateAccountRemoteTrimsEndpointSlash(t *testing.T) {
+	t.Setenv("AWS_ACCESS_KEY_ID", "AKIAEXAMPLE")
+	t.Setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+
+	var got http.Request
+	srv := newAdminTestServer(t, http.StatusOK, createAccountOK, &got)
+
+	_, err := cmd.CreateAccountRemote(t.Context(),
+		cmd.AdminTarget(srv.URL+"/", "us-west-1", writeServerCA(t, srv)),
+		gateway.CreateAccountRequest{Name: "ben@example.com", ClientToken: strings.Repeat("a", 32)})
+	require.NoError(t, err)
+	assert.Equal(t, "/admin/CreateAccount", got.URL.Path)
+}
+
+func TestCreateAccountRemoteSurfacesGatewayError(t *testing.T) {
+	t.Setenv("AWS_ACCESS_KEY_ID", "AKIAEXAMPLE")
+	t.Setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+
+	var got http.Request
+	srv := newAdminTestServer(t, http.StatusConflict,
+		`{"error":{"code":"AccountAlreadyExists","message":"taken"},"requestId":"req-1"}`, &got)
+
+	_, err := cmd.CreateAccountRemote(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)),
+		gateway.CreateAccountRequest{Name: "ben@example.com", ClientToken: strings.Repeat("a", 32)})
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "AccountAlreadyExists")
+	assert.Contains(t, err.Error(), "req-1")
+	assert.False(t, cmd.RetryableAdminError(err), "a taken name is not fixed by retrying")
+}
+
+// An untrusted certificate must fail rather than fall back to no verification.
+func TestCreateAccountRemoteRefusesUntrustedServer(t *testing.T) {
+	t.Setenv("AWS_ACCESS_KEY_ID", "AKIAEXAMPLE")
+	t.Setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+
+	var got http.Request
+	srv := newAdminTestServer(t, http.StatusOK, createAccountOK, &got)
+
+	_, err := cmd.CreateAccountRemote(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", ""),
+		gateway.CreateAccountRequest{Name: "ben@example.com", ClientToken: strings.Repeat("a", 32)})
+	assert.Error(t, err)
+}
+
+// Only a lost or in-flight result is worth retrying. Suggesting it for the
+// rest invites a fresh token, which is what creates a duplicate account.
+func TestDecodeAdminErrorRetryability(t *testing.T) {
+	tests := []struct {
+		name        string
+		status      int
+		body        string
+		wantRetry   bool
+		wantMessage string
+	}{
+		{
+			name:        "in progress is retryable",
+			status:      http.StatusConflict,
+			body:        `{"error":{"code":"OperationInProgress","message":"in flight"},"requestId":"r"}`,
+			wantRetry:   true,
+			wantMessage: "OperationInProgress",
+		},
+		{
+			name:        "service unavailable is retryable",
+			status:      http.StatusServiceUnavailable,
+			body:        `{"error":{"code":"ServiceUnavailable","message":"down"},"requestId":"r"}`,
+			wantRetry:   true,
+			wantMessage: "ServiceUnavailable",
+		},
+		{
+			name:        "internal error is retryable",
+			status:      http.StatusInternalServerError,
+			body:        `{"error":{"code":"InternalError","message":"boom"},"requestId":"r"}`,
+			wantRetry:   true,
+			wantMessage: "InternalError",
+		},
+		{
+			name:        "access denied is not",
+			status:      http.StatusForbidden,
+			body:        `{"error":{"code":"AccessDenied","message":"no"},"requestId":"r"}`,
+			wantMessage: "AccessDenied",
+		},
+		{
+			name:        "idempotent mismatch is not",
+			status:      http.StatusBadRequest,
+			body:        `{"error":{"code":"IdempotentParameterMismatch","message":"no"},"requestId":"r"}`,
+			wantMessage: "IdempotentParameterMismatch",
+		},
+		{
+			name:        "a non-envelope body is reported verbatim",
+			status:      http.StatusBadGateway,
+			body:        `<html>nginx</html>`,
+			wantMessage: "nginx",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			err := cmd.DecodeAdminError(tc.status, []byte(tc.body))
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tc.wantMessage)
+			assert.Equal(t, tc.wantRetry, cmd.RetryableAdminError(err))
+		})
+	}
+}

cmd/spinifex/cmd/admin_test.go

@@ -768,8 +768,7 @@ func predastoreMultinodeNodes() []admin.PredastoreNodeConfig {
 type predastoreBucketAuth struct {
 	Buckets []struct {
 		Name string `toml:"name"`
-		Type string `toml:"type"`
-	} `toml:"buckets"`
+	} `toml:"bucket"`
 	Auth []struct {
 		AccessKeyID     string `toml:"access_key_id"`
 		SecretAccessKey string `toml:"secret_access_key"`
@@ -821,16 +820,10 @@ func TestPredastoreMultinodeTemplate_NorthstarProvisioned(t *testing.T) {
 	var cfg predastoreBucketAuth
 	require.NoError(t, toml.Unmarshal([]byte(content), &cfg))
 
+	// The zone bucket is declared once for the whole cluster: every host's gate
+	// reads the same config, so a resolver on any node can serve it.
 	assert.True(t, cfg.hasBucket(admin.NorthstarBucketName), "northstar zone bucket not rendered")
 
-	// The zone bucket must be readable from every node's local predastore, so
-	// a resolver on any node can serve it.
-	for _, b := range cfg.Buckets {
-		if b.Name == admin.NorthstarBucketName {
-			assert.Equal(t, "distributed", b.Type)
-		}
-	}
-
 	// BootstrapBaseZone writes zone files with the system key, not the
 	// resolver's. Without this grant, seeding fails on every node.
 	systemActions, ok := cfg.policyFor("AK", admin.NorthstarBucketName)
@@ -1055,15 +1048,15 @@ func TestSpinifexTomlTemplate_PeerNorthstarStanzaIsPathOnly(t *testing.T) {
 }
 
 // Unset knob must leave production config byte-identical: no [compaction] block
-// and the original trailing bytes unchanged (single-node has no trailing
-// newline, multinode keeps one).
+// and the original trailing bytes unchanged — the [iam] table both templates
+// end on, and the newline after it.
 func TestPredastoreTemplates_UnsetCompactionEmitsNoBlock(t *testing.T) {
 	single := renderSingleNodePredastore(t, admin.ConfigSettings{
 		Region: "ap-southeast-2", BindIP: "0.0.0.0", ConfigDir: "/cfg",
 		AccessKey: "AK", SecretKey: "SK", NatsToken: "tok",
 	})
 	assert.NotContains(t, single, "[compaction]")
-	assert.True(t, strings.HasSuffix(single, `access_keys_bucket = "spinifex-iam-access-keys"`),
+	assert.True(t, strings.HasSuffix(single, "access_keys_bucket = \"spinifex-iam-access-keys\"\n"),
 		"unset single-node tail changed: %q", single[len(single)-40:])
 
 	multi, err := admin.GenerateMultiNodePredastoreConfig(