release: update go.mod for v1.18.0

The local helper masked x-amz-security-token by line prefix, which matched the
header form and missed the presigned query form entirely, leaking the token into
the exported logs on every signature mismatch. Defer to bluebottle's
RedactedCanonicalRequest, which knows the structure and masks both forms.

.github/actions/junit-to-es/action.yml

@@ -0,0 +1,43 @@
+name: JUnit to Elasticsearch
+description: >-
+  Bulk-index one doc per JUnit test case into the mulga Elasticsearch sink,
+  so per-test pass rates outlive the 90-day artifact retention. Call it once
+  per job right after the JUnit XML is produced (with `if: always()`), next to
+  the junit-report action that renders the same files into the job summary.
+
+inputs:
+  junit-glob:
+    description: Shell glob matching the JUnit XML files to ship.
+    required: true
+  cell:
+    description: >-
+      Cell label stored as labels.ci_cell — the same value the job puts in
+      ci.cell for its OTel attributes (single, multi, gpu, ddil, 20, ...).
+      Without it, results from different topologies collapse into one bucket.
+    required: true
+  es-sink:
+    description: host:port of the Elasticsearch sink.
+    required: false
+    default: "192.168.1.13:9200"
+  index:
+    description: Target index.
+    required: false
+    default: mulga-ci-test-results
+
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      env:
+        JUNIT_GLOB: ${{ inputs.junit-glob }}
+        JUNIT_CELL: ${{ inputs.cell }}
+        MULGA_ES_SINK: ${{ inputs.es-sink }}
+        ES_INDEX: ${{ inputs.index }}
+        RUN_ID: ${{ github.run_id }}
+        RUN_ATTEMPT: ${{ github.run_attempt }}
+        WORKFLOW: ${{ github.workflow }}
+        BRANCH: ${{ github.head_ref || github.ref_name }}
+        SHA: ${{ github.sha }}
+        ARTIFACT_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+      run: |
+        python3 "${{ github.action_path }}/ship.py"

.github/actions/junit-to-es/ship.py

@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+"""Ship per-test JUnit results to the Elasticsearch sink.
+
+One doc per leaf test case, bulk-indexed into mulga-ci-test-results. The
+mulga-ci-runs summary says which suite failed; this says which test, with the
+failure text, so a per-test pass-rate trend outlives artifact retention.
+
+Inputs (env):
+    JUNIT_GLOB      Glob matching JUnit XML files. No matches is not an error:
+                    a leg that never ran produces none.
+    JUNIT_CELL      Cell label for these results (single, multi, gpu, 20, ...).
+    MULGA_ES_SINK   host:port of the sink. Anonymous requests are superuser
+                    there, so no credentials are involved.
+    ES_INDEX        Target index; defaults to mulga-ci-test-results.
+    RUN_ID, RUN_ATTEMPT, WORKFLOW, BRANCH, SHA, ARTIFACT_URL
+"""
+
+from __future__ import annotations
+
+import glob
+import json
+import os
+import re
+import sys
+import urllib.error
+import urllib.request
+import xml.etree.ElementTree as ET
+from datetime import datetime, timezone
+
+# Go test output carries colour codes into the JUnit CDATA (harness.Phase
+# emits them), which are noise in a stored failure message.
+_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
+
+# Enough failure text to recognise a failure; the artifact log stays the full
+# record.
+_MAX_MESSAGE = 4000
+
+# go-junit-report emits this when a `=== RUN` line has no matching result line,
+# which means the log it was fed was truncated or filtered. It is a parse gap,
+# not a test outcome, so it is stored under its own status and left out of every
+# pass rate rather than counted as a failure.
+_NO_RESULT = "No test result found"
+
+
+def _strip_ansi(s: str) -> str:
+    return _ANSI_RE.sub("", s)
+
+
+def _truncate(s: str) -> str:
+    s = s.strip()
+    return s if len(s) <= _MAX_MESSAGE else s[: _MAX_MESSAGE - 1] + "…"
+
+
+def _suite_from_path(path: str) -> str:
+    base = os.path.basename(path)
+    if base.endswith(".xml"):
+        base = base[:-4]
+    if base.startswith("junit-"):
+        base = base[len("junit-") :]
+    return base
+
+
+def _cases(path: str) -> list[dict]:
+    """Parse one JUnit file into leaf test cases."""
+    try:
+        root = ET.parse(path).getroot()
+    except ET.ParseError as exc:
+        print(f"::warning::junit-to-es: {path} is not parseable ({exc}), skipped")
+        return []
+
+    # JUnit XML has two shapes: <testsuite> as root, or <testsuites> wrapping
+    # several. .iter() handles both.
+    out: list[dict] = []
+    for s in root.iter("testsuite"):
+        for tc in s.findall("testcase"):
+            status = "pass"
+            fail_type = ""
+            message = ""
+            for tag in ("failure", "error"):
+                node = tc.find(tag)
+                if node is not None:
+                    status = "fail" if tag == "failure" else "error"
+                    fail_type = tag
+                    message = _strip_ansi(
+                        ((node.get("message") or "") + "\n" + (node.text or "")).strip()
+                    )
+                    if (node.get("message") or "").strip() == _NO_RESULT:
+                        status = "unknown"
+                    break
+            if status == "pass" and tc.find("skipped") is not None:
+                status = "skip"
+            out.append(
+                {
+                    "name": tc.get("name", ""),
+                    "classname": tc.get("classname", ""),
+                    "time": float(tc.get("time", 0) or 0),
+                    "status": status,
+                    "fail_type": fail_type,
+                    "message": message,
+                }
+            )
+
+    # Drop container cases: Go reports a parent t.Run as a rollup of its
+    # children, so keeping both double-counts one outcome. Same rule the job
+    # summary renderer applies.
+    names = {c["name"] for c in out}
+    containers = {n.rsplit("/", 1)[0] for n in names if "/" in n}
+    return [c for c in out if c["name"] not in containers]
+
+
+def _doc(cell: str, suite: str, case: dict, ts: str, ci: dict) -> dict:
+    full = case["name"]
+    top, _, sub = full.partition("/")
+    doc = {
+        "@timestamp": ts,
+        "suite": suite,
+        "status": case["status"],
+        "duration_seconds": case["time"],
+        "artifact_url": ci["artifact_url"],
+        "labels": {"ci_run_id": ci["run_id"], "ci_cell": cell},
+        "test": {
+            "name": top,
+            "subtest": sub,
+            "full_name": full,
+            "classname": case["classname"],
+        },
+        "ci": {
+            "workflow": ci["workflow"],
+            "branch": ci["branch"],
+            "sha": ci["sha"],
+            "run_attempt": ci["run_attempt"],
+        },
+    }
+    if case["fail_type"]:
+        doc["failure"] = {
+            "type": case["fail_type"],
+            "message": _truncate(case["message"]),
+        }
+    return doc
+
+
+def _doc_id(ci: dict, cell: str, suite: str, full_name: str) -> str:
+    # Deterministic so a replayed post overwrites instead of double-counting.
+    # run_attempt is part of it because a re-run is a distinct result.
+    return f"{ci['run_id']}-{ci['run_attempt']}-{cell}-{suite}-{full_name}"
+
+
+def _bulk(sink: str, index: str, body: str) -> int:
+    req = urllib.request.Request(
+        f"http://{sink}/{index}/_bulk",
+        data=body.encode("utf-8"),
+        headers={"Content-Type": "application/x-ndjson"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            payload = json.loads(resp.read().decode("utf-8"))
+    except (urllib.error.URLError, OSError) as exc:
+        print(f"::error::junit-to-es: bulk POST to {sink}/{index} failed: {exc}")
+        return 1
+
+    if payload.get("errors"):
+        for item in payload.get("items", []):
+            err = next(iter(item.values())).get("error")
+            if err:
+                print(f"::error::junit-to-es: bulk rejected a doc: {json.dumps(err)}")
+                return 1
+    return 0
+
+
+def main() -> int:
+    pattern = os.environ.get("JUNIT_GLOB", "").strip()
+    if not pattern:
+        print("::error::junit-to-es: JUNIT_GLOB is empty")
+        return 1
+    cell = os.environ.get("JUNIT_CELL", "").strip()
+    if not cell:
+        print("::error::junit-to-es: JUNIT_CELL is empty")
+        return 1
+    sink = os.environ.get("MULGA_ES_SINK", "").strip()
+    if not sink:
+        print("::error::junit-to-es: MULGA_ES_SINK is empty")
+        return 1
+    index = os.environ.get("ES_INDEX", "").strip() or "mulga-ci-test-results"
+
+    ci = {
+        "run_id": os.environ.get("RUN_ID", ""),
+        "run_attempt": os.environ.get("RUN_ATTEMPT", ""),
+        "workflow": os.environ.get("WORKFLOW", ""),
+        "branch": os.environ.get("BRANCH", ""),
+        "sha": os.environ.get("SHA", ""),
+        "artifact_url": os.environ.get("ARTIFACT_URL", ""),
+    }
+    ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    lines: list[str] = []
+    counts: dict[str, int] = {"pass": 0, "fail": 0, "error": 0, "skip": 0, "unknown": 0}
+    for path in sorted(glob.glob(pattern)):
+        suite = _suite_from_path(path)
+        for case in _cases(path):
+            doc = _doc(cell, suite, case, ts, ci)
+            counts[case["status"]] = counts.get(case["status"], 0) + 1
+            meta = {"index": {"_id": _doc_id(ci, cell, suite, doc["test"]["full_name"])}}
+            lines.append(json.dumps(meta))
+            lines.append(json.dumps(doc))
+
+    if not lines:
+        # Not an error on its own: the leg may have been skipped, or died
+        # before any test ran. The run summary doc records that separately.
+        print(f"junit-to-es: no JUnit files matched {pattern}, nothing to post")
+        return 0
+
+    rc = _bulk(sink, index, "\n".join(lines) + "\n")
+    if rc == 0:
+        print(
+            f"junit-to-es: posted {sum(counts.values())} test docs "
+            f"(cell {cell}) to {sink}/{index} — {counts['pass']} pass, "
+            f"{counts['fail']} fail, {counts['error']} error, "
+            f"{counts['skip']} skip, {counts['unknown']} unknown"
+        )
+    return rc
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/resolve-sibling-refs.sh

@@ -1,20 +1,21 @@
 #!/usr/bin/env bash
-# Resolve which branch of each sibling repo (viperblock, predastore) the CI
-# workspace should build against. A cross-repo change is developed on a branch
+# Resolve which branch of each sibling repo (viperblock, predastore, northstar,
+# bluebottle) the CI workspace should build against. A cross-repo change is developed on a branch
 # of the SAME name in each repo; build against that branch when it exists so the
 # set can be tested together, and fall back to dev otherwise. This mirrors the
 # resolution the e2e path already does in
 # mulga/scripts/tofu-cluster/build-install-artifacts.sh, so unit CI and e2e CI
 # agree on how sibling branches are selected.
 #
-# Reads $BRANCH (the triggering head/ref name) and writes viperblock=<ref> and
-# predastore=<ref> to $GITHUB_OUTPUT for the checkout steps to consume.
+# Reads $BRANCH (the triggering head/ref name) and writes viperblock=<ref>,
+# predastore=<ref>, northstar=<ref> and bluebottle=<ref> to $GITHUB_OUTPUT for
+# the checkout steps to consume.
 set -euo pipefail
 
 : "${BRANCH:?BRANCH must be set to the triggering branch name}"
 : "${GITHUB_OUTPUT:?GITHUB_OUTPUT must be set}"
 
-for repo in viperblock predastore; do
+for repo in viperblock predastore northstar bluebottle; do
   if git ls-remote --exit-code --heads \
       "https://github.com/mulgadc/${repo}.git" "$BRANCH" >/dev/null 2>&1; then
     ref="$BRANCH"

.github/scripts/suite-sets.sh

@@ -24,10 +24,14 @@
 # behaviour, so a second node adds no assertion. Its DB VMs are capped at four
 # concurrent by the suite's own semaphore, which is what keeps it inside the
 # job's 50-minute budget alongside the suites above.
-E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds"
+# quota belongs to both topologies because the gate it asserts is gateway code
+# that runs identically on either, and because a quota left switched on would
+# cap every suite sharing the environment — so it must be proved to restore
+# itself wherever it runs, not only on one node.
+E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota"
 
 # Suites runnable against a multi-node environment.
-E2E_SUITES_MULTI="multinode lb cert"
+E2E_SUITES_MULTI="multinode lb cert quota"
 
 # grep -xE alternation form, for narrowing a requested set to the eligible one.
 E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"

.github/workflows/e2e-baremetal.yml

@@ -143,6 +143,16 @@ jobs:
           [ -f "$LOG" ] || exit 0
           go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-baremetal.xml" || true
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: "20"
+
       - name: Analyze e2e failures
         if: always()
         uses: ./.github/actions/e2e-analyze

.github/workflows/e2e-ddil.yml

@@ -180,6 +180,16 @@ jobs:
           junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
           title: "DDIL scenario results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
+          cell: ddil
+
       - name: Collect Cluster Logs
         if: failure()
         working-directory: mulga/scripts/tofu-cluster

.github/workflows/e2e-nightly.yml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
     inputs:
       cells:
-        description: "Comma-separated cell numbers to run (blank = all)"
+        description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-19 tofu matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection"
         required: false
 
 permissions:
@@ -342,6 +342,14 @@ jobs:
               | tar -C "${ARTIFACT_DIR}/vm-console-node${idx}" -xf - 2>/dev/null
             ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
               > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
+            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \
+              > "${ARTIFACT_DIR}/openvswitch-ipsec-node${idx}.log" 2>/dev/null
+            # ovn-controller and ovs-vswitchd log to files, never to journald:
+            # without these a cross-chassis black-hole cannot be attributed to
+            # IPsec versus SB port binding.
+            mkdir -p "${ARTIFACT_DIR}/ovs-logs-node${idx}"
+            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo tar -C /var/log -cf - ovn openvswitch 2>/dev/null || true' \
+              | tar -C "${ARTIFACT_DIR}/ovs-logs-node${idx}" -xf - 2>/dev/null
           done
 
           cat "${ARTIFACT_DIR}"/spinifex-journal-node*.log \
@@ -363,6 +371,16 @@ jobs:
             go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
           done
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always() && steps.gate.outputs.selected == 'true'
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: ${{ matrix.cell }}
+
       - name: Analyze e2e failures
         if: always() && steps.gate.outputs.selected == 'true'
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -379,8 +397,11 @@ jobs:
           if-no-files-found: warn
           retention-days: 14
 
+      # Gated like every other step, not just always(): a cell the dispatch
+      # deselected never checked out or ran tofu init, so destroying would fail
+      # on missing plugins and report the cell red for correctly doing nothing.
       - name: Teardown
-        if: always()
+        if: always() && steps.gate.outputs.selected == 'true'
         working-directory: mulga/scripts/tofu-cluster
         env:
           MATRIX_HOST: ${{ matrix.host }}
@@ -397,7 +418,7 @@ jobs:
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
 
       - name: Upload tofu log
-        if: always()
+        if: always() && steps.gate.outputs.selected == 'true'
         uses: actions/upload-artifact@v7
         with:
           name: tofu-log-cell-${{ matrix.cell }}-${{ github.run_id }}
@@ -426,3 +447,615 @@ jobs:
       # There's no matrix leg here, just the one fixed baremetal cell.
       otel-attrs: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=20,ci.matrix=baremetal/pxe/real-hw
     secrets: inherit
+
+  # ISO cells. Unlike 1-19 these need no tofu: the harnesses run QEMU directly
+  # on the runner. The nodes are therefore L1, which keeps the instances they
+  # launch at L2 and their benchmark numbers comparable between the two cells.
+  #
+  # Serialised against each other because the guest MACs are fixed
+  # (52:54:00:12:...): stable MACs are what make DHCP discovery work, and two
+  # concurrent runs would put duplicates on the rack LAN, which is a confusing
+  # failure for everyone on it, not just CI.
+  iso-single:
+    name: cell-21 (iso/dhcp/single)
+    if: >-
+      github.event_name == 'schedule' ||
+      inputs.cells == '' ||
+      contains(format(',{0},', inputs.cells), ',21,')
+    runs-on: [self-hosted, ci-single]
+    # ISO build, install, boot and pool re-form is ~35 of these on its own,
+    # before the workload step imports an AMI and launches an instance.
+    timeout-minutes: 90
+    concurrency:
+      group: iso-qemu-rack
+      cancel-in-progress: false
+    env:
+      CI_WAN_MODE: dhcp
+      CI_PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
+      # What a real node gets. The harness default of 4G is a dev-box size, and
+      # importing an 820 MiB AMI on top of the running stack does not fit in it.
+      MEM: 8G
+      SMP: 8
+      # .230 is consumed by the OVN router's SNAT, leaving .231-.234 for
+      # instances. Disjoint from cell-22's pool so a run that dies holding its
+      # SNAT address cannot collide with the next cell.
+      EXT_POOL_START: 192.168.0.230
+      EXT_POOL_END: 192.168.0.234
+      EXT_GATEWAY: 192.168.1.1
+      EXT_PREFIX_LEN: 23
+      # Pins where the benchmark writes, so collecting and removing it later are
+      # both exact. Left to itself the harness picks a timestamped name.
+      BENCH_OUT: /tmp/cell21-bench-${{ github.run_id }}
+    steps:
+      - name: Clean Workspace
+        run: |
+          sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          path: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ env.GIT_BRANCH }}
+          path: mulga/spinifex
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ env.GIT_BRANCH }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: mulga/spinifex
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: mulga/viperblock
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: mulga/predastore
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: mulga/northstar
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: mulga/bluebottle
+
+      # Assert the host can do what the cell assumes, before spending 40
+      # minutes finding out it cannot. A checklist in a plan doc goes stale;
+      # this does not.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          ip link show "${BRIDGE:-br-wan}" >/dev/null ||
+            { echo "::error::bridge br-wan not found on this runner"; exit 1; }
+          grep -qE "^allow +${BRIDGE:-br-wan}$" /etc/qemu/bridge.conf ||
+            { echo "::error::br-wan not permitted in /etc/qemu/bridge.conf"; exit 1; }
+          command -v sshpass >/dev/null ||
+            { echo "::error::sshpass not installed"; exit 1; }
+          free_mb=$(free -m | awk '/^Mem:/ {print $7}')
+          [ "$free_mb" -ge 8192 ] ||
+            { echo "::error::need 8 GiB available, have ${free_mb} MiB"; exit 1; }
+          # A 40G image on tmpfs competes with the 8 GiB the VM itself needs.
+          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
+          [ "$fstype" != tmpfs ] ||
+            { echo "::error::$TMPDIR is tmpfs — the node disk would be in RAM"; exit 1; }
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          [ "$avail_gb" -ge 60 ] ||
+            { echo "::error::need 60 GiB on $TMPDIR, have ${avail_gb} GiB"; exit 1; }
+
+      - name: Build artefacts
+        working-directory: mulga/scripts/iso-builder
+        run: make build-rootfs build-squashfs build-initramfs
+
+      # One key for both halves of the cell. test-workload.sh SSHes to the node
+      # AND registers the same key as the EC2 key pair it uses to reach the
+      # instances, so the key test-smoke.sh installs has to be that one — its
+      # default ~/.ssh/id_*.pub would leave the workload step locked out.
+      - name: Prepare SSH key
+        run: |
+          set -euo pipefail
+          mkdir -p "$HOME/.ssh" && chmod 700 "$HOME/.ssh"
+          # Two keys, because the guest key must not share a name with the one
+          # smoke-test.sh registers from the node: the node holds that private
+          # half, and IMDS caches material by key name for five minutes, so
+          # re-importing over it is invisible to the guests.
+          for k in spinifex-key spinifex-workload-key; do
+            [ -f "$HOME/.ssh/$k" ] ||
+              ssh-keygen -t ed25519 -N "" -f "$HOME/.ssh/$k" -C "$k-ci"
+          done
+
+      # Boots the ISO, installs unattended, reads the lease off the serial
+      # console, runs the in-guest smoke test, and installs the key for the
+      # next step.
+      - name: Install and smoke test
+        id: smoke
+        working-directory: mulga/scripts/iso-builder
+        run: |
+          set -euo pipefail
+          # KEEP_NODE: the harness owns the only VM there is and tears it down
+          # on exit, so without this the workload step below has no node left.
+          KEEP_NODE=1 SSH_IDENTITY="$HOME/.ssh/spinifex-key.pub" \
+            ./test/test-smoke.sh | tee /tmp/smoke-$GITHUB_RUN_ID.log
+          ip=$(sed -n 's/^\[smoke\] node address: //p' /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
+          [ -n "$ip" ] || { echo "::error::smoke test did not report a node address"; exit 1; }
+          echo "node_ip=$ip" >> "$GITHUB_OUTPUT"
+
+      # The customer path: AMI, key, VPC, instance, SSH in. --benchmark records
+      # host and guest disk numbers so a regression shows up as a number rather
+      # than a story.
+      - name: Workload and benchmark
+        working-directory: mulga/scripts/iso-builder
+        run: |
+          set -euo pipefail
+          cat > /tmp/cell21.env <<EOF
+          NODES=(${{ steps.smoke.outputs.node_ip }})
+          NODE_NAMES=(spinifex-ci)
+          SSH_USER=spinifex
+          SSH_KEY_PATH=$HOME/.ssh/spinifex-workload-key
+          # SSH_KEY_PATH reaches the guests, SSH_IDENTITY reaches the node, and
+          # they are deliberately different keys. Without the second, node SSH
+          # falls back to the runner's default identity, which the node has
+          # never seen.
+          SSH_IDENTITY=$HOME/.ssh/spinifex-key
+          EXT_POOL_START=$EXT_POOL_START
+          EXT_POOL_END=$EXT_POOL_END
+          EXT_GATEWAY=$EXT_GATEWAY
+          EXT_PREFIX_LEN=$EXT_PREFIX_LEN
+          EOF
+          ./test/test-workload.sh --env /tmp/cell21.env --benchmark
+
+      - name: Collect logs
+        if: always()
+        run: |
+          mkdir -p /tmp/cell21-artifacts-$GITHUB_RUN_ID
+          cp /tmp/smoke-$GITHUB_RUN_ID.log /tmp/cell21-artifacts-$GITHUB_RUN_ID/ ||
+            echo "::warning::no smoke log to collect"
+          # fio JSON and the summary table. Warned about rather than swallowed:
+          # a benchmark that ran and produced nothing is worth knowing about.
+          if [ -d "$BENCH_OUT" ]; then
+            cp -r "$BENCH_OUT" /tmp/cell21-artifacts-$GITHUB_RUN_ID/benchmark
+          else
+            echo "::warning::no benchmark results at $BENCH_OUT"
+          fi
+          # The guest's own view. Without it a node that dies mid-test is only
+          # visible as the harness command that stopped producing output.
+          serial=$(sed -n 's/^\[smoke\] serial log preserved: //p' \
+                   /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
+          [ -n "$serial" ] && cp "$serial" /tmp/cell21-artifacts-$GITHUB_RUN_ID/serial.log || true
+          node=$(sed -n 's/^\[smoke\] node logs preserved: //p' \
+                 /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
+          [ -n "$node" ] && cp "$node" /tmp/cell21-artifacts-$GITHUB_RUN_ID/node.log || true
+          # QEMU's stderr no longer reaches the step log: it is redirected so a
+          # kept-alive VM cannot hold the step's stdout open.
+          qemu=$(sed -n 's/^\[smoke\] qemu log preserved: //p' \
+                 /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
+          [ -n "$qemu" ] && cp "$qemu" /tmp/cell21-artifacts-$GITHUB_RUN_ID/qemu.log || true
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: e2e-artifacts-cell-21-${{ github.run_id }}
+          path: /tmp/cell21-artifacts-${{ github.run_id }}
+          if-no-files-found: warn
+          # Matches the predastore benchmark in cell-23: a night's numbers are
+          # only useful against the nights before them.
+          retention-days: 90
+
+      - name: Teardown
+        if: always()
+        working-directory: mulga/scripts/iso-builder
+        run: |
+          # No env file means the workload step never ran, so there is nothing
+          # of its to clean up.
+          if [ -f /tmp/cell21.env ]; then
+            ./test/test-workload.sh --env /tmp/cell21.env --cleanup || true
+          fi
+          sudo pkill -f spinifex-smoke-test || true
+          rm -rf /tmp/cell21-artifacts-$GITHUB_RUN_ID /tmp/cell21.env \
+                 /tmp/smoke-$GITHUB_RUN_ID.log "$BENCH_OUT"
+          # Runs before BENCH_OUT existed left a timestamped directory behind
+          # every night. Sweep anything older than a day.
+          find /tmp -maxdepth 1 -name 'spinifex-benchmark-*' -type d -mmin +1440 \
+               -exec rm -rf {} + 2>/dev/null || true
+          # KEEP_NODE leaves the node's disk and serial log behind for us.
+          sleep 2
+          rm -f "$TMPDIR"/tmp.*.qcow2 "$TMPDIR"/tmp.*.log "$TMPDIR"/tmp.*-node.log
+
+  iso-multi:
+    name: cell-22 (iso/dhcp/three-node)
+    # After cell-21 rather than beside it: same concurrency group, and running
+    # them together would need 32 GiB on one box.
+    needs: [iso-single]
+    if: >-
+      always() && (
+      github.event_name == 'schedule' ||
+      inputs.cells == '' ||
+      contains(format(',{0},', inputs.cells), ',22,') )
+    runs-on: [self-hosted, ci-multi]
+    timeout-minutes: 90
+    concurrency:
+      group: iso-qemu-rack
+      cancel-in-progress: false
+    env:
+      MEM: 8G
+      SMP: 8
+      PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
+      EXT_POOL_START: 192.168.0.235
+      EXT_POOL_END: 192.168.0.239
+      EXT_GATEWAY: 192.168.1.1
+      EXT_PREFIX_LEN: 23
+    steps:
+      - name: Clean Workspace
+        run: |
+          sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+          # Pinned so the per-node serial logs can be collected afterwards, and
+          # kept beside TMPDIR rather than in /tmp: it also holds nine 40G node
+          # disks, which is what the tmpfs assertion below is about.
+          echo "WORK_DIR=$E2E_ROOT/work/cell22-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          path: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ env.GIT_BRANCH }}
+          path: mulga/spinifex
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ env.GIT_BRANCH }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: mulga/spinifex
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: mulga/viperblock
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: mulga/predastore
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: mulga/northstar
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: mulga/bluebottle
+
+      # 3 x 8 GiB plus 9 sparse 40G disks, before the instances the cluster
+      # launches on top.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          free_mb=$(free -m | awk '/^Mem:/ {print $7}')
+          [ "$free_mb" -ge 26624 ] ||
+            { echo "::error::need 26 GiB available, have ${free_mb} MiB"; exit 1; }
+          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
+          [ "$fstype" != tmpfs ] ||
+            { echo "::error::$TMPDIR is tmpfs — nine node disks would be in RAM"; exit 1; }
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          [ "$avail_gb" -ge 120 ] ||
+            { echo "::error::need 120 GiB on $TMPDIR ($fstype), have ${avail_gb} GiB"; exit 1; }
+
+      - name: Build artefacts
+        working-directory: mulga/scripts/iso-builder
+        run: make build-rootfs build-squashfs build-initramfs
+
+      # Four phases: install three nodes, convert to one cluster, assert it
+      # formed (including the external pool), then pull node 1's power and
+      # assert the survivors hold quorum.
+      #
+      # Phase 5 (test-workload.sh, as cell-21 runs it) is not wired yet: the
+      # nodes are still on user-mode NAT and reachable only through the
+      # harness's own port forwards, and test-cluster.sh does not open the
+      # formation port. Both land with mulga-yxtda and the bridged WAN mode;
+      # until then this cell proves the cluster forms and survives, not that a
+      # customer can use it.
+      - name: Install, form and verify the cluster
+        working-directory: mulga/scripts/iso-builder
+        run: |
+          set -euo pipefail
+          ./test/test-cluster.sh 2>&1 | tee /tmp/cell22-$GITHUB_RUN_ID.log
+
+      - name: Collect logs
+        if: always()
+        run: |
+          mkdir -p /tmp/cell22-artifacts-$GITHUB_RUN_ID
+          cp /tmp/cell22-$GITHUB_RUN_ID.log /tmp/cell22-artifacts-$GITHUB_RUN_ID/ ||
+            echo "::warning::no cluster log to collect"
+          # Serial only. The same directory holds the nodes' disk images, which
+          # are gigabytes and must not go into an artifact.
+          cp "$WORK_DIR"/*-serial.log /tmp/cell22-artifacts-$GITHUB_RUN_ID/ 2>/dev/null ||
+            echo "no per-node serial logs — the run passed and cleaned up after itself"
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: e2e-artifacts-cell-22-${{ github.run_id }}
+          path: /tmp/cell22-artifacts-${{ github.run_id }}
+          if-no-files-found: warn
+          retention-days: 90
+
+      - name: Teardown
+        if: always()
+        run: |
+          sudo pkill -f spinifex-cluster-test || true
+          rm -rf /tmp/cell22-artifacts-$GITHUB_RUN_ID /tmp/cell22-$GITHUB_RUN_ID.log \
+                 "$WORK_DIR"
+
+  # Predastore throughput ("cell 23"). Predastore has no runners of its own and
+  # spinifex depends on it, so the benchmark lives here and checks predastore
+  # out as a sibling.
+  #
+  # needs: is the whole point. A throughput benchmark sharing a host with a cell
+  # measures the neighbour, and ci-single and ci-multi both resolve to
+  # bottlebrush, so "a different label" would not have separated them. needs: on
+  # a matrix job waits for every leg, so this starts on a quiet machine.
+  #
+  # always() because a failed cell must not silently skip the benchmark and
+  # leave a green-looking night with no number in it.
+  performance:
+    name: cell-23 (predastore/throughput)
+    needs: [matrix, baremetal, iso-single, iso-multi]
+    if: >-
+      always() && (
+      github.event_name == 'schedule' ||
+      inputs.cells == '' ||
+      contains(format(',{0},', inputs.cells), ',23,') )
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 60
+    env:
+      # Two-minute samples. The 30s smoke preset is too short to compare nights.
+      PERF_PRESET: compare
+    steps:
+      - name: Clean Workspace
+        run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+          echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
+
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      # Which predastore branch to measure: the same-named branch when one
+      # exists, dev otherwise. Matches how the e2e path already picks siblings,
+      # so a cross-repo change is benchmarked as a set.
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      # The benchmark works in $TMPDIR and predastore refuses writes below a 5%
+      # free-space watermark. Without this the symptom is several hundred
+      # "storage pool is at capacity" lines from warp and no statement of how
+      # much space there actually was.
+      #
+      # TMPDIR is pointed at the runner's disk above, and the tmpfs assertion is
+      # the one that matters: a storage benchmark on a RAM-backed filesystem
+      # measures memcpy. It would report a number that looks excellent, sit in
+      # the baseline, and never notice a disk regression.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
+          case "$fstype" in
+            tmpfs|ramfs)
+              echo "::error::$TMPDIR is $fstype — a storage benchmark there measures RAM"
+              exit 1 ;;
+          esac
+          # Work dirs from runs that were killed before their cleanup trap ran.
+          # -mtime +0 so a concurrent run's directory is never in scope.
+          find "$TMPDIR" -maxdepth 1 -name 'predastore-e2e-performance.*' \
+               -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          pct_used=$(df --output=pcent "$TMPDIR" | tail -1 | tr -dc '0-9')
+          if [ "$pct_used" -ge 90 ] || [ "$avail_gb" -lt 20 ]; then
+            echo "::error::$TMPDIR ($fstype) is ${pct_used}% full with ${avail_gb} GiB free."
+            echo "::error::The benchmark needs headroom above predastore's 5% watermark."
+            exit 1
+          fi
+
+      # PERF_PORT_OFFSET defaults to 10000, so the benchmark's own nodes do not
+      # collide with a cluster left on the default ports.
+      - name: Run throughput benchmark
+        working-directory: predastore
+        run: |
+          set -euo pipefail
+          # Into the results directory, so the artifact reads on its own rather
+          # than needing the job log it was printed into. The benchmark only
+          # ever creates this directory, so a log placed here first survives.
+          mkdir -p "$PERF_RESULTS_ROOT"
+          make e2e-performance 2>&1 | tee "$PERF_RESULTS_ROOT/e2e-performance.log"
+
+      # Report-only. The baseline comparison and the -10% gate are mulga-wpjsx;
+      # until the run-to-run spread is known from real nights, a threshold would
+      # be a guess, and a gate that cries wolf gets muted.
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: predastore-perf-${{ github.run_id }}
+          path: ${{ env.PERF_RESULTS_ROOT }}
+          if-no-files-found: warn
+          retention-days: 90
+
+      - name: Clean up results
+        if: always()
+        run: rm -rf "$PERF_RESULTS_ROOT"
+
+  # Fault injection, not throughput. Cell-23 asks how fast predastore is when
+  # every node is healthy; this asks whether it survives one that is not.
+  #
+  # After cell-23 rather than beside it: both want a quiet single runner, and a
+  # benchmark sharing a machine with a fault injection run measures the wrong
+  # thing. The port offset keeps them apart even so.
+  stress:
+    name: cell-24 (predastore/fault-injection)
+    needs: [matrix, baremetal, iso-single, iso-multi, performance]
+    if: >-
+      always() && (
+      github.event_name == 'schedule' ||
+      inputs.cells == '' ||
+      contains(format(',{0},', inputs.cells), ',24,') )
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 30
+    env:
+      # Freezes a follower. STRESS_HOST=leader is a known failure: a gate keeps
+      # the frozen node first in its meta read order and pays the full client
+      # timeout per key, so listing slows in proportion to object count.
+      STRESS_HOST: follower
+      # Clear of cell-23's default 10000, so the two never collide on a runner
+      # that lets them overlap.
+      STRESS_PORT_OFFSET: "20000"
+    steps:
+      - name: Clean Workspace
+        run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+          echo "STRESS_RESULTS_ROOT=$E2E_ROOT/predastore-stress-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
+
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      # A frozen node is indistinguishable from a slow one, so a run on a
+      # loaded or nearly full disk fails as a timeout and reads as a real
+      # fault. The checks are the cheap way to tell those apart afterwards.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          find "$TMPDIR" -maxdepth 1 -name 'predastore-e2e-stress.*' \
+               -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          if [ "$avail_gb" -lt 10 ]; then
+            echo "::error::$TMPDIR has ${avail_gb} GiB free; the run needs headroom above predastore's 5% watermark."
+            exit 1
+          fi
+
+      # Leftovers from a run killed before its cleanup trap: a surviving node
+      # holds its ports and fails this run somewhere unrelated.
+      - name: Clear stale nodes
+        run: pkill -KILL -f '/bin/s3d -config' || true
+
+      - name: Run fault injection
+        working-directory: predastore
+        run: |
+          set -euo pipefail
+          # events.txt carries the timeline and the FAIL line, but not the build
+          # output or anything Warp wrote before the harness took over. The
+          # harness only creates this directory, so a log placed here survives.
+          mkdir -p "$STRESS_RESULTS_ROOT"
+          make e2e-stress 2>&1 | tee "$STRESS_RESULTS_ROOT/e2e-stress.log"
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: predastore-stress-${{ github.run_id }}
+          path: ${{ env.STRESS_RESULTS_ROOT }}
+          if-no-files-found: warn
+          retention-days: 90
+
+      - name: Clean up
+        if: always()
+        run: |
+          pkill -KILL -f '/bin/s3d -config' || true
+          rm -rf "$STRESS_RESULTS_ROOT"

.github/workflows/e2e-suites.yml

@@ -394,6 +394,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "E2E ${{ matrix.suite }} results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: single
+
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -796,6 +806,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "Multi-Node E2E ${{ matrix.suite }} results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: multi
+
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze

.github/workflows/e2e.yml

@@ -52,7 +52,7 @@ jobs:
     # inputs.suites.
     if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-iam') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-ecs') || contains(github.event.inputs.suites, 'e2e-storagegrowth') || contains(github.event.inputs.suites, 'e2e-partialblock') || contains(github.event.inputs.suites, 'e2e-rds')
     runs-on: [self-hosted, ci-single]
-    timeout-minutes: 50
+    timeout-minutes: 60
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
@@ -250,53 +250,58 @@ jobs:
           EOF
           ok "kubectl installed on VM"
 
-      # CreateDBInstance boots the DB VM from the rds-postgres AMI, so that image
+      # CreateDBInstance boots the DB VM from an RDS AMI, so both engines' images
       # must be in the VM's image store before the suite runs — same split as the
       # eks-node pair above: build on the runner (needs source + network), import
-      # on the VM (needs predastore/NATS). The suite's psql client needs no step
-      # here: it runs inside a client VM that installs postgresql-client from its
-      # own user-data, because a private-only endpoint is unreachable from the
-      # runner and from the cluster host alike.
-      - name: Build rds-postgres AMI (libguestfs, runner)
+      # on the VM (needs predastore/NATS). The suite's psql and mariadb clients
+      # need no step here: they run inside a client VM that installs them from
+      # its own user-data, because a private-only endpoint is unreachable from
+      # the runner and from the cluster host alike.
+      - name: Build RDS AMIs (libguestfs, runner)
         if: contains(env.SUITES, 'e2e-rds')
         working-directory: spinifex
         run: |
           set -euo pipefail
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          banner "build rds-postgres AMI (libguestfs — no host devices)"
           sudo apt-get update -qq
           sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
           export LIBGUESTFS_BACKEND=direct
           sudo chmod 0644 /boot/vmlinuz-* || true
-          group_open "make build-rds-postgres-image"
-          make build-rds-postgres-image
-          group_close
-          ok "raw: $(du -h /tmp/rds-postgres-image-build/rds-postgres-alpine.raw | cut -f1)"
+          for IMAGE in rds-postgres rds-mariadb; do
+            banner "build ${IMAGE} AMI (libguestfs — no host devices)"
+            group_open "make build-${IMAGE}-image"
+            make "build-${IMAGE}-image"
+            group_close
+            ok "raw: $(du -h "/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" | cut -f1)"
+          done
 
-      - name: Import rds-postgres AMI on VM
+      - name: Import RDS AMIs on VM
         if: contains(env.SUITES, 'e2e-rds')
         run: |
           set -euo pipefail
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
-            spinifex/scripts/images/rds-postgres/manifest.conf)"
-          # SYSTEM_TAG carries three tags, not one: managed-by marks it a system
-          # image and engine/engine-version are what an EngineVersion request
-          # resolves against. --tag is repeatable, so expand rather than pass the
-          # whole string as a single value.
-          TAG_ARGS=""
-          for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
           SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
           VM=tf-user@${{ steps.vm.outputs.wan_ip }}
-          banner "import rds-postgres AMI to VM image store"
-          scp "${SSH_OPTS[@]}" \
-            /tmp/rds-postgres-image-build/rds-postgres-alpine.raw \
-            "$VM":/tmp/rds-postgres-alpine.raw
-          ssh "${SSH_OPTS[@]}" "$VM" \
-            "spx admin images import --file /tmp/rds-postgres-alpine.raw \
-               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
-               --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
-          ok "rds-postgres AMI imported"
+          for IMAGE in rds-postgres rds-mariadb; do
+            eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
+              "spinifex/scripts/images/${IMAGE}/manifest.conf")"
+            # SYSTEM_TAG carries four tags, not one: managed-by marks it a system
+            # image, engine/engine-version are what an EngineVersion request
+            # resolves against, and the contract tag gates the data-volume format
+            # grant. --tag is repeatable, so expand rather than pass the whole
+            # string as a single value.
+            TAG_ARGS=""
+            for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
+            banner "import ${IMAGE} AMI to VM image store"
+            scp "${SSH_OPTS[@]}" \
+              "/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" \
+              "$VM":"/tmp/${IMAGE}-alpine.raw"
+            ssh "${SSH_OPTS[@]}" "$VM" \
+              "spx admin images import --file /tmp/${IMAGE}-alpine.raw \
+                 --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
+                 --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
+            ok "${IMAGE} AMI imported"
+          done
 
       - name: Build E2E test binaries on runner
         working-directory: spinifex/tests/e2e
@@ -383,9 +388,10 @@ jobs:
                echo \"::group::suite \$SUITE\"; \
                printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
+               case \$SUITE in rds) TMO=50m ;; *) TMO=30m ;; esac; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
-                 ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
+                 ./\$SUITE.test -test.v -test.timeout=\$TMO ${TEST_RUN_FLAG} \
                  2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \
                       | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
                rc=\${PIPESTATUS[0]}; \
@@ -458,6 +464,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "E2E results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: single
+
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -897,6 +913,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "Multi-Node E2E results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: multi
+
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze

.github/workflows/gpu-e2e.yml

@@ -287,6 +287,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "GPU E2E results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always() && steps.gate.outputs.paused != 'true'
+        continue-on-error: true
+        uses: ./.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: gpu
+
       - name: Analyze e2e failures
         if: always() && steps.gate.outputs.paused != 'true'
         uses: ./.github/actions/e2e-analyze

.github/workflows/publish-system-ami.yml

@@ -2,23 +2,26 @@ name: Publish System AMIs
 
 # Builds the system AMIs (libguestfs, on the runner) and publishes each
 # compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket that backs
-# iso.mulgadc.com/system-ami/. Five images ship:
+# iso.mulgadc.com/system-ami/. Six images ship:
 #   spinifex-eks-node       Alpine EKS K3s node (server|agent, role at boot)
 #   spinifex-ecs-node       Alpine ECS container instance
 #   spinifex-eks-node-gpu   Ubuntu EKS K3s GPU worker (NVIDIA driver + toolkit)
 #   spinifex-ecs-node-gpu   Ubuntu ECS GPU container instance
 #   spinifex-rds-postgres   Alpine RDS PostgreSQL instance
+#   spinifex-rds-mariadb    Alpine RDS MariaDB instance
 # Operators then install them on any cluster with:
 #   spx admin images import --name spinifex-eks-node
 #   spx admin images import --name spinifex-ecs-node
 #   spx admin images import --name spinifex-eks-node-gpu
 #   spx admin images import --name spinifex-ecs-node-gpu
 #   spx admin images import --name spinifex-rds-postgres
+#   spx admin images import --name spinifex-rds-mariadb
 #
-# Manual-only: publishing overwrites the mirror artifact, so it is a deliberate
-# action, not something every push should trigger. The `variant` input scopes
-# which images build — the Ubuntu GPU legs are 16G and much slower (NVIDIA
-# driver download + DKMS build), so publishing only the CPU AMIs stays cheap.
+# Publishing overwrites the mirror artifact, so it is a deliberate action, not
+# something every push should trigger: it runs manually, or from release.yml
+# once a tag has passed its gates. The `variant` input scopes which images
+# build — the Ubuntu GPU legs are 16G and much slower (NVIDIA driver download +
+# DKMS build), so publishing only the CPU AMIs stays cheap.
 
 on:
   workflow_dispatch:
@@ -36,6 +39,16 @@ on:
         description: "Build + convert only; skip the R2 upload"
         type: boolean
         default: false
+  workflow_call:
+    inputs:
+      variant:
+        description: "Which images to build (all|cpu|gpu|rds)"
+        type: string
+        default: all
+      dry_run:
+        description: "Build + convert only; skip the R2 upload"
+        type: boolean
+        default: false
 
 permissions:
   contents: read
@@ -59,7 +72,7 @@ jobs:
           set -euo pipefail
           cpu='{"name":"spinifex-eks-node","manifest":"scripts/images/eks-node/manifest.conf"},{"name":"spinifex-ecs-node","manifest":"scripts/images/ecs-agent/manifest.conf"}'
           gpu='{"name":"spinifex-eks-node-gpu","manifest":"scripts/images/eks-node-gpu/manifest.conf"},{"name":"spinifex-ecs-node-gpu","manifest":"scripts/images/ecs-agent-gpu/manifest.conf"}'
-          rds='{"name":"spinifex-rds-postgres","manifest":"scripts/images/rds-postgres/manifest.conf"}'
+          rds='{"name":"spinifex-rds-postgres","manifest":"scripts/images/rds-postgres/manifest.conf"},{"name":"spinifex-rds-mariadb","manifest":"scripts/images/rds-mariadb/manifest.conf"}'
           case "${VARIANT}" in
             cpu) items="${cpu}" ;;
             gpu) items="${gpu}" ;;
@@ -99,6 +112,20 @@ jobs:
           ref: dev
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: dev
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: dev
+          path: bluebottle
+
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
@@ -116,7 +143,7 @@ jobs:
         # "go.work already exists".
         run: |
           rm -f go.work go.work.sum
-          go work init ./spinifex ./viperblock ./predastore
+          go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Install build + upload tooling
         run: |

.github/workflows/release.yml

@@ -28,13 +28,25 @@ jobs:
           repository: mulgadc/viperblock
           path: viperblock
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          path: bluebottle
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
 
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
-          go-version: "1.26.5"
+          go-version: "1.27.0"
 
       - name: Build microVM artifacts (lb-agent + kernel + initramfs)
         run: cd spinifex && make build-microvm-image
@@ -73,6 +85,18 @@ jobs:
           repository: mulgadc/viperblock
           path: viperblock
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          path: bluebottle
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
 
@@ -207,6 +231,17 @@ jobs:
             echo "Release ${TAG} promoted to latest"
           fi
 
+  # Refresh the system AMIs on the R2 mirror only once the release is out of
+  # draft. Shares the ci-single runner with `iso`, so it queues behind it.
+  system-images:
+    needs: promote
+    permissions:
+      contents: read
+    uses: ./.github/workflows/publish-system-ami.yml
+    with:
+      variant: all
+    secrets: inherit
+
   # Annotate the (still-draft) release if either gate failed.
   mark-failed:
     needs: [ verify, baremetal ]
@@ -268,6 +303,18 @@ jobs:
           repository: mulgadc/viperblock
           path: viperblock
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          path: bluebottle
+
       - name: Checkout Mulga iso-builder
         uses: actions/checkout@v7
         with:
@@ -286,7 +333,7 @@ jobs:
           cache: false
 
       - name: Configure Go workspace
-        run: go work init ./spinifex ./predastore ./viperblock
+        run: go work init ./spinifex ./predastore ./viperblock ./northstar ./bluebottle
 
       - name: Install ISO builder dependencies
         run: |

.github/workflows/segscan-oracle.yml

@@ -51,6 +51,20 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
       - uses: actions/create-github-app-token@v3
         id: app-token
         with:
@@ -73,7 +87,7 @@ jobs:
           go-version-file: "spinifex/go.mod"
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Run Segscan Oracle Test
         env:

.github/workflows/spinifex.yml

@@ -53,14 +53,65 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
+      # the last go.sum change and never refreshed it. Both caches are managed
+      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
-          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
+          cache: false
+
+      # Module contents follow the five go.sum files, so a content key is exact.
+      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
+      # dependencies out of the key and re-downloading on every run.
+      - name: Cache Go modules
+        uses: actions/cache@v6
+        with:
+          path: ~/go/pkg/mod
+          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
+          restore-keys: gomod-
+
+      # Build artifacts follow source, so the key rolls per run and falls back to
+      # this job's newest entry. Keyed per job because -race, -covermode=atomic
+      # and plain builds produce disjoint entries and would evict each other.
+      - name: Cache Go build
+        uses: actions/cache@v6
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
+
+      # The EBS provider contract suite needs a real implementation on each
+      # side of the boundary: qemu-img/qemu-nbd/qemu-io back qemunbdd, which
+      # shares no code with viperblock, and nbdkit plus viperblock's plugin
+      # back viperblockd, the provider production actually uses. Without them
+      # the suite runs against MemoryProvider alone. nbdkit-plugin-dev carries
+      # the headers and nbdkit.pc that cgo needs to build that plugin, and
+      # libnbd-bin supplies the third-party NBD client (nbdinfo/nbdcopy) that
+      # judges the published export by something outside this codebase.
+      - name: Install conformance tools
+        run: sudo apt-get update && sudo apt-get install -y --no-install-recommends qemu-utils nbdkit nbdkit-plugin-dev libnbd-bin
+
+      - name: Build viperblock nbdkit plugin
+        run: make -C viperblock go_build_nbd
 
       - name: Build Spinifex
         run: make -C spinifex build
@@ -69,10 +120,13 @@ jobs:
         run: make -C spinifex manifest-check manifest-lint
 
       - name: Run Tests with Coverage
-        run: make -C spinifex test-cover
+        env:
+          SPINIFEX_REQUIRE_CONFORMANCE_TOOLS: "1"
+          VIPERBLOCK_NBDKIT_PLUGIN: ${{ github.workspace }}/viperblock/lib/nbdkit-viperblock-plugin.so
+        run: make -C spinifex test-cover GOTESTFLAGS=-count=1
 
       - name: Run Action Tests
-        run: make -C spinifex test-actions
+        run: make -C spinifex test-actions GOTESTFLAGS=-count=1
 
       - name: Check Diff Coverage
         run: cd spinifex && scripts/diff-coverage.sh coverage.out --base HEAD~1
@@ -114,14 +168,51 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
+      # the last go.sum change and never refreshed it. Both caches are managed
+      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
-          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
+          cache: false
+
+      # Module contents follow the five go.sum files, so a content key is exact.
+      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
+      # dependencies out of the key and re-downloading on every run.
+      - name: Cache Go modules
+        uses: actions/cache@v6
+        with:
+          path: ~/go/pkg/mod
+          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
+          restore-keys: gomod-
+
+      # Build artifacts follow source, so the key rolls per run and falls back to
+      # this job's newest entry. Keyed per job because -race, -covermode=atomic
+      # and plain builds produce disjoint entries and would evict each other.
+      - name: Cache Go build
+        uses: actions/cache@v6
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Run Integration Tests
         run: make -C spinifex test-integration
@@ -157,17 +248,54 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
+      # the last go.sum change and never refreshed it. Both caches are managed
+      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
-          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
+          cache: false
+
+      # Module contents follow the five go.sum files, so a content key is exact.
+      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
+      # dependencies out of the key and re-downloading on every run.
+      - name: Cache Go modules
+        uses: actions/cache@v6
+        with:
+          path: ~/go/pkg/mod
+          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
+          restore-keys: gomod-
+
+      # Build artifacts follow source, so the key rolls per run and falls back to
+      # this job's newest entry. Keyed per job because -race, -covermode=atomic
+      # and plain builds produce disjoint entries and would evict each other.
+      - name: Cache Go build
+        uses: actions/cache@v6
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Run Tests with Race Detector
-        run: make -C spinifex test-race
+        run: make -C spinifex test-race GOTESTFLAGS=-count=1
 
   lint_and_security:
     name: Lint and Security
@@ -200,19 +328,56 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
+      # the last go.sum change and never refreshed it. Both caches are managed
+      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
-          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
+          cache: false
+
+      # Module contents follow the five go.sum files, so a content key is exact.
+      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
+      # dependencies out of the key and re-downloading on every run.
+      - name: Cache Go modules
+        uses: actions/cache@v6
+        with:
+          path: ~/go/pkg/mod
+          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
+          restore-keys: gomod-
+
+      # Build artifacts follow source, so the key rolls per run and falls back to
+      # this job's newest entry. Keyed per job because -race, -covermode=atomic
+      # and plain builds produce disjoint entries and would evict each other.
+      - name: Cache Go build
+        uses: actions/cache@v6
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Lint
         uses: golangci/golangci-lint-action@v9
         with:
-          version: v2.12.2
+          version: v2.13.0
           working-directory: spinifex
 
       - name: Govulncheck

.gitignore

@@ -34,9 +34,10 @@ Thumbs.db
 # Environment
 .env
 .venv/
+.cache/
+
+# VScode
+.vscode/
 
 # Test reports
 *-report.txt
-
-# Per-worktree golangci-lint cache
-/.cache/

.golangci.yml

@@ -26,7 +26,7 @@ linters:
     - errorlint # catches ==/!= on errors, type assertions on errors, and %v instead of %w — all break on wrapped errors
     - exptostd # detects golang.org/x/exp/ functions replaceable by stdlib
     - fatcontext # detects nested contexts in loops
-    - forbidigo # blocks the legacy JetStream/KV symbols
+    - forbidigo # blocks forbidden identifiers
     - gocheckcompilerdirectives # validates go compiler directive comments (//go:)
     - godot # checks that declaration comments end in a period
     - goprintffuncname # checks that printf-like functions are named with f at the end
@@ -69,6 +69,18 @@ linters:
             - pkg: math/rand$
               desc: Use math/rand/v2 instead, see https://go.dev/blog/randv2
 
+        # The EC2 control plane reaches block storage through ebsprovider, not
+        # by importing the engine. viperblock.New starts a chunk uploader, so
+        # every construction is a writer against a volume viperblockd serves.
+        # Nothing under handlers/ may import viperblock, in any form, tests
+        # included. A types-only import counts: one became an engine import.
+        ec2-control-plane:
+          files:
+            - '**/spinifex/handlers/**'
+          deny:
+            - pkg: github.com/mulgadc/viperblock
+              desc: Use the ebsprovider.EBSProvider interface; the control plane must not construct or name viperblock types.
+
     embeddedstructfieldcheck:
       forbid-mutex: true
 
@@ -80,6 +92,8 @@ linters:
     forbidigo:
       analyze-types: true
       forbid:
+        - pattern: '^panic$'
+          msg: panic terminates the process; return an error or explicitly justify the panic.
         - pattern: '\.JetStream$'
           pkg: '^github\.com/nats-io/nats\.go$'
           msg: legacy nc.JetStream() is superseded; construct with jetstream.New(nc).
@@ -104,6 +118,7 @@ linters:
 
     modernize:
       disable:
+        - embedlit # its fix promotes embedded fields into the outer literal, which does not compile
         - newexpr # don't replace aws.String() with new() — AWS SDK idiom
         - stringsbuilder # don't replace small string concatenations with strings.Builder
 
@@ -154,6 +169,7 @@ linters:
           - bodyclose
           - errcheck
           - errchkjson
+          - forbidigo
           - gosec
 
       # spinifex-specific: gosec exclusions

Makefile

@@ -1,10 +1,10 @@
 GO_PROJECT_NAME := spx
 SHELL := /bin/bash
 
-# golangci-lint's default cache (~/.cache/golangci-lint) is keyed by module
-# path, not worktree path, so parallel worktrees of this module collide and
-# surface each other's issues. Scope it per worktree.
-export GOLANGCI_LINT_CACHE := $(CURDIR)/.cache/golangci-lint
+# GOFIPS140 is a global build-config change: it alters every package's action ID,
+# stdlib included, so a target that omits it shares no build-cache entry with one
+# that sets it. Exported once here rather than per-recipe so the two can't drift.
+export GOFIPS140 := v1.0.0
 
 # Detect architecture for cross-platform support
 ARCH := $(shell uname -m)
@@ -63,19 +63,23 @@ LDFLAGS := -s -w -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Version=$(VERSI
 
 go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
-	GOFIPS140=v1.0.0 go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
+	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
 
 build-installer:
 	@echo -e "\n....Building spinifex-installer"
-	GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
+	go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
 
 build-lb-agent:
 	@echo -e "\n....Building lb-agent (static)"
-	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
+	CGO_ENABLED=0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
 
 build-ecs-agent: ## Build the ecs-agent (ships in the ECS-AMI guest; not in host `build`)
 	@echo -e "\n....Building ecs-agent (static)"
-	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w -X main.version=$(VERSION)" -o ./bin/ecs-agent ./cmd/ecs-agent
+	CGO_ENABLED=0 go build -ldflags "-s -w -X main.version=$(VERSION)" -o ./bin/ecs-agent ./cmd/ecs-agent
+
+build-loadgen: ## Build spx-loadgen (drives the stress harness's measured load; not shipped to nodes)
+	@echo -e "\n....Building spx-loadgen"
+	GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/spx-loadgen ./cmd/spx-loadgen
 
 build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-node)
 ifndef IMAGE
@@ -111,6 +115,12 @@ build-rds-postgres-image: ## Build the spinifex-rds-postgres AMI (Alpine + Postg
 import-rds-postgres-image: ## Build + register the rds-postgres AMI (requires a running cluster)
 	$(MAKE) build-system-image IMAGE=rds-postgres IMPORT=1
 
+build-rds-mariadb-image: ## Build the spinifex-rds-mariadb AMI (Alpine + MariaDB 11.8 + rds-init; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=rds-mariadb
+
+import-rds-mariadb-image: ## Build + register the rds-mariadb AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=rds-mariadb IMPORT=1
+
 MICROVM_OUT_DIR := build/microvm
 MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
 MICROVM_INPUTS := scripts/build-microvm-image.sh $(MICROVM_OUT_DIR)/init.sh $(MICROVM_OUT_DIR)/inittab bin/lb-agent
@@ -134,9 +144,22 @@ install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share
 # the pre-commit gate: manifest checks, lint, vuln, and the unit and e2e-harness tiers.
 # integration and race tests skipped to keep quick, they run in CI.
 preflight:
-	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-harness
+	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-package-check test-harness test-build-scripts
 	@echo -e "\n ✅ Preflight passed — safe to commit."
 
+# Shell suites + shellcheck for build/scripts/, the systemd-unit helpers that
+# ship on every node (unlike scripts/images/, kept in preflight: a wrong
+# restart decision here is an outage, not asset churn).
+test-build-scripts:
+	@echo -e "\n....Running build/scripts/**/*_test.sh...."
+	@for t in $$(find build/scripts -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		sh "$$t" || exit 1; \
+	done
+	@echo -e "\n....Running shellcheck over build/scripts/**/*.sh...."
+	shellcheck -S warning $$(find build/scripts -name '*.sh' | sort)
+	@echo "  test-build-scripts ok"
+
 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
 # default `go test ./spinifex/...`. Runs with mocked AWS clients — no
 # infrastructure required, safe to run in CI without a cluster.
@@ -149,9 +172,13 @@ test-harness:
 AWS_MODEL_CONFORMANCE_REPORT ?= $(CURDIR)/.cache/aws-model-conformance-report.txt
 AWS_MODEL_CONFORMANCE_MODE ?= fail
 AWS_MODEL_OPERATION_COVERAGE_REPORT ?= $(CURDIR)/docs/aws-model-operation-coverage.md
+# -count=1 is load-bearing: the conformance report is written by the test binary,
+# so a cached pass skips the run, leaves no report, and the cat below fails the
+# target. It also keeps the conformance gate honest — a cached result would mean
+# the check never ran against this commit.
 test-integration:
 	@echo -e "\n....Running in-process integration tests...."
-	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -count=1 -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
 	@cat $(AWS_MODEL_CONFORMANCE_REPORT)
 	@$(MAKE) --no-print-directory generate-aws-model-coverage
 	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_REPORT)"
@@ -192,25 +219,30 @@ manifest-lint-update:
 # Run unit tests
 test:
 	@echo -e "\n....Running tests for $(GO_PROJECT_NAME)...."
-	GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/...
+	LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/...
+
+# Empty locally, where reusing a cached result between runs is the point.
+# CI passes -count=1 so a green run means the suite executed against that commit
+# rather than replaying results the persisted build cache carried over.
+GOTESTFLAGS ?=
 
 # Run unit tests with coverage profile
 COVERPROFILE ?= coverage.out
 test-cover:
 	@echo -e "\n....Running tests with coverage for $(GO_PROJECT_NAME)...."
-	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
 	@scripts/check-coverage.sh $(COVERPROFILE) $(QUIET)
 
 # Run unit tests with race detector
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
-	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... $(_RACEQ)
 
 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
 test-actions:
 	@echo -e "\n....Running action tests...."
-	LOG_IGNORE=1 go test -timeout 60s ./.github/actions/...
+	LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 60s ./.github/actions/...
 
 # Shell suites + shellcheck for scripts/images/ and images/mkosi.profiles/
 # helpers baked into system images. Kept out of `preflight` (a dedicated CI
@@ -249,6 +281,10 @@ test-images:
 diff-coverage: test-cover
 	@QUIET=$(QUIET) scripts/diff-coverage.sh $(COVERPROFILE)
 
+# Check that newly added test files use an external test package
+test-package-check:
+	@QUIET=$(QUIET) scripts/check-test-package.sh
+
 bench:
 	@echo -e "\n....Running benchmarks for $(GO_PROJECT_NAME)...."
 	LOG_IGNORE=1 go test -benchmem -run=. -bench=. ./...
@@ -303,9 +339,9 @@ install-system:
 		ovn-central ovn-host openvswitch-switch dhcpcd-base
 
 install-go:
-	@echo -e "\n....Installing Go 1.26.5 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.27.0 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.26.5.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.27.0.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi
@@ -388,7 +424,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
+.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

README.md

@@ -3,7 +3,7 @@
 </p>
 
 <p align="center">
-  <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.26+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
+  <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.27+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
   <a href="https://mulgadc.com"><img src="https://img.shields.io/badge/home-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
 </p>
@@ -13,9 +13,10 @@
   <a href="#the-platform">Platform</a> ·
   <a href="#three-ways-to-deploy">Deploy</a> ·
   <a href="#aws-compatibility">AWS Compatibility</a> ·
+  <a href="#live-demo">Live Demo</a> ·
   <a href="#core-components">Components</a> ·
   <a href="#architecture-at-a-glance">Architecture</a> ·
-  <a href="#installation">Installation</a> ·
+  <a href="#self-hosted-installation">Installation</a> ·
   <a href="https://docs.mulgadc.com">Docs</a>
 </p>
 
@@ -29,11 +30,9 @@ Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your
 
 Spinifex is an open-source infrastructure platform that recreates the AWS service surface (EC2, EBS, S3, VPC, IAM) on bare-metal, edge, and on-premises environments. Run cloud-native software without a hyperscaler.
 
-Most AWS alternatives wrap an API and call it a cloud. We rebuilt the engineering underneath: distributed object storage with erasure coding, block storage built to survive failure, and compute on bare metal. That depth is the reason your software runs unchanged, instead of rewritten.
+Built for teams that need:
 
-It's built for teams that need:
-
-- The AWS tooling they already use (AWS CLI, SDKs, Terraform), with nothing to rewrite
+- The AWS tooling you already use (AWS CLI, SDKs, Terraform), with nothing to rewrite
 - Full control of the stack: your hardware, your network, your data, your keys
 - A cloud that keeps running offline, through disconnection, with no external control plane
 - An open core (AGPL-3.0), auditable and yours to keep, with no lock-in
@@ -50,15 +49,15 @@ From commodity hardware up to unmodified AWS tooling, every layer is replaceable
 
 ## Three Ways To Deploy
 
-One AWS-compatible surface, three deployment shapes. Pick the one that matches your reality; the platform on top is the same.
+The same AWS-compatible platform runs in three shapes. Pick the one that matches your hardware.
 
 ### Neocloud
 
-Lift and shift onto a partner Neocloud. Move workloads off the hyperscalers and onto our Neocloud partner ecosystem without rewriting them. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
+Lift and shift onto a partner Neocloud without rewriting workloads. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
 
 ### On-premise
 
-Bring your own hardware and host Spinifex in your own data centre. A real multi-node HA cluster integrated with your storage and networking, with full control of stack, data, and jurisdiction. A predictable bill, no egress surprises, an auditable open-source core.
+Host Spinifex on your own hardware in your own data centre. You get a multi-node HA cluster integrated with your storage and networking, so the stack, the data, and the jurisdiction stay yours.
 
 ### Edge
 
@@ -66,7 +65,7 @@ Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels, factorie
 
 ## AWS Compatibility
 
-Speak the AWS API surface, natively. The AWS SDKs, AWS CLI, and Terraform: everything you deploy on AWS deploys on Spinifex unchanged. At the edge, on-premise, or on a partner Neocloud.
+Spinifex speaks the AWS API surface. The AWS SDKs, AWS CLI, and Terraform all work unchanged, whether you deploy at the edge, on-premise, or on a partner Neocloud.
 
 | Service | What it is | Status |
 | --- | --- | --- |
@@ -84,15 +83,33 @@ Speak the AWS API surface, natively. The AWS SDKs, AWS CLI, and Terraform: every
 
 Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
 
+## Live Demo
+
+Create a live sandbox in under a minute. The fastest way to experience the stack. Try Spinifex against one of your own workloads on infrastructure we run for you.
+
+* Live in under a minute. No card, no install.
+* Yours for 72 hours to test and experiment.
+* Real AWS-compatible endpoint, access keys and console.
+* EC2, EBS, S3, VPC, IAM, EKS and RDS, ready to call.
+* Point your existing AWS CLI or Terraform straight at it.
+
+Sign up at [https://mulgadc.com/signup](https://mulgadc.com/signup)
+
 ## Core Components
 
-### Spinifex (Compute Service – EC2 Alternative)
+### Spinifex (Compute and Platform Services – EC2, EKS, ECS, RDS)
 
-Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs similar to EC2. It manages lifecycle operations like start, stop, and terminate, using QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs using the AWS CLI, SDKs, or Terraform—without needing Kubernetes or heavyweight orchestrators. Keep in mind, you can also setup a Kubernetes environment using Spinifex with underlying instances.
+Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing an EC2 compatible API. It manages lifecycle operations like start, stop, and terminate through QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs with the AWS CLI, SDKs, or Terraform, without Kubernetes or a heavyweight orchestrator.
 
-- EC2-like VM management on bare metal
-- Launches with cloud-init metadata support
-- Works with standard AWS tooling
+The managed services build on that same instance layer. An EKS node group, an ECS container instance, and an RDS database are Spinifex VMs on your own hardware, each driven through its own AWS API.
+
+- **EC2** – VM lifecycle on bare metal, cloud-init and IMDS metadata, key pairs, tags, spot instances, capacity reservations, and GPU passthrough
+- **EKS** – Managed Kubernetes on k3s, with HA control planes, managed node groups, cluster add-ons, an OIDC provider, and IAM access entries
+- **ECS** – Task and service scheduling across container instances, with awsvpc ENIs, capacity providers, rolling deployments, and GPU-aware placement
+- **ECR** – Private container registry backed by Predastore, with content-addressed blobs deduplicated per account and lifecycle policies
+- **RDS** – Managed PostgreSQL and MariaDB, with parameter groups, automated backups, and DNS endpoints served by Northstar
+- **ALB / NLB** – Layer 7 load balancing on HAProxy and layer 4 on nginx `stream`, with health checks and ACM-issued listener certificates
+- **VPC and IAM** – OVN-backed networking and security groups, with roles, policies, and SigV4 auth applied across every service
 
 ### Viperblock (Block Storage – EBS Alternative)
 
@@ -101,12 +118,12 @@ Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs
 - Fast, durable virtual disks
 - Replication for resilience
 - Exposed over NBD or embedded in VMs
-- Supports high performance WAL logs using local NVMe drives to reduce IO traffic to S3.
-- In memory read/write block cache for blazing performance.
+- High performance WAL logs on local NVMe drives, reducing IO traffic to S3
+- In-memory read/write block cache
 
 ### Predastore (Object Storage – S3-Compatible)
 
-[Predastore](https://github.com/mulgadc/predastore) is a fully S3-compatible object storage system. It supports the AWS S3 API, including Signature V4 authentication, multipart uploads, and Terraform provisioning. Data is chunked and distributed across nodes using Reed-Solomon erasure coding, making it fault-tolerant and ideal for large-scale or low-bandwidth scenarios.
+[Predastore](https://github.com/mulgadc/predastore) is a fully S3-compatible object storage system. It supports the AWS S3 API, including Signature V4 authentication, multipart uploads, and Terraform provisioning. Predastore chunks data and distributes it across nodes using Reed-Solomon erasure coding, so it survives node loss and suits large-scale or low-bandwidth sites.
 
 - S3-compatible API and auth
 - Multipart uploads, streaming reads/writes
@@ -128,16 +145,7 @@ Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs
 
 Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless: scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Just systemd units, a NATS cluster, and your hardware. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
 
-## Key Features
-
-- **AWS-compatible APIs.** Use the AWS CLI, SDKs, and Terraform you already know. Repoint your endpoint and ship; nothing to rewrite.
-- **Zero cloud dependency.** Runs entirely on your hardware. No phone-home, no control plane, no external authority. Works fully offline.
-- **Bare-metal compute.** QEMU-based with direct hardware access. No hypervisor tax, no abstraction overhead.
-- **Built-in storage.** Block and object storage included: NVMe caching, Reed-Solomon erasure coding, and replication out of the box.
-- **Edge-first architecture.** Designed for disconnected, contested, and resource-constrained environments from day one.
-- **Open core, no lock-in.** AGPL-3.0 with a commercial option. Inspect it, modify it, deploy it. The platform is yours to keep.
-
-## Installation
+## Self-hosted Installation
 
 Installation requires an Ubuntu 26.04 or Debian 13 system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for installing Spinifex.
 
@@ -173,31 +181,21 @@ aws ec2 describe-instance-types
 
 For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation.
 
-### Component Repositories
-
-Spinifex coordinates these independent components:
-
-- **[Predastore](https://github.com/mulgadc/predastore)** - S3-compatible object storage
-- **[Viperblock](https://github.com/mulgadc/viperblock)** - EBS-compatible block storage
-- **[Northstar](https://github.com/mulgadc/northstar)** - Authoritative DNS server
-
-Each component can be developed independently. See component-specific documentation for focused development guides.
-
 ## Spinifex UI
 
-Spinifex ships with a built-in web console — an optional alternative to the AWS CLI, SDKs, and Terraform. If you're familiar with the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
+Spinifex ships with a built-in web console, an optional alternative to the AWS CLI, SDKs, and Terraform. If you know the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
 
 <p align="center">
-  <img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console — dashboard view" width="900">
+  <img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console dashboard" width="900">
 </p>
 
-The console is served by each node on port `3000` over TLS, and becomes available as soon as `spinifex.target` is up:
+Each node serves the console on port `3000` over TLS, and it comes up with `spinifex.target`:
 
-- **Same API, different surface.** Every action in the UI is the same AWS SigV4 call the CLI makes — so RBAC, audit trails, and IAM policies apply uniformly.
-- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed — no separate user database.
-- **Self-hosted, works offline.** The UI is embedded in the Spinifex binary and served from the node itself. No external CDN, no analytics calls, no cloud dependency.
+- **Same API, different surface.** The UI issues the same AWS SigV4 calls the CLI does, so RBAC, audit trails, and IAM policies apply the same way.
+- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed. There is no separate user database.
+- **Self-hosted, works offline.** The Spinifex binary embeds the UI and each node serves it directly, with no external CDN or analytics calls.
 
-For the full walkthrough — first-time TLS certificate trust, login, and feature tour — see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
+For the full walkthrough, covering first-time TLS certificate trust, login, and a feature tour, see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
 
 ## Development Philosophy
 
@@ -205,16 +203,9 @@ For the full walkthrough — first-time TLS certificate trust, login, and featur
 
 Spinifex is developed by experienced infrastructure engineers with deep AWS expertise, including former AWS team members who understand the intricacies of building production-grade cloud services. Our team brings decades of combined experience from AWS, enterprise infrastructure, and edge computing environments.
 
-**Real-World Experience:**
-
-- Production AWS service development and operations
-- Large-scale infrastructure deployment and management
-- Edge computing and resource-constrained environments
-- Enterprise security and compliance requirements
-
 ### AI-Assisted Development
 
-While Spinifex is architected and implemented by experienced engineers, we leverage **Claude Code** (Anthropic's AI coding assistant) to accelerate certain development tasks. This approach combines human expertise with AI efficiency:
+Spinifex is architected and implemented by experienced engineers, with **Claude Code** accelerating certain development tasks. The split keeps the quality and reliability standards that production infrastructure demands:
 
 **How We Use Claude Code:**
 
@@ -230,12 +221,10 @@ While Spinifex is architected and implemented by experienced engineers, we lever
 - **Performance Optimization**: Real-world performance tuning and benchmarking
 - **Production Operations**: Deployment strategies and operational procedures
 
-This hybrid approach ensures Spinifex benefits from both proven engineering expertise and modern development acceleration, while maintaining the quality and reliability standards required for production infrastructure.
-
 ## Trademarks
 
 "AWS", "Amazon Web Services", and all related service names (EC2, EBS, S3, VPC, IAM, and others) are trademarks of Amazon.com, Inc. or its affiliates. Mulga Defense Corporation and Spinifex are independent and not affiliated with, endorsed by, or sponsored by Amazon.com, Inc. or Amazon Web Services, Inc. References to AWS services describe interoperability and compatibility only.
 
 ## License
 
-Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE). You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.
+Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE), with a commercial license available. You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.

build/Dockerfile.distro

@@ -1,6 +1,6 @@
 ARG VERSION=dev
 
-FROM golang:1.26-bookworm AS builder
+FROM golang:1.27-bookworm AS builder
 
 ARG VERSION
 
@@ -16,9 +16,11 @@ COPY predastore/go.mod predastore/go.sum ./predastore/
 COPY viperblock/go.mod viperblock/go.sum ./viperblock/
 COPY viperblock/nbd/go.mod viperblock/nbd/go.sum ./viperblock/nbd/
 COPY viperblock/nbd/libguestfs.org/nbdkit/go.mod ./viperblock/nbd/libguestfs.org/nbdkit/
+COPY northstar/go.mod northstar/go.sum ./northstar/
+COPY bluebottle/go.mod bluebottle/go.sum ./bluebottle/
 
 # Generate go.work (gitignored, so not in checkout)
-RUN cd spinifex && go work init . ../predastore ../viperblock ../viperblock/nbd ../viperblock/nbd/libguestfs.org/nbdkit
+RUN cd spinifex && go work init . ../predastore ../viperblock ../viperblock/nbd ../viperblock/nbd/libguestfs.org/nbdkit ../northstar ../bluebottle
 
 # Download dependencies (cached layer)
 RUN cd spinifex && go mod download
@@ -27,6 +29,8 @@ RUN cd spinifex && go mod download
 COPY spinifex/ ./spinifex/
 COPY predastore/ ./predastore/
 COPY viperblock/ ./viperblock/
+COPY northstar/ ./northstar/
+COPY bluebottle/ ./bluebottle/
 
 # Build spinifex binary (spx)
 RUN COMMIT=$(cd spinifex && git rev-parse --short HEAD 2>/dev/null || echo "unknown") && \

build/scripts/nats-js-watchdog.sh

@@ -18,22 +18,97 @@
 # So a health check alone misses case 2 entirely. Both probes run here, and a
 # sustained failure of either restarts the service. The multi-sample window
 # avoids restarting during transient startup/quorum 503s.
+#
+# Three more guards sit around that decision, because "restart when the
+# probes fail" is not safe on its own:
+#
+#   - Age gate: spinifex-nats.service is Type=simple, so it reports active the
+#     instant the process forks, long before JetStream finishes replaying its
+#     state. Acting before a recovery budget has elapsed restarts a server
+#     that is still coming up, not one that has failed.
+#   - Cooldown: a restart that did not fix things must not be retried by the
+#     very next run, seconds later.
+#   - Escalation: a restart that has not fixed things after several attempts
+#     is not going to fix it on the next one either, and an unrunnable spx
+#     must never be read as proof of a JetStream failure at all.
 
 set -eu
 
 HEALTH_URL="http://127.0.0.1:8222/healthz?js-enabled-only=true"
 SERVICE="spinifex-nats.service"
 SPX="${SPX_BIN:-/usr/local/bin/spx}"
-SAMPLES=3        # consecutive failing samples required before acting
-SAMPLE_GAP=2     # seconds between samples
-PROBE_TIMEOUT=5  # seconds for one canary round-trip
+SAMPLES="${SAMPLES:-3}"           # consecutive failing samples required before acting
+SAMPLE_GAP="${SAMPLE_GAP:-2}"     # seconds between samples
+PROBE_TIMEOUT="${PROBE_TIMEOUT:-5}"  # seconds for one canary round-trip
 
-# Only act on a running service: a stopped/activating unit is systemd's to manage,
-# and try-restart on it would either no-op or fight an in-flight (re)start.
-if ! systemctl is-active --quiet "$SERVICE"; then
+# Recovery budget: on casuarina NATS needed 3m49s between "Started" and
+# JetStream actually accepting writes, and the bead recorded ~6min elsewhere.
+# 600s clears both with margin.
+NATS_MIN_AGE="${NATS_MIN_AGE:-600}"
+
+# Minimum time between restarts this watchdog performs, independent of the
+# age gate: closes the case where a restart did not fix things and the very
+# next run pulls the trigger again before the new process could recover.
+RESTART_COOLDOWN="${RESTART_COOLDOWN:-600}"
+
+# A restart that has not fixed things after this many attempts is not going
+# to fix it on the next one either, and looping costs more than stopping.
+RESTART_ESCALATE_AT="${RESTART_ESCALATE_AT:-3}"
+
+STATE_DIR="${STATE_DIR:-/run/spinifex}"
+COOLDOWN_STAMP="$STATE_DIR/nats-js-watchdog.last-restart"
+RESTART_COUNT_FILE="$STATE_DIR/nats-js-watchdog.restart-count"
+# Marks that the escalation message has already been printed for the current
+# latch, so a node that stays broken says so once instead of once per tick.
+ESCALATED_STAMP="$STATE_DIR/nats-js-watchdog.escalated"
+
+# Monotonic clock source, so a wall-clock step during boot (NTP correcting an
+# unset RTC) cannot manufacture a false age or cooldown. Overridable so tests
+# can stub a clock without faking /proc.
+UPTIME_FILE="${UPTIME_FILE:-/proc/uptime}"
+
+uptime_seconds() {
+    awk '{printf "%d", $1}' "$UPTIME_FILE"
+}
+
+# nats_age_seconds prints how long $SERVICE has been running. It reads
+# ExecMainStartTimestampMonotonic rather than is-active: that property goes
+# true the instant Type=simple forks, long before JetStream has replayed its
+# state, so it cannot express the recovery window this gate exists to
+# protect. A unit systemd has never started prints 0, which also covers a
+# stopped/activating unit the way the old is-active guard did.
+nats_age_seconds() {
+    start_usec=$(systemctl show --property=ExecMainStartTimestampMonotonic --value "$SERVICE" 2>/dev/null) || start_usec=""
+    case "$start_usec" in
+        ''|*[!0-9]*) echo 0; return ;;
+    esac
+    if [ "$start_usec" -eq 0 ]; then
+        echo 0
+        return
+    fi
+    echo $(( $(uptime_seconds) - start_usec / 1000000 ))
+}
+
+age=$(nats_age_seconds)
+if [ "$age" -lt "$NATS_MIN_AGE" ]; then
     exit 0
 fi
 
+if [ -f "$COOLDOWN_STAMP" ]; then
+    last=$(cat "$COOLDOWN_STAMP" 2>/dev/null) || last=""
+    case "$last" in ''|*[!0-9]*) last=0 ;; esac
+    if [ $(( $(uptime_seconds) - last )) -lt "$RESTART_COOLDOWN" ]; then
+        exit 0
+    fi
+fi
+
+restart_count=$(cat "$RESTART_COUNT_FILE" 2>/dev/null) || restart_count=""
+case "$restart_count" in ''|*[!0-9]*) restart_count=0 ;; esac
+
+# The escalation count is read here but acted on only after the probes, below.
+# Checking it up front would return before the one line that clears it, so a
+# latched watchdog could never observe recovery and never re-arm itself.
+
 # curl is the probe transport; if it is unavailable, do nothing rather than
 # false-restart a healthy server.
 if ! command -v curl >/dev/null 2>&1; then
@@ -45,13 +120,22 @@ js_healthy() {
     curl -fsS -o /dev/null --max-time 5 "$HEALTH_URL" 2>/dev/null
 }
 
-# Round-trips a canary key through the cluster-state bucket. Absent spx this
-# degrades to the health check alone rather than restarting on a missing tool.
+# Round-trips a canary key through the cluster-state bucket. Exit 0 means the
+# write succeeded and exit 1 means JetStream refused it; anything else
+# (spx's own exit 2, a missing binary, or one that cannot run at all) means
+# the probe observed nothing about JetStream and must never be read as
+# evidence of a failure.
 js_writable() {
     if [ ! -x "$SPX" ]; then
-        return 0
+        return 2
     fi
     "$SPX" admin node js-probe --timeout "${PROBE_TIMEOUT}s" >/dev/null 2>&1
+    rc=$?
+    case "$rc" in
+        0) return 0 ;;
+        1) return 1 ;;
+        *) return 2 ;;
+    esac
 }
 
 failure=""
@@ -59,6 +143,18 @@ i=1
 while [ "$i" -le "$SAMPLES" ]; do
     if js_healthy; then
         if js_writable; then
+            wrc=0
+        else
+            wrc=$?
+        fi
+        if [ "$wrc" -eq 0 ]; then
+            # Recovery: drop the history so a later failure gets the full
+            # restart budget again, and re-arm the escalation message.
+            rm -f "$RESTART_COUNT_FILE" "$ESCALATED_STAMP"
+            exit 0
+        fi
+        if [ "$wrc" -eq 2 ]; then
+            echo "nats-js-watchdog: js-probe inconclusive (config, connect, bucket-open, or spx itself failed); not evidence of a JetStream failure, taking no action" >&2
             exit 0
         fi
         failure="JetStream accepts no writes (canary round-trip failed) while $HEALTH_URL reports healthy"
@@ -71,5 +167,24 @@ while [ "$i" -le "$SAMPLES" ]; do
     i=$((i + 1))
 done
 
+# The failure is real and current, so the restart history now means what it
+# says. Above the threshold another restart will not help, so stop acting —
+# but keep probing on later runs, because that is what notices a recovery.
+if [ "$restart_count" -ge "$RESTART_ESCALATE_AT" ]; then
+    if [ ! -f "$ESCALATED_STAMP" ]; then
+        mkdir -p "$STATE_DIR"
+        : > "$ESCALATED_STAMP"
+        echo "nats-js-watchdog: already restarted $SERVICE $restart_count time(s) without recovery; giving up rather than looping (clear $RESTART_COUNT_FILE to re-arm)" >&2
+    fi
+    exit 0
+fi
+
 echo "nats-js-watchdog: $failure on $SAMPLES consecutive probes; restarting $SERVICE" >&2
+mkdir -p "$STATE_DIR"
+uptime_seconds > "$COOLDOWN_STAMP"
+restart_count=$((restart_count + 1))
+echo "$restart_count" > "$RESTART_COUNT_FILE"
+if [ "$restart_count" -ge "$RESTART_ESCALATE_AT" ]; then
+    echo "nats-js-watchdog: restart $restart_count of $RESTART_ESCALATE_AT before escalation; if $SERVICE is still failing after this, the watchdog will stop acting until $RESTART_COUNT_FILE is cleared" >&2
+fi
 systemctl try-restart "$SERVICE"

build/scripts/nats-js-watchdog_test.sh

@@ -0,0 +1,197 @@
+#!/bin/sh
+# Self-contained POSIX test for nats-js-watchdog.sh's restart-decision logic:
+# the age gate, cooldown, escalation, and js-probe exit-2 handling. No real
+# NATS or spx: systemctl, curl, and spx are all stubbed on PATH ahead of the
+# real ones, driven entirely by env vars this test controls.
+#
+# Run: sh build/scripts/nats-js-watchdog_test.sh
+set -eu
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+SCRIPT="${SCRIPT_DIR}/nats-js-watchdog.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "${WORK}"' EXIT
+
+FAKE_BIN="${WORK}/bin"
+mkdir -p "${FAKE_BIN}"
+
+FAILS=0
+fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); }
+pass() { echo "ok: $*"; }
+
+# Fake systemctl: `show` reports a controllable ExecMainStartTimestampMonotonic,
+# and `try-restart` just records that it was called rather than touching a
+# real unit.
+cat > "${FAKE_BIN}/systemctl" <<'EOF'
+#!/bin/sh
+case "$1" in
+    show)
+        echo "${FAKE_NATS_START_USEC:-0}"
+        ;;
+    try-restart)
+        echo "try-restart $2" >> "${FAKE_RESTART_LOG:-/dev/null}"
+        ;;
+esac
+EOF
+chmod +x "${FAKE_BIN}/systemctl"
+
+# Fake curl: js_healthy() only cares about the exit status.
+cat > "${FAKE_BIN}/curl" <<'EOF'
+#!/bin/sh
+[ "${FAKE_HEALTHY:-1}" = "1" ]
+EOF
+chmod +x "${FAKE_BIN}/curl"
+
+# Fake spx: js-probe's whole contract from the watchdog's side is its exit
+# code (0 ok, 1 JetStream refused, 2 inconclusive).
+cat > "${FAKE_BIN}/spx" <<'EOF'
+#!/bin/sh
+exit "${FAKE_SPX_EXIT:-0}"
+EOF
+chmod +x "${FAKE_BIN}/spx"
+
+PATH="${FAKE_BIN}:${PATH}"
+export PATH
+
+# reset_env <name>: fresh per-case dirs and a default env that already clears
+# the age, cooldown, and escalation gates (old enough NATS, no cooldown
+# stamp, no restart history) and a healthy, single-sample probe, so a case
+# only has to override what it is actually testing.
+reset_env() {
+    CASE="${WORK}/$1"
+    mkdir -p "${CASE}/state"
+    UPTIME_FILE="${CASE}/uptime"
+    printf '700.00 0.00\n' > "${UPTIME_FILE}"
+    STATE_DIR="${CASE}/state"
+    FAKE_RESTART_LOG="${CASE}/restart.log"
+    FAKE_NATS_START_USEC=1000000   # 1s: 700s uptime - 1s start = 699s age, clears the 600s default gate
+    FAKE_HEALTHY=1
+    FAKE_SPX_EXIT=0
+    SAMPLES=1
+    SAMPLE_GAP=0
+    NATS_MIN_AGE=600
+    RESTART_COOLDOWN=600
+    RESTART_ESCALATE_AT=3
+    SPX_BIN="${FAKE_BIN}/spx"
+    STDOUT="${CASE}/stdout"
+    STDERR="${CASE}/stderr"
+    export UPTIME_FILE STATE_DIR FAKE_RESTART_LOG FAKE_NATS_START_USEC \
+        FAKE_HEALTHY FAKE_SPX_EXIT SAMPLES SAMPLE_GAP NATS_MIN_AGE \
+        RESTART_COOLDOWN RESTART_ESCALATE_AT SPX_BIN
+}
+
+# Run the watchdog, capturing its exit code (without tripping set -e) + streams.
+invoke() { rc=0; sh "${SCRIPT}" >"${STDOUT}" 2>"${STDERR}" || rc=$?; }
+
+restarted() { [ -s "${FAKE_RESTART_LOG}" ]; }
+
+# --- Case 1: NATS younger than the age gate -> no restart ---
+reset_env young
+FAKE_NATS_START_USEC=690000000  # 690s: 700s uptime - 690s start = 10s age, well under 600s
+export FAKE_NATS_START_USEC
+invoke
+[ "${rc}" -eq 0 ] && pass "young: exit 0" || fail "young: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "young: must not restart while still in the recovery window" || pass "young: no restart"
+
+# --- Case 2: cooldown stamp fresh -> no restart ---
+reset_env cooldown
+echo 650 > "${STATE_DIR}/nats-js-watchdog.last-restart"
+invoke
+[ "${rc}" -eq 0 ] && pass "cooldown: exit 0" || fail "cooldown: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "cooldown: must not restart again inside the cooldown window" || pass "cooldown: no restart"
+
+# --- Case 3: escalation threshold reached while still failing -> no restart,
+# loud log. The probe must fail: a healthy probe at the threshold is case 3b,
+# and it re-arms rather than giving up. ---
+reset_env escalated
+FAKE_SPX_EXIT=1
+export FAKE_SPX_EXIT
+echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count"
+invoke
+[ "${rc}" -eq 0 ] && pass "escalated: exit 0" || fail "escalated: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "escalated: must not restart past the escalation threshold" || pass "escalated: no restart"
+grep -q 'giving up rather than looping' "${STDERR}" \
+    && pass "escalated: loud log emitted" || fail "escalated: escalation log missing: $(cat "${STDERR}")"
+
+# Second identical run: the node is still broken, so the decision stands, but
+# repeating the message every timer tick is what buried the real signal for 42
+# hours on node1.
+invoke
+grep -q 'giving up rather than looping' "${STDERR}" \
+    && fail "escalated: message repeated on every run instead of once per latch" \
+    || pass "escalated: message not repeated"
+
+# --- Case 3b: escalation threshold reached but JetStream has recovered ->
+# the watchdog must re-arm itself. This is the production bug: the escalation
+# check used to run before the probe, so the clear below was unreachable and
+# the node stayed disarmed until reboot. ---
+reset_env escalated_recovered
+echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count"
+: > "${STATE_DIR}/nats-js-watchdog.escalated"
+invoke
+[ "${rc}" -eq 0 ] && pass "escalated-recovered: exit 0" || fail "escalated-recovered: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "escalated-recovered: must not restart a recovered server" || pass "escalated-recovered: no restart"
+[ -f "${STATE_DIR}/nats-js-watchdog.restart-count" ] \
+    && fail "escalated-recovered: restart count must clear once JetStream is writable again" \
+    || pass "escalated-recovered: restart count cleared"
+[ -f "${STATE_DIR}/nats-js-watchdog.escalated" ] \
+    && fail "escalated-recovered: escalation marker must clear so a later failure is reported" \
+    || pass "escalated-recovered: escalation marker cleared"
+[ -s "${STDERR}" ] \
+    && fail "escalated-recovered: a recovered node must be silent: $(cat "${STDERR}")" \
+    || pass "escalated-recovered: silent"
+
+# --- Case 3c: re-armed watchdog acts again. One healthy run clears the latch,
+# and the next genuine failure must restart -- proving it is armed, not just
+# quiet. ---
+reset_env escalated_rearm
+echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count"
+invoke
+FAKE_SPX_EXIT=1
+export FAKE_SPX_EXIT
+invoke
+restarted && pass "rearm: restarts again after recovery cleared the latch" \
+    || fail "rearm: expected a restart once re-armed: $(cat "${STDERR}")"
+[ "$(cat "${STATE_DIR}/nats-js-watchdog.restart-count" 2>/dev/null)" = "1" ] \
+    && pass "rearm: restart count restarts from 1" \
+    || fail "rearm: expected count 1, got $(cat "${STATE_DIR}/nats-js-watchdog.restart-count" 2>/dev/null)"
+
+# --- Case 4: js-probe exits 2 (inconclusive) -> no restart ---
+reset_env inconclusive
+FAKE_SPX_EXIT=2
+export FAKE_SPX_EXIT
+invoke
+[ "${rc}" -eq 0 ] && pass "inconclusive: exit 0" || fail "inconclusive: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "inconclusive: exit 2 must never be read as a JetStream failure" || pass "inconclusive: no restart"
+grep -q 'inconclusive' "${STDERR}" \
+    && pass "inconclusive: reason logged" || fail "inconclusive: log missing: $(cat "${STDERR}")"
+
+# --- Case 5: genuine JetStream write failure -> restart happens ---
+reset_env genuine_failure
+FAKE_SPX_EXIT=1
+export FAKE_SPX_EXIT
+echo 2 > "${STATE_DIR}/nats-js-watchdog.restart-count"
+invoke
+[ "${rc}" -eq 0 ] && pass "genuine-failure: exit 0" || fail "genuine-failure: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && pass "genuine-failure: restart triggered" || fail "genuine-failure: expected a restart: $(cat "${STDERR}")"
+[ "$(cat "${STATE_DIR}/nats-js-watchdog.restart-count" 2>/dev/null)" = "3" ] \
+    && pass "genuine-failure: restart count incremented" || fail "genuine-failure: restart count not incremented"
+[ -f "${STATE_DIR}/nats-js-watchdog.last-restart" ] \
+    && pass "genuine-failure: cooldown stamp written" || fail "genuine-failure: cooldown stamp missing"
+
+# --- Case 6: healthy NATS -> no restart, and prior restart history clears ---
+reset_env healthy
+echo 2 > "${STATE_DIR}/nats-js-watchdog.restart-count"
+invoke
+[ "${rc}" -eq 0 ] && pass "healthy: exit 0" || fail "healthy: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "healthy: must not restart a healthy server" || pass "healthy: no restart"
+[ -f "${STATE_DIR}/nats-js-watchdog.restart-count" ] \
+    && fail "healthy: restart count must clear once JetStream is confirmed writable" \
+    || pass "healthy: restart count cleared"
+
+if [ "${FAILS}" -eq 0 ]; then
+    echo "PASS: all nats-js-watchdog cases"
+    exit 0
+fi
+echo "FAILED: ${FAILS} case(s)"
+exit 1

build/scripts/wait-for-nats.sh

@@ -26,6 +26,7 @@ fi
 # Final fallback
 NATS_HOST="${NATS_HOST:-127.0.0.1}"
 
+# shellcheck disable=SC2034  # loop count only, index unused
 for i in $(seq 1 "$TIMEOUT"); do
     if nc -z "$NATS_HOST" "$NATS_PORT" 2>/dev/null; then
         exit 0

build/systemd/regenerate-ssh-host-keys.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Regenerate SSH host keys
 ConditionPathExists=!/etc/ssh/ssh_host_ed25519_key

build/systemd/spinifex-awsgw.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex AWS Gateway
 PartOf=spinifex.target

build/systemd/spinifex-daemon.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 3
 [Unit]
 Description=Spinifex Daemon (VM Orchestration)
 PartOf=spinifex.target
@@ -38,12 +39,13 @@ PrivateDevices=no
 DevicePolicy=closed
 DeviceAllow=/dev/kvm rw
 DeviceAllow=/dev/net/tun rw
+DeviceAllow=/dev/vhost-net rw
 DeviceAllow=char-vfio rw
 DeviceAllow=/dev/vfio/vfio rw
 ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
-# /proc/sys/net/ipv4/conf: per-tap IMDS endpoints set rp_filter/accept_local, which
-# ProtectKernelTunables=yes would otherwise keep read-only.
-ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf
+# /proc/sys/net/ipv4/conf: ProtectKernelTunables would keep the per-tap IMDS
+# rp_filter/accept_local read-only; /etc/spinifex/firewall: the root helper writes peers.nft.
+ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf -/etc/spinifex/firewall
 
 # RG-9: hardening baseline
 ProtectHome=yes

build/systemd/spinifex-firewall.service

@@ -0,0 +1,25 @@
+# spinifex-unit-version: 1
+[Unit]
+Description=Spinifex host firewall (nftables input policy)
+Documentation=file:/etc/spinifex/firewall/spinifex.nft
+Wants=network-pre.target
+Before=network-pre.target shutdown.target
+DefaultDependencies=no
+ConditionPathExists=/etc/spinifex/firewall/peers.nft
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+StandardInput=null
+ProtectSystem=full
+ProtectHome=true
+ExecStart=/usr/local/lib/spinifex/spinifex-firewall-apply
+ExecReload=/usr/local/lib/spinifex/spinifex-firewall-apply
+TimeoutStopSec=10s
+
+# No ExecStop: flushing on stop would take vpcd's MASQUERADE and per-EIP FORWARD
+# rules with it. Remove the policy deliberately with
+# `nft delete table inet spinifex_filter`.
+
+[Install]
+WantedBy=sysinit.target

build/systemd/spinifex-guests.slice

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex guest VM tier
 Before=slices.target

build/systemd/spinifex-nats-watchdog.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 2
 [Unit]
 Description=Spinifex NATS JetStream liveness watchdog
 PartOf=spinifex.target
@@ -15,6 +16,10 @@ ExecStart=/var/lib/spinifex/nats-js-watchdog.sh
 # cutting it off. Only the failing path runs that long; a healthy node exits on
 # the first sample.
 TimeoutStopSec=45
+# Same 34s worst case bounds the start side: without this the unit inherits
+# oneshot's infinite default, so a genuinely hung run would sit in
+# activating(start) forever and could not be retriggered by its timer.
+TimeoutStartSec=120
 
 # Hardening (kept compatible with talking to PID1 over /run/systemd).
 NoNewPrivileges=yes

build/systemd/spinifex-nats-watchdog.timer

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex NATS JetStream liveness watchdog timer
 PartOf=spinifex.target

build/systemd/spinifex-nats.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex NATS Message Bus
 PartOf=spinifex.target

build/systemd/spinifex-northstar.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Northstar (Authoritative DNS)
 PartOf=spinifex.target

build/systemd/spinifex-predastore.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 3
 [Unit]
 Description=Spinifex Predastore (S3-compatible Object Storage)
 PartOf=spinifex.target
@@ -6,9 +7,9 @@ Wants=spinifex-nats.service
 
 [Service]
 EnvironmentFile=-/etc/spinifex/telemetry.env
-# Guest block I/O arrives with no caller trace; sample those roots at 5% so
-# they don't drown request traces. Request-linked spans are always kept.
-Environment=MULGA_ROOT_TRACE_RATIO=0.05
+# No MULGA_ROOT_TRACE_RATIO here: the sink's tail sampler sheds guest block I/O
+# instead, and it can see whether a trace errored. A head ratio decides before
+# the error exists, so it discarded failures at the same rate as successes.
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-storage
@@ -17,9 +18,9 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service predastore start
 Restart=on-failure
 RestartSec=5
-# Matches viperblock's drain budget: viperblock's WAL flush writes into
-# predastore, and After= already stops viperblock first, but this is the
-# backstop if that ordering is ever bypassed (direct stop, future edit).
+# Bounds predastore's own graceful stop. viperblock's WAL flush writes into
+# predastore, but After= already stops viperblock first (within its own 150s
+# budget); this is the backstop if that ordering is ever bypassed.
 TimeoutStopSec=30
 OOMScoreAdjust=-500
 # Legitimately the most memory-hungry infra service (S3 buffering/caching);

build/systemd/spinifex-qmp-collector.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex QMP Telemetry Collector (guest metrics)
 PartOf=spinifex.target

build/systemd/spinifex-shutdown.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex graceful guest drain
 PartOf=spinifex.target
@@ -11,11 +12,11 @@ RemainAfterExit=yes
 ExecStart=/bin/true
 # ExecStop fires on ANY target stop, including the stop half of `systemctl
 # restart spinifex.target` (a rolling deploy) -- PartOf propagates regardless
-# of whether the host is actually going down. --only-if-host-stopping drains
-# only when systemd itself is unwinding into shutdown.target; a plain restart
-# or `systemctl stop spinifex.target` falls through so QEMU/nbdkit survive
-# (KillMode=process) and the daemon reconnects to them on the way back up.
-ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --only-if-host-stopping
+# of whether the host is actually going down. --unless-restarting drains on a
+# real shutdown/reboot and on a plain `systemctl stop spinifex.target`; it
+# skips only when a restart of the stack is already queued, so QEMU/nbdkit
+# survive (KillMode=process) and the daemon reconnects on the way back up.
+ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --unless-restarting
 TimeoutStopSec=180
 
 [Install]

build/systemd/spinifex-system.slice

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex control/infra tier (NATS, predastore, viperblock, daemon, awsgw, vpcd, ui)
 Before=slices.target

build/systemd/spinifex-ui.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Web UI
 PartOf=spinifex.target

build/systemd/spinifex-viperblock.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 2
 [Unit]
 Description=Spinifex Viperblock (Block Storage)
 PartOf=spinifex.target
@@ -23,8 +24,10 @@ MemoryMax=25%
 # way guest QEMU survives a daemon restart. Tearing one out mid-write corrupts
 # that guest's filesystem; idle nbdkit is reaped explicitly on shutdown.
 KillMode=process
-# Bounds this process's own WAL drain, which the plugin caps at 20s.
-TimeoutStopSec=30
+# One worst-case nbdkit reap: killProcessGracePeriod (120s, utils.KillProcess)
+# plus its own shutdownDrainTimeout (20s, viperblock/nbd) and headroom — must
+# strictly exceed 120s now shutdownVolumes reaps concurrently, not serially.
+TimeoutStopSec=150
 EnvironmentFile=/etc/spinifex/systemd.env
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/
@@ -48,7 +51,8 @@ ProtectKernelTunables=yes
 ProtectKernelModules=yes
 ProtectKernelLogs=yes
 ProtectControlGroups=yes
-ProtectProc=invisible
+# ProtectProc must stay default: hidepid=invisible hides spinifex-daemon's
+# QEMU /proc entries (different uid) orphan nbdkit reaping must see.
 SystemCallArchitectures=native
 MemoryDenyWriteExecute=yes
 RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK

build/systemd/spinifex-vpcd.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex VPC Daemon (OVN Networking)
 PartOf=spinifex.target

build/systemd/spinifex.slice

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex resource-bounded slice (host ceiling)
 Before=slices.target

build/systemd/spinifex.target

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Infrastructure Platform
 After=network-online.target

cmd/ecs-agent/ecrresolver.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"net/http"
+	"net/url"
 	"strings"
 	"sync"
 
@@ -60,12 +61,25 @@ func (e *ecrResolver) client() (*http.Client, error) {
 	return c, nil
 }
 
-// isECRRef reports whether ref targets an ECR registry, i.e. its registry host
-// is "<acct>.dkr.ecr.<region>.<domain>". Only ECR pulls need a minted token;
-// public registries (docker.io and friends) pull anonymously, so refs without
-// an ECR host must not drag in IMDS host credentials.
-func isECRRef(ref string) bool {
+// gatewayRegistryHost returns the host[:port] of the co-located ECR registry
+// from the gateway URL; empty/unparseable yields "" (matches no ref). The
+// registry shares the gateway listener, so its port is the gateway port.
+func gatewayRegistryHost(gatewayURL string) string {
+	u, err := url.Parse(gatewayURL)
+	if err != nil {
+		return ""
+	}
+	return u.Host
+}
+
+// isECRRef reports whether ref targets an ECR registry: an "<acct>.dkr.ecr."
+// host, or one equal to gatewayHost (the agent's own co-located registry
+// advertised as a bare host/IP). Other refs pull anonymously, no token.
+func isECRRef(ref, gatewayHost string) bool {
 	host, _, _ := strings.Cut(ref, "/")
+	if gatewayHost != "" && host == gatewayHost {
+		return true
+	}
 	if !strings.Contains(host, ".") && !strings.Contains(host, ":") {
 		return false // no registry host -> docker.io default
 	}
@@ -75,7 +89,7 @@ func isECRRef(ref string) bool {
 // Authorize returns the ECR token's user/password and proxy endpoint for ref.
 // Non-ECR refs resolve to anonymous pull (empty credentials).
 func (e *ecrResolver) Authorize(ctx context.Context, ref string) (user, pass, endpoint string, err error) {
-	if !isECRRef(ref) {
+	if !isECRRef(ref, gatewayRegistryHost(e.gatewayURL)) {
 		return "", "", "", nil
 	}
 	c, err := e.creds.Retrieve(ctx)

cmd/ecs-agent/ecrresolver_test.go

@@ -115,17 +115,44 @@ func TestECRResolver_NonECRAnonymous(t *testing.T) {
 }
 
 func TestIsECRRef(t *testing.T) {
-	cases := map[string]bool{
-		"nginx:alpine":            false,
-		"docker.io/library/nginx": false,
-		"ghcr.io/org/app:1":       false,
-		"registry:5000/app:1":     false,
-		"123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1": true,
-		"123456789012.dkr.ecr.ap-southeast-2.amazonaws.com/x":    true,
+	cases := []struct {
+		ref, gatewayHost string
+		want             bool
+	}{
+		{"nginx:alpine", "192.168.1.13:9999", false},
+		{"docker.io/library/nginx", "192.168.1.13:9999", false},
+		{"ghcr.io/org/app:1", "192.168.1.13:9999", false},
+		{"registry:5000/app:1", "192.168.1.13:9999", false},
+		{"123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1", "192.168.1.13:9999", true},
+		{"123456789012.dkr.ecr.ap-southeast-2.amazonaws.com/x", "192.168.1.13:9999", true},
+		// Co-located gateway registry advertised as a bare host/IP.
+		{"192.168.1.13:9999/bunyip:latest", "192.168.1.13:9999", true},
+		{"192.168.1.13:9999/bunyip:latest", "", false},
 	}
-	for ref, want := range cases {
-		if got := isECRRef(ref); got != want {
-			t.Errorf("isECRRef(%q) = %v, want %v", ref, got, want)
+	for _, c := range cases {
+		if got := isECRRef(c.ref, c.gatewayHost); got != c.want {
+			t.Errorf("isECRRef(%q, %q) = %v, want %v", c.ref, c.gatewayHost, got, c.want)
 		}
 	}
 }
+
+// TestECRResolver_Authorize_BareHostGateway asserts that a ref whose host
+// equals the agent's own gateway host authenticates via the co-located ECR
+// registry, even though it lacks a ".dkr.ecr." name.
+func TestECRResolver_Authorize_BareHostGateway(t *testing.T) {
+	srv := gatewayStub(t, "AWS:jwt", "https://192.168.1.13:9999", http.StatusOK)
+	creds := stubCreds{c: credentials.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "sess",
+		Expiration: time.Now().Add(time.Hour),
+	}}
+	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
+
+	host := strings.TrimPrefix(srv.URL, "https://")
+	user, pass, _, err := r.Authorize(context.Background(), host+"/bunyip:latest")
+	if err != nil {
+		t.Fatalf("Authorize: %v", err)
+	}
+	if user != "AWS" || pass != "jwt" {
+		t.Errorf("got %q:%q, want AWS:jwt", user, pass)
+	}
+}

cmd/ecs-agent/runtime/runtime_test.go

@@ -1,71 +1,9 @@
 package runtime
 
 import (
-	"context"
-	"errors"
 	"testing"
 )
 
-// staticResolver is a test Resolver returning fixed creds or an error.
-type staticResolver struct {
-	user, pass, endpoint string
-	err                  error
-}
-
-func (s staticResolver) Authorize(_ context.Context, _ string) (string, string, string, error) {
-	return s.user, s.pass, s.endpoint, s.err
-}
-
-func TestFakePuller_RecordsAndDefaultsRef(t *testing.T) {
-	f := &FakePuller{}
-	img, err := f.Pull(context.Background(), PullSpec{Ref: "repo/app:1"}, staticResolver{user: "AWS"})
-	if err != nil {
-		t.Fatalf("Pull: %v", err)
-	}
-	if img.Ref != "repo/app:1" {
-		t.Errorf("Ref defaulted wrong: %q", img.Ref)
-	}
-	if len(f.Pulls) != 1 || f.Pulls[0].Ref != "repo/app:1" {
-		t.Errorf("Pulls = %+v", f.Pulls)
-	}
-	if len(f.Authzd) != 1 || f.Authzd[0] != "AWS" {
-		t.Errorf("Authzd = %+v", f.Authzd)
-	}
-}
-
-func TestFakePuller_ResolverError(t *testing.T) {
-	f := &FakePuller{}
-	_, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, staticResolver{err: errors.New("boom")})
-	if err == nil {
-		t.Fatal("expected resolver error to propagate")
-	}
-}
-
-func TestFakePuller_ProgrammedError(t *testing.T) {
-	f := &FakePuller{Err: errors.New("pull failed")}
-	if _, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil); err == nil {
-		t.Fatal("expected programmed error")
-	}
-}
-
-func TestFakePuller_OnPullOverrides(t *testing.T) {
-	want := Image{Ref: "x", Digest: "sha256:abc", Size: 42}
-	f := &FakePuller{Err: errors.New("ignored"), OnPull: func(PullSpec) (Image, error) {
-		return want, nil
-	}}
-	got, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil)
-	if err != nil || got != want {
-		t.Fatalf("got %+v, %v; want %+v", got, err, want)
-	}
-}
-
-func TestFakePuller_Close(t *testing.T) {
-	f := &FakePuller{}
-	if err := f.Close(); err != nil || !f.Closed {
-		t.Fatalf("Close: err=%v closed=%v", err, f.Closed)
-	}
-}
-
 func TestNormalizeRef(t *testing.T) {
 	cases := map[string]string{
 		"nginx:alpine":      "docker.io/library/nginx:alpine",

cmd/installer/firstboot/firstboot.go

@@ -146,6 +146,21 @@ hostnamectl set-hostname %s
 printf "nameserver 1.1.1.1\nnameserver 8.8.8.8\n" > /etc/resolvconf/resolv.conf.d/base
 resolvconf -u
 
+# The RTC on a fresh machine can be badly wrong — a flat CMOS battery is routine
+# on refurbished hardware. Credentials and TLS certificates are minted further
+# down, and a certificate issued against a wrong clock carries a NotBefore that
+# outlives the correction, so the clock has to be right before that happens.
+# chronyd is already running; makestep forces an immediate correction instead of
+# waiting out the slew. Bounded and non-fatal: an air-gapped install must still
+# complete, and rtcsync writes the corrected time back so later boots start close.
+echo "[firstboot] syncing clock before credentials are minted"
+chronyc -a makestep >/dev/null 2>&1 || true
+if chronyc waitsync 12 0.1 0 5 >/dev/null 2>&1; then
+    echo "[firstboot] clock synced: $(date -uIs)"
+else
+    echo "[firstboot] WARNING: clock not synced, continuing with $(date -uIs)"
+fi
+
 # Configure OVN networking.
 # br-wan (and br-lan if present) are Linux bridges managed by systemd-networkd
 # (declared in /etc/systemd/network/ by the installer). setup-ovn.sh
@@ -207,6 +222,12 @@ fi
 
 # Enable services to start, on reboot
 systemctl enable spinifex.target spinifex-banner.service
+# setup.sh enables this on the package install path, but the ISO chroot returns
+# before that line (no systemd to talk to), so the ISO needs it here. An nft
+# ruleset is runtime state and does not survive a reboot; without this the unit
+# never reapplies the policy and the node comes back up open. Enabled, not
+# started: ConditionPathExists holds it inert until the daemon writes a peer file.
+systemctl enable spinifex-firewall.service
 # WantedBy=timers.target only self-activates once enabled — without this the
 # JetStream ENOSPC-latch watchdog never runs and a full disk needs a manual restart.
 systemctl enable --now spinifex-nats-watchdog.timer

cmd/installer/install/steps_test.go

@@ -887,13 +887,6 @@ func TestInstallBootToolZFSStopsWhenADiskFails(t *testing.T) {
 	f.mustNotRun(t, "SPINIFEX_BOOT_TOOL_NVRAM=1")
 }
 
-func TestRefreshBootToolNoteOnlyAppliesToZFS(t *testing.T) {
-	// Log-only, but it is the operator's only record of the disk-replacement
-	// command, so both branches are walked.
-	refreshBootToolNote(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)})
-	refreshBootToolNote(DiskConfig{FS: FSExt4, Disks: disks(1, 40)})
-}
-
 func TestMaskSystemdUnitIsIdempotent(t *testing.T) {
 	root := t.TempDir()
 	if err := os.MkdirAll(filepath.Join(root, "etc/systemd/system"), 0o755); err != nil {
@@ -913,14 +906,6 @@ func TestMaskSystemdUnitIsIdempotent(t *testing.T) {
 	}
 }
 
-func TestCopyAssetsAreBestEffort(t *testing.T) {
-	root := t.TempDir()
-	// Neither asset exists outside the live ISO; a missing one must not fail
-	// the install, only the branding.
-	copyGrubFont(root)
-	copySplashImage(root)
-}
-
 func TestWriteFstabOnZFSHoldsNothingButSwap(t *testing.T) {
 	root := withMountRoot(t)
 	targetSkeleton(t, root)

cmd/installer/install/zfs.go

@@ -182,8 +182,7 @@ func ImportablePools() ([]string, error) {
 		// zpool exits non-zero when there is nothing to import, which is the
 		// normal case. Anything else — no binary, no /dev/zfs — must surface, or
 		// a machine with an existing pool looks empty to the caller.
-		var exitErr *exec.ExitError
-		if errors.As(err, &exitErr) {
+		if _, ok := errors.AsType[*exec.ExitError](err); ok {
 			return nil, nil
 		}
 		return nil, fmt.Errorf("zpool import: %w", err)

cmd/lb-agent/main.go

@@ -18,8 +18,8 @@ import (
 	"os/signal"
 	"syscall"
 
+	"github.com/mulgadc/bluebottle/pkg/otelsetup"
 	"github.com/mulgadc/spinifex/spinifex/lbagent"
-	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 
 	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
@@ -28,7 +28,7 @@ func main() {
 	// Without this the agent falls back to slog's default text handler, which folds
 	// attributes into the message so the console scraper recovers a level and nothing
 	// else. No OTLP export: the agent's only egress is the gateway it is watching.
-	otelsetup.SetDefaultJSONLogger(slog.LevelInfo)
+	otelsetup.SetDefaultJSONLogger("lb-agent", slog.LevelInfo)
 
 	var (
 		lbID       string

cmd/rds-agent/agent.go

@@ -15,6 +15,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/gwsign"
 	"github.com/mulgadc/spinifex/internal/rdsgw"
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 )
 
 // Overridable via -ldflags "-X main.version=...".
@@ -180,7 +181,7 @@ type Agent struct {
 	id              identity
 	cp              controlPlane
 	probe           *engineProbe
-	engine          *postgresEngine
+	engine          engine
 	handoffWriter   func(string, *handlers_rds.GetDBBootstrapConfigOutput) error
 	dataMountWaiter func(context.Context, string, string) error
 
@@ -195,10 +196,7 @@ type Agent struct {
 
 // Assembles from already-built parts, so tests can pass fakes; New builds the
 // production control plane and delegates here.
-func newAgent(cfg config, cp controlPlane, probe *engineProbe) *Agent {
-	if cfg.DataMount == "" {
-		cfg.DataMount = defaultDataMount
-	}
+func newAgent(cfg config, cp controlPlane, probe *engineProbe) (*Agent, error) {
 	if cfg.MountsFile == "" {
 		cfg.MountsFile = defaultMountsFile
 	}
@@ -211,11 +209,15 @@ func newAgent(cfg config, cp controlPlane, probe *engineProbe) *Agent {
 		cfg: cfg, id: id, cp: cp, probe: probe,
 		handoffWriter: writeHandoff, dataMountWaiter: waitForDataMount,
 	}
-	a.engine = newPostgresEngine(cfg, execCommandRunner, execSessionRunner, probe)
+	eng, err := newEngine(cfg, execCommandRunner, execSessionRunner, probe)
+	if err != nil {
+		return nil, err
+	}
+	a.engine = eng
 	a.hb = newHeartbeater(cp, probe, a.engine, handlers_rds.HeartbeatInterval)
 	a.cmd = newCommander(cp, newCommandRegistry(a.engine, newGuestStorage(cfg, execCommandRunner)), cfg.PollWait)
 	a.guard = newParamGuard(a.engine, probe, cp)
-	return a
+	return a, nil
 }
 
 // Does not wait for IMDS: the register loop rides out a datapath still coming up.
@@ -227,14 +229,17 @@ func New(cfg config) (*Agent, error) {
 	if err != nil {
 		return nil, err
 	}
-	return newAgent(cfg, cp, newEngineProbe(cfg, execProbeRunner)), nil
+	probe, err := newProbe(cfg, execProbeRunner)
+	if err != nil {
+		return nil, err
+	}
+	return newAgent(cfg, cp, probe)
 }
 
 func (a *Agent) Run(ctx context.Context) error {
-	// Named before the first dial. When register cannot reach the gateway the
-	// retry loop is all that survives, and the address it is dialing is what
-	// separates a mgmt NIC that came up without an address from a control plane
-	// that is genuinely down.
+	// Named before the first dial: when register cannot reach the gateway, the
+	// address it is dialing is what separates a mgmt NIC that came up without one
+	// from a control plane that is genuinely down.
 	slog.Info("rds-agent: starting",
 		"gatewayURL", a.cfg.GatewayURL, "dbInstanceIdentifier", a.id.DBInstanceIdentifier,
 		"agentVersion", a.id.AgentVersion)
@@ -245,6 +250,13 @@ func (a *Agent) Run(ctx context.Context) error {
 	}
 	a.hb.id, a.cmd.id, a.guard.id = a.id, a.id, a.id
 
+	// Ahead of the handoff, so rds-init never unblocks and no datadir is touched.
+	// An agent running on the wrong image would initialise one engine over
+	// another's data, coming up healthy and empty beside data nothing references.
+	if err := a.refuseEngineMismatch(ctx); err != nil {
+		return err
+	}
+
 	// Beating before the bootstrap keeps a stuck boot visible as a live VM with
 	// a down engine rather than as silence.
 	go a.hb.Run(ctx)
@@ -279,6 +291,26 @@ func (a *Agent) Run(ctx context.Context) error {
 	return nil
 }
 
+// The engine cloud-init says this VM was launched as, against the engine the
+// image bakes. An empty expectation is no assertion rather than a disagreement:
+// a VM launched before the control plane emitted RDS_ENGINE carries none.
+func (a *Agent) refuseEngineMismatch(ctx context.Context) error {
+	expected := strings.ToLower(strings.TrimSpace(a.cfg.Engine))
+	if expected == "" || expected == a.cfg.BakedEngine {
+		return nil
+	}
+
+	message := fmt.Sprintf("this VM was launched as engine %q but its image bakes %q; refusing to bootstrap",
+		expected, a.cfg.BakedEngine)
+	slog.ErrorContext(ctx, "rds-agent: "+message)
+	// Reported before returning, so the control plane learns why the instance
+	// never came up rather than only that it never did.
+	if _, err := a.cp.SubmitState(ctx, a.id, handlers_rds.EngineHealthUnhealthy, message); err != nil {
+		slog.ErrorContext(ctx, "rds-agent: reporting the engine mismatch failed", "err", err)
+	}
+	return errors.New(message)
+}
+
 const dataMountPollInterval = time.Second
 
 // waitForDataMount blocks the stateful agent loops until the kernel reports the
@@ -375,10 +407,10 @@ func retryObserved(ctx context.Context, what string, fn func(context.Context) er
 		}
 		if attempt >= retryErrorAttempt {
 			slog.ErrorContext(ctx, "rds-agent: "+what+" still failing, retrying",
-				"attempt", attempt, "retryIn", delay, "err", err)
+				"attempt", attempt, "retry_in_ms", otelsetup.Millis(delay), "err", err)
 		} else {
 			slog.WarnContext(ctx, "rds-agent: "+what+" failed, retrying",
-				"attempt", attempt, "retryIn", delay, "err", err)
+				"attempt", attempt, "retry_in_ms", otelsetup.Millis(delay), "err", err)
 		}
 
 		select {

cmd/rds-agent/agent_test.go

@@ -168,23 +168,31 @@ func (f *fakeControlPlane) snapshotReplies() [][]handlers_rds.CommandReply {
 	return out
 }
 
-// testConfig points the agent's file output at a temp dir and its probe at a
-// name no real binary answers to.
+// testConfig stamps the engine a real rds-postgres image bakes, so the agent
+// resolves the same guest layout it would in a guest, and points its file output
+// at a temp dir.
 func testConfig(t *testing.T) config {
 	t.Helper()
-	return config{
-		GatewayURL: "https://gw.test",
-		HandoffDir: filepath.Join(t.TempDir(), "spinifex-rds"),
-		EngineHost: defaultEngineHost,
-		EnginePort: defaultEnginePort,
-		PGIsReady:  "pg_isready",
-		PollWait:   time.Second,
+	cfg := testLoadConfig(t, enginePostgres)
+	cfg.GatewayURL = "https://gw.test"
+	cfg.HandoffDir = filepath.Join(t.TempDir(), "spinifex-rds")
+	cfg.PollWait = time.Second
+	return cfg
+}
+
+// newTestAgent builds the agent over a probe the test drives directly.
+func newTestAgent(t *testing.T, cfg config, cp controlPlane, run probeRunner) *Agent {
+	t.Helper()
+	a, err := newAgent(cfg, cp, newPostgresProbe(cfg, run))
+	if err != nil {
+		t.Fatalf("newAgent: %v", err)
 	}
+	return a
 }
 
 // staticProbe is a probeRunner that always reports the same exit code.
 func staticProbe(code int) probeRunner {
-	return func(context.Context, string, ...string) (int, error) { return code, nil }
+	return func(context.Context, string, ...string) (int, string, error) { return code, "", nil }
 }
 
 func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput {
@@ -199,6 +207,8 @@ func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput
 		MasterUsername:       "master",
 		MasterUserPassword:   password,
 		Port:                 5432,
+		ListenAddress:        testListenAddress,
+		ClientCIDR:           testClientCIDR,
 		DataVolumeID:         "vol-data-01",
 		DataVolumeSerial:     "voldata01",
 		VMGeneration:         1,
@@ -209,6 +219,13 @@ func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput
 
 const testPayloadID = "bp-0123456789abcdef"
 
+// The customer ENI address the engine binds, and the VPC range its client
+// authentication admits. writeHandoff refuses a config without either.
+const (
+	testListenAddress = "10.0.1.20"
+	testClientCIDR    = "10.0.0.0/16"
+)
+
 // Writes the completion receipt rds-init leaves behind, so the acknowledgement
 // step finds what a finished initialization would have produced.
 func writeReceipt(t *testing.T, a *Agent, payloadID, dbInstanceIdentifier string) {
@@ -272,7 +289,7 @@ func TestRun_RegistersThenDeliversBootstrap(t *testing.T) {
 	cp.bootstrapOut = bootstrapOutput(&password)
 
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	if err := runAgent(t, a); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
@@ -303,7 +320,7 @@ func TestRun_AttachModeNeedsNoPassword(t *testing.T) {
 	cp.bootstrapOut = bootstrapOutput(nil)
 
 	cfg := testConfig(t)
-	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
 
@@ -325,7 +342,7 @@ func TestRun_RetriesRegisterAndBootstrap(t *testing.T) {
 	cp.bootstrapErrs = []error{errors.New("ServerInternal")}
 
 	cfg := testConfig(t)
-	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
 
@@ -344,7 +361,7 @@ func TestBootstrap_RetriesHandoffWithoutRefetching(t *testing.T) {
 	password := "staged-secret"
 	cp.bootstrapOut = bootstrapOutput(&password)
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 
 	writes := 0
 	a.handoffWriter = func(_ string, got *handlers_rds.GetDBBootstrapConfigOutput) error {
@@ -385,7 +402,7 @@ func TestBootstrap_RetriesInitializeWithNoPassword(t *testing.T) {
 	}}
 	cp.bootstrapOut = bootstrapOutput(&password)
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 
 	var handed []*handlers_rds.GetDBBootstrapConfigOutput
 	a.handoffWriter = func(_ string, got *handlers_rds.GetDBBootstrapConfigOutput) error {
@@ -408,7 +425,7 @@ func TestAcknowledgeBootstrap_WaitsForTheReceiptThenRetiresIt(t *testing.T) {
 	cp := newFakeControlPlane()
 	cfg := testConfig(t)
 	cfg.DataMount = t.TempDir()
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	a.id = identity{DBInstanceIdentifier: "db-resolved"}
 	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1, dataVolumeID: "vol-data-01"}
 
@@ -447,7 +464,7 @@ func TestAcknowledgeBootstrap_IgnoresAForeignReceipt(t *testing.T) {
 			cp := newFakeControlPlane()
 			cfg := testConfig(t)
 			cfg.DataMount = t.TempDir()
-			a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+			a := newTestAgent(t, cfg, cp, staticProbe(0))
 			a.id = identity{DBInstanceIdentifier: "db-resolved"}
 			a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
 			writeReceipt(t, a, tc.payloadID, tc.dbID)
@@ -470,7 +487,7 @@ func TestAcknowledgeBootstrap_IsANoOpOnAttach(t *testing.T) {
 	cp := newFakeControlPlane()
 	cfg := testConfig(t)
 	cfg.DataMount = t.TempDir()
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 
 	if err := a.acknowledgeBootstrap(t.Context()); err != nil {
 		t.Fatalf("acknowledgeBootstrap: %v", err)
@@ -480,6 +497,52 @@ func TestAcknowledgeBootstrap_IsANoOpOnAttach(t *testing.T) {
 	}
 }
 
+// The image is the authority on which engine the agent runs. A VM launched as
+// another engine has to stop ahead of the handoff: rds-init would otherwise
+// initialise one engine over the other's datadir and come up healthy and empty
+// beside data nothing references.
+func TestRun_RefusesAnEngineTheImageDoesNotBake(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+
+	cfg := testConfig(t)
+	cfg.Engine = "mariadb"
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+
+	err := a.Run(t.Context())
+	if err == nil {
+		t.Fatal("Run returned nil for a VM launched as an engine this image does not bake")
+	}
+	if !strings.Contains(err.Error(), "mariadb") {
+		t.Errorf("error = %v, want it to name the engine the VM was launched as", err)
+	}
+	if _, statErr := os.Stat(filepath.Join(cfg.HandoffDir, handoffEnvFile)); !os.IsNotExist(statErr) {
+		t.Error("the agent wrote a bootstrap handoff for an engine it refused to run")
+	}
+	if n := len(cp.bootstrapReqs); n != 0 {
+		t.Errorf("bootstrap fetches = %d, want none before the refusal", n)
+	}
+	// Reported rather than only logged, so the control plane learns why the
+	// instance never came up rather than only that it never did.
+	states := cp.snapshotStates()
+	if len(states) != 1 || states[0].health != handlers_rds.EngineHealthUnhealthy {
+		t.Fatalf("submitted states = %+v, want a single unhealthy report", states)
+	}
+}
+
+// The same assertion when it agrees is not a gate: the boot goes on exactly as
+// it does for a VM launched before the control plane sent one at all.
+func TestRun_ProceedsWhenTheDeliveredEngineMatchesTheImage(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+
+	cfg := testConfig(t)
+	cfg.Engine = enginePostgres
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+}
+
 // A configured identifier is asserted on the wire so a mis-provisioned VM is
 // rejected by the gateway instead of adopting whatever it resolves to.
 func TestRun_SendsConfiguredIdentifier(t *testing.T) {
@@ -489,7 +552,7 @@ func TestRun_SendsConfiguredIdentifier(t *testing.T) {
 	cfg := testConfig(t)
 	cfg.DBInstanceIdentifier = "db-configured"
 	cfg.EngineVersion = "18.1"
-	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
 
@@ -503,12 +566,12 @@ func TestRun_WaitsForDataMountBeforePollingCommands(t *testing.T) {
 	cp := newFakeControlPlane()
 	cp.bootstrapOut = bootstrapOutput(nil)
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(2)))
+	a := newTestAgent(t, cfg, cp, staticProbe(2))
 
 	waiting := make(chan struct{})
 	release := make(chan struct{})
 	a.dataMountWaiter = func(ctx context.Context, mountsFile, target string) error {
-		if mountsFile != defaultMountsFile || target != defaultDataMount {
+		if mountsFile != defaultMountsFile || target != cfg.DataMount {
 			t.Errorf("mount waiter got %q/%q, want configured defaults", mountsFile, target)
 		}
 		close(waiting)
@@ -580,7 +643,7 @@ func TestRun_HeartbeatIncludesBootstrapFailure(t *testing.T) {
 	cp.registerOut.HeartbeatIntervalSeconds = 0
 
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(2)))
+	a := newTestAgent(t, cfg, cp, staticProbe(2))
 	a.hb.interval = 10 * time.Millisecond
 
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
@@ -609,7 +672,7 @@ func TestRun_HeartbeatsEngineHealth(t *testing.T) {
 	cp.registerOut.HeartbeatIntervalSeconds = 0
 
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	a.hb.interval = 10 * time.Millisecond
 
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
@@ -639,7 +702,7 @@ func TestRegister_AdoptsControlPlaneCadence(t *testing.T) {
 	cp.registerOut.HeartbeatIntervalSeconds = 45
 
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	if err := runAgent(t, a); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
@@ -656,12 +719,15 @@ func TestRun_HeartbeatReportsDownEngine(t *testing.T) {
 	cp.registerOut.HeartbeatIntervalSeconds = 0
 
 	cfg := testConfig(t)
-	probe := newEngineProbe(cfg, staticProbe(2))
+	probe := newPostgresProbe(cfg, staticProbe(2))
 	// The engine has served before, so a silent one now is a failure rather
 	// than a boot still in progress.
 	probe.seenHealthy = true
 
-	a := newAgent(cfg, cp, probe)
+	a, err := newAgent(cfg, cp, probe)
+	if err != nil {
+		t.Fatalf("newAgent: %v", err)
+	}
 	a.hb.interval = 10 * time.Millisecond
 
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
@@ -752,7 +818,7 @@ func TestAcknowledgeBootstrap_StopsOnATerminalDenial(t *testing.T) {
 	}}
 	cfg := testConfig(t)
 	cfg.DataMount = t.TempDir()
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	a.id = identity{DBInstanceIdentifier: "db-resolved"}
 	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
 	writeReceipt(t, a, testPayloadID, "db-resolved")
@@ -780,7 +846,7 @@ func TestAcknowledgeBootstrap_DoesNotReportAnAbsentReceiptAsAFailure(t *testing.
 	cp := newFakeControlPlane()
 	cfg := testConfig(t)
 	cfg.DataMount = t.TempDir()
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	a.id = identity{DBInstanceIdentifier: "db-resolved"}
 	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
 

cmd/rds-agent/bootstrap.go

@@ -32,10 +32,9 @@ func (a *Agent) bootstrap(ctx context.Context) error {
 		if fetched == nil {
 			return fmt.Errorf("bootstrap fetch returned no config")
 		}
-		// A control plane that predates the encrypted payload answers initialize
-		// with nothing to initialise with. Retrying re-dials the worker queue
-		// group until an upgraded node answers, so a mixed-version fleet heals
-		// within one boot rather than needing a reboot.
+		// A control plane predating the encrypted payload answers initialize with
+		// nothing to initialise with. Retrying re-dials the queue group until an
+		// upgraded node answers, so a mixed-version fleet heals within one boot.
 		if fetched.Mode == handlers_rds.BootstrapModeInitialize &&
 			(fetched.MasterUserPassword == nil || *fetched.MasterUserPassword == "") {
 			return fmt.Errorf("bootstrap fetch returned mode=%s with no master password", fetched.Mode)
@@ -61,7 +60,6 @@ func (a *Agent) bootstrap(ctx context.Context) error {
 	a.hb.clearBootstrapFailure()
 	if cfg.Port > 0 {
 		a.probe.setPort(int(cfg.Port))
-		a.engine.setPort(int(cfg.Port))
 	}
 	if cfg.BootstrapPending && cfg.PayloadID != "" {
 		a.pending = &pendingBootstrap{
@@ -98,6 +96,15 @@ func writeHandoff(dir string, cfg *handlers_rds.GetDBBootstrapConfigOutput) erro
 	if cfg.MasterUsername == "" {
 		return fmt.Errorf("bootstrap config carries no master username")
 	}
+	// Half a handoff would leave rds-init deciding what to bind and who to admit,
+	// and the only answers it could reach on its own are the wildcards this
+	// contract exists to remove.
+	if cfg.ListenAddress == "" {
+		return fmt.Errorf("bootstrap config carries no listen address for the engine to bind")
+	}
+	if cfg.ClientCIDR == "" {
+		return fmt.Errorf("bootstrap config carries no client CIDR to scope client authentication to")
+	}
 	if err := os.MkdirAll(dir, handoffDirMode); err != nil {
 		return fmt.Errorf("create handoff dir %s: %w", dir, err)
 	}
@@ -107,7 +114,11 @@ func writeHandoff(dir string, cfg *handlers_rds.GetDBBootstrapConfigOutput) erro
 		return fmt.Errorf("secure handoff dir %s: %w", dir, err)
 	}
 
-	if err := writeHandoffFile(dir, handoffParamsFile, renderParameters(cfg.Parameters)); err != nil {
+	params, err := renderParameters(cfg.Engine, cfg.Parameters)
+	if err != nil {
+		return err
+	}
+	if err := writeHandoffFile(dir, handoffParamsFile, params); err != nil {
 		return err
 	}
 	// Cert and key are written as a pair; half of one would start the engine
@@ -132,6 +143,8 @@ func renderBootstrapEnv(cfg *handlers_rds.GetDBBootstrapConfigOutput) string {
 	writeEnvLine(&b, "RDS_MODE", cfg.Mode)
 	writeEnvLine(&b, "RDS_DB_INSTANCE_IDENTIFIER", cfg.DBInstanceIdentifier)
 	writeEnvLine(&b, "RDS_MASTER_USERNAME", cfg.MasterUsername)
+	writeEnvLine(&b, "RDS_LISTEN_ADDRESS", cfg.ListenAddress)
+	writeEnvLine(&b, "RDS_CLIENT_CIDR", cfg.ClientCIDR)
 	writeEnvLine(&b, "RDS_DATA_VOLUME_ID", cfg.DataVolumeID)
 	writeEnvLine(&b, "RDS_DATA_VOLUME_SERIAL", cfg.DataVolumeSerial)
 	writeEnvLine(&b, "RDS_VM_GENERATION", strconv.FormatInt(cfg.VMGeneration, 10))
@@ -177,18 +190,23 @@ func shellQuote(s string) string {
 	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
 }
 
-// postgresql.conf syntax. Values are quoted so a setting with a space or unit
-// suffix survives; the engine accepts quoted numerics and booleans too.
-func renderParameters(params []handlers_rds.Parameter) string {
+// postgresql.conf syntax, which MariaDB's option files also parse. Values are
+// quoted so a space or unit suffix survives. Names are the engine's startup
+// spellings, not the customer's: the wrong one leaves the engine unable to boot.
+func renderParameters(engineName string, params []handlers_rds.Parameter) (string, error) {
+	engine, err := handlers_rds.LookupEngine(engineName)
+	if err != nil {
+		return "", fmt.Errorf("render the resolved parameters: %w", err)
+	}
 	var b strings.Builder
 	b.WriteString("# Resolved parameter group, written by rds-agent.\n")
 	for _, p := range params {
 		if p.Name == "" {
 			continue
 		}
-		fmt.Fprintf(&b, "%s = '%s'\n", p.Name, strings.ReplaceAll(p.Value, "'", "''"))
+		fmt.Fprintf(&b, "%s = '%s'\n", engine.OptionFileName(p.Name), strings.ReplaceAll(p.Value, "'", "''"))
 	}
-	return b.String()
+	return b.String(), nil
 }
 
 // Atomic: a temp file in the same directory, renamed over the target. The mode

cmd/rds-agent/bootstrap_test.go

@@ -16,7 +16,10 @@ func TestWriteHandoff_WritesEveryFileRootOnly(t *testing.T) {
 
 	err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:               handlers_rds.BootstrapModeInitialize,
+		Engine:             "postgres",
 		MasterUsername:     "master",
+		ListenAddress:      "10.0.1.20",
+		ClientCIDR:         "10.0.0.0/16",
 		MasterUserPassword: &password,
 		DBName:             "appdb",
 		Port:               5433,
@@ -82,7 +85,10 @@ func TestWriteHandoff_AttachOmitsPassword(t *testing.T) {
 	dir := filepath.Join(t.TempDir(), "handoff")
 	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:           handlers_rds.BootstrapModeAttach,
+		Engine:         "postgres",
 		MasterUsername: "master",
+		ListenAddress:  "10.0.1.20",
+		ClientCIDR:     "10.0.0.0/16",
 		Port:           5432,
 	}); err != nil {
 		t.Fatalf("writeHandoff: %v", err)
@@ -112,7 +118,10 @@ func TestWriteHandoff_QuotesShellMetacharacters(t *testing.T) {
 
 	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:               handlers_rds.BootstrapModeInitialize,
+		Engine:             "postgres",
 		MasterUsername:     "master",
+		ListenAddress:      "10.0.1.20",
+		ClientCIDR:         "10.0.0.0/16",
 		MasterUserPassword: &password,
 		Port:               5432,
 	}); err != nil {
@@ -132,7 +141,10 @@ func TestWriteHandoff_SkipsHalfATLSPair(t *testing.T) {
 	dir := filepath.Join(t.TempDir(), "handoff")
 	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:               handlers_rds.BootstrapModeAttach,
+		Engine:             "postgres",
 		MasterUsername:     "master",
+		ListenAddress:      "10.0.1.20",
+		ClientCIDR:         "10.0.0.0/16",
 		Port:               5432,
 		ServingCertificate: "CERT",
 	}); err != nil {
@@ -149,8 +161,9 @@ func TestWriteHandoff_SkipsHalfATLSPair(t *testing.T) {
 func TestWriteHandoff_RejectsConfigWithNoMasterUser(t *testing.T) {
 	dir := filepath.Join(t.TempDir(), "handoff")
 	err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
-		Mode: handlers_rds.BootstrapModeAttach,
-		Port: 5432,
+		Mode:   handlers_rds.BootstrapModeAttach,
+		Engine: "postgres",
+		Port:   5432,
 	})
 	if err == nil {
 		t.Fatal("writeHandoff accepted a config with no master username, want an error")
@@ -170,7 +183,10 @@ func TestWriteHandoff_TightensExistingDirectory(t *testing.T) {
 
 	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:           handlers_rds.BootstrapModeAttach,
+		Engine:         "postgres",
 		MasterUsername: "master",
+		ListenAddress:  "10.0.1.20",
+		ClientCIDR:     "10.0.0.0/16",
 		Port:           5432,
 	}); err != nil {
 		t.Fatalf("writeHandoff: %v", err)
@@ -219,3 +235,46 @@ func findLine(t *testing.T, body, prefix string) string {
 	t.Fatalf("no line starting %q in:\n%s", prefix, body)
 	return ""
 }
+
+// The include rds-init installs is what the engine starts on, so the spelling
+// written here is the one that has to boot.
+func TestRenderParameters_UsesTheEngineStartupSpelling(t *testing.T) {
+	params := []handlers_rds.Parameter{
+		{Name: "time_zone", Value: "SYSTEM"},
+		{Name: "max_connections", Value: "85"},
+	}
+
+	rendered, err := renderParameters("mariadb", params)
+	if err != nil {
+		t.Fatalf("renderParameters: %v", err)
+	}
+	if !strings.Contains(rendered, "default_time_zone = 'SYSTEM'\n") {
+		t.Errorf("time_zone was not written under its startup spelling:\n%s", rendered)
+	}
+	if strings.Contains(rendered, "\ntime_zone = ") {
+		t.Errorf("time_zone reached the option file, which mariadbd refuses:\n%s", rendered)
+	}
+	if !strings.Contains(rendered, "max_connections = '85'\n") {
+		t.Errorf("an unmapped parameter was not written unchanged:\n%s", rendered)
+	}
+}
+
+func TestRenderParameters_PostgresKeepsEveryName(t *testing.T) {
+	rendered, err := renderParameters("postgres", []handlers_rds.Parameter{
+		{Name: "shared_buffers", Value: "262144"},
+	})
+	if err != nil {
+		t.Fatalf("renderParameters: %v", err)
+	}
+	if !strings.Contains(rendered, "shared_buffers = '262144'\n") {
+		t.Errorf("postgres name was rewritten:\n%s", rendered)
+	}
+}
+
+// Failing closed beats writing a file the engine may refuse: the bad include
+// would already be on the data volume by the time the server rejected it.
+func TestRenderParameters_RefusesAnUnknownEngine(t *testing.T) {
+	if _, err := renderParameters("", nil); err == nil {
+		t.Fatal("renderParameters accepted a config naming no engine, want an error")
+	}
+}

cmd/rds-agent/config.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/mulgadc/spinifex/internal/guestenv"
@@ -11,18 +13,11 @@ const (
 	defaultEnvFile    = "/etc/spinifex-rds/agent.env"
 	defaultGatewayCA  = "/etc/spinifex-rds/gateway-ca.pem"
 	defaultHandoffDir = "/run/spinifex-rds"
-	defaultEngineHost = "127.0.0.1"
-	defaultEnginePort = 5432
-	defaultPGIsReady  = "pg_isready"
-
-	// The guest layout rds-init lays down. Overridable for the same reason it is
-	// there: a sibling engine preset points at its own paths.
-	defaultPGBin         = "/usr/libexec/postgresql18"
-	defaultPGData        = "/var/lib/postgresql/18/data"
-	defaultSocketDir     = "/run/postgresql"
-	defaultPGUser        = "postgres"
-	defaultRCService     = "rc-service"
-	defaultEngineSvcName = "postgresql"
+	// Where setup.sh stamps the engine the image bakes. The agent builds its
+	// engine implementation from this file rather than from anything delivered.
+	defaultEngineFile         = "/etc/spinifex-rds/engine"
+	defaultRCService          = "rc-service"
+	defaultEngineProbeTimeout = 5 * time.Second
 	// The long-poll window the agent asks the gateway to hold a request open
 	// for. The gateway caps it at 20s.
 	defaultPollWait = 20 * time.Second
@@ -42,21 +37,34 @@ type config struct {
 	// version alone rather than clearing it.
 	EngineVersion string
 	HandoffDir    string
-	EngineHost    string
 	EnginePort    int
-	// Overridable so a test or a sibling engine preset can point at its own.
-	PGIsReady string
-	PollWait  time.Duration
+	PollWait      time.Duration
+
+	// The engine the control plane launched this VM as, delivered by cloud-init,
+	// and the engine the image itself bakes. Two independent statements of the
+	// same fact: the agent refuses to bootstrap when they disagree.
+	Engine      string
+	EngineFile  string
+	BakedEngine string
 
 	// Where the command handlers reach the engine: the client binaries, the
 	// datadir the parameter file is installed into, the socket they connect
 	// over, the OS user they drop to, and the service the stop goes through.
-	PGBin         string
-	PGData        string
+	EngineBinDir  string
+	EngineDataDir string
 	SocketDir     string
-	PGUser        string
+	EngineUser    string
 	RCService     string
 	EngineService string
+	// Where the engine records the pid its probe checks for liveness, for an
+	// engine whose own client cannot tell a server still recovering from one that
+	// is not running at all.
+	EnginePidFile      string
+	EngineProbeTimeout time.Duration
+	// Where the engine records why it would not start, which the probe quotes
+	// when the server is up but not serving. Overridable so a test can point it
+	// at a fixture.
+	EngineErrorLog string
 
 	// Where the data volume is mounted, and the two kernel surfaces a storage
 	// grow resolves its device from. Overridable so a test can point them at
@@ -75,19 +83,21 @@ func loadConfig(envFile string) config {
 		GatewayCA:            get("RDS_GATEWAY_CA"),
 		Region:               get("RDS_REGION"),
 		DBInstanceIdentifier: get("RDS_DB_INSTANCE_IDENTIFIER"),
+		Engine:               get("RDS_ENGINE"),
+		EngineFile:           get("RDS_ENGINE_FILE"),
 		EngineVersion:        get("RDS_ENGINE_VERSION"),
 		HandoffDir:           get("RDS_HANDOFF_DIR"),
-		EngineHost:           get("RDS_ENGINE_HOST"),
-		PGIsReady:            get("RDS_PG_ISREADY"),
-		PGBin:                get("RDS_PG_BIN"),
-		PGData:               get("RDS_PGDATA"),
+		EngineBinDir:         get("RDS_ENGINE_BIN"),
+		EngineDataDir:        get("RDS_ENGINE_DATA"),
 		SocketDir:            get("RDS_SOCKET_DIR"),
-		PGUser:               get("RDS_PG_USER"),
+		EngineUser:           get("RDS_ENGINE_USER"),
 		RCService:            get("RDS_RC_SERVICE"),
 		EngineService:        get("RDS_ENGINE_SERVICE"),
+		EnginePidFile:        get("RDS_ENGINE_PIDFILE"),
+		EngineErrorLog:       get("RDS_ENGINE_LOG"),
 		DataMount:            get("RDS_DATA_MOUNT"),
-		EnginePort:           defaultEnginePort,
 		PollWait:             defaultPollWait,
+		EngineProbeTimeout:   defaultEngineProbeTimeout,
 	}
 	if cfg.GatewayCA == "" {
 		cfg.GatewayCA = defaultGatewayCA
@@ -95,35 +105,20 @@ func loadConfig(envFile string) config {
 	if cfg.HandoffDir == "" {
 		cfg.HandoffDir = defaultHandoffDir
 	}
-	if cfg.EngineHost == "" {
-		cfg.EngineHost = defaultEngineHost
-	}
-	if cfg.PGIsReady == "" {
-		cfg.PGIsReady = defaultPGIsReady
-	}
-	if cfg.PGBin == "" {
-		cfg.PGBin = defaultPGBin
-	}
-	if cfg.PGData == "" {
-		cfg.PGData = defaultPGData
-	}
-	if cfg.SocketDir == "" {
-		cfg.SocketDir = defaultSocketDir
-	}
-	if cfg.PGUser == "" {
-		cfg.PGUser = defaultPGUser
+	if cfg.EngineFile == "" {
+		cfg.EngineFile = defaultEngineFile
 	}
 	if cfg.RCService == "" {
 		cfg.RCService = defaultRCService
 	}
-	if cfg.EngineService == "" {
-		cfg.EngineService = defaultEngineSvcName
-	}
+	// The image's own stamp rather than the delivered engine: an agent that took
+	// its layout from the payload could run one engine's implementation against
+	// another engine's datadir, which is the failure the stamp exists to prevent.
+	cfg.BakedEngine = readBakedEngine(cfg.EngineFile)
+	cfg.applyLayout(engineLayouts[cfg.BakedEngine])
+
 	// rds-datadir reads the same RDS_DATA_MOUNT, so the two agree on where the
 	// volume landed without either asserting it to the other.
-	if cfg.DataMount == "" {
-		cfg.DataMount = defaultDataMount
-	}
 	cfg.MountsFile = defaultMountsFile
 	cfg.SysBlock = defaultSysBlock
 	// The authoritative port comes from the bootstrap config; this is only what
@@ -138,5 +133,53 @@ func loadConfig(envFile string) config {
 			cfg.PollWait = d
 		}
 	}
+	if v := get("RDS_ENGINE_PROBE_TIMEOUT"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.EngineProbeTimeout = d
+		}
+	}
 	return cfg
 }
+
+// Fills in whatever the delivered configuration left unset. An override always
+// wins, so a test can point any of these at a fixture. An unrecognised engine
+// leaves the layout empty, and New refuses rather than guessing at one.
+func (c *config) applyLayout(layout engineLayout) {
+	if c.EngineBinDir == "" {
+		c.EngineBinDir = layout.binDir
+	}
+	if c.EngineDataDir == "" {
+		c.EngineDataDir = layout.dataDir
+	}
+	if c.SocketDir == "" {
+		c.SocketDir = layout.socketDir
+	}
+	if c.EngineUser == "" {
+		c.EngineUser = layout.osUser
+	}
+	if c.EngineService == "" {
+		c.EngineService = layout.service
+	}
+	if c.EnginePidFile == "" {
+		c.EnginePidFile = layout.pidFile
+	}
+	if c.EngineErrorLog == "" {
+		c.EngineErrorLog = layout.errorLog
+	}
+	if c.DataMount == "" {
+		c.DataMount = layout.dataMount
+	}
+	if c.EnginePort == 0 {
+		c.EnginePort = layout.port
+	}
+}
+
+// The engine the image bakes. An absent or unreadable stamp reads as empty,
+// which New refuses: an agent that guessed would defeat the stamp.
+func readBakedEngine(path string) string {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return ""
+	}
+	return strings.ToLower(strings.TrimSpace(string(raw)))
+}

cmd/rds-agent/config_test.go

@@ -3,21 +3,34 @@ package main
 import (
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 )
 
+// Stamps the engine a real image bakes and loads the configuration against it,
+// which is what selects the guest layout the agent's defaults come from.
+func testLoadConfig(t *testing.T, engine string) config {
+	t.Helper()
+	stamp := filepath.Join(t.TempDir(), "engine")
+	if err := os.WriteFile(stamp, []byte(engine+"\n"), 0o444); err != nil {
+		t.Fatalf("write the engine stamp: %v", err)
+	}
+	t.Setenv("RDS_ENGINE_FILE", stamp)
+	return loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+}
+
 func TestLoadConfig_Defaults(t *testing.T) {
-	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+	cfg := testLoadConfig(t, enginePostgres)
 
 	if cfg.GatewayCA != defaultGatewayCA || cfg.HandoffDir != defaultHandoffDir {
 		t.Errorf("CA/handoff = %q/%q, want the built-in defaults", cfg.GatewayCA, cfg.HandoffDir)
 	}
-	if cfg.EngineHost != defaultEngineHost || cfg.EnginePort != defaultEnginePort {
-		t.Errorf("engine endpoint = %s:%d, want the local default", cfg.EngineHost, cfg.EnginePort)
+	if cfg.PollWait != defaultPollWait {
+		t.Errorf("pollWait = %v, want the built-in default", cfg.PollWait)
 	}
-	if cfg.PGIsReady != defaultPGIsReady || cfg.PollWait != defaultPollWait {
-		t.Errorf("probe/pollWait = %q/%v, want the built-in defaults", cfg.PGIsReady, cfg.PollWait)
+	if cfg.EngineProbeTimeout != defaultEngineProbeTimeout {
+		t.Errorf("engine probe timeout = %v, want %v", cfg.EngineProbeTimeout, defaultEngineProbeTimeout)
 	}
 	// Identity is optional: the gateway resolves it from the caller's
 	// credentials, so an agent with no configured identifier is normal.
@@ -26,11 +39,94 @@ func TestLoadConfig_Defaults(t *testing.T) {
 	}
 }
 
+// The guest layout is the image's to state, not the control plane's: everything
+// the agent reaches the engine through comes from the stamp it was built with.
+func TestLoadConfig_TakesTheLayoutFromTheBakedEngine(t *testing.T) {
+	cfg := testLoadConfig(t, enginePostgres)
+
+	if cfg.BakedEngine != enginePostgres {
+		t.Fatalf("baked engine = %q, want %q", cfg.BakedEngine, enginePostgres)
+	}
+	layout := engineLayouts[enginePostgres]
+	if cfg.EngineBinDir != layout.binDir || cfg.EngineDataDir != layout.dataDir {
+		t.Errorf("bin/data = %q/%q, want the PostgreSQL layout", cfg.EngineBinDir, cfg.EngineDataDir)
+	}
+	if cfg.EngineUser != layout.osUser || cfg.EngineService != layout.service {
+		t.Errorf("user/service = %q/%q, want the PostgreSQL layout", cfg.EngineUser, cfg.EngineService)
+	}
+	if cfg.SocketDir != layout.socketDir || cfg.DataMount != layout.dataMount {
+		t.Errorf("socket/mount = %q/%q, want the PostgreSQL layout", cfg.SocketDir, cfg.DataMount)
+	}
+	if cfg.EnginePort != layout.port {
+		t.Errorf("port = %d, want the PostgreSQL default %d", cfg.EnginePort, layout.port)
+	}
+}
+
+// An image with no stamp cannot say which engine it is, and an agent that
+// guessed would be the failure the stamp exists to prevent.
+func TestNewEngine_RefusesAnImageWithNoEngineStamp(t *testing.T) {
+	t.Setenv("RDS_ENGINE_FILE", filepath.Join(t.TempDir(), "absent-engine"))
+	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+
+	if _, err := newProbe(cfg, staticProbe(0)); err == nil {
+		t.Error("newProbe built a probe for an image carrying no engine stamp")
+	}
+	if _, err := newEngine(cfg, nil, nil, nil); err == nil {
+		t.Error("newEngine built an engine for an image carrying no engine stamp")
+	}
+}
+
+func TestNewEngine_RefusesAnEngineItDoesNotImplement(t *testing.T) {
+	cfg := testLoadConfig(t, "oracle")
+
+	_, err := newEngine(cfg, nil, nil, nil)
+	if err == nil {
+		t.Fatal("newEngine built an engine this agent does not implement")
+	}
+	if !strings.Contains(err.Error(), "oracle") {
+		t.Errorf("error = %v, want it to name the engine the image bakes", err)
+	}
+}
+
+// The implementation is the agent's, but the definition it validates names and
+// classifies parameters against is the control plane's, which now offers MariaDB.
+func TestNewEngine_BuildsMariaDBForAMariaDBImage(t *testing.T) {
+	cfg := testLoadConfig(t, engineMariaDB)
+	probe, err := newProbe(cfg, staticProbe(0))
+	if err != nil {
+		t.Fatalf("newProbe: %v", err)
+	}
+
+	built, err := newEngine(cfg, nil, nil, probe)
+	if err != nil {
+		t.Fatalf("newEngine: %v", err)
+	}
+	if _, ok := built.(*mariadbEngine); !ok {
+		t.Errorf("newEngine built %T for a MariaDB image", built)
+	}
+}
+
+// The probe reads the pidfile Alpine's packaged service starts mariadbd with,
+// because nothing in the generated configuration can move it: the service passes
+// --pid-file on the command line, which beats an option file.
+func TestLoadConfig_TakesTheMariaDBPidfileFromTheLayout(t *testing.T) {
+	cfg := testLoadConfig(t, engineMariaDB)
+
+	if cfg.EnginePidFile != engineLayouts[engineMariaDB].pidFile {
+		t.Errorf("pidfile = %q, want the MariaDB layout's", cfg.EnginePidFile)
+	}
+	if cfg.EnginePidFile == "" {
+		t.Error("an engine whose probe needs a pidfile was given none")
+	}
+}
+
 func TestLoadConfig_ReadsEnvFile(t *testing.T) {
+	t.Setenv("RDS_ENGINE_FILE", filepath.Join(t.TempDir(), "absent-engine"))
 	path := filepath.Join(t.TempDir(), "agent.env")
 	body := "RDS_GATEWAY_URL=https://gw.internal:9999\n" +
 		"RDS_REGION=ap-southeast-2\n" +
 		"RDS_DB_INSTANCE_IDENTIFIER=db-1\n" +
+		"RDS_ENGINE=postgres\n" +
 		"RDS_ENGINE_VERSION=18.1\n" +
 		"RDS_ENGINE_PORT=6543\n" +
 		"RDS_POLL_WAIT=5s\n"
@@ -45,19 +141,22 @@ func TestLoadConfig_ReadsEnvFile(t *testing.T) {
 	if cfg.DBInstanceIdentifier != "db-1" || cfg.EngineVersion != "18.1" {
 		t.Errorf("identity = %q/%q, want db-1/18.1", cfg.DBInstanceIdentifier, cfg.EngineVersion)
 	}
+	if cfg.Engine != enginePostgres {
+		t.Errorf("delivered engine = %q, want the one cloud-init carried", cfg.Engine)
+	}
 	if cfg.EnginePort != 6543 || cfg.PollWait != 5*time.Second {
 		t.Errorf("port/pollWait = %d/%v, want 6543/5s", cfg.EnginePort, cfg.PollWait)
 	}
 }
 
-// An unparseable override falls back to the default rather than leaving the
-// agent with a zero port or a zero-length long poll.
+// An unparseable override falls back to the layout's own default rather than
+// leaving the agent with a zero port or a zero-length long poll.
 func TestLoadConfig_IgnoresUnusableOverrides(t *testing.T) {
 	t.Setenv("RDS_ENGINE_PORT", "not-a-port")
 	t.Setenv("RDS_POLL_WAIT", "forever")
 
-	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
-	if cfg.EnginePort != defaultEnginePort || cfg.PollWait != defaultPollWait {
+	cfg := testLoadConfig(t, enginePostgres)
+	if cfg.EnginePort != engineLayouts[enginePostgres].port || cfg.PollWait != defaultPollWait {
 		t.Errorf("port/pollWait = %d/%v, want the defaults", cfg.EnginePort, cfg.PollWait)
 	}
 }

cmd/rds-agent/defaultparams_test.go

@@ -0,0 +1,73 @@
+package main
+
+//test:in-package — the agent is a main package, which has no external test
+// package to import it from, and this covers the unexported default parameter
+// set the image bakes.
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The include the image build validates against a real mariadbd. It is checked
+// in rather than generated during the build because the build runs inside the
+// libguestfs appliance, where there is no Go toolchain to resolve the catalog.
+const mariadbDefaultParametersFixture = "../../scripts/images/rds-mariadb/default-parameters.cnf"
+
+// Byte-for-byte what rds-init installs on a smallest-class instance that
+// overrides nothing. Regenerate with RDS_UPDATE_FIXTURES=1 go test ./cmd/rds-agent/.
+//
+// Keeping it in step matters because setup.sh feeds exactly this file to
+// mariadbd: a catalog that grew a name the server refuses at startup would
+// otherwise reach a customer as an instance that never boots.
+func TestMariaDBDefaultParametersFixture_MatchesTheCatalog(t *testing.T) {
+	engine, err := handlers_rds.LookupEngine("mariadb")
+	if err != nil {
+		t.Fatalf("LookupEngine: %v", err)
+	}
+	class := handlers_rds.SmallestInstanceClass()
+	params, err := engine.ResolveEffectiveParameters(class, nil)
+	if err != nil {
+		t.Fatalf("ResolveEffectiveParameters(%s): %v", class, err)
+	}
+	body, err := renderParameters("mariadb", params)
+	if err != nil {
+		t.Fatalf("renderParameters: %v", err)
+	}
+	want := mariadbParametersHeader + body
+
+	path := filepath.FromSlash(mariadbDefaultParametersFixture)
+	if os.Getenv("RDS_UPDATE_FIXTURES") == "1" {
+		if err := os.WriteFile(path, []byte(want), 0o644); err != nil {
+			t.Fatalf("update %s: %v", path, err)
+		}
+		return
+	}
+
+	got, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read %s: %v", path, err)
+	}
+	if string(got) != want {
+		t.Errorf("%s is stale for class %s.\nRegenerate with RDS_UPDATE_FIXTURES=1 go test ./cmd/rds-agent/\n\ngot:\n%s\nwant:\n%s",
+			path, class, got, want)
+	}
+}
+
+// The one thing the fixture exists to prove cannot regress without the image
+// build catching it, asserted here too so a plain preflight names it.
+func TestMariaDBDefaultParametersFixture_CarriesNoSetOnlyName(t *testing.T) {
+	got, err := os.ReadFile(filepath.FromSlash(mariadbDefaultParametersFixture))
+	if err != nil {
+		t.Fatalf("read the fixture: %v", err)
+	}
+	for _, name := range []string{"\ntime_zone ", "\ntx_isolation ", "\ninnodb_buffer_pool_instances "} {
+		if strings.Contains(string(got), name) {
+			t.Errorf("the default include carries %q, which mariadbd refuses at startup", name)
+		}
+	}
+}

cmd/rds-agent/engine.go

@@ -3,7 +3,6 @@ package main
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"os"
 	"os/exec"
@@ -11,17 +10,15 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-	"sync"
-	"sync/atomic"
 	"syscall"
 	"time"
 
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
 
-// The in-guest half of the engine seam. Everything PostgreSQL-shaped that a
-// control-plane command needs lives behind this, so the command registry above
-// stays engine-agnostic.
+// The in-guest half of the engine seam. Everything engine-shaped that a
+// control-plane command needs lives behind this, so the command registry stays
+// engine-agnostic.
 type engineOps interface {
 	// Rotates the master role's password live. Never persisted anywhere in the
 	// guest: it exists only for the length of this call.
@@ -44,18 +41,127 @@ type engineOps interface {
 // a control-plane directive: nothing issues it, the agent runs it when the
 // engine fails to come up after a parameter change.
 type parameterRecovery interface {
-	// Reports whether the installed set differed and was replaced.
+	// Reports whether the installed set differed and was replaced. True with a
+	// non-nil error is a set already on disk whose re-derived TLS enforcement did
+	// not land: the caller still has to restart and report.
 	RestoreLastKnownGoodParameters(ctx context.Context) (bool, error)
 	Restart(ctx context.Context) error
 }
 
-// Records the parameter file only after the probe observes the postmaster
-// serving it. This is separate from apply, where static values are installed
-// but not adopted until a later restart.
+// Records the parameter file only after the probe observes the engine serving
+// it. This is separate from apply, where static values are installed but not
+// adopted until a later restart.
 type servingParameterRecorder interface {
 	RecordServingParameters(ctx context.Context) error
 }
 
+// Everything one engine implementation owns. The narrow interfaces above are
+// what the heartbeat, the commander and the rollback guard each hold; this is
+// only what the agent has to construct.
+type engine interface {
+	engineOps
+	parameterRecovery
+	servingParameterRecorder
+}
+
+// The guest layout an engine preset lays down, keyed by the engine the image
+// bakes. Every default that belongs to one engine lives here, so nothing
+// outside an implementation names its paths, service, port or probe.
+type engineLayout struct {
+	binDir    string
+	dataDir   string
+	socketDir string
+	osUser    string
+	service   string
+	dataMount string
+	port      int
+	// Where the engine records the pid the probe checks for liveness. Empty for
+	// an engine whose probe answers without one.
+	pidFile string
+	// Where the engine records why it would not start. Empty for an engine whose
+	// image does not direct one, which leaves the probe reporting its own reason
+	// alone rather than guessing at a path.
+	errorLog string
+	// The engine's own liveness signal, which is the only part of engineProbe
+	// that is not shared.
+	newProbe func(cfg config, run probeRunner) *engineProbe
+	// The implementation every control-plane directive is served by. It fails
+	// when the control plane this agent was built against does not offer the
+	// engine the image bakes, rather than serving one it has no definition for.
+	newEngine func(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error)
+}
+
+// The names the control plane knows these implementations by, which are also
+// what setup.sh stamps into each image.
+const (
+	enginePostgres = "postgres"
+	engineMariaDB  = "mariadb"
+)
+
+var engineLayouts = map[string]engineLayout{
+	enginePostgres: {
+		binDir:    "/usr/libexec/postgresql18",
+		dataDir:   "/var/lib/postgresql/18/data",
+		socketDir: "/run/postgresql",
+		osUser:    "postgres",
+		service:   "postgresql",
+		dataMount: "/var/lib/postgresql",
+		port:      5432,
+		newProbe:  newPostgresProbe,
+		newEngine: newPostgresEngineFromCatalog,
+	},
+	engineMariaDB: {
+		binDir:    "/usr/bin",
+		dataDir:   "/var/lib/mysql/data",
+		socketDir: "/run/mysqld",
+		osUser:    "mysql",
+		service:   "mariadb",
+		dataMount: "/var/lib/mysql",
+		port:      3306,
+		// Alpine's service passes --pid-file=/run/mysqld/$RC_SVCNAME.pid, and the
+		// command line beats the option file, so this is the path rather than
+		// anything the drop-in names. setup.sh asserts it at build time.
+		pidFile: "/run/mysqld/mariadb.pid",
+		// log_error in the platform drop-in rds-init writes. Without it
+		// mysqld_safe would send the server's errors to syslog, which nothing
+		// collects off a guest that has no SSH.
+		errorLog:  "/var/log/mysql/error.log",
+		newProbe:  newMariaDBProbe,
+		newEngine: newMariaDBEngineFromCatalog,
+	},
+}
+
+// Builds the health probe the image's engine supplies.
+func newProbe(cfg config, run probeRunner) (*engineProbe, error) {
+	layout, err := layoutFor(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return layout.newProbe(cfg, run), nil
+}
+
+// Builds the implementation the image bakes. Nothing here consults the control
+// plane for which engine to be: the agent runs the engine it is made of, and the
+// delivered engine is only ever checked against it.
+func newEngine(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+	layout, err := layoutFor(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return layout.newEngine(cfg, run, startSess, probe)
+}
+
+func layoutFor(cfg config) (engineLayout, error) {
+	if cfg.BakedEngine == "" {
+		return engineLayout{}, fmt.Errorf("this image carries no engine stamp at %s", cfg.EngineFile)
+	}
+	layout, ok := engineLayouts[cfg.BakedEngine]
+	if !ok {
+		return engineLayout{}, fmt.Errorf("this image bakes engine %q, which this agent does not implement", cfg.BakedEngine)
+	}
+	return layout, nil
+}
+
 // One child process. Env replaces the agent's own environment rather than
 // extending it, so a secret placed here reaches only the process that needs it.
 type command struct {
@@ -65,10 +171,14 @@ type command struct {
 	Stdin string
 	// The OS user to drop to before exec. Empty runs as the agent itself.
 	User string
+	// The statement that makes this client print the session sentinel. Read only
+	// when the command is started as a held session, whose reader has to know
+	// when the engine has finished each script it was fed.
+	SentinelStatement string
 }
 
-// Returns stdout. stderr is folded into the error, because psql reports the
-// actual SQL failure there while exiting with a bare status.
+// Returns stdout. stderr is folded into the error, because both clients report
+// the actual SQL failure there while exiting with a bare status.
 type commandRunner func(ctx context.Context, c command) (string, error)
 
 func execCommandRunner(ctx context.Context, c command) (string, error) {
@@ -113,417 +223,51 @@ func lookupCredential(username string) (*syscall.Credential, error) {
 	return &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)}, nil
 }
 
-// The engine is reached over its unix socket under peer authentication, so the
-// agent drops to the postgres OS user rather than holding a password of its own.
-type postgresEngine struct {
-	run       commandRunner
-	startSess sessionRunner
-	psql      string
-	rcService string
-	service   string
-	pgData    string
-	socketDir string
-	osUser    string
-	probe     *engineProbe
-	// Set from the bootstrap config, on a different goroutine than the commands
-	// that read it.
-	port atomic.Int64
-
-	// Serializes parameter installs, serving snapshots and rollback restores so
-	// none can copy or replace an intermediate configuration.
-	paramMu       sync.Mutex
-	repairTimeout time.Duration
-	repairPoll    time.Duration
-
-	// The backup mode currently held, if any. Guarded because the expiry timer
-	// releases it on its own goroutine.
-	mu   sync.Mutex
-	held *quiesceHold
-}
-
-var (
-	_ engineOps                = (*postgresEngine)(nil)
-	_ parameterRecovery        = (*postgresEngine)(nil)
-	_ servingParameterRecorder = (*postgresEngine)(nil)
-)
-
-const (
-	parameterRepairTimeout = 90 * time.Second
-	parameterRepairPoll    = time.Second
-)
-
-func newPostgresEngine(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) *postgresEngine {
-	e := &postgresEngine{
-		run:           run,
-		startSess:     startSess,
-		psql:          filepath.Join(cfg.PGBin, "psql"),
-		rcService:     cfg.RCService,
-		service:       cfg.EngineService,
-		pgData:        cfg.PGData,
-		socketDir:     cfg.SocketDir,
-		osUser:        cfg.PGUser,
-		probe:         probe,
-		repairTimeout: parameterRepairTimeout,
-		repairPoll:    parameterRepairPoll,
-	}
-	e.port.Store(int64(cfg.EnginePort))
-	return e
-}
-
-func (e *postgresEngine) setPort(port int) {
-	e.port.Store(int64(port))
-}
-
-// The role name and the password ride the environment and are re-quoted by
-// psql, so neither reaches a shell word or an argv another process can read.
-func (e *postgresEngine) SetPassword(ctx context.Context, username, password string) error {
-	if username == "" || password == "" {
-		return fmt.Errorf("set-password requires both %s and %s",
-			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
-	}
-	// The statement below runs as the cluster superuser over the socket under
-	// peer auth, so a reserved name in the command payload would hand the
-	// customer the bootstrap superuser rather than rotate their own role.
-	if err := handlers_rds.EnginePostgres().ValidateUsernameNotReserved(username); err != nil {
-		return fmt.Errorf("refusing to set the password of a role the engine reserves: %w", err)
-	}
-	// psql interpolates the password into the ALTER ROLE before the server sees
-	// it, and these three are what would write it to the log — the last on any
-	// failure at its own default. SUSET, so the parameter group cannot win.
-	const sql = `SET log_statement = 'none';
-SET log_min_duration_statement = -1;
-SET log_min_error_statement = 'panic';
-\getenv master RDS_MASTER_USERNAME
-\getenv password RDS_MASTER_PASSWORD
-ALTER ROLE :"master" WITH LOGIN PASSWORD :'password';
-`
-	_, err := e.psqlRun(ctx, sql,
-		"RDS_MASTER_USERNAME="+username,
-		"RDS_MASTER_PASSWORD="+password,
-	)
-	if err != nil {
-		// psql echoes the failing statement, which here would carry the new
-		// password back to the control plane and into the event ring.
-		return fmt.Errorf("apply the master password: %s", redact(err.Error(), password))
+// Through the service manager rather than the engine's own control tool, so the
+// supervisor records the state and does not restart the engine underneath a VM
+// that is going down.
+func serviceAction(ctx context.Context, run commandRunner, rcService, service, action string) error {
+	if _, err := run(ctx, command{
+		Name: rcService,
+		Args: []string{service, action},
+		Env:  []string{"PATH=" + defaultGuestPath},
+	}); err != nil {
+		return fmt.Errorf("%s the %s service: %w", action, service, err)
 	}
 	return nil
 }
 
-// The include the resolved parameter set is rendered to, and the copy of the
-// last one the engine accepted. Both live beside the data rather than in /etc,
-// matching rds-init: a class change boots a fresh root volume, which would
-// otherwise revert them when a fresh root volume is used.
-const (
-	parametersFileName = "10-rds-parameters.conf"
-	// Deliberately not a .conf name: include_dir globs *.conf, so the rollback
-	// copy must not be read as a second set of settings.
-	lastGoodFileName = "10-rds-parameters.last-good"
-)
-
-// Installs the resolved set, validates it with the engine's own config parser
-// before the engine ever adopts it, and reloads. A value the engine refuses is
-// rolled back here rather than left on the data volume, where it would survive
-// every VM replace and turn the next restart into a boot loop.
-func (e *postgresEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
-	e.paramMu.Lock()
-	defer e.paramMu.Unlock()
-
-	initialState, _ := e.probe.state(ctx)
-	if initialState == engineRecovering {
-		return nil, errors.New("apply parameters while the engine is still starting or recovering")
-	}
-	if initialState == engineServing {
-		// A command can arrive before the first heartbeat. Seed the configuration
-		// currently being served before replacing its file in that window.
-		if err := e.recordServingParametersLocked(ctx); err != nil {
-			return nil, fmt.Errorf("record the parameters serving before the apply: %w", err)
-		}
-	}
-
-	// The check has to run against the file in place, because the engine parses
-	// the datadir's own include_dir. The window that leaves is closed from both
-	// ends: the rollback below, and the last-known-good restore the agent runs at
-	// boot when the engine does not come up.
-	_, restore, err := e.installParameters(params)
-	if err != nil {
-		return nil, err
-	}
-	if err := e.checkConfig(ctx); err != nil {
-		return nil, errors.Join(fmt.Errorf("the engine rejected the parameter set: %w", err), restore())
-	}
-
-	state, _ := e.probe.state(ctx)
-	switch state {
-	case engineAbsent:
-		return e.restartOnRepairSetLocked(ctx)
-	case engineRecovering:
-		return nil, errors.Join(errors.New("the engine entered startup or recovery during the parameter apply"), restore())
-	}
-
-	if _, err := e.psqlRun(ctx, "SELECT pg_reload_conf();\n"); err != nil {
-		// The engine may have gone down after the first probe. In that case start
-		// it on the new, parser-checked repair set rather than restoring the set
-		// that already left it unable to serve.
-		if state, _ := e.probe.state(ctx); state == engineAbsent {
-			return e.restartOnRepairSetLocked(ctx)
-		}
-		return nil, errors.Join(fmt.Errorf("reload the engine configuration: %w", err), restore())
-	}
-	return e.pendingRestartParameters(ctx)
-}
-
-func (e *postgresEngine) restartOnRepairSetLocked(ctx context.Context) ([]string, error) {
-	repairCtx, cancel := context.WithTimeout(ctx, e.repairTimeout)
-	defer cancel()
-
-	if err := e.Restart(repairCtx); err != nil {
-		return nil, fmt.Errorf("start the engine on the repaired parameter set: %w", err)
-	}
-
-	timer := time.NewTimer(0)
-	defer timer.Stop()
-	lastMessage := "engine did not respond"
-	for {
-		select {
-		case <-repairCtx.Done():
-			if ctx.Err() != nil {
-				return nil, fmt.Errorf("wait for the engine on the repaired parameter set: %w", ctx.Err())
-			}
-			return nil, fmt.Errorf("wait for the engine on the repaired parameter set: %s: %w", lastMessage, repairCtx.Err())
-		case <-timer.C:
-		}
-
-		state, message := e.probe.state(repairCtx)
-		if state == engineServing {
-			pending, err := e.pendingRestartParameters(repairCtx)
-			if err == nil {
-				return pending, nil
-			}
-			lastMessage = err.Error()
-		} else if message != "" {
-			lastMessage = message
-		}
-		timer.Reset(e.repairPoll)
-	}
-}
-
-func (e *postgresEngine) pendingRestartParameters(ctx context.Context) ([]string, error) {
-	out, err := e.psqlRun(ctx, "SELECT name FROM pg_settings WHERE pending_restart ORDER BY name;\n")
-	if err != nil {
-		return nil, fmt.Errorf("read the settings pending a restart: %w", err)
-	}
-	var pending []string
-	for line := range strings.SplitSeq(out, "\n") {
-		if name := strings.TrimSpace(line); name != "" {
-			pending = append(pending, name)
-		}
-	}
-	return pending, nil
-}
-
-func (e *postgresEngine) parametersPath() string {
-	return filepath.Join(e.pgData, "conf.d", parametersFileName)
-}
-
-func (e *postgresEngine) lastGoodPath() string {
-	return filepath.Join(e.pgData, "conf.d", lastGoodFileName)
-}
-
-// Written as root and handed to the engine's user, at the same path and mode
-// rds-init installs it at, so a later boot overwrites rather than shadows it.
-// Returns the content that was there and a func putting it back, so the caller
-// can undo the install without re-rendering anything.
-func (e *postgresEngine) installParameters(params []handlers_rds.Parameter) (previous []byte, restore func() error, err error) {
-	path := e.parametersPath()
-	// A missing file is the first-ever apply, and its rollback is a removal.
-	previous, readErr := os.ReadFile(path)
-	if readErr != nil && !os.IsNotExist(readErr) {
-		return nil, nil, fmt.Errorf("read the installed parameters at %s: %w", path, readErr)
-	}
-	existed := readErr == nil
-
-	if err := e.writeEngineFile(path, []byte(renderParameters(params))); err != nil {
-		return nil, nil, err
-	}
-	return previous, func() error {
-		if !existed {
-			if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
-				return fmt.Errorf("withdraw the rejected parameters at %s: %w", path, err)
-			}
-			return nil
-		}
-		return e.writeEngineFile(path, previous)
-	}, nil
-}
-
-// Snapshots the include only when the running postmaster has no settings still
-// pending a restart. The parameter mutex keeps an apply from replacing the file
-// between that check and the copy.
-func (e *postgresEngine) RecordServingParameters(ctx context.Context) error {
-	e.paramMu.Lock()
-	defer e.paramMu.Unlock()
-	return e.recordServingParametersLocked(ctx)
-}
-
-func (e *postgresEngine) recordServingParametersLocked(ctx context.Context) error {
-	if _, err := os.Stat(e.parametersPath()); err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return fmt.Errorf("inspect the serving parameters: %w", err)
-	}
-	pending, err := e.pendingRestartParameters(ctx)
-	if err != nil {
-		return err
-	}
-	if len(pending) > 0 {
-		return nil
-	}
-
-	content, err := os.ReadFile(e.parametersPath())
-	if err != nil {
-		return fmt.Errorf("read the serving parameters: %w", err)
-	}
-	lastGood, err := os.ReadFile(e.lastGoodPath())
-	if err == nil && bytes.Equal(content, lastGood) {
-		return nil
-	}
-	if err != nil && !os.IsNotExist(err) {
-		return fmt.Errorf("read the last known good parameters: %w", err)
-	}
-	return e.writeEngineFile(e.lastGoodPath(), content)
-}
-
-// Puts the last set the engine accepted back in place, for a restart that failed
-// after a parameter change. The probe is checked again under the parameter lock
-// so a repair that just brought the engine back cannot be reversed.
-func (e *postgresEngine) RestoreLastKnownGoodParameters(ctx context.Context) (bool, error) {
-	e.paramMu.Lock()
-	defer e.paramMu.Unlock()
-
-	if state, _ := e.probe.state(ctx); state != engineAbsent {
-		return false, nil
-	}
-	lastGood, err := os.ReadFile(e.lastGoodPath())
-	if err != nil {
-		if os.IsNotExist(err) {
-			return false, nil
-		}
-		return false, fmt.Errorf("read the last known good parameters: %w", err)
-	}
-	current, err := os.ReadFile(e.parametersPath())
+// Whether the installed set requires TLS of client connections. Read back from
+// the file rather than taken from the set being applied, because the restore and
+// repair paths put a file in place without holding one. Both engines' generated
+// parameter files are written in the same syntax, so one reader serves them.
+func installedTLSEnforcement(name, installedPath string) (bool, error) {
+	values, err := readOptionFile(installedPath)
 	if err != nil && !os.IsNotExist(err) {
 		return false, fmt.Errorf("read the installed parameters: %w", err)
 	}
-	if bytes.Equal(current, lastGood) {
+	value, ok := values[name]
+	if !ok {
+		// The ordinary state of an instance whose resolved set predates the
+		// parameter, and the whole of the migration story: it begins enforcing at
+		// its next boot with no control-plane work at all.
+		return true, nil
+	}
+	switch value {
+	case "1":
+		return true, nil
+	case "0":
 		return false, nil
 	}
-	if err := e.writeEngineFile(e.parametersPath(), lastGood); err != nil {
-		return false, err
-	}
-	return true, nil
-}
-
-// Atomic: a temp file in the same directory, renamed over the target. The temp
-// name deliberately does not end in .conf, so a crash between write and rename
-// cannot leave the engine reading a half-written include.
-func (e *postgresEngine) writeEngineFile(path string, content []byte) error {
-	tmp := path + ".new"
-	if err := os.WriteFile(tmp, content, 0o600); err != nil {
-		return fmt.Errorf("write %s: %w", tmp, err)
-	}
-	if err := e.chownToEngine(tmp); err != nil {
-		os.Remove(tmp)
-		return err
-	}
-	if err := os.Rename(tmp, path); err != nil {
-		os.Remove(tmp)
-		return fmt.Errorf("install %s: %w", path, err)
-	}
-	return nil
-}
-
-// The engine's own parser, run offline against the datadir. Reading one setting
-// back is enough: postgres parses postgresql.conf and every include first, and
-// exits non-zero naming the file and line of an unknown parameter or a value
-// outside its range.
-func (e *postgresEngine) checkConfig(ctx context.Context) error {
-	if _, err := e.run(ctx, command{
-		Name: filepath.Join(filepath.Dir(e.psql), "postgres"),
-		Args: []string{"-D", e.pgData, "-C", "shared_buffers"},
-		Env:  []string{"PATH=" + defaultGuestPath},
-		User: e.osUser,
-	}); err != nil {
-		return err
-	}
-	return nil
-}
-
-// The engine reads its own config, so a root-owned file would be unreadable to
-// it. A guest with no such user is a broken image, not something to work around.
-func (e *postgresEngine) chownToEngine(path string) error {
-	credential, err := lookupCredential(e.osUser)
-	if err != nil {
-		return err
-	}
-	if err := os.Chown(path, int(credential.Uid), int(credential.Gid)); err != nil {
-		return fmt.Errorf("hand %s to %s: %w", path, e.osUser, err)
-	}
-	return nil
-}
-
-// Through the service manager rather than pg_ctl, so the supervisor records the
-// engine as stopped and does not restart it underneath a VM that is going down.
-func (e *postgresEngine) Stop(ctx context.Context) error {
-	return e.serviceAction(ctx, "stop")
-}
-
-// Only the parameter rollback calls this: a restart the control plane wants goes
-// through RebootDBInstance, which cycles the VM.
-func (e *postgresEngine) Restart(ctx context.Context) error {
-	return e.serviceAction(ctx, "restart")
-}
-
-func (e *postgresEngine) serviceAction(ctx context.Context, action string) error {
-	if _, err := e.run(ctx, command{
-		Name: e.rcService,
-		Args: []string{e.service, action},
-		Env:  []string{"PATH=" + defaultGuestPath},
-	}); err != nil {
-		return fmt.Errorf("%s the %s service: %w", action, e.service, err)
-	}
-	return nil
+	// The resolver canonicalises every boolean, so anything else means the file
+	// was written by something other than the platform. The permissive reading is
+	// the one that must not be chosen for a setting that turns TLS off.
+	return false, fmt.Errorf("the installed parameters set %s to %q, which is neither 1 nor 0", name, value)
 }
 
 // A minimal environment for a child that is not carrying secrets.
 const defaultGuestPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 
-// Feeds sql on stdin rather than as an argument, so a statement is never
-// visible in the process table.
-func (e *postgresEngine) psqlRun(ctx context.Context, sql string, env ...string) (string, error) {
-	return e.run(ctx, command{
-		Name:  e.psql,
-		Args:  e.psqlArgs(),
-		Env:   append([]string{"PATH=" + defaultGuestPath}, env...),
-		Stdin: sql,
-		User:  e.osUser,
-	})
-}
-
-// Reading the script from stdin is what lets one invocation serve both a
-// one-shot run and a session held open across several statements.
-func (e *postgresEngine) psqlArgs() []string {
-	return []string{
-		"--no-psqlrc", "--quiet", "--no-align", "--tuples-only",
-		"-v", "ON_ERROR_STOP=1",
-		"-h", e.socketDir,
-		"-p", strconv.FormatInt(e.port.Load(), 10),
-		"-U", e.osUser,
-		"-d", "postgres",
-		"-f", "-",
-	}
-}
-
 func redact(s, secret string) string {
 	if secret == "" {
 		return s

cmd/rds-agent/engine_contract_test.go

@@ -0,0 +1,333 @@
+package main
+
+//test:in-package — the agent is a main package, which has no external test
+// package to import it from, and these cases are built on the unexported
+// engine interface and each implementation's own constructor.
+
+import (
+	"context"
+	"errors"
+	"os"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// A probe the test drives directly, in place of whichever binaries the engine's
+// own state function would have run. Each implementation's state function is
+// covered on its own; what these cases exercise is what the engine does with the
+// answer.
+func drivenProbe(state *engineState) *engineProbe {
+	return newEngineProbe(0, func(context.Context, int64) (engineState, string) {
+		return *state, "driven by the test"
+	})
+}
+
+// One engine behind the interfaces the heartbeat, the commander and the rollback
+// guard hold, plus the few facts a shared case needs to say what it means.
+type engineHarness struct {
+	engine  engine
+	runner  *recordingRunner
+	params  parameterStore
+	service string
+	// The state the probe reports, moved by the test rather than by the engine.
+	state *engineState
+	// Points the next quiesce at a session the test can inspect, and returns the
+	// commands those sessions were opened with.
+	useSession func(*fakeSession) *[]command
+	// A setting this engine's catalog really carries, so a shared case installs
+	// something the implementation would genuinely be handed.
+	parameter handlers_rds.Parameter
+	// What the engine's own include directive globs. Neither the rollback copy
+	// nor the serving copy may end in it.
+	includeSuffix string
+}
+
+func engineHarnesses() []struct {
+	name  string
+	build func(*testing.T) *engineHarness
+} {
+	return []struct {
+		name  string
+		build func(*testing.T) *engineHarness
+	}{
+		{name: enginePostgres, build: buildPostgresHarness},
+		{name: engineMariaDB, build: buildMariaDBHarness},
+	}
+}
+
+func forEachEngine(t *testing.T, run func(*testing.T, *engineHarness)) {
+	t.Helper()
+	for _, fixture := range engineHarnesses() {
+		t.Run(fixture.name, func(t *testing.T) { run(t, fixture.build(t)) })
+	}
+}
+
+func buildPostgresHarness(t *testing.T) *engineHarness {
+	t.Helper()
+	runner := &recordingRunner{}
+	e := newTestEngine(t, runner.run)
+	state := engineServing
+	e.probe = drivenProbe(&state)
+
+	return &engineHarness{
+		engine: e, runner: runner, params: e.params, service: e.service, state: &state,
+		useSession: func(session *fakeSession) *[]command {
+			started := &[]command{}
+			e.startSess = func(_ context.Context, c command) (engineSession, error) {
+				*started = append(*started, c)
+				return session, nil
+			}
+			return started
+		},
+		parameter:     handlers_rds.Parameter{Name: "work_mem", Value: "4096"},
+		includeSuffix: ".conf",
+	}
+}
+
+func buildMariaDBHarness(t *testing.T) *engineHarness {
+	t.Helper()
+	runner := &recordingRunner{}
+	e := newTestMariaDBEngine(t, runner.run)
+	state := engineServing
+	e.probe = drivenProbe(&state)
+
+	return &engineHarness{
+		engine: e, runner: runner, params: e.params, service: e.service, state: &state,
+		useSession: func(session *fakeSession) *[]command {
+			started := &[]command{}
+			e.startSess = func(_ context.Context, c command) (engineSession, error) {
+				*started = append(*started, c)
+				return session, nil
+			}
+			return started
+		},
+		parameter:     handlers_rds.Parameter{Name: "max_connections", Value: "200"},
+		includeSuffix: ".cnf",
+	}
+}
+
+// The parameters land where rds-init installs them, so the next boot overwrites
+// the file rather than shadowing it with a second copy.
+func TestEngineContract_ApplyParametersInstallsWhereRdsInitDoes(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		if _, err := h.engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{h.parameter}); err != nil {
+			t.Fatalf("ApplyParameters: %v", err)
+		}
+
+		installed, err := os.ReadFile(h.params.installedPath())
+		if err != nil {
+			t.Fatalf("read the installed parameters: %v", err)
+		}
+		if !strings.Contains(string(installed), h.parameter.Name+" = '"+h.parameter.Value+"'") {
+			t.Errorf("installed parameters = %q, want the resolved value", installed)
+		}
+		if !strings.HasSuffix(h.params.installedPath(), h.includeSuffix) {
+			t.Errorf("the installed set at %s is not a file the engine includes", h.params.installedPath())
+		}
+		// Comparison material, not a second set of settings: an include glob that
+		// read either of these would apply a withdrawn configuration.
+		copies := []string{h.params.lastGoodPath()}
+		if h.params.serving != "" {
+			copies = append(copies, h.params.servingPath())
+		}
+		for _, path := range copies {
+			if strings.HasSuffix(path, h.includeSuffix) {
+				t.Errorf("the copy at %s would be included by the engine", path)
+			}
+		}
+	})
+}
+
+// An engine still starting or replaying has not finished adopting the set it is
+// on, so replacing that set underneath it would be installing over a moving
+// target.
+func TestEngineContract_ApplyParametersRefusesWhileTheEngineIsRecovering(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		*h.state = engineRecovering
+
+		if _, err := h.engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{h.parameter}); err == nil {
+			t.Fatal("ApplyParameters succeeded against an engine that was still coming up")
+		}
+		if _, err := os.Stat(h.params.installedPath()); !os.IsNotExist(err) {
+			t.Errorf("the refused set was installed anyway (stat err = %v)", err)
+		}
+	})
+}
+
+// The boot-time half of the parameter safety net: the last set the engine
+// accepted goes back only when the engine is genuinely down, and only once.
+func TestEngineContract_RestoreLastKnownGoodOnlyRollsBackADownEngine(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		accepted := h.parameter
+		if _, err := h.engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{accepted}); err != nil {
+			t.Fatalf("ApplyParameters: %v", err)
+		}
+		if err := h.engine.RecordServingParameters(t.Context()); err != nil {
+			t.Fatalf("RecordServingParameters: %v", err)
+		}
+		if _, err := h.engine.ApplyParameters(t.Context(),
+			[]handlers_rds.Parameter{{Name: accepted.Name, Value: accepted.Value + "0"}}); err != nil {
+			t.Fatalf("second ApplyParameters: %v", err)
+		}
+
+		// A guard whose deadline expired while an apply brought the engine back must
+		// not reverse that repair.
+		restored, err := h.engine.RestoreLastKnownGoodParameters(t.Context())
+		if err != nil {
+			t.Fatalf("RestoreLastKnownGoodParameters against a serving engine: %v", err)
+		}
+		if restored {
+			t.Fatal("rolled the parameters back while the engine was serving")
+		}
+
+		*h.state = engineAbsent
+		restored, err = h.engine.RestoreLastKnownGoodParameters(t.Context())
+		if err != nil {
+			t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
+		}
+		if !restored {
+			t.Fatal("did not roll back to a set that differs from the last accepted one")
+		}
+		installed, err := os.ReadFile(h.params.installedPath())
+		if err != nil {
+			t.Fatalf("read the installed parameters: %v", err)
+		}
+		if !strings.Contains(string(installed), accepted.Name+" = '"+accepted.Value+"'") {
+			t.Errorf("installed = %q, want the last accepted set", installed)
+		}
+
+		// Repeating a rollback would only churn a cluster that is failing for
+		// another reason.
+		restored, err = h.engine.RestoreLastKnownGoodParameters(t.Context())
+		if err != nil {
+			t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
+		}
+		if restored {
+			t.Error("rolled back again when the installed set already matched")
+		}
+	})
+}
+
+// Through the service manager rather than the engine's own control tool, so the
+// supervisor records the engine as stopped and does not restart it underneath a
+// VM that is going down.
+func TestEngineContract_StopGoesThroughTheServiceManager(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		if err := h.engine.Stop(t.Context()); err != nil {
+			t.Fatalf("Stop: %v", err)
+		}
+		if len(h.runner.calls) != 1 {
+			t.Fatalf("ran %d commands, want the service stop", len(h.runner.calls))
+		}
+		call := h.runner.calls[0]
+		if !slices.Equal(call.Args, []string{h.service, "stop"}) {
+			t.Errorf("stop ran %s %v, want the service manager", call.Name, call.Args)
+		}
+	})
+}
+
+// One backup at a time, released by Unquiesce or by its own deadline, whichever
+// comes first. The deadline is what keeps a control plane that dies mid-snapshot
+// from leaving the engine held.
+func TestEngineContract_QuiesceHoldsUntilReleased(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		session := &fakeSession{}
+		started := h.useSession(session)
+
+		if err := h.engine.Quiesce(t.Context(), "orders-db-pre-upgrade", time.Minute); err != nil {
+			t.Fatalf("Quiesce: %v", err)
+		}
+		if len(*started) != 1 {
+			t.Fatalf("started %d sessions, want 1", len(*started))
+		}
+		if session.closes() != 0 {
+			t.Fatal("the session was closed while the backup was meant to be held")
+		}
+
+		err := h.engine.Quiesce(t.Context(), "second", time.Minute)
+		if err == nil {
+			t.Fatal("a second quiesce was accepted while the first was held")
+		}
+		if !strings.Contains(err.Error(), "orders-db-pre-upgrade") {
+			t.Errorf("error %q does not name the backup already holding the engine", err)
+		}
+		if len(*started) != 1 {
+			t.Errorf("started %d sessions, want the second refused before opening one", len(*started))
+		}
+
+		if err := h.engine.Unquiesce(t.Context()); err != nil {
+			t.Fatalf("Unquiesce: %v", err)
+		}
+		if session.closes() != 1 {
+			t.Errorf("closed %d times, want the session ended exactly once", session.closes())
+		}
+	})
+}
+
+// A hold that expired mid-snapshot means the snapshot was not taken against a
+// held checkpoint, so the release reports it rather than succeeding silently.
+func TestEngineContract_QuiesceReleasesItselfOnItsDeadline(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		session := &fakeSession{}
+		h.useSession(session)
+
+		if err := h.engine.Quiesce(t.Context(), "orders-db-pre-upgrade", 10*time.Millisecond); err != nil {
+			t.Fatalf("Quiesce: %v", err)
+		}
+		deadline := time.Now().Add(2 * time.Second)
+		for session.closes() == 0 && time.Now().Before(deadline) {
+			time.Sleep(time.Millisecond)
+		}
+		if session.closes() != 1 {
+			t.Fatal("the backup session outlived its deadline")
+		}
+
+		err := h.engine.Unquiesce(t.Context())
+		if err == nil {
+			t.Fatal("Unquiesce succeeded after the hold expired")
+		}
+		if !strings.Contains(err.Error(), "expired") {
+			t.Errorf("error %q does not say the hold had expired", err)
+		}
+	})
+}
+
+// A failed start leaves nothing held, so a retry is not refused by a hold that
+// never took effect.
+func TestEngineContract_QuiesceEndsTheSessionWhenTheBackupWillNotStart(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		session := &fakeSession{execErr: errors.New("the engine refused the backup")}
+		h.useSession(session)
+
+		if err := h.engine.Quiesce(t.Context(), "orders-db-pre-upgrade", time.Minute); err == nil {
+			t.Fatal("Quiesce succeeded against an engine that refused the backup")
+		}
+		if session.closes() != 1 {
+			t.Errorf("closed %d times, want the failed session ended", session.closes())
+		}
+		if err := h.engine.Unquiesce(t.Context()); err == nil {
+			t.Error("a failed quiesce left a hold behind")
+		}
+	})
+}
+
+// Every command the control plane can issue is served by both implementations,
+// so a directive cannot be answered as unsupported on one engine and honoured on
+// the other.
+func TestEngineContract_ServesEveryRegisteredCommand(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		registry := newCommandRegistry(h.engine, &fakeStorage{})
+		for _, name := range []string{
+			handlers_rds.CommandSetPassword, handlers_rds.CommandApplyParams, handlers_rds.CommandStopEngine,
+			handlers_rds.CommandGrowFilesystem, handlers_rds.CommandQuiesce, handlers_rds.CommandUnquiesce,
+		} {
+			if _, ok := registry[name]; !ok {
+				t.Errorf("command %q has no handler", name)
+			}
+		}
+	})
+}

cmd/rds-agent/engine_mariadb.go

@@ -0,0 +1,814 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The control plane's own statements about this engine, resolved once at
+// startup. Held as functions rather than as the Engine value so nothing here
+// re-derives a rule the API already owns.
+type controlPlaneRules struct {
+	// The whole master-username rule rather than only the reserved set. MariaDB's
+	// client has no identifier quoting, so the name is interpolated into SQL by
+	// this process and every part of the rule is load-bearing here.
+	validateUsername func(username string) error
+	// Whether a setting takes effect only at a restart. Already customer-facing
+	// and authoritative, since the API refuses ApplyMethod=immediate on one.
+	isStatic func(name string) bool
+	// The catalog name behind a spelling read back out of an option file. Only a
+	// setting whose startup spelling differs from the customer's moves.
+	catalogName func(optionFileName string) string
+	// The catalog's data type for a setting, which decides how its value is
+	// rendered into SQL. Empty for a name the catalog does not carry.
+	dataType func(name string) string
+	// The engine's own name for the setting that requires TLS of a client
+	// connection. Held rather than spelled out here, so the guest derives
+	// enforcement from the same key the control plane resolves it under.
+	tlsEnforcementParameter string
+}
+
+func controlPlaneRulesFrom(meta handlers_rds.Engine) controlPlaneRules {
+	// Built once per engine: everything read back out of a generated file has to
+	// return to the catalog's namespace before it is classified or reported, or a
+	// startup spelling would read as an unknown name.
+	catalogNames := map[string]string{}
+	for _, name := range meta.CatalogParameterNames() {
+		if optionFileName := meta.OptionFileName(name); optionFileName != name {
+			catalogNames[optionFileName] = name
+		}
+	}
+
+	return controlPlaneRules{
+		validateUsername: meta.ValidateMasterUsername,
+		isStatic: func(name string) bool {
+			spec, ok := meta.LookupParameter(name)
+			// A name the catalog does not carry cannot be shown to have been adopted
+			// without a restart, and is never issued as a live SET GLOBAL either.
+			return !ok || spec.ApplyType == handlers_rds.ApplyTypeStatic
+		},
+		catalogName: func(optionFileName string) string {
+			if name, ok := catalogNames[optionFileName]; ok {
+				return name
+			}
+			return optionFileName
+		},
+		dataType: func(name string) string {
+			spec, ok := meta.LookupParameter(name)
+			if !ok {
+				return ""
+			}
+			return spec.DataType
+		},
+		tlsEnforcementParameter: meta.TLSEnforcementParameter(),
+	}
+}
+
+// Reached over its unix socket as root, which the datadir's unix_socket plugin
+// authenticates from the connecting process' own uid. That is why the agent
+// holds no password of its own.
+type mariadbEngine struct {
+	quiesceState
+	parameterManager
+
+	rules     controlPlaneRules
+	run       commandRunner
+	startSess sessionRunner
+	client    string
+	socket    string
+	rcService string
+	service   string
+}
+
+var _ engine = (*mariadbEngine)(nil)
+
+const (
+	mariadbClientBinary = "mariadb"
+	mariadbAdminBinary  = "mariadb-admin"
+	// The account mariadb-install-db creates for the unix_socket plugin, which
+	// rds-init keeps for the platform and never hands to the customer.
+	mariadbSuperuser                  = "root"
+	mariadbProbeConnectTimeoutSeconds = 3
+	// How much of the engine's error log a probe reason carries. The server names
+	// what it refused on in its last few lines, and the reason reaches a customer
+	// through StatusInfos, so this quotes the tail rather than the whole file.
+	mariadbErrorLogTailBytes = 4096
+	mariadbErrorLogTailLines = 4
+	// How much of the probe client's stderr a reason carries, on the same grounds
+	// as the log tail above.
+	mariadbProbeStderrMaxBytes = 512
+	// Named by the platform drop-in rds-init writes, so both halves reach the
+	// same socket without either asserting it to the other.
+	mariadbSocketFile = "mysqld.sock"
+	// The customer reaches the instance over its customer ENI, so the master is
+	// created and rotated on the wildcard host rather than on localhost.
+	mariadbMasterHost = "%"
+)
+
+// The generated drop-ins on the data volume. The rollback and serving copies
+// deliberately do not end in .cnf: !includedir reads *.cnf, and neither is a
+// second set of settings.
+const (
+	mariadbParametersFile = "10-rds-parameters.cnf"
+	mariadbLastGoodFile   = "10-rds-parameters.last-good"
+	mariadbServingFile    = "10-rds-parameters.serving"
+	// MariaDB treats a setting before any group as a fatal parsing error, so
+	// every generated file carries the header rds-init writes.
+	mariadbParametersHeader = "[mysqld]\n"
+)
+
+// The layout's factory. The rules come from the control plane's own definition
+// of this engine, so a build whose control plane does not offer MariaDB refuses
+// to run this implementation rather than inventing a definition for it.
+func newMariaDBEngineFromCatalog(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+	meta, err := handlers_rds.LookupEngine(engineMariaDB)
+	if err != nil {
+		return nil, fmt.Errorf("this image bakes %s, which this build's control plane does not offer: %w", engineMariaDB, err)
+	}
+	return newMariaDBEngine(cfg, controlPlaneRulesFrom(meta), run, startSess, probe), nil
+}
+
+func newMariaDBEngine(cfg config, rules controlPlaneRules, run commandRunner, startSess sessionRunner, probe *engineProbe) *mariadbEngine {
+	return &mariadbEngine{
+		rules:     rules,
+		run:       run,
+		startSess: startSess,
+		client:    filepath.Join(cfg.EngineBinDir, mariadbClientBinary),
+		socket:    mariadbSocketPath(cfg),
+		rcService: cfg.RCService,
+		service:   cfg.EngineService,
+		parameterManager: parameterManager{
+			probe: probe,
+			params: parameterStore{
+				// The mount point rather than the datadir one level inside it: the
+				// include directory has to outlive the sweep a failed bootstrap runs.
+				dir:       filepath.Join(cfg.DataMount, "conf.d"),
+				installed: mariadbParametersFile,
+				lastGood:  mariadbLastGoodFile,
+				serving:   mariadbServingFile,
+				header:    mariadbParametersHeader,
+				osUser:    cfg.EngineUser,
+				engine:    engineMariaDB,
+			},
+			repairTimeout: parameterRepairTimeout,
+			repairPoll:    parameterRepairPoll,
+		},
+	}
+}
+
+func mariadbSocketPath(cfg config) string {
+	return filepath.Join(cfg.SocketDir, mariadbSocketFile)
+}
+
+// Every local connection, from the probe's ping to a held quiesce session.
+// --no-defaults so no option file can move it; --skip-ssl because the serving
+// cert names the endpoint rather than this socket, so a verifying client fails.
+func mariadbSocketConnectArgs(socket string) []string {
+	return []string{
+		"--no-defaults", "--protocol=socket", "--socket=" + socket,
+		"--user=" + mariadbSuperuser, "--skip-ssl",
+	}
+}
+
+// Reports whether a pid is a live process. EPERM is a process this agent may not
+// signal rather than one that is gone, which matters because a stale pidfile
+// read as "alive" is far less dangerous than a live engine read as "absent".
+type processLivenessFn func(pid int) bool
+
+func processAlive(pid int) bool {
+	if pid <= 0 {
+		return false
+	}
+	err := syscall.Kill(pid, 0)
+	return err == nil || errors.Is(err, syscall.EPERM)
+}
+
+func newMariaDBProbe(cfg config, run probeRunner) *engineProbe {
+	return newEngineProbe(cfg.EnginePort, mariadbProbeState(cfg, run, processAlive))
+}
+
+// Three stages: during InnoDB crash recovery mariadbd opens neither socket nor
+// port, so a ping cannot tell a recovering engine from an absent one — and
+// reading recovery as absent has the rollback guard restart a server mid-replay.
+func mariadbProbeState(cfg config, run probeRunner, alive processLivenessFn) probeStateFn {
+	pidFile, socket := cfg.EnginePidFile, mariadbSocketPath(cfg)
+	admin, client := filepath.Join(cfg.EngineBinDir, mariadbAdminBinary), filepath.Join(cfg.EngineBinDir, mariadbClientBinary)
+	// The probe alone bounds the connect: it must come back and report, where a
+	// statement the agent issues is allowed to take as long as the engine needs.
+	connect := append(mariadbSocketConnectArgs(socket),
+		fmt.Sprintf("--connect-timeout=%d", mariadbProbeConnectTimeoutSeconds))
+	probeTimeout := cfg.EngineProbeTimeout
+	if probeTimeout <= 0 {
+		probeTimeout = defaultEngineProbeTimeout
+	}
+	runBounded := func(ctx context.Context, name string, args ...string) (int, string, error) {
+		probeCtx, cancel := context.WithTimeout(ctx, probeTimeout)
+		defer cancel()
+		return run(probeCtx, name, args...)
+	}
+
+	return func(ctx context.Context, _ int64) (engineState, string) {
+		pid, err := readPidFile(pidFile)
+		switch {
+		case os.IsNotExist(err):
+			return engineAbsent, fmt.Sprintf("the engine has written no pidfile at %s", pidFile)
+		case err != nil:
+			return engineAbsent, fmt.Sprintf("the engine pidfile %s could not be read: %v", pidFile, err)
+		case !alive(pid):
+			return engineAbsent, fmt.Sprintf("the engine pidfile %s names pid %d, which is not running", pidFile, pid)
+		}
+
+		// The process is up from here on, so every remaining failure is an engine
+		// not serving yet rather than one that is gone. Reporting absent against a
+		// live server would have the rollback guard restart one making progress.
+		switch code, stderr, err := runBounded(ctx, admin, append(slices.Clone(connect), "ping")...); {
+		case err != nil:
+			return engineRecovering, fmt.Sprintf("engine probe could not run: %v", err)
+		case code != 0:
+			// A server that is genuinely recovering and one that refused the client
+			// are indistinguishable from the socket. The server states its side in
+			// its own log, and the client states its side on stderr.
+			return engineRecovering, withProbeStderr(withEngineLogTail(
+				"engine is not answering on its socket yet (startup or crash recovery)",
+				cfg.EngineErrorLog), stderr)
+		}
+
+		// ping answers even on ER_ACCESS_DENIED, so on its own it certifies a server
+		// that may be unable to execute anything. The statement rides argv rather
+		// than stdin: it carries nothing secret and a probe reads no result back.
+		query := append(slices.Clone(connect), "--batch", "--skip-column-names", "--execute=SELECT 1")
+		switch code, stderr, err := runBounded(ctx, client, query...); {
+		case err != nil:
+			return engineRecovering, fmt.Sprintf("engine probe could not run: %v", err)
+		case code != 0:
+			return engineRecovering, withProbeStderr(
+				"engine answered its socket but would not execute a statement", stderr)
+		}
+		return engineServing, ""
+	}
+}
+
+// Appends what the probe client said on stderr. A client that fails during the
+// handshake is refused by the server without the server ever logging why, so
+// this is the only account of that half of the exchange.
+func withProbeStderr(reason, stderr string) string {
+	stderr = strings.TrimSpace(stderr)
+	if stderr == "" {
+		return reason
+	}
+	return reason + "; the probe client reported: " + collapseProbeStderr(stderr)
+}
+
+// One line, bounded: the reason reaches a customer through StatusInfos, and a
+// client that repeats itself per attempt would otherwise crowd out the rest.
+func collapseProbeStderr(stderr string) string {
+	line := strings.Join(strings.Fields(strings.ReplaceAll(stderr, "\n", " ")), " ")
+	if len(line) > mariadbProbeStderrMaxBytes {
+		return line[:mariadbProbeStderrMaxBytes] + "..."
+	}
+	return line
+}
+
+// Appends the last few lines of the engine's error log to reason. An unset path,
+// an unreadable file or an empty one leaves reason alone: the probe's own
+// statement is still true, and a guessed cause would be worse than none.
+func withEngineLogTail(reason, path string) string {
+	tail := engineLogTail(path)
+	if tail == "" {
+		return reason
+	}
+	return reason + "; the engine's log ends: " + tail
+}
+
+func engineLogTail(path string) string {
+	if path == "" {
+		return ""
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return ""
+	}
+	defer func() { _ = f.Close() }()
+
+	info, err := f.Stat()
+	if err != nil {
+		return ""
+	}
+	offset := max(info.Size()-mariadbErrorLogTailBytes, 0)
+	buf := make([]byte, info.Size()-offset)
+	if _, err := f.ReadAt(buf, offset); err != nil {
+		return ""
+	}
+
+	lines := strings.Split(string(buf), "\n")
+	// A read that started mid-file almost certainly started mid-line, and half a
+	// line is more misleading than one fewer.
+	if offset > 0 && len(lines) > 0 {
+		lines = lines[1:]
+	}
+	kept := make([]string, 0, mariadbErrorLogTailLines)
+	for i := len(lines) - 1; i >= 0 && len(kept) < mariadbErrorLogTailLines; i-- {
+		if line := strings.TrimSpace(lines[i]); line != "" {
+			kept = append(kept, line)
+		}
+	}
+	slices.Reverse(kept)
+	return strings.Join(kept, " | ")
+}
+
+func readPidFile(path string) (int, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return 0, err
+	}
+	pid, err := strconv.Atoi(strings.TrimSpace(string(raw)))
+	if err != nil {
+		return 0, fmt.Errorf("%s does not hold a pid: %w", path, err)
+	}
+	return pid, nil
+}
+
+func (e *mariadbEngine) Stop(ctx context.Context) error {
+	return serviceAction(ctx, e.run, e.rcService, e.service, "stop")
+}
+
+// Only the parameter rollback and a failed apply call this: a restart the
+// control plane wants goes through RebootDBInstance, which cycles the VM. The
+// serving copy is refreshed first, since the server starts on what is installed.
+func (e *mariadbEngine) Restart(ctx context.Context) error {
+	if err := e.recordServingCopy(); err != nil {
+		return err
+	}
+	return serviceAction(ctx, e.run, e.rcService, e.service, "restart")
+}
+
+// The installed set copied to the name the pending-restart comparison reads. A
+// live SET GLOBAL apply deliberately does not touch it: that value was adopted
+// without a restart, which is what "not pending" means.
+func (e *mariadbEngine) recordServingCopy() error {
+	content, err := os.ReadFile(e.params.installedPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("read the installed parameters: %w", err)
+	}
+	return e.params.write(e.params.servingPath(), content)
+}
+
+// A value safe to interpolate into a single-quoted SQL literal. The quote is
+// doubled, which holds in every sql_mode, and the backslash is doubled, which is
+// why every statement built here pins a mode with backslash escapes on.
+func sqlLiteral(value string) string {
+	return strings.ReplaceAll(strings.ReplaceAll(value, `\`, `\\`), `'`, `''`)
+}
+
+// The spellings MariaDB parses for a boolean system variable, and the only ones
+// the catalog's own enum narrows a boolean to.
+var mariadbBooleanValues = map[string]string{
+	"on": "ON", "off": "OFF", "true": "ON", "false": "OFF", "1": "ON", "0": "OFF",
+}
+
+// One SET GLOBAL right-hand side. MariaDB refuses a quoted literal for a numeric
+// or boolean with ER_WRONG_TYPE_FOR_VAR, so the catalog's data type picks the
+// form; re-rendering from the parsed value is what keeps it safe to interpolate.
+func mariadbSetValue(dataType, value string) (string, error) {
+	switch dataType {
+	case handlers_rds.ParamTypeInteger:
+		n, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64)
+		if err != nil {
+			return "", fmt.Errorf("value %q is not an integer", value)
+		}
+		return strconv.FormatInt(n, 10), nil
+	case handlers_rds.ParamTypeReal:
+		f, err := strconv.ParseFloat(strings.TrimSpace(value), 64)
+		if err != nil {
+			return "", fmt.Errorf("value %q is not a real number", value)
+		}
+		return strconv.FormatFloat(f, 'g', -1, 64), nil
+	case handlers_rds.ParamTypeBoolean:
+		if literal, ok := mariadbBooleanValues[strings.ToLower(strings.TrimSpace(value))]; ok {
+			return literal, nil
+		}
+		return "", fmt.Errorf("value %q is not a boolean", value)
+	default:
+		return "'" + sqlLiteral(value) + "'", nil
+	}
+}
+
+// Turns off the two logs that would otherwise copy a statement's text, and pins
+// the mode the escaping above assumes. All three are session-scoped: a rotation
+// must not leave the customer's own general log switched off behind it.
+const mariadbSessionGuard = `SET SESSION sql_log_off = 1;
+SET SESSION slow_query_log = 0;
+SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';
+`
+
+// Rotates the master's password. The statement is built here rather than by the
+// client, which has neither psql's identifier interpolation nor its parameter
+// quoting, and it rides stdin so it is never visible in the process table.
+func (e *mariadbEngine) SetPassword(ctx context.Context, username, password string) error {
+	if username == "" || password == "" {
+		return fmt.Errorf("set-password requires both %s and %s",
+			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
+	}
+	// This runs as the server's own superuser, so a reserved name in the payload
+	// would rotate the platform's account. The whole rule matters here, not just
+	// the reserved set: the name is interpolated below rather than client-quoted.
+	if err := e.rules.validateUsername(username); err != nil {
+		return fmt.Errorf("refusing to set the password of role %q: %w", username, err)
+	}
+
+	escaped := sqlLiteral(password)
+	sql := fmt.Sprintf("%sALTER USER '%s'@'%s' IDENTIFIED BY '%s';\n",
+		mariadbSessionGuard, username, mariadbMasterHost, escaped)
+	if _, err := e.clientRun(ctx, sql); err != nil {
+		// The client echoes the failing statement, which carries the escaped form
+		// of the password rather than the raw one — so redacting only the raw one
+		// would leak exactly the passwords that needed escaping.
+		return fmt.Errorf("apply the master password: %s", redact(redact(err.Error(), password), escaped))
+	}
+	return nil
+}
+
+// Installs the resolved set and applies the half a running server can adopt.
+// MariaDB re-reads no file while up, so SET GLOBAL is the only immediate path.
+// It has no offline parser, so the safety net is the boot-time rollback.
+func (e *mariadbEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+
+	initialState, _ := e.probe.state(ctx)
+	if initialState == engineRecovering {
+		return nil, errors.New("apply parameters while the engine is still starting or recovering")
+	}
+	if initialState == engineServing {
+		// A command can arrive before the first heartbeat. Seed the configuration
+		// currently being served before replacing its file in that window.
+		if err := e.recordServingParametersLocked(ctx); err != nil {
+			return nil, fmt.Errorf("record the parameters serving before the apply: %w", err)
+		}
+	}
+
+	restore, err := e.params.install(params)
+	if err != nil {
+		return nil, err
+	}
+
+	state, _ := e.probe.state(ctx)
+	switch state {
+	case engineAbsent:
+		return e.restartOnRepairSetLocked(ctx)
+	case engineRecovering:
+		return nil, errors.Join(errors.New("the engine entered startup or recovery during the parameter apply"), restore())
+	}
+
+	// Before the set reaches the server: enforcement is one of the values below,
+	// and a server told to require a transport it cannot offer would refuse every
+	// client the statement after.
+	if err := e.requireServingTLSForEnforcement(ctx); err != nil {
+		return nil, errors.Join(err, restore())
+	}
+
+	if err := e.applyDynamicParameters(ctx, params); err != nil {
+		// The engine may have gone down after the first probe. In that case start it
+		// on the newly installed set rather than restoring one that already left it
+		// unable to serve.
+		if state, _ := e.probe.state(ctx); state == engineAbsent {
+			return e.restartOnRepairSetLocked(ctx)
+		}
+		return nil, errors.Join(err, restore())
+	}
+	// Last, and on its own: the batch above is not a transaction, so moving the
+	// TLS posture inside it would leave enforcement changed on a set the server
+	// went on to refuse, with only the parameter file rolled back.
+	if err := e.applyTLSEnforcement(ctx); err != nil {
+		if state, _ := e.probe.state(ctx); state == engineAbsent {
+			return e.restartOnRepairSetLocked(ctx)
+		}
+		return nil, errors.Join(err, restore())
+	}
+	return e.pendingRestartParameters(ctx)
+}
+
+// The dynamic half, in one invocation that stops at the first refusal so the
+// server names the setting it would not take. Only catalog names are emitted,
+// which is what makes the identifier safe to build into the statement.
+//
+// TLS enforcement is excluded: applyTLSEnforcement issues it after this batch
+// has succeeded.
+func (e *mariadbEngine) applyDynamicParameters(ctx context.Context, params []handlers_rds.Parameter) error {
+	var b strings.Builder
+	b.WriteString("SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';\n")
+	applied := 0
+	for _, p := range params {
+		if p.Name == "" || p.Name == e.rules.tlsEnforcementParameter || e.rules.isStatic(p.Name) {
+			continue
+		}
+		value, err := mariadbSetValue(e.rules.dataType(p.Name), p.Value)
+		if err != nil {
+			return fmt.Errorf("apply the dynamic parameters: %s: %w", p.Name, err)
+		}
+		fmt.Fprintf(&b, "SET GLOBAL %s = %s;\n", p.Name, value)
+		applied++
+	}
+	if applied == 0 {
+		return nil
+	}
+	if _, err := e.clientRun(ctx, b.String()); err != nil {
+		return fmt.Errorf("apply the dynamic parameters: %w", err)
+	}
+	return nil
+}
+
+// Puts the installed set's enforcement on the running server. The value comes
+// from the installed file rather than the set in hand, so a set that predates
+// the parameter enforces here rather than leaving the server on mariadbd's own
+// default of off while the API reports enforcement.
+func (e *mariadbEngine) applyTLSEnforcement(ctx context.Context) error {
+	enforce, err := installedTLSEnforcement(e.rules.tlsEnforcementParameter, e.params.installedPath())
+	if err != nil {
+		return err
+	}
+	previous, err := e.readTLSEnforcement(ctx)
+	if err != nil {
+		return err
+	}
+	if previous == enforce {
+		return nil
+	}
+	applyErr := e.setTLSEnforcement(ctx, enforce)
+	if applyErr == nil {
+		return nil
+	}
+	// The client can fail without saying whether the server ran the statement, so
+	// the posture it left behind is read rather than assumed unchanged: the one
+	// outcome that must not survive a failed apply is plaintext nobody asked for.
+	live, readErr := e.readTLSEnforcement(ctx)
+	if readErr != nil {
+		return errors.Join(applyErr, readErr)
+	}
+	if live == previous {
+		return applyErr
+	}
+	return errors.Join(applyErr, e.setTLSEnforcement(ctx, previous))
+}
+
+func (e *mariadbEngine) readTLSEnforcement(ctx context.Context) (bool, error) {
+	name := e.rules.tlsEnforcementParameter
+	out, err := e.clientRun(ctx, "SELECT @@global."+name+";\n")
+	if err != nil {
+		return false, fmt.Errorf("read whether the engine requires TLS of clients: %w", err)
+	}
+	switch value := strings.TrimSpace(out); value {
+	case "1":
+		return true, nil
+	case "0":
+		return false, nil
+	default:
+		return false, fmt.Errorf("the engine reports %s as %q, which is neither 1 nor 0", name, value)
+	}
+}
+
+func (e *mariadbEngine) setTLSEnforcement(ctx context.Context, enforce bool) error {
+	name, value := e.rules.tlsEnforcementParameter, "0"
+	if enforce {
+		value = "1"
+	}
+	// Rendered the way the batch renders every other boolean, so the statement the
+	// server sees does not depend on which path issued it.
+	literal, err := mariadbSetValue(handlers_rds.ParamTypeBoolean, value)
+	if err != nil {
+		return fmt.Errorf("render %s: %w", name, err)
+	}
+	sql := fmt.Sprintf("SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';\nSET GLOBAL %s = %s;\n", name, literal)
+	if _, err := e.clientRun(ctx, sql); err != nil {
+		return fmt.Errorf("set %s to %s: %w", name, literal, err)
+	}
+	return nil
+}
+
+// Refuses to require TLS of clients from a server that is not serving it, which
+// would reject every TCP connection while the parameter path reported success.
+// Read back from the installed file rather than taken from the set in hand, so
+// what is checked is also what the server reads at its next start.
+func (e *mariadbEngine) requireServingTLSForEnforcement(ctx context.Context) error {
+	enforce, err := installedTLSEnforcement(e.rules.tlsEnforcementParameter, e.params.installedPath())
+	if err != nil {
+		return err
+	}
+	if !enforce {
+		return nil
+	}
+	// DISABLED is a server built with TLS and started without a certificate,
+	// which is what a boot that was handed none leaves behind. The agent's own
+	// connections are unaffected either way: MariaDB counts a unix socket as a
+	// secure transport.
+	out, err := e.clientRun(ctx, "SELECT @@global.have_ssl;\n")
+	if err != nil {
+		return fmt.Errorf("read whether the engine is serving TLS: %w", err)
+	}
+	if serving := strings.TrimSpace(out); serving != "YES" {
+		return fmt.Errorf("the engine is serving without TLS (have_ssl = %q), so requiring it of clients would reject every connection", serving)
+	}
+	return nil
+}
+
+func (e *mariadbEngine) restartOnRepairSetLocked(ctx context.Context) ([]string, error) {
+	return awaitRepairedEngine(ctx, e.probe, e.Restart, e.pendingRestartParameters, e.repairTimeout, e.repairPoll)
+}
+
+// MariaDB has no pending_restart, so this is computed from the files: the
+// catalog-static keys whose installed value differs from the one the server
+// started on. Live values are not compared — mysqld silently rewrites them.
+func (e *mariadbEngine) pendingRestartParameters(context.Context) ([]string, error) {
+	installed, err := readOptionFile(e.params.installedPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			// Nothing is installed, so nothing is waiting to be adopted.
+			return nil, nil
+		}
+		return nil, fmt.Errorf("read the installed parameters: %w", err)
+	}
+	serving, err := readOptionFile(e.params.servingPath())
+	if err != nil && !os.IsNotExist(err) {
+		return nil, fmt.Errorf("read the parameters the engine started on: %w", err)
+	}
+	// Back to the catalog's names before anything is classified or reported: the
+	// files are written in the engine's startup spellings, and the answer is a
+	// customer-facing list the API refuses ApplyMethod=immediate against.
+	installed = e.catalogKeyed(installed)
+	serving = e.catalogKeyed(serving)
+
+	// An absent serving copy is a set the engine has not started on: rds-init
+	// writes the two together on every boot, so the whole static half counts as
+	// pending rather than being promoted on the strength of a missing file.
+	var pending []string
+	for name, value := range installed {
+		if !e.rules.isStatic(name) {
+			continue
+		}
+		if served, ok := serving[name]; !ok || served != value {
+			pending = append(pending, name)
+		}
+	}
+	// A static setting the group stopped naming reverts to its default at the next
+	// start, which the running server has not adopted either.
+	for name := range serving {
+		if _, ok := installed[name]; !ok && e.rules.isStatic(name) {
+			pending = append(pending, name)
+		}
+	}
+	slices.Sort(pending)
+	return pending, nil
+}
+
+// Snapshots the installed set as the rollback target, but only when nothing in
+// it is still waiting for a restart. The parameter mutex keeps an apply from
+// replacing the file between that check and the copy.
+func (e *mariadbEngine) RecordServingParameters(ctx context.Context) error {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return e.recordServingParametersLocked(ctx)
+}
+
+func (e *mariadbEngine) recordServingParametersLocked(ctx context.Context) error {
+	return recordLastGood(ctx, e.params, e.pendingRestartParameters)
+}
+
+// Puts the last set the engine accepted back in place, for a start that failed
+// after a parameter change.
+func (e *mariadbEngine) RestoreLastKnownGoodParameters(ctx context.Context) (bool, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return restoreLastGood(ctx, e.params, e.probe)
+}
+
+// The same settings under the names the catalog and the API know them by, since
+// a file is written in the engine's startup spellings and those are not always
+// the customer's.
+func (e *mariadbEngine) catalogKeyed(values map[string]string) map[string]string {
+	out := make(map[string]string, len(values))
+	for name, value := range values {
+		out[e.rules.catalogName(name)] = value
+	}
+	return out
+}
+
+// The subset of MariaDB's option-file syntax the generated drop-ins are written
+// in: a group header, comments, and one `name = value` per line. Nothing else
+// reaches these two files, because only rds-init and this agent write them.
+func readOptionFile(path string) (map[string]string, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	values := make(map[string]string)
+	for line := range strings.SplitSeq(string(raw), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") || strings.HasPrefix(line, "[") {
+			continue
+		}
+		// A bare option such as skip-log-bin carries no value; it is still a setting
+		// whose presence or absence differs.
+		name, value, _ := strings.Cut(line, "=")
+		values[normaliseOptionName(name)] = strings.Trim(strings.TrimSpace(value), `"'`)
+	}
+	return values, nil
+}
+
+// MariaDB reads - and _ as the same character in an option name, so the two
+// spellings of one setting must not compare as two settings.
+func normaliseOptionName(name string) string {
+	return strings.ReplaceAll(strings.TrimSpace(name), "-", "_")
+}
+
+// How long any one BACKUP STAGE waits for its metadata locks. Well under the
+// control plane's quiesce timeout, so a stage that cannot take its lock is
+// reported rather than left queued in front of live traffic.
+const mariadbQuiesceLockWait = 20 * time.Second
+
+// MariaDB's own backup API rather than FLUSH TABLES WITH READ LOCK: FTWRL makes
+// the whole database read-only for the full hold, and its acquisition phase is
+// unbounded. BACKUP STAGE blocks commits instead, and covers Aria and MyISAM.
+func (e *mariadbEngine) Quiesce(ctx context.Context, label string, hold time.Duration) error {
+	if err := validateQuiesceRequest(label, hold); err != nil {
+		return err
+	}
+
+	// Held across the whole start, so a second quiesce waits and then finds the
+	// first one's hold rather than opening a concurrent backup alongside it.
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.held != nil {
+		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)
+	}
+
+	// Deliberately not the command's context: the session has to outlive the call
+	// that started it, and its own deadline is what bounds it instead.
+	session, err := e.startSess(context.WithoutCancel(ctx), command{
+		Name:              e.client,
+		Args:              e.clientArgs(),
+		Env:               []string{"PATH=" + defaultGuestPath},
+		SentinelStatement: "SELECT '" + sessionSentinel + "';\n",
+	})
+	if err != nil {
+		return fmt.Errorf("open a backup session: %w", err)
+	}
+
+	// In order and on one connection: MariaDB releases the hold with the session
+	// that took it, which bounds a control plane that dies mid-snapshot. Fed one
+	// at a time so a stage that cannot take its locks is named.
+	stages := []string{
+		fmt.Sprintf("SET SESSION lock_wait_timeout = %d;\n", int(mariadbQuiesceLockWait.Seconds())),
+		"BACKUP STAGE START;\n",
+		"BACKUP STAGE FLUSH;\n",
+		"BACKUP STAGE BLOCK_DDL;\n",
+		"BACKUP STAGE BLOCK_COMMIT;\n",
+	}
+	for _, sql := range stages {
+		if err := session.Exec(ctx, sql); err != nil {
+			if closeErr := session.Close(); closeErr != nil {
+				slog.Warn("rds-agent: closing a failed backup session", "err", closeErr)
+			}
+			return fmt.Errorf("put the engine into backup mode at %q: %w", strings.TrimSpace(sql), err)
+		}
+	}
+
+	e.beginHoldLocked(label, session, hold)
+	return nil
+}
+
+func (e *mariadbEngine) Unquiesce(ctx context.Context) error {
+	return e.releaseHold(ctx, "BACKUP STAGE END;\n")
+}
+
+// Feeds sql on stdin rather than as an argument, so a statement is never visible
+// in the process table.
+func (e *mariadbEngine) clientRun(ctx context.Context, sql string) (string, error) {
+	return e.run(ctx, command{
+		Name:  e.client,
+		Args:  e.clientArgs(),
+		Env:   []string{"PATH=" + defaultGuestPath},
+		Stdin: sql,
+	})
+}
+
+// --unbuffered so a held session's output reaches the reader as each statement
+// finishes rather than when a pipe buffer fills. No User on the command: the
+// connection args carry it, and the agent runs as the account they name.
+func (e *mariadbEngine) clientArgs() []string {
+	return append(mariadbSocketConnectArgs(e.socket),
+		"--batch", "--skip-column-names", "--unbuffered")
+}

cmd/rds-agent/engine_mariadb_log_test.go

@@ -0,0 +1,211 @@
+package main
+
+//test:in-package — the agent is a main package, which has no external test
+// package to import it from, and this covers the unexported engineLogTail and
+// probe-reason helpers directly.
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+)
+
+const mariadbRDSInitScript = "../../scripts/images/rds-mariadb/rds-init"
+
+func writeEngineLog(t *testing.T, body string) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "error.log")
+	if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
+		t.Fatalf("write engine log: %v", err)
+	}
+	return path
+}
+
+// The whole point of the log: "not answering on its socket" is the same message
+// for a server still recovering and for one that refused to start, and only the
+// engine's own log separates them.
+func TestMariaDBProbe_QuotesTheEngineLogWhenTheSocketIsSilent(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	cfg.EngineErrorLog = writeEngineLog(t, strings.Join([]string{
+		"2026-08-12  1:49:26 0 [Note] Starting MariaDB 11.8.0",
+		"2026-08-12  1:49:26 0 [ERROR] InnoDB: Cannot allocate memory for the buffer pool",
+		"2026-08-12  1:49:26 0 [ERROR] Plugin 'InnoDB' registration as a STORAGE ENGINE failed.",
+		"",
+	}, "\n"))
+	writePid(t, cfg.EnginePidFile, os.Getpid())
+
+	state, message := mariadbProbeState(cfg,
+		func(context.Context, string, ...string) (int, string, error) { return 1, "", nil },
+		func(int) bool { return true })(t.Context(), int64(cfg.EnginePort))
+
+	if state != engineRecovering {
+		t.Errorf("state = %v, want recovering", state)
+	}
+	if !strings.Contains(message, "Cannot allocate memory for the buffer pool") {
+		t.Errorf("message = %q, want the engine's own refusal quoted", message)
+	}
+}
+
+func TestMariaDBProbe_SurvivesAnAbsentEngineLog(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	cfg.EngineErrorLog = filepath.Join(t.TempDir(), "does-not-exist.log")
+	writePid(t, cfg.EnginePidFile, os.Getpid())
+
+	state, message := mariadbProbeState(cfg,
+		func(context.Context, string, ...string) (int, string, error) { return 1, "", nil },
+		func(int) bool { return true })(t.Context(), int64(cfg.EnginePort))
+
+	if state != engineRecovering {
+		t.Errorf("state = %v, want recovering", state)
+	}
+	if message != "engine is not answering on its socket yet (startup or crash recovery)" {
+		t.Errorf("message = %q, want the bare reason when there is no log to quote", message)
+	}
+}
+
+// A client refused during the handshake is closed by the server without the
+// server ever logging a cause, so the engine log alone leaves the failure
+// unexplained and only the client's own stderr names it.
+func TestMariaDBProbe_QuotesTheProbeClientWhenItIsRefused(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	cfg.EngineErrorLog = writeEngineLog(t, "2026-08-12  4:32:39 148 [Warning] Aborted connection 148\n")
+	writePid(t, cfg.EnginePidFile, os.Getpid())
+
+	state, message := mariadbProbeState(cfg,
+		func(context.Context, string, ...string) (int, string, error) {
+			return 1, "ERROR 2026 (HY000): TLS/SSL error: self-signed certificate\n", nil
+		},
+		func(int) bool { return true })(t.Context(), int64(cfg.EnginePort))
+
+	if state != engineRecovering {
+		t.Errorf("state = %v, want recovering", state)
+	}
+	if !strings.Contains(message, "self-signed certificate") {
+		t.Errorf("message = %q, want the probe client's own error quoted", message)
+	}
+	if !strings.Contains(message, "Aborted connection") {
+		t.Errorf("message = %q, want the engine log kept alongside the client's error", message)
+	}
+}
+
+func TestCollapseProbeStderr(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{name: "folds to one line", in: "ERROR 2026:\n  TLS error\n", want: "ERROR 2026: TLS error"},
+		{name: "empty stays empty", in: "   \n", want: ""},
+		{
+			name: "bounded",
+			in:   strings.Repeat("y", mariadbProbeStderrMaxBytes+50),
+			want: strings.Repeat("y", mariadbProbeStderrMaxBytes) + "...",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := collapseProbeStderr(tt.in); got != tt.want {
+				t.Errorf("collapseProbeStderr() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+// A probe that says nothing must not append an empty clause: the reason reaches
+// a customer, and a dangling "reported:" would read as truncated output.
+func TestWithProbeStderrLeavesReasonAloneWhenTheClientIsSilent(t *testing.T) {
+	const reason = "engine is not answering on its socket yet"
+	if got := withProbeStderr(reason, "  \n "); got != reason {
+		t.Errorf("withProbeStderr() = %q, want the reason unchanged", got)
+	}
+}
+
+func TestEngineLogTail(t *testing.T) {
+	tests := []struct {
+		name        string
+		body        string
+		want        string
+		wantMissing string
+	}{
+		{
+			name: "keeps only the last lines",
+			body: "one\ntwo\nthree\nfour\nfive\nsix\n",
+			want: "three | four | five | six",
+		},
+		{
+			name: "skips blank lines",
+			body: "alpha\n\n\nbeta\n\n",
+			want: "alpha | beta",
+		},
+		{
+			name: "empty file yields nothing",
+			body: "",
+			want: "",
+		},
+		{
+			name:        "drops a line the byte cap cut in half",
+			body:        strings.Repeat("x", mariadbErrorLogTailBytes) + "\nlast\n",
+			want:        "last",
+			wantMissing: "x",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := engineLogTail(writeEngineLog(t, tt.body))
+			if got != tt.want {
+				t.Errorf("engineLogTail() = %q, want %q", got, tt.want)
+			}
+			if tt.wantMissing != "" && strings.Contains(got, tt.wantMissing) {
+				t.Errorf("engineLogTail() = %q, want no partial line", got)
+			}
+		})
+	}
+}
+
+func TestEngineLogTailUnsetPath(t *testing.T) {
+	if got := engineLogTail(""); got != "" {
+		t.Errorf("engineLogTail(\"\") = %q, want empty", got)
+	}
+}
+
+// The agent reads a path the image writes, and nothing at runtime would notice
+// the two drifting apart: the log would simply always be missing and every
+// refusal would go back unexplained again.
+func TestMariaDBErrorLogPathMatchesTheImage(t *testing.T) {
+	raw, err := os.ReadFile(mariadbRDSInitScript)
+	if err != nil {
+		t.Fatalf("read rds-init: %v", err)
+	}
+	script := string(raw)
+
+	logDir := shellDefault(t, script, "LOG_DIR", "RDS_LOG_DIR")
+	engineLog := shellDefault(t, script, "ENGINE_LOG", "RDS_ENGINE_LOG")
+	want := strings.Replace(engineLog, "${LOG_DIR}", logDir, 1)
+
+	if got := engineLayouts[engineMariaDB].errorLog; got != want {
+		t.Errorf("layout errorLog = %q, but rds-init writes log_error = %q", got, want)
+	}
+}
+
+// Pulls the fallback out of a `NAME="${OVERRIDE:-fallback}"` assignment.
+func shellDefault(t *testing.T, script, name, override string) string {
+	t.Helper()
+	re := regexp.MustCompile(`(?m)^` + name + `="\$\{` + override + `:-(.*)\}"$`)
+	m := re.FindStringSubmatch(script)
+	if m == nil {
+		t.Fatalf("rds-init has no %s assignment defaulting from %s", name, override)
+	}
+	return m[1]
+}
+
+func TestWithEngineLogTailLeavesReasonAloneWithoutALog(t *testing.T) {
+	const reason = "engine is not answering on its socket yet"
+	if got := withEngineLogTail(reason, ""); got != reason {
+		t.Errorf("withEngineLogTail() = %q, want the reason unchanged", got)
+	}
+}

cmd/rds-agent/engine_postgres.go

@@ -0,0 +1,548 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The engine is reached over its unix socket under peer authentication, so the
+// agent drops to the postgres OS user rather than holding a password of its own.
+type postgresEngine struct {
+	quiesceState
+	parameterManager
+
+	// The control plane's own metadata for this engine, resolved once at
+	// startup: the live password apply runs as the cluster superuser, so it
+	// re-checks the role name against the same reserved set the API validates.
+	meta      handlers_rds.Engine
+	run       commandRunner
+	startSess sessionRunner
+	psql      string
+	rcService string
+	service   string
+	pgData    string
+	socketDir string
+	osUser    string
+}
+
+var _ engine = (*postgresEngine)(nil)
+
+// pg_isready gives PostgreSQL three states for free: exit 0 is serving, exit 1
+// is a postmaster up and rejecting connections, anything else is an engine that
+// is not there. Resolved on PATH, where the client package puts it.
+const postgresProbeBinary = "pg_isready"
+
+// The socket directory rather than loopback: the engine binds the customer ENI
+// and nothing else, so there is no TCP path for the probe to take. It also keeps
+// liveness independent of the network path, of the pg_hba scope and of the TLS
+// enforcement rule — libpq reads a host beginning with / as a socket directory,
+// and the generated `local ... peer` line covers it.
+func newPostgresProbe(cfg config, run probeRunner) *engineProbe {
+	return newEngineProbe(cfg.EnginePort, postgresProbeState(cfg.SocketDir, run))
+}
+
+func postgresProbeState(host string, run probeRunner) probeStateFn {
+	return func(ctx context.Context, port int64) (engineState, string) {
+		portArg := strconv.FormatInt(port, 10)
+		code, _, err := run(ctx, postgresProbeBinary, "-h", host, "-p", portArg, "-q")
+		switch {
+		case err != nil:
+			// A missing binary or broken image. Reporting healthy on the strength of
+			// nothing would hide it, so this reads as absent like an engine that did
+			// not answer.
+			return engineAbsent, fmt.Sprintf("engine probe could not run: %v", err)
+		case code == 0:
+			return engineServing, ""
+		case code == 1:
+			return engineRecovering, "engine is rejecting connections (startup or recovery)"
+		default:
+			return engineAbsent, fmt.Sprintf("engine did not respond on %s:%s", host, portArg)
+		}
+	}
+}
+
+// The include the resolved set is rendered to, and the copy of the last one the
+// engine accepted. Both live beside the data rather than in /etc, matching
+// rds-init: a class change boots a fresh root volume and would revert them.
+const (
+	postgresParametersFile = "10-rds-parameters.conf"
+	// Deliberately not a .conf name: include_dir globs *.conf, so the rollback
+	// copy must not be read as a second set of settings.
+	postgresLastGoodFile = "10-rds-parameters.last-good"
+)
+
+// The pg_hba rule that implements TLS enforcement, and the directory the
+// generated pg_hba includes it from. Its content is a constant, so the copy
+// rds-init writes in shell and the one written here have no template to drift
+// apart on: enforcement is the presence of the file and nothing more.
+const (
+	postgresHBADir           = "hba.d"
+	postgresForceSSLRuleFile = "20-rds-force-ssl.conf"
+	// An explicit reject rather than an omitted permit: both refuse the
+	// connection, and only this one puts the reason in the server log and in the
+	// client's error.
+	postgresForceSSLRules = `# Managed by the platform: the rule that requires TLS of client connections.
+# Written by rds-init at boot and by rds-agent on a parameter apply.
+hostnossl all all 0.0.0.0/0 reject
+hostnossl all all ::/0 reject
+`
+	// What the read-back must count once the file is in place. Asserting the shape
+	// rather than only the absence of parse errors is what catches the failure
+	// that will actually happen: the derivation above writing the wrong thing.
+	postgresForceSSLRuleCount = 2
+)
+
+// The layout's factory resolves the control plane metadata during startup.
+func newPostgresEngineFromCatalog(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+	meta, err := handlers_rds.LookupEngine(enginePostgres)
+	if err != nil {
+		return nil, fmt.Errorf("this image bakes %s, which this build's control plane does not offer: %w", enginePostgres, err)
+	}
+	return newPostgresEngine(cfg, meta, run, startSess, probe), nil
+}
+
+func newPostgresEngine(cfg config, meta handlers_rds.Engine, run commandRunner, startSess sessionRunner, probe *engineProbe) *postgresEngine {
+	return &postgresEngine{
+		meta:      meta,
+		run:       run,
+		startSess: startSess,
+		psql:      filepath.Join(cfg.EngineBinDir, "psql"),
+		rcService: cfg.RCService,
+		service:   cfg.EngineService,
+		pgData:    cfg.EngineDataDir,
+		socketDir: cfg.SocketDir,
+		osUser:    cfg.EngineUser,
+		parameterManager: parameterManager{
+			probe: probe,
+			params: parameterStore{
+				dir:       filepath.Join(cfg.EngineDataDir, "conf.d"),
+				installed: postgresParametersFile,
+				lastGood:  postgresLastGoodFile,
+				osUser:    cfg.EngineUser,
+				engine:    enginePostgres,
+			},
+			repairTimeout: parameterRepairTimeout,
+			repairPoll:    parameterRepairPoll,
+		},
+	}
+}
+
+// The role name and the password ride the environment and are re-quoted by
+// psql, so neither reaches a shell word or an argv another process can read.
+func (e *postgresEngine) SetPassword(ctx context.Context, username, password string) error {
+	if username == "" || password == "" {
+		return fmt.Errorf("set-password requires both %s and %s",
+			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
+	}
+	// The statement below runs as the cluster superuser over the socket under
+	// peer auth, so a reserved name in the command payload would hand the
+	// customer the bootstrap superuser rather than rotate their own role.
+	if err := e.meta.ValidateUsernameNotReserved(username); err != nil {
+		return fmt.Errorf("refusing to set the password of a role the engine reserves: %w", err)
+	}
+	// psql interpolates the password into the ALTER ROLE before the server sees
+	// it, and these three are what would write it to the log — the last on any
+	// failure at its own default. SUSET, so the parameter group cannot win.
+	const sql = `SET log_statement = 'none';
+SET log_min_duration_statement = -1;
+SET log_min_error_statement = 'panic';
+\getenv master RDS_MASTER_USERNAME
+\getenv password RDS_MASTER_PASSWORD
+ALTER ROLE :"master" WITH LOGIN PASSWORD :'password';
+`
+	_, err := e.psqlRun(ctx, sql,
+		"RDS_MASTER_USERNAME="+username,
+		"RDS_MASTER_PASSWORD="+password,
+	)
+	if err != nil {
+		// psql echoes the failing statement, which here would carry the new
+		// password back to the control plane and into the event ring.
+		return fmt.Errorf("apply the master password: %s", redact(err.Error(), password))
+	}
+	return nil
+}
+
+// Installs the resolved set, validates it with the engine's own parser before
+// the engine adopts it, and reloads. A refused value is rolled back here rather
+// than left on the data volume, where it would survive every VM replace.
+func (e *postgresEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+
+	initialState, _ := e.probe.state(ctx)
+	if initialState == engineRecovering {
+		return nil, errors.New("apply parameters while the engine is still starting or recovering")
+	}
+	if initialState == engineServing {
+		// A command can arrive before the first heartbeat. Seed the configuration
+		// currently being served before replacing its file in that window.
+		if err := e.recordServingParametersLocked(ctx); err != nil {
+			return nil, fmt.Errorf("record the parameters serving before the apply: %w", err)
+		}
+	}
+
+	// The check runs against the file in place, because the engine parses the
+	// datadir's own include_dir. The window that leaves is closed by the rollback
+	// below and the last-known-good restore the agent runs at boot.
+	restoreParameters, err := e.params.install(params)
+	if err != nil {
+		return nil, err
+	}
+	if err := e.checkConfig(ctx); err != nil {
+		return nil, errors.Join(fmt.Errorf("the engine rejected the parameter set: %w", err), restoreParameters())
+	}
+
+	state, _ := e.probe.state(ctx)
+	switch state {
+	case engineAbsent:
+		return e.restartOnRepairSetLocked(ctx)
+	case engineRecovering:
+		return nil, errors.Join(errors.New("the engine entered startup or recovery during the parameter apply"), restoreParameters())
+	}
+
+	// After the engine's own parser has passed the set and before the reload that
+	// adopts both files at once.
+	enforce, restoreEnforcement, err := e.syncTLSEnforcement(ctx, state)
+	if err != nil {
+		return nil, errors.Join(fmt.Errorf("apply the TLS enforcement of the parameter set: %w", err), restoreParameters())
+	}
+	// The two files carry one apply from here: a rejected set withdrawn without
+	// its enforcement rule would leave the instance serving the old parameters
+	// under the new set's TLS posture.
+	restore := func() error { return errors.Join(restoreEnforcement(), restoreParameters()) }
+
+	if err := e.reload(ctx); err != nil {
+		// The engine may have gone down after the first probe. In that case start
+		// it on the new, parser-checked repair set rather than restoring the set
+		// that already left it unable to serve.
+		if state, _ := e.probe.state(ctx); state == engineAbsent {
+			return e.restartOnRepairSetLocked(ctx)
+		}
+		return nil, errors.Join(err, restore())
+	}
+	if err := e.verifyTLSEnforcement(ctx, enforce); err != nil {
+		// The reload already happened, so putting the files back is not enough: a
+		// pg_hba that parsed but says the wrong thing has been adopted, and only a
+		// second reload takes the engine off it.
+		return nil, errors.Join(err, restore(), e.reload(ctx))
+	}
+	return e.pendingRestartParameters(ctx)
+}
+
+// Re-derived before the restart: the engine reads pg_hba when it starts, so a
+// repair set started under the previous set's enforcement would serve one TLS
+// posture while the API reported the other.
+func (e *postgresEngine) restartOnRepairSetLocked(ctx context.Context) ([]string, error) {
+	if _, _, err := e.syncTLSEnforcement(ctx, engineAbsent); err != nil {
+		return nil, fmt.Errorf("apply the TLS enforcement of the repair set: %w", err)
+	}
+	return awaitRepairedEngine(ctx, e.probe, e.Restart, e.pendingRestartParameters, e.repairTimeout, e.repairPoll)
+}
+
+// The function's own answer is the reload, not psql's exit status: it returns f
+// when it could not signal the postmaster, and the read-back that follows parses
+// pg_hba from disk, so an unsignalled reload would otherwise verify clean.
+func (e *postgresEngine) reload(ctx context.Context) error {
+	out, err := e.psqlRun(ctx, "SELECT pg_reload_conf();\n")
+	if err != nil {
+		return fmt.Errorf("reload the engine configuration: %w", err)
+	}
+	if result := strings.TrimSpace(out); result != "t" {
+		return fmt.Errorf("reload the engine configuration: pg_reload_conf() returned %q", result)
+	}
+	return nil
+}
+
+// The postmaster's own answer: a static setting it has stored but not adopted is
+// what it reports here, so nothing in the guest has to classify one.
+func (e *postgresEngine) pendingRestartParameters(ctx context.Context) ([]string, error) {
+	out, err := e.psqlRun(ctx, "SELECT name FROM pg_settings WHERE pending_restart ORDER BY name;\n")
+	if err != nil {
+		return nil, fmt.Errorf("read the settings pending a restart: %w", err)
+	}
+	var pending []string
+	for line := range strings.SplitSeq(out, "\n") {
+		if name := strings.TrimSpace(line); name != "" {
+			pending = append(pending, name)
+		}
+	}
+	return pending, nil
+}
+
+// Snapshots the include only when the running postmaster has no settings still
+// pending a restart. The parameter mutex keeps an apply from replacing the file
+// between that check and the copy.
+func (e *postgresEngine) RecordServingParameters(ctx context.Context) error {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return e.recordServingParametersLocked(ctx)
+}
+
+func (e *postgresEngine) recordServingParametersLocked(ctx context.Context) error {
+	return recordLastGood(ctx, e.params, e.pendingRestartParameters)
+}
+
+// Puts the last set the engine accepted back in place, for a restart that failed
+// after a parameter change.
+func (e *postgresEngine) RestoreLastKnownGoodParameters(ctx context.Context) (bool, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	restored, err := restoreLastGood(ctx, e.params, e.probe)
+	if err != nil || !restored {
+		return restored, err
+	}
+	// A service restart does not re-run rds-init, so a restore that stopped at the
+	// parameter file would bring the last known good set up under the enforcement
+	// of the set that was just rejected. restoreLastGood only replaces the file of
+	// an engine that is down.
+	if _, _, err := e.syncTLSEnforcement(ctx, engineAbsent); err != nil {
+		return true, fmt.Errorf("apply the TLS enforcement of the restored parameter set: %w", err)
+	}
+	return true, nil
+}
+
+// Turns the installed enforcement value into the pg_hba fact that implements it.
+// PostgreSQL has no server setting for this, so the parameter file on its own
+// would report the change applied and enforce nothing.
+//
+// The serving-TLS guard runs only against an engine that is up. The restore and
+// repair paths reach this with the engine down, where the rule is read by the
+// start that follows rather than by a reload.
+func (e *postgresEngine) syncTLSEnforcement(ctx context.Context, state engineState) (enforce bool, restore func() error, err error) {
+	enforce, err = installedTLSEnforcement(e.meta.TLSEnforcementParameter(), e.params.installedPath())
+	if err != nil {
+		return false, nil, err
+	}
+	if enforce && state == engineServing {
+		if err := e.requireServingTLS(ctx); err != nil {
+			return false, nil, err
+		}
+	}
+	restore, err = e.writeTLSEnforcement(enforce)
+	if err != nil {
+		return false, nil, err
+	}
+	return enforce, restore, nil
+}
+
+// Writes or removes the enforcement rule, returning a func putting back whatever
+// was there. Removing matters as much as writing: a restored data volume carries
+// hba.d with it, so a stale rule would keep enforcing under a parameter group
+// that turns enforcement off.
+func (e *postgresEngine) writeTLSEnforcement(enforce bool) (func() error, error) {
+	path := e.forceSSLRulePath()
+	previous, readErr := os.ReadFile(path)
+	if readErr != nil && !os.IsNotExist(readErr) {
+		return nil, fmt.Errorf("read the TLS enforcement rule at %s: %w", path, readErr)
+	}
+	existed := readErr == nil
+	restore := func() error {
+		if !existed {
+			return e.removeTLSEnforcement()
+		}
+		return e.params.write(path, previous)
+	}
+
+	if !enforce {
+		if err := e.removeTLSEnforcement(); err != nil {
+			return nil, err
+		}
+		return restore, nil
+	}
+	// rds-init lays the directory down on every boot, so this covers an agent that
+	// reached an apply before one ran rather than the ordinary case.
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, 0o700); err != nil {
+		return nil, fmt.Errorf("create %s: %w", dir, err)
+	}
+	if err := e.params.chownToEngine(dir); err != nil {
+		return nil, err
+	}
+	if err := e.params.write(path, []byte(postgresForceSSLRules)); err != nil {
+		return nil, err
+	}
+	return restore, nil
+}
+
+func (e *postgresEngine) removeTLSEnforcement() error {
+	path := e.forceSSLRulePath()
+	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("withdraw the TLS enforcement rule at %s: %w", path, err)
+	}
+	return nil
+}
+
+func (e *postgresEngine) forceSSLRulePath() string {
+	return filepath.Join(e.pgData, postgresHBADir, postgresForceSSLRuleFile)
+}
+
+// Refuses to require TLS of clients from an engine that is not serving it, which
+// would reject every connection while the parameter path reported success.
+func (e *postgresEngine) requireServingTLS(ctx context.Context) error {
+	out, err := e.psqlRun(ctx, "SHOW ssl;\n")
+	if err != nil {
+		return fmt.Errorf("read whether the engine is serving TLS: %w", err)
+	}
+	if serving := strings.TrimSpace(out); serving != "on" {
+		return fmt.Errorf("the engine is serving without TLS (ssl = %q), so requiring it of clients would reject every connection", serving)
+	}
+	return nil
+}
+
+// PostgreSQL discards a pg_hba it cannot parse on reload and keeps the rules it
+// is already serving, which presents as an apply that succeeded and enforces
+// nothing. So the file is read back.
+//
+// pg_hba_file_rules re-parses from disk, so this proves the file would be
+// adopted rather than that the postmaster has adopted it. The inference holds
+// because a reloaded pg_hba is discarded only on a parse error.
+const postgresHBAReadBack = `SELECT count(*) FILTER (WHERE type = 'hostnossl' AND auth_method = 'reject'),
+       count(*) FILTER (WHERE error IS NOT NULL) FROM pg_hba_file_rules;
+`
+
+func (e *postgresEngine) verifyTLSEnforcement(ctx context.Context, enforce bool) error {
+	out, err := e.psqlRun(ctx, postgresHBAReadBack)
+	if err != nil {
+		return fmt.Errorf("read back the client authentication rules: %w", err)
+	}
+	rules, unparsable, err := parseHBAReadBack(out)
+	if err != nil {
+		return err
+	}
+	if unparsable > 0 {
+		return fmt.Errorf("the client authentication file carries %d rules the engine could not parse, so it would keep the rules it is already serving", unparsable)
+	}
+	want := 0
+	if enforce {
+		want = postgresForceSSLRuleCount
+	}
+	if rules != want {
+		return fmt.Errorf("the client authentication file carries %d TLS enforcement rules, want %d", rules, want)
+	}
+	return nil
+}
+
+func parseHBAReadBack(out string) (rules, unparsable int, err error) {
+	answer := strings.TrimSpace(out)
+	malformed := fmt.Errorf("the client authentication read-back returned %q, want two counts", answer)
+	counts := strings.Split(answer, "|")
+	if len(counts) != 2 {
+		return 0, 0, malformed
+	}
+	parsed := make([]int, len(counts))
+	for i, count := range counts {
+		if parsed[i], err = strconv.Atoi(strings.TrimSpace(count)); err != nil {
+			return 0, 0, malformed
+		}
+	}
+	return parsed[0], parsed[1], nil
+}
+
+// The engine's own parser, run offline against the datadir. Reading one setting
+// back is enough: postgres parses every include first and exits non-zero naming
+// the file and line of an unknown parameter or an out-of-range value.
+func (e *postgresEngine) checkConfig(ctx context.Context) error {
+	if _, err := e.run(ctx, command{
+		Name: filepath.Join(filepath.Dir(e.psql), "postgres"),
+		Args: []string{"-D", e.pgData, "-C", "shared_buffers"},
+		Env:  []string{"PATH=" + defaultGuestPath},
+		User: e.osUser,
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (e *postgresEngine) Stop(ctx context.Context) error {
+	return serviceAction(ctx, e.run, e.rcService, e.service, "stop")
+}
+
+// Only the parameter rollback calls this: a restart the control plane wants goes
+// through RebootDBInstance, which cycles the VM.
+func (e *postgresEngine) Restart(ctx context.Context) error {
+	return serviceAction(ctx, e.run, e.rcService, e.service, "restart")
+}
+
+// Feeds sql on stdin rather than as an argument, so a statement is never
+// visible in the process table.
+func (e *postgresEngine) psqlRun(ctx context.Context, sql string, env ...string) (string, error) {
+	return e.run(ctx, command{
+		Name:  e.psql,
+		Args:  e.psqlArgs(),
+		Env:   append([]string{"PATH=" + defaultGuestPath}, env...),
+		Stdin: sql,
+		User:  e.osUser,
+	})
+}
+
+// Reading the script from stdin is what lets one invocation serve both a
+// one-shot run and a session held open across several statements.
+func (e *postgresEngine) psqlArgs() []string {
+	return []string{
+		"--no-psqlrc", "--quiet", "--no-align", "--tuples-only",
+		"-v", "ON_ERROR_STOP=1",
+		"-h", e.socketDir,
+		"-p", strconv.FormatInt(e.probe.port.Load(), 10),
+		"-U", e.osUser,
+		"-d", "postgres",
+		"-f", "-",
+	}
+}
+
+// Puts the engine into backup mode: the datadir is checkpointed and the engine
+// stops writing over the pages a snapshot is about to read. The hold is released
+// by Unquiesce, or by its own deadline, whichever comes first.
+func (e *postgresEngine) Quiesce(ctx context.Context, label string, hold time.Duration) error {
+	if err := validateQuiesceRequest(label, hold); err != nil {
+		return err
+	}
+
+	// Held across the whole start, so a second quiesce waits and then finds the
+	// first one's hold rather than opening a concurrent backup alongside it.
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.held != nil {
+		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)
+	}
+
+	// Deliberately not the command's context: the session has to outlive the call
+	// that started it, and its own deadline is what bounds it instead.
+	session, err := e.startSess(context.WithoutCancel(ctx), command{
+		Name:              e.psql,
+		Args:              e.psqlArgs(),
+		Env:               []string{"PATH=" + defaultGuestPath, "RDS_BACKUP_LABEL=" + label},
+		User:              e.osUser,
+		SentinelStatement: `\echo ` + sessionSentinel + "\n",
+	})
+	if err != nil {
+		return fmt.Errorf("open a backup session: %w", err)
+	}
+
+	// fast forces an immediate checkpoint rather than spreading it over the
+	// checkpoint interval, which would hold the snapshot open for minutes.
+	const sql = `\getenv label RDS_BACKUP_LABEL
+SELECT pg_backup_start(:'label', fast => true);
+`
+	if err := session.Exec(ctx, sql); err != nil {
+		if closeErr := session.Close(); closeErr != nil {
+			slog.Warn("rds-agent: closing a failed backup session", "err", closeErr)
+		}
+		return fmt.Errorf("put the engine into backup mode: %w", err)
+	}
+
+	e.beginHoldLocked(label, session, hold)
+	return nil
+}
+
+func (e *postgresEngine) Unquiesce(ctx context.Context) error {
+	return e.releaseHold(ctx, "SELECT pg_backup_stop();\n")
+}

cmd/rds-agent/engine_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"errors"
+	"fmt"
 	"os"
 	"os/user"
 	"path/filepath"
@@ -139,11 +140,35 @@ func (r *recordingRunner) run(_ context.Context, c command) (string, error) {
 	return r.out, r.err
 }
 
+func testPostgresEngineMeta(t *testing.T) handlers_rds.Engine {
+	t.Helper()
+	meta, err := handlers_rds.LookupEngine(enginePostgres)
+	if err != nil {
+		t.Fatalf("LookupEngine(%s): %v", enginePostgres, err)
+	}
+	return meta
+}
+
 func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
 	t.Helper()
-	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
-	cfg.PGData = t.TempDir()
-	if err := os.MkdirAll(filepath.Join(cfg.PGData, "conf.d"), 0o700); err != nil {
+	cfg := testEngineConfig(t)
+	return newPostgresEngine(cfg, testPostgresEngineMeta(t), withPostgresReadBacks(cfg.EngineDataDir, run), nil, newPostgresProbe(cfg, staticProbe(0)))
+}
+
+// The same engine with the read-backs left to the runner, for the cases that are
+// about what the engine does with each answer rather than about the rest of an
+// apply.
+func newScriptedTestEngine(t *testing.T, run commandRunner) *postgresEngine {
+	t.Helper()
+	cfg := testEngineConfig(t)
+	return newPostgresEngine(cfg, testPostgresEngineMeta(t), run, nil, newPostgresProbe(cfg, staticProbe(0)))
+}
+
+func testEngineConfig(t *testing.T) config {
+	t.Helper()
+	cfg := testLoadConfig(t, enginePostgres)
+	cfg.EngineDataDir = t.TempDir()
+	if err := os.MkdirAll(filepath.Join(cfg.EngineDataDir, "conf.d"), 0o700); err != nil {
 		t.Fatalf("create conf.d: %v", err)
 	}
 	// The guest's postgres user does not exist here, and the chown of the
@@ -152,8 +177,36 @@ func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
 	if err != nil {
 		t.Fatalf("resolve the current user: %v", err)
 	}
-	cfg.PGUser = current.Username
-	return newPostgresEngine(cfg, run, nil, newEngineProbe(cfg, staticProbe(0)))
+	cfg.EngineUser = current.Username
+	return cfg
+}
+
+// Answers the read-backs every apply runs against a live postmaster, from the
+// datadir the case is working in: an engine serving TLS, a reload the postmaster
+// took, and a pg_hba carrying whatever the agent last wrote. The call still
+// reaches the runner, so a case counting commands sees them; one that is about
+// the read-backs themselves builds its own engine.
+func withPostgresReadBacks(dataDir string, run commandRunner) commandRunner {
+	rule := filepath.Join(dataDir, postgresHBADir, postgresForceSSLRuleFile)
+	return func(ctx context.Context, c command) (string, error) {
+		out, err := run(ctx, c)
+		if err != nil {
+			return out, err
+		}
+		switch {
+		case strings.Contains(c.Stdin, "SHOW ssl"):
+			return "on\n", nil
+		case strings.Contains(c.Stdin, "pg_reload_conf"):
+			return "t\n", nil
+		case strings.Contains(c.Stdin, "pg_hba_file_rules"):
+			rules := 0
+			if _, statErr := os.Stat(rule); statErr == nil {
+				rules = postgresForceSSLRuleCount
+			}
+			return fmt.Sprintf("%d|0\n", rules), nil
+		}
+		return out, err
+	}
 }
 
 // The password reaches psql through the environment and is re-quoted there,
@@ -288,19 +341,27 @@ func TestPostgresEngine_ApplyParametersInstallsAndReloads(t *testing.T) {
 	if !strings.Contains(string(installed), "shared_buffers = '256MB'") {
 		t.Errorf("installed parameters = %q, want the resolved value", installed)
 	}
-	if len(runner.calls) != 3 {
-		t.Fatalf("ran %d commands, want a config check, a reload and a pending-restart read", len(runner.calls))
+	if len(runner.calls) != 5 {
+		t.Fatalf("ran %d commands, want a config check, the TLS guard, a reload, the rule read-back and a pending-restart read", len(runner.calls))
 	}
-	// The check runs before the reload, so a value the engine would refuse never
-	// reaches a running cluster.
+	// The check and the TLS guard both run before the reload, so neither a value
+	// the engine would refuse nor a rule rejecting every client reaches a running
+	// cluster. The read-back of the rules comes after, since the reload is what
+	// puts them in force.
 	if !slices.Contains(runner.calls[0].Args, "-C") {
 		t.Errorf("first command = %v, want the offline config check", runner.calls[0].Args)
 	}
-	if !strings.Contains(runner.calls[1].Stdin, "pg_reload_conf") {
-		t.Errorf("second statement = %q, want the reload", runner.calls[1].Stdin)
+	if !strings.Contains(runner.calls[1].Stdin, "SHOW ssl") {
+		t.Errorf("second statement = %q, want the serving-TLS guard", runner.calls[1].Stdin)
 	}
-	if !strings.Contains(runner.calls[2].Stdin, "pending_restart") {
-		t.Errorf("third statement = %q, want the pending-restart read", runner.calls[2].Stdin)
+	if !strings.Contains(runner.calls[2].Stdin, "pg_reload_conf") {
+		t.Errorf("third statement = %q, want the reload", runner.calls[2].Stdin)
+	}
+	if !strings.Contains(runner.calls[3].Stdin, "pg_hba_file_rules") {
+		t.Errorf("fourth statement = %q, want the client authentication read-back", runner.calls[3].Stdin)
+	}
+	if !strings.Contains(runner.calls[4].Stdin, "pending_restart") {
+		t.Errorf("fifth statement = %q, want the pending-restart read", runner.calls[4].Stdin)
 	}
 	if len(pending) != 2 || pending[0] != "shared_buffers" || pending[1] != "max_connections" {
 		t.Errorf("pending = %v, want both settings the engine reported", pending)
@@ -319,6 +380,21 @@ func TestPostgresEngine_ApplyParametersSurfacesAFailedReload(t *testing.T) {
 	}
 }
 
+// pg_reload_conf() returns f when it could not signal the postmaster, and psql
+// exits 0 all the same. Every read-back after it parses files from disk, so an
+// apply that trusted the exit status would verify clean against a server still
+// serving the old rules.
+func TestPostgresEngine_ApplyParametersSurfacesAnUnsignalledReload(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount, reload: "f"}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	_, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "work_mem", Value: "8MB"}})
+	if err == nil || !strings.Contains(err.Error(), `pg_reload_conf() returned "f"`) {
+		t.Fatalf("ApplyParameters error = %v, want a refusal naming the answer the postmaster gave", err)
+	}
+}
+
 type reloadFailingRunner struct {
 	recordingRunner
 
@@ -341,10 +417,10 @@ func TestPostgresEngine_ApplyParametersRestartsOnARepairSetWhileEngineIsDown(t *
 	codes := []int{2, 2, 0}
 	probeCall := 0
 	cfg := testProbeConfig()
-	engine.probe = newEngineProbe(cfg, func(context.Context, string, ...string) (int, error) {
+	engine.probe = newPostgresProbe(cfg, func(context.Context, string, ...string) (int, string, error) {
 		code := codes[min(probeCall, len(codes)-1)]
 		probeCall++
-		return code, nil
+		return code, "", nil
 	})
 	engine.repairTimeout = 100 * time.Millisecond
 	engine.repairPoll = time.Millisecond
@@ -372,7 +448,7 @@ func TestPostgresEngine_ApplyParametersRestartsOnARepairSetWhileEngineIsDown(t *
 	if !strings.Contains(runner.calls[2].Stdin, "pending_restart") {
 		t.Errorf("third call = %+v, want the pending-restart read", runner.calls[2])
 	}
-	installed, err := os.ReadFile(engine.parametersPath())
+	installed, err := os.ReadFile(engine.params.installedPath())
 	if err != nil {
 		t.Fatalf("read installed parameters: %v", err)
 	}
@@ -381,6 +457,225 @@ func TestPostgresEngine_ApplyParametersRestartsOnARepairSetWhileEngineIsDown(t *
 	}
 }
 
+// Answers the two read-backs an apply runs from fields the case sets, so what
+// the engine does with each answer is what is under test.
+type tlsReadBackRunner struct {
+	recordingRunner
+
+	ssl        string
+	ruleRows   int
+	brokenRows int
+	// What pg_reload_conf() answered. Empty is the postmaster having taken the
+	// signal, which is every case but the one about it not having.
+	reload string
+}
+
+func (r *tlsReadBackRunner) run(ctx context.Context, c command) (string, error) {
+	switch {
+	case strings.Contains(c.Stdin, "SHOW ssl"):
+		r.calls = append(r.calls, c)
+		return r.ssl + "\n", nil
+	case strings.Contains(c.Stdin, "pg_reload_conf"):
+		r.calls = append(r.calls, c)
+		if r.reload == "" {
+			return "t\n", nil
+		}
+		return r.reload + "\n", nil
+	case strings.Contains(c.Stdin, "pg_hba_file_rules"):
+		r.calls = append(r.calls, c)
+		return fmt.Sprintf("%d|%d\n", r.ruleRows, r.brokenRows), nil
+	}
+	return r.recordingRunner.run(ctx, c)
+}
+
+func forceSSLParameter(t *testing.T, value string) []handlers_rds.Parameter {
+	t.Helper()
+	return []handlers_rds.Parameter{
+		{Name: "work_mem", Value: "4096"},
+		{Name: testPostgresEngineMeta(t).TLSEnforcementParameter(), Value: value},
+	}
+}
+
+// PostgreSQL has no server setting for this, so the value in the parameter file
+// is inert: the rule the generated pg_hba includes is the whole of it.
+func TestPostgresEngine_ApplyParametersWritesTheEnforcementRule(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1")); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	rule, err := os.ReadFile(engine.forceSSLRulePath())
+	if err != nil {
+		t.Fatalf("read the enforcement rule: %v", err)
+	}
+	if string(rule) != postgresForceSSLRules {
+		t.Errorf("enforcement rule = %q, want the constant rds-init also writes", rule)
+	}
+}
+
+// A snapshot carries hba.d with it, so restoring one taken while enforcing into
+// a group that turns enforcement off has to take the rule with it.
+func TestPostgresEngine_ApplyParametersRemovesAStaleEnforcementRule(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on"}
+	engine := newScriptedTestEngine(t, runner.run)
+	if err := os.MkdirAll(filepath.Dir(engine.forceSSLRulePath()), 0o700); err != nil {
+		t.Fatalf("create hba.d: %v", err)
+	}
+	if err := os.WriteFile(engine.forceSSLRulePath(), []byte(postgresForceSSLRules), 0o600); err != nil {
+		t.Fatalf("write the rule the restored volume carried: %v", err)
+	}
+
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "0")); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if _, err := os.Stat(engine.forceSSLRulePath()); !os.IsNotExist(err) {
+		t.Errorf("the stale enforcement rule is still in place (stat err = %v)", err)
+	}
+}
+
+// The ordinary state of an instance whose resolved set predates the parameter,
+// and the whole of the migration: it begins requiring TLS with no control-plane
+// work at all.
+func TestPostgresEngine_ApplyParametersEnforcesOnAnAbsentKey(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(t.Context(),
+		[]handlers_rds.Parameter{{Name: "work_mem", Value: "4096"}}); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if _, err := os.Stat(engine.forceSSLRulePath()); err != nil {
+		t.Errorf("a set predating the parameter did not enforce (stat err = %v)", err)
+	}
+}
+
+// The resolver canonicalises every boolean, so a value that is neither means the
+// file was written by something other than the platform. Reading an unparsable
+// security setting as off is the one choice not open here.
+func TestPostgresEngine_ApplyParametersRefusesAnUnparsableEnforcementValue(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on"}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "yes"))
+	if err == nil || !strings.Contains(err.Error(), "neither 1 nor 0") {
+		t.Fatalf("ApplyParameters error = %v, want a refusal naming the unreadable value", err)
+	}
+	if _, statErr := os.Stat(engine.params.installedPath()); !os.IsNotExist(statErr) {
+		t.Errorf("the refused set is still installed (stat err = %v)", statErr)
+	}
+}
+
+// Without this, a live enable against an engine started with no certificate
+// would reject every subsequent connection and report the parameter applied.
+func TestPostgresEngine_ApplyParametersRefusesToEnforceWithoutTLS(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "off"}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1"))
+	if err == nil || !strings.Contains(err.Error(), "serving without TLS") {
+		t.Fatalf("ApplyParameters error = %v, want a refusal naming the engine's own TLS state", err)
+	}
+	if _, statErr := os.Stat(engine.forceSSLRulePath()); !os.IsNotExist(statErr) {
+		t.Errorf("the enforcement rule was written anyway (stat err = %v)", statErr)
+	}
+}
+
+// PostgreSQL discards a pg_hba it cannot parse and keeps the rules it is already
+// serving, which presents as an apply that succeeded and enforces nothing. The
+// shape is asserted as well as the errors, because our own derivation writing
+// the wrong thing is the failure that will actually happen.
+func TestPostgresEngine_ApplyParametersFailsAnUnverifiableEnforcement(t *testing.T) {
+	tests := []struct {
+		name       string
+		value      string
+		ruleRows   int
+		brokenRows int
+		want       string
+	}{
+		{name: "rule missing", value: "1", ruleRows: 0, want: "carries 0 TLS enforcement rules, want 2"},
+		{name: "rule present when it should not be", value: "0", ruleRows: 2, want: "carries 2 TLS enforcement rules, want 0"},
+		{name: "a rule the engine could not parse", value: "1", ruleRows: 2, brokenRows: 1, want: "could not parse"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			runner := &tlsReadBackRunner{ssl: "on", ruleRows: tc.ruleRows, brokenRows: tc.brokenRows}
+			engine := newScriptedTestEngine(t, runner.run)
+
+			_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, tc.value))
+			if err == nil || !strings.Contains(err.Error(), tc.want) {
+				t.Fatalf("ApplyParameters error = %v, want %q", err, tc.want)
+			}
+		})
+	}
+}
+
+// A rejected set has to take its enforcement with it. Withdrawing one without
+// the other would leave the instance serving the previous parameters under the
+// rejected set's TLS posture.
+func TestPostgresEngine_ApplyParametersWithdrawsBothFilesTogether(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
+	engine := newScriptedTestEngine(t, runner.run)
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1")); err != nil {
+		t.Fatalf("first ApplyParameters: %v", err)
+	}
+
+	// Turning enforcement off, verified against a pg_hba that still carries the
+	// rule — so the apply fails after both files have already been replaced.
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "0")); err == nil {
+		t.Fatal("ApplyParameters succeeded against an unverifiable enforcement")
+	}
+	installed, err := os.ReadFile(engine.params.installedPath())
+	if err != nil {
+		t.Fatalf("read the installed parameters: %v", err)
+	}
+	if !strings.Contains(string(installed), "rds.force_ssl = '1'") {
+		t.Errorf("installed parameters = %q, want the previous accepted set restored", installed)
+	}
+	if _, err := os.Stat(engine.forceSSLRulePath()); err != nil {
+		t.Errorf("the enforcement of the previous set was not restored with it (stat err = %v)", err)
+	}
+	// The reload already adopted the rejected files, so putting them back is not
+	// enough on its own.
+	if !strings.Contains(runner.calls[len(runner.calls)-1].Stdin, "pg_reload_conf") {
+		t.Errorf("last statement = %q, want a reload taking the engine off the withdrawn set", runner.calls[len(runner.calls)-1].Stdin)
+	}
+}
+
+// A service restart does not re-run rds-init, so a restore stopping at the
+// parameter file would bring the last known good set up under the enforcement of
+// the set that was just rejected.
+func TestPostgresEngine_RestoreLastKnownGoodReDerivesEnforcement(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1")); err != nil {
+		t.Fatalf("ApplyParameters(enforcing): %v", err)
+	}
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("RecordServingParameters: %v", err)
+	}
+	runner.ruleRows = 0
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "0")); err != nil {
+		t.Fatalf("ApplyParameters(not enforcing): %v", err)
+	}
+	if _, err := os.Stat(engine.forceSSLRulePath()); !os.IsNotExist(err) {
+		t.Fatalf("the enforcement rule survived the set that turned it off (stat err = %v)", err)
+	}
+
+	engine.probe = stubProbe(t, 2, nil)
+	restored, err := engine.RestoreLastKnownGoodParameters(t.Context())
+	if err != nil {
+		t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
+	}
+	if !restored {
+		t.Fatal("did not restore a set that differs from the last known good")
+	}
+	if _, err := os.Stat(engine.forceSSLRulePath()); err != nil {
+		t.Errorf("the restored set's enforcement was not re-derived (stat err = %v)", err)
+	}
+}
+
 func TestPostgresEngine_ApplyParametersKeepsRepairSetWhenRestartTimesOut(t *testing.T) {
 	runner := &recordingRunner{}
 	engine := newTestEngine(t, runner.run)
@@ -392,7 +687,7 @@ func TestPostgresEngine_ApplyParametersKeepsRepairSetWhenRestartTimesOut(t *test
 	if err == nil || !strings.Contains(err.Error(), "wait for the engine") {
 		t.Fatalf("ApplyParameters error = %v, want the repair wait failure", err)
 	}
-	installed, readErr := os.ReadFile(engine.parametersPath())
+	installed, readErr := os.ReadFile(engine.params.installedPath())
 	if readErr != nil {
 		t.Fatalf("read installed parameters: %v", readErr)
 	}
@@ -400,24 +695,3 @@ func TestPostgresEngine_ApplyParametersKeepsRepairSetWhenRestartTimesOut(t *test
 		t.Errorf("installed parameters = %q, want the checked repair set retained", installed)
 	}
 }
-
-// Through the service manager, so the supervisor records the engine as stopped
-// and does not restart it underneath a VM that is going down.
-func TestPostgresEngine_StopGoesThroughTheServiceManager(t *testing.T) {
-	runner := &recordingRunner{}
-	engine := newTestEngine(t, runner.run)
-
-	if err := engine.Stop(context.Background()); err != nil {
-		t.Fatalf("Stop: %v", err)
-	}
-	if len(runner.calls) != 1 {
-		t.Fatalf("ran %d commands, want 1", len(runner.calls))
-	}
-	call := runner.calls[0]
-	if filepath.Base(call.Name) != "rc-service" {
-		t.Errorf("ran %q, want rc-service", call.Name)
-	}
-	if len(call.Args) != 2 || call.Args[0] != "postgresql" || call.Args[1] != "stop" {
-		t.Errorf("args = %v, want [postgresql stop]", call.Args)
-	}
-}

cmd/rds-agent/health.go

@@ -1,38 +1,49 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"errors"
-	"fmt"
 	"os/exec"
-	"strconv"
+	"strings"
 	"sync/atomic"
 
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
 
 // A non-zero exit is a result, not a fault, so it comes back as a code; err is
-// reserved for the probe failing to run at all.
-type probeRunner func(ctx context.Context, name string, args ...string) (int, error)
+// reserved for the probe failing to run at all. The client's stderr comes back
+// with it: on a refusal it is the only place the reason is ever stated.
+type probeRunner func(ctx context.Context, name string, args ...string) (int, string, error)
 
-func execProbeRunner(ctx context.Context, name string, args ...string) (int, error) {
-	err := exec.CommandContext(ctx, name, args...).Run()
+func execProbeRunner(ctx context.Context, name string, args ...string) (int, string, error) {
+	var stderr bytes.Buffer
+	cmd := exec.CommandContext(ctx, name, args...)
+	cmd.Stderr = &stderr
+	err := cmd.Run()
+	out := strings.TrimSpace(stderr.String())
 	if err == nil {
-		return 0, nil
+		return 0, out, nil
 	}
-	var exitErr *exec.ExitError
-	if errors.As(err, &exitErr) {
-		return exitErr.ExitCode(), nil
+	if ctxErr := ctx.Err(); ctxErr != nil {
+		return -1, out, ctxErr
 	}
-	return -1, err
+	if exitErr, ok := errors.AsType[*exec.ExitError](err); ok {
+		return exitErr.ExitCode(), out, nil
+	}
+	return -1, out, err
 }
 
-// Shells out to pg_isready rather than dialling the port, since only a startup
-// exchange separates an engine that is serving from one still in recovery.
+// What the engine reports about itself, which is the only part of the probe
+// that differs between engines. The port is passed in rather than read from the
+// engine, so the atomic below stays the one place it is followed from.
+type probeStateFn func(ctx context.Context, port int64) (engineState, string)
+
+// Engine-neutral: the latch, the port, the degraded mapping and the translation
+// to a reported health are shared, and only the state function is supplied by
+// the engine the image bakes.
 type engineProbe struct {
-	run    probeRunner
-	binary string
-	host   string
+	stateFn probeStateFn
 	// Set from the bootstrap config once it lands, from a different goroutine
 	// than the heartbeat that reads it.
 	port atomic.Int64
@@ -41,9 +52,9 @@ type engineProbe struct {
 	seenHealthy bool
 }
 
-func newEngineProbe(cfg config, run probeRunner) *engineProbe {
-	p := &engineProbe{run: run, binary: cfg.PGIsReady, host: cfg.EngineHost}
-	p.port.Store(int64(cfg.EnginePort))
+func newEngineProbe(port int, stateFn probeStateFn) *engineProbe {
+	p := &engineProbe{stateFn: stateFn}
+	p.port.Store(int64(port))
 	return p
 }
 
@@ -52,15 +63,14 @@ func (p *engineProbe) setPort(port int) {
 }
 
 // What the engine is doing, before the seenHealthy latch collapses "never up"
-// and "was up and went away" into one health. The parameter rollback needs them
-// apart: a postmaster that is up and replaying WAL will come back on its own, one
-// that is not running at all after a parameter change will not.
+// and "went away" into one health. The rollback needs them apart: an engine
+// replaying WAL comes back on its own, one that is not running will not.
 type engineState int
 
 const (
 	// Not answering at all, or the probe could not run.
 	engineAbsent engineState = iota
-	// The postmaster is up and rejecting connections: startup or crash recovery.
+	// The process is up but not yet serving: startup or crash recovery.
 	engineRecovering
 	engineServing
 )
@@ -83,21 +93,7 @@ func (p *engineProbe) Check(ctx context.Context) (handlers_rds.EngineHealth, str
 // The raw probe result. Safe to call from a goroutine other than the
 // heartbeat's: it touches no latched state.
 func (p *engineProbe) state(ctx context.Context) (engineState, string) {
-	port := strconv.FormatInt(p.port.Load(), 10)
-	code, err := p.run(ctx, p.binary, "-h", p.host, "-p", port, "-q")
-	switch {
-	case err != nil:
-		// A missing binary or broken image. Reporting healthy on the strength of
-		// nothing would hide it, so this reads as absent like an engine that did
-		// not answer.
-		return engineAbsent, fmt.Sprintf("engine probe could not run: %v", err)
-	case code == 0:
-		return engineServing, ""
-	case code == 1:
-		return engineRecovering, "engine is rejecting connections (startup or recovery)"
-	default:
-		return engineAbsent, fmt.Sprintf("engine did not respond on %s:%s", p.host, port)
-	}
+	return p.stateFn(ctx, p.port.Load())
 }
 
 // A non-answering engine: starting until it has answered once, unhealthy after.

cmd/rds-agent/health_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"strings"
 	"testing"
+	"time"
 
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
@@ -28,8 +29,8 @@ func TestEngineProbe_MapsExitCodesToHealth(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			probe := newEngineProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
-				return tt.code, tt.runErr
+			probe := newPostgresProbe(testProbeConfig(), func(context.Context, string, ...string) (int, string, error) {
+				return tt.code, "", tt.runErr
 			})
 			probe.seenHealthy = tt.seenHealthy
 
@@ -51,8 +52,8 @@ func TestEngineProbe_MapsExitCodesToHealth(t *testing.T) {
 // has to flip the first time the engine answers.
 func TestEngineProbe_LatchesAfterFirstHealthy(t *testing.T) {
 	code := 0
-	probe := newEngineProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
-		return code, nil
+	probe := newPostgresProbe(testProbeConfig(), func(context.Context, string, ...string) (int, string, error) {
+		return code, "", nil
 	})
 
 	if got, _ := probe.Check(context.Background()); got != handlers_rds.EngineHealthHealthy {
@@ -68,9 +69,9 @@ func TestEngineProbe_LatchesAfterFirstHealthy(t *testing.T) {
 // must follow it rather than keep asking the default.
 func TestEngineProbe_ProbesTheAssignedPort(t *testing.T) {
 	var gotArgs []string
-	probe := newEngineProbe(testProbeConfig(), func(_ context.Context, _ string, args ...string) (int, error) {
+	probe := newPostgresProbe(testProbeConfig(), func(_ context.Context, _ string, args ...string) (int, string, error) {
 		gotArgs = args
-		return 0, nil
+		return 0, "", nil
 	})
 
 	probe.setPort(6543)
@@ -81,6 +82,65 @@ func TestEngineProbe_ProbesTheAssignedPort(t *testing.T) {
 	}
 }
 
-func testProbeConfig() config {
-	return config{EngineHost: defaultEngineHost, EnginePort: defaultEnginePort, PGIsReady: defaultPGIsReady}
+func TestExecProbeRunner_PreservesTheProbeDeadline(t *testing.T) {
+	ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
+	defer cancel()
+
+	code, _, err := execProbeRunner(ctx, "sh", "-c", "while :; do :; done")
+	if code != -1 {
+		t.Errorf("exit code = %d, want -1 for a timed-out probe", code)
+	}
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Errorf("error = %v, want the context deadline", err)
+	}
+}
+
+// The client's account of a refusal is the reason a probe failure is legible at
+// all, so a runner that dropped it would leave the caller nothing to report.
+func TestExecProbeRunner_CapturesTheClientsStderr(t *testing.T) {
+	code, stderr, err := execProbeRunner(t.Context(), "sh", "-c", "echo 'TLS handshake failed' >&2; exit 1")
+	if err != nil {
+		t.Fatalf("execProbeRunner: %v", err)
+	}
+	if code != 1 {
+		t.Errorf("exit code = %d, want 1", code)
+	}
+	if stderr != "TLS handshake failed" {
+		t.Errorf("stderr = %q, want the client's own message", stderr)
+	}
+}
+
+// The engine binds the customer ENI and nothing else, so a loopback probe would
+// report a serving engine as absent. libpq reads a host beginning with / as a
+// socket directory, which is also what keeps liveness off the network path.
+func TestPostgresProbe_ProbesTheUnixSocket(t *testing.T) {
+	cfg := testProbeConfig()
+	var probed []string
+	probe := newPostgresProbe(cfg, func(_ context.Context, name string, args ...string) (int, string, error) {
+		probed = append([]string{name}, args...)
+		return 0, "", nil
+	})
+
+	if state, _ := probe.state(context.Background()); state != engineServing {
+		t.Fatalf("probe state = %v, want engineServing", state)
+	}
+	host := ""
+	for i, arg := range probed {
+		if arg == "-h" && i+1 < len(probed) {
+			host = probed[i+1]
+		}
+	}
+	if host != cfg.SocketDir {
+		t.Errorf("probe host = %q, want the socket directory %q", host, cfg.SocketDir)
+	}
+	if !strings.HasPrefix(host, "/") {
+		t.Errorf("probe host = %q, which libpq reads as a TCP host rather than a socket directory", host)
+	}
+}
+
+func testProbeConfig() config {
+	return config{
+		SocketDir:  engineLayouts[enginePostgres].socketDir,
+		EnginePort: engineLayouts[enginePostgres].port,
+	}
 }

cmd/rds-agent/heartbeat_test.go

@@ -17,7 +17,7 @@ func TestHeartbeater_FirstServingProbeSeedsLastKnownGood(t *testing.T) {
 	runner := &recordingRunner{}
 	engine := newTestEngine(t, runner.run)
 	content := []byte("work_mem = '4096'\n")
-	if err := os.WriteFile(engine.parametersPath(), content, 0o600); err != nil {
+	if err := os.WriteFile(engine.params.installedPath(), content, 0o600); err != nil {
 		t.Fatalf("write installed parameters: %v", err)
 	}
 
@@ -25,7 +25,7 @@ func TestHeartbeater_FirstServingProbeSeedsLastKnownGood(t *testing.T) {
 	h := newHeartbeater(cp, engine.probe, engine, 0)
 	h.beat(context.Background())
 
-	lastGood, err := os.ReadFile(engine.lastGoodPath())
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
 	if err != nil {
 		t.Fatalf("read last known good: %v", err)
 	}
@@ -39,7 +39,7 @@ func TestHeartbeater_FirstServingProbeSeedsLastKnownGood(t *testing.T) {
 }
 
 func TestHeartbeater_BoundsBootstrapFailure(t *testing.T) {
-	h := newHeartbeater(newFakeControlPlane(), newEngineProbe(testProbeConfig(), staticProbe(2)), nil, 0)
+	h := newHeartbeater(newFakeControlPlane(), newPostgresProbe(testProbeConfig(), staticProbe(2)), nil, 0)
 	h.setBootstrapFailure("bootstrap fetch", errors.New(strings.Repeat("界", 1000)))
 
 	failure := h.bootstrapFailure.Load()
@@ -60,8 +60,8 @@ func TestHeartbeater_BoundsBootstrapFailure(t *testing.T) {
 func TestHeartbeater_ChecksServingParametersOnEveryHealthyProbe(t *testing.T) {
 	code := 0
 	cfg := testProbeConfig()
-	probe := newEngineProbe(cfg, func(context.Context, string, ...string) (int, error) {
-		return code, nil
+	probe := newPostgresProbe(cfg, func(context.Context, string, ...string) (int, string, error) {
+		return code, "", nil
 	})
 	recorder := &countingServingRecorder{}
 	h := newHeartbeater(newFakeControlPlane(), probe, recorder, 0)

cmd/rds-agent/parameters.go

@@ -0,0 +1,233 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The generated parameter files and what it takes to install one. They live on
+// the data volume at the paths rds-init writes: a class change replaces the VM,
+// so a set under /etc would revert to defaults while the API reported otherwise.
+type parameterStore struct {
+	dir string
+	// The include the resolved set is rendered to.
+	installed string
+	// The copy of the last set the engine accepted, which a boot loop is rolled
+	// back to. Deliberately not a name the engine's own include glob reads, so it
+	// is never parsed as a second set of settings.
+	lastGood string
+	// What the engine actually started on, for an engine that cannot be asked
+	// which of its settings are still pending a restart. Empty for one that can.
+	serving string
+	// Prepended to every file written here, for a configuration format that has
+	// no setting outside a group.
+	header string
+	// The engine parses its own configuration, so a root-owned file would be
+	// unreadable to it.
+	osUser string
+	// Names the engine whose option-file spellings these files are written in.
+	engine string
+}
+
+func (s parameterStore) installedPath() string { return filepath.Join(s.dir, s.installed) }
+func (s parameterStore) lastGoodPath() string  { return filepath.Join(s.dir, s.lastGood) }
+func (s parameterStore) servingPath() string   { return filepath.Join(s.dir, s.serving) }
+
+// Byte-for-byte what rds-init writes for the same set, so the copies compared
+// against it are comparing values rather than formatting.
+func (s parameterStore) render(params []handlers_rds.Parameter) ([]byte, error) {
+	body, err := renderParameters(s.engine, params)
+	if err != nil {
+		return nil, err
+	}
+	return []byte(s.header + body), nil
+}
+
+// Installs the resolved set at the path rds-init uses, so a later boot
+// overwrites rather than shadows it. Returns a func putting back whatever was
+// there, so the caller can undo the install without re-rendering anything.
+func (s parameterStore) install(params []handlers_rds.Parameter) (restore func() error, err error) {
+	path := s.installedPath()
+	// A missing file is the first-ever apply, and its rollback is a removal.
+	previous, readErr := os.ReadFile(path)
+	if readErr != nil && !os.IsNotExist(readErr) {
+		return nil, fmt.Errorf("read the installed parameters at %s: %w", path, readErr)
+	}
+	existed := readErr == nil
+
+	rendered, err := s.render(params)
+	if err != nil {
+		return nil, err
+	}
+	if err := s.write(path, rendered); err != nil {
+		return nil, err
+	}
+	return func() error {
+		if !existed {
+			if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+				return fmt.Errorf("withdraw the rejected parameters at %s: %w", path, err)
+			}
+			return nil
+		}
+		return s.write(path, previous)
+	}, nil
+}
+
+// Atomic: a temp file in the same directory, renamed over the target. The temp
+// name deliberately does not end in the engine's include suffix, so a crash
+// between write and rename cannot leave the engine reading a half-written file.
+func (s parameterStore) write(path string, content []byte) error {
+	tmp := path + ".new"
+	if err := os.WriteFile(tmp, content, 0o600); err != nil {
+		return fmt.Errorf("write %s: %w", tmp, err)
+	}
+	if err := s.chownToEngine(tmp); err != nil {
+		os.Remove(tmp)
+		return err
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("install %s: %w", path, err)
+	}
+	return nil
+}
+
+// A guest with no engine user is a broken image, not something to work around.
+func (s parameterStore) chownToEngine(path string) error {
+	credential, err := lookupCredential(s.osUser)
+	if err != nil {
+		return err
+	}
+	if err := os.Chown(path, int(credential.Uid), int(credential.Gid)); err != nil {
+		return fmt.Errorf("hand %s to %s: %w", path, s.osUser, err)
+	}
+	return nil
+}
+
+// Snapshots the installed set as the rollback target, but only once the engine
+// is running all of it. A set still pending a restart has not been shown to
+// serve, so promoting it would point the rollback at a config about to fail.
+func recordLastGood(ctx context.Context, store parameterStore, pending pendingRestartFn) error {
+	if _, err := os.Stat(store.installedPath()); err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("inspect the serving parameters: %w", err)
+	}
+	names, err := pending(ctx)
+	if err != nil {
+		return err
+	}
+	if len(names) > 0 {
+		return nil
+	}
+
+	content, err := os.ReadFile(store.installedPath())
+	if err != nil {
+		return fmt.Errorf("read the serving parameters: %w", err)
+	}
+	lastGood, err := os.ReadFile(store.lastGoodPath())
+	if err == nil && bytes.Equal(content, lastGood) {
+		return nil
+	}
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("read the last known good parameters: %w", err)
+	}
+	return store.write(store.lastGoodPath(), content)
+}
+
+// Puts the last set the engine accepted back in place, for a restart that failed
+// after a parameter change. The probe is checked again under the caller's
+// parameter lock, so a repair that just brought the engine back is not reversed.
+func restoreLastGood(ctx context.Context, store parameterStore, probe *engineProbe) (bool, error) {
+	if state, _ := probe.state(ctx); state != engineAbsent {
+		return false, nil
+	}
+	lastGood, err := os.ReadFile(store.lastGoodPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, fmt.Errorf("read the last known good parameters: %w", err)
+	}
+	current, err := os.ReadFile(store.installedPath())
+	if err != nil && !os.IsNotExist(err) {
+		return false, fmt.Errorf("read the installed parameters: %w", err)
+	}
+	if bytes.Equal(current, lastGood) {
+		return false, nil
+	}
+	if err := store.write(store.installedPath(), lastGood); err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+// The settings the engine has accepted but will not honour until it restarts.
+// Each implementation answers it its own way, and both the apply and the serving
+// snapshot are gated on the same answer.
+type pendingRestartFn func(ctx context.Context) ([]string, error)
+
+// The parameter-file state every engine keeps. The operations over it are free
+// functions above rather than methods here, so each implementation names its own
+// restart and pending answers at the call site where the compiler checks them.
+type parameterManager struct {
+	params parameterStore
+	probe  *engineProbe
+	// Serializes parameter installs, serving snapshots and rollback restores so
+	// none can copy or replace an intermediate configuration.
+	paramMu       sync.Mutex
+	repairTimeout time.Duration
+	repairPoll    time.Duration
+}
+
+const (
+	parameterRepairTimeout = 90 * time.Second
+	parameterRepairPoll    = time.Second
+)
+
+// Starts a down engine on the set just installed and waits for it to serve.
+// Reached when the engine went away during an apply: the installed set is the
+// newer, so starting on it beats restoring one that already failed.
+func awaitRepairedEngine(ctx context.Context, probe *engineProbe, restart func(context.Context) error,
+	pending pendingRestartFn, timeout, poll time.Duration) ([]string, error) {
+	repairCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	if err := restart(repairCtx); err != nil {
+		return nil, fmt.Errorf("start the engine on the repaired parameter set: %w", err)
+	}
+
+	timer := time.NewTimer(0)
+	defer timer.Stop()
+	lastMessage := "engine did not respond"
+	for {
+		select {
+		case <-repairCtx.Done():
+			if ctx.Err() != nil {
+				return nil, fmt.Errorf("wait for the engine on the repaired parameter set: %w", ctx.Err())
+			}
+			return nil, fmt.Errorf("wait for the engine on the repaired parameter set: %s: %w", lastMessage, repairCtx.Err())
+		case <-timer.C:
+		}
+
+		state, message := probe.state(repairCtx)
+		if state == engineServing {
+			names, err := pending(repairCtx)
+			if err == nil {
+				return names, nil
+			}
+			lastMessage = err.Error()
+		} else if message != "" {
+			lastMessage = message
+		}
+		timer.Reset(poll)
+	}
+}

cmd/rds-agent/paramrecovery.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"log/slog"
 	"time"
 
@@ -49,10 +50,16 @@ func (g *paramGuard) Run(ctx context.Context) {
 		return
 	}
 
+	// A restore that replaced the file and then failed has still moved the
+	// instance onto the last accepted set, so the restart and the report below
+	// are what it needs; returning here would leave it down and unaccounted for.
 	restored, err := g.engine.RestoreLastKnownGoodParameters(ctx)
 	if err != nil {
-		slog.Error("rds-agent: could not restore the last known good parameters", "err", err)
-		return
+		slog.Error("rds-agent: restoring the last known good parameters failed",
+			"err", err, "parametersReplaced", restored)
+		if !restored {
+			return
+		}
 	}
 	if !restored {
 		slog.Warn("rds-agent: engine is down but no different last accepted parameter set is available; not rolling back")
@@ -60,7 +67,7 @@ func (g *paramGuard) Run(ctx context.Context) {
 	}
 
 	message := handlers_rds.ParameterRollbackMessage
-	slog.Error("rds-agent: "+message, "after", g.after)
+	slog.Error("rds-agent: "+message, "after_ms", otelsetup.Millis(g.after))
 	if g.cp != nil {
 		if _, err := g.cp.SubmitState(ctx, g.id, handlers_rds.EngineHealthUnhealthy, message); err != nil {
 			slog.Error("rds-agent: reporting the parameter rollback failed", "err", err)

cmd/rds-agent/paramrecovery_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"os"
-	"path/filepath"
 	"slices"
 	"strings"
 	"testing"
@@ -50,7 +49,7 @@ func TestPostgresEngine_ApplyParametersRollsBackAValueTheEngineRejects(t *testin
 		t.Fatal("ApplyParameters succeeded against a config the engine rejected")
 	}
 
-	installed, err := os.ReadFile(engine.parametersPath())
+	installed, err := os.ReadFile(engine.params.installedPath())
 	if err != nil {
 		t.Fatalf("read the installed parameters: %v", err)
 	}
@@ -69,7 +68,7 @@ func TestPostgresEngine_ApplyParametersWithdrawsTheFirstRejectedSet(t *testing.T
 		[]handlers_rds.Parameter{{Name: "work_mem", Value: "0"}}); err == nil {
 		t.Fatal("ApplyParameters succeeded against a config the engine rejected")
 	}
-	if _, err := os.Stat(engine.parametersPath()); !os.IsNotExist(err) {
+	if _, err := os.Stat(engine.params.installedPath()); !os.IsNotExist(err) {
 		t.Errorf("the rejected include is still installed (stat err = %v)", err)
 	}
 }
@@ -97,7 +96,7 @@ func TestPostgresEngine_ApplyBeforeFirstHeartbeatSeedsLastKnownGood(t *testing.T
 	runner := &pendingAfterReloadRunner{}
 	engine := newTestEngine(t, runner.run)
 	first := []byte("shared_buffers = '32768'\n")
-	if err := os.WriteFile(engine.parametersPath(), first, 0o600); err != nil {
+	if err := os.WriteFile(engine.params.installedPath(), first, 0o600); err != nil {
 		t.Fatalf("write serving parameter set: %v", err)
 	}
 
@@ -108,7 +107,7 @@ func TestPostgresEngine_ApplyBeforeFirstHeartbeatSeedsLastKnownGood(t *testing.T
 	if !slices.Equal(pending, []string{"shared_buffers"}) {
 		t.Errorf("pending = %v, want shared_buffers", pending)
 	}
-	lastGood, err := os.ReadFile(engine.lastGoodPath())
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
 	if err != nil {
 		t.Fatalf("read last known good: %v", err)
 	}
@@ -153,10 +152,10 @@ func TestPostgresEngine_ServingSnapshotWaitsForParameterApply(t *testing.T) {
 	}
 	engine := newTestEngine(t, runner.run)
 	first := []byte("shared_buffers = '32768'\n")
-	if err := os.WriteFile(engine.parametersPath(), first, 0o600); err != nil {
+	if err := os.WriteFile(engine.params.installedPath(), first, 0o600); err != nil {
 		t.Fatalf("write serving parameter set: %v", err)
 	}
-	if err := os.WriteFile(engine.lastGoodPath(), first, 0o600); err != nil {
+	if err := os.WriteFile(engine.params.lastGoodPath(), first, 0o600); err != nil {
 		t.Fatalf("write last known good: %v", err)
 	}
 
@@ -192,7 +191,7 @@ func TestPostgresEngine_ServingSnapshotWaitsForParameterApply(t *testing.T) {
 		t.Error("serving snapshot completed while the parameter apply held its transaction")
 	}
 
-	lastGood, err := os.ReadFile(engine.lastGoodPath())
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
 	if err != nil {
 		t.Fatalf("read last known good: %v", err)
 	}
@@ -216,7 +215,7 @@ func TestPostgresEngine_LastKnownGoodTracksTheServingSet(t *testing.T) {
 	}
 	assertLastGood := func(value string) {
 		t.Helper()
-		lastGood, err := os.ReadFile(engine.lastGoodPath())
+		lastGood, err := os.ReadFile(engine.params.lastGoodPath())
 		if err != nil {
 			t.Fatalf("read the last known good parameters: %v", err)
 		}
@@ -240,8 +239,8 @@ func TestPostgresEngine_LastKnownGoodTracksTheServingSet(t *testing.T) {
 	apply("16384")
 	assertLastGood("8192")
 
-	if strings.HasSuffix(engine.lastGoodPath(), ".conf") {
-		t.Errorf("the rollback copy at %s would be included by the engine", engine.lastGoodPath())
+	if strings.HasSuffix(engine.params.lastGoodPath(), ".conf") {
+		t.Errorf("the rollback copy at %s would be included by the engine", engine.params.lastGoodPath())
 	}
 }
 
@@ -249,7 +248,7 @@ func TestPostgresEngine_RecordServingParametersSkipsPendingRestart(t *testing.T)
 	runner := &recordingRunner{}
 	engine := newTestEngine(t, runner.run)
 	first := []byte("shared_buffers = '32768'\n")
-	if err := os.WriteFile(engine.parametersPath(), first, 0o600); err != nil {
+	if err := os.WriteFile(engine.params.installedPath(), first, 0o600); err != nil {
 		t.Fatalf("write first parameter set: %v", err)
 	}
 	if err := engine.RecordServingParameters(t.Context()); err != nil {
@@ -257,14 +256,14 @@ func TestPostgresEngine_RecordServingParametersSkipsPendingRestart(t *testing.T)
 	}
 
 	second := []byte("shared_buffers = '65536'\n")
-	if err := os.WriteFile(engine.parametersPath(), second, 0o600); err != nil {
+	if err := os.WriteFile(engine.params.installedPath(), second, 0o600); err != nil {
 		t.Fatalf("write pending parameter set: %v", err)
 	}
 	runner.out = "shared_buffers\n"
 	if err := engine.RecordServingParameters(t.Context()); err != nil {
 		t.Fatalf("record with a pending restart: %v", err)
 	}
-	lastGood, err := os.ReadFile(engine.lastGoodPath())
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
 	if err != nil {
 		t.Fatalf("read last known good: %v", err)
 	}
@@ -278,7 +277,7 @@ func TestPostgresEngine_RecordServingParametersSkipsPendingRestart(t *testing.T)
 	if err := engine.RecordServingParameters(t.Context()); err != nil {
 		t.Fatalf("record the restarted parameter set: %v", err)
 	}
-	lastGood, err = os.ReadFile(engine.lastGoodPath())
+	lastGood, err = os.ReadFile(engine.params.lastGoodPath())
 	if err != nil {
 		t.Fatalf("read advanced last known good: %v", err)
 	}
@@ -330,7 +329,7 @@ func TestPostgresEngine_RestoreLastKnownGoodParameters(t *testing.T) {
 	if !restored {
 		t.Fatal("did not restore a set that differs from the last known good")
 	}
-	installed, err := os.ReadFile(engine.parametersPath())
+	installed, err := os.ReadFile(engine.params.installedPath())
 	if err != nil {
 		t.Fatalf("read the installed parameters: %v", err)
 	}
@@ -369,9 +368,9 @@ func (f *fakeRecovery) Restart(context.Context) error {
 // A probe whose result the test drives directly, standing in for pg_isready.
 func stubProbe(t *testing.T, code int, err error) *engineProbe {
 	t.Helper()
-	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
-	return newEngineProbe(cfg, func(context.Context, string, ...string) (int, error) {
-		return code, err
+	cfg := testLoadConfig(t, enginePostgres)
+	return newPostgresProbe(cfg, func(context.Context, string, ...string) (int, string, error) {
+		return code, "", err
 	})
 }
 
@@ -405,6 +404,47 @@ func TestParamGuard_RollsBackReportsAndRestartsWhenTheEngineNeverComesUp(t *test
 	}
 }
 
+// PostgreSQL re-derives TLS enforcement from the set it has just put back, so
+// the file can be replaced and the call still fail. The rollback has happened
+// either way, and the worse outcome is an instance left down and unreported.
+func TestParamGuard_ReportsAndRestartsWhenOnlyTheEnforcementFailed(t *testing.T) {
+	engine := &fakeRecovery{restored: true, restoreErr: errors.New("write the TLS enforcement rule: read-only file system")}
+	cp := newFakeControlPlane()
+	guard := newParamGuard(engine, stubProbe(t, 2, nil), cp)
+	guard.id = identity{DBInstanceIdentifier: "db-1"}
+	guard.after, guard.poll = 20*time.Millisecond, time.Millisecond
+	guard.Run(t.Context())
+
+	if engine.restarts != 1 {
+		t.Errorf("restarts = %d, want the engine restarted on the set already put back", engine.restarts)
+	}
+	states := cp.snapshotStates()
+	if len(states) != 1 {
+		t.Fatalf("submitted states = %d, want the rollback still reported", len(states))
+	}
+	if states[0].health != handlers_rds.EngineHealthUnhealthy || states[0].message != handlers_rds.ParameterRollbackMessage {
+		t.Errorf("rollback state = %+v, want the unhealthy rollback report", states[0])
+	}
+}
+
+// The other half of the same contract: a restore that replaced nothing leaves
+// the instance on the set it is already failing under, so restarting it would
+// only churn a cluster that is down for a reason the rollback cannot fix.
+func TestParamGuard_DoesNotRestartWhenTheRestoreReplacedNothing(t *testing.T) {
+	engine := &fakeRecovery{restored: false, restoreErr: errors.New("read the last known good parameters: input/output error")}
+	cp := newFakeControlPlane()
+	guard := newParamGuard(engine, stubProbe(t, 2, nil), cp)
+	guard.after, guard.poll = 20*time.Millisecond, time.Millisecond
+	guard.Run(t.Context())
+
+	if engine.restarts != 0 {
+		t.Errorf("restarts = %d, want none when nothing was rolled back", engine.restarts)
+	}
+	if states := cp.snapshotStates(); len(states) != 0 {
+		t.Errorf("submitted states = %d, want no rollback report", len(states))
+	}
+}
+
 func TestParamGuard_DoesNothingWhenTheEngineIsServing(t *testing.T) {
 	engine := &fakeRecovery{restored: true}
 	newTestGuard(engine, stubProbe(t, 0, nil)).Run(t.Context())

cmd/rds-agent/quiesce.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"io"
 	"log/slog"
 	"os/exec"
@@ -15,10 +16,9 @@ import (
 	"time"
 )
 
-// PostgreSQL's non-exclusive backup API ties the backup to the session that
-// started it: when that session ends, for any reason, the backup is aborted.
-// That is what makes the hold self-expiring — a control plane that dies mid
-// snapshot cannot leave the engine in backup mode forever.
+// Both engines tie a backup hold to the session that took it: PostgreSQL aborts
+// a non-exclusive backup when its session ends, MariaDB releases a BACKUP STAGE
+// with its connection. That is what makes the hold self-expiring.
 type engineSession interface {
 	// Runs sql and waits for the engine to finish it.
 	Exec(ctx context.Context, sql string) error
@@ -31,7 +31,8 @@ type engineSession interface {
 type sessionRunner func(ctx context.Context, c command) (engineSession, error)
 
 // Written after every statement so the reader knows the engine is done with it.
-// The statements this carries return LSNs, so it cannot appear in a result.
+// The statements this carries return LSNs and backup stages, so it cannot appear
+// in a result.
 const sessionSentinel = "--rds-agent-ready--"
 
 // The backup mode currently held, and the timer that ends it regardless of what
@@ -42,76 +43,66 @@ type quiesceHold struct {
 	expiry  *time.Timer
 }
 
-// Puts the engine into backup mode: the datadir is checkpointed and the engine
-// stops writing over the pages a snapshot is about to read. The hold is released
-// by Unquiesce, or by its own deadline, whichever comes first.
-func (e *postgresEngine) Quiesce(ctx context.Context, label string, hold time.Duration) error {
+// The hold bookkeeping, which is the same for both engines: only the statements
+// that take and release it differ, not the single-hold rule or the deadline that
+// releases it whatever happens to the caller.
+type quiesceState struct {
+	// Guarded because the expiry timer releases the hold on its own goroutine.
+	mu   sync.Mutex
+	held *quiesceHold
+}
+
+// The two things every quiesce needs, checked before a session is opened so a
+// malformed command costs the engine nothing.
+func validateQuiesceRequest(label string, hold time.Duration) error {
 	if label == "" {
 		return errors.New("quiesce requires a backup label")
 	}
 	if hold <= 0 {
 		return errors.New("quiesce requires a positive hold deadline")
 	}
-
-	// Held across the whole start, so a second quiesce waits and then finds the
-	// first one's hold rather than opening a concurrent backup alongside it.
-	e.mu.Lock()
-	defer e.mu.Unlock()
-	if e.held != nil {
-		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)
-	}
-
-	// Deliberately not the command's context: the session has to outlive the call
-	// that started it, and its own deadline is what bounds it instead.
-	session, err := e.startSess(context.WithoutCancel(ctx), command{
-		Name:  e.psql,
-		Args:  e.psqlArgs(),
-		Env:   []string{"PATH=" + defaultGuestPath, "RDS_BACKUP_LABEL=" + label},
-		Stdin: "",
-		User:  e.osUser,
-	})
-	if err != nil {
-		return fmt.Errorf("open a backup session: %w", err)
-	}
-
-	// fast forces an immediate checkpoint rather than spreading it over the
-	// checkpoint interval, which would hold the snapshot open for minutes.
-	const sql = `\getenv label RDS_BACKUP_LABEL
-SELECT pg_backup_start(:'label', fast => true);
-`
-	if err := session.Exec(ctx, sql); err != nil {
-		if closeErr := session.Close(); closeErr != nil {
-			slog.Warn("rds-agent: closing a failed backup session", "err", closeErr)
-		}
-		return fmt.Errorf("put the engine into backup mode: %w", err)
-	}
-
-	e.held = &quiesceHold{
-		label:   label,
-		session: session,
-		expiry:  time.AfterFunc(hold, func() { e.expireQuiesce(label) }),
-	}
-	slog.Info("rds-agent: engine quiesced for backup", "label", label, "hold", hold)
 	return nil
 }
 
-// Ends the backup cleanly. A missing hold is an error rather than a silent
-// success: it means the deadline fired first, so the snapshot the control plane
-// just took was not taken against a held checkpoint.
-func (e *postgresEngine) Unquiesce(ctx context.Context) error {
-	e.mu.Lock()
-	held := e.held
-	e.held = nil
-	e.mu.Unlock()
+// Records the hold and arms its deadline. Called with mu held, so a second
+// quiesce waits and then finds this hold rather than opening a concurrent
+// backup alongside it.
+func (q *quiesceState) beginHoldLocked(label string, session engineSession, hold time.Duration) {
+	q.held = &quiesceHold{
+		label:   label,
+		session: session,
+		expiry:  time.AfterFunc(hold, func() { q.expire(label) }),
+	}
+	slog.Info("rds-agent: engine quiesced for backup", "label", label, "hold_ms", otelsetup.Millis(hold))
+}
 
+// Takes the hold off the engine, if one is still there, and disarms its
+// deadline. The caller ends the session, because the statement that releases the
+// backup has to run on it first.
+func (q *quiesceState) takeHold() *quiesceHold {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	held := q.held
+	q.held = nil
+	if held != nil {
+		held.expiry.Stop()
+	}
+	return held
+}
+
+// Ends the backup cleanly, running releaseSQL on the session that took the hold.
+// A missing hold is an error rather than a silent success: it means the deadline
+// fired first, so what the control plane just snapshotted was not held.
+func (q *quiesceState) releaseHold(ctx context.Context, releaseSQL string) error {
+	held := q.takeHold()
 	if held == nil {
 		return errors.New("the engine is not quiesced; the backup hold had already expired")
 	}
-	held.expiry.Stop()
 
-	// The stop has to run on the session that started the backup, and the session
-	// is closed either way — the engine aborts an unstopped backup with it.
-	execErr := held.session.Exec(ctx, "SELECT pg_backup_stop();\n")
+	// The release has to run on the session that took the hold, and the session is
+	// closed either way — both engines end an unreleased backup with it.
+	execErr := held.session.Exec(ctx, releaseSQL)
 	closeErr := held.session.Close()
 	if execErr != nil {
 		return fmt.Errorf("take the engine out of backup mode: %w", errors.Join(execErr, closeErr))
@@ -123,17 +114,17 @@ func (e *postgresEngine) Unquiesce(ctx context.Context) error {
 	return nil
 }
 
-// The deadline. Ending the session is enough — the engine aborts the backup it
-// was holding — so this never has to reach the engine itself.
-func (e *postgresEngine) expireQuiesce(label string) {
-	e.mu.Lock()
-	held := e.held
+// The deadline. Ending the session is enough — both engines release the hold
+// with it — so this never has to reach the engine itself.
+func (q *quiesceState) expire(label string) {
+	q.mu.Lock()
+	held := q.held
 	if held == nil || held.label != label {
-		e.mu.Unlock()
+		q.mu.Unlock()
 		return
 	}
-	e.held = nil
-	e.mu.Unlock()
+	q.held = nil
+	q.mu.Unlock()
 
 	slog.Warn("rds-agent: backup hold expired; releasing the engine", "label", label)
 	if err := held.session.Close(); err != nil {
@@ -141,21 +132,27 @@ func (e *postgresEngine) expireQuiesce(label string) {
 	}
 }
 
-// A psql child with its stdin held open, so statements can be fed to it one at a
-// time and the engine keeps seeing a single session.
-type psqlSession struct {
+// A client child with its stdin held open, so statements can be fed to it one at
+// a time and the engine keeps seeing a single session.
+type clientSession struct {
 	cmd    *exec.Cmd
 	stdin  io.WriteCloser
 	stdout *bufio.Reader
 	stderr *bytes.Buffer
+	// The statement that makes this client print the sentinel, which is the only
+	// part of a held session that differs between engines.
+	sentinel string
 
 	mu     sync.Mutex
 	closed bool
 }
 
-var _ engineSession = (*psqlSession)(nil)
+var _ engineSession = (*clientSession)(nil)
 
 func execSessionRunner(ctx context.Context, c command) (engineSession, error) {
+	if c.SentinelStatement == "" {
+		return nil, fmt.Errorf("%s was started as a session with no sentinel statement", c.Name)
+	}
 	cmd := exec.CommandContext(ctx, c.Name, c.Args...)
 	cmd.Env = c.Env
 	if c.User != "" {
@@ -180,24 +177,30 @@ func execSessionRunner(ctx context.Context, c command) (engineSession, error) {
 	if err := cmd.Start(); err != nil {
 		return nil, fmt.Errorf("start %s: %w", c.Name, err)
 	}
-	return &psqlSession{cmd: cmd, stdin: stdin, stdout: bufio.NewReader(stdout), stderr: &stderr}, nil
+	return &clientSession{
+		cmd:      cmd,
+		stdin:    stdin,
+		stdout:   bufio.NewReader(stdout),
+		stderr:   &stderr,
+		sentinel: c.SentinelStatement,
+	}, nil
 }
 
-// Feeds sql and waits for the sentinel that follows it. ON_ERROR_STOP makes psql
-// exit on a failed statement, so a sentinel that never arrives is the failure
-// itself and the engine's own message is on stderr.
-func (s *psqlSession) Exec(ctx context.Context, sql string) error {
+// Feeds sql and waits for the sentinel that follows it. Both clients are run in
+// a mode that exits on a failed statement, so a sentinel that never arrives is
+// the failure itself and the engine's own message is on stderr.
+func (s *clientSession) Exec(ctx context.Context, sql string) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.closed {
 		return errors.New("the backup session is closed")
 	}
-	if _, err := io.WriteString(s.stdin, sql+"\\echo "+sessionSentinel+"\n"); err != nil {
+	if _, err := io.WriteString(s.stdin, sql+s.sentinel); err != nil {
 		return fmt.Errorf("%w: %s", err, strings.TrimSpace(s.stderr.String()))
 	}
 
-	// Read on its own goroutine so a psql that never answers costs the caller its
-	// deadline rather than blocking it forever.
+	// Read on its own goroutine so a client that never answers costs the caller
+	// its deadline rather than blocking it forever.
 	done := make(chan error, 1)
 	go func() { done <- s.awaitSentinel() }()
 	select {
@@ -208,7 +211,7 @@ func (s *psqlSession) Exec(ctx context.Context, sql string) error {
 	}
 }
 
-func (s *psqlSession) awaitSentinel() error {
+func (s *clientSession) awaitSentinel() error {
 	for {
 		line, err := s.stdout.ReadString('\n')
 		if strings.TrimSpace(line) == sessionSentinel {
@@ -226,7 +229,7 @@ func (s *psqlSession) awaitSentinel() error {
 // Closing stdin ends the script, and the wait reaps the child. A session that
 // will not end is killed, because leaving it is what would keep the engine in
 // backup mode.
-func (s *psqlSession) Close() error {
+func (s *clientSession) Close() error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.closed {

cmd/rds-agent/quiesce_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"errors"
 	"slices"
 	"strings"
 	"sync"
@@ -113,81 +112,6 @@ func TestPostgresEngine_UnquiesceStopsTheBackupAndEndsTheSession(t *testing.T) {
 	}
 }
 
-// A hold that expired mid-snapshot means the snapshot was not taken against a
-// held checkpoint, so the release reports it rather than succeeding silently.
-func TestPostgresEngine_UnquiesceWithoutAHoldIsAnError(t *testing.T) {
-	engine, _ := newQuiesceEngine(t, &fakeSession{})
-
-	err := engine.Unquiesce(context.Background())
-	if err == nil {
-		t.Fatal("Unquiesce succeeded with no hold to release")
-	}
-	if !strings.Contains(err.Error(), "expired") {
-		t.Errorf("error %q does not say the hold had expired", err)
-	}
-}
-
-// The engine aborts the backup when the session ends, which is what makes the
-// hold self-expiring: a control plane that dies cannot leave it held forever.
-func TestPostgresEngine_QuiesceReleasesItselfOnItsDeadline(t *testing.T) {
-	session := &fakeSession{}
-	engine, _ := newQuiesceEngine(t, session)
-
-	if err := engine.Quiesce(context.Background(), "orders-db-pre-upgrade", 10*time.Millisecond); err != nil {
-		t.Fatalf("Quiesce: %v", err)
-	}
-	deadline := time.Now().Add(2 * time.Second)
-	for session.closes() == 0 && time.Now().Before(deadline) {
-		time.Sleep(time.Millisecond)
-	}
-
-	if session.closes() != 1 {
-		t.Fatal("the backup session outlived its deadline")
-	}
-	// The hold is gone with it, so a later release is honest about that.
-	if err := engine.Unquiesce(context.Background()); err == nil {
-		t.Error("Unquiesce succeeded after the hold expired")
-	}
-}
-
-// One backup at a time: a second start would open a concurrent backup alongside
-// the first rather than extending it.
-func TestPostgresEngine_QuiesceRefusesASecondHold(t *testing.T) {
-	session := &fakeSession{}
-	engine, started := newQuiesceEngine(t, session)
-	if err := engine.Quiesce(context.Background(), "first", time.Minute); err != nil {
-		t.Fatalf("Quiesce: %v", err)
-	}
-
-	err := engine.Quiesce(context.Background(), "second", time.Minute)
-	if err == nil {
-		t.Fatal("a second quiesce was accepted while the first was held")
-	}
-	if !strings.Contains(err.Error(), "first") {
-		t.Errorf("error %q does not name the backup already holding the engine", err)
-	}
-	if len(*started) != 1 {
-		t.Errorf("started %d sessions, want the second refused before opening one", len(*started))
-	}
-}
-
-// A failed start leaves nothing held, so a retry is not refused by a hold that
-// never took effect.
-func TestPostgresEngine_QuiesceEndsTheSessionWhenTheBackupWillNotStart(t *testing.T) {
-	session := &fakeSession{execErr: errors.New(`ERROR:  a backup is already in progress`)}
-	engine, _ := newQuiesceEngine(t, session)
-
-	if err := engine.Quiesce(context.Background(), "orders-db-pre-upgrade", time.Minute); err == nil {
-		t.Fatal("Quiesce succeeded against an engine that refused the backup")
-	}
-	if session.closes() != 1 {
-		t.Errorf("closed %d times, want the failed session ended", session.closes())
-	}
-	if engine.held != nil {
-		t.Error("a failed quiesce left a hold behind")
-	}
-}
-
 func TestCommandRegistry_QuiesceCarriesTheLabelAndDeadline(t *testing.T) {
 	engine := &fakeEngine{}
 	reply := newCommander(nil, newCommandRegistry(engine, &fakeStorage{}), 0).execute(context.Background(),

cmd/rds-agent/register.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"log/slog"
 	"time"
 )
@@ -20,7 +21,7 @@ func (a *Agent) register(ctx context.Context) error {
 		a.hb.setInterval(time.Duration(out.HeartbeatIntervalSeconds) * time.Second)
 
 		slog.Info("rds-agent: registered",
-			"dbInstanceIdentifier", a.id.DBInstanceIdentifier, "heartbeat", a.hb.interval)
+			"dbInstanceIdentifier", a.id.DBInstanceIdentifier, "heartbeat_ms", otelsetup.Millis(a.hb.interval))
 		return nil
 	})
 }

cmd/rds-agent/storage.go

@@ -10,10 +10,9 @@ import (
 	"strings"
 )
 
-// The in-guest half of a storage grow. The control plane has already
-// grown the block device by the time this runs; all that is left is to put the
-// filesystem on the capacity that is now there. Both ext4 and XFS grow while
-// mounted, so this needs no ordering against the engine start.
+// The in-guest half of a storage grow: the control plane has already grown the
+// block device, so all that is left is the filesystem. Both ext4 and XFS grow
+// while mounted, so this needs no ordering against the engine start.
 type storageOps interface {
 	// Extends the filesystem at the data mount onto its whole device, and
 	// reports what it did for the command reply.
@@ -28,8 +27,6 @@ const (
 	defaultXFSGrowfs  = "xfs_growfs"
 	defaultMountsFile = "/proc/mounts"
 	defaultSysBlock   = "/sys/class/block"
-	// Where rds-datadir mounts the data volume; the datadir sits one level in.
-	defaultDataMount = "/var/lib/postgresql"
 	// growpart says this when the partition already fills the disk, which a
 	// resumed grow is the normal way to reach.
 	growpartNoChange = "NOCHANGE"

cmd/rds-agent/storage_test.go

@@ -61,12 +61,12 @@ func newTestStorage(t *testing.T, device, fstype string, run commandRunner) *gue
 	mounts := filepath.Join(dir, "mounts")
 	content := "proc /proc proc rw 0 0\n"
 	if device != "" {
-		content += device + " " + defaultDataMount + " " + fstype + " rw,relatime 0 0\n"
+		content += device + " " + testDataMount + " " + fstype + " rw,relatime 0 0\n"
 	}
 	if err := os.WriteFile(mounts, []byte(content), 0o600); err != nil {
 		t.Fatalf("write the mount table: %v", err)
 	}
-	cfg := config{DataMount: defaultDataMount, MountsFile: mounts, SysBlock: filepath.Join(dir, "block")}
+	cfg := config{DataMount: testDataMount, MountsFile: mounts, SysBlock: filepath.Join(dir, "block")}
 	if err := os.MkdirAll(cfg.SysBlock, 0o755); err != nil {
 		t.Fatalf("create the sysfs tree: %v", err)
 	}
@@ -93,6 +93,9 @@ func seedPartition(t *testing.T, g *guestStorage, disk, name, number string) {
 
 // The data volume is formatted whole, so the usual grow is resize2fs against
 // the device with no partition to push out first.
+// Where rds-datadir mounts the data volume in the PostgreSQL preset.
+const testDataMount = "/var/lib/postgresql"
+
 func TestGrowFilesystem_ResizesAnExt4DeviceOnline(t *testing.T) {
 	log := &runLog{}
 	g := newTestStorage(t, "/dev/vdb", "ext4", log.run)
@@ -108,7 +111,7 @@ func TestGrowFilesystem_ResizesAnExt4DeviceOnline(t *testing.T) {
 	if args := log.calls[0].Args; len(args) != 1 || args[0] != "/dev/vdb" {
 		t.Errorf("resize2fs args = %v, want the device", args)
 	}
-	if !strings.Contains(message, "/dev/vdb") || !strings.Contains(message, defaultDataMount) {
+	if !strings.Contains(message, "/dev/vdb") || !strings.Contains(message, testDataMount) {
 		t.Errorf("reply message = %q, want the device and mount it grew", message)
 	}
 }
@@ -125,7 +128,7 @@ func TestGrowFilesystem_GrowsXFSByMountPoint(t *testing.T) {
 	if got := log.names(); len(got) != 1 || got[0] != defaultXFSGrowfs {
 		t.Fatalf("ran %v, want just %s", got, defaultXFSGrowfs)
 	}
-	if args := log.calls[0].Args; len(args) != 1 || args[0] != defaultDataMount {
+	if args := log.calls[0].Args; len(args) != 1 || args[0] != testDataMount {
 		t.Errorf("xfs_growfs args = %v, want the mount point", args)
 	}
 }

cmd/spinifex/cmd/admin.go

@@ -25,23 +25,23 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"uuid"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/iam"
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/ebsmetadata"
+	"github.com/mulgadc/spinifex/spinifex/ebsprovider"
 	"github.com/mulgadc/spinifex/spinifex/formation"
 	"github.com/mulgadc/spinifex/spinifex/gpu"
+	handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot"
 	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
 	"github.com/mulgadc/spinifex/spinifex/hostdns"
 	"github.com/mulgadc/spinifex/spinifex/network/host"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/utils"
-	"github.com/mulgadc/viperblock/viperblock"
-	"github.com/mulgadc/viperblock/viperblock/backends/s3"
-	"github.com/mulgadc/viperblock/viperblock/v_utils"
 	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/pterm/pterm"
 	"github.com/spf13/cobra"
 )
@@ -118,8 +118,9 @@ var nodeDrainCmd = &cobra.Command{
 its guests via QMP and unmount their volumes (flushing the viperblock WAL) while
 every service is still running. STORAGE/PERSIST/INFRA are left to systemd's
 ordered unit teardown. This is what spinifex-shutdown.service's ExecStop runs,
-with --only-if-host-stopping so it only drains on a genuine host
-shutdown/reboot -- not on a plain "systemctl restart/stop spinifex.target".
+with --unless-restarting so it drains on a genuine host shutdown/reboot and on
+a plain "systemctl stop spinifex.target", but skips the drain when a restart
+of the stack (e.g. "systemctl restart spinifex.target") is already queued.
 Run by hand without that flag, it always drains unconditionally.`,
 	Run: runNodeDrainLocal,
 }
@@ -197,6 +198,25 @@ takes effect immediately. Prompts for confirmation unless --yes is passed.`,
 	Run: runimagesPromoteCmd,
 }
 
+var volumesCmd = &cobra.Command{
+	Use:   "volumes",
+	Short: "Inspect block storage volumes",
+	Long:  `Inspect the volumes the storage provider holds, independently of the EC2 API.`,
+}
+
+var volumesOrphansCmd = &cobra.Command{
+	Use:   "orphans",
+	Short: "Report volumes the provider holds with no control-plane record",
+	Long: `List volumes the storage provider holds that the control plane has no record
+of. Their blocks are consuming space but they have no API handle, so they
+cannot be described or deleted through the EC2 API.
+
+This command only reports. It never deletes: the evidence that an orphan is
+still wanted is the record that went missing, so removal is an operator
+decision made per volume.`,
+	Run: runVolumesOrphansCmd,
+}
+
 var accountCmd = &cobra.Command{
 	Use:   "account",
 	Short: "Manage Spinifex accounts",
@@ -283,7 +303,9 @@ func init() {
 	nodeCmd.AddCommand(nodeDrainCmd)
 	nodeDrainCmd.Flags().Bool("local", false, "Drain the local node only (required)")
 	nodeDrainCmd.Flags().Duration("timeout", 120*time.Second, "Maximum time to wait per phase")
-	nodeDrainCmd.Flags().Bool("only-if-host-stopping", false, "Skip the drain unless systemd is unwinding into shutdown.target (real reboot/poweroff); unset, always drains")
+	nodeDrainCmd.Flags().Bool("unless-restarting", false, "Drain on a real host shutdown or a plain target stop; skip only when a restart of the stack is already queued. Unset, always drains")
+	nodeDrainCmd.Flags().Bool("only-if-host-stopping", false, "Deprecated: use --unless-restarting")
+	_ = nodeDrainCmd.Flags().MarkDeprecated("only-if-host-stopping", "use --unless-restarting instead")
 	nodeCmd.AddCommand(nodeJSProbeCmd)
 	nodeJSProbeCmd.Flags().Duration("timeout", 10*time.Second, "Maximum time to wait for the canary round-trip")
 
@@ -293,6 +315,9 @@ func init() {
 	imagesCmd.AddCommand(imagesRemoveCmd)
 	imagesCmd.AddCommand(imagesPromoteCmd)
 
+	adminCmd.AddCommand(volumesCmd)
+	volumesCmd.AddCommand(volumesOrphansCmd)
+
 	adminCmd.AddCommand(accountCmd)
 	accountCmd.AddCommand(accountCreateCmd)
 	accountCmd.AddCommand(accountListCmd)
@@ -305,6 +330,9 @@ func init() {
 
 	adminCmd.AddCommand(upgradeCmd)
 	upgradeCmd.Flags().Bool("yes", false, "Apply migrations without prompting")
+	upgradeCmd.Flags().Bool("dry-run", false, "Report pending config and unit changes without applying them")
+	upgradeCmd.Flags().Bool("units-only", false, "Reconcile systemd units only, skip config migrations")
+	upgradeCmd.Flags().Bool("skip-units", false, "Apply config migrations only, skip systemd unit reconciliation")
 	certRenewCmd.Flags().StringSlice("extra-ip", nil, "Additional IP addresses to include in SANs")
 	certRenewCmd.Flags().StringSlice("extra-dns", nil, "Additional DNS names to include in SANs")
 	accountCreateCmd.Flags().String("name", "", "Account name (required)")
@@ -387,19 +415,19 @@ func init() {
 	imagesRemoveCmd.Flags().String("image-id", "", "AMI ID to remove (required)")
 	imagesRemoveCmd.Flags().Bool("force", false, "Bypass dependency, ownership and config-corrupt checks (salvage mode)")
 	imagesRemoveCmd.Flags().Bool("yes", false, "Skip interactive confirmation prompt")
-	if err := imagesRemoveCmd.MarkFlagRequired("image-id"); err != nil {
-		panic(err)
-	}
+	_ = imagesRemoveCmd.MarkFlagRequired("image-id")
 
 	imagesPromoteCmd.Flags().String("image-id", "", "AMI ID to promote to system image (required)")
 	imagesPromoteCmd.Flags().Bool("yes", false, "Skip interactive confirmation prompt")
-	if err := imagesPromoteCmd.MarkFlagRequired("image-id"); err != nil {
-		panic(err)
-	}
+	_ = imagesPromoteCmd.MarkFlagRequired("image-id")
 }
 
 const bytesPerGiB = 1024 * 1024 * 1024
 
+// imageImportTimeout bounds each provider call the import makes. Generous
+// because the snapshot at the end runs over the whole imported volume.
+const imageImportTimeout = 30 * time.Minute
+
 // amiVolumeSizeGiB returns the smallest whole GiB that still holds sizeBytes.
 //
 // Rounding up is load-bearing. The image is copied into a root volume of
@@ -427,16 +455,10 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	var imageStat os.FileInfo
 	var err error
 
-	cfgFile, _ := cmd.Flags().GetString("config")
 	forceCmd, _ := cmd.Flags().GetBool("force")
 	skipVerify, _ := cmd.Flags().GetBool("skip-verify")
 	ostmpDir, _ := cmd.Flags().GetString("tmp-dir")
 
-	// Use default config path
-	if cfgFile == "" {
-		cfgFile = DefaultConfigFile()
-	}
-
 	//configDir, _ := cmd.Flags().GetString("config-dir")
 	baseDir, _ := cmd.Flags().GetString("spinifex-dir")
 
@@ -543,9 +565,11 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	// Create the specified image directory
 	imagePath := fmt.Sprintf("%s/%s/%s/%s", imageDir, image.Distro, image.Version, image.Arch)
 
-	// Create config directory
-	if err := os.MkdirAll(imagePath, 0700); err != nil {
-		fmt.Fprintf(os.Stderr, "Error creating config directory: %v\n", err)
+	// 0770, matching the root:spinifex convention setup.sh gives the shared data
+	// dirs. At 0700 the first sudo import leaves a directory no group member can
+	// descend into, so every later import without sudo fails on mkdir.
+	if err := os.MkdirAll(imagePath, 0770); err != nil {
+		fmt.Fprintf(os.Stderr, "Error creating image directory %s: %v\n", imagePath, err)
 		os.Exit(1)
 	}
 
@@ -607,57 +631,46 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	// Create the specified manifest file to describe the image/AMI
-	manifest := viperblock.VolumeConfig{}
-
-	// Calculate the size
-
-	manifest.AMIMetadata.Name = fmt.Sprintf("ami-%s-%s-%s", image.Distro, image.Version, image.Arch)
-	if amiNameOverride != "" {
-		manifest.AMIMetadata.Name = amiNameOverride
-	}
+	// Describe the image/AMI as the control-plane document it will become.
 	volumeId := utils.GenerateResourceID("ami")
-	manifest.AMIMetadata.ImageID = volumeId
+	amiName := fmt.Sprintf("ami-%s-%s-%s", image.Distro, image.Version, image.Arch)
+	if amiNameOverride != "" {
+		amiName = amiNameOverride
+	}
 
-	manifest.AMIMetadata.Description = fmt.Sprintf("%s cloud image prepared for Spinifex", manifest.AMIMetadata.Name)
-	manifest.AMIMetadata.Architecture = image.Arch
-	manifest.AMIMetadata.PlatformDetails = image.Platform
-	manifest.AMIMetadata.CreationDate = time.Now()
-	manifest.AMIMetadata.RootDeviceType = "ebs"
-	manifest.AMIMetadata.Virtualization = "hvm"
-	manifest.AMIMetadata.ImageOwnerAlias = "system"
-	manifest.AMIMetadata.VolumeSizeGiB = amiVolumeSizeGiB(imageStat.Size())
-	manifest.AMIMetadata.BootMode = image.BootMode
-	manifest.AMIMetadata.Distro = image.Distro
-	manifest.AMIMetadata.DistroFamily = utils.DistroFamily(image.Distro)
+	ami := ebsmetadata.AMI{
+		ImageID:         volumeId,
+		Name:            amiName,
+		Description:     fmt.Sprintf("%s cloud image prepared for Spinifex", amiName),
+		Architecture:    image.Arch,
+		PlatformDetails: image.Platform,
+		CreationDate:    time.Now(),
+		RootDeviceType:  "ebs",
+		Virtualization:  "hvm",
+		ImageOwnerAlias: "system",
+		VolumeSizeGiB:   amiVolumeSizeGiB(imageStat.Size()),
+		SnapshotID:      admin.SnapPrefix(volumeId),
+		BootMode:        image.BootMode,
+		Distro:          image.Distro,
+		DistroFamily:    utils.DistroFamily(image.Distro),
+	}
 
 	// Copy catalog-provided tags (e.g. spinifex:managed-by for system AMIs)
 	// onto the imported AMI so the UI can filter them out.
 	if len(image.Tags) > 0 {
-		manifest.AMIMetadata.Tags = make(map[string]string, len(image.Tags))
-		maps.Copy(manifest.AMIMetadata.Tags, image.Tags)
+		ami.Tags = make(map[string]string, len(image.Tags))
+		maps.Copy(ami.Tags, image.Tags)
 	}
 
-	// Volume Data
-	manifest.VolumeMetadata.VolumeID = volumeId // TODO: Confirm if unique, e.g vol-, if ami- used
-	manifest.VolumeMetadata.VolumeName = manifest.AMIMetadata.Name
-	manifest.VolumeMetadata.TenantID = "system"
-	manifest.VolumeMetadata.SizeGiB = manifest.AMIMetadata.VolumeSizeGiB
-	manifest.VolumeMetadata.State = "available"
-	manifest.VolumeMetadata.AvailabilityZone = "" // TODO: Confirm
-	manifest.VolumeMetadata.CreatedAt = time.Now()
-	manifest.VolumeMetadata.VolumeType = "gp3"
-	manifest.VolumeMetadata.IOPS = 1000
-
 	// Write the manifest to disk
 	// Save as JSON
-	jsonData, err := json.Marshal(manifest)
+	jsonData, err := json.Marshal(ami)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Could not marshal manifest: %v\n", err)
 		os.Exit(1)
 	}
 
-	manifestFilename := fmt.Sprintf("%s/%s.json", imagePath, manifest.AMIMetadata.Name)
+	manifestFilename := fmt.Sprintf("%s/%s.json", imagePath, ami.Name)
 	// Write to file
 	err = os.WriteFile(manifestFilename, jsonData, 0600)
 	if err != nil {
@@ -665,145 +678,118 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	// Upload the image to S3 (predastore)
-
-	appConfig, err := config.LoadConfig(cfgFile)
-
-	if err != nil {
-		fmt.Println("Error loading config file:", err)
-		return
-	}
-
-	s3Config := s3.S3Config{
-		VolumeName: volumeId,
-		VolumeSize: utils.SafeInt64ToUint64(imageStat.Size()),
-		Bucket:     appConfig.Nodes[appConfig.Node].Predastore.Bucket,
-		Region:     appConfig.Nodes[appConfig.Node].Predastore.Region,
-		AccessKey:  appConfig.Nodes[appConfig.Node].Predastore.AccessKey,
-		SecretKey:  appConfig.Nodes[appConfig.Node].Predastore.SecretKey,
-		Host:       appConfig.Nodes[appConfig.Node].Predastore.Host,
-	}
-
-	mkey, err := utils.LoadViperblockMasterKey(appConfig.Nodes[appConfig.Node].Viperblock.EncryptionKeyFile)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not load viperblock encryption key: %v\n", err)
-		os.Exit(1)
-	}
-
-	// Import is an interactive command whose output is a progress bar, so the
-	// volume's routine mount chatter is noise here: a fresh import has no prior
-	// state, making the "no state found" / 404 lines expected rather than
-	// notable. Scoping the logger to this VB (New copies it onto the backend
-	// too) keeps it off the process-wide default. Errors still surface, both
-	// through this logger and as the returned error the caller prints.
-	vbConfig := viperblock.VB{
-		VolumeName: volumeId,
-		VolumeSize: utils.SafeInt64ToUint64(imageStat.Size()),
-		BaseDir:    tmpDir,
-		Cache: viperblock.Cache{
-			Config: viperblock.CacheConfig{
-				Size: 0,
-			},
-		},
-		VolumeConfig:      manifest,
-		MasterKey:         mkey,
-		EncryptionEnabled: mkey != nil,
-		Logger:            slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})),
-	}
-
-	// Bake the deployment CA into the image trust store so a stock cloud image
-	// trusts the gateway from first boot. Best-effort: never fails the import.
-	// The CA path derives from the --spinifex-dir data root (its config/ symlink
-	// holds ca.pem), not NodeBaseDir(): node.BaseDir is unset in the import
-	// command, which would collapse the path to a relative config/ca.pem.
-	bakeCACertIntoImage(extractedImagePath, filepath.Join(baseDir, "config", "ca.pem"))
-
-	// Render the flush bar here rather than inside viperblock, which stays a
-	// pure storage library. The bar is built lazily on the first update so it is
-	// never drawn if the import fails before any bytes flush; viperblock
-	// throttles the callback to ≤101 invocations, so the human-readable title
-	// renders without the per-block render-frequency regression.
-	var flushBar *pterm.ProgressbarPrinter
-	var flushUpdate func(current uint64)
-	progress := func(current, total uint64) {
-		if flushBar == nil {
-			flushBar, flushUpdate = utils.NewByteProgressBar("Flushing image to storage", total)
-		}
-		flushUpdate(current)
-	}
-
-	err = v_utils.ImportDiskImage(&s3Config, &vbConfig, extractedImagePath, progress)
-
-	if flushBar != nil {
-		_, _ = flushBar.Stop()
-	}
-
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not import image to predastore: %v\n", err)
-		os.Exit(1)
-	}
-
 	defer os.RemoveAll(tmpDir)
 
+	appConfig, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not connect to the cluster: %v\n", err)
+		os.Exit(1)
+	}
+	defer nc.Close()
+	node := appConfig.Nodes[appConfig.Node]
+
+	fmt.Println("Writing image to storage ...")
+	provider := ebsprovider.NewNATSProvider(nc, imageImportTimeout)
+	err = admin.ImportImage(context.Background(), provider, admin.ImportOpts{
+		VolumeID:         volumeId,
+		NodeID:           appConfig.Node,
+		SizeBytes:        utils.SafeUint64ToInt64(ami.VolumeSizeGiB * bytesPerGiB),
+		AvailabilityZone: node.AZ,
+		SourcePath:       extractedImagePath,
+		Snapshot:         true,
+		Progress:         os.Stdout,
+	})
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not import image: %v\n", err)
+		os.Exit(1)
+	}
+
+	// admin.ImportImage only wrote the provider's half of the snapshot (its
+	// blocks); this writes the EC2 control plane's, without which
+	// DescribeSnapshots and GetAMISourceVolumeID cannot resolve the import.
+	metaStore := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
+	if err := registerImportedAMISnapshot(metaStore, node.Predastore.Bucket, ami, node.AZ, node.Viperblock.EncryptionKeyFile != ""); err != nil {
+		fmt.Fprintf(os.Stderr, "Imported %s but could not register its snapshot: %v\n", volumeId, err)
+		os.Exit(1)
+	}
+
+	// The document is what DescribeImages enumerates, so it is written last:
+	// until it exists the AMI is not launchable, and an import that failed
+	// partway leaves no half-registered image behind.
+	ami.State = "available"
+	if err := ebsmetadata.NewStore(metaStore, node.Predastore.Bucket).PutAMI(context.Background(), ami); err != nil {
+		fmt.Fprintf(os.Stderr, "Imported %s but could not register it: %v\n", volumeId, err)
+		os.Exit(1)
+	}
+
 	fmt.Printf("✅ Image import complete. Image-ID (AMI): %s\n", volumeId)
 }
 
-// caBakeRunCommand installs the uploaded CA into the guest trust store across
-// the distro families, then removes the staged copy. Each branch is gated on its
-// updater existing so only the matching distro's branch writes anything (install
-// -D creates the anchor dir on minimal images). Debian/Ubuntu use
-// update-ca-certificates, RHEL/Rocky use update-ca-trust, and stock Alpine ships
-// only ca-certificates-bundle (no updater, no anchor dir) so it falls back to
-// appending the PEM to the static bundle. The install chain's exit code is
-// preserved past the cleanup rm so a failure surfaces instead of a false success.
-// The RHEL branch restorecon's only the trust-store paths it touched so the cert
-// is correctly labelled without the image-wide --no-selinux-relabel (below)
-// leaving an unlabelled bundle; restorecon is best-effort and never fails the bake.
-const caBakeRunCommand = `( { command -v update-ca-certificates >/dev/null && install -D -m644 /tmp/spinifex-ca.pem /usr/local/share/ca-certificates/spinifex.crt && update-ca-certificates; } || ` +
-	`{ command -v update-ca-trust >/dev/null && install -D -m644 /tmp/spinifex-ca.pem /etc/pki/ca-trust/source/anchors/spinifex.crt && update-ca-trust && ` +
-	`{ command -v restorecon >/dev/null 2>&1 && restorecon -RF /etc/pki/ca-trust/source/anchors/spinifex.crt /etc/pki/ca-trust/extracted >/dev/null 2>&1; true; }; } || ` +
-	`{ cat /tmp/spinifex-ca.pem >> /etc/ssl/certs/ca-certificates.crt; } ); ` +
-	`rc=$?; rm -f /tmp/spinifex-ca.pem; exit $rc`
-
-// caBakeTimeout bounds the virt-customize run so a stalled libguestfs appliance
-// cannot hang the import indefinitely; on timeout the bake degrades to a skip.
-const caBakeTimeout = 5 * time.Minute
-
-// caBakeCmd builds the virt-customize invocation that uploads the deployment CA
-// into the disk image at imagePath and installs it into the guest trust store.
-// --no-selinux-relabel stops virt-customize flagging the image for a first-boot
-// SELinux autorelabel: that relabel+reboot corrupts XFS roots (RHEL/Rocky) on the
-// reboot, so the run-command relabels only the touched trust paths instead.
-func caBakeCmd(ctx context.Context, imagePath, caCertPath string) *exec.Cmd {
-	return exec.CommandContext(ctx, "virt-customize", "-a", imagePath,
-		"--no-selinux-relabel",
-		"--upload", caCertPath+":/tmp/spinifex-ca.pem",
-		"--run-command", caBakeRunCommand)
+// registerImportedAMISnapshot writes the EC2 control plane's snapshot document
+// for a catalog-imported AMI. The provider-backed import path only writes the
+// storage backend's blocks under ami.SnapshotID, so without this a launch or
+// DescribeSnapshots call cannot resolve it. The volume ID matches ami.ImageID:
+// admin.ImportImage creates the volume under the AMI's own ID.
+func registerImportedAMISnapshot(store objectstore.ObjectStore, bucket string, ami ebsmetadata.AMI, az string, encrypted bool) error {
+	cfg := &handlers_ec2_snapshot.SnapshotConfig{
+		SnapshotID:       ami.SnapshotID,
+		VolumeID:         ami.ImageID,
+		VolumeSize:       utils.SafeUint64ToInt64(ami.VolumeSizeGiB),
+		State:            "completed",
+		Progress:         "100%",
+		StartTime:        time.Now(),
+		Description:      fmt.Sprintf("Imported AMI volume for %s", ami.Name),
+		Encrypted:        encrypted,
+		OwnerID:          utils.GlobalAccountID,
+		AvailabilityZone: az,
+	}
+	if err := handlers_ec2_snapshot.WriteSnapshotConfig(store, bucket, ami.SnapshotID, cfg); err != nil {
+		return fmt.Errorf("register snapshot metadata: %w", err)
+	}
+	return nil
 }
 
-// caBakeRunner resolves virt-customize and runs the CA bake; overridable in tests.
-var caBakeRunner = func(imagePath, caCertPath string) ([]byte, error) {
-	if _, err := exec.LookPath("virt-customize"); err != nil {
-		return nil, fmt.Errorf("virt-customize not found: %w", err)
+func runVolumesOrphansCmd(_ *cobra.Command, _ []string) {
+	appConfig, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not connect to the cluster: %v\n", err)
+		os.Exit(1)
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), caBakeTimeout)
-	defer cancel()
-	return caBakeCmd(ctx, imagePath, caCertPath).CombinedOutput()
-}
+	defer nc.Close()
 
-// bakeCACertIntoImage uploads the deployment CA into the image's trust store via
-// virt-customize so an imported stock image trusts the gateway from first boot.
-// Best-effort: a missing CA, an absent virt-customize, or an image libguestfs
-// cannot inspect logs and continues — image import never fails on the CA.
-func bakeCACertIntoImage(imagePath, caCertPath string) {
-	if _, err := os.Stat(caCertPath); err != nil {
-		slog.Warn("CA bake skipped: deployment CA not found; imported image will not auto-trust the gateway", "ca", caCertPath, "err", err)
+	node := appConfig.Nodes[appConfig.Node]
+	store := objectstore.NewS3ObjectStoreFromConfig(
+		node.Predastore.Host,
+		node.Predastore.Region,
+		node.Predastore.AccessKey,
+		node.Predastore.SecretKey,
+	)
+
+	orphans, err := admin.FindOrphanVolumes(
+		context.Background(),
+		ebsprovider.NewNATSProvider(nc, imageImportTimeout),
+		ebsmetadata.NewStore(store, node.Predastore.Bucket),
+	)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Orphan scan failed:", err)
+		os.Exit(1)
+	}
+	if len(orphans) == 0 {
+		fmt.Println("No orphaned volumes: every volume the provider holds has a control-plane record.")
 		return
 	}
-	if out, err := caBakeRunner(imagePath, caCertPath); err != nil {
-		slog.Warn("CA bake skipped: virt-customize could not customize image; imported image will not auto-trust the gateway", "err", err, "output", string(out))
-		return
+
+	fmt.Printf("%d orphaned volume(s) — held by the provider, unknown to the control plane:\n\n", len(orphans))
+	for _, orphan := range orphans {
+		fmt.Printf("  %s\n", orphan.VolumeID)
+		fmt.Printf("    handle:  %s\n", orphan.Handle)
+		if orphan.Derived {
+			fmt.Println("    note:    derived volume; its base volume is also unknown")
+		}
 	}
+	fmt.Println()
+	fmt.Println("Nothing was deleted. Each of these holds data that cannot be reached")
+	fmt.Println("through the EC2 API; removing one is a per-volume operator decision.")
 }
 
 func runimagesRemoveCmd(cmd *cobra.Command, args []string) {
@@ -1342,18 +1328,32 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	if masterKeyExisted {
 		// Preserve path: reuse the existing identity. The system credentials must
 		// match what seeded the NATS KV `system` secret, so load them rather than
-		// mint new ones. The admin credentials are not recovered: bootstrap.json is
-		// consumed and deleted by awsgw after first boot, and the operator's copy
-		// already lives in ~/.aws/credentials. Leaving them empty makes
-		// finalizeNodeSetup refresh only ~/.aws/config (endpoint/CA for a changed
-		// bind IP), not the credentials.
+		// mint new ones.
 		accessKey, secretKey, err = loadSystemCredentials(configDir)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error loading preserved system credentials: %v\n", err)
 			os.Exit(1)
 		}
-		fmt.Println("\n🔐 Preserved existing identity (master key, credentials and CA unchanged)")
+
+		// The admin pair is minted afresh rather than preserved, because it cannot
+		// be read back: bootstrap.json is consumed and deleted by awsgw on first
+		// start. The half of the identity that survives a re-init is on local disk,
+		// but the IAM store lives in NATS and predastore — so forming a cluster out
+		// of already-initialized nodes rebuilds it and the old access key stops
+		// existing. Carrying an empty pair through instead is worse than useless:
+		// it reaches every joiner's bootstrap.json, where seeding rejects the empty
+		// key ID and awsgw fails to start on every restart, forever.
+		bootstrapResult, err := writeBootstrapFiles(configDir, bootstrapDir, masterKey, accessKey, secretKey, accountID)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error writing bootstrap files: %v\n", err)
+			os.Exit(1)
+		}
+		adminAccessKey = bootstrapResult.AdminAccessKey
+		adminSecretKey = bootstrapResult.AdminSecretKey
+
+		fmt.Println("\n🔐 Preserved existing identity (master key and CA unchanged)")
 		fmt.Printf("   Master key: %s\n", filepath.Join(configDir, "master.key"))
+		fmt.Printf("   Admin credentials reissued in %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
 	} else {
 		// Fresh install: mint system + admin credentials and seed the bootstrap files.
 		accessKey, err = admin.GenerateAWSAccessKey()
@@ -1808,6 +1808,10 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		os.Exit(1)
 	}
 
+	// The host firewall scopes this port to known peers, and a node joining is
+	// not one yet. Closed again on every exit below.
+	closeFormationPort := openFormationPort(port)
+
 	fmt.Printf("\n📡 Formation server started on %s\n", formationAddr)
 	fmt.Printf("   Waiting for %d more node(s) to join...\n", expectedNodes-1)
 	fmt.Printf("   Token expires in %s\n\n", tokenTTL)
@@ -1818,6 +1822,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	if err := fs.WaitForCompletion(formationTimeout); err != nil {
 		fmt.Fprintf(os.Stderr, "❌ %v\n", err)
 		fs.Shutdown(context.Background())
+		closeFormationPort()
 		os.Remove(tokenPath)
 		os.Exit(1)
 	}
@@ -1936,6 +1941,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 
 	// Shutdown formation server
 	fs.Shutdown(context.Background())
+	closeFormationPort()
 	if err := os.Remove(tokenPath); err != nil && !errors.Is(err, os.ErrNotExist) {
 		slog.Warn("Failed to remove join token file", "path", tokenPath, "error", err)
 	}
@@ -2589,13 +2595,24 @@ func initIAMServiceFromConfig() (*handlers_iam.IAMServiceImpl, *config.ClusterCo
 	return svc, cfg, nc, func() { nc.Close() }, nil
 }
 
-// adminAccessPolicyDocument is the AdministratorAccess policy document that
-// grants full access to all actions and resources.
-const adminAccessPolicyDocument = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"*","Resource":"*"}]}`
+// openAccountNameIndex opens the account-name reservation index over an
+// existing NATS connection.
+func openAccountNameIndex(ctx context.Context, nc *nats.Conn) (*handlers_iam.AccountNameIndex, error) {
+	js, err := jetstream.New(nc)
+	if err != nil {
+		return nil, fmt.Errorf("get JetStream context: %w", err)
+	}
+	return handlers_iam.NewAccountNameIndex(ctx, js)
+}
 
 func runAccountCreate(cmd *cobra.Command, args []string) {
 	name, _ := cmd.Flags().GetString("name")
 
+	if remote, _ := cmd.Flags().GetBool("remote"); remote {
+		runAccountCreateRemote(cmd, name)
+		return
+	}
+
 	svc, cfg, nc, cleanup, err := initIAMServiceFromConfig()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
@@ -2603,60 +2620,73 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 	}
 	defer cleanup()
 
-	// 1. Create the account
-	account, err := svc.CreateAccount(name)
+	ctx := context.Background()
+
+	// Reject a duplicate before allocating an account ID. CreateAccount does not
+	// check names, so without this two runs of the same command silently produce
+	// two accounts that differ only by ID.
+	existing, err := handlers_iam.FindAccountByName(svc, name)
 	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error checking existing accounts: %v\n", err)
+		os.Exit(1)
+	}
+	if existing != nil {
+		fmt.Fprintf(os.Stderr, "Error: account %q already exists (account ID %s)\n",
+			existing.AccountName, existing.AccountID)
+		os.Exit(1)
+	}
+
+	// Reserve the name so a concurrent create — including one through
+	// /admin/CreateAccount — cannot take it between the check above and the
+	// create below. The CLI has no client token, so it owns the reservation by
+	// a per-invocation identifier.
+	names, err := openAccountNameIndex(ctx, nc)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	reservationOwner := "spx-cli-" + uuid.NewV4().String()
+	switch err := names.Reserve(ctx, name, reservationOwner); {
+	case errors.Is(err, handlers_iam.ErrNameTaken):
+		fmt.Fprintf(os.Stderr, "Error: account %q already exists\n", name)
+		os.Exit(1)
+	case errors.Is(err, handlers_iam.ErrNameInFlight):
+		fmt.Fprintf(os.Stderr, "Error: another account creation for %q is in progress\n", name)
+		os.Exit(1)
+	case err != nil:
+		fmt.Fprintf(os.Stderr, "Error reserving account name: %v\n", err)
+		os.Exit(1)
+	}
+
+	account, err := svc.CreateAccount(handlers_iam.NormalizeAccountName(name))
+	if err != nil {
+		if relErr := names.Release(ctx, name, reservationOwner); relErr != nil {
+			fmt.Fprintf(os.Stderr, "Warning: could not release name reservation: %v\n", relErr)
+		}
 		fmt.Fprintf(os.Stderr, "Error creating account: %v\n", err)
 		os.Exit(1)
 	}
 	accountID := account.AccountID
 
+	if err := names.Commit(ctx, name, accountID, reservationOwner); err != nil {
+		fmt.Fprintf(os.Stderr, "Warning: could not index account name: %v\n", err)
+	}
+
 	// Create default VPC for the new account (belt-and-suspenders: daemon also
 	// does this via iam.account.created event, but daemon may not be running).
 	nodeConfig := cfg.Nodes[cfg.Node]
-	vpcSvc, vpcErr := handlers_ec2_vpc.NewVPCServiceImplWithNATS(context.Background(), &nodeConfig, nc)
+	vpcSvc, vpcErr := handlers_ec2_vpc.NewVPCServiceImplWithNATS(ctx, &nodeConfig, nc)
 	if vpcErr != nil {
 		fmt.Fprintf(os.Stderr, "Warning: could not create default VPC service: %v\n", vpcErr)
 	} else if _, vpcErr = vpcSvc.EnsureDefaultVPC(accountID); vpcErr != nil {
 		fmt.Fprintf(os.Stderr, "Warning: could not create default VPC: %v\n", vpcErr)
 	}
 
-	// 2. Create admin user
-	_, err = svc.CreateUser(accountID, &iam.CreateUserInput{
-		UserName: aws.String("admin"),
-	})
+	// Shared with /admin/CreateAccount so the CLI and the remote endpoint cannot
+	// drift in what an account is made of.
+	provisioned, err := handlers_iam.ProvisionAccount(svc, accountID, account.AccountName)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error creating admin user: %v\n", err)
-		os.Exit(1)
-	}
-
-	// 3. Create access key for admin user
-	akOut, err := svc.CreateAccessKey(accountID, &iam.CreateAccessKeyInput{
-		UserName: aws.String("admin"),
-	})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error creating access key: %v\n", err)
-		os.Exit(1)
-	}
-
-	// 4. Create AdministratorAccess policy scoped to this account
-	policyARN := fmt.Sprintf("arn:aws:iam::%s:policy/AdministratorAccess", accountID)
-	_, err = svc.CreatePolicy(accountID, &iam.CreatePolicyInput{
-		PolicyName:     aws.String("AdministratorAccess"),
-		PolicyDocument: aws.String(adminAccessPolicyDocument),
-	})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error creating admin policy: %v\n", err)
-		os.Exit(1)
-	}
-
-	// 5. Attach policy to admin user
-	_, err = svc.AttachUserPolicy(accountID, &iam.AttachUserPolicyInput{
-		UserName:  aws.String("admin"),
-		PolicyArn: aws.String(policyARN),
-	})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error attaching policy: %v\n", err)
+		fmt.Fprintf(os.Stderr, "Error provisioning account: %v\n", err)
 		os.Exit(1)
 	}
 
@@ -2678,8 +2708,8 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 	configPath := filepath.Join(homeDir, ".aws", "config")
 
 	if err := admin.UpdateAWSINIFile(credPath, profileName, map[string]string{
-		"aws_access_key_id":     *akOut.AccessKey.AccessKeyId,
-		"aws_secret_access_key": *akOut.AccessKey.SecretAccessKey,
+		"aws_access_key_id":     provisioned.AccessKeyID,
+		"aws_secret_access_key": provisioned.SecretAccessKey,
 	}); err != nil {
 		fmt.Fprintf(os.Stderr, "Warning: could not update AWS credentials: %v\n", err)
 	}
@@ -2699,16 +2729,24 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 	// Print credentials
 	fmt.Println("\nAccount created successfully!")
 	fmt.Printf("  Account ID:        %s\n", accountID)
-	fmt.Printf("  Account Name:      %s\n", name)
-	fmt.Printf("  Admin User:        admin\n")
-	fmt.Printf("  Access Key ID:     %s\n", *akOut.AccessKey.AccessKeyId)
-	fmt.Printf("  Secret Access Key: %s\n", *akOut.AccessKey.SecretAccessKey)
+	fmt.Printf("  Account Name:      %s\n", provisioned.AccountName)
+	fmt.Printf("  Admin User:        %s\n", provisioned.AdminUser)
+	fmt.Printf("  Access Key ID:     %s\n", provisioned.AccessKeyID)
+	fmt.Printf("  Secret Access Key: %s\n", provisioned.SecretAccessKey)
 	fmt.Printf("  AWS Profile:       %s\n", profileName)
 	fmt.Println("\nUse with:")
 	fmt.Printf("  AWS_PROFILE=%s aws ec2 describe-instances\n", profileName)
 }
 
 func runAccountList(cmd *cobra.Command, args []string) {
+	if remote, _ := cmd.Flags().GetBool("remote"); remote {
+		if err := runAccountListRemote(cmd); err != nil {
+			fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+			os.Exit(1)
+		}
+		return
+	}
+
 	svc, _, _, cleanup, err := initIAMServiceFromConfig()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
@@ -2722,20 +2760,7 @@ func runAccountList(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	if len(accounts) == 0 {
-		fmt.Println("No accounts found.")
-		return
-	}
-
-	fmt.Printf("%-14s %-20s %-10s %s\n", "ACCOUNT ID", "NAME", "STATUS", "CREATED")
-	fmt.Printf("%-14s %-20s %-10s %s\n", "----------", "----", "------", "-------")
-	for _, a := range accounts {
-		created := a.CreatedAt
-		if t, err := time.Parse(time.RFC3339, a.CreatedAt); err == nil {
-			created = t.Format("2006-01-02 15:04")
-		}
-		fmt.Printf("%-14s %-20s %-10s %s\n", a.AccountID, a.AccountName, a.Status, created)
-	}
+	printAccountTable(accountSummaries(accounts))
 }
 
 func runCertRenew(cmd *cobra.Command, _ []string) {

cmd/spinifex/cmd/admin_account_delete.go

@@ -0,0 +1,200 @@
+package cmd
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/accountteardown"
+	"github.com/mulgadc/spinifex/spinifex/admin"
+	"github.com/mulgadc/spinifex/spinifex/objectstore"
+	"github.com/spf13/cobra"
+)
+
+// accountDeleteTimeout is the outer bound on the whole teardown. It must stay
+// well above the sum of the per-stage drain budgets, which are what actually
+// decide when to give up on a resource; cutting in before them would abandon a
+// large account midway with no stuck report to show for it.
+const accountDeleteTimeout = 2 * time.Hour
+
+var accountDeleteCmd = &cobra.Command{
+	Use:   "delete <account-id>",
+	Short: "Tear down an account's infrastructure and remove the account",
+	Long: `Remove an account and everything it owns.
+
+The account is marked TERMINATING first, which blocks its credentials, and then
+its resources are deleted in dependency order: compute, attachments, storage,
+network, platform, identity, account. Nothing is removed until the account can
+no longer create more.
+
+The account name must be given to confirm, so a mistyped account id fails
+rather than emptying a live tenant. The system account (000000000000) and the
+super admin (000000000001) can never be deleted.
+
+  spx admin account delete 000000000042 --dry-run
+  spx admin account delete 000000000042 --name tenant@example.com
+
+--force is for resources the ordinary API refuses to delete: a volume attached
+to an instance that will not terminate leaves both stranded and the account can
+never be emptied. It clears the attachment in the control plane without the
+guest's cooperation, so use it only once the guest is being destroyed.`,
+	Args: cobra.ExactArgs(1),
+	RunE: runAccountDelete,
+
+	SilenceUsage: true,
+}
+
+func init() {
+	accountCmd.AddCommand(accountDeleteCmd)
+
+	accountDeleteCmd.Flags().Bool("dry-run", false, "Report what would be deleted and change nothing")
+	accountDeleteCmd.Flags().Bool("force", false, "Escalate past state guards for resources that will not delete")
+	accountDeleteCmd.Flags().Bool("yes", false, "Skip the interactive confirmation (requires --name)")
+	accountDeleteCmd.Flags().String("name", "", "Account name, which must match the stored record")
+	accountDeleteCmd.Flags().Bool("remote", false, "Delete over POST /admin/DeleteAccount instead of connecting to NATS")
+	accountDeleteCmd.Flags().String("endpoint", "", "Gateway endpoint for --remote (default: this node's AWS gateway)")
+	accountDeleteCmd.Flags().String("region", "", "SigV4 region for --remote (default: this node's region)")
+	accountDeleteCmd.Flags().String("ca-bundle", "", "CA certificate for --remote (default: this node's CA)")
+	accountDeleteCmd.Flags().String("client-token", "", "Idempotency token for --remote (default: generated; reuse it to follow the same teardown)")
+}
+
+func runAccountDelete(cmd *cobra.Command, args []string) error {
+	accountID := args[0]
+	if remote, _ := cmd.Flags().GetBool("remote"); remote {
+		return runAccountDeleteRemote(cmd, accountID)
+	}
+
+	dryRun, _ := cmd.Flags().GetBool("dry-run")
+	force, _ := cmd.Flags().GetBool("force")
+	assumeYes, _ := cmd.Flags().GetBool("yes")
+	name, _ := cmd.Flags().GetString("name")
+
+	svc, cfg, nc, cleanup, err := initIAMServiceFromConfig()
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), accountDeleteTimeout)
+	defer cancel()
+
+	node := cfg.Nodes[cfg.Node]
+	buckets := objectstore.NewS3ObjectStoreFromConfig(
+		admin.DialTarget(node.Predastore.Host),
+		node.Predastore.Region,
+		node.Predastore.AccessKey,
+		node.Predastore.SecretKey,
+	)
+
+	engine, err := accountteardown.NewClusterEngine(ctx, nc, len(cfg.Nodes), svc, buckets)
+	if err != nil {
+		return err
+	}
+
+	if dryRun {
+		result, err := engine.Inventory(ctx, accountID)
+		if err != nil {
+			return err
+		}
+		printTeardownPlan(result)
+		fmt.Printf("\nDry run: nothing was deleted. %d resources would be removed.\n", result.DeletedCount())
+		return nil
+	}
+
+	// Always show the inventory before destroying it. An operator who has not
+	// seen what is about to go has not really confirmed anything.
+	plan, err := engine.Inventory(ctx, accountID)
+	if err != nil {
+		return err
+	}
+	printTeardownPlan(plan)
+
+	if name == "" {
+		if assumeYes {
+			return errors.New("--yes requires --name so the account is still confirmed")
+		}
+		name, err = promptAccountName(accountID)
+		if err != nil {
+			return err
+		}
+	}
+
+	result, err := engine.Teardown(ctx, accountteardown.Request{
+		AccountID: accountID, AccountName: name, Force: force,
+	})
+	if result != nil {
+		printTeardownResult(result)
+	}
+	if errors.Is(err, accountteardown.ErrResourcesStuck) {
+		fmt.Fprintf(os.Stderr, "\nThe account is still TERMINATING so its leftovers keep an owner.\n")
+		if !force {
+			fmt.Fprintf(os.Stderr, "Re-run with --force to clear resources the ordinary API refuses to delete.\n")
+		}
+		return err
+	}
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("\nAccount %s deleted. %d resources removed.\n", accountID, result.DeletedCount())
+	return nil
+}
+
+// promptAccountName asks for the confirmation name on a terminal.
+func promptAccountName(accountID string) (string, error) {
+	fmt.Printf("\nThis permanently deletes account %s and everything above.\n", accountID)
+	fmt.Print("Type the account name to confirm: ")
+
+	reader := bufio.NewReader(os.Stdin)
+	entered, err := reader.ReadString('\n')
+	if err != nil {
+		return "", fmt.Errorf("read confirmation: %w", err)
+	}
+	entered = strings.TrimSpace(entered)
+	if entered == "" {
+		return "", errors.New("cancelled: no account name given")
+	}
+	return entered, nil
+}
+
+func printTeardownPlan(result *accountteardown.Result) {
+	fmt.Printf("Account %s (%s)\n\n", result.AccountID, result.AccountName)
+	if result.DeletedCount() == 0 {
+		fmt.Println("The account holds no resources.")
+		return
+	}
+	for _, stage := range result.Stages {
+		if len(stage.Deleted) == 0 {
+			continue
+		}
+		fmt.Printf("  %s\n", stage.Stage)
+		for _, resource := range stage.Deleted {
+			fmt.Printf("    %s\n", resource)
+		}
+	}
+}
+
+func printTeardownResult(result *accountteardown.Result) {
+	fmt.Println()
+	for _, stage := range result.Stages {
+		printTeardownStage(stage)
+	}
+}
+
+// printTeardownStage reports one finished stage. A stage that removed nothing
+// and left nothing behind is not printed: an account with no ECS clusters
+// should not read as seven lines of activity.
+func printTeardownStage(stage accountteardown.StageResult) {
+	if len(stage.Deleted) == 0 && len(stage.Stuck) == 0 {
+		return
+	}
+	fmt.Printf("  %-12s %d deleted, %d stuck (%s)\n",
+		stage.Stage, len(stage.Deleted), len(stage.Stuck), stage.Elapsed)
+	for _, stuck := range stage.Stuck {
+		fmt.Printf("    STUCK %s: %s\n", stuck.Resource, stuck.Reason)
+	}
+}

cmd/spinifex/cmd/admin_account_delete_output_test.go

@@ -0,0 +1,162 @@
+package cmd_test
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+	"github.com/mulgadc/spinifex/spinifex/accountteardown"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// captureOutput redirects os.Stdout for the duration of fn and returns what was
+// written. The teardown printers are the operator's only view of what is about
+// to be destroyed, so what they print is the thing under test.
+func captureOutput(t *testing.T, fn func()) string {
+	t.Helper()
+	r, w, err := os.Pipe()
+	require.NoError(t, err)
+	orig := os.Stdout
+	os.Stdout = w //nolint:reassign // test-local stdout capture, restored below
+
+	fn()
+
+	require.NoError(t, w.Close())
+	os.Stdout = orig //nolint:reassign // restoring the real os.Stdout captured above
+
+	var buf bytes.Buffer
+	_, err = io.Copy(&buf, r)
+	require.NoError(t, err)
+	return buf.String()
+}
+
+// feedStdin swaps os.Stdin for a pipe carrying content, mirroring the stdout
+// capture above.
+func feedStdin(t *testing.T, content string, fn func()) {
+	t.Helper()
+	r, w, err := os.Pipe()
+	require.NoError(t, err)
+	orig := os.Stdin
+	os.Stdin = r                          //nolint:reassign // test-local stdin swap, restored below
+	t.Cleanup(func() { os.Stdin = orig }) //nolint:reassign // restoring the real os.Stdin captured above
+
+	go func() {
+		_, _ = io.WriteString(w, content)
+		_ = w.Close()
+	}()
+
+	fn()
+}
+
+func teardownPlan() *accountteardown.Result {
+	return &accountteardown.Result{
+		AccountID:   "000000000042",
+		AccountName: "tenant@example.com",
+		DryRun:      true,
+		Stages: []accountteardown.StageResult{
+			{
+				Stage:   accountteardown.StageCompute,
+				Deleted: []accountteardown.Resource{{Kind: "instance", ID: "i-1", Detail: "running"}},
+				Elapsed: "0s",
+			},
+			{Stage: accountteardown.StageAttachments, Elapsed: "0s"},
+			{
+				Stage:   accountteardown.StageStorage,
+				Deleted: []accountteardown.Resource{{Kind: "volume", ID: "vol-1"}},
+				Elapsed: "0s",
+			},
+		},
+	}
+}
+
+// The plan is what the operator confirms against, so every resource has to
+// appear under its stage — and an empty stage must not pad the list.
+func TestPrintTeardownPlanListsEveryResource(t *testing.T) {
+	output := captureOutput(t, func() { cmd.PrintTeardownPlan(teardownPlan()) })
+
+	assert.Contains(t, output, "000000000042")
+	assert.Contains(t, output, "tenant@example.com")
+	assert.Contains(t, output, "i-1")
+	assert.Contains(t, output, "vol-1")
+	assert.Contains(t, output, "compute")
+	assert.Contains(t, output, "storage")
+	assert.NotContains(t, output, "attachments", "an empty stage is noise, not information")
+}
+
+// An account with nothing in it must say so. A bare header reads as truncated
+// output, which is exactly when an operator stops and asks.
+func TestPrintTeardownPlanSaysWhenTheAccountIsEmpty(t *testing.T) {
+	output := captureOutput(t, func() {
+		cmd.PrintTeardownPlan(&accountteardown.Result{
+			AccountID:   "000000000042",
+			AccountName: "tenant@example.com",
+			Stages:      []accountteardown.StageResult{{Stage: accountteardown.StageCompute, Elapsed: "0s"}},
+		})
+	})
+
+	assert.Contains(t, output, "holds no resources")
+}
+
+// A stuck resource is the most valuable thing a teardown reports: each one is a
+// delete path that does not work. It must be named, with the reason.
+func TestPrintTeardownResultNamesStuckResources(t *testing.T) {
+	result := &accountteardown.Result{
+		AccountID: "000000000042",
+		Stages: []accountteardown.StageResult{
+			{
+				Stage:   accountteardown.StageCompute,
+				Deleted: []accountteardown.Resource{{Kind: "instance", ID: "i-1"}},
+				Elapsed: "5s",
+			},
+			{Stage: accountteardown.StageAttachments, Elapsed: "0s"},
+			{
+				Stage: accountteardown.StageStorage,
+				Stuck: []accountteardown.Stuck{{
+					Resource: accountteardown.Resource{Kind: "volume", ID: "vol-1"},
+					Reason:   "still attached to i-1",
+				}},
+				Elapsed: "5m0s",
+			},
+		},
+	}
+
+	output := captureOutput(t, func() { cmd.PrintTeardownResult(result) })
+
+	assert.Contains(t, output, "1 deleted, 0 stuck")
+	assert.Contains(t, output, "STUCK")
+	assert.Contains(t, output, "vol-1")
+	assert.Contains(t, output, "still attached to i-1")
+	assert.NotContains(t, output, "attachments")
+}
+
+// The typed name is the confirmation. Surrounding whitespace is a paste
+// artefact rather than a mismatch, but an empty line is a cancellation.
+func TestPromptAccountNameReadsTheConfirmation(t *testing.T) {
+	var name string
+	var err error
+
+	captureOutput(t, func() {
+		feedStdin(t, "  tenant@example.com  \n", func() {
+			name, err = cmd.PromptAccountName("000000000042")
+		})
+	})
+
+	require.NoError(t, err)
+	assert.Equal(t, "tenant@example.com", name)
+}
+
+func TestPromptAccountNameTreatsAnEmptyReplyAsCancelled(t *testing.T) {
+	var err error
+
+	captureOutput(t, func() {
+		feedStdin(t, "\n", func() {
+			_, err = cmd.PromptAccountName("000000000042")
+		})
+	})
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "cancelled")
+}

cmd/spinifex/cmd/admin_account_delete_remote.go

@@ -0,0 +1,142 @@
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/accountteardown"
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	"github.com/spf13/cobra"
+)
+
+// accountDeletePollInterval is how often --remote asks the gateway how far the
+// teardown has got. The work is minutes long, so a tighter poll would only add
+// load without telling the operator anything new.
+var accountDeletePollInterval = 10 * time.Second
+
+// runAccountDeleteRemote drives POST /admin/DeleteAccount and then follows the
+// job to completion, so the remote path reports the same outcome the local one
+// prints rather than only that the teardown started.
+func runAccountDeleteRemote(cmd *cobra.Command, accountID string) error {
+	dryRun, _ := cmd.Flags().GetBool("dry-run")
+	force, _ := cmd.Flags().GetBool("force")
+	assumeYes, _ := cmd.Flags().GetBool("yes")
+	name, _ := cmd.Flags().GetString("name")
+	clientToken, _ := cmd.Flags().GetString("client-token")
+
+	target, err := resolveAdminTarget(cmd)
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), accountDeleteTimeout)
+	defer cancel()
+
+	// The inventory is fetched first even for a real deletion: an operator who
+	// has not seen what is about to go has not really confirmed anything.
+	plan, err := deleteAccountRemote(ctx, target, gateway.DeleteAccountRequest{
+		AccountID: accountID, DryRun: true,
+	})
+	if err != nil {
+		return err
+	}
+	if plan.Inventory != nil {
+		printTeardownPlan(plan.Inventory)
+	}
+	if dryRun {
+		fmt.Printf("\nDry run: nothing was deleted. %d resources would be removed.\n", plan.Inventory.DeletedCount())
+		return nil
+	}
+
+	if name == "" {
+		if assumeYes {
+			return errors.New("--yes requires --name so the account is still confirmed")
+		}
+		if name, err = promptAccountName(accountID); err != nil {
+			return err
+		}
+	}
+	if clientToken == "" {
+		clientToken = newClientToken()
+	}
+
+	started, err := deleteAccountRemote(ctx, target, gateway.DeleteAccountRequest{
+		AccountID: accountID, AccountName: name, ClientToken: clientToken, Force: force,
+	})
+	if err != nil {
+		var adminErr *adminError
+		if errors.As(err, &adminErr) && retryableAdminErrors[adminErr.Code] {
+			fmt.Fprintf(os.Stderr, "Retry with --client-token %s to follow the same teardown.\n", clientToken)
+		}
+		return err
+	}
+	fmt.Printf("\nTeardown started: %s\n", started.DeletionID)
+
+	return followAccountDeletion(ctx, target, accountID)
+}
+
+// followAccountDeletion polls until the job reaches a terminal state, printing
+// each stage as it lands.
+func followAccountDeletion(ctx context.Context, target adminTarget, accountID string) error {
+	reported := 0
+	for {
+		job, err := describeAccountDeletionRemote(ctx, target, accountID)
+		if err != nil {
+			return err
+		}
+
+		for ; reported < len(job.Stages); reported++ {
+			printTeardownStage(job.Stages[reported])
+		}
+
+		switch job.State {
+		case gateway.DeletionStateCompleted:
+			fmt.Printf("\nAccount %s deleted.\n", accountID)
+			return nil
+		case gateway.DeletionStateFailed:
+			fmt.Fprintf(os.Stderr, "\nThe account is still TERMINATING so its leftovers keep an owner.\n")
+			return fmt.Errorf("teardown failed: %s", job.Error)
+		}
+
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("stopped following teardown of %s: %w", accountID, ctx.Err())
+		case <-time.After(accountDeletePollInterval):
+		}
+	}
+}
+
+func deleteAccountRemote(
+	ctx context.Context,
+	target adminTarget,
+	req gateway.DeleteAccountRequest,
+) (*gateway.DeleteAccountResponse, error) {
+	var out gateway.DeleteAccountResponse
+	if err := callAdmin(ctx, target, "DeleteAccount", req, &out); err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func describeAccountDeletionRemote(ctx context.Context, target adminTarget, accountID string) (*accountDeletionStatus, error) {
+	var out accountDeletionStatus
+	err := callAdmin(ctx, target, "DescribeAccountDeletion",
+		gateway.DescribeAccountDeletionRequest{AccountID: accountID}, &out)
+	if err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+// accountDeletionStatus is the subset of the stored job this command reports.
+// The gateway's own record type is unexported, and the CLI needs no more than
+// the state, the stages and why it stopped.
+type accountDeletionStatus struct {
+	DeletionID string                        `json:"deletionId"`
+	State      string                        `json:"state"`
+	Stages     []accountteardown.StageResult `json:"stages"`
+	Error      string                        `json:"error"`
+}

cmd/spinifex/cmd/admin_account_delete_remote_test.go

@@ -0,0 +1,199 @@
+package cmd_test
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// adminTestCredentials puts a static key in the environment so the SigV4 signer
+// resolves without reaching for IMDS.
+func adminTestCredentials(t *testing.T) {
+	t.Helper()
+	t.Setenv("AWS_ACCESS_KEY_ID", "AKIAEXAMPLE")
+	t.Setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+}
+
+// newScriptedAdminServer answers each successive request with the next body,
+// repeating the last one, which is how a job that changes state between polls
+// is reproduced.
+func newScriptedAdminServer(t *testing.T, bodies ...string) (*httptest.Server, *atomic.Int32) {
+	t.Helper()
+	var calls atomic.Int32
+
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		index := int(calls.Add(1)) - 1
+		if index >= len(bodies) {
+			index = len(bodies) - 1
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(bodies[index]))
+	}))
+	t.Cleanup(srv.Close)
+	return srv, &calls
+}
+
+// The delete request must reach /admin/DeleteAccount signed for the spinifex
+// service: a key signed for another service is treated as probing and denied.
+func TestDeleteAccountRemoteSignsAndPosts(t *testing.T) {
+	adminTestCredentials(t)
+
+	var got http.Request
+	srv := newAdminTestServer(t, http.StatusOK,
+		`{"deletionId":"d-1","accountId":"000000000042","state":"RUNNING"}`, &got)
+
+	out, err := cmd.DeleteAccountRemote(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)),
+		gateway.DeleteAccountRequest{
+			AccountID:   "000000000042",
+			AccountName: "tenant@example.com",
+			ClientToken: strings.Repeat("a", 32),
+		})
+	require.NoError(t, err)
+
+	assert.Equal(t, "d-1", out.DeletionID)
+	assert.Equal(t, gateway.DeletionStateRunning, out.State)
+
+	assert.Equal(t, http.MethodPost, got.Method)
+	assert.Equal(t, "/admin/DeleteAccount", got.URL.Path)
+	assert.Contains(t, got.Header.Get("Authorization"), "/us-west-1/spinifex/aws4_request")
+}
+
+// A dry run answers inline with the inventory, which is what the operator
+// confirms against before the real call.
+func TestDeleteAccountRemoteDryRunCarriesTheInventory(t *testing.T) {
+	adminTestCredentials(t)
+
+	var got http.Request
+	srv := newAdminTestServer(t, http.StatusOK,
+		`{"accountId":"000000000042","state":"DRY_RUN","dryRun":true,"inventory":{"accountId":"000000000042",`+
+			`"dryRun":true,"stages":[{"stage":1,"deleted":[{"kind":"instance","id":"i-1"}],"elapsed":"0s"}]}}`, &got)
+
+	out, err := cmd.DeleteAccountRemote(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)),
+		gateway.DeleteAccountRequest{AccountID: "000000000042", DryRun: true})
+	require.NoError(t, err)
+
+	assert.True(t, out.DryRun)
+	require.NotNil(t, out.Inventory)
+	assert.Equal(t, 1, out.Inventory.DeletedCount())
+}
+
+// A refusal must surface as the gateway's code, not as a decode failure, so the
+// operator can tell a protected account from an unreachable cluster.
+func TestDeleteAccountRemoteSurfacesGatewayError(t *testing.T) {
+	adminTestCredentials(t)
+
+	var got http.Request
+	srv := newAdminTestServer(t, http.StatusForbidden,
+		`{"error":{"code":"AccessDenied","message":"denied"},"requestId":"req-9"}`, &got)
+
+	_, err := cmd.DeleteAccountRemote(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)),
+		gateway.DeleteAccountRequest{
+			AccountID:   "000000000001",
+			AccountName: "spinifex",
+			ClientToken: strings.Repeat("a", 32),
+		})
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "AccessDenied")
+	assert.Contains(t, err.Error(), "req-9")
+	assert.False(t, cmd.RetryableAdminError(err), "a protected account is not fixed by retrying")
+}
+
+func TestDescribeAccountDeletionRemoteReportsProgress(t *testing.T) {
+	adminTestCredentials(t)
+
+	var got http.Request
+	srv := newAdminTestServer(t, http.StatusOK,
+		`{"deletionId":"d-1","state":"RUNNING","stages":[{"stage":1,"elapsed":"5s"}]}`, &got)
+
+	out, err := cmd.DescribeAccountDeletionRemote(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042")
+	require.NoError(t, err)
+
+	assert.Equal(t, "/admin/DescribeAccountDeletion", got.URL.Path)
+	assert.Equal(t, gateway.DeletionStateRunning, out.State)
+	assert.Len(t, out.Stages, 1)
+}
+
+// The follow loop is what makes --remote report the same outcome the local
+// path prints. It must keep polling while the job runs and stop on the first
+// terminal state.
+func TestFollowAccountDeletionPollsUntilComplete(t *testing.T) {
+	adminTestCredentials(t)
+	defer cmd.SetAccountDeletePollInterval(time.Millisecond)()
+
+	srv, calls := newScriptedAdminServer(t,
+		`{"deletionId":"d-1","state":"RUNNING","stages":[{"stage":1,"elapsed":"5s"}]}`,
+		`{"deletionId":"d-1","state":"RUNNING","stages":[{"stage":1,"elapsed":"5s"},{"stage":2,"elapsed":"1s"}]}`,
+		`{"deletionId":"d-1","state":"COMPLETED","stages":[{"stage":1,"elapsed":"5s"},{"stage":2,"elapsed":"1s"}]}`,
+	)
+
+	err := cmd.FollowAccountDeletion(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042")
+
+	require.NoError(t, err)
+	assert.Equal(t, int32(3), calls.Load(), "the loop must keep polling while the job runs")
+}
+
+// A failed teardown must exit non-zero with the stored reason: the account is
+// left TERMINATING, and an operator who reads "done" would not go looking.
+func TestFollowAccountDeletionReportsFailure(t *testing.T) {
+	adminTestCredentials(t)
+	defer cmd.SetAccountDeletePollInterval(time.Millisecond)()
+
+	srv, _ := newScriptedAdminServer(t,
+		`{"deletionId":"d-1","state":"FAILED","stages":[],"error":"2 resources left"}`)
+
+	err := cmd.FollowAccountDeletion(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042")
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "2 resources left")
+}
+
+// Giving up on following is not the same as the teardown stopping, so the
+// message has to name the account the caller must go back to.
+func TestFollowAccountDeletionStopsWhenTheContextEnds(t *testing.T) {
+	adminTestCredentials(t)
+	defer cmd.SetAccountDeletePollInterval(50 * time.Millisecond)()
+
+	srv, _ := newScriptedAdminServer(t, `{"deletionId":"d-1","state":"RUNNING","stages":[]}`)
+
+	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+	defer cancel()
+
+	err := cmd.FollowAccountDeletion(ctx,
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042")
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "000000000042")
+}
+
+// An unreadable job is not a finished one. Reporting it as complete would hide
+// a teardown still holding an account's resources.
+func TestFollowAccountDeletionFailsOnAnUnreadableJob(t *testing.T) {
+	adminTestCredentials(t)
+	defer cmd.SetAccountDeletePollInterval(time.Millisecond)()
+
+	var got http.Request
+	srv := newAdminTestServer(t, http.StatusNotFound,
+		`{"error":{"code":"NoSuchEntity","message":"no job"},"requestId":"req-2"}`, &got)
+
+	err := cmd.FollowAccountDeletion(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042")
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "NoSuchEntity")
+}

cmd/spinifex/cmd/admin_account_list_remote.go

@@ -0,0 +1,67 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	"github.com/spf13/cobra"
+)
+
+// runAccountListRemote reads the listing over POST /admin/ListAccounts, which
+// is how an off-cluster operator or the load-test harness sees the tenants it
+// created without SSH access to a node.
+func runAccountListRemote(cmd *cobra.Command) error {
+	target, err := resolveAdminTarget(cmd)
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), adminRequestTimeout)
+	defer cancel()
+
+	var out gateway.ListAccountsResponse
+	if err := callAdmin(ctx, target, "ListAccounts", struct{}{}, &out); err != nil {
+		return err
+	}
+
+	printAccountTable(out.Accounts)
+	return nil
+}
+
+// accountSummaries adapts the local listing to the shape the remote one
+// returns, so both paths print through one function and cannot drift.
+func accountSummaries(accounts []*handlers_iam.Account) []gateway.AccountSummary {
+	summaries := make([]gateway.AccountSummary, 0, len(accounts))
+	for _, account := range accounts {
+		if account == nil {
+			continue
+		}
+		summaries = append(summaries, gateway.AccountSummary{
+			AccountID:   account.AccountID,
+			AccountName: account.AccountName,
+			Status:      account.Status,
+			CreatedAt:   account.CreatedAt,
+		})
+	}
+	return summaries
+}
+
+func printAccountTable(accounts []gateway.AccountSummary) {
+	if len(accounts) == 0 {
+		fmt.Println("No accounts found.")
+		return
+	}
+
+	fmt.Printf("%-14s %-30s %-12s %s\n", "ACCOUNT ID", "NAME", "STATUS", "CREATED")
+	fmt.Printf("%-14s %-30s %-12s %s\n", "----------", "----", "------", "-------")
+	for _, account := range accounts {
+		created := account.CreatedAt
+		if parsed, err := time.Parse(time.RFC3339, account.CreatedAt); err == nil {
+			created = parsed.Format("2006-01-02 15:04")
+		}
+		fmt.Printf("%-14s %-30s %-12s %s\n", account.AccountID, account.AccountName, account.Status, created)
+	}
+}

cmd/spinifex/cmd/admin_account_quota.go

@@ -0,0 +1,185 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strconv"
+
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	handlers_quota "github.com/mulgadc/spinifex/spinifex/handlers/quota"
+	"github.com/spf13/cobra"
+)
+
+// quotaDimensions maps each CLI flag to the override field it sets. One table
+// drives the flag registration and the parse, so a new dimension cannot be
+// added to one and forgotten in the other.
+var quotaDimensions = []struct {
+	flag  string
+	usage string
+	field func(*handlers_quota.Overrides) **int
+}{
+	{"vcpus", "Total vCPUs across running and stopped instances", func(o *handlers_quota.Overrides) **int { return &o.VCPUs }},
+	{"vpcs", "VPCs", func(o *handlers_quota.Overrides) **int { return &o.VPCs }},
+	{"subnets", "Subnets, counted per account rather than per VPC", func(o *handlers_quota.Overrides) **int { return &o.Subnets }},
+	{"eips", "Elastic IP addresses", func(o *handlers_quota.Overrides) **int { return &o.EIPs }},
+	{"volumes", "EBS volumes", func(o *handlers_quota.Overrides) **int { return &o.Volumes }},
+	{"volumes-gib", "Total EBS capacity in GiB", func(o *handlers_quota.Overrides) **int { return &o.VolumesGiB }},
+	{"rds-instances", "RDS database instances", func(o *handlers_quota.Overrides) **int { return &o.RDSInstances }},
+	{"load-balancers", "Load balancers, ALB and NLB together", func(o *handlers_quota.Overrides) **int { return &o.LoadBalancers }},
+}
+
+var accountQuotaCmd = &cobra.Command{
+	Use:   "quota",
+	Short: "Inspect and adjust an account's service quotas",
+	Long: `Inspect and adjust the service quotas applied to one account.
+
+Every account inherits the [quota] limits from awsgw.toml. An override raises or
+lowers individual dimensions for one account without affecting any other, so a
+customer can be given more vCPUs while every other tenant stays on the baseline.
+
+Only the dimensions passed as flags are changed; the rest keep inheriting. Use
+-1 for no limit on a dimension, and --clear to drop every override.`,
+}
+
+var accountQuotaGetCmd = &cobra.Command{
+	Use:   "get <account-id>",
+	Short: "Show the limits in force for an account",
+	Long: `Show the limits in force for an account and where each came from:
+"config" for an inherited baseline, "override" for one set on this account.`,
+	Args: cobra.ExactArgs(1),
+	RunE: runAccountQuotaGet,
+}
+
+var accountQuotaSetCmd = &cobra.Command{
+	Use:   "set <account-id>",
+	Short: "Override one or more limits for an account",
+	Long: `Override one or more limits for an account. Dimensions not named keep
+inheriting the configured baseline.
+
+  spx admin account quota set 000000000005 --vcpus 32
+  spx admin account quota set 000000000005 --clear`,
+	Args: cobra.ExactArgs(1),
+	RunE: runAccountQuotaSet,
+}
+
+// Both subcommands are served only by the /admin surface, so unlike the account
+// commands there is no NATS path and no --remote to opt into.
+func init() {
+	accountCmd.AddCommand(accountQuotaCmd)
+	accountQuotaCmd.AddCommand(accountQuotaGetCmd)
+	accountQuotaCmd.AddCommand(accountQuotaSetCmd)
+
+	for _, c := range []*cobra.Command{accountQuotaGetCmd, accountQuotaSetCmd} {
+		c.Flags().String("endpoint", "", "Gateway endpoint (default: this node's AWS gateway)")
+		c.Flags().String("region", "", "SigV4 region (default: this node's region)")
+		c.Flags().String("ca-bundle", "", "CA certificate (default: this node's CA)")
+	}
+
+	for _, d := range quotaDimensions {
+		accountQuotaSetCmd.Flags().Int(d.flag, 0, d.usage+" (-1 for no limit)")
+	}
+	accountQuotaSetCmd.Flags().Bool("clear", false, "Remove every override, returning the account to the configured limits")
+}
+
+func runAccountQuotaGet(cmd *cobra.Command, args []string) error {
+	target, err := resolveAdminTarget(cmd)
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), adminRequestTimeout)
+	defer cancel()
+
+	var out gateway.AccountQuotaResponse
+	req := gateway.AccountQuotaRequest{AccountID: args[0]}
+	if err := callAdmin(ctx, target, "GetAccountQuota", req, &out); err != nil {
+		return err
+	}
+	printAccountQuota(&out)
+	return nil
+}
+
+func runAccountQuotaSet(cmd *cobra.Command, args []string) error {
+	target, err := resolveAdminTarget(cmd)
+	if err != nil {
+		return err
+	}
+
+	over, changed, err := quotaOverridesFromFlags(cmd)
+	if err != nil {
+		return err
+	}
+	clearAll, err := cmd.Flags().GetBool("clear")
+	if err != nil {
+		return err
+	}
+	// Refusing both is not pedantry: --clear silently winning would discard
+	// limits the caller believed it was setting in the same breath.
+	if clearAll && changed {
+		return fmt.Errorf("--clear cannot be combined with a dimension flag")
+	}
+	if !clearAll && !changed {
+		return fmt.Errorf("no dimension given; pass at least one limit flag, or --clear to remove every override")
+	}
+	if clearAll {
+		over = handlers_quota.Overrides{}
+	}
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), adminRequestTimeout)
+	defer cancel()
+
+	var out gateway.AccountQuotaResponse
+	req := gateway.AccountQuotaRequest{AccountID: args[0], Overrides: over}
+	if err := callAdmin(ctx, target, "PutAccountQuota", req, &out); err != nil {
+		return err
+	}
+	printAccountQuota(&out)
+	return nil
+}
+
+// quotaOverridesFromFlags builds the override set from the flags the caller
+// actually passed. An unset flag is left nil so the dimension keeps inheriting,
+// which is why Changed is consulted rather than the flag's value.
+func quotaOverridesFromFlags(cmd *cobra.Command) (handlers_quota.Overrides, bool, error) {
+	var over handlers_quota.Overrides
+	changed := false
+	for _, d := range quotaDimensions {
+		if !cmd.Flags().Changed(d.flag) {
+			continue
+		}
+		value, err := cmd.Flags().GetInt(d.flag)
+		if err != nil {
+			return over, false, err
+		}
+		if value < handlers_quota.Unlimited {
+			return over, false, fmt.Errorf("--%s must be 0 or greater, or %d for no limit", d.flag, handlers_quota.Unlimited)
+		}
+		*d.field(&over) = &value
+		changed = true
+	}
+	return over, changed, nil
+}
+
+func printAccountQuota(resp *gateway.AccountQuotaResponse) {
+	fmt.Printf("Account %s\n\n", resp.AccountID)
+	fmt.Printf("%-16s %10s  %s\n", "DIMENSION", "LIMIT", "SOURCE")
+	fmt.Printf("%-16s %10s  %s\n", "---------", "-----", "------")
+
+	names := make([]string, 0, len(resp.Limits))
+	for name := range resp.Limits {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	for _, name := range names {
+		limit := strconv.Itoa(resp.Limits[name])
+		if resp.Limits[name] == handlers_quota.Unlimited {
+			limit = "unlimited"
+		}
+		fmt.Printf("%-16s %10s  %s\n", name, limit, resp.Source[name])
+	}
+
+	if resp.Overrides.UpdatedAt != "" {
+		fmt.Printf("\nOverride set by %s at %s\n", resp.Overrides.UpdatedBy, resp.Overrides.UpdatedAt)
+	}
+}

cmd/spinifex/cmd/admin_account_quota_test.go

@@ -0,0 +1,140 @@
+//test:in-package — drives the unexported flag table, parser and printer the quota subcommands are built from.
+package cmd
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	handlers_quota "github.com/mulgadc/spinifex/spinifex/handlers/quota"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/require"
+)
+
+// newQuotaSetCmd builds a command carrying the same flags as the real one, so
+// the parse is tested without mutating the package-level command between tests.
+func newQuotaSetCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "set"}
+	for _, d := range quotaDimensions {
+		cmd.Flags().Int(d.flag, 0, d.usage)
+	}
+	cmd.Flags().Bool("clear", false, "")
+	return cmd
+}
+
+// Only the flags actually passed become overrides. Every other dimension must
+// stay nil so it keeps inheriting, which is the whole point of a sparse record.
+func TestQuotaOverridesFromFlagsIsSparse(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--vcpus", "32", "--rds-instances", "8"}))
+
+	over, changed, err := quotaOverridesFromFlags(cmd)
+	require.NoError(t, err)
+	require.True(t, changed)
+
+	require.NotNil(t, over.VCPUs)
+	require.Equal(t, 32, *over.VCPUs)
+	require.NotNil(t, over.RDSInstances)
+	require.Equal(t, 8, *over.RDSInstances)
+
+	require.Nil(t, over.VPCs)
+	require.Nil(t, over.EIPs)
+	require.Nil(t, over.Volumes)
+	require.Nil(t, over.VolumesGiB)
+	require.Nil(t, over.Subnets)
+	require.Nil(t, over.LoadBalancers)
+}
+
+// A flag left unset must not be read as an explicit 0, which would deny every
+// request on that dimension instead of inheriting it.
+func TestQuotaOverridesFromFlagsUnsetIsNotZero(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--vpcs", "8"}))
+
+	over, changed, err := quotaOverridesFromFlags(cmd)
+	require.NoError(t, err)
+	require.True(t, changed)
+	require.Nil(t, over.VCPUs, "an unset flag must not become an explicit zero")
+}
+
+// Passing 0 deliberately is a real limit and must be preserved.
+func TestQuotaOverridesFromFlagsExplicitZero(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--eips", "0"}))
+
+	over, changed, err := quotaOverridesFromFlags(cmd)
+	require.NoError(t, err)
+	require.True(t, changed)
+	require.NotNil(t, over.EIPs)
+	require.Equal(t, 0, *over.EIPs)
+}
+
+func TestQuotaOverridesFromFlagsUnlimited(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--vcpus", "-1"}))
+
+	over, _, err := quotaOverridesFromFlags(cmd)
+	require.NoError(t, err)
+	require.NotNil(t, over.VCPUs)
+	require.Equal(t, handlers_quota.Unlimited, *over.VCPUs)
+}
+
+// Anything below the Unlimited sentinel is a typo, not a smaller limit.
+func TestQuotaOverridesFromFlagsRejectsBelowUnlimited(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--vcpus", "-2"}))
+
+	_, _, err := quotaOverridesFromFlags(cmd)
+	require.Error(t, err)
+}
+
+// No flags at all means nothing changed, which the caller turns into an error
+// rather than a write that silently does nothing.
+func TestQuotaOverridesFromFlagsNoneChanged(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags(nil))
+
+	over, changed, err := quotaOverridesFromFlags(cmd)
+	require.NoError(t, err)
+	require.False(t, changed)
+	require.True(t, over.Empty())
+}
+
+// The table must name every dimension, label its source, and render the
+// Unlimited sentinel as a word rather than as -1, which reads like a bug.
+func TestPrintAccountQuota(t *testing.T) {
+	resp := &gateway.AccountQuotaResponse{
+		AccountID: "000000000042",
+		Limits:    map[string]int{"vcpus": 32, "vpcs": handlers_quota.Unlimited, "eips": 4},
+		Source:    map[string]string{"vcpus": "override", "vpcs": "override", "eips": "config"},
+		Overrides: handlers_quota.Overrides{UpdatedBy: "operator", UpdatedAt: "2026-08-20T01:00:00Z"},
+	}
+
+	out := captureStdout(t, func() { printAccountQuota(resp) })
+
+	require.Contains(t, out, "000000000042")
+	require.Contains(t, out, "vcpus")
+	require.Contains(t, out, "32")
+	require.Contains(t, out, "override")
+	require.Contains(t, out, "config")
+	require.Contains(t, out, "unlimited")
+	require.NotContains(t, out, "-1")
+	require.Contains(t, out, "operator")
+}
+
+// Every dimension the override record carries must be reachable from the CLI,
+// or a limit exists that an operator cannot set.
+func TestQuotaDimensionsCoverEveryOverrideField(t *testing.T) {
+	var over handlers_quota.Overrides
+	for _, d := range quotaDimensions {
+		v := 1
+		*d.field(&over) = &v
+	}
+	require.False(t, over.Empty())
+
+	seen := map[string]bool{}
+	for _, d := range quotaDimensions {
+		require.False(t, seen[d.flag], "duplicate flag %q", d.flag)
+		seen[d.flag] = true
+	}
+	require.Len(t, quotaDimensions, 8)
+}

cmd/spinifex/cmd/admin_acm.go

@@ -60,16 +60,12 @@ func init() {
 	certCreateTenantCACmd.Flags().StringSlice("domain", nil, "Domain the tenant CA is permitted to sign for (repeatable)")
 	certCreateTenantCACmd.Flags().Bool("regenerate", false, "Replace an existing tenant CA with a fresh root (invalidates every device's trust store — requires confirmation)")
 	certCreateTenantCACmd.Flags().Bool("yes", false, "Skip the --regenerate confirmation prompt")
-	if err := certCreateTenantCACmd.MarkFlagRequired("domain"); err != nil {
-		panic(err)
-	}
+	_ = certCreateTenantCACmd.MarkFlagRequired("domain")
 
 	certCmd.AddCommand(certForceRenewCmd)
 	certForceRenewCmd.Flags().String("arn", "", "Certificate ARN to force-renew (required)")
 	certForceRenewCmd.Flags().String("account", "", "Account ID that owns the certificate; defaults to the bootstrap account")
-	if err := certForceRenewCmd.MarkFlagRequired("arn"); err != nil {
-		panic(err)
-	}
+	_ = certForceRenewCmd.MarkFlagRequired("arn")
 }
 
 func runCertForceRenew(cmd *cobra.Command, _ []string) {

cmd/spinifex/cmd/admin_cabake_test.go

@@ -1,67 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"errors"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestCABakeCmd_Args(t *testing.T) {
-	cmd := caBakeCmd(context.Background(), "/img/disk.raw", "/data/config/ca.pem")
-	args := cmd.Args
-
-	assert.Equal(t, "virt-customize", filepath.Base(args[0]))
-	assert.Contains(t, args, "-a")
-	assert.Contains(t, args, "/img/disk.raw")
-	assert.Contains(t, args, "--upload")
-	assert.Contains(t, args, "/data/config/ca.pem:/tmp/spinifex-ca.pem")
-	assert.Contains(t, args, "--run-command")
-
-	// --no-selinux-relabel stops virt-customize forcing a first-boot autorelabel,
-	// whose relabel+reboot corrupts XFS roots on RHEL/Rocky.
-	assert.Contains(t, args, "--no-selinux-relabel")
-
-	// The run-command covers Debian/Ubuntu (update-ca-certificates), RHEL/Rocky
-	// (update-ca-trust), and the stock-Alpine fallback that appends to the static
-	// ca-certificates bundle when no updater is present. The RHEL branch relabels
-	// only the touched trust paths so the cert is labelled without a full relabel.
-	joined := strings.Join(args, " ")
-	assert.Contains(t, joined, "update-ca-certificates")
-	assert.Contains(t, joined, "update-ca-trust")
-	assert.Contains(t, joined, "restorecon")
-	assert.Contains(t, joined, "/etc/ssl/certs/ca-certificates.crt")
-}
-
-func TestBakeCACertIntoImage_SkipsWhenCAMissing(t *testing.T) {
-	called := false
-	orig := caBakeRunner
-	caBakeRunner = func(string, string) ([]byte, error) { called = true; return nil, nil }
-	defer func() { caBakeRunner = orig }()
-
-	// A missing CA must not invoke virt-customize and must not panic.
-	bakeCACertIntoImage("/img/disk.raw", filepath.Join(t.TempDir(), "absent-ca.pem"))
-	assert.False(t, called, "runner must not be called when the CA is absent")
-}
-
-func TestBakeCACertIntoImage_ContinuesOnRunnerError(t *testing.T) {
-	caPath := filepath.Join(t.TempDir(), "ca.pem")
-	require.NoError(t, os.WriteFile(caPath, []byte("ca"), 0600))
-
-	called := false
-	orig := caBakeRunner
-	caBakeRunner = func(string, string) ([]byte, error) {
-		called = true
-		return []byte("libguestfs: cannot inspect image"), errors.New("exit status 1")
-	}
-	defer func() { caBakeRunner = orig }()
-
-	// An image libguestfs cannot inspect must skip-and-continue, never panic/fail.
-	bakeCACertIntoImage("/img/disk.raw", caPath)
-	assert.True(t, called, "runner must be invoked when the CA is present")
-}

cmd/spinifex/cmd/admin_firewall.go

@@ -0,0 +1,54 @@
+package cmd
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+	"os/exec"
+	"strconv"
+)
+
+// firewallApplyHelper is the fixed-verb root helper setup.sh installs. Absent on
+// a developer machine and on any node installed with --firewall=off, where there
+// is no policy to punch a hole in.
+var firewallApplyHelper = "/usr/local/lib/spinifex/spinifex-firewall-apply"
+
+// openFormationPort opens the formation port to any source and returns the
+// function that closes it again.
+//
+// The port is peer-scoped in the steady-state policy, where it carries the
+// daemon cluster manager. A node dialling in to join is not a peer yet, so
+// without this the join handshake is dropped rather than refused and the joiner
+// retries in silence until it times out. The window is bounded by formation
+// itself, and the handshake behind it is TLS 1.3 with a bearer token.
+//
+// Best effort: a node with no policy installed needs nothing done, and a
+// failure to open must not stop an operator forming a cluster.
+func openFormationPort(port int) func() {
+	if _, err := os.Stat(firewallApplyHelper); err != nil {
+		return func() {}
+	}
+
+	if err := runFirewallHelper("open-port", port); err != nil {
+		fmt.Printf("⚠️  Could not open port %d in the host firewall: %v\n", port, err)
+		fmt.Printf("   Joining nodes may be unable to reach the formation server.\n")
+		return func() {}
+	}
+
+	return func() {
+		if err := runFirewallHelper("close-port", port); err != nil {
+			// Loud, not fatal: formation has already succeeded by here, and the
+			// port stays open until the next reconcile or setup.sh run.
+			slog.Error("Failed to close the formation port in the host firewall",
+				"port", port, "helper", firewallApplyHelper, "err", err)
+		}
+	}
+}
+
+func runFirewallHelper(verb string, port int) error {
+	cmd := exec.Command(firewallApplyHelper, verb, strconv.Itoa(port))
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("%s: %w: %s", verb, err, out)
+	}
+	return nil
+}

cmd/spinifex/cmd/admin_firewall_test.go

@@ -0,0 +1,70 @@
+package cmd_test
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// stubFirewallHelper points the firewall helper at a script that logs its
+// arguments, and returns the path of that log.
+func stubFirewallHelper(t *testing.T, body string) string {
+	t.Helper()
+	dir := t.TempDir()
+	log := filepath.Join(dir, "calls")
+	helper := filepath.Join(dir, "spinifex-firewall-apply")
+	script := "#!/bin/sh\necho \"$*\" >> " + log + "\n" + body
+	require.NoError(t, os.WriteFile(helper, []byte(script), 0o755))
+	t.Cleanup(cmd.SetFirewallApplyHelper(helper))
+	return log
+}
+
+func helperCalls(t *testing.T, log string) []string {
+	t.Helper()
+	data, err := os.ReadFile(log)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	require.NoError(t, err)
+	return strings.Fields(strings.TrimSpace(strings.ReplaceAll(string(data), "\n", " ")))
+}
+
+func TestOpenFormationPort(t *testing.T) {
+	log := stubFirewallHelper(t, "exit 0\n")
+
+	closePort := cmd.OpenFormationPort(4432)
+	assert.Equal(t, []string{"open-port", "4432"}, helperCalls(t, log))
+
+	closePort()
+	assert.Equal(t, []string{"open-port", "4432", "close-port", "4432"}, helperCalls(t, log),
+		"the window must be closed again once formation ends")
+}
+
+// A machine with no policy installed has nothing to open, and formation must not
+// be held up by looking for a helper that was never installed.
+func TestOpenFormationPort_NoHelperInstalled(t *testing.T) {
+	t.Cleanup(cmd.SetFirewallApplyHelper(filepath.Join(t.TempDir(), "absent")))
+
+	closePort := cmd.OpenFormationPort(4432)
+	require.NotNil(t, closePort)
+	closePort()
+}
+
+// Opening is best effort: an operator forming a cluster must not be blocked by a
+// firewall helper that fails. The close must then not run either, since there is
+// nothing open to close.
+func TestOpenFormationPort_FailureDoesNotBlockFormation(t *testing.T) {
+	log := stubFirewallHelper(t, "exit 1\n")
+
+	closePort := cmd.OpenFormationPort(4432)
+	require.NotNil(t, closePort)
+	closePort()
+
+	assert.Equal(t, []string{"open-port", "4432"}, helperCalls(t, log),
+		"a failed open must not be followed by a close")
+}

cmd/spinifex/cmd/admin_jsprobe.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"time"
@@ -17,6 +18,12 @@ import (
 // other consumer reads.
 const canaryKey = "_watchdog.canary"
 
+// errProbeInconclusive marks a failure that happened before the probe ever
+// reached JetStream: config load, connect, or bucket open. The watchdog must
+// treat this as "learned nothing", not as JetStream refusing a write — during
+// recovery the bucket may simply not exist yet.
+var errProbeInconclusive = errors.New("probe did not reach JetStream")
+
 var nodeJSProbeCmd = &cobra.Command{
 	Use:   "js-probe",
 	Short: "Verify JetStream still accepts writes on this node",
@@ -42,6 +49,9 @@ func runNodeJSProbe(cmd *cobra.Command, _ []string) {
 
 	if err := probeJetStreamWrite(ctx); err != nil {
 		fmt.Fprintf(os.Stderr, "js-probe: %v\n", err)
+		if errors.Is(err, errProbeInconclusive) {
+			os.Exit(2)
+		}
 		os.Exit(1)
 	}
 	fmt.Println("js-probe: ok")
@@ -52,21 +62,26 @@ func runNodeJSProbe(cmd *cobra.Command, _ []string) {
 // than a bucket of its own: a dedicated bucket is a separate stream that can be
 // placed on a different node, so it could stay writable while the bucket the
 // control plane actually depends on is wedged.
+//
+// Every stage before canaryRoundTrip is wrapped in errProbeInconclusive: the
+// probe has observed nothing about JetStream's write path yet, so a caller
+// must not read config, connect, or bucket-open failures as a JetStream
+// failure.
 func probeJetStreamWrite(ctx context.Context) error {
 	_, nc, err := loadConfigAndConnect()
 	if err != nil {
-		return fmt.Errorf("connect: %w", err)
+		return fmt.Errorf("%w: connect: %w", errProbeInconclusive, err)
 	}
 	defer nc.Close()
 
 	js, err := jetstream.New(nc)
 	if err != nil {
-		return fmt.Errorf("jetstream: %w", err)
+		return fmt.Errorf("%w: jetstream: %w", errProbeInconclusive, err)
 	}
 
 	kv, err := js.KeyValue(ctx, daemon.ClusterStateBucket)
 	if err != nil {
-		return fmt.Errorf("open bucket %s: %w", daemon.ClusterStateBucket, err)
+		return fmt.Errorf("%w: open bucket %s: %w", errProbeInconclusive, daemon.ClusterStateBucket, err)
 	}
 	return canaryRoundTrip(ctx, kv)
 }

cmd/spinifex/cmd/admin_jsprobe_test.go

@@ -2,12 +2,17 @@ package cmd
 
 import (
 	"context"
+	"fmt"
+	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
 	"github.com/nats-io/nats.go/jetstream"
+	"github.com/spf13/viper"
 	"github.com/stretchr/testify/require"
 
+	"github.com/mulgadc/spinifex/spinifex/daemon"
 	"github.com/mulgadc/spinifex/spinifex/testutil"
 )
 
@@ -73,3 +78,82 @@ func TestCanaryRoundTrip_FailsWhenBucketIsGone(t *testing.T) {
 func TestCanaryKeyIsNamespaced(t *testing.T) {
 	require.True(t, len(canaryKey) > 0 && canaryKey[0] == '_')
 }
+
+// nodeTomlWithNATSHost writes a minimal single-node spinifex.toml pointing at
+// the given NATS host, the shape probeJetStreamWrite's loadConfigAndConnect
+// call needs to resolve node1's NATS endpoint.
+func nodeTomlWithNATSHost(t *testing.T, natsHost string) string {
+	t.Helper()
+	return writeSpinifexToml(t, fmt.Sprintf(`
+version = "1.0"
+epoch = 1
+node = "node1"
+
+[nodes.node1]
+node = "node1"
+region = "us-east-1"
+az = "us-east-1a"
+
+[nodes.node1.nats]
+host = %q
+`, natsHost))
+}
+
+// A config file viper cannot parse must never read as a JetStream failure:
+// the probe has not connected to anything yet.
+func TestProbeJetStreamWrite_ConfigLoadFailureIsInconclusive(t *testing.T) {
+	resetGlobalViper(t)
+
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	require.NoError(t, os.WriteFile(path, []byte("not [[[ valid toml"), 0o600))
+	viper.Set("config", path)
+
+	err := probeJetStreamWrite(context.Background())
+	require.Error(t, err)
+	require.ErrorIs(t, err, errProbeInconclusive, "config-load failure must be inconclusive")
+}
+
+// A NATS endpoint that refuses the connection must also read as inconclusive,
+// not as JetStream refusing a write — this is the class of failure that
+// turned a broken spx binary into a cluster-wide NATS restart.
+func TestProbeJetStreamWrite_ConnectFailureIsInconclusive(t *testing.T) {
+	resetGlobalViper(t)
+
+	viper.Set("config", nodeTomlWithNATSHost(t, "nats://127.0.0.1:1"))
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	err := probeJetStreamWrite(ctx)
+	require.Error(t, err)
+	require.ErrorIs(t, err, errProbeInconclusive, "connect failure must be inconclusive")
+}
+
+// A reachable NATS with no cluster-state bucket yet (recovery, or a bucket
+// that has not been created) must not read as a JetStream write failure.
+func TestProbeJetStreamWrite_BucketOpenFailureIsInconclusive(t *testing.T) {
+	resetGlobalViper(t)
+
+	ns, _, _ := testutil.StartTestJetStream(t)
+	viper.Set("config", nodeTomlWithNATSHost(t, ns.ClientURL()))
+
+	err := probeJetStreamWrite(context.Background())
+	require.Error(t, err)
+	require.ErrorIs(t, err, errProbeInconclusive, "missing bucket must be inconclusive")
+}
+
+// The positive end-to-end path: once the bucket exists and NATS accepts
+// writes, the probe succeeds through every stage rather than just at the
+// canaryRoundTrip unit level covered above.
+func TestProbeJetStreamWrite_SucceedsAgainstRealClusterStateBucket(t *testing.T) {
+	resetGlobalViper(t)
+
+	ns, _, js := testutil.StartTestJetStream(t)
+	_, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: daemon.ClusterStateBucket})
+	require.NoError(t, err)
+
+	viper.Set("config", nodeTomlWithNATSHost(t, ns.ClientURL()))
+
+	require.NoError(t, probeJetStreamWrite(context.Background()))
+}

cmd/spinifex/cmd/admin_ochre.go

@@ -1,10 +1,11 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
-	"log/slog"
 	"os"
 	"os/exec"
 	"path"
@@ -15,16 +16,16 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/mulgadc/predastore/pkg/masterkey"
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/ebsprovider"
 	gateway_bedrock "github.com/mulgadc/spinifex/spinifex/gateway/bedrock"
+	"github.com/mulgadc/spinifex/spinifex/gateway/bedrock/hfhub"
 	handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/utils"
-	"github.com/mulgadc/viperblock/viperblock"
-	vbs3 "github.com/mulgadc/viperblock/viperblock/backends/s3"
-	"github.com/mulgadc/viperblock/viperblock/v_utils"
+	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/pterm/pterm"
 	"github.com/spf13/cobra"
@@ -77,12 +78,51 @@ explicit act.`,
 	Run: runOchreWeightsRemove,
 }
 
+var ochreWeightsPullCmd = &cobra.Command{
+	Use:   "pull",
+	Short: "Fetch a self-host model's weights from Hugging Face into predastore",
+	Long: `pull resolves --revision to an immutable commit SHA, lists the repo's file
+tree, downloads only *.safetensors shards plus the fixed config/tokenizer
+set, and streams each into predastore under --s3-uri (or a keyed default
+when omitted). It never touches the offline 'stage' path or the weights KV
+store itself; it prints the resulting s3:// URI for 'weights stage --s3-uri'.
+
+Refuses before any object lands if --model-id is not a self-host catalog
+entry, the repo has no safetensors, or the hub returns 401/403 (a gated repo
+with no usable token). A failure partway through removes every object
+already written so 'stage' never sees a half-model.`,
+	Run: runOchreWeightsPull,
+}
+
+// ochreCredentialsCmd groups vendor credential management for pull's
+// optional stored-token fallback (D2); it holds no Run of its own.
+var ochreCredentialsCmd = &cobra.Command{
+	Use:   "credentials",
+	Short: "Manage stored provider credentials for self-host model pulls",
+}
+
+var ochreCredentialsSetCmd = &cobra.Command{
+	Use:   "set",
+	Short: "Store a vendor API token/licence credential",
+	Long: `set stores --vendor's token, encrypted at rest under the cluster's IAM master
+key, in the bedrock-credentials KV bucket. --account defaults to the
+platform account, since self-host models are platform-level, not per-tenant.
+
+The token itself is never a command-line argument: --token must be '-', and
+the secret is read from stdin, so it never appears in shell history or a
+process listing.`,
+	Run: runOchreCredentialsSet,
+}
+
 func init() {
 	adminCmd.AddCommand(ochreCmd)
 	ochreCmd.AddCommand(ochreWeightsCmd)
+	ochreCmd.AddCommand(ochreCredentialsCmd)
 	ochreWeightsCmd.AddCommand(ochreWeightsStageCmd)
 	ochreWeightsCmd.AddCommand(ochreWeightsListCmd)
 	ochreWeightsCmd.AddCommand(ochreWeightsRemoveCmd)
+	ochreWeightsCmd.AddCommand(ochreWeightsPullCmd)
+	ochreCredentialsCmd.AddCommand(ochreCredentialsSetCmd)
 
 	ochreWeightsStageCmd.Flags().String("model-id", "", "Catalog model ID to stage weights for (required)")
 	ochreWeightsStageCmd.Flags().String("s3-uri", "", "predastore S3 URI holding the model's Hugging Face files, e.g. s3://bucket/prefix/ (required)")
@@ -92,6 +132,19 @@ func init() {
 
 	ochreWeightsRemoveCmd.Flags().String("model-id", "", "Model ID to remove from staged weights (required)")
 	_ = ochreWeightsRemoveCmd.MarkFlagRequired("model-id")
+
+	ochreWeightsPullCmd.Flags().String("model-id", "", "Self-host catalog model ID this pull is for (required)")
+	ochreWeightsPullCmd.Flags().String("hf-repo", "", "Hugging Face repo, e.g. meta-llama/Llama-3.2-1B-Instruct (default: the catalog entry's repo)")
+	ochreWeightsPullCmd.Flags().String("revision", "main", "Hugging Face branch, tag or commit SHA to resolve and pin (default: the catalog entry's pinned revision)")
+	ochreWeightsPullCmd.Flags().String("s3-uri", "", "predastore destination prefix, e.g. s3://bucket/prefix/ (default: s3://ochre-weights/<repo>/<sha>/)")
+	ochreWeightsPullCmd.Flags().String("hf-token", "", "Hugging Face token for a gated repo (falls back to HF_TOKEN, then a stored platform credential)")
+	_ = ochreWeightsPullCmd.MarkFlagRequired("model-id")
+
+	ochreCredentialsSetCmd.Flags().String("vendor", "", "Vendor to store a credential for, e.g. huggingface (required)")
+	ochreCredentialsSetCmd.Flags().String("account", "", "Account ID to store the credential under (default: platform account)")
+	ochreCredentialsSetCmd.Flags().String("token", "", "Must be '-': reads the secret from stdin (required)")
+	_ = ochreCredentialsSetCmd.MarkFlagRequired("vendor")
+	_ = ochreCredentialsSetCmd.MarkFlagRequired("token")
 }
 
 // requiredWeightsFiles are the fixed-name Hugging Face artefacts stage
@@ -109,6 +162,264 @@ var requiredWeightsFiles = []string{
 // tokenizer.model under original/, not at the prefix root. Either is enough.
 var tokenizerFileNames = []string{"tokenizer.json", "tokenizer.model"}
 
+// vendorHuggingFace is the CredentialStore vendor key 'ochre credentials
+// set' and pull's stored-credential fallback share for the Hugging Face
+// licence token (D2).
+const vendorHuggingFace = "huggingface"
+
+// ochrePullManifestFile is the fixed name 'weights pull' writes into the
+// destination prefix and 'weights stage' reads back (D3), recording which
+// upstream commit a staged model's weights came from.
+const ochrePullManifestFile = "ochre-pull.json"
+
+// ochrePullManifest is the JSON body of ochrePullManifestFile.
+type ochrePullManifest struct {
+	HFRepo      string    `json:"hf_repo"`
+	RevisionSHA string    `json:"revision_sha"`
+	PulledAt    time.Time `json:"pulled_at"`
+}
+
+// rootOnlyPullFileNames are the fixed-name config artefacts pull fetches only
+// at the repo root. A same-named file in a subdir is a component config, not
+// the model's -- e.g. sentence-transformers' 1_Pooling/config.json -- and must
+// never flatten onto the root config.json, so nested matches are dropped.
+var rootOnlyPullFileNames = map[string]bool{
+	"config.json":                  true,
+	"tokenizer_config.json":        true,
+	"generation_config.json":       true,
+	"special_tokens_map.json":      true,
+	"model.safetensors.index.json": true,
+}
+
+// nestablePullFileNames are the tokenizer artefacts matched by basename at any
+// depth: some repos keep them in a subdir (e.g. Llama's original/tokenizer.model).
+var nestablePullFileNames = map[string]bool{
+	"tokenizer.json":  true,
+	"tokenizer.model": true,
+}
+
+// selectPullFiles filters a Hugging Face tree listing down to the
+// safetensors-only set pull downloads (D4): every *.safetensors file plus
+// the fixed config/tokenizer set. Anything else -- notably .bin/.pt pickle
+// checkpoints, which can execute code on load -- is dropped silently rather
+// than aborting the whole pull.
+func selectPullFiles(entries []hfhub.TreeEntry) []hfhub.TreeEntry {
+	var selected []hfhub.TreeEntry
+	for _, e := range entries {
+		if e.Type != "file" {
+			continue
+		}
+		base := path.Base(e.Path)
+		nested := strings.Contains(e.Path, "/")
+		switch {
+		case strings.HasSuffix(e.Path, ".safetensors"):
+			selected = append(selected, e)
+		case nestablePullFileNames[base]:
+			selected = append(selected, e)
+		case rootOnlyPullFileNames[base] && !nested:
+			selected = append(selected, e)
+		}
+	}
+	return selected
+}
+
+// anySafetensors reports whether entries contains at least one *.safetensors
+// file. selectPullFiles's config/tokenizer set alone is never enough --
+// D4 aborts a pull with no actual weights to serve.
+func anySafetensors(entries []hfhub.TreeEntry) bool {
+	for _, e := range entries {
+		if strings.HasSuffix(e.Path, ".safetensors") {
+			return true
+		}
+	}
+	return false
+}
+
+// defaultPullPrefix derives D7's fallback destination when --s3-uri is
+// omitted: keyed by the exact repo and resolved commit SHA, so re-pulling
+// the same commit lands at the same, self-deduping URI.
+func defaultPullPrefix(hfRepo, sha string) string {
+	return fmt.Sprintf("s3://ochre-weights/%s/%s/", strings.Trim(hfRepo, "/"), sha)
+}
+
+// putPullManifest writes ochrePullManifestFile into bucket/prefix, the last
+// step of a successful pull (D3).
+func putPullManifest(ctx context.Context, store objectstore.ObjectStore, bucket, prefix string, manifest ochrePullManifest) error {
+	data, err := json.Marshal(manifest)
+	if err != nil {
+		return fmt.Errorf("encode pull manifest: %w", err)
+	}
+	key := prefix + ochrePullManifestFile
+	if _, err := store.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String(key), Body: bytes.NewReader(data)}); err != nil {
+		return fmt.Errorf("put pull manifest s3://%s/%s: %w", bucket, key, err)
+	}
+	return nil
+}
+
+// readPullManifest reads and decodes bucket/prefix's ochre-pull.json, if
+// present. A missing manifest is not an error: stage's offline path (an
+// operator-supplied prefix with no pull manifest) must keep working
+// unchanged, so ok is simply false.
+func readPullManifest(ctx context.Context, store objectstore.ObjectStore, bucket, prefix string) (ochrePullManifest, bool, error) {
+	key := prefix + ochrePullManifestFile
+	out, err := store.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
+	if err != nil {
+		if objectstore.IsNoSuchKeyError(err) {
+			return ochrePullManifest{}, false, nil
+		}
+		return ochrePullManifest{}, false, fmt.Errorf("get pull manifest s3://%s/%s: %w", bucket, key, err)
+	}
+	defer out.Body.Close()
+
+	var manifest ochrePullManifest
+	if err := json.NewDecoder(out.Body).Decode(&manifest); err != nil {
+		return ochrePullManifest{}, false, fmt.Errorf("decode pull manifest s3://%s/%s: %w", bucket, key, err)
+	}
+	return manifest, true, nil
+}
+
+// cleanupPulledObjects best-effort deletes every key already written by a
+// pull that failed partway through (D5): a half-uploaded prefix must never
+// be left for 'stage' to mistake for a complete model. A delete failure is
+// logged, not returned -- the original pull error is what the operator needs.
+func cleanupPulledObjects(ctx context.Context, store objectstore.ObjectStore, bucket string, keys []string) {
+	for _, key := range keys {
+		if _, err := store.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)}); err != nil {
+			fmt.Fprintf(os.Stderr, "warning: could not clean up s3://%s/%s after a failed pull: %v\n", bucket, key, err)
+		}
+	}
+}
+
+// pullOneFile downloads hfPath from the hub to a local temp file and PUTs it
+// to bucket/key. A temp file is required, not a direct HTTP-to-S3 pipe:
+// predastore's PutObject signs the request over an io.ReadSeeker, which an
+// HTTP response body is not. tmpDir is cleaned up by the caller.
+func pullOneFile(ctx context.Context, hf *hfhub.Client, store objectstore.ObjectStore, hfRepo, sha, hfPath, bucket, key, tmpDir string) error {
+	body, _, err := hf.DownloadFile(ctx, hfRepo, sha, hfPath)
+	if err != nil {
+		return err
+	}
+	defer body.Close()
+
+	tmpFile, err := os.CreateTemp(tmpDir, "part-*")
+	if err != nil {
+		return fmt.Errorf("create temp file: %w", err)
+	}
+	defer os.Remove(tmpFile.Name())
+	defer tmpFile.Close()
+
+	if _, err := io.Copy(tmpFile, body); err != nil {
+		return fmt.Errorf("download to temp file: %w", err)
+	}
+	if _, err := tmpFile.Seek(0, io.SeekStart); err != nil {
+		return fmt.Errorf("seek temp file: %w", err)
+	}
+
+	if _, err := store.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String(key), Body: tmpFile}); err != nil {
+		return fmt.Errorf("put s3://%s/%s: %w", bucket, key, err)
+	}
+	return nil
+}
+
+// pullFilesToPrefix downloads and uploads every selected file, one at a time
+// so at most one file is ever held on local disk (D6). Keys are flattened to
+// each file's basename to match the flat layout 'stage' validates -- e.g.
+// Llama's original/tokenizer.model lands at <prefix>tokenizer.model. It
+// returns the keys written so far even on error, so the caller can clean up
+// a partial prefix (D5).
+func pullFilesToPrefix(ctx context.Context, hf *hfhub.Client, store objectstore.ObjectStore, hfRepo, sha, bucket, prefix string, files []hfhub.TreeEntry) ([]string, error) {
+	tmpDir, err := os.MkdirTemp("", "spinifex-ochre-pull-*")
+	if err != nil {
+		return nil, fmt.Errorf("create temp dir: %w", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	var uploaded []string
+	seen := make(map[string]string, len(files))
+	for _, f := range files {
+		key := prefix + path.Base(f.Path)
+		if prev, dup := seen[key]; dup {
+			return uploaded, fmt.Errorf("refusing to overwrite %s: both %s and %s flatten to it", key, prev, f.Path)
+		}
+		seen[key] = f.Path
+		if err := pullOneFile(ctx, hf, store, hfRepo, sha, f.Path, bucket, key, tmpDir); err != nil {
+			return uploaded, fmt.Errorf("pull %s: %w", f.Path, err)
+		}
+		uploaded = append(uploaded, key)
+	}
+	return uploaded, nil
+}
+
+// runPullWeights holds 'ochre weights pull' decision and side-effect logic:
+// catalog validation, ref resolution to an immutable commit SHA (D3), tree
+// listing and safetensors-only filtering (D4), streaming each selected file
+// into predastore (D6), and finally the ochre-pull.json manifest. It never
+// touches the weights KV store; 'stage' consumes the printed s3:// URI
+// separately. A failure removes every object already written (D5).
+func runPullWeights(ctx context.Context, hf *hfhub.Client, store objectstore.ObjectStore, modelID, hfRepo, revision string, revisionExplicit bool, s3URIFlag string) (string, error) {
+	spec, found, selfHost := gateway_bedrock.LookupServingSpec(modelID)
+	if !found || !selfHost {
+		if !found {
+			return "", fmt.Errorf("unknown model ID %q: not present in the Ochre catalog", modelID)
+		}
+		return "", fmt.Errorf("%q is a provider-served model, not self-host; weights pull does not apply", modelID)
+	}
+
+	// The catalog carries the canonical repo and a pinned revision so a bare
+	// pull cannot silently grab a regressed upstream default; an explicit flag
+	// still overrides either.
+	if hfRepo == "" {
+		hfRepo = spec.HFRepo
+	}
+	if hfRepo == "" {
+		return "", fmt.Errorf("no --hf-repo given and %q has no catalog HF repo", modelID)
+	}
+	if !revisionExplicit && spec.HFRevision != "" {
+		revision = spec.HFRevision
+	}
+
+	fmt.Printf("Resolving %s@%s ...\n", hfRepo, revision)
+	sha, err := hf.ResolveRevision(ctx, hfRepo, revision)
+	if err != nil {
+		return "", err
+	}
+
+	fmt.Printf("Listing files at %s@%s ...\n", hfRepo, sha)
+	tree, err := hf.ListTree(ctx, hfRepo, sha)
+	if err != nil {
+		return "", err
+	}
+
+	selected := selectPullFiles(tree)
+	if !anySafetensors(selected) {
+		return "", fmt.Errorf("%s@%s has no *.safetensors files; refusing to pull pickle-format (.bin) weights", hfRepo, sha)
+	}
+
+	s3URI := s3URIFlag
+	if s3URI == "" {
+		s3URI = defaultPullPrefix(hfRepo, sha)
+	}
+	bucket, prefix, err := parseWeightsS3URI(s3URI)
+	if err != nil {
+		return "", err
+	}
+
+	fmt.Printf("Pulling %d file(s) into s3://%s/%s ...\n", len(selected), bucket, prefix)
+	uploaded, err := pullFilesToPrefix(ctx, hf, store, hfRepo, sha, bucket, prefix, selected)
+	if err != nil {
+		cleanupPulledObjects(ctx, store, bucket, uploaded)
+		return "", err
+	}
+
+	manifest := ochrePullManifest{HFRepo: hfRepo, RevisionSHA: sha, PulledAt: time.Now().UTC()}
+	if err := putPullManifest(ctx, store, bucket, prefix, manifest); err != nil {
+		cleanupPulledObjects(ctx, store, bucket, uploaded)
+		return "", err
+	}
+
+	return fmt.Sprintf("s3://%s/%s", bucket, prefix), nil
+}
+
 // parseWeightsS3URI splits an s3://bucket/prefix URI into its bucket and
 // prefix. The prefix is normalised to end with '/' so downstream listing and
 // validation scope to the directory rather than any key merely sharing the
@@ -307,63 +618,6 @@ func buildWeightsImage(srcDir, imagePath string, contentBytes int64) error {
 	return mkfsExt4Runner(srcDir, imagePath)
 }
 
-// snapshotImportedWeightsVolume snapshots a viperblock volume that
-// v_utils.ImportDiskImage just wrote and closed -- never attached, never
-// nbdkit-served, so there is no live checkpoint to load. This mirrors the
-// offline snapshot sequence handlers/ec2/image/service_impl.go's
-// snapshotStoppedVolume uses for a stopped instance's root volume: reopen
-// read-only, load the numbered checkpoint Close() wrote, then CreateSnapshot.
-//
-// az names the zone recorded on the snapshot for DescribeSnapshots; it is not
-// used to resolve the clone.
-func snapshotImportedWeightsVolume(s3Config *vbs3.S3Config, volumeID string, volumeSize uint64, walDir, az string, mkey *masterkey.Key) (string, error) {
-	vbConfig := viperblock.VB{
-		VolumeName:        volumeID,
-		VolumeSize:        volumeSize,
-		BaseDir:           walDir,
-		Cache:             viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}},
-		MasterKey:         mkey,
-		EncryptionEnabled: mkey != nil,
-		Logger:            slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})),
-	}
-
-	vb, err := viperblock.New(&vbConfig, "s3", *s3Config)
-	if err != nil {
-		return "", fmt.Errorf("new viperblock: %w", err)
-	}
-	defer func() {
-		vb.StopChunkUploader()
-		vb.StopWALSyncer()
-	}()
-
-	if err := vb.Backend.Init(); err != nil {
-		return "", fmt.Errorf("backend init: %w", err)
-	}
-	if err := vb.LoadState(); err != nil {
-		return "", fmt.Errorf("load state: %w", err)
-	}
-	if err := vb.LoadBlockState(); err != nil {
-		return "", fmt.Errorf("load block state: %w", err)
-	}
-	defer func() {
-		if err := vb.RemoveLocalFiles(); err != nil {
-			slog.Warn("snapshotImportedWeightsVolume: failed to remove local files", "volumeId", volumeID, "err", err)
-		}
-	}()
-
-	snapshotID := admin.SnapPrefix(volumeID)
-	if _, err := vb.CreateSnapshot(snapshotID); err != nil {
-		return "", fmt.Errorf("create snapshot: %w", err)
-	}
-
-	store := objectstore.NewS3ObjectStoreFromConfig(s3Config.Host, s3Config.Region, s3Config.AccessKey, s3Config.SecretKey)
-	if err := registerWeightsSnapshot(store, s3Config.Bucket, snapshotID, volumeID, volumeSize, az, mkey != nil); err != nil {
-		return "", err
-	}
-
-	return snapshotID, nil
-}
-
 // registerWeightsSnapshot writes the EC2 control plane's half of the snapshot
 // prefix. CreateSnapshot above writes only viperblock's half -- the block
 // checkpoint and config.json -- but CreateVolume resolves a SnapshotId through
@@ -447,6 +701,16 @@ func runStageWeights(ctx context.Context, store objectstore.ObjectStore, weights
 		return "", err
 	}
 
+	// A pull manifest is optional: absent means an operator-supplied prefix
+	// from the offline path, which stage has always supported and must keep
+	// supporting unchanged.
+	var sourceRevision string
+	if manifest, ok, err := readPullManifest(ctx, store, bucket, prefix); err != nil {
+		return "", err
+	} else if ok {
+		sourceRevision = manifest.RevisionSHA
+	}
+
 	tmpDir, err := os.MkdirTemp(tmpDirFlag, "spinifex-weights-tmp-*")
 	if err != nil {
 		return "", fmt.Errorf("could not create temp dir: %w", err)
@@ -469,7 +733,7 @@ func runStageWeights(ctx context.Context, store objectstore.ObjectStore, weights
 		return "", err
 	}
 
-	if err := weightsStore.PutWeights(ctx, modelID, sourceURI, snapshotID); err != nil {
+	if err := weightsStore.PutWeightsWithRevision(ctx, modelID, sourceURI, snapshotID, sourceRevision); err != nil {
 		return "", err
 	}
 
@@ -485,12 +749,11 @@ func runStageWeights(ctx context.Context, store objectstore.ObjectStore, weights
 }
 
 // materializeWeightsVolume is the real weightsMaterializer: it packages
-// downloadDir into a filesystem image, imports it into a new viperblock
-// volume backed by node's predastore, and snapshots it. This is the
-// expensive, environment-dependent half of stage -- it shells out to
-// mkfs.ext4 and talks to a live S3-shaped backend -- so it is exercised live
-// (see the plan's Testing section) rather than in unit tests.
-func materializeWeightsVolume(node config.Config, downloadDir string, contentBytes int64, mkey *masterkey.Key) (string, error) {
+// downloadDir into a filesystem image, imports it into a new volume through
+// the EBS provider, and snapshots it. This is the expensive,
+// environment-dependent half of stage -- it shells out to mkfs.ext4 and needs
+// a live provider -- so it is exercised live rather than in unit tests.
+func materializeWeightsVolume(ctx context.Context, provider ebsprovider.EBSProvider, nodeID string, node config.Config, downloadDir string, contentBytes int64) (string, error) {
 	tmpDir := filepath.Dir(downloadDir)
 	volumeId := utils.GenerateResourceID("vol")
 	imagePath := filepath.Join(tmpDir, volumeId+".img")
@@ -505,74 +768,32 @@ func materializeWeightsVolume(node config.Config, downloadDir string, contentByt
 		return "", fmt.Errorf("could not stat image: %w", err)
 	}
 
-	// Round the volume up to a whole GiB rather than using the raw image size.
-	// viperblock rejects a tail write whose block overruns VolumeSize, so an
-	// unaligned size fails at the last block. This also keeps the byte size and
-	// the manifest's SizeGiB describing the same volume.
+	// Round the volume up to a whole GiB rather than using the raw image size,
+	// so the byte size and the recorded SizeGiB describe the same volume.
 	sizeGiB := amiVolumeSizeGiB(imageStat.Size())
 	volumeBytes := sizeGiB * bytesPerGiB
 
-	s3Config := vbs3.S3Config{
-		VolumeName: volumeId,
-		VolumeSize: volumeBytes,
-		Bucket:     node.Predastore.Bucket,
-		Region:     node.Predastore.Region,
-		AccessKey:  node.Predastore.AccessKey,
-		SecretKey:  node.Predastore.SecretKey,
-		Host:       node.Predastore.Host,
-	}
-
-	// AMIMetadata is deliberately left zero-valued: a weights volume is not
-	// bootable and must never be registered as a launchable AMI. Leaving it
-	// unset also means ImportDiskImage does not attempt its own automatic
-	// snapshot -- that happens explicitly below, after the volume is closed.
-	manifest := viperblock.VolumeConfig{}
-	manifest.VolumeMetadata.VolumeID = volumeId
-	manifest.VolumeMetadata.VolumeName = volumeId
-	manifest.VolumeMetadata.TenantID = "system"
-	manifest.VolumeMetadata.SizeGiB = sizeGiB
-	manifest.VolumeMetadata.State = "available"
-	manifest.VolumeMetadata.AvailabilityZone = node.Predastore.Region
-	manifest.VolumeMetadata.CreatedAt = time.Now()
-	manifest.VolumeMetadata.VolumeType = "gp3"
-	manifest.VolumeMetadata.IOPS = 1000
-
-	vbConfig := viperblock.VB{
-		VolumeName:        volumeId,
-		VolumeSize:        volumeBytes,
-		BaseDir:           tmpDir,
-		Cache:             viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}},
-		VolumeConfig:      manifest,
-		MasterKey:         mkey,
-		EncryptionEnabled: mkey != nil,
-		Logger:            slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})),
-	}
-
-	var flushBar *pterm.ProgressbarPrinter
-	var flushUpdate func(current uint64)
-	progress := func(current, total uint64) {
-		if flushBar == nil {
-			flushBar, flushUpdate = utils.NewByteProgressBar("Flushing weights to storage", total)
-		}
-		flushUpdate(current)
-	}
-
 	// Name the volume in the failure path: a part-written import leaves blocks
 	// in predastore, and without the ID an operator cannot find them to reclaim.
-	if err := v_utils.ImportDiskImage(&s3Config, &vbConfig, imagePath, progress); err != nil {
-		if flushBar != nil {
-			_, _ = flushBar.Stop()
-		}
+	fmt.Println("Writing weights to storage ...")
+	if err := admin.ImportImage(ctx, provider, admin.ImportOpts{
+		VolumeID:         volumeId,
+		NodeID:           nodeID,
+		SizeBytes:        utils.SafeUint64ToInt64(volumeBytes),
+		AvailabilityZone: node.AZ,
+		SourcePath:       imagePath,
+		Snapshot:         true,
+		Progress:         os.Stdout,
+	}); err != nil {
 		return "", fmt.Errorf("could not import weights volume %s: %w", volumeId, err)
 	}
-	if flushBar != nil {
-		_, _ = flushBar.Stop()
-	}
 
-	fmt.Println("Snapshotting weights volume ...")
-	snapshotID, err := snapshotImportedWeightsVolume(&s3Config, volumeId, volumeBytes, tmpDir, node.AZ, mkey)
-	if err != nil {
-		return "", fmt.Errorf("could not snapshot weights volume: %w", err)
+	// The provider wrote viperblock's half of the snapshot prefix; this writes
+	// the EC2 control plane's, without which a launch cannot resolve it.
+	snapshotID := admin.SnapPrefix(volumeId)
+	store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
+	if err := registerWeightsSnapshot(store, node.Predastore.Bucket, snapshotID, volumeId, volumeBytes, node.AZ, node.Viperblock.EncryptionKeyFile != ""); err != nil {
+		return "", err
 	}
 	return snapshotID, nil
 }
@@ -583,8 +804,8 @@ func materializeWeightsVolume(node config.Config, downloadDir string, contentByt
 // stage time -- and treats a missing object as gone. Any other read failure
 // is a real error, not a signal to guess the snapshot's existence.
 func newWeightsSnapshotChecker(store objectstore.ObjectStore, bucket string) weightsSnapshotChecker {
-	return func(_ context.Context, snapshotID string) (bool, error) {
-		if _, err := handlers_ec2_snapshot.ReadSnapshotConfig(store, bucket, snapshotID); err != nil {
+	return func(ctx context.Context, snapshotID string) (bool, error) {
+		if _, err := handlers_ec2_snapshot.ReadSnapshotConfig(ctx, store, bucket, snapshotID); err != nil {
 			if objectstore.IsNoSuchKeyError(err) {
 				return false, nil
 			}
@@ -629,15 +850,9 @@ func runOchreWeightsStage(cmd *cobra.Command, _ []string) {
 	weightsStore := gateway_bedrock.NewWeightsStore(js, len(appConfig.Nodes))
 	store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
 
-	mkey, err := utils.LoadViperblockMasterKey(node.Viperblock.EncryptionKeyFile)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not load viperblock encryption key: %v\n", err)
-		ochreExit(1)
-		return
-	}
-
-	materialize := func(_ context.Context, downloadDir string, contentBytes int64) (string, error) {
-		return materializeWeightsVolume(node, downloadDir, contentBytes, mkey)
+	provider := ebsprovider.NewNATSProvider(nc, imageImportTimeout)
+	materialize := func(ctx context.Context, downloadDir string, contentBytes int64) (string, error) {
+		return materializeWeightsVolume(ctx, provider, appConfig.Node, node, downloadDir, contentBytes)
 	}
 	checkSnapshotLive := newWeightsSnapshotChecker(store, node.Predastore.Bucket)
 
@@ -673,7 +888,7 @@ func listWeightsOutput(ctx context.Context, weightsStore *gateway_bedrock.Weight
 		return "No models staged.", nil
 	}
 
-	tableData := pterm.TableData{{"MODEL ID", "SOURCE URI", "SNAPSHOT ID", "STATUS"}}
+	tableData := pterm.TableData{{"MODEL ID", "SOURCE URI", "REVISION", "SNAPSHOT ID", "STATUS"}}
 	for _, e := range entries {
 		live, err := checkSnapshotLive(ctx, e.SnapshotID)
 		if err != nil {
@@ -683,7 +898,11 @@ func listWeightsOutput(ctx context.Context, weightsStore *gateway_bedrock.Weight
 		if !live {
 			status = weightsStatusDangling
 		}
-		tableData = append(tableData, []string{e.ModelID, e.SourceURI, e.SnapshotID, status})
+		revision := e.SourceRevision
+		if revision == "" {
+			revision = "-"
+		}
+		tableData = append(tableData, []string{e.ModelID, e.SourceURI, revision, e.SnapshotID, status})
 	}
 	return pterm.DefaultTable.WithHasHeader().WithLeftAlignment().WithData(tableData).Srender()
 }
@@ -766,6 +985,142 @@ func runOchreWeightsRemove(cmd *cobra.Command, _ []string) {
 	fmt.Printf("✅ Removed staged-weights entry for %s (snapshot %s and source objects untouched).\n", modelID, entry.SnapshotID)
 }
 
+// resolveHFToken applies pull's token resolution order (D2): the --hf-token
+// flag, then HF_TOKEN env (the must-have path, self-sufficient on its own),
+// then a stored platform credential for vendor "huggingface". The stored
+// credential step is a best-effort fallback: a cluster with no IAM master
+// key provisioned yet simply yields no token rather than failing the pull.
+func resolveHFToken(ctx context.Context, cmd *cobra.Command, cfg *config.ClusterConfig, nc *nats.Conn) string {
+	if token, _ := cmd.Flags().GetString("hf-token"); token != "" {
+		return token
+	}
+	if token := os.Getenv("HF_TOKEN"); token != "" {
+		return token
+	}
+
+	masterKeyPath := filepath.Join(cfg.NodeBaseDir(), "config", "master.key")
+	masterKey, err := handlers_iam.LoadMasterKey(masterKeyPath)
+	if err != nil {
+		return ""
+	}
+	js, err := jetstream.New(nc)
+	if err != nil {
+		return ""
+	}
+	credStore := gateway_bedrock.NewCredentialStore(js, masterKey, len(cfg.Nodes), nil)
+	token, ok, err := credStore.Resolve(ctx, utils.GlobalAccountID, vendorHuggingFace)
+	if err != nil || !ok {
+		return ""
+	}
+	return token
+}
+
+func runOchreWeightsPull(cmd *cobra.Command, _ []string) {
+	modelID, _ := cmd.Flags().GetString("model-id")
+	hfRepo, _ := cmd.Flags().GetString("hf-repo")
+	revision, _ := cmd.Flags().GetString("revision")
+	revisionExplicit := cmd.Flags().Changed("revision")
+	s3URI, _ := cmd.Flags().GetString("s3-uri")
+
+	appConfig, nc, err := loadConfigAndConnectFn()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+	defer nc.Close()
+	node := appConfig.Nodes[appConfig.Node]
+
+	ctx := context.Background()
+	token := resolveHFToken(ctx, cmd, appConfig, nc)
+
+	hf := hfhub.NewClient(token)
+	store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
+
+	finalURI, err := runPullWeights(ctx, hf, store, modelID, hfRepo, revision, revisionExplicit, s3URI)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		ochreExit(1)
+		return
+	}
+	fmt.Printf("✅ Pulled into %s\n", finalURI)
+	fmt.Println(finalURI)
+}
+
+// readTokenFromStdin reads a credential from r, trimming exactly one
+// trailing newline (matching 'printf "%s" "$TOKEN" | cmd --token -' or a
+// piped 'echo'). The token is never logged.
+func readTokenFromStdin(r io.Reader) (string, error) {
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return "", fmt.Errorf("read token from stdin: %w", err)
+	}
+	token := strings.TrimSuffix(strings.TrimSuffix(string(data), "\n"), "\r")
+	if token == "" {
+		return "", fmt.Errorf("no token read from stdin")
+	}
+	return token, nil
+}
+
+// ochreCredentialsStore connects to the cluster, loads the node's IAM master
+// key from disk -- the same on-disk key initIAMServiceFromConfig uses -- and
+// returns a CredentialStore ready for PutCredential/Resolve.
+func ochreCredentialsStore() (*gateway_bedrock.CredentialStore, func(), error) {
+	cfg, nc, err := loadConfigAndConnectFn()
+	if err != nil {
+		return nil, nil, fmt.Errorf("connect to cluster: %w", err)
+	}
+	masterKeyPath := filepath.Join(cfg.NodeBaseDir(), "config", "master.key")
+	masterKey, err := handlers_iam.LoadMasterKey(masterKeyPath)
+	if err != nil {
+		nc.Close()
+		return nil, nil, fmt.Errorf("load master key: %w", err)
+	}
+	js, err := jetstream.New(nc)
+	if err != nil {
+		nc.Close()
+		return nil, nil, fmt.Errorf("jetstream context: %w", err)
+	}
+	return gateway_bedrock.NewCredentialStore(js, masterKey, len(cfg.Nodes), nil), func() { nc.Close() }, nil
+}
+
+func runOchreCredentialsSet(cmd *cobra.Command, _ []string) {
+	vendor, _ := cmd.Flags().GetString("vendor")
+	accountID, _ := cmd.Flags().GetString("account")
+	tokenFlag, _ := cmd.Flags().GetString("token")
+
+	if tokenFlag != "-" {
+		fmt.Fprintln(os.Stderr, "Error: --token must be '-' (the secret is read from stdin, never a command-line argument)")
+		ochreExit(1)
+		return
+	}
+	if accountID == "" {
+		accountID = utils.GlobalAccountID
+	}
+
+	token, err := readTokenFromStdin(os.Stdin)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+
+	store, cleanup, err := ochreCredentialsStore()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+	defer cleanup()
+
+	if err := store.PutCredential(context.Background(), accountID, vendor, token); err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+	fmt.Printf("✅ Stored %s credential for account %s\n", vendor, accountID)
+}
+
 var adminOchreAccessCmd = &cobra.Command{
 	Use:   "access",
 	Short: "Manage per-account model access grants",
@@ -814,15 +1169,11 @@ func init() {
 		c.Flags().String("account-id", "", "12-digit account ID to change (required)")
 		c.Flags().String("model-id", "", "Model ID to change (e.g. meta.llama3-2-1b-instruct-v1:0)")
 		c.Flags().Bool("all-models", false, "Apply to every model in the platform catalog")
-		if err := c.MarkFlagRequired("account-id"); err != nil {
-			panic(err)
-		}
+		_ = c.MarkFlagRequired("account-id")
 	}
 
 	adminOchreAccessListCmd.Flags().String("account-id", "", "12-digit account ID to inspect (required)")
-	if err := adminOchreAccessListCmd.MarkFlagRequired("account-id"); err != nil {
-		panic(err)
-	}
+	_ = adminOchreAccessListCmd.MarkFlagRequired("account-id")
 }
 
 // ochreAccessStore connects to the cluster and returns the grant store along

cmd/spinifex/cmd/admin_ochre_appliance.go

@@ -0,0 +1,97 @@
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	handlers_ochrevector "github.com/mulgadc/spinifex/spinifex/handlers/ochrevector"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/spf13/cobra"
+)
+
+// applianceTeardownTimeout bounds the teardown NATS round trip: it covers an
+// RDS DeleteDBInstance call plus a KV delete, not a VM boot, so it is far
+// shorter than the appliance's own launch timeout.
+const applianceTeardownTimeout = 5 * time.Minute
+
+var ochreApplianceCmd = &cobra.Command{
+	Use:   "appliance",
+	Short: "Manage the shared platform Postgres appliance backing every Ochre vector index",
+}
+
+var ochreApplianceTeardownCmd = &cobra.Command{
+	Use:   "teardown",
+	Short: "Destroy the shared platform Postgres appliance",
+	Long: `teardown destroys the platform appliance singleton: its backing RDS DB
+instance, this node's VPC host port, and its KV record.
+
+This is a destructive, cluster-wide operator action -- every account's vector
+index data stored in the appliance is lost with it. By default the index
+registry and KB/DataSource metadata survive, exactly as they survive a
+restart: a future daemon startup re-provisions a fresh appliance (a new
+instance, a new password) and reconciles each surviving index by re-creating
+it and re-ingesting from its recorded source, with no operator action and no
+change to its id. --purge-metadata additionally wipes that metadata for an
+intentional full destroy, after which nothing is auto-restored.
+
+Requires --confirm.`,
+	Run: runOchreApplianceTeardown,
+}
+
+func init() {
+	ochreCmd.AddCommand(ochreApplianceCmd)
+	ochreApplianceCmd.AddCommand(ochreApplianceTeardownCmd)
+
+	ochreApplianceTeardownCmd.Flags().Bool("confirm", false,
+		"Required: confirms you intend to destroy the shared platform appliance and every account's vector index data with it")
+	ochreApplianceTeardownCmd.Flags().Bool("purge-metadata", false,
+		"Also wipe the index registry and KB/DataSource records, disabling auto-restore on the next re-provision")
+}
+
+// applianceTeardownFn indirects the NATS call so runApplianceTeardownGuarded
+// is testable without a live daemon, mirroring vectorServiceFn/
+// endpointServiceFn.
+var applianceTeardownFn = func(ctx context.Context, purgeMetadata bool) error {
+	_, nc, err := loadConfigAndConnectFn()
+	if err != nil {
+		return err
+	}
+	defer nc.Close()
+	_, err = utils.NATSRequest[handlers_ochrevector.TeardownApplianceResponse](ctx, nc,
+		handlers_ochrevector.SubjectTeardownAppliance, &handlers_ochrevector.TeardownApplianceRequest{PurgeMetadata: purgeMetadata},
+		applianceTeardownTimeout, utils.GlobalAccountID)
+	return err
+}
+
+// runApplianceTeardownGuarded is the testable core of 'ochre appliance
+// teardown': it refuses without --confirm before making any NATS call, so a
+// missing flag never even opens a connection to the cluster.
+func runApplianceTeardownGuarded(ctx context.Context, confirmed, purgeMetadata bool, teardown func(context.Context, bool) error) (string, error) {
+	if !confirmed {
+		return "", errors.New("refusing to tear down the shared platform appliance without --confirm: " +
+			"this destroys every account's vector index data and is not automatically rebuilt")
+	}
+	if err := teardown(ctx, purgeMetadata); err != nil {
+		return "", err
+	}
+	if purgeMetadata {
+		return "✅ Platform appliance torn down, index/KB metadata purged.", nil
+	}
+	return "✅ Platform appliance torn down.", nil
+}
+
+func runOchreApplianceTeardown(cmd *cobra.Command, _ []string) {
+	confirmed, _ := cmd.Flags().GetBool("confirm")
+	purgeMetadata, _ := cmd.Flags().GetBool("purge-metadata")
+
+	msg, err := runApplianceTeardownGuarded(context.Background(), confirmed, purgeMetadata, applianceTeardownFn)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		ochreExit(1)
+		return
+	}
+	fmt.Println(msg)
+}

cmd/spinifex/cmd/admin_ochre_appliance_test.go

@@ -0,0 +1,153 @@
+// Exercises unexported ochre appliance CLI command internals with no
+// exported surface to drive them through.
+//
+//test:in-package
+package cmd
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/testutil"
+	"github.com/stretchr/testify/require"
+)
+
+// TestRunApplianceTeardownGuarded_RefusesWithoutConfirm proves the guard
+// refuses before ever invoking teardown -- a missing --confirm must never
+// reach a NATS call, let alone destroy the appliance.
+func TestRunApplianceTeardownGuarded_RefusesWithoutConfirm(t *testing.T) {
+	called := false
+	teardown := func(context.Context, bool) error {
+		called = true
+		return nil
+	}
+
+	_, err := runApplianceTeardownGuarded(context.Background(), false, false, teardown)
+	require.Error(t, err)
+	require.ErrorContains(t, err, "--confirm")
+	require.False(t, called, "teardown must not run without --confirm")
+}
+
+// TestRunApplianceTeardownGuarded_ConfirmedInvokesTeardown proves --confirm
+// drives the teardown call through and reports success.
+func TestRunApplianceTeardownGuarded_ConfirmedInvokesTeardown(t *testing.T) {
+	called := false
+	teardown := func(context.Context, bool) error {
+		called = true
+		return nil
+	}
+
+	msg, err := runApplianceTeardownGuarded(context.Background(), true, false, teardown)
+	require.NoError(t, err)
+	require.True(t, called)
+	require.Contains(t, msg, "torn down")
+}
+
+// TestRunApplianceTeardownGuarded_TeardownErrorSurfaces proves a teardown
+// failure (e.g. the daemon reporting no appliance up) surfaces to the caller
+// rather than being swallowed behind a generic success message.
+func TestRunApplianceTeardownGuarded_TeardownErrorSurfaces(t *testing.T) {
+	teardown := func(context.Context, bool) error {
+		return errors.New("ochrevector: platform appliance is not enabled or not up on this node")
+	}
+
+	_, err := runApplianceTeardownGuarded(context.Background(), true, false, teardown)
+	require.Error(t, err)
+	require.ErrorContains(t, err, "not enabled or not up")
+}
+
+// TestRunApplianceTeardownGuarded_PurgeMetadataThreadsThrough proves the
+// --purge-metadata flag reaches the teardown call and is reflected in the
+// success message, so an operator can tell which mode ran.
+func TestRunApplianceTeardownGuarded_PurgeMetadataThreadsThrough(t *testing.T) {
+	var gotPurge bool
+	teardown := func(_ context.Context, purgeMetadata bool) error {
+		gotPurge = purgeMetadata
+		return nil
+	}
+
+	msg, err := runApplianceTeardownGuarded(context.Background(), true, true, teardown)
+	require.NoError(t, err)
+	require.True(t, gotPurge, "purgeMetadata must reach the teardown call")
+	require.Contains(t, msg, "metadata purged")
+}
+
+// TestApplianceTeardownFn_ThreadsPurgeMetadataOverNATS drives the real
+// (unstubbed) applianceTeardownFn through runApplianceTeardownGuarded, so the
+// actual NATSRequest call -- including the PurgeMetadata field on the wire
+// request -- executes rather than a test double standing in for it. No
+// daemon is subscribed to the teardown subject in this test, so the call
+// fails fast; the point is that it's reached and built with PurgeMetadata set.
+func TestApplianceTeardownFn_ThreadsPurgeMetadataOverNATS(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	stubConnect(t, fakeClusterConfig(), nc, nil)
+
+	_, err := runApplianceTeardownGuarded(context.Background(), true, true, applianceTeardownFn)
+	require.Error(t, err, "no daemon is subscribed to the teardown subject in this test")
+}
+
+// withApplianceTeardown swaps applianceTeardownFn for one that never dials
+// NATS, so the Run wrapper's real flag/exit control flow is exercised
+// without a live daemon.
+func withApplianceTeardown(t *testing.T, fn func(context.Context, bool) error) {
+	t.Helper()
+	orig := applianceTeardownFn
+	t.Cleanup(func() { applianceTeardownFn = orig })
+	applianceTeardownFn = fn
+}
+
+func TestRunOchreApplianceTeardown_MissingConfirmExits1(t *testing.T) {
+	withApplianceTeardown(t, func(context.Context, bool) error {
+		t.Fatal("teardown must not be called without --confirm")
+		return nil
+	})
+
+	cmd := *ochreApplianceTeardownCmd
+	code := withOchreExitCapture(t, func() { runOchreApplianceTeardown(&cmd, nil) })
+	require.Equal(t, 1, code)
+}
+
+func TestRunOchreApplianceTeardown_ConfirmedPrintsSuccess(t *testing.T) {
+	withApplianceTeardown(t, func(context.Context, bool) error { return nil })
+
+	cmd := *ochreApplianceTeardownCmd
+	require.NoError(t, cmd.Flags().Set("confirm", "true"))
+
+	var out string
+	code := withOchreExitCapture(t, func() {
+		out = captureStdout(t, func() { runOchreApplianceTeardown(&cmd, nil) })
+	})
+	require.Equal(t, -1, code)
+	require.Contains(t, out, "torn down")
+}
+
+func TestRunOchreApplianceTeardown_TeardownErrorExits1(t *testing.T) {
+	withApplianceTeardown(t, func(context.Context, bool) error {
+		return errors.New("ochrevector: platform appliance is not enabled or not up on this node")
+	})
+
+	cmd := *ochreApplianceTeardownCmd
+	require.NoError(t, cmd.Flags().Set("confirm", "true"))
+
+	code := withOchreExitCapture(t, func() { runOchreApplianceTeardown(&cmd, nil) })
+	require.Equal(t, 1, code)
+}
+
+// TestRunOchreApplianceTeardown_PurgeMetadataFlagReachesTeardownFn proves
+// the --purge-metadata flag is read and passed through the Run wrapper.
+func TestRunOchreApplianceTeardown_PurgeMetadataFlagReachesTeardownFn(t *testing.T) {
+	var gotPurge bool
+	withApplianceTeardown(t, func(_ context.Context, purgeMetadata bool) error {
+		gotPurge = purgeMetadata
+		return nil
+	})
+
+	cmd := *ochreApplianceTeardownCmd
+	require.NoError(t, cmd.Flags().Set("confirm", "true"))
+	require.NoError(t, cmd.Flags().Set("purge-metadata", "true"))
+
+	code := withOchreExitCapture(t, func() { runOchreApplianceTeardown(&cmd, nil) })
+	require.Equal(t, -1, code)
+	require.True(t, gotPurge, "--purge-metadata must reach applianceTeardownFn")
+}