release: update go.mod for v1.18.0

The local helper masked x-amz-security-token by line prefix, which matched the
header form and missed the presigned query form entirely, leaking the token into
the exported logs on every signature mismatch. Defer to bluebottle's
RedactedCanonicalRequest, which knows the structure and masks both forms.

.github/actions/junit-to-es/action.yml

@@ -0,0 +1,43 @@
+name: JUnit to Elasticsearch
+description: >-
+  Bulk-index one doc per JUnit test case into the mulga Elasticsearch sink,
+  so per-test pass rates outlive the 90-day artifact retention. Call it once
+  per job right after the JUnit XML is produced (with `if: always()`), next to
+  the junit-report action that renders the same files into the job summary.
+
+inputs:
+  junit-glob:
+    description: Shell glob matching the JUnit XML files to ship.
+    required: true
+  cell:
+    description: >-
+      Cell label stored as labels.ci_cell — the same value the job puts in
+      ci.cell for its OTel attributes (single, multi, gpu, ddil, 20, ...).
+      Without it, results from different topologies collapse into one bucket.
+    required: true
+  es-sink:
+    description: host:port of the Elasticsearch sink.
+    required: false
+    default: "192.168.1.13:9200"
+  index:
+    description: Target index.
+    required: false
+    default: mulga-ci-test-results
+
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      env:
+        JUNIT_GLOB: ${{ inputs.junit-glob }}
+        JUNIT_CELL: ${{ inputs.cell }}
+        MULGA_ES_SINK: ${{ inputs.es-sink }}
+        ES_INDEX: ${{ inputs.index }}
+        RUN_ID: ${{ github.run_id }}
+        RUN_ATTEMPT: ${{ github.run_attempt }}
+        WORKFLOW: ${{ github.workflow }}
+        BRANCH: ${{ github.head_ref || github.ref_name }}
+        SHA: ${{ github.sha }}
+        ARTIFACT_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+      run: |
+        python3 "${{ github.action_path }}/ship.py"

.github/actions/junit-to-es/ship.py

@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+"""Ship per-test JUnit results to the Elasticsearch sink.
+
+One doc per leaf test case, bulk-indexed into mulga-ci-test-results. The
+mulga-ci-runs summary says which suite failed; this says which test, with the
+failure text, so a per-test pass-rate trend outlives artifact retention.
+
+Inputs (env):
+    JUNIT_GLOB      Glob matching JUnit XML files. No matches is not an error:
+                    a leg that never ran produces none.
+    JUNIT_CELL      Cell label for these results (single, multi, gpu, 20, ...).
+    MULGA_ES_SINK   host:port of the sink. Anonymous requests are superuser
+                    there, so no credentials are involved.
+    ES_INDEX        Target index; defaults to mulga-ci-test-results.
+    RUN_ID, RUN_ATTEMPT, WORKFLOW, BRANCH, SHA, ARTIFACT_URL
+"""
+
+from __future__ import annotations
+
+import glob
+import json
+import os
+import re
+import sys
+import urllib.error
+import urllib.request
+import xml.etree.ElementTree as ET
+from datetime import datetime, timezone
+
+# Go test output carries colour codes into the JUnit CDATA (harness.Phase
+# emits them), which are noise in a stored failure message.
+_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
+
+# Enough failure text to recognise a failure; the artifact log stays the full
+# record.
+_MAX_MESSAGE = 4000
+
+# go-junit-report emits this when a `=== RUN` line has no matching result line,
+# which means the log it was fed was truncated or filtered. It is a parse gap,
+# not a test outcome, so it is stored under its own status and left out of every
+# pass rate rather than counted as a failure.
+_NO_RESULT = "No test result found"
+
+
+def _strip_ansi(s: str) -> str:
+    return _ANSI_RE.sub("", s)
+
+
+def _truncate(s: str) -> str:
+    s = s.strip()
+    return s if len(s) <= _MAX_MESSAGE else s[: _MAX_MESSAGE - 1] + "…"
+
+
+def _suite_from_path(path: str) -> str:
+    base = os.path.basename(path)
+    if base.endswith(".xml"):
+        base = base[:-4]
+    if base.startswith("junit-"):
+        base = base[len("junit-") :]
+    return base
+
+
+def _cases(path: str) -> list[dict]:
+    """Parse one JUnit file into leaf test cases."""
+    try:
+        root = ET.parse(path).getroot()
+    except ET.ParseError as exc:
+        print(f"::warning::junit-to-es: {path} is not parseable ({exc}), skipped")
+        return []
+
+    # JUnit XML has two shapes: <testsuite> as root, or <testsuites> wrapping
+    # several. .iter() handles both.
+    out: list[dict] = []
+    for s in root.iter("testsuite"):
+        for tc in s.findall("testcase"):
+            status = "pass"
+            fail_type = ""
+            message = ""
+            for tag in ("failure", "error"):
+                node = tc.find(tag)
+                if node is not None:
+                    status = "fail" if tag == "failure" else "error"
+                    fail_type = tag
+                    message = _strip_ansi(
+                        ((node.get("message") or "") + "\n" + (node.text or "")).strip()
+                    )
+                    if (node.get("message") or "").strip() == _NO_RESULT:
+                        status = "unknown"
+                    break
+            if status == "pass" and tc.find("skipped") is not None:
+                status = "skip"
+            out.append(
+                {
+                    "name": tc.get("name", ""),
+                    "classname": tc.get("classname", ""),
+                    "time": float(tc.get("time", 0) or 0),
+                    "status": status,
+                    "fail_type": fail_type,
+                    "message": message,
+                }
+            )
+
+    # Drop container cases: Go reports a parent t.Run as a rollup of its
+    # children, so keeping both double-counts one outcome. Same rule the job
+    # summary renderer applies.
+    names = {c["name"] for c in out}
+    containers = {n.rsplit("/", 1)[0] for n in names if "/" in n}
+    return [c for c in out if c["name"] not in containers]
+
+
+def _doc(cell: str, suite: str, case: dict, ts: str, ci: dict) -> dict:
+    full = case["name"]
+    top, _, sub = full.partition("/")
+    doc = {
+        "@timestamp": ts,
+        "suite": suite,
+        "status": case["status"],
+        "duration_seconds": case["time"],
+        "artifact_url": ci["artifact_url"],
+        "labels": {"ci_run_id": ci["run_id"], "ci_cell": cell},
+        "test": {
+            "name": top,
+            "subtest": sub,
+            "full_name": full,
+            "classname": case["classname"],
+        },
+        "ci": {
+            "workflow": ci["workflow"],
+            "branch": ci["branch"],
+            "sha": ci["sha"],
+            "run_attempt": ci["run_attempt"],
+        },
+    }
+    if case["fail_type"]:
+        doc["failure"] = {
+            "type": case["fail_type"],
+            "message": _truncate(case["message"]),
+        }
+    return doc
+
+
+def _doc_id(ci: dict, cell: str, suite: str, full_name: str) -> str:
+    # Deterministic so a replayed post overwrites instead of double-counting.
+    # run_attempt is part of it because a re-run is a distinct result.
+    return f"{ci['run_id']}-{ci['run_attempt']}-{cell}-{suite}-{full_name}"
+
+
+def _bulk(sink: str, index: str, body: str) -> int:
+    req = urllib.request.Request(
+        f"http://{sink}/{index}/_bulk",
+        data=body.encode("utf-8"),
+        headers={"Content-Type": "application/x-ndjson"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            payload = json.loads(resp.read().decode("utf-8"))
+    except (urllib.error.URLError, OSError) as exc:
+        print(f"::error::junit-to-es: bulk POST to {sink}/{index} failed: {exc}")
+        return 1
+
+    if payload.get("errors"):
+        for item in payload.get("items", []):
+            err = next(iter(item.values())).get("error")
+            if err:
+                print(f"::error::junit-to-es: bulk rejected a doc: {json.dumps(err)}")
+                return 1
+    return 0
+
+
+def main() -> int:
+    pattern = os.environ.get("JUNIT_GLOB", "").strip()
+    if not pattern:
+        print("::error::junit-to-es: JUNIT_GLOB is empty")
+        return 1
+    cell = os.environ.get("JUNIT_CELL", "").strip()
+    if not cell:
+        print("::error::junit-to-es: JUNIT_CELL is empty")
+        return 1
+    sink = os.environ.get("MULGA_ES_SINK", "").strip()
+    if not sink:
+        print("::error::junit-to-es: MULGA_ES_SINK is empty")
+        return 1
+    index = os.environ.get("ES_INDEX", "").strip() or "mulga-ci-test-results"
+
+    ci = {
+        "run_id": os.environ.get("RUN_ID", ""),
+        "run_attempt": os.environ.get("RUN_ATTEMPT", ""),
+        "workflow": os.environ.get("WORKFLOW", ""),
+        "branch": os.environ.get("BRANCH", ""),
+        "sha": os.environ.get("SHA", ""),
+        "artifact_url": os.environ.get("ARTIFACT_URL", ""),
+    }
+    ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    lines: list[str] = []
+    counts: dict[str, int] = {"pass": 0, "fail": 0, "error": 0, "skip": 0, "unknown": 0}
+    for path in sorted(glob.glob(pattern)):
+        suite = _suite_from_path(path)
+        for case in _cases(path):
+            doc = _doc(cell, suite, case, ts, ci)
+            counts[case["status"]] = counts.get(case["status"], 0) + 1
+            meta = {"index": {"_id": _doc_id(ci, cell, suite, doc["test"]["full_name"])}}
+            lines.append(json.dumps(meta))
+            lines.append(json.dumps(doc))
+
+    if not lines:
+        # Not an error on its own: the leg may have been skipped, or died
+        # before any test ran. The run summary doc records that separately.
+        print(f"junit-to-es: no JUnit files matched {pattern}, nothing to post")
+        return 0
+
+    rc = _bulk(sink, index, "\n".join(lines) + "\n")
+    if rc == 0:
+        print(
+            f"junit-to-es: posted {sum(counts.values())} test docs "
+            f"(cell {cell}) to {sink}/{index} — {counts['pass']} pass, "
+            f"{counts['fail']} fail, {counts['error']} error, "
+            f"{counts['skip']} skip, {counts['unknown']} unknown"
+        )
+    return rc
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/resolve-sibling-refs.sh

@@ -1,20 +1,21 @@
 #!/usr/bin/env bash
-# Resolve which branch of each sibling repo (viperblock, predastore) the CI
-# workspace should build against. A cross-repo change is developed on a branch
+# Resolve which branch of each sibling repo (viperblock, predastore, northstar,
+# bluebottle) the CI workspace should build against. A cross-repo change is developed on a branch
 # of the SAME name in each repo; build against that branch when it exists so the
 # set can be tested together, and fall back to dev otherwise. This mirrors the
 # resolution the e2e path already does in
 # mulga/scripts/tofu-cluster/build-install-artifacts.sh, so unit CI and e2e CI
 # agree on how sibling branches are selected.
 #
-# Reads $BRANCH (the triggering head/ref name) and writes viperblock=<ref> and
-# predastore=<ref> to $GITHUB_OUTPUT for the checkout steps to consume.
+# Reads $BRANCH (the triggering head/ref name) and writes viperblock=<ref>,
+# predastore=<ref>, northstar=<ref> and bluebottle=<ref> to $GITHUB_OUTPUT for
+# the checkout steps to consume.
 set -euo pipefail
 
 : "${BRANCH:?BRANCH must be set to the triggering branch name}"
 : "${GITHUB_OUTPUT:?GITHUB_OUTPUT must be set}"
 
-for repo in viperblock predastore; do
+for repo in viperblock predastore northstar bluebottle; do
   if git ls-remote --exit-code --heads \
       "https://github.com/mulgadc/${repo}.git" "$BRANCH" >/dev/null 2>&1; then
     ref="$BRANCH"

.github/scripts/suite-sets.sh

@@ -24,10 +24,14 @@
 # behaviour, so a second node adds no assertion. Its DB VMs are capped at four
 # concurrent by the suite's own semaphore, which is what keeps it inside the
 # job's 50-minute budget alongside the suites above.
-E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds"
+# quota belongs to both topologies because the gate it asserts is gateway code
+# that runs identically on either, and because a quota left switched on would
+# cap every suite sharing the environment — so it must be proved to restore
+# itself wherever it runs, not only on one node.
+E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota"
 
 # Suites runnable against a multi-node environment.
-E2E_SUITES_MULTI="multinode lb cert"
+E2E_SUITES_MULTI="multinode lb cert quota"
 
 # grep -xE alternation form, for narrowing a requested set to the eligible one.
 E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"

.github/workflows/e2e-baremetal.yml

@@ -143,6 +143,16 @@ jobs:
           [ -f "$LOG" ] || exit 0
           go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-baremetal.xml" || true
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: "20"
+
       - name: Analyze e2e failures
         if: always()
         uses: ./.github/actions/e2e-analyze

.github/workflows/e2e-ddil.yml

@@ -180,6 +180,16 @@ jobs:
           junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
           title: "DDIL scenario results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
+          cell: ddil
+
       - name: Collect Cluster Logs
         if: failure()
         working-directory: mulga/scripts/tofu-cluster

.github/workflows/e2e-nightly.yml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
     inputs:
       cells:
-        description: "Comma-separated cell numbers to run (blank = all)"
+        description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-19 tofu matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection"
         required: false
 
 permissions:
@@ -342,6 +342,14 @@ jobs:
               | tar -C "${ARTIFACT_DIR}/vm-console-node${idx}" -xf - 2>/dev/null
             ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
               > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
+            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \
+              > "${ARTIFACT_DIR}/openvswitch-ipsec-node${idx}.log" 2>/dev/null
+            # ovn-controller and ovs-vswitchd log to files, never to journald:
+            # without these a cross-chassis black-hole cannot be attributed to
+            # IPsec versus SB port binding.
+            mkdir -p "${ARTIFACT_DIR}/ovs-logs-node${idx}"
+            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo tar -C /var/log -cf - ovn openvswitch 2>/dev/null || true' \
+              | tar -C "${ARTIFACT_DIR}/ovs-logs-node${idx}" -xf - 2>/dev/null
           done
 
           cat "${ARTIFACT_DIR}"/spinifex-journal-node*.log \
@@ -363,6 +371,16 @@ jobs:
             go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
           done
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always() && steps.gate.outputs.selected == 'true'
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: ${{ matrix.cell }}
+
       - name: Analyze e2e failures
         if: always() && steps.gate.outputs.selected == 'true'
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -379,8 +397,11 @@ jobs:
           if-no-files-found: warn
           retention-days: 14
 
+      # Gated like every other step, not just always(): a cell the dispatch
+      # deselected never checked out or ran tofu init, so destroying would fail
+      # on missing plugins and report the cell red for correctly doing nothing.
       - name: Teardown
-        if: always()
+        if: always() && steps.gate.outputs.selected == 'true'
         working-directory: mulga/scripts/tofu-cluster
         env:
           MATRIX_HOST: ${{ matrix.host }}
@@ -397,7 +418,7 @@ jobs:
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
 
       - name: Upload tofu log
-        if: always()
+        if: always() && steps.gate.outputs.selected == 'true'
         uses: actions/upload-artifact@v7
         with:
           name: tofu-log-cell-${{ matrix.cell }}-${{ github.run_id }}
@@ -426,3 +447,615 @@ jobs:
       # There's no matrix leg here, just the one fixed baremetal cell.
       otel-attrs: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=20,ci.matrix=baremetal/pxe/real-hw
     secrets: inherit
+
+  # ISO cells. Unlike 1-19 these need no tofu: the harnesses run QEMU directly
+  # on the runner. The nodes are therefore L1, which keeps the instances they
+  # launch at L2 and their benchmark numbers comparable between the two cells.
+  #
+  # Serialised against each other because the guest MACs are fixed
+  # (52:54:00:12:...): stable MACs are what make DHCP discovery work, and two
+  # concurrent runs would put duplicates on the rack LAN, which is a confusing
+  # failure for everyone on it, not just CI.
+  iso-single:
+    name: cell-21 (iso/dhcp/single)
+    if: >-
+      github.event_name == 'schedule' ||
+      inputs.cells == '' ||
+      contains(format(',{0},', inputs.cells), ',21,')
+    runs-on: [self-hosted, ci-single]
+    # ISO build, install, boot and pool re-form is ~35 of these on its own,
+    # before the workload step imports an AMI and launches an instance.
+    timeout-minutes: 90
+    concurrency:
+      group: iso-qemu-rack
+      cancel-in-progress: false
+    env:
+      CI_WAN_MODE: dhcp
+      CI_PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
+      # What a real node gets. The harness default of 4G is a dev-box size, and
+      # importing an 820 MiB AMI on top of the running stack does not fit in it.
+      MEM: 8G
+      SMP: 8
+      # .230 is consumed by the OVN router's SNAT, leaving .231-.234 for
+      # instances. Disjoint from cell-22's pool so a run that dies holding its
+      # SNAT address cannot collide with the next cell.
+      EXT_POOL_START: 192.168.0.230
+      EXT_POOL_END: 192.168.0.234
+      EXT_GATEWAY: 192.168.1.1
+      EXT_PREFIX_LEN: 23
+      # Pins where the benchmark writes, so collecting and removing it later are
+      # both exact. Left to itself the harness picks a timestamped name.
+      BENCH_OUT: /tmp/cell21-bench-${{ github.run_id }}
+    steps:
+      - name: Clean Workspace
+        run: |
+          sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          path: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ env.GIT_BRANCH }}
+          path: mulga/spinifex
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ env.GIT_BRANCH }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: mulga/spinifex
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: mulga/viperblock
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: mulga/predastore
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: mulga/northstar
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: mulga/bluebottle
+
+      # Assert the host can do what the cell assumes, before spending 40
+      # minutes finding out it cannot. A checklist in a plan doc goes stale;
+      # this does not.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          ip link show "${BRIDGE:-br-wan}" >/dev/null ||
+            { echo "::error::bridge br-wan not found on this runner"; exit 1; }
+          grep -qE "^allow +${BRIDGE:-br-wan}$" /etc/qemu/bridge.conf ||
+            { echo "::error::br-wan not permitted in /etc/qemu/bridge.conf"; exit 1; }
+          command -v sshpass >/dev/null ||
+            { echo "::error::sshpass not installed"; exit 1; }
+          free_mb=$(free -m | awk '/^Mem:/ {print $7}')
+          [ "$free_mb" -ge 8192 ] ||
+            { echo "::error::need 8 GiB available, have ${free_mb} MiB"; exit 1; }
+          # A 40G image on tmpfs competes with the 8 GiB the VM itself needs.
+          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
+          [ "$fstype" != tmpfs ] ||
+            { echo "::error::$TMPDIR is tmpfs — the node disk would be in RAM"; exit 1; }
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          [ "$avail_gb" -ge 60 ] ||
+            { echo "::error::need 60 GiB on $TMPDIR, have ${avail_gb} GiB"; exit 1; }
+
+      - name: Build artefacts
+        working-directory: mulga/scripts/iso-builder
+        run: make build-rootfs build-squashfs build-initramfs
+
+      # One key for both halves of the cell. test-workload.sh SSHes to the node
+      # AND registers the same key as the EC2 key pair it uses to reach the
+      # instances, so the key test-smoke.sh installs has to be that one — its
+      # default ~/.ssh/id_*.pub would leave the workload step locked out.
+      - name: Prepare SSH key
+        run: |
+          set -euo pipefail
+          mkdir -p "$HOME/.ssh" && chmod 700 "$HOME/.ssh"
+          # Two keys, because the guest key must not share a name with the one
+          # smoke-test.sh registers from the node: the node holds that private
+          # half, and IMDS caches material by key name for five minutes, so
+          # re-importing over it is invisible to the guests.
+          for k in spinifex-key spinifex-workload-key; do
+            [ -f "$HOME/.ssh/$k" ] ||
+              ssh-keygen -t ed25519 -N "" -f "$HOME/.ssh/$k" -C "$k-ci"
+          done
+
+      # Boots the ISO, installs unattended, reads the lease off the serial
+      # console, runs the in-guest smoke test, and installs the key for the
+      # next step.
+      - name: Install and smoke test
+        id: smoke
+        working-directory: mulga/scripts/iso-builder
+        run: |
+          set -euo pipefail
+          # KEEP_NODE: the harness owns the only VM there is and tears it down
+          # on exit, so without this the workload step below has no node left.
+          KEEP_NODE=1 SSH_IDENTITY="$HOME/.ssh/spinifex-key.pub" \
+            ./test/test-smoke.sh | tee /tmp/smoke-$GITHUB_RUN_ID.log
+          ip=$(sed -n 's/^\[smoke\] node address: //p' /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
+          [ -n "$ip" ] || { echo "::error::smoke test did not report a node address"; exit 1; }
+          echo "node_ip=$ip" >> "$GITHUB_OUTPUT"
+
+      # The customer path: AMI, key, VPC, instance, SSH in. --benchmark records
+      # host and guest disk numbers so a regression shows up as a number rather
+      # than a story.
+      - name: Workload and benchmark
+        working-directory: mulga/scripts/iso-builder
+        run: |
+          set -euo pipefail
+          cat > /tmp/cell21.env <<EOF
+          NODES=(${{ steps.smoke.outputs.node_ip }})
+          NODE_NAMES=(spinifex-ci)
+          SSH_USER=spinifex
+          SSH_KEY_PATH=$HOME/.ssh/spinifex-workload-key
+          # SSH_KEY_PATH reaches the guests, SSH_IDENTITY reaches the node, and
+          # they are deliberately different keys. Without the second, node SSH
+          # falls back to the runner's default identity, which the node has
+          # never seen.
+          SSH_IDENTITY=$HOME/.ssh/spinifex-key
+          EXT_POOL_START=$EXT_POOL_START
+          EXT_POOL_END=$EXT_POOL_END
+          EXT_GATEWAY=$EXT_GATEWAY
+          EXT_PREFIX_LEN=$EXT_PREFIX_LEN
+          EOF
+          ./test/test-workload.sh --env /tmp/cell21.env --benchmark
+
+      - name: Collect logs
+        if: always()
+        run: |
+          mkdir -p /tmp/cell21-artifacts-$GITHUB_RUN_ID
+          cp /tmp/smoke-$GITHUB_RUN_ID.log /tmp/cell21-artifacts-$GITHUB_RUN_ID/ ||
+            echo "::warning::no smoke log to collect"
+          # fio JSON and the summary table. Warned about rather than swallowed:
+          # a benchmark that ran and produced nothing is worth knowing about.
+          if [ -d "$BENCH_OUT" ]; then
+            cp -r "$BENCH_OUT" /tmp/cell21-artifacts-$GITHUB_RUN_ID/benchmark
+          else
+            echo "::warning::no benchmark results at $BENCH_OUT"
+          fi
+          # The guest's own view. Without it a node that dies mid-test is only
+          # visible as the harness command that stopped producing output.
+          serial=$(sed -n 's/^\[smoke\] serial log preserved: //p' \
+                   /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
+          [ -n "$serial" ] && cp "$serial" /tmp/cell21-artifacts-$GITHUB_RUN_ID/serial.log || true
+          node=$(sed -n 's/^\[smoke\] node logs preserved: //p' \
+                 /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
+          [ -n "$node" ] && cp "$node" /tmp/cell21-artifacts-$GITHUB_RUN_ID/node.log || true
+          # QEMU's stderr no longer reaches the step log: it is redirected so a
+          # kept-alive VM cannot hold the step's stdout open.
+          qemu=$(sed -n 's/^\[smoke\] qemu log preserved: //p' \
+                 /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
+          [ -n "$qemu" ] && cp "$qemu" /tmp/cell21-artifacts-$GITHUB_RUN_ID/qemu.log || true
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: e2e-artifacts-cell-21-${{ github.run_id }}
+          path: /tmp/cell21-artifacts-${{ github.run_id }}
+          if-no-files-found: warn
+          # Matches the predastore benchmark in cell-23: a night's numbers are
+          # only useful against the nights before them.
+          retention-days: 90
+
+      - name: Teardown
+        if: always()
+        working-directory: mulga/scripts/iso-builder
+        run: |
+          # No env file means the workload step never ran, so there is nothing
+          # of its to clean up.
+          if [ -f /tmp/cell21.env ]; then
+            ./test/test-workload.sh --env /tmp/cell21.env --cleanup || true
+          fi
+          sudo pkill -f spinifex-smoke-test || true
+          rm -rf /tmp/cell21-artifacts-$GITHUB_RUN_ID /tmp/cell21.env \
+                 /tmp/smoke-$GITHUB_RUN_ID.log "$BENCH_OUT"
+          # Runs before BENCH_OUT existed left a timestamped directory behind
+          # every night. Sweep anything older than a day.
+          find /tmp -maxdepth 1 -name 'spinifex-benchmark-*' -type d -mmin +1440 \
+               -exec rm -rf {} + 2>/dev/null || true
+          # KEEP_NODE leaves the node's disk and serial log behind for us.
+          sleep 2
+          rm -f "$TMPDIR"/tmp.*.qcow2 "$TMPDIR"/tmp.*.log "$TMPDIR"/tmp.*-node.log
+
+  iso-multi:
+    name: cell-22 (iso/dhcp/three-node)
+    # After cell-21 rather than beside it: same concurrency group, and running
+    # them together would need 32 GiB on one box.
+    needs: [iso-single]
+    if: >-
+      always() && (
+      github.event_name == 'schedule' ||
+      inputs.cells == '' ||
+      contains(format(',{0},', inputs.cells), ',22,') )
+    runs-on: [self-hosted, ci-multi]
+    timeout-minutes: 90
+    concurrency:
+      group: iso-qemu-rack
+      cancel-in-progress: false
+    env:
+      MEM: 8G
+      SMP: 8
+      PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
+      EXT_POOL_START: 192.168.0.235
+      EXT_POOL_END: 192.168.0.239
+      EXT_GATEWAY: 192.168.1.1
+      EXT_PREFIX_LEN: 23
+    steps:
+      - name: Clean Workspace
+        run: |
+          sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+          # Pinned so the per-node serial logs can be collected afterwards, and
+          # kept beside TMPDIR rather than in /tmp: it also holds nine 40G node
+          # disks, which is what the tmpfs assertion below is about.
+          echo "WORK_DIR=$E2E_ROOT/work/cell22-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          path: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ env.GIT_BRANCH }}
+          path: mulga/spinifex
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ env.GIT_BRANCH }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: mulga/spinifex
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: mulga/viperblock
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: mulga/predastore
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: mulga/northstar
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: mulga/bluebottle
+
+      # 3 x 8 GiB plus 9 sparse 40G disks, before the instances the cluster
+      # launches on top.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          free_mb=$(free -m | awk '/^Mem:/ {print $7}')
+          [ "$free_mb" -ge 26624 ] ||
+            { echo "::error::need 26 GiB available, have ${free_mb} MiB"; exit 1; }
+          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
+          [ "$fstype" != tmpfs ] ||
+            { echo "::error::$TMPDIR is tmpfs — nine node disks would be in RAM"; exit 1; }
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          [ "$avail_gb" -ge 120 ] ||
+            { echo "::error::need 120 GiB on $TMPDIR ($fstype), have ${avail_gb} GiB"; exit 1; }
+
+      - name: Build artefacts
+        working-directory: mulga/scripts/iso-builder
+        run: make build-rootfs build-squashfs build-initramfs
+
+      # Four phases: install three nodes, convert to one cluster, assert it
+      # formed (including the external pool), then pull node 1's power and
+      # assert the survivors hold quorum.
+      #
+      # Phase 5 (test-workload.sh, as cell-21 runs it) is not wired yet: the
+      # nodes are still on user-mode NAT and reachable only through the
+      # harness's own port forwards, and test-cluster.sh does not open the
+      # formation port. Both land with mulga-yxtda and the bridged WAN mode;
+      # until then this cell proves the cluster forms and survives, not that a
+      # customer can use it.
+      - name: Install, form and verify the cluster
+        working-directory: mulga/scripts/iso-builder
+        run: |
+          set -euo pipefail
+          ./test/test-cluster.sh 2>&1 | tee /tmp/cell22-$GITHUB_RUN_ID.log
+
+      - name: Collect logs
+        if: always()
+        run: |
+          mkdir -p /tmp/cell22-artifacts-$GITHUB_RUN_ID
+          cp /tmp/cell22-$GITHUB_RUN_ID.log /tmp/cell22-artifacts-$GITHUB_RUN_ID/ ||
+            echo "::warning::no cluster log to collect"
+          # Serial only. The same directory holds the nodes' disk images, which
+          # are gigabytes and must not go into an artifact.
+          cp "$WORK_DIR"/*-serial.log /tmp/cell22-artifacts-$GITHUB_RUN_ID/ 2>/dev/null ||
+            echo "no per-node serial logs — the run passed and cleaned up after itself"
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: e2e-artifacts-cell-22-${{ github.run_id }}
+          path: /tmp/cell22-artifacts-${{ github.run_id }}
+          if-no-files-found: warn
+          retention-days: 90
+
+      - name: Teardown
+        if: always()
+        run: |
+          sudo pkill -f spinifex-cluster-test || true
+          rm -rf /tmp/cell22-artifacts-$GITHUB_RUN_ID /tmp/cell22-$GITHUB_RUN_ID.log \
+                 "$WORK_DIR"
+
+  # Predastore throughput ("cell 23"). Predastore has no runners of its own and
+  # spinifex depends on it, so the benchmark lives here and checks predastore
+  # out as a sibling.
+  #
+  # needs: is the whole point. A throughput benchmark sharing a host with a cell
+  # measures the neighbour, and ci-single and ci-multi both resolve to
+  # bottlebrush, so "a different label" would not have separated them. needs: on
+  # a matrix job waits for every leg, so this starts on a quiet machine.
+  #
+  # always() because a failed cell must not silently skip the benchmark and
+  # leave a green-looking night with no number in it.
+  performance:
+    name: cell-23 (predastore/throughput)
+    needs: [matrix, baremetal, iso-single, iso-multi]
+    if: >-
+      always() && (
+      github.event_name == 'schedule' ||
+      inputs.cells == '' ||
+      contains(format(',{0},', inputs.cells), ',23,') )
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 60
+    env:
+      # Two-minute samples. The 30s smoke preset is too short to compare nights.
+      PERF_PRESET: compare
+    steps:
+      - name: Clean Workspace
+        run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+          echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
+
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      # Which predastore branch to measure: the same-named branch when one
+      # exists, dev otherwise. Matches how the e2e path already picks siblings,
+      # so a cross-repo change is benchmarked as a set.
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      # The benchmark works in $TMPDIR and predastore refuses writes below a 5%
+      # free-space watermark. Without this the symptom is several hundred
+      # "storage pool is at capacity" lines from warp and no statement of how
+      # much space there actually was.
+      #
+      # TMPDIR is pointed at the runner's disk above, and the tmpfs assertion is
+      # the one that matters: a storage benchmark on a RAM-backed filesystem
+      # measures memcpy. It would report a number that looks excellent, sit in
+      # the baseline, and never notice a disk regression.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
+          case "$fstype" in
+            tmpfs|ramfs)
+              echo "::error::$TMPDIR is $fstype — a storage benchmark there measures RAM"
+              exit 1 ;;
+          esac
+          # Work dirs from runs that were killed before their cleanup trap ran.
+          # -mtime +0 so a concurrent run's directory is never in scope.
+          find "$TMPDIR" -maxdepth 1 -name 'predastore-e2e-performance.*' \
+               -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          pct_used=$(df --output=pcent "$TMPDIR" | tail -1 | tr -dc '0-9')
+          if [ "$pct_used" -ge 90 ] || [ "$avail_gb" -lt 20 ]; then
+            echo "::error::$TMPDIR ($fstype) is ${pct_used}% full with ${avail_gb} GiB free."
+            echo "::error::The benchmark needs headroom above predastore's 5% watermark."
+            exit 1
+          fi
+
+      # PERF_PORT_OFFSET defaults to 10000, so the benchmark's own nodes do not
+      # collide with a cluster left on the default ports.
+      - name: Run throughput benchmark
+        working-directory: predastore
+        run: |
+          set -euo pipefail
+          # Into the results directory, so the artifact reads on its own rather
+          # than needing the job log it was printed into. The benchmark only
+          # ever creates this directory, so a log placed here first survives.
+          mkdir -p "$PERF_RESULTS_ROOT"
+          make e2e-performance 2>&1 | tee "$PERF_RESULTS_ROOT/e2e-performance.log"
+
+      # Report-only. The baseline comparison and the -10% gate are mulga-wpjsx;
+      # until the run-to-run spread is known from real nights, a threshold would
+      # be a guess, and a gate that cries wolf gets muted.
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: predastore-perf-${{ github.run_id }}
+          path: ${{ env.PERF_RESULTS_ROOT }}
+          if-no-files-found: warn
+          retention-days: 90
+
+      - name: Clean up results
+        if: always()
+        run: rm -rf "$PERF_RESULTS_ROOT"
+
+  # Fault injection, not throughput. Cell-23 asks how fast predastore is when
+  # every node is healthy; this asks whether it survives one that is not.
+  #
+  # After cell-23 rather than beside it: both want a quiet single runner, and a
+  # benchmark sharing a machine with a fault injection run measures the wrong
+  # thing. The port offset keeps them apart even so.
+  stress:
+    name: cell-24 (predastore/fault-injection)
+    needs: [matrix, baremetal, iso-single, iso-multi, performance]
+    if: >-
+      always() && (
+      github.event_name == 'schedule' ||
+      inputs.cells == '' ||
+      contains(format(',{0},', inputs.cells), ',24,') )
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 30
+    env:
+      # Freezes a follower. STRESS_HOST=leader is a known failure: a gate keeps
+      # the frozen node first in its meta read order and pays the full client
+      # timeout per key, so listing slows in proportion to object count.
+      STRESS_HOST: follower
+      # Clear of cell-23's default 10000, so the two never collide on a runner
+      # that lets them overlap.
+      STRESS_PORT_OFFSET: "20000"
+    steps:
+      - name: Clean Workspace
+        run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+          echo "STRESS_RESULTS_ROOT=$E2E_ROOT/predastore-stress-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
+
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      # A frozen node is indistinguishable from a slow one, so a run on a
+      # loaded or nearly full disk fails as a timeout and reads as a real
+      # fault. The checks are the cheap way to tell those apart afterwards.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          find "$TMPDIR" -maxdepth 1 -name 'predastore-e2e-stress.*' \
+               -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          if [ "$avail_gb" -lt 10 ]; then
+            echo "::error::$TMPDIR has ${avail_gb} GiB free; the run needs headroom above predastore's 5% watermark."
+            exit 1
+          fi
+
+      # Leftovers from a run killed before its cleanup trap: a surviving node
+      # holds its ports and fails this run somewhere unrelated.
+      - name: Clear stale nodes
+        run: pkill -KILL -f '/bin/s3d -config' || true
+
+      - name: Run fault injection
+        working-directory: predastore
+        run: |
+          set -euo pipefail
+          # events.txt carries the timeline and the FAIL line, but not the build
+          # output or anything Warp wrote before the harness took over. The
+          # harness only creates this directory, so a log placed here survives.
+          mkdir -p "$STRESS_RESULTS_ROOT"
+          make e2e-stress 2>&1 | tee "$STRESS_RESULTS_ROOT/e2e-stress.log"
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: predastore-stress-${{ github.run_id }}
+          path: ${{ env.STRESS_RESULTS_ROOT }}
+          if-no-files-found: warn
+          retention-days: 90
+
+      - name: Clean up
+        if: always()
+        run: |
+          pkill -KILL -f '/bin/s3d -config' || true
+          rm -rf "$STRESS_RESULTS_ROOT"

.github/workflows/e2e-suites.yml

@@ -394,6 +394,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "E2E ${{ matrix.suite }} results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: single
+
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -796,6 +806,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "Multi-Node E2E ${{ matrix.suite }} results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: multi
+
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze

.github/workflows/e2e.yml

@@ -388,7 +388,7 @@ jobs:
                echo \"::group::suite \$SUITE\"; \
                printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
-               case \$SUITE in rds) TMO=45m ;; *) TMO=30m ;; esac; \
+               case \$SUITE in rds) TMO=50m ;; *) TMO=30m ;; esac; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
                  ./\$SUITE.test -test.v -test.timeout=\$TMO ${TEST_RUN_FLAG} \
@@ -464,6 +464,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "E2E results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: single
+
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -903,6 +913,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "Multi-Node E2E results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: multi
+
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze

.github/workflows/gpu-e2e.yml

@@ -287,6 +287,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "GPU E2E results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always() && steps.gate.outputs.paused != 'true'
+        continue-on-error: true
+        uses: ./.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: gpu
+
       - name: Analyze e2e failures
         if: always() && steps.gate.outputs.paused != 'true'
         uses: ./.github/actions/e2e-analyze

.github/workflows/publish-system-ami.yml

@@ -17,10 +17,11 @@ name: Publish System AMIs
 #   spx admin images import --name spinifex-rds-postgres
 #   spx admin images import --name spinifex-rds-mariadb
 #
-# Manual-only: publishing overwrites the mirror artifact, so it is a deliberate
-# action, not something every push should trigger. The `variant` input scopes
-# which images build — the Ubuntu GPU legs are 16G and much slower (NVIDIA
-# driver download + DKMS build), so publishing only the CPU AMIs stays cheap.
+# Publishing overwrites the mirror artifact, so it is a deliberate action, not
+# something every push should trigger: it runs manually, or from release.yml
+# once a tag has passed its gates. The `variant` input scopes which images
+# build — the Ubuntu GPU legs are 16G and much slower (NVIDIA driver download +
+# DKMS build), so publishing only the CPU AMIs stays cheap.
 
 on:
   workflow_dispatch:
@@ -38,6 +39,16 @@ on:
         description: "Build + convert only; skip the R2 upload"
         type: boolean
         default: false
+  workflow_call:
+    inputs:
+      variant:
+        description: "Which images to build (all|cpu|gpu|rds)"
+        type: string
+        default: all
+      dry_run:
+        description: "Build + convert only; skip the R2 upload"
+        type: boolean
+        default: false
 
 permissions:
   contents: read
@@ -101,6 +112,20 @@ jobs:
           ref: dev
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: dev
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: dev
+          path: bluebottle
+
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
@@ -118,7 +143,7 @@ jobs:
         # "go.work already exists".
         run: |
           rm -f go.work go.work.sum
-          go work init ./spinifex ./viperblock ./predastore
+          go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Install build + upload tooling
         run: |

.github/workflows/release.yml

@@ -28,13 +28,25 @@ jobs:
           repository: mulgadc/viperblock
           path: viperblock
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          path: bluebottle
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
 
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
-          go-version: "1.26.5"
+          go-version: "1.27.0"
 
       - name: Build microVM artifacts (lb-agent + kernel + initramfs)
         run: cd spinifex && make build-microvm-image
@@ -73,6 +85,18 @@ jobs:
           repository: mulgadc/viperblock
           path: viperblock
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          path: bluebottle
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
 
@@ -207,6 +231,17 @@ jobs:
             echo "Release ${TAG} promoted to latest"
           fi
 
+  # Refresh the system AMIs on the R2 mirror only once the release is out of
+  # draft. Shares the ci-single runner with `iso`, so it queues behind it.
+  system-images:
+    needs: promote
+    permissions:
+      contents: read
+    uses: ./.github/workflows/publish-system-ami.yml
+    with:
+      variant: all
+    secrets: inherit
+
   # Annotate the (still-draft) release if either gate failed.
   mark-failed:
     needs: [ verify, baremetal ]
@@ -268,6 +303,18 @@ jobs:
           repository: mulgadc/viperblock
           path: viperblock
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          path: bluebottle
+
       - name: Checkout Mulga iso-builder
         uses: actions/checkout@v7
         with:
@@ -286,7 +333,7 @@ jobs:
           cache: false
 
       - name: Configure Go workspace
-        run: go work init ./spinifex ./predastore ./viperblock
+        run: go work init ./spinifex ./predastore ./viperblock ./northstar ./bluebottle
 
       - name: Install ISO builder dependencies
         run: |

.github/workflows/segscan-oracle.yml

@@ -51,6 +51,20 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
       - uses: actions/create-github-app-token@v3
         id: app-token
         with:
@@ -73,7 +87,7 @@ jobs:
           go-version-file: "spinifex/go.mod"
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Run Segscan Oracle Test
         env:

.github/workflows/spinifex.yml

@@ -53,14 +53,65 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
+      # the last go.sum change and never refreshed it. Both caches are managed
+      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
-          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
+          cache: false
+
+      # Module contents follow the five go.sum files, so a content key is exact.
+      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
+      # dependencies out of the key and re-downloading on every run.
+      - name: Cache Go modules
+        uses: actions/cache@v6
+        with:
+          path: ~/go/pkg/mod
+          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
+          restore-keys: gomod-
+
+      # Build artifacts follow source, so the key rolls per run and falls back to
+      # this job's newest entry. Keyed per job because -race, -covermode=atomic
+      # and plain builds produce disjoint entries and would evict each other.
+      - name: Cache Go build
+        uses: actions/cache@v6
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
+
+      # The EBS provider contract suite needs a real implementation on each
+      # side of the boundary: qemu-img/qemu-nbd/qemu-io back qemunbdd, which
+      # shares no code with viperblock, and nbdkit plus viperblock's plugin
+      # back viperblockd, the provider production actually uses. Without them
+      # the suite runs against MemoryProvider alone. nbdkit-plugin-dev carries
+      # the headers and nbdkit.pc that cgo needs to build that plugin, and
+      # libnbd-bin supplies the third-party NBD client (nbdinfo/nbdcopy) that
+      # judges the published export by something outside this codebase.
+      - name: Install conformance tools
+        run: sudo apt-get update && sudo apt-get install -y --no-install-recommends qemu-utils nbdkit nbdkit-plugin-dev libnbd-bin
+
+      - name: Build viperblock nbdkit plugin
+        run: make -C viperblock go_build_nbd
 
       - name: Build Spinifex
         run: make -C spinifex build
@@ -69,10 +120,13 @@ jobs:
         run: make -C spinifex manifest-check manifest-lint
 
       - name: Run Tests with Coverage
-        run: make -C spinifex test-cover
+        env:
+          SPINIFEX_REQUIRE_CONFORMANCE_TOOLS: "1"
+          VIPERBLOCK_NBDKIT_PLUGIN: ${{ github.workspace }}/viperblock/lib/nbdkit-viperblock-plugin.so
+        run: make -C spinifex test-cover GOTESTFLAGS=-count=1
 
       - name: Run Action Tests
-        run: make -C spinifex test-actions
+        run: make -C spinifex test-actions GOTESTFLAGS=-count=1
 
       - name: Check Diff Coverage
         run: cd spinifex && scripts/diff-coverage.sh coverage.out --base HEAD~1
@@ -114,14 +168,51 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
+      # the last go.sum change and never refreshed it. Both caches are managed
+      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
-          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
+          cache: false
+
+      # Module contents follow the five go.sum files, so a content key is exact.
+      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
+      # dependencies out of the key and re-downloading on every run.
+      - name: Cache Go modules
+        uses: actions/cache@v6
+        with:
+          path: ~/go/pkg/mod
+          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
+          restore-keys: gomod-
+
+      # Build artifacts follow source, so the key rolls per run and falls back to
+      # this job's newest entry. Keyed per job because -race, -covermode=atomic
+      # and plain builds produce disjoint entries and would evict each other.
+      - name: Cache Go build
+        uses: actions/cache@v6
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Run Integration Tests
         run: make -C spinifex test-integration
@@ -157,17 +248,54 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
+      # the last go.sum change and never refreshed it. Both caches are managed
+      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
-          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
+          cache: false
+
+      # Module contents follow the five go.sum files, so a content key is exact.
+      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
+      # dependencies out of the key and re-downloading on every run.
+      - name: Cache Go modules
+        uses: actions/cache@v6
+        with:
+          path: ~/go/pkg/mod
+          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
+          restore-keys: gomod-
+
+      # Build artifacts follow source, so the key rolls per run and falls back to
+      # this job's newest entry. Keyed per job because -race, -covermode=atomic
+      # and plain builds produce disjoint entries and would evict each other.
+      - name: Cache Go build
+        uses: actions/cache@v6
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Run Tests with Race Detector
-        run: make -C spinifex test-race
+        run: make -C spinifex test-race GOTESTFLAGS=-count=1
 
   lint_and_security:
     name: Lint and Security
@@ -200,19 +328,56 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
+      # the last go.sum change and never refreshed it. Both caches are managed
+      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
-          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
+          cache: false
+
+      # Module contents follow the five go.sum files, so a content key is exact.
+      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
+      # dependencies out of the key and re-downloading on every run.
+      - name: Cache Go modules
+        uses: actions/cache@v6
+        with:
+          path: ~/go/pkg/mod
+          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
+          restore-keys: gomod-
+
+      # Build artifacts follow source, so the key rolls per run and falls back to
+      # this job's newest entry. Keyed per job because -race, -covermode=atomic
+      # and plain builds produce disjoint entries and would evict each other.
+      - name: Cache Go build
+        uses: actions/cache@v6
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Lint
         uses: golangci/golangci-lint-action@v9
         with:
-          version: v2.12.2
+          version: v2.13.0
           working-directory: spinifex
 
       - name: Govulncheck

.gitignore

@@ -34,9 +34,10 @@ Thumbs.db
 # Environment
 .env
 .venv/
+.cache/
+
+# VScode
+.vscode/
 
 # Test reports
 *-report.txt
-
-# Per-worktree golangci-lint cache
-/.cache/

.golangci.yml

@@ -26,7 +26,7 @@ linters:
     - errorlint # catches ==/!= on errors, type assertions on errors, and %v instead of %w — all break on wrapped errors
     - exptostd # detects golang.org/x/exp/ functions replaceable by stdlib
     - fatcontext # detects nested contexts in loops
-    - forbidigo # blocks the legacy JetStream/KV symbols
+    - forbidigo # blocks forbidden identifiers
     - gocheckcompilerdirectives # validates go compiler directive comments (//go:)
     - godot # checks that declaration comments end in a period
     - goprintffuncname # checks that printf-like functions are named with f at the end
@@ -69,6 +69,18 @@ linters:
             - pkg: math/rand$
               desc: Use math/rand/v2 instead, see https://go.dev/blog/randv2
 
+        # The EC2 control plane reaches block storage through ebsprovider, not
+        # by importing the engine. viperblock.New starts a chunk uploader, so
+        # every construction is a writer against a volume viperblockd serves.
+        # Nothing under handlers/ may import viperblock, in any form, tests
+        # included. A types-only import counts: one became an engine import.
+        ec2-control-plane:
+          files:
+            - '**/spinifex/handlers/**'
+          deny:
+            - pkg: github.com/mulgadc/viperblock
+              desc: Use the ebsprovider.EBSProvider interface; the control plane must not construct or name viperblock types.
+
     embeddedstructfieldcheck:
       forbid-mutex: true
 
@@ -80,6 +92,8 @@ linters:
     forbidigo:
       analyze-types: true
       forbid:
+        - pattern: '^panic$'
+          msg: panic terminates the process; return an error or explicitly justify the panic.
         - pattern: '\.JetStream$'
           pkg: '^github\.com/nats-io/nats\.go$'
           msg: legacy nc.JetStream() is superseded; construct with jetstream.New(nc).
@@ -104,6 +118,7 @@ linters:
 
     modernize:
       disable:
+        - embedlit # its fix promotes embedded fields into the outer literal, which does not compile
         - newexpr # don't replace aws.String() with new() — AWS SDK idiom
         - stringsbuilder # don't replace small string concatenations with strings.Builder
 
@@ -154,6 +169,7 @@ linters:
           - bodyclose
           - errcheck
           - errchkjson
+          - forbidigo
           - gosec
 
       # spinifex-specific: gosec exclusions

Makefile

@@ -1,10 +1,10 @@
 GO_PROJECT_NAME := spx
 SHELL := /bin/bash
 
-# golangci-lint's default cache (~/.cache/golangci-lint) is keyed by module
-# path, not worktree path, so parallel worktrees of this module collide and
-# surface each other's issues. Scope it per worktree.
-export GOLANGCI_LINT_CACHE := $(CURDIR)/.cache/golangci-lint
+# GOFIPS140 is a global build-config change: it alters every package's action ID,
+# stdlib included, so a target that omits it shares no build-cache entry with one
+# that sets it. Exported once here rather than per-recipe so the two can't drift.
+export GOFIPS140 := v1.0.0
 
 # Detect architecture for cross-platform support
 ARCH := $(shell uname -m)
@@ -63,19 +63,23 @@ LDFLAGS := -s -w -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Version=$(VERSI
 
 go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
-	GOFIPS140=v1.0.0 go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
+	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
 
 build-installer:
 	@echo -e "\n....Building spinifex-installer"
-	GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
+	go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
 
 build-lb-agent:
 	@echo -e "\n....Building lb-agent (static)"
-	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
+	CGO_ENABLED=0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
 
 build-ecs-agent: ## Build the ecs-agent (ships in the ECS-AMI guest; not in host `build`)
 	@echo -e "\n....Building ecs-agent (static)"
-	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w -X main.version=$(VERSION)" -o ./bin/ecs-agent ./cmd/ecs-agent
+	CGO_ENABLED=0 go build -ldflags "-s -w -X main.version=$(VERSION)" -o ./bin/ecs-agent ./cmd/ecs-agent
+
+build-loadgen: ## Build spx-loadgen (drives the stress harness's measured load; not shipped to nodes)
+	@echo -e "\n....Building spx-loadgen"
+	GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/spx-loadgen ./cmd/spx-loadgen
 
 build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-node)
 ifndef IMAGE
@@ -140,9 +144,22 @@ install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share
 # the pre-commit gate: manifest checks, lint, vuln, and the unit and e2e-harness tiers.
 # integration and race tests skipped to keep quick, they run in CI.
 preflight:
-	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-harness
+	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-package-check test-harness test-build-scripts
 	@echo -e "\n ✅ Preflight passed — safe to commit."
 
+# Shell suites + shellcheck for build/scripts/, the systemd-unit helpers that
+# ship on every node (unlike scripts/images/, kept in preflight: a wrong
+# restart decision here is an outage, not asset churn).
+test-build-scripts:
+	@echo -e "\n....Running build/scripts/**/*_test.sh...."
+	@for t in $$(find build/scripts -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		sh "$$t" || exit 1; \
+	done
+	@echo -e "\n....Running shellcheck over build/scripts/**/*.sh...."
+	shellcheck -S warning $$(find build/scripts -name '*.sh' | sort)
+	@echo "  test-build-scripts ok"
+
 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
 # default `go test ./spinifex/...`. Runs with mocked AWS clients — no
 # infrastructure required, safe to run in CI without a cluster.
@@ -155,9 +172,13 @@ test-harness:
 AWS_MODEL_CONFORMANCE_REPORT ?= $(CURDIR)/.cache/aws-model-conformance-report.txt
 AWS_MODEL_CONFORMANCE_MODE ?= fail
 AWS_MODEL_OPERATION_COVERAGE_REPORT ?= $(CURDIR)/docs/aws-model-operation-coverage.md
+# -count=1 is load-bearing: the conformance report is written by the test binary,
+# so a cached pass skips the run, leaves no report, and the cat below fails the
+# target. It also keeps the conformance gate honest — a cached result would mean
+# the check never ran against this commit.
 test-integration:
 	@echo -e "\n....Running in-process integration tests...."
-	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -count=1 -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
 	@cat $(AWS_MODEL_CONFORMANCE_REPORT)
 	@$(MAKE) --no-print-directory generate-aws-model-coverage
 	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_REPORT)"
@@ -198,25 +219,30 @@ manifest-lint-update:
 # Run unit tests
 test:
 	@echo -e "\n....Running tests for $(GO_PROJECT_NAME)...."
-	GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/...
+	LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/...
+
+# Empty locally, where reusing a cached result between runs is the point.
+# CI passes -count=1 so a green run means the suite executed against that commit
+# rather than replaying results the persisted build cache carried over.
+GOTESTFLAGS ?=
 
 # Run unit tests with coverage profile
 COVERPROFILE ?= coverage.out
 test-cover:
 	@echo -e "\n....Running tests with coverage for $(GO_PROJECT_NAME)...."
-	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
 	@scripts/check-coverage.sh $(COVERPROFILE) $(QUIET)
 
 # Run unit tests with race detector
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
-	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... $(_RACEQ)
 
 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
 test-actions:
 	@echo -e "\n....Running action tests...."
-	LOG_IGNORE=1 go test -timeout 60s ./.github/actions/...
+	LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 60s ./.github/actions/...
 
 # Shell suites + shellcheck for scripts/images/ and images/mkosi.profiles/
 # helpers baked into system images. Kept out of `preflight` (a dedicated CI
@@ -255,6 +281,10 @@ test-images:
 diff-coverage: test-cover
 	@QUIET=$(QUIET) scripts/diff-coverage.sh $(COVERPROFILE)
 
+# Check that newly added test files use an external test package
+test-package-check:
+	@QUIET=$(QUIET) scripts/check-test-package.sh
+
 bench:
 	@echo -e "\n....Running benchmarks for $(GO_PROJECT_NAME)...."
 	LOG_IGNORE=1 go test -benchmem -run=. -bench=. ./...
@@ -309,9 +339,9 @@ install-system:
 		ovn-central ovn-host openvswitch-switch dhcpcd-base
 
 install-go:
-	@echo -e "\n....Installing Go 1.26.5 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.27.0 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.26.5.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.27.0.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi
@@ -394,7 +424,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
+.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

README.md

@@ -3,7 +3,7 @@
 </p>
 
 <p align="center">
-  <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.26+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
+  <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.27+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
   <a href="https://mulgadc.com"><img src="https://img.shields.io/badge/home-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
 </p>
@@ -12,10 +12,11 @@
   <a href="#what-is-spinifex">What is Spinifex?</a> ·
   <a href="#the-platform">Platform</a> ·
   <a href="#three-ways-to-deploy">Deploy</a> ·
-  <a href="#aws-compatibility">AWS services</a> ·
+  <a href="#aws-compatibility">AWS Compatibility</a> ·
+  <a href="#live-demo">Live Demo</a> ·
   <a href="#core-components">Components</a> ·
   <a href="#architecture-at-a-glance">Architecture</a> ·
-  <a href="#installation">Installation</a> ·
+  <a href="#self-hosted-installation">Installation</a> ·
   <a href="https://docs.mulgadc.com">Docs</a>
 </p>
 
@@ -29,11 +30,9 @@ Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your
 
 Spinifex is an open-source infrastructure platform that recreates the AWS service surface (EC2, EBS, S3, VPC, IAM) on bare-metal, edge, and on-premises environments. Run cloud-native software without a hyperscaler.
 
-Most AWS alternatives wrap an API and call it a cloud. We rebuilt the engineering underneath: distributed object storage with erasure coding, block storage built to survive failure, and compute on bare metal. That depth is the reason your software runs unchanged, instead of rewritten.
+Built for teams that need:
 
-It's built for teams that need:
-
-- The AWS tooling they already use (AWS CLI, SDKs, Terraform), with nothing to rewrite
+- The AWS tooling you already use (AWS CLI, SDKs, Terraform), with nothing to rewrite
 - Full control of the stack: your hardware, your network, your data, your keys
 - A cloud that keeps running offline, through disconnection, with no external control plane
 - An open core (AGPL-3.0), auditable and yours to keep, with no lock-in
@@ -48,51 +47,69 @@ You change *where* your software runs, not *what* it is.
 
 From commodity hardware up to unmodified AWS tooling, every layer is replaceable and yours to own.
 
-## Three ways to deploy
+## Three Ways To Deploy
 
-One AWS-compatible surface, three deployment shapes. Pick the one that matches your reality; the platform on top is the same.
+The same AWS-compatible platform runs in three shapes. Pick the one that matches your hardware.
 
 ### Neocloud
 
-Lift and shift onto a partner Neocloud. Move workloads off the hyperscalers and onto our Neocloud partner ecosystem without rewriting them. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
+Lift and shift onto a partner Neocloud without rewriting workloads. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
 
 ### On-premise
 
-Bring your own hardware and host Spinifex in your own data centre. A real multi-node HA cluster integrated with your storage and networking, with full control of stack, data, and jurisdiction. A predictable bill, no egress surprises, an auditable open-source core.
+Host Spinifex on your own hardware in your own data centre. You get a multi-node HA cluster integrated with your storage and networking, so the stack, the data, and the jurisdiction stay yours.
 
 ### Edge
 
 Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels, factories, and clinics. Compute next to the data, running through disconnection, on hardware you own. The same AWS APIs your software already uses.
 
-## AWS compatibility
+## AWS Compatibility
 
-Speak the AWS API surface, natively. The AWS SDKs, AWS CLI, and Terraform: everything you deploy on AWS deploys on Spinifex unchanged. At the edge, on-premise, or on a partner Neocloud.
+Spinifex speaks the AWS API surface. The AWS SDKs, AWS CLI, and Terraform all work unchanged, whether you deploy at the edge, on-premise, or on a partner Neocloud.
 
 | Service | What it is | Status |
 | --- | --- | --- |
 | **EC2** | Compute | Available |
-| **EBS** | Block storage | Available |
-| **S3** | Object storage | Available |
+| **EBS** | Block Storage | Available |
+| **S3** | Object Storage | Available |
 | **VPC** | Networking | Available |
-| **IAM** | Identity & auth | Available |
-| **ALB / NLB** | Load balancers | Available |
+| **IAM** | Identity & Auth | Available |
+| **ALB / NLB** | Load Balancers | Available |
 | **EKS** | Kubernetes | Available |
-| **ECR** | Container registry | Available |
-| **ECS** | Container service | Available |
+| **ECR** | Container Registry | Available |
+| **ECS** | Container Service | Available |
 | **RDS** | Databases | Available |
 | **Bedrock** | AI Deployment | Q3 2026 |
 
 Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
 
+## Live Demo
+
+Create a live sandbox in under a minute. The fastest way to experience the stack. Try Spinifex against one of your own workloads on infrastructure we run for you.
+
+* Live in under a minute. No card, no install.
+* Yours for 72 hours to test and experiment.
+* Real AWS-compatible endpoint, access keys and console.
+* EC2, EBS, S3, VPC, IAM, EKS and RDS, ready to call.
+* Point your existing AWS CLI or Terraform straight at it.
+
+Sign up at [https://mulgadc.com/signup](https://mulgadc.com/signup)
+
 ## Core Components
 
-### Spinifex (Compute Service – EC2 Alternative)
+### Spinifex (Compute and Platform Services – EC2, EKS, ECS, RDS)
 
-Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs similar to EC2. It manages lifecycle operations like start, stop, and terminate, using QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs using the AWS CLI, SDKs, or Terraform—without needing Kubernetes or heavyweight orchestrators. Keep in mind, you can also setup a Kubernetes environment using Spinifex with underlying instances.
+Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing an EC2 compatible API. It manages lifecycle operations like start, stop, and terminate through QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs with the AWS CLI, SDKs, or Terraform, without Kubernetes or a heavyweight orchestrator.
 
-- EC2-like VM management on bare metal
-- Launches with cloud-init metadata support
-- Works with standard AWS tooling
+The managed services build on that same instance layer. An EKS node group, an ECS container instance, and an RDS database are Spinifex VMs on your own hardware, each driven through its own AWS API.
+
+- **EC2** – VM lifecycle on bare metal, cloud-init and IMDS metadata, key pairs, tags, spot instances, capacity reservations, and GPU passthrough
+- **EKS** – Managed Kubernetes on k3s, with HA control planes, managed node groups, cluster add-ons, an OIDC provider, and IAM access entries
+- **ECS** – Task and service scheduling across container instances, with awsvpc ENIs, capacity providers, rolling deployments, and GPU-aware placement
+- **ECR** – Private container registry backed by Predastore, with content-addressed blobs deduplicated per account and lifecycle policies
+- **RDS** – Managed PostgreSQL and MariaDB, with parameter groups, automated backups, and DNS endpoints served by Northstar
+- **ALB / NLB** – Layer 7 load balancing on HAProxy and layer 4 on nginx `stream`, with health checks and ACM-issued listener certificates
+- **VPC and IAM** – OVN-backed networking and security groups, with roles, policies, and SigV4 auth applied across every service
 
 ### Viperblock (Block Storage – EBS Alternative)
 
@@ -101,17 +118,25 @@ Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs
 - Fast, durable virtual disks
 - Replication for resilience
 - Exposed over NBD or embedded in VMs
-- Supports high performance WAL logs using local NVMe drives to reduce IO traffic to S3.
-- In memory read/write block cache for blazing performance.
+- High performance WAL logs on local NVMe drives, reducing IO traffic to S3
+- In-memory read/write block cache
 
 ### Predastore (Object Storage – S3-Compatible)
 
-[Predastore](https://github.com/mulgadc/predastore) is a fully S3-compatible object storage system. It supports the AWS S3 API, including Signature V4 authentication, multipart uploads, and Terraform provisioning. Data is chunked and distributed across nodes using Reed-Solomon erasure coding, making it fault-tolerant and ideal for large-scale or low-bandwidth scenarios.
+[Predastore](https://github.com/mulgadc/predastore) is a fully S3-compatible object storage system. It supports the AWS S3 API, including Signature V4 authentication, multipart uploads, and Terraform provisioning. Predastore chunks data and distributes it across nodes using Reed-Solomon erasure coding, so it survives node loss and suits large-scale or low-bandwidth sites.
 
 - S3-compatible API and auth
 - Multipart uploads, streaming reads/writes
 - Data redundancy with Reed-Solomon encoding
 
+### Northstar (Authoritative DNS)
+
+[Northstar](https://github.com/mulgadc/northstar) is a lightweight authoritative DNS server that handles both internal service discovery and public-facing DNS for a Spinifex cluster. Zones are human-readable TOML files, served from the local filesystem or synced from Predastore, keeping DNS configuration in the same object storage as the rest of your infrastructure.
+
+- Authoritative DNS over UDP, TCP, and DNS-over-TLS
+- Service discovery for cluster components via SRV records
+- Zone files stored locally or in S3-compatible storage
+
 ## Architecture at a Glance
 
 <p align="center">
@@ -120,18 +145,9 @@ Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs
 
 Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless: scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Just systemd units, a NATS cluster, and your hardware. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
 
-## Key Features
+## Self-hosted Installation
 
-- **AWS-compatible APIs.** Use the AWS CLI, SDKs, and Terraform you already know. Repoint your endpoint and ship; nothing to rewrite.
-- **Zero cloud dependency.** Runs entirely on your hardware. No phone-home, no control plane, no external authority. Works fully offline.
-- **Bare-metal compute.** QEMU-based with direct hardware access. No hypervisor tax, no abstraction overhead.
-- **Built-in storage.** Block and object storage included: NVMe caching, Reed-Solomon erasure coding, and replication out of the box.
-- **Edge-first architecture.** Designed for disconnected, contested, and resource-constrained environments from day one.
-- **Open core, no lock-in.** AGPL-3.0 with a commercial option. Inspect it, modify it, deploy it. The platform is yours to keep.
-
-## Installation
-
-Installation requires an Ubuntu / Debian system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for maintaining and installing Spinifex.
+Installation requires an Ubuntu 26.04 or Debian 13 system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for installing Spinifex.
 
 ### Bare Metal ISO
 
@@ -141,16 +157,11 @@ The recommended installation is a [bootable x86 installer](https://iso.mulgadc.c
 curl -fLO https://iso.mulgadc.com/spinifex.iso
 ```
 
-Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware. The install guide walks through the full process.
-
+Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware.
 
 ### Single Node Install
 
-The installation is straightforward to set up and running on a single node for testing purposes. Debian 13 is currently supported, additional Linux distributions are on the immediate roadmap.
-
->*Prerequisite:* Linux bridge for networking.
-
-Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) prerequisites for setup details.
+>*Prerequisite:* Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) for setup details.
 
 ```bash
 curl -fsSL https://install.mulgadc.com | bash
@@ -168,36 +179,23 @@ aws ec2 describe-instance-types
 
 ### Development Setup
 
-For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation
-
-### Component Repositories
-
-Spinifex coordinates these independent components:
-
-- **[Predastore](https://github.com/mulgadc/predastore)** - S3-compatible object storage
-- **[Viperblock](https://github.com/mulgadc/viperblock)** - EBS-compatible block storage
-
-Each component can be developed independently. See component-specific documentation for focused development guides.
+For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation.
 
 ## Spinifex UI
 
-Spinifex ships with a built-in web console — an optional alternative to the AWS CLI, SDKs, and Terraform. If you're familiar with the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
+Spinifex ships with a built-in web console, an optional alternative to the AWS CLI, SDKs, and Terraform. If you know the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
 
 <p align="center">
-  <img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console — dashboard view" width="900">
+  <img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console dashboard" width="900">
 </p>
 
-The console is served by each node on port `3000` over TLS, and becomes available as soon as `spinifex.target` is up:
+Each node serves the console on port `3000` over TLS, and it comes up with `spinifex.target`:
 
-```bash
-open https://YOUR_NODE_IP:3000
-```
+- **Same API, different surface.** The UI issues the same AWS SigV4 calls the CLI does, so RBAC, audit trails, and IAM policies apply the same way.
+- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed. There is no separate user database.
+- **Self-hosted, works offline.** The Spinifex binary embeds the UI and each node serves it directly, with no external CDN or analytics calls.
 
-- **Same API, different surface.** Every action in the UI is the same AWS SigV4 call the CLI makes — so RBAC, audit trails, and IAM policies apply uniformly.
-- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed — no separate user database.
-- **Self-hosted, works offline.** The UI is embedded in the Spinifex binary and served from the node itself. No external CDN, no analytics calls, no cloud dependency.
-
-For the full walkthrough — first-time TLS certificate trust, login, and feature tour — see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
+For the full walkthrough, covering first-time TLS certificate trust, login, and a feature tour, see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
 
 ## Development Philosophy
 
@@ -205,16 +203,9 @@ For the full walkthrough — first-time TLS certificate trust, login, and featur
 
 Spinifex is developed by experienced infrastructure engineers with deep AWS expertise, including former AWS team members who understand the intricacies of building production-grade cloud services. Our team brings decades of combined experience from AWS, enterprise infrastructure, and edge computing environments.
 
-**Real-World Experience:**
-
-- Production AWS service development and operations
-- Large-scale infrastructure deployment and management
-- Edge computing and resource-constrained environments
-- Enterprise security and compliance requirements
-
 ### AI-Assisted Development
 
-While Spinifex is architected and implemented by experienced engineers, we leverage **Claude Code** (Anthropic's AI coding assistant) to accelerate certain development tasks. This approach combines human expertise with AI efficiency:
+Spinifex is architected and implemented by experienced engineers, with **Claude Code** accelerating certain development tasks. The split keeps the quality and reliability standards that production infrastructure demands:
 
 **How We Use Claude Code:**
 
@@ -230,12 +221,10 @@ While Spinifex is architected and implemented by experienced engineers, we lever
 - **Performance Optimization**: Real-world performance tuning and benchmarking
 - **Production Operations**: Deployment strategies and operational procedures
 
-This hybrid approach ensures Spinifex benefits from both proven engineering expertise and modern development acceleration, while maintaining the quality and reliability standards required for production infrastructure.
-
 ## Trademarks
 
 "AWS", "Amazon Web Services", and all related service names (EC2, EBS, S3, VPC, IAM, and others) are trademarks of Amazon.com, Inc. or its affiliates. Mulga Defense Corporation and Spinifex are independent and not affiliated with, endorsed by, or sponsored by Amazon.com, Inc. or Amazon Web Services, Inc. References to AWS services describe interoperability and compatibility only.
 
 ## License
 
-Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE). You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.
+Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE), with a commercial license available. You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.

build/Dockerfile.distro

@@ -1,6 +1,6 @@
 ARG VERSION=dev
 
-FROM golang:1.26-bookworm AS builder
+FROM golang:1.27-bookworm AS builder
 
 ARG VERSION
 
@@ -16,9 +16,11 @@ COPY predastore/go.mod predastore/go.sum ./predastore/
 COPY viperblock/go.mod viperblock/go.sum ./viperblock/
 COPY viperblock/nbd/go.mod viperblock/nbd/go.sum ./viperblock/nbd/
 COPY viperblock/nbd/libguestfs.org/nbdkit/go.mod ./viperblock/nbd/libguestfs.org/nbdkit/
+COPY northstar/go.mod northstar/go.sum ./northstar/
+COPY bluebottle/go.mod bluebottle/go.sum ./bluebottle/
 
 # Generate go.work (gitignored, so not in checkout)
-RUN cd spinifex && go work init . ../predastore ../viperblock ../viperblock/nbd ../viperblock/nbd/libguestfs.org/nbdkit
+RUN cd spinifex && go work init . ../predastore ../viperblock ../viperblock/nbd ../viperblock/nbd/libguestfs.org/nbdkit ../northstar ../bluebottle
 
 # Download dependencies (cached layer)
 RUN cd spinifex && go mod download
@@ -27,6 +29,8 @@ RUN cd spinifex && go mod download
 COPY spinifex/ ./spinifex/
 COPY predastore/ ./predastore/
 COPY viperblock/ ./viperblock/
+COPY northstar/ ./northstar/
+COPY bluebottle/ ./bluebottle/
 
 # Build spinifex binary (spx)
 RUN COMMIT=$(cd spinifex && git rev-parse --short HEAD 2>/dev/null || echo "unknown") && \

build/scripts/nats-js-watchdog.sh

@@ -18,22 +18,97 @@
 # So a health check alone misses case 2 entirely. Both probes run here, and a
 # sustained failure of either restarts the service. The multi-sample window
 # avoids restarting during transient startup/quorum 503s.
+#
+# Three more guards sit around that decision, because "restart when the
+# probes fail" is not safe on its own:
+#
+#   - Age gate: spinifex-nats.service is Type=simple, so it reports active the
+#     instant the process forks, long before JetStream finishes replaying its
+#     state. Acting before a recovery budget has elapsed restarts a server
+#     that is still coming up, not one that has failed.
+#   - Cooldown: a restart that did not fix things must not be retried by the
+#     very next run, seconds later.
+#   - Escalation: a restart that has not fixed things after several attempts
+#     is not going to fix it on the next one either, and an unrunnable spx
+#     must never be read as proof of a JetStream failure at all.
 
 set -eu
 
 HEALTH_URL="http://127.0.0.1:8222/healthz?js-enabled-only=true"
 SERVICE="spinifex-nats.service"
 SPX="${SPX_BIN:-/usr/local/bin/spx}"
-SAMPLES=3        # consecutive failing samples required before acting
-SAMPLE_GAP=2     # seconds between samples
-PROBE_TIMEOUT=5  # seconds for one canary round-trip
+SAMPLES="${SAMPLES:-3}"           # consecutive failing samples required before acting
+SAMPLE_GAP="${SAMPLE_GAP:-2}"     # seconds between samples
+PROBE_TIMEOUT="${PROBE_TIMEOUT:-5}"  # seconds for one canary round-trip
 
-# Only act on a running service: a stopped/activating unit is systemd's to manage,
-# and try-restart on it would either no-op or fight an in-flight (re)start.
-if ! systemctl is-active --quiet "$SERVICE"; then
+# Recovery budget: on casuarina NATS needed 3m49s between "Started" and
+# JetStream actually accepting writes, and the bead recorded ~6min elsewhere.
+# 600s clears both with margin.
+NATS_MIN_AGE="${NATS_MIN_AGE:-600}"
+
+# Minimum time between restarts this watchdog performs, independent of the
+# age gate: closes the case where a restart did not fix things and the very
+# next run pulls the trigger again before the new process could recover.
+RESTART_COOLDOWN="${RESTART_COOLDOWN:-600}"
+
+# A restart that has not fixed things after this many attempts is not going
+# to fix it on the next one either, and looping costs more than stopping.
+RESTART_ESCALATE_AT="${RESTART_ESCALATE_AT:-3}"
+
+STATE_DIR="${STATE_DIR:-/run/spinifex}"
+COOLDOWN_STAMP="$STATE_DIR/nats-js-watchdog.last-restart"
+RESTART_COUNT_FILE="$STATE_DIR/nats-js-watchdog.restart-count"
+# Marks that the escalation message has already been printed for the current
+# latch, so a node that stays broken says so once instead of once per tick.
+ESCALATED_STAMP="$STATE_DIR/nats-js-watchdog.escalated"
+
+# Monotonic clock source, so a wall-clock step during boot (NTP correcting an
+# unset RTC) cannot manufacture a false age or cooldown. Overridable so tests
+# can stub a clock without faking /proc.
+UPTIME_FILE="${UPTIME_FILE:-/proc/uptime}"
+
+uptime_seconds() {
+    awk '{printf "%d", $1}' "$UPTIME_FILE"
+}
+
+# nats_age_seconds prints how long $SERVICE has been running. It reads
+# ExecMainStartTimestampMonotonic rather than is-active: that property goes
+# true the instant Type=simple forks, long before JetStream has replayed its
+# state, so it cannot express the recovery window this gate exists to
+# protect. A unit systemd has never started prints 0, which also covers a
+# stopped/activating unit the way the old is-active guard did.
+nats_age_seconds() {
+    start_usec=$(systemctl show --property=ExecMainStartTimestampMonotonic --value "$SERVICE" 2>/dev/null) || start_usec=""
+    case "$start_usec" in
+        ''|*[!0-9]*) echo 0; return ;;
+    esac
+    if [ "$start_usec" -eq 0 ]; then
+        echo 0
+        return
+    fi
+    echo $(( $(uptime_seconds) - start_usec / 1000000 ))
+}
+
+age=$(nats_age_seconds)
+if [ "$age" -lt "$NATS_MIN_AGE" ]; then
     exit 0
 fi
 
+if [ -f "$COOLDOWN_STAMP" ]; then
+    last=$(cat "$COOLDOWN_STAMP" 2>/dev/null) || last=""
+    case "$last" in ''|*[!0-9]*) last=0 ;; esac
+    if [ $(( $(uptime_seconds) - last )) -lt "$RESTART_COOLDOWN" ]; then
+        exit 0
+    fi
+fi
+
+restart_count=$(cat "$RESTART_COUNT_FILE" 2>/dev/null) || restart_count=""
+case "$restart_count" in ''|*[!0-9]*) restart_count=0 ;; esac
+
+# The escalation count is read here but acted on only after the probes, below.
+# Checking it up front would return before the one line that clears it, so a
+# latched watchdog could never observe recovery and never re-arm itself.
+
 # curl is the probe transport; if it is unavailable, do nothing rather than
 # false-restart a healthy server.
 if ! command -v curl >/dev/null 2>&1; then
@@ -45,13 +120,22 @@ js_healthy() {
     curl -fsS -o /dev/null --max-time 5 "$HEALTH_URL" 2>/dev/null
 }
 
-# Round-trips a canary key through the cluster-state bucket. Absent spx this
-# degrades to the health check alone rather than restarting on a missing tool.
+# Round-trips a canary key through the cluster-state bucket. Exit 0 means the
+# write succeeded and exit 1 means JetStream refused it; anything else
+# (spx's own exit 2, a missing binary, or one that cannot run at all) means
+# the probe observed nothing about JetStream and must never be read as
+# evidence of a failure.
 js_writable() {
     if [ ! -x "$SPX" ]; then
-        return 0
+        return 2
     fi
     "$SPX" admin node js-probe --timeout "${PROBE_TIMEOUT}s" >/dev/null 2>&1
+    rc=$?
+    case "$rc" in
+        0) return 0 ;;
+        1) return 1 ;;
+        *) return 2 ;;
+    esac
 }
 
 failure=""
@@ -59,6 +143,18 @@ i=1
 while [ "$i" -le "$SAMPLES" ]; do
     if js_healthy; then
         if js_writable; then
+            wrc=0
+        else
+            wrc=$?
+        fi
+        if [ "$wrc" -eq 0 ]; then
+            # Recovery: drop the history so a later failure gets the full
+            # restart budget again, and re-arm the escalation message.
+            rm -f "$RESTART_COUNT_FILE" "$ESCALATED_STAMP"
+            exit 0
+        fi
+        if [ "$wrc" -eq 2 ]; then
+            echo "nats-js-watchdog: js-probe inconclusive (config, connect, bucket-open, or spx itself failed); not evidence of a JetStream failure, taking no action" >&2
             exit 0
         fi
         failure="JetStream accepts no writes (canary round-trip failed) while $HEALTH_URL reports healthy"
@@ -71,5 +167,24 @@ while [ "$i" -le "$SAMPLES" ]; do
     i=$((i + 1))
 done
 
+# The failure is real and current, so the restart history now means what it
+# says. Above the threshold another restart will not help, so stop acting —
+# but keep probing on later runs, because that is what notices a recovery.
+if [ "$restart_count" -ge "$RESTART_ESCALATE_AT" ]; then
+    if [ ! -f "$ESCALATED_STAMP" ]; then
+        mkdir -p "$STATE_DIR"
+        : > "$ESCALATED_STAMP"
+        echo "nats-js-watchdog: already restarted $SERVICE $restart_count time(s) without recovery; giving up rather than looping (clear $RESTART_COUNT_FILE to re-arm)" >&2
+    fi
+    exit 0
+fi
+
 echo "nats-js-watchdog: $failure on $SAMPLES consecutive probes; restarting $SERVICE" >&2
+mkdir -p "$STATE_DIR"
+uptime_seconds > "$COOLDOWN_STAMP"
+restart_count=$((restart_count + 1))
+echo "$restart_count" > "$RESTART_COUNT_FILE"
+if [ "$restart_count" -ge "$RESTART_ESCALATE_AT" ]; then
+    echo "nats-js-watchdog: restart $restart_count of $RESTART_ESCALATE_AT before escalation; if $SERVICE is still failing after this, the watchdog will stop acting until $RESTART_COUNT_FILE is cleared" >&2
+fi
 systemctl try-restart "$SERVICE"

build/scripts/nats-js-watchdog_test.sh

@@ -0,0 +1,197 @@
+#!/bin/sh
+# Self-contained POSIX test for nats-js-watchdog.sh's restart-decision logic:
+# the age gate, cooldown, escalation, and js-probe exit-2 handling. No real
+# NATS or spx: systemctl, curl, and spx are all stubbed on PATH ahead of the
+# real ones, driven entirely by env vars this test controls.
+#
+# Run: sh build/scripts/nats-js-watchdog_test.sh
+set -eu
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+SCRIPT="${SCRIPT_DIR}/nats-js-watchdog.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "${WORK}"' EXIT
+
+FAKE_BIN="${WORK}/bin"
+mkdir -p "${FAKE_BIN}"
+
+FAILS=0
+fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); }
+pass() { echo "ok: $*"; }
+
+# Fake systemctl: `show` reports a controllable ExecMainStartTimestampMonotonic,
+# and `try-restart` just records that it was called rather than touching a
+# real unit.
+cat > "${FAKE_BIN}/systemctl" <<'EOF'
+#!/bin/sh
+case "$1" in
+    show)
+        echo "${FAKE_NATS_START_USEC:-0}"
+        ;;
+    try-restart)
+        echo "try-restart $2" >> "${FAKE_RESTART_LOG:-/dev/null}"
+        ;;
+esac
+EOF
+chmod +x "${FAKE_BIN}/systemctl"
+
+# Fake curl: js_healthy() only cares about the exit status.
+cat > "${FAKE_BIN}/curl" <<'EOF'
+#!/bin/sh
+[ "${FAKE_HEALTHY:-1}" = "1" ]
+EOF
+chmod +x "${FAKE_BIN}/curl"
+
+# Fake spx: js-probe's whole contract from the watchdog's side is its exit
+# code (0 ok, 1 JetStream refused, 2 inconclusive).
+cat > "${FAKE_BIN}/spx" <<'EOF'
+#!/bin/sh
+exit "${FAKE_SPX_EXIT:-0}"
+EOF
+chmod +x "${FAKE_BIN}/spx"
+
+PATH="${FAKE_BIN}:${PATH}"
+export PATH
+
+# reset_env <name>: fresh per-case dirs and a default env that already clears
+# the age, cooldown, and escalation gates (old enough NATS, no cooldown
+# stamp, no restart history) and a healthy, single-sample probe, so a case
+# only has to override what it is actually testing.
+reset_env() {
+    CASE="${WORK}/$1"
+    mkdir -p "${CASE}/state"
+    UPTIME_FILE="${CASE}/uptime"
+    printf '700.00 0.00\n' > "${UPTIME_FILE}"
+    STATE_DIR="${CASE}/state"
+    FAKE_RESTART_LOG="${CASE}/restart.log"
+    FAKE_NATS_START_USEC=1000000   # 1s: 700s uptime - 1s start = 699s age, clears the 600s default gate
+    FAKE_HEALTHY=1
+    FAKE_SPX_EXIT=0
+    SAMPLES=1
+    SAMPLE_GAP=0
+    NATS_MIN_AGE=600
+    RESTART_COOLDOWN=600
+    RESTART_ESCALATE_AT=3
+    SPX_BIN="${FAKE_BIN}/spx"
+    STDOUT="${CASE}/stdout"
+    STDERR="${CASE}/stderr"
+    export UPTIME_FILE STATE_DIR FAKE_RESTART_LOG FAKE_NATS_START_USEC \
+        FAKE_HEALTHY FAKE_SPX_EXIT SAMPLES SAMPLE_GAP NATS_MIN_AGE \
+        RESTART_COOLDOWN RESTART_ESCALATE_AT SPX_BIN
+}
+
+# Run the watchdog, capturing its exit code (without tripping set -e) + streams.
+invoke() { rc=0; sh "${SCRIPT}" >"${STDOUT}" 2>"${STDERR}" || rc=$?; }
+
+restarted() { [ -s "${FAKE_RESTART_LOG}" ]; }
+
+# --- Case 1: NATS younger than the age gate -> no restart ---
+reset_env young
+FAKE_NATS_START_USEC=690000000  # 690s: 700s uptime - 690s start = 10s age, well under 600s
+export FAKE_NATS_START_USEC
+invoke
+[ "${rc}" -eq 0 ] && pass "young: exit 0" || fail "young: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "young: must not restart while still in the recovery window" || pass "young: no restart"
+
+# --- Case 2: cooldown stamp fresh -> no restart ---
+reset_env cooldown
+echo 650 > "${STATE_DIR}/nats-js-watchdog.last-restart"
+invoke
+[ "${rc}" -eq 0 ] && pass "cooldown: exit 0" || fail "cooldown: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "cooldown: must not restart again inside the cooldown window" || pass "cooldown: no restart"
+
+# --- Case 3: escalation threshold reached while still failing -> no restart,
+# loud log. The probe must fail: a healthy probe at the threshold is case 3b,
+# and it re-arms rather than giving up. ---
+reset_env escalated
+FAKE_SPX_EXIT=1
+export FAKE_SPX_EXIT
+echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count"
+invoke
+[ "${rc}" -eq 0 ] && pass "escalated: exit 0" || fail "escalated: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "escalated: must not restart past the escalation threshold" || pass "escalated: no restart"
+grep -q 'giving up rather than looping' "${STDERR}" \
+    && pass "escalated: loud log emitted" || fail "escalated: escalation log missing: $(cat "${STDERR}")"
+
+# Second identical run: the node is still broken, so the decision stands, but
+# repeating the message every timer tick is what buried the real signal for 42
+# hours on node1.
+invoke
+grep -q 'giving up rather than looping' "${STDERR}" \
+    && fail "escalated: message repeated on every run instead of once per latch" \
+    || pass "escalated: message not repeated"
+
+# --- Case 3b: escalation threshold reached but JetStream has recovered ->
+# the watchdog must re-arm itself. This is the production bug: the escalation
+# check used to run before the probe, so the clear below was unreachable and
+# the node stayed disarmed until reboot. ---
+reset_env escalated_recovered
+echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count"
+: > "${STATE_DIR}/nats-js-watchdog.escalated"
+invoke
+[ "${rc}" -eq 0 ] && pass "escalated-recovered: exit 0" || fail "escalated-recovered: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "escalated-recovered: must not restart a recovered server" || pass "escalated-recovered: no restart"
+[ -f "${STATE_DIR}/nats-js-watchdog.restart-count" ] \
+    && fail "escalated-recovered: restart count must clear once JetStream is writable again" \
+    || pass "escalated-recovered: restart count cleared"
+[ -f "${STATE_DIR}/nats-js-watchdog.escalated" ] \
+    && fail "escalated-recovered: escalation marker must clear so a later failure is reported" \
+    || pass "escalated-recovered: escalation marker cleared"
+[ -s "${STDERR}" ] \
+    && fail "escalated-recovered: a recovered node must be silent: $(cat "${STDERR}")" \
+    || pass "escalated-recovered: silent"
+
+# --- Case 3c: re-armed watchdog acts again. One healthy run clears the latch,
+# and the next genuine failure must restart -- proving it is armed, not just
+# quiet. ---
+reset_env escalated_rearm
+echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count"
+invoke
+FAKE_SPX_EXIT=1
+export FAKE_SPX_EXIT
+invoke
+restarted && pass "rearm: restarts again after recovery cleared the latch" \
+    || fail "rearm: expected a restart once re-armed: $(cat "${STDERR}")"
+[ "$(cat "${STATE_DIR}/nats-js-watchdog.restart-count" 2>/dev/null)" = "1" ] \
+    && pass "rearm: restart count restarts from 1" \
+    || fail "rearm: expected count 1, got $(cat "${STATE_DIR}/nats-js-watchdog.restart-count" 2>/dev/null)"
+
+# --- Case 4: js-probe exits 2 (inconclusive) -> no restart ---
+reset_env inconclusive
+FAKE_SPX_EXIT=2
+export FAKE_SPX_EXIT
+invoke
+[ "${rc}" -eq 0 ] && pass "inconclusive: exit 0" || fail "inconclusive: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "inconclusive: exit 2 must never be read as a JetStream failure" || pass "inconclusive: no restart"
+grep -q 'inconclusive' "${STDERR}" \
+    && pass "inconclusive: reason logged" || fail "inconclusive: log missing: $(cat "${STDERR}")"
+
+# --- Case 5: genuine JetStream write failure -> restart happens ---
+reset_env genuine_failure
+FAKE_SPX_EXIT=1
+export FAKE_SPX_EXIT
+echo 2 > "${STATE_DIR}/nats-js-watchdog.restart-count"
+invoke
+[ "${rc}" -eq 0 ] && pass "genuine-failure: exit 0" || fail "genuine-failure: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && pass "genuine-failure: restart triggered" || fail "genuine-failure: expected a restart: $(cat "${STDERR}")"
+[ "$(cat "${STATE_DIR}/nats-js-watchdog.restart-count" 2>/dev/null)" = "3" ] \
+    && pass "genuine-failure: restart count incremented" || fail "genuine-failure: restart count not incremented"
+[ -f "${STATE_DIR}/nats-js-watchdog.last-restart" ] \
+    && pass "genuine-failure: cooldown stamp written" || fail "genuine-failure: cooldown stamp missing"
+
+# --- Case 6: healthy NATS -> no restart, and prior restart history clears ---
+reset_env healthy
+echo 2 > "${STATE_DIR}/nats-js-watchdog.restart-count"
+invoke
+[ "${rc}" -eq 0 ] && pass "healthy: exit 0" || fail "healthy: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "healthy: must not restart a healthy server" || pass "healthy: no restart"
+[ -f "${STATE_DIR}/nats-js-watchdog.restart-count" ] \
+    && fail "healthy: restart count must clear once JetStream is confirmed writable" \
+    || pass "healthy: restart count cleared"
+
+if [ "${FAILS}" -eq 0 ]; then
+    echo "PASS: all nats-js-watchdog cases"
+    exit 0
+fi
+echo "FAILED: ${FAILS} case(s)"
+exit 1

build/scripts/wait-for-nats.sh

@@ -26,6 +26,7 @@ fi
 # Final fallback
 NATS_HOST="${NATS_HOST:-127.0.0.1}"
 
+# shellcheck disable=SC2034  # loop count only, index unused
 for i in $(seq 1 "$TIMEOUT"); do
     if nc -z "$NATS_HOST" "$NATS_PORT" 2>/dev/null; then
         exit 0

build/systemd/regenerate-ssh-host-keys.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Regenerate SSH host keys
 ConditionPathExists=!/etc/ssh/ssh_host_ed25519_key

build/systemd/spinifex-awsgw.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex AWS Gateway
 PartOf=spinifex.target

build/systemd/spinifex-daemon.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 3
 [Unit]
 Description=Spinifex Daemon (VM Orchestration)
 PartOf=spinifex.target
@@ -38,12 +39,13 @@ PrivateDevices=no
 DevicePolicy=closed
 DeviceAllow=/dev/kvm rw
 DeviceAllow=/dev/net/tun rw
+DeviceAllow=/dev/vhost-net rw
 DeviceAllow=char-vfio rw
 DeviceAllow=/dev/vfio/vfio rw
 ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
-# /proc/sys/net/ipv4/conf: per-tap IMDS endpoints set rp_filter/accept_local, which
-# ProtectKernelTunables=yes would otherwise keep read-only.
-ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf
+# /proc/sys/net/ipv4/conf: ProtectKernelTunables would keep the per-tap IMDS
+# rp_filter/accept_local read-only; /etc/spinifex/firewall: the root helper writes peers.nft.
+ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf -/etc/spinifex/firewall
 
 # RG-9: hardening baseline
 ProtectHome=yes

build/systemd/spinifex-firewall.service

@@ -0,0 +1,25 @@
+# spinifex-unit-version: 1
+[Unit]
+Description=Spinifex host firewall (nftables input policy)
+Documentation=file:/etc/spinifex/firewall/spinifex.nft
+Wants=network-pre.target
+Before=network-pre.target shutdown.target
+DefaultDependencies=no
+ConditionPathExists=/etc/spinifex/firewall/peers.nft
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+StandardInput=null
+ProtectSystem=full
+ProtectHome=true
+ExecStart=/usr/local/lib/spinifex/spinifex-firewall-apply
+ExecReload=/usr/local/lib/spinifex/spinifex-firewall-apply
+TimeoutStopSec=10s
+
+# No ExecStop: flushing on stop would take vpcd's MASQUERADE and per-EIP FORWARD
+# rules with it. Remove the policy deliberately with
+# `nft delete table inet spinifex_filter`.
+
+[Install]
+WantedBy=sysinit.target

build/systemd/spinifex-guests.slice

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex guest VM tier
 Before=slices.target

build/systemd/spinifex-nats-watchdog.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 2
 [Unit]
 Description=Spinifex NATS JetStream liveness watchdog
 PartOf=spinifex.target
@@ -15,6 +16,10 @@ ExecStart=/var/lib/spinifex/nats-js-watchdog.sh
 # cutting it off. Only the failing path runs that long; a healthy node exits on
 # the first sample.
 TimeoutStopSec=45
+# Same 34s worst case bounds the start side: without this the unit inherits
+# oneshot's infinite default, so a genuinely hung run would sit in
+# activating(start) forever and could not be retriggered by its timer.
+TimeoutStartSec=120
 
 # Hardening (kept compatible with talking to PID1 over /run/systemd).
 NoNewPrivileges=yes

build/systemd/spinifex-nats-watchdog.timer

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex NATS JetStream liveness watchdog timer
 PartOf=spinifex.target

build/systemd/spinifex-nats.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex NATS Message Bus
 PartOf=spinifex.target

build/systemd/spinifex-northstar.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Northstar (Authoritative DNS)
 PartOf=spinifex.target

build/systemd/spinifex-predastore.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 3
 [Unit]
 Description=Spinifex Predastore (S3-compatible Object Storage)
 PartOf=spinifex.target
@@ -6,9 +7,9 @@ Wants=spinifex-nats.service
 
 [Service]
 EnvironmentFile=-/etc/spinifex/telemetry.env
-# Guest block I/O arrives with no caller trace; sample those roots at 5% so
-# they don't drown request traces. Request-linked spans are always kept.
-Environment=MULGA_ROOT_TRACE_RATIO=0.05
+# No MULGA_ROOT_TRACE_RATIO here: the sink's tail sampler sheds guest block I/O
+# instead, and it can see whether a trace errored. A head ratio decides before
+# the error exists, so it discarded failures at the same rate as successes.
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-storage
@@ -17,9 +18,9 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service predastore start
 Restart=on-failure
 RestartSec=5
-# Matches viperblock's drain budget: viperblock's WAL flush writes into
-# predastore, and After= already stops viperblock first, but this is the
-# backstop if that ordering is ever bypassed (direct stop, future edit).
+# Bounds predastore's own graceful stop. viperblock's WAL flush writes into
+# predastore, but After= already stops viperblock first (within its own 150s
+# budget); this is the backstop if that ordering is ever bypassed.
 TimeoutStopSec=30
 OOMScoreAdjust=-500
 # Legitimately the most memory-hungry infra service (S3 buffering/caching);

build/systemd/spinifex-qmp-collector.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex QMP Telemetry Collector (guest metrics)
 PartOf=spinifex.target

build/systemd/spinifex-shutdown.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex graceful guest drain
 PartOf=spinifex.target
@@ -11,11 +12,11 @@ RemainAfterExit=yes
 ExecStart=/bin/true
 # ExecStop fires on ANY target stop, including the stop half of `systemctl
 # restart spinifex.target` (a rolling deploy) -- PartOf propagates regardless
-# of whether the host is actually going down. --only-if-host-stopping drains
-# only when systemd itself is unwinding into shutdown.target; a plain restart
-# or `systemctl stop spinifex.target` falls through so QEMU/nbdkit survive
-# (KillMode=process) and the daemon reconnects to them on the way back up.
-ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --only-if-host-stopping
+# of whether the host is actually going down. --unless-restarting drains on a
+# real shutdown/reboot and on a plain `systemctl stop spinifex.target`; it
+# skips only when a restart of the stack is already queued, so QEMU/nbdkit
+# survive (KillMode=process) and the daemon reconnects on the way back up.
+ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --unless-restarting
 TimeoutStopSec=180
 
 [Install]

build/systemd/spinifex-system.slice

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex control/infra tier (NATS, predastore, viperblock, daemon, awsgw, vpcd, ui)
 Before=slices.target

build/systemd/spinifex-ui.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Web UI
 PartOf=spinifex.target

build/systemd/spinifex-viperblock.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 2
 [Unit]
 Description=Spinifex Viperblock (Block Storage)
 PartOf=spinifex.target
@@ -23,8 +24,10 @@ MemoryMax=25%
 # way guest QEMU survives a daemon restart. Tearing one out mid-write corrupts
 # that guest's filesystem; idle nbdkit is reaped explicitly on shutdown.
 KillMode=process
-# Bounds this process's own WAL drain, which the plugin caps at 20s.
-TimeoutStopSec=30
+# One worst-case nbdkit reap: killProcessGracePeriod (120s, utils.KillProcess)
+# plus its own shutdownDrainTimeout (20s, viperblock/nbd) and headroom — must
+# strictly exceed 120s now shutdownVolumes reaps concurrently, not serially.
+TimeoutStopSec=150
 EnvironmentFile=/etc/spinifex/systemd.env
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/
@@ -48,7 +51,8 @@ ProtectKernelTunables=yes
 ProtectKernelModules=yes
 ProtectKernelLogs=yes
 ProtectControlGroups=yes
-ProtectProc=invisible
+# ProtectProc must stay default: hidepid=invisible hides spinifex-daemon's
+# QEMU /proc entries (different uid) orphan nbdkit reaping must see.
 SystemCallArchitectures=native
 MemoryDenyWriteExecute=yes
 RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK

build/systemd/spinifex-vpcd.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex VPC Daemon (OVN Networking)
 PartOf=spinifex.target

build/systemd/spinifex.slice

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex resource-bounded slice (host ceiling)
 Before=slices.target

build/systemd/spinifex.target

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Infrastructure Platform
 After=network-online.target

cmd/ecs-agent/ecrresolver.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"net/http"
+	"net/url"
 	"strings"
 	"sync"
 
@@ -60,12 +61,25 @@ func (e *ecrResolver) client() (*http.Client, error) {
 	return c, nil
 }
 
-// isECRRef reports whether ref targets an ECR registry, i.e. its registry host
-// is "<acct>.dkr.ecr.<region>.<domain>". Only ECR pulls need a minted token;
-// public registries (docker.io and friends) pull anonymously, so refs without
-// an ECR host must not drag in IMDS host credentials.
-func isECRRef(ref string) bool {
+// gatewayRegistryHost returns the host[:port] of the co-located ECR registry
+// from the gateway URL; empty/unparseable yields "" (matches no ref). The
+// registry shares the gateway listener, so its port is the gateway port.
+func gatewayRegistryHost(gatewayURL string) string {
+	u, err := url.Parse(gatewayURL)
+	if err != nil {
+		return ""
+	}
+	return u.Host
+}
+
+// isECRRef reports whether ref targets an ECR registry: an "<acct>.dkr.ecr."
+// host, or one equal to gatewayHost (the agent's own co-located registry
+// advertised as a bare host/IP). Other refs pull anonymously, no token.
+func isECRRef(ref, gatewayHost string) bool {
 	host, _, _ := strings.Cut(ref, "/")
+	if gatewayHost != "" && host == gatewayHost {
+		return true
+	}
 	if !strings.Contains(host, ".") && !strings.Contains(host, ":") {
 		return false // no registry host -> docker.io default
 	}
@@ -75,7 +89,7 @@ func isECRRef(ref string) bool {
 // Authorize returns the ECR token's user/password and proxy endpoint for ref.
 // Non-ECR refs resolve to anonymous pull (empty credentials).
 func (e *ecrResolver) Authorize(ctx context.Context, ref string) (user, pass, endpoint string, err error) {
-	if !isECRRef(ref) {
+	if !isECRRef(ref, gatewayRegistryHost(e.gatewayURL)) {
 		return "", "", "", nil
 	}
 	c, err := e.creds.Retrieve(ctx)

cmd/ecs-agent/ecrresolver_test.go

@@ -115,17 +115,44 @@ func TestECRResolver_NonECRAnonymous(t *testing.T) {
 }
 
 func TestIsECRRef(t *testing.T) {
-	cases := map[string]bool{
-		"nginx:alpine":            false,
-		"docker.io/library/nginx": false,
-		"ghcr.io/org/app:1":       false,
-		"registry:5000/app:1":     false,
-		"123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1": true,
-		"123456789012.dkr.ecr.ap-southeast-2.amazonaws.com/x":    true,
+	cases := []struct {
+		ref, gatewayHost string
+		want             bool
+	}{
+		{"nginx:alpine", "192.168.1.13:9999", false},
+		{"docker.io/library/nginx", "192.168.1.13:9999", false},
+		{"ghcr.io/org/app:1", "192.168.1.13:9999", false},
+		{"registry:5000/app:1", "192.168.1.13:9999", false},
+		{"123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1", "192.168.1.13:9999", true},
+		{"123456789012.dkr.ecr.ap-southeast-2.amazonaws.com/x", "192.168.1.13:9999", true},
+		// Co-located gateway registry advertised as a bare host/IP.
+		{"192.168.1.13:9999/bunyip:latest", "192.168.1.13:9999", true},
+		{"192.168.1.13:9999/bunyip:latest", "", false},
 	}
-	for ref, want := range cases {
-		if got := isECRRef(ref); got != want {
-			t.Errorf("isECRRef(%q) = %v, want %v", ref, got, want)
+	for _, c := range cases {
+		if got := isECRRef(c.ref, c.gatewayHost); got != c.want {
+			t.Errorf("isECRRef(%q, %q) = %v, want %v", c.ref, c.gatewayHost, got, c.want)
 		}
 	}
 }
+
+// TestECRResolver_Authorize_BareHostGateway asserts that a ref whose host
+// equals the agent's own gateway host authenticates via the co-located ECR
+// registry, even though it lacks a ".dkr.ecr." name.
+func TestECRResolver_Authorize_BareHostGateway(t *testing.T) {
+	srv := gatewayStub(t, "AWS:jwt", "https://192.168.1.13:9999", http.StatusOK)
+	creds := stubCreds{c: credentials.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "sess",
+		Expiration: time.Now().Add(time.Hour),
+	}}
+	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
+
+	host := strings.TrimPrefix(srv.URL, "https://")
+	user, pass, _, err := r.Authorize(context.Background(), host+"/bunyip:latest")
+	if err != nil {
+		t.Fatalf("Authorize: %v", err)
+	}
+	if user != "AWS" || pass != "jwt" {
+		t.Errorf("got %q:%q, want AWS:jwt", user, pass)
+	}
+}

cmd/ecs-agent/runtime/runtime_test.go

@@ -1,71 +1,9 @@
 package runtime
 
 import (
-	"context"
-	"errors"
 	"testing"
 )
 
-// staticResolver is a test Resolver returning fixed creds or an error.
-type staticResolver struct {
-	user, pass, endpoint string
-	err                  error
-}
-
-func (s staticResolver) Authorize(_ context.Context, _ string) (string, string, string, error) {
-	return s.user, s.pass, s.endpoint, s.err
-}
-
-func TestFakePuller_RecordsAndDefaultsRef(t *testing.T) {
-	f := &FakePuller{}
-	img, err := f.Pull(context.Background(), PullSpec{Ref: "repo/app:1"}, staticResolver{user: "AWS"})
-	if err != nil {
-		t.Fatalf("Pull: %v", err)
-	}
-	if img.Ref != "repo/app:1" {
-		t.Errorf("Ref defaulted wrong: %q", img.Ref)
-	}
-	if len(f.Pulls) != 1 || f.Pulls[0].Ref != "repo/app:1" {
-		t.Errorf("Pulls = %+v", f.Pulls)
-	}
-	if len(f.Authzd) != 1 || f.Authzd[0] != "AWS" {
-		t.Errorf("Authzd = %+v", f.Authzd)
-	}
-}
-
-func TestFakePuller_ResolverError(t *testing.T) {
-	f := &FakePuller{}
-	_, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, staticResolver{err: errors.New("boom")})
-	if err == nil {
-		t.Fatal("expected resolver error to propagate")
-	}
-}
-
-func TestFakePuller_ProgrammedError(t *testing.T) {
-	f := &FakePuller{Err: errors.New("pull failed")}
-	if _, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil); err == nil {
-		t.Fatal("expected programmed error")
-	}
-}
-
-func TestFakePuller_OnPullOverrides(t *testing.T) {
-	want := Image{Ref: "x", Digest: "sha256:abc", Size: 42}
-	f := &FakePuller{Err: errors.New("ignored"), OnPull: func(PullSpec) (Image, error) {
-		return want, nil
-	}}
-	got, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil)
-	if err != nil || got != want {
-		t.Fatalf("got %+v, %v; want %+v", got, err, want)
-	}
-}
-
-func TestFakePuller_Close(t *testing.T) {
-	f := &FakePuller{}
-	if err := f.Close(); err != nil || !f.Closed {
-		t.Fatalf("Close: err=%v closed=%v", err, f.Closed)
-	}
-}
-
 func TestNormalizeRef(t *testing.T) {
 	cases := map[string]string{
 		"nginx:alpine":      "docker.io/library/nginx:alpine",

cmd/installer/firstboot/firstboot.go

@@ -146,6 +146,21 @@ hostnamectl set-hostname %s
 printf "nameserver 1.1.1.1\nnameserver 8.8.8.8\n" > /etc/resolvconf/resolv.conf.d/base
 resolvconf -u
 
+# The RTC on a fresh machine can be badly wrong — a flat CMOS battery is routine
+# on refurbished hardware. Credentials and TLS certificates are minted further
+# down, and a certificate issued against a wrong clock carries a NotBefore that
+# outlives the correction, so the clock has to be right before that happens.
+# chronyd is already running; makestep forces an immediate correction instead of
+# waiting out the slew. Bounded and non-fatal: an air-gapped install must still
+# complete, and rtcsync writes the corrected time back so later boots start close.
+echo "[firstboot] syncing clock before credentials are minted"
+chronyc -a makestep >/dev/null 2>&1 || true
+if chronyc waitsync 12 0.1 0 5 >/dev/null 2>&1; then
+    echo "[firstboot] clock synced: $(date -uIs)"
+else
+    echo "[firstboot] WARNING: clock not synced, continuing with $(date -uIs)"
+fi
+
 # Configure OVN networking.
 # br-wan (and br-lan if present) are Linux bridges managed by systemd-networkd
 # (declared in /etc/systemd/network/ by the installer). setup-ovn.sh
@@ -207,6 +222,12 @@ fi
 
 # Enable services to start, on reboot
 systemctl enable spinifex.target spinifex-banner.service
+# setup.sh enables this on the package install path, but the ISO chroot returns
+# before that line (no systemd to talk to), so the ISO needs it here. An nft
+# ruleset is runtime state and does not survive a reboot; without this the unit
+# never reapplies the policy and the node comes back up open. Enabled, not
+# started: ConditionPathExists holds it inert until the daemon writes a peer file.
+systemctl enable spinifex-firewall.service
 # WantedBy=timers.target only self-activates once enabled — without this the
 # JetStream ENOSPC-latch watchdog never runs and a full disk needs a manual restart.
 systemctl enable --now spinifex-nats-watchdog.timer

cmd/installer/install/blockdev.go

@@ -45,6 +45,7 @@ var (
 	sysClassBlockDir = "/sys/class/block"
 	devByIDDir       = "/dev/disk/by-id"
 	procMountsPath   = "/proc/mounts"
+	procSwapsPath    = "/proc/swaps"
 )
 
 // ListDisks enumerates install-candidate disks from /sys/block. It reads sysfs
@@ -132,6 +133,144 @@ func partitionPath(disk string, n int) string {
 	return fmt.Sprintf("%s%d", disk, n)
 }
 
+// readKernelPartitions returns the partition device names the kernel currently
+// publishes for a disk, sorted.
+//
+// It reads sysfs rather than /dev deliberately. A partition device node
+// outlives the table that created it, so a stale node is indistinguishable
+// from a fresh one by existence alone — which is how a disk whose old table
+// the kernel never dropped passes every check and then gets formatted at the
+// previous layout's offsets.
+func readKernelPartitions(disk string) ([]string, error) {
+	name := filepath.Base(disk)
+	entries, err := os.ReadDir(filepath.Join(sysBlockDir, name))
+	if err != nil {
+		return nil, fmt.Errorf("read the kernel's partition list for %s: %w", disk, err)
+	}
+	var out []string
+	for _, e := range entries {
+		// Partitions are the child directories carrying a "partition" attribute.
+		// "queue", "device" and the plain attribute files sit alongside them.
+		if _, err := os.Stat(filepath.Join(sysBlockDir, name, e.Name(), "partition")); err != nil {
+			continue
+		}
+		out = append(out, e.Name())
+	}
+	slices.Sort(out)
+	return out, nil
+}
+
+// kernelPartitions is indirected so tests can drive the partition-table
+// assertions without a real block device.
+var kernelPartitions = readKernelPartitions
+
+// partitionDevice maps a kernel partition name back to its device node,
+// resolved alongside the disk it belongs to rather than assuming /dev.
+func partitionDevice(disk, part string) string {
+	return filepath.Join(filepath.Dir(disk), part)
+}
+
+// diskHolders names the md, LVM or device-mapper devices claiming this disk or
+// one of its partitions. A holder is the usual reason the kernel refuses to
+// re-read a partition table, so it is what the operator needs told when the
+// installer gives up on a disk.
+func diskHolders(disk string) []string {
+	name := filepath.Base(disk)
+	dirs := []string{filepath.Join(sysBlockDir, name, "holders")}
+	parts, _ := kernelPartitions(disk)
+	for _, p := range parts {
+		dirs = append(dirs, filepath.Join(sysBlockDir, name, p, "holders"))
+	}
+
+	seen := map[string]bool{}
+	var out []string
+	for _, dir := range dirs {
+		entries, err := os.ReadDir(dir)
+		if err != nil {
+			continue
+		}
+		for _, e := range entries {
+			if seen[e.Name()] {
+				continue
+			}
+			seen[e.Name()] = true
+			out = append(out, e.Name())
+		}
+	}
+	slices.Sort(out)
+	return out
+}
+
+// diskMount is one filesystem mounted from a disk the installer is about to
+// erase.
+type diskMount struct {
+	Device     string
+	Mountpoint string
+}
+
+// devicesOnDisk is the set of device paths belonging to a disk: the whole disk
+// and every partition the kernel currently publishes. Matching /proc tables
+// against a set rather than a string prefix keeps /dev/nvme0n1 from claiming
+// /dev/nvme0n11.
+func devicesOnDisk(disk string) map[string]bool {
+	out := map[string]bool{disk: true}
+	parts, _ := kernelPartitions(disk)
+	for _, p := range parts {
+		out[partitionDevice(disk, p)] = true
+	}
+	return out
+}
+
+// mountsOnDisk returns the filesystems mounted from a disk, deepest mountpoint
+// first so a nested mount is released before the one it sits inside.
+func mountsOnDisk(disk string) []diskMount {
+	devs := devicesOnDisk(disk)
+	var out []diskMount
+	for _, fields := range procTable(procMountsPath, 0) {
+		if len(fields) >= 2 && devs[fields[0]] {
+			out = append(out, diskMount{Device: fields[0], Mountpoint: fields[1]})
+		}
+	}
+	slices.SortFunc(out, func(a, b diskMount) int {
+		return strings.Count(b.Mountpoint, "/") - strings.Count(a.Mountpoint, "/")
+	})
+	return out
+}
+
+// swapsOnDisk returns the active swap areas backed by a disk.
+func swapsOnDisk(disk string) []string {
+	devs := devicesOnDisk(disk)
+	var out []string
+	// The first line of /proc/swaps is a column header, not an entry.
+	for _, fields := range procTable(procSwapsPath, 1) {
+		if len(fields) >= 1 && devs[fields[0]] {
+			out = append(out, fields[0])
+		}
+	}
+	return out
+}
+
+// procTable splits a whitespace-delimited /proc table into fields per line,
+// dropping the first skip lines and any blank ones.
+func procTable(path string, skip int) [][]string {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil
+	}
+	var out [][]string
+	n := 0
+	for line := range strings.SplitSeq(string(data), "\n") {
+		n++
+		if n <= skip {
+			continue
+		}
+		if fields := strings.Fields(line); len(fields) > 0 {
+			out = append(out, fields)
+		}
+	}
+	return out
+}
+
 // byIDPreference orders the /dev/disk/by-id/ namespaces we prefer to name a
 // pool member by. wwn- ids are last: they are stable but opaque, so a degraded
 // pool reports a number nobody can map to a drive bay.

cmd/installer/install/blockdev_test.go

@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 	"slices"
 	"strconv"
+	"strings"
 	"testing"
 )
 
@@ -40,6 +41,7 @@ func newFakeSysfs(t *testing.T, disks ...fakeDisk) *fakeSysfs {
 	swap(&sysClassBlockDir, filepath.Join(f.root, "sys/class/block"))
 	swap(&devByIDDir, filepath.Join(f.root, "dev/disk/by-id"))
 	swap(&procMountsPath, filepath.Join(f.root, "proc/mounts"))
+	swap(&procSwapsPath, filepath.Join(f.root, "proc/swaps"))
 
 	f.mkdirAll(sysBlockDir)
 	f.mkdirAll(sysClassBlockDir)
@@ -113,6 +115,18 @@ func (f *fakeSysfs) setMounts(content string) {
 	f.write(procMountsPath, content)
 }
 
+func (f *fakeSysfs) setSwaps(content string) {
+	f.t.Helper()
+	f.write(procSwapsPath, content)
+}
+
+// addHolder marks a partition as claimed by md, LVM or device-mapper — the
+// state that makes the kernel refuse to re-read the disk's partition table.
+func (f *fakeSysfs) addHolder(disk, part, holder string) {
+	f.t.Helper()
+	f.mkdirAll(filepath.Join(sysBlockDir, disk, part, "holders", holder))
+}
+
 func boolAttr(b bool) string {
 	if b {
 		return "1\n"
@@ -352,3 +366,119 @@ func TestHasAnyPrefix(t *testing.T) {
 		t.Error("sda should not match")
 	}
 }
+
+func TestReadKernelPartitionsListsOnlyPartitions(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{
+		name: "nvme0n1", sectors: 1 << 22,
+		partitions: []string{"nvme0n1p3", "nvme0n1p1", "nvme0n1p2"},
+	})
+
+	got, err := readKernelPartitions("/dev/nvme0n1")
+	if err != nil {
+		t.Fatalf("readKernelPartitions: %v", err)
+	}
+	// Sorted, and "queue" and "device" alongside them are not partitions.
+	want := []string{"nvme0n1p1", "nvme0n1p2", "nvme0n1p3"}
+	if !slices.Equal(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+}
+
+func TestReadKernelPartitionsIsEmptyForAnUnpartitionedDisk(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{name: "nvme2n1", sectors: 1 << 22})
+
+	got, err := readKernelPartitions("/dev/nvme2n1")
+	if err != nil {
+		t.Fatalf("readKernelPartitions: %v", err)
+	}
+	if len(got) != 0 {
+		t.Errorf("got %v, want none", got)
+	}
+}
+
+func TestReadKernelPartitionsFailsOnAnUnknownDisk(t *testing.T) {
+	newFakeSysfs(t)
+
+	if _, err := readKernelPartitions("/dev/nvme9n1"); err == nil {
+		t.Fatal("expected an error for a disk with no sysfs entry")
+	}
+}
+
+func TestDiskHoldersNamesEveryClaimant(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{
+		name: "sda", sectors: 1 << 22, partitions: []string{"sda1", "sda2"},
+	})
+	f.addHolder("sda", "sda1", "md0")
+	f.addHolder("sda", "sda2", "dm-0")
+
+	if got, want := diskHolders("/dev/sda"), []string{"dm-0", "md0"}; !slices.Equal(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+}
+
+func TestDiskHoldersIsEmptyWhenNothingClaimsTheDisk(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1"}})
+
+	if got := diskHolders("/dev/sda"); len(got) != 0 {
+		t.Errorf("got %v, want none", got)
+	}
+}
+
+func TestMountsOnDiskFindsOnlyThisDiskDeepestFirst(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1", "sda2"}},
+		fakeDisk{name: "sdb", sectors: 1 << 22, partitions: []string{"sdb1"}},
+	)
+	f.setMounts(strings.Join([]string{
+		"/dev/sda1 /oldroot ext4 rw 0 0",
+		"/dev/sda2 /oldroot/boot/efi vfat rw 0 0",
+		"/dev/sdb1 /other ext4 rw 0 0",
+		"proc /proc proc rw 0 0",
+	}, "\n") + "\n")
+
+	got := mountsOnDisk("/dev/sda")
+	want := []diskMount{
+		{Device: "/dev/sda2", Mountpoint: "/oldroot/boot/efi"},
+		{Device: "/dev/sda1", Mountpoint: "/oldroot"},
+	}
+	if !slices.Equal(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+}
+
+// /dev/nvme0n1 must not claim /dev/nvme0n11, which a string-prefix match would.
+func TestMountsOnDiskDoesNotMatchASiblingNamespace(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "nvme0n1", sectors: 1 << 22, partitions: []string{"nvme0n1p1"}},
+		fakeDisk{name: "nvme0n11", sectors: 1 << 22, partitions: []string{"nvme0n11p1"}},
+	)
+	f.setMounts("/dev/nvme0n11p1 /elsewhere ext4 rw 0 0\n")
+
+	if got := mountsOnDisk("/dev/nvme0n1"); len(got) != 0 {
+		t.Errorf("got %v, want none", got)
+	}
+}
+
+func TestSwapsOnDiskSkipsTheHeaderAndOtherDisks(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1"}},
+		fakeDisk{name: "sdb", sectors: 1 << 22, partitions: []string{"sdb1"}},
+	)
+	f.setSwaps(strings.Join([]string{
+		"Filename\t\t\t\tType\t\tSize\tUsed\tPriority",
+		"/dev/sda1 partition 8388604 0 -2",
+		"/dev/sdb1 partition 8388604 0 -3",
+	}, "\n") + "\n")
+
+	if got, want := swapsOnDisk("/dev/sda"), []string{"/dev/sda1"}; !slices.Equal(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+}
+
+func TestSwapsOnDiskIsEmptyWithoutASwapsTable(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1"}})
+
+	if got := swapsOnDisk("/dev/sda"); len(got) != 0 {
+		t.Errorf("got %v, want none", got)
+	}
+}

cmd/installer/install/disk.go

@@ -4,6 +4,9 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	"path/filepath"
+	"slices"
+	"strings"
 	"time"
 )
 
@@ -47,6 +50,95 @@ const (
 	tailReserveMiB = 1024
 )
 
+// partitionSettleTimeout bounds the wait for the kernel and udev to publish a
+// disk's new partition table. A var so tests do not pay it.
+var partitionSettleTimeout = 15 * time.Second
+
+// releaseDisk detaches whatever the live environment has attached to a disk
+// before it is erased. A mounted filesystem or an active swap area makes
+// BLKRRPART return EBUSY, and the kernel then keeps serving the disk's old
+// partition table while sgdisk writes the new one and reports success.
+func releaseDisk(d Disk) {
+	for _, m := range mountsOnDisk(d.Path) {
+		err := runQuiet("umount", m.Device)
+		if err != nil {
+			// Lazy detach: a busy mount must not stall an install that is about to
+			// erase the filesystem underneath it anyway.
+			err = runQuiet("umount", "-l", m.Device)
+		}
+		if err != nil {
+			slog.Warn("could not unmount before erase", "device", m.Device, "mountpoint", m.Mountpoint, "err", err)
+			continue
+		}
+		slog.Info("released mount before erase", "device", m.Device, "mountpoint", m.Mountpoint)
+	}
+	for _, s := range swapsOnDisk(d.Path) {
+		if err := runQuiet("swapoff", s); err != nil {
+			slog.Warn("could not disable swap before erase", "device", s, "err", err)
+			continue
+		}
+		slog.Info("disabled swap before erase", "device", s)
+	}
+}
+
+// settlePartitions forces the kernel to re-read a disk's partition table and
+// blocks until its view matches want, naming the partitions by number.
+//
+// An empty want asserts the disk has no partitions at all. That is the only
+// state no leftover can satisfy: a disk whose previous table had the same
+// partition numbers as the one being written — an old OS install is p1, p2, p3,
+// exactly what the boot layout writes — passes an "are the expected partitions
+// there" check on stale nodes alone.
+func settlePartitions(d Disk, want ...int) error {
+	if err := runQuiet("partprobe", d.Path); err != nil {
+		// partprobe rescans the whole disk and fails if it cannot open any part of
+		// it; the ioctl on its own sometimes still gets through.
+		if err := runQuiet("blockdev", "--rereadpt", d.Path); err != nil {
+			slog.Warn("could not force a partition table re-read", "disk", d.Path, "err", err)
+		}
+	}
+	_ = runQuiet("udevadm", "settle", "--timeout=10")
+
+	wanted := make([]string, 0, len(want))
+	for _, n := range want {
+		wanted = append(wanted, filepath.Base(d.PartitionPath(n)))
+	}
+	slices.Sort(wanted)
+
+	deadline := time.Now().Add(partitionSettleTimeout)
+	for {
+		got, err := kernelPartitions(d.Path)
+		if err != nil {
+			return err
+		}
+		if slices.Equal(got, wanted) {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			return staleTableError(d, wanted, got)
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+// staleTableError explains a kernel partition view that never caught up. The
+// erase case gets its own wording because it is the dangerous one — proceeding
+// would format the previous layout — and because the holder is usually the
+// whole diagnosis.
+func staleTableError(d Disk, wanted, got []string) error {
+	if len(wanted) == 0 {
+		msg := fmt.Sprintf("%s still shows partitions [%s] after its table was erased — the kernel is using the old table, so formatting would write at the previous layout's offsets",
+			d.Path, strings.Join(got, ", "))
+		if h := diskHolders(d.Path); len(h) > 0 {
+			return fmt.Errorf("%s; it is held by %s, which must be stopped before this disk can be installed to",
+				msg, strings.Join(h, ", "))
+		}
+		return fmt.Errorf("%s; something in the live environment is holding it open", msg)
+	}
+	return fmt.Errorf("%s: the kernel shows partitions [%s] but the new table has [%s] — the table was written but never picked up",
+		d.Path, strings.Join(got, ", "), strings.Join(wanted, ", "))
+}
+
 // espSizeMiB picks the ESP size for a disk.
 func espSizeMiB(d Disk) int {
 	if d.Bytes >= espLargeDiskGiB<<30 {
@@ -98,7 +190,7 @@ func partitionDataDisk(rm RoleMount) error {
 	if err := run("sgdisk", "-G", rm.Disk.Path); err != nil {
 		slog.Warn("could not randomise GPT GUIDs", "disk", rm.Disk.Path, "err", err)
 	}
-	return nil
+	return settlePartitions(rm.Disk, dataPartNum)
 }
 
 // partitionOne writes the GPT for a single disk.
@@ -129,25 +221,15 @@ func partitionOne(d Disk, rootType string) error {
 	if err := run("sgdisk", "-G", d.Path); err != nil {
 		slog.Warn("could not randomise GPT GUIDs", "disk", d.Path, "err", err)
 	}
-	return nil
+	return settlePartitions(d, biosPartNum, espPartNum, rootPartNum)
 }
 
 // waitForPartitions ensures every partition device node the installer is about
-// to use exists. Trixie's udev is slow enough after a table rewrite that mkfs
-// races it and fails with ENOENT on a device the kernel has already accepted.
+// to use exists. The kernel's own view was already asserted per disk as each
+// table was written; udev publishes the /dev nodes and by-id symlinks after
+// that, and Trixie is slow enough about it that mkfs races them.
 func waitForPartitions(cfg DiskConfig) error {
-	for _, d := range cfg.Disks {
-		// Best-effort: partprobe failing is not fatal, since udev may still act
-		// on the BLKRRPART ioctl sgdisk issued itself.
-		if err := run("partprobe", d.Path); err != nil {
-			slog.Warn("partprobe failed, continuing", "disk", d.Path, "err", err)
-		}
-	}
-	if err := run("udevadm", "settle", "--timeout=10"); err != nil {
-		slog.Warn("udevadm settle failed, continuing", "err", err)
-	}
-
-	deadline := time.Now().Add(15 * time.Second)
+	deadline := time.Now().Add(partitionSettleTimeout)
 	for _, d := range cfg.bootDisks() {
 		wanted := []string{d.PartitionPath(espPartNum), d.PartitionPath(rootPartNum)}
 		// The pool is built from by-id paths, so their symlinks have to exist

cmd/installer/install/steps_test.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"slices"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -16,17 +18,69 @@ import (
 type fakeExec struct {
 	calls []string
 	fail  func(name string, args []string) error
+
+	// parts is the kernel's partition view per disk. Seeded by the test and then
+	// moved by the sgdisk calls the installer makes, so the partition-table
+	// assertions can run without a block device.
+	parts map[string][]string
+
+	// stale marks disks whose view never moves however many tables are written
+	// to them: something holds the disk open, BLKRRPART returns EBUSY, and the
+	// kernel keeps serving the old table while sgdisk reports success.
+	stale map[string]bool
+}
+
+// holdOpen makes a disk keep its current partition view forever.
+func (f *fakeExec) holdOpen(disk string) { f.stale[disk] = true }
+
+// seedPartitions gives a disk a pre-existing partition table — the state a
+// drive is in when the node has been installed on before.
+func (f *fakeExec) seedPartitions(disk string, nums ...int) {
+	for _, n := range nums {
+		f.parts[disk] = append(f.parts[disk], filepath.Base(partitionPath(disk, n)))
+	}
+}
+
+// applySgdisk moves the fake kernel view in step with a table rewrite, the way
+// a kernel that honoured every BLKRRPART would: -Z empties it, each -n adds the
+// partition it creates.
+func (f *fakeExec) applySgdisk(args []string) {
+	if len(args) == 0 {
+		return
+	}
+	disk := args[len(args)-1]
+	if f.stale[disk] {
+		return
+	}
+	for i, a := range args {
+		switch {
+		case a == "-Z":
+			f.parts[disk] = nil
+		case a == "-n" && i+1 < len(args):
+			num, _, _ := strings.Cut(args[i+1], ":")
+			n, err := strconv.Atoi(num)
+			if err != nil {
+				continue
+			}
+			f.parts[disk] = append(f.parts[disk], filepath.Base(partitionPath(disk, n)))
+		}
+	}
 }
 
 func fakeCommands(t *testing.T) *fakeExec {
 	t.Helper()
-	f := &fakeExec{}
-	prevRun, prevQuiet, prevEnv := run, runQuiet, runEnv
+	f := &fakeExec{parts: map[string][]string{}, stale: map[string]bool{}}
+	prevRun, prevQuiet, prevEnv, prevParts := run, runQuiet, runEnv, kernelPartitions
 
 	record := func(name string, args ...string) error {
 		f.calls = append(f.calls, strings.TrimSpace(name+" "+strings.Join(args, " ")))
 		if f.fail != nil {
-			return f.fail(name, args)
+			if err := f.fail(name, args); err != nil {
+				return err
+			}
+		}
+		if name == "sgdisk" {
+			f.applySgdisk(args)
 		}
 		return nil
 	}
@@ -35,8 +89,21 @@ func fakeCommands(t *testing.T) *fakeExec {
 	runEnv = func(env []string, name string, args ...string) error {
 		return record(name, append(env, args...)...)
 	}
+	kernelPartitions = func(disk string) ([]string, error) {
+		got := slices.Clone(f.parts[disk])
+		slices.Sort(got)
+		return got, nil
+	}
 
-	t.Cleanup(func() { run, runQuiet, runEnv = prevRun, prevQuiet, prevEnv })
+	// A short settle so a test asserting the stale-table failure does not wait
+	// out the install's real budget.
+	prevTimeout := partitionSettleTimeout
+	partitionSettleTimeout = 10 * time.Millisecond
+
+	t.Cleanup(func() {
+		run, runQuiet, runEnv, kernelPartitions = prevRun, prevQuiet, prevEnv, prevParts
+		partitionSettleTimeout = prevTimeout
+	})
 	return f
 }
 
@@ -114,6 +181,24 @@ func targetSkeleton(t *testing.T, root string) {
 
 // tempDisks builds disks whose device nodes are real files, so the steps that
 // wait for partition nodes to appear can complete.
+// fakeProcTables points the mount and swap readers at fixture tables for the
+// duration of a test.
+func fakeProcTables(t *testing.T, mounts, swaps string) {
+	t.Helper()
+	dir := t.TempDir()
+	point := func(dst *string, name, content string) {
+		prev := *dst
+		p := filepath.Join(dir, name)
+		if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
+			t.Fatalf("write %s: %v", p, err)
+		}
+		*dst = p
+		t.Cleanup(func() { *dst = prev })
+	}
+	point(&procMountsPath, "mounts", mounts)
+	point(&procSwapsPath, "swaps", swaps)
+}
+
 func tempDisks(t *testing.T, n int, gib int64) []Disk {
 	t.Helper()
 	dir := t.TempDir()
@@ -237,21 +322,154 @@ func TestEspSizeMiB(t *testing.T) {
 	}
 }
 
-func TestClearDiskOnlyLabelclearsAnExistingPartition(t *testing.T) {
+// A drive arriving with a previous install's layout must have every partition
+// cleared, not just the one the root filesystem used to be on: a data drive's
+// partition is p1, so a p3-only labelclear leaves a former pool member's ZFS
+// label in place at both ends of the device.
+func TestClearDiskWipesEveryPartitionItFinds(t *testing.T) {
 	f := fakeCommands(t)
-	disks := tempDisks(t, 1, 40)
-	if err := clearDisk(disks[0]); err != nil {
+	d := tempDisks(t, 1, 40)[0]
+	f.seedPartitions(d.Path, biosPartNum, espPartNum, rootPartNum)
+
+	if err := clearDisk(d); err != nil {
 		t.Fatalf("clearDisk: %v", err)
 	}
-	f.mustRun(t, "zpool labelclear -f "+disks[0].PartitionPath(rootPartNum))
+	for _, n := range []int{biosPartNum, espPartNum, rootPartNum} {
+		f.mustRun(t, "zpool labelclear -f "+d.PartitionPath(n))
+		f.mustRun(t, "wipefs -a "+d.PartitionPath(n))
+	}
+	f.mustRun(t, "wipefs -a "+d.Path)
+	f.mustRun(t, "sgdisk -Z "+d.Path)
+}
 
-	f2 := fakeCommands(t)
+func TestClearDiskProbesNoPartitionsOnABlankDrive(t *testing.T) {
+	f := fakeCommands(t)
 	blank := Disk{Path: filepath.Join(t.TempDir(), "sdb")}
+
 	if err := clearDisk(blank); err != nil {
 		t.Fatalf("clearDisk: %v", err)
 	}
-	f2.mustNotRun(t, "labelclear")
-	f2.mustRun(t, "wipefs -a "+blank.Path)
+	f.mustNotRun(t, "labelclear")
+	f.mustRun(t, "wipefs -a "+blank.Path)
+}
+
+// The reported field failure: a drive whose old table the kernel never dropped.
+// Proceeding would format the previous layout's offsets, so the install must
+// stop, and the message must say the table is stale rather than blaming the
+// device node that did not appear.
+func TestClearDiskFailsWhenTheKernelKeepsTheOldTable(t *testing.T) {
+	f := fakeCommands(t)
+	d := tempDisks(t, 1, 40)[0]
+	f.seedPartitions(d.Path, biosPartNum, espPartNum, rootPartNum)
+	// sgdisk writes to the platters and exits 0, but BLKRRPART returned EBUSY,
+	// so the kernel's view never changes.
+	f.holdOpen(d.Path)
+
+	err := clearDisk(d)
+	if err == nil {
+		t.Fatal("clearDisk succeeded against a stale partition table")
+	}
+	for _, want := range []string{d.Path, "still shows partitions", "previous layout"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("err = %q, want it to mention %q", err, want)
+		}
+	}
+}
+
+func TestClearDiskReleasesMountsAndSwapFirst(t *testing.T) {
+	f := fakeCommands(t)
+	d := tempDisks(t, 1, 40)[0]
+	f.seedPartitions(d.Path, biosPartNum, espPartNum, rootPartNum)
+
+	root, esp := d.PartitionPath(rootPartNum), d.PartitionPath(espPartNum)
+	swapDev := d.PartitionPath(biosPartNum)
+	fakeProcTables(t,
+		fmt.Sprintf("%s /oldroot ext4 rw 0 0\n%s /oldroot/boot/efi vfat rw 0 0\n", root, esp),
+		fmt.Sprintf("Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n%s partition 1000 0 -2\n", swapDev),
+	)
+
+	if err := clearDisk(d); err != nil {
+		t.Fatalf("clearDisk: %v", err)
+	}
+	f.mustRun(t, "swapoff "+swapDev)
+
+	// Deepest mountpoint first: unmounting /oldroot while /oldroot/boot/efi is
+	// still attached fails with EBUSY and leaves the disk held.
+	var order []string
+	for _, c := range f.calls {
+		if c == "umount "+root || c == "umount "+esp {
+			order = append(order, c)
+		}
+	}
+	if want := []string{"umount " + esp, "umount " + root}; !slices.Equal(order, want) {
+		t.Errorf("unmount order:\n got %v\nwant %v", order, want)
+	}
+}
+
+// A drive held by md or LVM cannot be released — the ISO ships neither tool —
+// so the operator has to be told what is holding it.
+func TestStaleTableErrorNamesTheHolder(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 20, partitions: []string{"sda1"}})
+	f.addHolder("sda", "sda1", "md0")
+
+	prev := kernelPartitions
+	kernelPartitions = readKernelPartitions
+	t.Cleanup(func() { kernelPartitions = prev })
+
+	err := staleTableError(Disk{Path: "/dev/sda"}, nil, []string{"sda1"})
+	if err == nil || !strings.Contains(err.Error(), "held by md0") {
+		t.Fatalf("err = %v, want it to name the md0 holder", err)
+	}
+}
+
+// The reported field failure as a unit test: two drives arrive carrying a
+// previous install's partitions and the third has never been partitioned. All
+// three must be taken over, and the blank one must not be the only one to get
+// a new table.
+func TestPartitionDisksTakesOverDrivesThatAlreadyHavePartitions(t *testing.T) {
+	f := fakeCommands(t)
+	nodes := tempDisks(t, 3, 200)
+	f.seedPartitions(nodes[0].Path, biosPartNum, espPartNum, rootPartNum) // held an OS
+	f.seedPartitions(nodes[1].Path, dataPartNum)                          // held data
+	// nodes[2] arrives blank.
+
+	cfg := DiskConfig{FS: FSExt4}.WithRoles([]RoleMount{
+		{Role: RoleOS, Disk: nodes[0]},
+		{Role: RoleSpinifex, Disk: nodes[1]},
+		{Role: RolePredastore, Disk: nodes[2]},
+	})
+	if err := partitionDisks(cfg); err != nil {
+		t.Fatalf("partitionDisks: %v", err)
+	}
+	for _, d := range nodes {
+		f.mustRun(t, "sgdisk -Z "+d.Path)
+	}
+	f.mustRun(t, "sgdisk", nodes[0].Path, "-n 3:0:-1024M")
+	for _, d := range nodes[1:] {
+		f.mustRun(t, "sgdisk", d.Path, "-n 1:1M:-1024M", "-t 1:8300")
+	}
+}
+
+// A previous OS install leaves p1, p2, p3 — exactly the numbering the boot
+// layout writes. Asserting the expected partitions exist would pass on those
+// stale nodes and the install would format the old offsets, so the erase-stage
+// assertion on an empty table is what has to catch it.
+func TestPartitionOneStopsWhenAnOldTableHasTheSameLayout(t *testing.T) {
+	f := fakeCommands(t)
+	d := tempDisks(t, 1, 200)[0]
+	f.seedPartitions(d.Path, biosPartNum, espPartNum, rootPartNum)
+	f.holdOpen(d.Path)
+
+	err := partitionOne(d, typeLinuxFS)
+	if err == nil {
+		t.Fatal("partitionOne succeeded against a stale partition table")
+	}
+	// Specifically the erase-stage message: reaching the post-write check would
+	// mean the stale nodes had already been accepted.
+	if !strings.Contains(err.Error(), "still shows partitions") {
+		t.Errorf("err = %q, want the erase-stage stale-table message", err)
+	}
+	f.mustNotRun(t, "sgdisk -n")
 }
 
 func TestFormatESPsLabelsEveryMember(t *testing.T) {
@@ -669,13 +887,6 @@ func TestInstallBootToolZFSStopsWhenADiskFails(t *testing.T) {
 	f.mustNotRun(t, "SPINIFEX_BOOT_TOOL_NVRAM=1")
 }
 
-func TestRefreshBootToolNoteOnlyAppliesToZFS(t *testing.T) {
-	// Log-only, but it is the operator's only record of the disk-replacement
-	// command, so both branches are walked.
-	refreshBootToolNote(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)})
-	refreshBootToolNote(DiskConfig{FS: FSExt4, Disks: disks(1, 40)})
-}
-
 func TestMaskSystemdUnitIsIdempotent(t *testing.T) {
 	root := t.TempDir()
 	if err := os.MkdirAll(filepath.Join(root, "etc/systemd/system"), 0o755); err != nil {
@@ -695,14 +906,6 @@ func TestMaskSystemdUnitIsIdempotent(t *testing.T) {
 	}
 }
 
-func TestCopyAssetsAreBestEffort(t *testing.T) {
-	root := t.TempDir()
-	// Neither asset exists outside the live ISO; a missing one must not fail
-	// the install, only the branding.
-	copyGrubFont(root)
-	copySplashImage(root)
-}
-
 func TestWriteFstabOnZFSHoldsNothingButSwap(t *testing.T) {
 	root := withMountRoot(t)
 	targetSkeleton(t, root)

cmd/installer/install/zfs.go

@@ -182,8 +182,7 @@ func ImportablePools() ([]string, error) {
 		// zpool exits non-zero when there is nothing to import, which is the
 		// normal case. Anything else — no binary, no /dev/zfs — must surface, or
 		// a machine with an existing pool looks empty to the caller.
-		var exitErr *exec.ExitError
-		if errors.As(err, &exitErr) {
+		if _, ok := errors.AsType[*exec.ExitError](err); ok {
 			return nil, nil
 		}
 		return nil, fmt.Errorf("zpool import: %w", err)
@@ -340,23 +339,35 @@ func exportPool() {
 	}
 }
 
-// clearDisk removes any prior filesystem, partition table and ZFS label from a
-// disk so a retry after a partial install is not rejected for stale metadata —
-// a failure that otherwise reads as mysterious rather than as leftover debris.
+// clearDisk takes a disk over unconditionally, whatever it holds: everything
+// attached to it is released, every signature on it is wiped, its partition
+// table is erased, and the kernel is made to agree the disk is now empty.
+//
+// The last part is the point. sgdisk writes to the platters and exits 0 even
+// when the kernel refuses to drop the old table, so without the assertion an
+// install proceeds against partition nodes describing the previous layout.
 //
 // The probes are quiet because a blank disk makes all of them fail, and their
 // stderr on the install console reads like the install is going wrong.
 func clearDisk(d Disk) error {
-	// ZFS labels live at both ends of the device and wipefs does not know about
-	// them, so an old pool member has to be cleared before its partition goes.
-	if old := d.PartitionPath(rootPartNum); exists(old) {
-		_ = runQuiet("zpool", "labelclear", "-f", old)
-	}
-	_ = runQuiet("wipefs", "-a", d.Path)
-	return runQuiet("sgdisk", "-Z", d.Path)
-}
+	releaseDisk(d)
 
-func exists(path string) bool {
-	_, err := os.Stat(path)
-	return err == nil
+	// Every partition, not just the root slice. ZFS labels live at both ends of
+	// a device and wipefs does not know about them, so a data drive that was
+	// once a pool member keeps its label and fails a later zpool create.
+	parts, err := kernelPartitions(d.Path)
+	if err != nil {
+		return err
+	}
+	for _, p := range parts {
+		dev := partitionDevice(d.Path, p)
+		_ = runQuiet("zpool", "labelclear", "-f", dev)
+		_ = runQuiet("wipefs", "-a", dev)
+	}
+
+	_ = runQuiet("wipefs", "-a", d.Path)
+	if err := runQuiet("sgdisk", "-Z", d.Path); err != nil {
+		return fmt.Errorf("erase partition table: %w", err)
+	}
+	return settlePartitions(d)
 }

cmd/lb-agent/main.go

@@ -18,8 +18,8 @@ import (
 	"os/signal"
 	"syscall"
 
+	"github.com/mulgadc/bluebottle/pkg/otelsetup"
 	"github.com/mulgadc/spinifex/spinifex/lbagent"
-	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 
 	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
@@ -28,7 +28,7 @@ func main() {
 	// Without this the agent falls back to slog's default text handler, which folds
 	// attributes into the message so the console scraper recovers a level and nothing
 	// else. No OTLP export: the agent's only egress is the gateway it is watching.
-	otelsetup.SetDefaultJSONLogger(slog.LevelInfo)
+	otelsetup.SetDefaultJSONLogger("lb-agent", slog.LevelInfo)
 
 	var (
 		lbID       string

cmd/rds-agent/agent.go

@@ -15,6 +15,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/gwsign"
 	"github.com/mulgadc/spinifex/internal/rdsgw"
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 )
 
 // Overridable via -ldflags "-X main.version=...".
@@ -236,10 +237,9 @@ func New(cfg config) (*Agent, error) {
 }
 
 func (a *Agent) Run(ctx context.Context) error {
-	// Named before the first dial. When register cannot reach the gateway the
-	// retry loop is all that survives, and the address it is dialing is what
-	// separates a mgmt NIC that came up without an address from a control plane
-	// that is genuinely down.
+	// Named before the first dial: when register cannot reach the gateway, the
+	// address it is dialing is what separates a mgmt NIC that came up without one
+	// from a control plane that is genuinely down.
 	slog.Info("rds-agent: starting",
 		"gatewayURL", a.cfg.GatewayURL, "dbInstanceIdentifier", a.id.DBInstanceIdentifier,
 		"agentVersion", a.id.AgentVersion)
@@ -407,10 +407,10 @@ func retryObserved(ctx context.Context, what string, fn func(context.Context) er
 		}
 		if attempt >= retryErrorAttempt {
 			slog.ErrorContext(ctx, "rds-agent: "+what+" still failing, retrying",
-				"attempt", attempt, "retryIn", delay, "err", err)
+				"attempt", attempt, "retry_in_ms", otelsetup.Millis(delay), "err", err)
 		} else {
 			slog.WarnContext(ctx, "rds-agent: "+what+" failed, retrying",
-				"attempt", attempt, "retryIn", delay, "err", err)
+				"attempt", attempt, "retry_in_ms", otelsetup.Millis(delay), "err", err)
 		}
 
 		select {

cmd/rds-agent/agent_test.go

@@ -192,7 +192,7 @@ func newTestAgent(t *testing.T, cfg config, cp controlPlane, run probeRunner) *A
 
 // staticProbe is a probeRunner that always reports the same exit code.
 func staticProbe(code int) probeRunner {
-	return func(context.Context, string, ...string) (int, error) { return code, nil }
+	return func(context.Context, string, ...string) (int, string, error) { return code, "", nil }
 }
 
 func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput {
@@ -207,6 +207,8 @@ func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput
 		MasterUsername:       "master",
 		MasterUserPassword:   password,
 		Port:                 5432,
+		ListenAddress:        testListenAddress,
+		ClientCIDR:           testClientCIDR,
 		DataVolumeID:         "vol-data-01",
 		DataVolumeSerial:     "voldata01",
 		VMGeneration:         1,
@@ -217,6 +219,13 @@ func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput
 
 const testPayloadID = "bp-0123456789abcdef"
 
+// The customer ENI address the engine binds, and the VPC range its client
+// authentication admits. writeHandoff refuses a config without either.
+const (
+	testListenAddress = "10.0.1.20"
+	testClientCIDR    = "10.0.0.0/16"
+)
+
 // Writes the completion receipt rds-init leaves behind, so the acknowledgement
 // step finds what a finished initialization would have produced.
 func writeReceipt(t *testing.T, a *Agent, payloadID, dbInstanceIdentifier string) {

cmd/rds-agent/bootstrap.go

@@ -32,10 +32,9 @@ func (a *Agent) bootstrap(ctx context.Context) error {
 		if fetched == nil {
 			return fmt.Errorf("bootstrap fetch returned no config")
 		}
-		// A control plane that predates the encrypted payload answers initialize
-		// with nothing to initialise with. Retrying re-dials the worker queue
-		// group until an upgraded node answers, so a mixed-version fleet heals
-		// within one boot rather than needing a reboot.
+		// A control plane predating the encrypted payload answers initialize with
+		// nothing to initialise with. Retrying re-dials the queue group until an
+		// upgraded node answers, so a mixed-version fleet heals within one boot.
 		if fetched.Mode == handlers_rds.BootstrapModeInitialize &&
 			(fetched.MasterUserPassword == nil || *fetched.MasterUserPassword == "") {
 			return fmt.Errorf("bootstrap fetch returned mode=%s with no master password", fetched.Mode)
@@ -61,7 +60,6 @@ func (a *Agent) bootstrap(ctx context.Context) error {
 	a.hb.clearBootstrapFailure()
 	if cfg.Port > 0 {
 		a.probe.setPort(int(cfg.Port))
-		a.engine.setPort(int(cfg.Port))
 	}
 	if cfg.BootstrapPending && cfg.PayloadID != "" {
 		a.pending = &pendingBootstrap{
@@ -98,6 +96,15 @@ func writeHandoff(dir string, cfg *handlers_rds.GetDBBootstrapConfigOutput) erro
 	if cfg.MasterUsername == "" {
 		return fmt.Errorf("bootstrap config carries no master username")
 	}
+	// Half a handoff would leave rds-init deciding what to bind and who to admit,
+	// and the only answers it could reach on its own are the wildcards this
+	// contract exists to remove.
+	if cfg.ListenAddress == "" {
+		return fmt.Errorf("bootstrap config carries no listen address for the engine to bind")
+	}
+	if cfg.ClientCIDR == "" {
+		return fmt.Errorf("bootstrap config carries no client CIDR to scope client authentication to")
+	}
 	if err := os.MkdirAll(dir, handoffDirMode); err != nil {
 		return fmt.Errorf("create handoff dir %s: %w", dir, err)
 	}
@@ -136,6 +143,8 @@ func renderBootstrapEnv(cfg *handlers_rds.GetDBBootstrapConfigOutput) string {
 	writeEnvLine(&b, "RDS_MODE", cfg.Mode)
 	writeEnvLine(&b, "RDS_DB_INSTANCE_IDENTIFIER", cfg.DBInstanceIdentifier)
 	writeEnvLine(&b, "RDS_MASTER_USERNAME", cfg.MasterUsername)
+	writeEnvLine(&b, "RDS_LISTEN_ADDRESS", cfg.ListenAddress)
+	writeEnvLine(&b, "RDS_CLIENT_CIDR", cfg.ClientCIDR)
 	writeEnvLine(&b, "RDS_DATA_VOLUME_ID", cfg.DataVolumeID)
 	writeEnvLine(&b, "RDS_DATA_VOLUME_SERIAL", cfg.DataVolumeSerial)
 	writeEnvLine(&b, "RDS_VM_GENERATION", strconv.FormatInt(cfg.VMGeneration, 10))
@@ -182,12 +191,8 @@ func shellQuote(s string) string {
 }
 
 // postgresql.conf syntax, which MariaDB's option files also parse. Values are
-// quoted so a setting with a space or unit suffix survives; the engine accepts
-// quoted numerics and booleans too.
-//
-// Names are the engine's rather than the customer's: a setting the server takes
-// only through SET has a different spelling at startup, and writing the wrong one
-// leaves the engine refusing to boot on a file already on the data volume.
+// quoted so a space or unit suffix survives. Names are the engine's startup
+// spellings, not the customer's: the wrong one leaves the engine unable to boot.
 func renderParameters(engineName string, params []handlers_rds.Parameter) (string, error) {
 	engine, err := handlers_rds.LookupEngine(engineName)
 	if err != nil {

cmd/rds-agent/bootstrap_test.go

@@ -18,6 +18,8 @@ func TestWriteHandoff_WritesEveryFileRootOnly(t *testing.T) {
 		Mode:               handlers_rds.BootstrapModeInitialize,
 		Engine:             "postgres",
 		MasterUsername:     "master",
+		ListenAddress:      "10.0.1.20",
+		ClientCIDR:         "10.0.0.0/16",
 		MasterUserPassword: &password,
 		DBName:             "appdb",
 		Port:               5433,
@@ -85,6 +87,8 @@ func TestWriteHandoff_AttachOmitsPassword(t *testing.T) {
 		Mode:           handlers_rds.BootstrapModeAttach,
 		Engine:         "postgres",
 		MasterUsername: "master",
+		ListenAddress:  "10.0.1.20",
+		ClientCIDR:     "10.0.0.0/16",
 		Port:           5432,
 	}); err != nil {
 		t.Fatalf("writeHandoff: %v", err)
@@ -116,6 +120,8 @@ func TestWriteHandoff_QuotesShellMetacharacters(t *testing.T) {
 		Mode:               handlers_rds.BootstrapModeInitialize,
 		Engine:             "postgres",
 		MasterUsername:     "master",
+		ListenAddress:      "10.0.1.20",
+		ClientCIDR:         "10.0.0.0/16",
 		MasterUserPassword: &password,
 		Port:               5432,
 	}); err != nil {
@@ -137,6 +143,8 @@ func TestWriteHandoff_SkipsHalfATLSPair(t *testing.T) {
 		Mode:               handlers_rds.BootstrapModeAttach,
 		Engine:             "postgres",
 		MasterUsername:     "master",
+		ListenAddress:      "10.0.1.20",
+		ClientCIDR:         "10.0.0.0/16",
 		Port:               5432,
 		ServingCertificate: "CERT",
 	}); err != nil {
@@ -177,6 +185,8 @@ func TestWriteHandoff_TightensExistingDirectory(t *testing.T) {
 		Mode:           handlers_rds.BootstrapModeAttach,
 		Engine:         "postgres",
 		MasterUsername: "master",
+		ListenAddress:  "10.0.1.20",
+		ClientCIDR:     "10.0.0.0/16",
 		Port:           5432,
 	}); err != nil {
 		t.Fatalf("writeHandoff: %v", err)

cmd/rds-agent/config.go

@@ -13,7 +13,6 @@ const (
 	defaultEnvFile    = "/etc/spinifex-rds/agent.env"
 	defaultGatewayCA  = "/etc/spinifex-rds/gateway-ca.pem"
 	defaultHandoffDir = "/run/spinifex-rds"
-	defaultEngineHost = "127.0.0.1"
 	// Where setup.sh stamps the engine the image bakes. The agent builds its
 	// engine implementation from this file rather than from anything delivered.
 	defaultEngineFile         = "/etc/spinifex-rds/engine"
@@ -38,7 +37,6 @@ type config struct {
 	// version alone rather than clearing it.
 	EngineVersion string
 	HandoffDir    string
-	EngineHost    string
 	EnginePort    int
 	PollWait      time.Duration
 
@@ -63,6 +61,10 @@ type config struct {
 	// is not running at all.
 	EnginePidFile      string
 	EngineProbeTimeout time.Duration
+	// Where the engine records why it would not start, which the probe quotes
+	// when the server is up but not serving. Overridable so a test can point it
+	// at a fixture.
+	EngineErrorLog string
 
 	// Where the data volume is mounted, and the two kernel surfaces a storage
 	// grow resolves its device from. Overridable so a test can point them at
@@ -85,7 +87,6 @@ func loadConfig(envFile string) config {
 		EngineFile:           get("RDS_ENGINE_FILE"),
 		EngineVersion:        get("RDS_ENGINE_VERSION"),
 		HandoffDir:           get("RDS_HANDOFF_DIR"),
-		EngineHost:           get("RDS_ENGINE_HOST"),
 		EngineBinDir:         get("RDS_ENGINE_BIN"),
 		EngineDataDir:        get("RDS_ENGINE_DATA"),
 		SocketDir:            get("RDS_SOCKET_DIR"),
@@ -93,6 +94,7 @@ func loadConfig(envFile string) config {
 		RCService:            get("RDS_RC_SERVICE"),
 		EngineService:        get("RDS_ENGINE_SERVICE"),
 		EnginePidFile:        get("RDS_ENGINE_PIDFILE"),
+		EngineErrorLog:       get("RDS_ENGINE_LOG"),
 		DataMount:            get("RDS_DATA_MOUNT"),
 		PollWait:             defaultPollWait,
 		EngineProbeTimeout:   defaultEngineProbeTimeout,
@@ -103,9 +105,6 @@ func loadConfig(envFile string) config {
 	if cfg.HandoffDir == "" {
 		cfg.HandoffDir = defaultHandoffDir
 	}
-	if cfg.EngineHost == "" {
-		cfg.EngineHost = defaultEngineHost
-	}
 	if cfg.EngineFile == "" {
 		cfg.EngineFile = defaultEngineFile
 	}
@@ -164,6 +163,9 @@ func (c *config) applyLayout(layout engineLayout) {
 	if c.EnginePidFile == "" {
 		c.EnginePidFile = layout.pidFile
 	}
+	if c.EngineErrorLog == "" {
+		c.EngineErrorLog = layout.errorLog
+	}
 	if c.DataMount == "" {
 		c.DataMount = layout.dataMount
 	}

cmd/rds-agent/config_test.go

@@ -26,8 +26,8 @@ func TestLoadConfig_Defaults(t *testing.T) {
 	if cfg.GatewayCA != defaultGatewayCA || cfg.HandoffDir != defaultHandoffDir {
 		t.Errorf("CA/handoff = %q/%q, want the built-in defaults", cfg.GatewayCA, cfg.HandoffDir)
 	}
-	if cfg.EngineHost != defaultEngineHost || cfg.PollWait != defaultPollWait {
-		t.Errorf("engine host/pollWait = %q/%v, want the built-in defaults", cfg.EngineHost, cfg.PollWait)
+	if cfg.PollWait != defaultPollWait {
+		t.Errorf("pollWait = %v, want the built-in default", cfg.PollWait)
 	}
 	if cfg.EngineProbeTimeout != defaultEngineProbeTimeout {
 		t.Errorf("engine probe timeout = %v, want %v", cfg.EngineProbeTimeout, defaultEngineProbeTimeout)

cmd/rds-agent/defaultparams_test.go

@@ -1,5 +1,9 @@
 package main
 
+//test:in-package — the agent is a main package, which has no external test
+// package to import it from, and this covers the unexported default parameter
+// set the image bakes.
+
 import (
 	"os"
 	"path/filepath"

cmd/rds-agent/engine.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"os"
 	"os/exec"
 	"os/user"
 	"path/filepath"
@@ -40,7 +41,9 @@ type engineOps interface {
 // a control-plane directive: nothing issues it, the agent runs it when the
 // engine fails to come up after a parameter change.
 type parameterRecovery interface {
-	// Reports whether the installed set differed and was replaced.
+	// Reports whether the installed set differed and was replaced. True with a
+	// non-nil error is a set already on disk whose re-derived TLS enforcement did
+	// not land: the caller still has to restart and report.
 	RestoreLastKnownGoodParameters(ctx context.Context) (bool, error)
 	Restart(ctx context.Context) error
 }
@@ -59,9 +62,6 @@ type engine interface {
 	engineOps
 	parameterRecovery
 	servingParameterRecorder
-	// Follows the port the control plane assigned, once the bootstrap fetch
-	// lands. Called from a different goroutine than the commands that read it.
-	setPort(port int)
 }
 
 // The guest layout an engine preset lays down, keyed by the engine the image
@@ -78,6 +78,10 @@ type engineLayout struct {
 	// Where the engine records the pid the probe checks for liveness. Empty for
 	// an engine whose probe answers without one.
 	pidFile string
+	// Where the engine records why it would not start. Empty for an engine whose
+	// image does not direct one, which leaves the probe reporting its own reason
+	// alone rather than guessing at a path.
+	errorLog string
 	// The engine's own liveness signal, which is the only part of engineProbe
 	// that is not shared.
 	newProbe func(cfg config, run probeRunner) *engineProbe
@@ -104,9 +108,7 @@ var engineLayouts = map[string]engineLayout{
 		dataMount: "/var/lib/postgresql",
 		port:      5432,
 		newProbe:  newPostgresProbe,
-		newEngine: func(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
-			return newPostgresEngine(cfg, run, startSess, probe), nil
-		},
+		newEngine: newPostgresEngineFromCatalog,
 	},
 	engineMariaDB: {
 		binDir:    "/usr/bin",
@@ -116,12 +118,14 @@ var engineLayouts = map[string]engineLayout{
 		service:   "mariadb",
 		dataMount: "/var/lib/mysql",
 		port:      3306,
-		// Alpine's packaged service starts mysqld_safe with
-		// --pid-file=/run/mysqld/$RC_SVCNAME.pid, and a command-line option beats
-		// the configuration file, so this is the path rather than anything the
-		// platform drop-in could name. setup.sh asserts it against the packaged
-		// init script at build time.
-		pidFile:   "/run/mysqld/mariadb.pid",
+		// Alpine's service passes --pid-file=/run/mysqld/$RC_SVCNAME.pid, and the
+		// command line beats the option file, so this is the path rather than
+		// anything the drop-in names. setup.sh asserts it at build time.
+		pidFile: "/run/mysqld/mariadb.pid",
+		// log_error in the platform drop-in rds-init writes. Without it
+		// mysqld_safe would send the server's errors to syslog, which nothing
+		// collects off a guest that has no SSH.
+		errorLog:  "/var/log/mysql/error.log",
 		newProbe:  newMariaDBProbe,
 		newEngine: newMariaDBEngineFromCatalog,
 	},
@@ -233,6 +237,34 @@ func serviceAction(ctx context.Context, run commandRunner, rcService, service, a
 	return nil
 }
 
+// Whether the installed set requires TLS of client connections. Read back from
+// the file rather than taken from the set being applied, because the restore and
+// repair paths put a file in place without holding one. Both engines' generated
+// parameter files are written in the same syntax, so one reader serves them.
+func installedTLSEnforcement(name, installedPath string) (bool, error) {
+	values, err := readOptionFile(installedPath)
+	if err != nil && !os.IsNotExist(err) {
+		return false, fmt.Errorf("read the installed parameters: %w", err)
+	}
+	value, ok := values[name]
+	if !ok {
+		// The ordinary state of an instance whose resolved set predates the
+		// parameter, and the whole of the migration story: it begins enforcing at
+		// its next boot with no control-plane work at all.
+		return true, nil
+	}
+	switch value {
+	case "1":
+		return true, nil
+	case "0":
+		return false, nil
+	}
+	// The resolver canonicalises every boolean, so anything else means the file
+	// was written by something other than the platform. The permissive reading is
+	// the one that must not be chosen for a setting that turns TLS off.
+	return false, fmt.Errorf("the installed parameters set %s to %q, which is neither 1 nor 0", name, value)
+}
+
 // A minimal environment for a child that is not carrying secrets.
 const defaultGuestPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 

cmd/rds-agent/engine_contract_test.go

@@ -1,5 +1,9 @@
 package main
 
+//test:in-package — the agent is a main package, which has no external test
+// package to import it from, and these cases are built on the unexported
+// engine interface and each implementation's own constructor.
+
 import (
 	"context"
 	"errors"

cmd/rds-agent/engine_mariadb.go

@@ -10,7 +10,6 @@ import (
 	"slices"
 	"strconv"
 	"strings"
-	"sync"
 	"syscall"
 	"time"
 
@@ -31,6 +30,13 @@ type controlPlaneRules struct {
 	// The catalog name behind a spelling read back out of an option file. Only a
 	// setting whose startup spelling differs from the customer's moves.
 	catalogName func(optionFileName string) string
+	// The catalog's data type for a setting, which decides how its value is
+	// rendered into SQL. Empty for a name the catalog does not carry.
+	dataType func(name string) string
+	// The engine's own name for the setting that requires TLS of a client
+	// connection. Held rather than spelled out here, so the guest derives
+	// enforcement from the same key the control plane resolves it under.
+	tlsEnforcementParameter string
 }
 
 func controlPlaneRulesFrom(meta handlers_rds.Engine) controlPlaneRules {
@@ -58,32 +64,31 @@ func controlPlaneRulesFrom(meta handlers_rds.Engine) controlPlaneRules {
 			}
 			return optionFileName
 		},
+		dataType: func(name string) string {
+			spec, ok := meta.LookupParameter(name)
+			if !ok {
+				return ""
+			}
+			return spec.DataType
+		},
+		tlsEnforcementParameter: meta.TLSEnforcementParameter(),
 	}
 }
 
-// The engine is reached over its unix socket as root, which the datadir's
-// unix_socket plugin authenticates from the connecting process' own uid — the
-// direct analogue of PostgreSQL's peer auth, and the reason the agent holds no
-// password of its own.
+// Reached over its unix socket as root, which the datadir's unix_socket plugin
+// authenticates from the connecting process' own uid. That is why the agent
+// holds no password of its own.
 type mariadbEngine struct {
 	quiesceState
+	parameterManager
 
 	rules     controlPlaneRules
 	run       commandRunner
 	startSess sessionRunner
 	client    string
-	admin     string
 	socket    string
 	rcService string
 	service   string
-	probe     *engineProbe
-
-	params parameterStore
-	// Serializes parameter installs, serving snapshots and rollback restores so
-	// none can copy or replace an intermediate configuration.
-	paramMu       sync.Mutex
-	repairTimeout time.Duration
-	repairPoll    time.Duration
 }
 
 var _ engine = (*mariadbEngine)(nil)
@@ -95,6 +100,14 @@ const (
 	// rds-init keeps for the platform and never hands to the customer.
 	mariadbSuperuser                  = "root"
 	mariadbProbeConnectTimeoutSeconds = 3
+	// How much of the engine's error log a probe reason carries. The server names
+	// what it refused on in its last few lines, and the reason reaches a customer
+	// through StatusInfos, so this quotes the tail rather than the whole file.
+	mariadbErrorLogTailBytes = 4096
+	mariadbErrorLogTailLines = 4
+	// How much of the probe client's stderr a reason carries, on the same grounds
+	// as the log tail above.
+	mariadbProbeStderrMaxBytes = 512
 	// Named by the platform drop-in rds-init writes, so both halves reach the
 	// same socket without either asserting it to the other.
 	mariadbSocketFile = "mysqld.sock"
@@ -132,24 +145,25 @@ func newMariaDBEngine(cfg config, rules controlPlaneRules, run commandRunner, st
 		run:       run,
 		startSess: startSess,
 		client:    filepath.Join(cfg.EngineBinDir, mariadbClientBinary),
-		admin:     filepath.Join(cfg.EngineBinDir, mariadbAdminBinary),
 		socket:    mariadbSocketPath(cfg),
 		rcService: cfg.RCService,
 		service:   cfg.EngineService,
-		probe:     probe,
-		params: parameterStore{
-			// The mount point rather than the datadir one level inside it: the
-			// include directory has to outlive the sweep a failed bootstrap runs.
-			dir:       filepath.Join(cfg.DataMount, "conf.d"),
-			installed: mariadbParametersFile,
-			lastGood:  mariadbLastGoodFile,
-			serving:   mariadbServingFile,
-			header:    mariadbParametersHeader,
-			osUser:    cfg.EngineUser,
-			engine:    engineMariaDB,
+		parameterManager: parameterManager{
+			probe: probe,
+			params: parameterStore{
+				// The mount point rather than the datadir one level inside it: the
+				// include directory has to outlive the sweep a failed bootstrap runs.
+				dir:       filepath.Join(cfg.DataMount, "conf.d"),
+				installed: mariadbParametersFile,
+				lastGood:  mariadbLastGoodFile,
+				serving:   mariadbServingFile,
+				header:    mariadbParametersHeader,
+				osUser:    cfg.EngineUser,
+				engine:    engineMariaDB,
+			},
+			repairTimeout: parameterRepairTimeout,
+			repairPoll:    parameterRepairPoll,
 		},
-		repairTimeout: parameterRepairTimeout,
-		repairPoll:    parameterRepairPoll,
 	}
 }
 
@@ -157,10 +171,15 @@ func mariadbSocketPath(cfg config) string {
 	return filepath.Join(cfg.SocketDir, mariadbSocketFile)
 }
 
-// Nothing this implementation runs needs the assigned port: every connection it
-// makes is over the engine's unix socket, and the platform drop-in rds-init
-// writes is what puts the server on the port. The probe follows it separately.
-func (e *mariadbEngine) setPort(int) {}
+// Every local connection, from the probe's ping to a held quiesce session.
+// --no-defaults so no option file can move it; --skip-ssl because the serving
+// cert names the endpoint rather than this socket, so a verifying client fails.
+func mariadbSocketConnectArgs(socket string) []string {
+	return []string{
+		"--no-defaults", "--protocol=socket", "--socket=" + socket,
+		"--user=" + mariadbSuperuser, "--skip-ssl",
+	}
+}
 
 // Reports whether a pid is a live process. EPERM is a process this agent may not
 // signal rather than one that is gone, which matters because a stale pidfile
@@ -179,33 +198,21 @@ func newMariaDBProbe(cfg config, run probeRunner) *engineProbe {
 	return newEngineProbe(cfg.EnginePort, mariadbProbeState(cfg, run, processAlive))
 }
 
-// Three stages, because MariaDB has no single signal that separates an engine
-// still coming up from one that is not there at all: during InnoDB crash
-// recovery mariadbd opens neither its socket nor its port, so a ping fails
-// exactly as it would against nothing.
-//
-// Losing that distinction would break the rollback guard, which treats a
-// recovering engine as a reason to reset its deadline. An instance killed hard
-// mid-write can spend minutes replaying its redo log; read as absent, the guard
-// would roll the parameter file back and restart the server mid-recovery,
-// discarding the work — a rollback that exists to break a boot loop would create
-// one, on an instance whose parameters were never at fault.
-//
-// The port is not consulted. Both stages below reach the server over its unix
-// socket, whose path does not move with the port.
+// Three stages: during InnoDB crash recovery mariadbd opens neither socket nor
+// port, so a ping cannot tell a recovering engine from an absent one — and
+// reading recovery as absent has the rollback guard restart a server mid-replay.
 func mariadbProbeState(cfg config, run probeRunner, alive processLivenessFn) probeStateFn {
 	pidFile, socket := cfg.EnginePidFile, mariadbSocketPath(cfg)
 	admin, client := filepath.Join(cfg.EngineBinDir, mariadbAdminBinary), filepath.Join(cfg.EngineBinDir, mariadbClientBinary)
-	connect := []string{
-		"--no-defaults", "--protocol=socket", "--socket=" + socket,
-		"--user=" + mariadbSuperuser,
-		fmt.Sprintf("--connect-timeout=%d", mariadbProbeConnectTimeoutSeconds),
-	}
+	// The probe alone bounds the connect: it must come back and report, where a
+	// statement the agent issues is allowed to take as long as the engine needs.
+	connect := append(mariadbSocketConnectArgs(socket),
+		fmt.Sprintf("--connect-timeout=%d", mariadbProbeConnectTimeoutSeconds))
 	probeTimeout := cfg.EngineProbeTimeout
 	if probeTimeout <= 0 {
 		probeTimeout = defaultEngineProbeTimeout
 	}
-	runBounded := func(ctx context.Context, name string, args ...string) (int, error) {
+	runBounded := func(ctx context.Context, name string, args ...string) (int, string, error) {
 		probeCtx, cancel := context.WithTimeout(ctx, probeTimeout)
 		defer cancel()
 		return run(probeCtx, name, args...)
@@ -223,31 +230,103 @@ func mariadbProbeState(cfg config, run probeRunner, alive processLivenessFn) pro
 		}
 
 		// The process is up from here on, so every remaining failure is an engine
-		// that is not serving yet rather than one that is gone — including a probe
-		// binary that will not run at all. Reporting absent against a live server
-		// would have the rollback guard restart one that is making progress.
-		switch code, err := runBounded(ctx, admin, append(slices.Clone(connect), "ping")...); {
+		// not serving yet rather than one that is gone. Reporting absent against a
+		// live server would have the rollback guard restart one making progress.
+		switch code, stderr, err := runBounded(ctx, admin, append(slices.Clone(connect), "ping")...); {
 		case err != nil:
 			return engineRecovering, fmt.Sprintf("engine probe could not run: %v", err)
 		case code != 0:
-			return engineRecovering, "engine is not answering on its socket yet (startup or crash recovery)"
+			// A server that is genuinely recovering and one that refused the client
+			// are indistinguishable from the socket. The server states its side in
+			// its own log, and the client states its side on stderr.
+			return engineRecovering, withProbeStderr(withEngineLogTail(
+				"engine is not answering on its socket yet (startup or crash recovery)",
+				cfg.EngineErrorLog), stderr)
 		}
 
-		// ping answers successfully even on ER_ACCESS_DENIED, so on its own it
-		// certifies a server that may be unable to execute anything at all. The
-		// statement is a literal in the argv rather than on stdin because it
-		// carries nothing secret and a probe reads no result back.
+		// ping answers even on ER_ACCESS_DENIED, so on its own it certifies a server
+		// that may be unable to execute anything. The statement rides argv rather
+		// than stdin: it carries nothing secret and a probe reads no result back.
 		query := append(slices.Clone(connect), "--batch", "--skip-column-names", "--execute=SELECT 1")
-		switch code, err := runBounded(ctx, client, query...); {
+		switch code, stderr, err := runBounded(ctx, client, query...); {
 		case err != nil:
 			return engineRecovering, fmt.Sprintf("engine probe could not run: %v", err)
 		case code != 0:
-			return engineRecovering, "engine answered its socket but would not execute a statement"
+			return engineRecovering, withProbeStderr(
+				"engine answered its socket but would not execute a statement", stderr)
 		}
 		return engineServing, ""
 	}
 }
 
+// Appends what the probe client said on stderr. A client that fails during the
+// handshake is refused by the server without the server ever logging why, so
+// this is the only account of that half of the exchange.
+func withProbeStderr(reason, stderr string) string {
+	stderr = strings.TrimSpace(stderr)
+	if stderr == "" {
+		return reason
+	}
+	return reason + "; the probe client reported: " + collapseProbeStderr(stderr)
+}
+
+// One line, bounded: the reason reaches a customer through StatusInfos, and a
+// client that repeats itself per attempt would otherwise crowd out the rest.
+func collapseProbeStderr(stderr string) string {
+	line := strings.Join(strings.Fields(strings.ReplaceAll(stderr, "\n", " ")), " ")
+	if len(line) > mariadbProbeStderrMaxBytes {
+		return line[:mariadbProbeStderrMaxBytes] + "..."
+	}
+	return line
+}
+
+// Appends the last few lines of the engine's error log to reason. An unset path,
+// an unreadable file or an empty one leaves reason alone: the probe's own
+// statement is still true, and a guessed cause would be worse than none.
+func withEngineLogTail(reason, path string) string {
+	tail := engineLogTail(path)
+	if tail == "" {
+		return reason
+	}
+	return reason + "; the engine's log ends: " + tail
+}
+
+func engineLogTail(path string) string {
+	if path == "" {
+		return ""
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return ""
+	}
+	defer func() { _ = f.Close() }()
+
+	info, err := f.Stat()
+	if err != nil {
+		return ""
+	}
+	offset := max(info.Size()-mariadbErrorLogTailBytes, 0)
+	buf := make([]byte, info.Size()-offset)
+	if _, err := f.ReadAt(buf, offset); err != nil {
+		return ""
+	}
+
+	lines := strings.Split(string(buf), "\n")
+	// A read that started mid-file almost certainly started mid-line, and half a
+	// line is more misleading than one fewer.
+	if offset > 0 && len(lines) > 0 {
+		lines = lines[1:]
+	}
+	kept := make([]string, 0, mariadbErrorLogTailLines)
+	for i := len(lines) - 1; i >= 0 && len(kept) < mariadbErrorLogTailLines; i-- {
+		if line := strings.TrimSpace(lines[i]); line != "" {
+			kept = append(kept, line)
+		}
+	}
+	slices.Reverse(kept)
+	return strings.Join(kept, " | ")
+}
+
 func readPidFile(path string) (int, error) {
 	raw, err := os.ReadFile(path)
 	if err != nil {
@@ -265,11 +344,8 @@ func (e *mariadbEngine) Stop(ctx context.Context) error {
 }
 
 // Only the parameter rollback and a failed apply call this: a restart the
-// control plane wants goes through RebootDBInstance, which cycles the VM.
-//
-// The serving copy is refreshed first, because the server is about to start on
-// whatever is installed. That is exactly what rds-init does on every boot, and
-// it is what keeps the two equal by construction whenever the engine starts.
+// control plane wants goes through RebootDBInstance, which cycles the VM. The
+// serving copy is refreshed first, since the server starts on what is installed.
 func (e *mariadbEngine) Restart(ctx context.Context) error {
 	if err := e.recordServingCopy(); err != nil {
 		return err
@@ -298,6 +374,39 @@ func sqlLiteral(value string) string {
 	return strings.ReplaceAll(strings.ReplaceAll(value, `\`, `\\`), `'`, `''`)
 }
 
+// The spellings MariaDB parses for a boolean system variable, and the only ones
+// the catalog's own enum narrows a boolean to.
+var mariadbBooleanValues = map[string]string{
+	"on": "ON", "off": "OFF", "true": "ON", "false": "OFF", "1": "ON", "0": "OFF",
+}
+
+// One SET GLOBAL right-hand side. MariaDB refuses a quoted literal for a numeric
+// or boolean with ER_WRONG_TYPE_FOR_VAR, so the catalog's data type picks the
+// form; re-rendering from the parsed value is what keeps it safe to interpolate.
+func mariadbSetValue(dataType, value string) (string, error) {
+	switch dataType {
+	case handlers_rds.ParamTypeInteger:
+		n, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64)
+		if err != nil {
+			return "", fmt.Errorf("value %q is not an integer", value)
+		}
+		return strconv.FormatInt(n, 10), nil
+	case handlers_rds.ParamTypeReal:
+		f, err := strconv.ParseFloat(strings.TrimSpace(value), 64)
+		if err != nil {
+			return "", fmt.Errorf("value %q is not a real number", value)
+		}
+		return strconv.FormatFloat(f, 'g', -1, 64), nil
+	case handlers_rds.ParamTypeBoolean:
+		if literal, ok := mariadbBooleanValues[strings.ToLower(strings.TrimSpace(value))]; ok {
+			return literal, nil
+		}
+		return "", fmt.Errorf("value %q is not a boolean", value)
+	default:
+		return "'" + sqlLiteral(value) + "'", nil
+	}
+}
+
 // Turns off the two logs that would otherwise copy a statement's text, and pins
 // the mode the escaping above assumes. All three are session-scoped: a rotation
 // must not leave the customer's own general log switched off behind it.
@@ -314,10 +423,9 @@ func (e *mariadbEngine) SetPassword(ctx context.Context, username, password stri
 		return fmt.Errorf("set-password requires both %s and %s",
 			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
 	}
-	// This runs as the server's own superuser over the socket, so a reserved name
-	// in the command payload would rotate the platform's account rather than the
-	// customer's. The rest of the rule matters just as much here, because the name
-	// is interpolated into the statement below rather than quoted by the client.
+	// This runs as the server's own superuser, so a reserved name in the payload
+	// would rotate the platform's account. The whole rule matters here, not just
+	// the reserved set: the name is interpolated below rather than client-quoted.
 	if err := e.rules.validateUsername(username); err != nil {
 		return fmt.Errorf("refusing to set the password of role %q: %w", username, err)
 	}
@@ -334,14 +442,9 @@ func (e *mariadbEngine) SetPassword(ctx context.Context, username, password stri
 	return nil
 }
 
-// Installs the resolved set and applies the half of it a running server can
-// adopt. MariaDB re-reads no configuration file while it is up, so SET GLOBAL is
-// the only way an immediate apply reaches it; the static half waits in the
-// installed file for the next start, and is what comes back as pending.
-//
-// There is no offline parser to check the set with first — MariaDB ships no
-// equivalent of postgres -C — so the safety net for a static value the server
-// will not start on is the boot-time rollback to the last accepted set.
+// Installs the resolved set and applies the half a running server can adopt.
+// MariaDB re-reads no file while up, so SET GLOBAL is the only immediate path.
+// It has no offline parser, so the safety net is the boot-time rollback.
 func (e *mariadbEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
 	e.paramMu.Lock()
 	defer e.paramMu.Unlock()
@@ -371,6 +474,13 @@ func (e *mariadbEngine) ApplyParameters(ctx context.Context, params []handlers_r
 		return nil, errors.Join(errors.New("the engine entered startup or recovery during the parameter apply"), restore())
 	}
 
+	// Before the set reaches the server: enforcement is one of the values below,
+	// and a server told to require a transport it cannot offer would refuse every
+	// client the statement after.
+	if err := e.requireServingTLSForEnforcement(ctx); err != nil {
+		return nil, errors.Join(err, restore())
+	}
+
 	if err := e.applyDynamicParameters(ctx, params); err != nil {
 		// The engine may have gone down after the first probe. In that case start it
 		// on the newly installed set rather than restoring one that already left it
@@ -380,26 +490,37 @@ func (e *mariadbEngine) ApplyParameters(ctx context.Context, params []handlers_r
 		}
 		return nil, errors.Join(err, restore())
 	}
+	// Last, and on its own: the batch above is not a transaction, so moving the
+	// TLS posture inside it would leave enforcement changed on a set the server
+	// went on to refuse, with only the parameter file rolled back.
+	if err := e.applyTLSEnforcement(ctx); err != nil {
+		if state, _ := e.probe.state(ctx); state == engineAbsent {
+			return e.restartOnRepairSetLocked(ctx)
+		}
+		return nil, errors.Join(err, restore())
+	}
 	return e.pendingRestartParameters(ctx)
 }
 
-// The dynamic half of the set, in one invocation that stops at the first refusal
-// so the server's own message names the setting it would not take.
+// The dynamic half, in one invocation that stops at the first refusal so the
+// server names the setting it would not take. Only catalog names are emitted,
+// which is what makes the identifier safe to build into the statement.
 //
-// Only names the catalog carries are emitted, which is also what keeps the
-// statement safe to build: a name is an identifier here, and the classification
-// that selects it comes from the same table the API validated the request
-// against. MariaDB cannot set several globals atomically, so a value set before
-// a refusal stays live until the next restart, when the restored file wins.
+// TLS enforcement is excluded: applyTLSEnforcement issues it after this batch
+// has succeeded.
 func (e *mariadbEngine) applyDynamicParameters(ctx context.Context, params []handlers_rds.Parameter) error {
 	var b strings.Builder
 	b.WriteString("SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';\n")
 	applied := 0
 	for _, p := range params {
-		if p.Name == "" || e.rules.isStatic(p.Name) {
+		if p.Name == "" || p.Name == e.rules.tlsEnforcementParameter || e.rules.isStatic(p.Name) {
 			continue
 		}
-		fmt.Fprintf(&b, "SET GLOBAL %s = '%s';\n", p.Name, sqlLiteral(p.Value))
+		value, err := mariadbSetValue(e.rules.dataType(p.Name), p.Value)
+		if err != nil {
+			return fmt.Errorf("apply the dynamic parameters: %s: %w", p.Name, err)
+		}
+		fmt.Fprintf(&b, "SET GLOBAL %s = %s;\n", p.Name, value)
 		applied++
 	}
 	if applied == 0 {
@@ -411,22 +532,106 @@ func (e *mariadbEngine) applyDynamicParameters(ctx context.Context, params []han
 	return nil
 }
 
+// Puts the installed set's enforcement on the running server. The value comes
+// from the installed file rather than the set in hand, so a set that predates
+// the parameter enforces here rather than leaving the server on mariadbd's own
+// default of off while the API reports enforcement.
+func (e *mariadbEngine) applyTLSEnforcement(ctx context.Context) error {
+	enforce, err := installedTLSEnforcement(e.rules.tlsEnforcementParameter, e.params.installedPath())
+	if err != nil {
+		return err
+	}
+	previous, err := e.readTLSEnforcement(ctx)
+	if err != nil {
+		return err
+	}
+	if previous == enforce {
+		return nil
+	}
+	applyErr := e.setTLSEnforcement(ctx, enforce)
+	if applyErr == nil {
+		return nil
+	}
+	// The client can fail without saying whether the server ran the statement, so
+	// the posture it left behind is read rather than assumed unchanged: the one
+	// outcome that must not survive a failed apply is plaintext nobody asked for.
+	live, readErr := e.readTLSEnforcement(ctx)
+	if readErr != nil {
+		return errors.Join(applyErr, readErr)
+	}
+	if live == previous {
+		return applyErr
+	}
+	return errors.Join(applyErr, e.setTLSEnforcement(ctx, previous))
+}
+
+func (e *mariadbEngine) readTLSEnforcement(ctx context.Context) (bool, error) {
+	name := e.rules.tlsEnforcementParameter
+	out, err := e.clientRun(ctx, "SELECT @@global."+name+";\n")
+	if err != nil {
+		return false, fmt.Errorf("read whether the engine requires TLS of clients: %w", err)
+	}
+	switch value := strings.TrimSpace(out); value {
+	case "1":
+		return true, nil
+	case "0":
+		return false, nil
+	default:
+		return false, fmt.Errorf("the engine reports %s as %q, which is neither 1 nor 0", name, value)
+	}
+}
+
+func (e *mariadbEngine) setTLSEnforcement(ctx context.Context, enforce bool) error {
+	name, value := e.rules.tlsEnforcementParameter, "0"
+	if enforce {
+		value = "1"
+	}
+	// Rendered the way the batch renders every other boolean, so the statement the
+	// server sees does not depend on which path issued it.
+	literal, err := mariadbSetValue(handlers_rds.ParamTypeBoolean, value)
+	if err != nil {
+		return fmt.Errorf("render %s: %w", name, err)
+	}
+	sql := fmt.Sprintf("SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';\nSET GLOBAL %s = %s;\n", name, literal)
+	if _, err := e.clientRun(ctx, sql); err != nil {
+		return fmt.Errorf("set %s to %s: %w", name, literal, err)
+	}
+	return nil
+}
+
+// Refuses to require TLS of clients from a server that is not serving it, which
+// would reject every TCP connection while the parameter path reported success.
+// Read back from the installed file rather than taken from the set in hand, so
+// what is checked is also what the server reads at its next start.
+func (e *mariadbEngine) requireServingTLSForEnforcement(ctx context.Context) error {
+	enforce, err := installedTLSEnforcement(e.rules.tlsEnforcementParameter, e.params.installedPath())
+	if err != nil {
+		return err
+	}
+	if !enforce {
+		return nil
+	}
+	// DISABLED is a server built with TLS and started without a certificate,
+	// which is what a boot that was handed none leaves behind. The agent's own
+	// connections are unaffected either way: MariaDB counts a unix socket as a
+	// secure transport.
+	out, err := e.clientRun(ctx, "SELECT @@global.have_ssl;\n")
+	if err != nil {
+		return fmt.Errorf("read whether the engine is serving TLS: %w", err)
+	}
+	if serving := strings.TrimSpace(out); serving != "YES" {
+		return fmt.Errorf("the engine is serving without TLS (have_ssl = %q), so requiring it of clients would reject every connection", serving)
+	}
+	return nil
+}
+
 func (e *mariadbEngine) restartOnRepairSetLocked(ctx context.Context) ([]string, error) {
 	return awaitRepairedEngine(ctx, e.probe, e.Restart, e.pendingRestartParameters, e.repairTimeout, e.repairPoll)
 }
 
-// MariaDB has no pending_restart. Nothing in information_schema reports a
-// setting the server has stored but not adopted, so the answer is computed from
-// the files instead: the catalog-static keys whose value in the installed
-// drop-in differs from the one the server actually started on.
-//
-// Comparing live values against the desired set was rejected. mysqld silently
-// rewrites values at startup — rounding the buffer pool and log file sizes, and
-// downgrading max_connections when it cannot obtain enough file descriptors —
-// and each rewrite would read as a permanent pending restart: un-clearable
-// pending-reboot state, permanent drift, and a last-known-good copy frozen for
-// good, silently disabling a recovery mechanism only exercised during an
-// incident.
+// MariaDB has no pending_restart, so this is computed from the files: the
+// catalog-static keys whose installed value differs from the one the server
+// started on. Live values are not compared — mysqld silently rewrites them.
 func (e *mariadbEngine) pendingRestartParameters(context.Context) ([]string, error) {
 	installed, err := readOptionFile(e.params.installedPath())
 	if err != nil {
@@ -529,20 +734,14 @@ func normaliseOptionName(name string) string {
 	return strings.ReplaceAll(strings.TrimSpace(name), "-", "_")
 }
 
-// How long any one BACKUP STAGE waits for the metadata locks it needs. Well
-// under the control plane's own quiesce timeout, so a stage that cannot take its
-// lock fails and is reported rather than being abandoned mid-wait with the
-// request still queued in front of live traffic.
+// How long any one BACKUP STAGE waits for its metadata locks. Well under the
+// control plane's quiesce timeout, so a stage that cannot take its lock is
+// reported rather than left queued in front of live traffic.
 const mariadbQuiesceLockWait = 20 * time.Second
 
-// MariaDB's own backup API rather than FLUSH TABLES WITH READ LOCK. FTWRL turns
-// a snapshot into a write outage: the whole database is read-only until it is
-// released, so a control plane that died after issuing quiesce would leave the
-// customer read-only for the full hold, on a schedule and unattended. Its
-// acquisition phase is unbounded too, with writes queueing behind a pending lock
-// request. BACKUP STAGE blocks commits rather than all writes and its flush
-// stage covers Aria and MyISAM, so it yields a better consistency point while
-// being far less disruptive — and it is what mariadb-backup itself uses.
+// MariaDB's own backup API rather than FLUSH TABLES WITH READ LOCK: FTWRL makes
+// the whole database read-only for the full hold, and its acquisition phase is
+// unbounded. BACKUP STAGE blocks commits instead, and covers Aria and MyISAM.
 func (e *mariadbEngine) Quiesce(ctx context.Context, label string, hold time.Duration) error {
 	if err := validateQuiesceRequest(label, hold); err != nil {
 		return err
@@ -568,10 +767,9 @@ func (e *mariadbEngine) Quiesce(ctx context.Context, label string, hold time.Dur
 		return fmt.Errorf("open a backup session: %w", err)
 	}
 
-	// The stages run in order and on one connection: MariaDB releases the whole
-	// hold with the session that took it, which is what bounds a control plane
-	// that dies mid-snapshot. Fed one at a time so a stage that cannot take its
-	// locks is named rather than reported as a line number.
+	// In order and on one connection: MariaDB releases the hold with the session
+	// that took it, which bounds a control plane that dies mid-snapshot. Fed one
+	// at a time so a stage that cannot take its locks is named.
 	stages := []string{
 		fmt.Sprintf("SET SESSION lock_wait_timeout = %d;\n", int(mariadbQuiesceLockWait.Seconds())),
 		"BACKUP STAGE START;\n",
@@ -592,27 +790,8 @@ func (e *mariadbEngine) Quiesce(ctx context.Context, label string, hold time.Dur
 	return nil
 }
 
-// Ends the backup cleanly. A missing hold is an error rather than a silent
-// success: it means the deadline fired first, so the snapshot the control plane
-// just took was not taken against a held stage.
 func (e *mariadbEngine) Unquiesce(ctx context.Context) error {
-	held := e.takeHold()
-	if held == nil {
-		return errors.New("the engine is not quiesced; the backup hold had already expired")
-	}
-
-	// The release has to run on the session that took the stages, and the session
-	// is closed either way — the engine ends an unreleased backup with it.
-	execErr := held.session.Exec(ctx, "BACKUP STAGE END;\n")
-	closeErr := held.session.Close()
-	if execErr != nil {
-		return fmt.Errorf("take the engine out of backup mode: %w", errors.Join(execErr, closeErr))
-	}
-	if closeErr != nil {
-		return fmt.Errorf("close the backup session: %w", closeErr)
-	}
-	slog.Info("rds-agent: engine released from backup mode", "label", held.label)
-	return nil
+	return e.releaseHold(ctx, "BACKUP STAGE END;\n")
 }
 
 // Feeds sql on stdin rather than as an argument, so a statement is never visible
@@ -626,13 +805,10 @@ func (e *mariadbEngine) clientRun(ctx context.Context, sql string) (string, erro
 	})
 }
 
-// --no-defaults so no option file can move the connection this agent makes, and
 // --unbuffered so a held session's output reaches the reader as each statement
-// finishes rather than when a pipe buffer fills. No User: the agent is root, and
-// root is what the datadir's unix_socket plugin authenticates.
+// finishes rather than when a pipe buffer fills. No User on the command: the
+// connection args carry it, and the agent runs as the account they name.
 func (e *mariadbEngine) clientArgs() []string {
-	return []string{
-		"--no-defaults", "--batch", "--skip-column-names", "--unbuffered",
-		"--protocol=socket", "--socket=" + e.socket, "--user=" + mariadbSuperuser,
-	}
+	return append(mariadbSocketConnectArgs(e.socket),
+		"--batch", "--skip-column-names", "--unbuffered")
 }

cmd/rds-agent/engine_mariadb_log_test.go

@@ -0,0 +1,211 @@
+package main
+
+//test:in-package — the agent is a main package, which has no external test
+// package to import it from, and this covers the unexported engineLogTail and
+// probe-reason helpers directly.
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+)
+
+const mariadbRDSInitScript = "../../scripts/images/rds-mariadb/rds-init"
+
+func writeEngineLog(t *testing.T, body string) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "error.log")
+	if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
+		t.Fatalf("write engine log: %v", err)
+	}
+	return path
+}
+
+// The whole point of the log: "not answering on its socket" is the same message
+// for a server still recovering and for one that refused to start, and only the
+// engine's own log separates them.
+func TestMariaDBProbe_QuotesTheEngineLogWhenTheSocketIsSilent(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	cfg.EngineErrorLog = writeEngineLog(t, strings.Join([]string{
+		"2026-08-12  1:49:26 0 [Note] Starting MariaDB 11.8.0",
+		"2026-08-12  1:49:26 0 [ERROR] InnoDB: Cannot allocate memory for the buffer pool",
+		"2026-08-12  1:49:26 0 [ERROR] Plugin 'InnoDB' registration as a STORAGE ENGINE failed.",
+		"",
+	}, "\n"))
+	writePid(t, cfg.EnginePidFile, os.Getpid())
+
+	state, message := mariadbProbeState(cfg,
+		func(context.Context, string, ...string) (int, string, error) { return 1, "", nil },
+		func(int) bool { return true })(t.Context(), int64(cfg.EnginePort))
+
+	if state != engineRecovering {
+		t.Errorf("state = %v, want recovering", state)
+	}
+	if !strings.Contains(message, "Cannot allocate memory for the buffer pool") {
+		t.Errorf("message = %q, want the engine's own refusal quoted", message)
+	}
+}
+
+func TestMariaDBProbe_SurvivesAnAbsentEngineLog(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	cfg.EngineErrorLog = filepath.Join(t.TempDir(), "does-not-exist.log")
+	writePid(t, cfg.EnginePidFile, os.Getpid())
+
+	state, message := mariadbProbeState(cfg,
+		func(context.Context, string, ...string) (int, string, error) { return 1, "", nil },
+		func(int) bool { return true })(t.Context(), int64(cfg.EnginePort))
+
+	if state != engineRecovering {
+		t.Errorf("state = %v, want recovering", state)
+	}
+	if message != "engine is not answering on its socket yet (startup or crash recovery)" {
+		t.Errorf("message = %q, want the bare reason when there is no log to quote", message)
+	}
+}
+
+// A client refused during the handshake is closed by the server without the
+// server ever logging a cause, so the engine log alone leaves the failure
+// unexplained and only the client's own stderr names it.
+func TestMariaDBProbe_QuotesTheProbeClientWhenItIsRefused(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	cfg.EngineErrorLog = writeEngineLog(t, "2026-08-12  4:32:39 148 [Warning] Aborted connection 148\n")
+	writePid(t, cfg.EnginePidFile, os.Getpid())
+
+	state, message := mariadbProbeState(cfg,
+		func(context.Context, string, ...string) (int, string, error) {
+			return 1, "ERROR 2026 (HY000): TLS/SSL error: self-signed certificate\n", nil
+		},
+		func(int) bool { return true })(t.Context(), int64(cfg.EnginePort))
+
+	if state != engineRecovering {
+		t.Errorf("state = %v, want recovering", state)
+	}
+	if !strings.Contains(message, "self-signed certificate") {
+		t.Errorf("message = %q, want the probe client's own error quoted", message)
+	}
+	if !strings.Contains(message, "Aborted connection") {
+		t.Errorf("message = %q, want the engine log kept alongside the client's error", message)
+	}
+}
+
+func TestCollapseProbeStderr(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{name: "folds to one line", in: "ERROR 2026:\n  TLS error\n", want: "ERROR 2026: TLS error"},
+		{name: "empty stays empty", in: "   \n", want: ""},
+		{
+			name: "bounded",
+			in:   strings.Repeat("y", mariadbProbeStderrMaxBytes+50),
+			want: strings.Repeat("y", mariadbProbeStderrMaxBytes) + "...",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := collapseProbeStderr(tt.in); got != tt.want {
+				t.Errorf("collapseProbeStderr() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+// A probe that says nothing must not append an empty clause: the reason reaches
+// a customer, and a dangling "reported:" would read as truncated output.
+func TestWithProbeStderrLeavesReasonAloneWhenTheClientIsSilent(t *testing.T) {
+	const reason = "engine is not answering on its socket yet"
+	if got := withProbeStderr(reason, "  \n "); got != reason {
+		t.Errorf("withProbeStderr() = %q, want the reason unchanged", got)
+	}
+}
+
+func TestEngineLogTail(t *testing.T) {
+	tests := []struct {
+		name        string
+		body        string
+		want        string
+		wantMissing string
+	}{
+		{
+			name: "keeps only the last lines",
+			body: "one\ntwo\nthree\nfour\nfive\nsix\n",
+			want: "three | four | five | six",
+		},
+		{
+			name: "skips blank lines",
+			body: "alpha\n\n\nbeta\n\n",
+			want: "alpha | beta",
+		},
+		{
+			name: "empty file yields nothing",
+			body: "",
+			want: "",
+		},
+		{
+			name:        "drops a line the byte cap cut in half",
+			body:        strings.Repeat("x", mariadbErrorLogTailBytes) + "\nlast\n",
+			want:        "last",
+			wantMissing: "x",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := engineLogTail(writeEngineLog(t, tt.body))
+			if got != tt.want {
+				t.Errorf("engineLogTail() = %q, want %q", got, tt.want)
+			}
+			if tt.wantMissing != "" && strings.Contains(got, tt.wantMissing) {
+				t.Errorf("engineLogTail() = %q, want no partial line", got)
+			}
+		})
+	}
+}
+
+func TestEngineLogTailUnsetPath(t *testing.T) {
+	if got := engineLogTail(""); got != "" {
+		t.Errorf("engineLogTail(\"\") = %q, want empty", got)
+	}
+}
+
+// The agent reads a path the image writes, and nothing at runtime would notice
+// the two drifting apart: the log would simply always be missing and every
+// refusal would go back unexplained again.
+func TestMariaDBErrorLogPathMatchesTheImage(t *testing.T) {
+	raw, err := os.ReadFile(mariadbRDSInitScript)
+	if err != nil {
+		t.Fatalf("read rds-init: %v", err)
+	}
+	script := string(raw)
+
+	logDir := shellDefault(t, script, "LOG_DIR", "RDS_LOG_DIR")
+	engineLog := shellDefault(t, script, "ENGINE_LOG", "RDS_ENGINE_LOG")
+	want := strings.Replace(engineLog, "${LOG_DIR}", logDir, 1)
+
+	if got := engineLayouts[engineMariaDB].errorLog; got != want {
+		t.Errorf("layout errorLog = %q, but rds-init writes log_error = %q", got, want)
+	}
+}
+
+// Pulls the fallback out of a `NAME="${OVERRIDE:-fallback}"` assignment.
+func shellDefault(t *testing.T, script, name, override string) string {
+	t.Helper()
+	re := regexp.MustCompile(`(?m)^` + name + `="\$\{` + override + `:-(.*)\}"$`)
+	m := re.FindStringSubmatch(script)
+	if m == nil {
+		t.Fatalf("rds-init has no %s assignment defaulting from %s", name, override)
+	}
+	return m[1]
+}
+
+func TestWithEngineLogTailLeavesReasonAloneWithoutALog(t *testing.T) {
+	const reason = "engine is not answering on its socket yet"
+	if got := withEngineLogTail(reason, ""); got != reason {
+		t.Errorf("withEngineLogTail() = %q, want the reason unchanged", got)
+	}
+}

cmd/rds-agent/engine_mariadb_test.go

@@ -1,5 +1,9 @@
 package main
 
+//test:in-package — the agent is a main package, which has no external test
+// package to import it from, and these cases drive the unexported mariadbEngine
+// and its parameter store directly.
+
 import (
 	"context"
 	"errors"
@@ -27,6 +31,20 @@ func testMariaDBRules() controlPlaneRules {
 		"max_connections":         handlers_rds.ApplyTypeDynamic,
 		"long_query_time":         handlers_rds.ApplyTypeDynamic,
 		"time_zone":               handlers_rds.ApplyTypeDynamic,
+
+		"innodb_adaptive_hash_index": handlers_rds.ApplyTypeDynamic,
+		"log_output":                 handlers_rds.ApplyTypeDynamic,
+		"require_secure_transport":   handlers_rds.ApplyTypeDynamic,
+	}
+	dataTypes := map[string]string{
+		"innodb_buffer_pool_size":    handlers_rds.ParamTypeInteger,
+		"innodb_log_file_size":       handlers_rds.ParamTypeInteger,
+		"max_connections":            handlers_rds.ParamTypeInteger,
+		"long_query_time":            handlers_rds.ParamTypeReal,
+		"time_zone":                  handlers_rds.ParamTypeString,
+		"innodb_adaptive_hash_index": handlers_rds.ParamTypeBoolean,
+		"log_output":                 handlers_rds.ParamTypeEnum,
+		"require_secure_transport":   handlers_rds.ParamTypeBoolean,
 	}
 	reserved := []string{"root", "mysql", "mariadb.sys", "rdsadmin", "public"}
 
@@ -53,6 +71,8 @@ func testMariaDBRules() controlPlaneRules {
 			}
 			return optionFileName
 		},
+		dataType:                func(name string) string { return dataTypes[name] },
+		tlsEnforcementParameter: "require_secure_transport",
 	}
 }
 
@@ -61,6 +81,32 @@ func isTestIdentifierRune(r rune) bool {
 }
 
 func newTestMariaDBEngine(t *testing.T, run commandRunner) *mariadbEngine {
+	t.Helper()
+	return newScriptedTestMariaDBEngine(t, withMariaDBReadBacks(run))
+}
+
+// Answers the read-backs an apply runs against a live server, and does not
+// record the calls: an apply against a set that says nothing about TLS still
+// asks both, and a case counting the statements it issued is about the set
+// rather than the guard.
+func withMariaDBReadBacks(run commandRunner) commandRunner {
+	return func(ctx context.Context, c command) (string, error) {
+		switch {
+		case strings.Contains(c.Stdin, "have_ssl"):
+			return "YES\n", nil
+		// Already enforcing, which is what rds-init leaves behind: an apply that
+		// does not move the posture issues no statement for it.
+		case strings.Contains(c.Stdin, "@@global.require_secure_transport"):
+			return "1\n", nil
+		}
+		return run(ctx, c)
+	}
+}
+
+// The same engine with the read-back left to the runner, for the cases that are
+// about what the engine does with that answer rather than about the rest of an
+// apply.
+func newScriptedTestMariaDBEngine(t *testing.T, run commandRunner) *mariadbEngine {
 	t.Helper()
 	cfg := testLoadConfig(t, engineMariaDB)
 	cfg.DataMount = t.TempDir()
@@ -83,10 +129,11 @@ func newTestMariaDBEngine(t *testing.T, run commandRunner) *mariadbEngine {
 // that is registered: what a spec says about a parameter is what the guest must
 // see, and an unlisted name must read as static rather than as dynamic.
 func TestControlPlaneRulesFrom_TakesTheClassificationFromTheCatalog(t *testing.T) {
-	rules := controlPlaneRulesFrom(postgresEngineMeta)
+	meta := testPostgresEngineMeta(t)
+	rules := controlPlaneRulesFrom(meta)
 
 	for name, want := range map[string]bool{"shared_buffers": true, "work_mem": false} {
-		spec, ok := postgresEngineMeta.LookupParameter(name)
+		spec, ok := meta.LookupParameter(name)
 		if !ok {
 			t.Fatalf("the catalog no longer carries %s", name)
 		}
@@ -133,6 +180,44 @@ func TestMariaDBEngine_SetPasswordKeepsTheSecretOutOfArgvAndEnv(t *testing.T) {
 	}
 }
 
+// The serving engine offers TLS whose cert names the endpoint rather than this
+// socket, so a client that verifies is refused during the handshake and every
+// statement the agent issues fails. The probe and the statement client are
+// separate argv sites, and only one of them carrying the flag is the bug.
+func TestMariaDBEngine_EveryLocalClientDeclinesTLS(t *testing.T) {
+	engine := newTestMariaDBEngine(t, (&recordingRunner{}).run)
+	cfg := mariadbTestProbeConfig(t)
+
+	sites := map[string][]string{
+		"statement client": engine.clientArgs(),
+		"probe":            mariadbSocketConnectArgs(mariadbSocketPath(cfg)),
+	}
+	for name, args := range sites {
+		t.Run(name, func(t *testing.T) {
+			if !slices.Contains(args, "--skip-ssl") {
+				t.Errorf("%s args = %v, want TLS declined on the unix socket", name, args)
+			}
+			if !slices.Contains(args, "--no-defaults") {
+				t.Errorf("%s args = %v, want no option file able to move the connection", name, args)
+			}
+		})
+	}
+}
+
+// The shared constructor hands each caller its own slice, so one appending its
+// own flags cannot reach into what another already holds.
+func TestMariaDBSocketConnectArgsDoNotAlias(t *testing.T) {
+	first := mariadbSocketConnectArgs("/run/mysqld/mysqld.sock")
+	appended := append(mariadbSocketConnectArgs("/run/mysqld/mysqld.sock"), "--connect-timeout=3")
+
+	if slices.Contains(first, "--connect-timeout=3") {
+		t.Errorf("args = %v, want the second caller's flag kept out of the first's slice", first)
+	}
+	if len(appended) != len(first)+1 {
+		t.Errorf("appended = %v, want exactly one flag more than %v", appended, first)
+	}
+}
+
 // ValidateMasterUserPassword permits both of these, and the client offers no
 // equivalent of psql's parameter quoting, so the escaping here is what keeps a
 // password from ending the literal it is inside.
@@ -268,7 +353,7 @@ func TestMariaDBEngine_ApplyParametersSetsOnlyTheDynamicHalfLive(t *testing.T) {
 		t.Fatalf("ran %d commands, want the one live apply", len(runner.calls))
 	}
 	sql := runner.calls[0].Stdin
-	if !strings.Contains(sql, "SET GLOBAL max_connections = '200';") {
+	if !strings.Contains(sql, "SET GLOBAL max_connections = 200;") {
 		t.Errorf("SQL %q does not set the dynamic value live", sql)
 	}
 	if strings.Contains(sql, "innodb_buffer_pool_size") {
@@ -279,6 +364,54 @@ func TestMariaDBEngine_ApplyParametersSetsOnlyTheDynamicHalfLive(t *testing.T) {
 	}
 }
 
+// MariaDB refuses a quoted literal for a numeric or boolean system variable, so
+// a single quoting rule for the whole set fails on the first number in it —
+// which the resolved set always carries, whatever the customer changed.
+func TestMariaDBEngine_ApplyParametersRendersEachValueByItsCatalogType(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(context.Background(), []handlers_rds.Parameter{
+		{Name: "max_connections", Value: "200"},
+		{Name: "long_query_time", Value: "2.5"},
+		{Name: "innodb_adaptive_hash_index", Value: "on"},
+		{Name: "log_output", Value: "FILE"},
+	}); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+
+	if len(runner.calls) != 1 {
+		t.Fatalf("ran %d commands, want the one live apply", len(runner.calls))
+	}
+	sql := runner.calls[0].Stdin
+	for _, want := range []string{
+		"SET GLOBAL max_connections = 200;",
+		"SET GLOBAL long_query_time = 2.5;",
+		"SET GLOBAL innodb_adaptive_hash_index = ON;",
+		"SET GLOBAL log_output = 'FILE';",
+	} {
+		if !strings.Contains(sql, want) {
+			t.Errorf("SQL %q does not contain %q", sql, want)
+		}
+	}
+}
+
+// A numeric that is not a number never reaches the server: an unquoted
+// right-hand side is only safe to interpolate because of this check.
+func TestMariaDBEngine_ApplyParametersRefusesANonNumericNumber(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+
+	_, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "max_connections", Value: "200; DROP DATABASE orders"}})
+	if err == nil {
+		t.Fatal("ApplyParameters accepted a non-numeric value for an integer parameter")
+	}
+	if len(runner.calls) != 0 {
+		t.Errorf("ran %d commands for a refused value, want 0", len(runner.calls))
+	}
+}
+
 // A set with nothing dynamic in it must not open a connection at all: the
 // installed file is the whole of the change.
 func TestMariaDBEngine_ApplyParametersRunsNoStatementForAStaticOnlySet(t *testing.T) {
@@ -320,6 +453,189 @@ func TestMariaDBEngine_ApplyParametersRollsBackAValueTheServerRefuses(t *testing
 	}
 }
 
+// Answers the two TLS read-backs from fields the case sets, so what the engine
+// does with each answer is what is under test. liveEnforcement is the value the
+// server is already on.
+//
+// failOn fails one statement and then stops: with liveEnforcementAfter it is the
+// client that failed without saying whether the server ran what it was sent.
+type mariadbTLSReadBackRunner struct {
+	recordingRunner
+
+	haveSSL         string
+	liveEnforcement string
+
+	failOn               string
+	failErr              error
+	liveEnforcementAfter string
+}
+
+func (r *mariadbTLSReadBackRunner) run(ctx context.Context, c command) (string, error) {
+	switch {
+	case strings.Contains(c.Stdin, "have_ssl"):
+		r.calls = append(r.calls, c)
+		return r.haveSSL + "\n", nil
+	case strings.Contains(c.Stdin, "@@global.require_secure_transport"):
+		r.calls = append(r.calls, c)
+		if r.liveEnforcement == "" {
+			return "0\n", nil
+		}
+		return r.liveEnforcement + "\n", nil
+	case r.failOn != "" && strings.Contains(c.Stdin, r.failOn):
+		r.calls = append(r.calls, c)
+		r.failOn, r.liveEnforcement = "", r.liveEnforcementAfter
+		return "", r.failErr
+	}
+	return r.recordingRunner.run(ctx, c)
+}
+
+func requireSecureTransport(value string) []handlers_rds.Parameter {
+	return []handlers_rds.Parameter{
+		{Name: "max_connections", Value: "200"},
+		{Name: "require_secure_transport", Value: value},
+	}
+}
+
+// Unlike PostgreSQL's, this is a real global system variable: the value in the
+// file is the enforcement, and a running server takes it as SET GLOBAL.
+func TestMariaDBEngine_ApplyParametersEnforcesTLSLive(t *testing.T) {
+	runner := &mariadbTLSReadBackRunner{haveSSL: "YES"}
+	engine := newScriptedTestMariaDBEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(context.Background(), requireSecureTransport("1")); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+
+	installed, err := os.ReadFile(engine.params.installedPath())
+	if err != nil {
+		t.Fatalf("read the installed parameters: %v", err)
+	}
+	if !strings.Contains(string(installed), "require_secure_transport = '1'") {
+		t.Errorf("installed parameters = %q, want the value the next start reads", installed)
+	}
+	sql := runner.calls[len(runner.calls)-1].Stdin
+	if !strings.Contains(sql, "SET GLOBAL require_secure_transport = ON;") {
+		t.Errorf("SQL %q does not require TLS of clients without a restart", sql)
+	}
+}
+
+// Without this the apply would reject every client of an engine started with no
+// certificate, and report the parameter applied.
+func TestMariaDBEngine_ApplyParametersRefusesToEnforceWithoutTLS(t *testing.T) {
+	runner := &mariadbTLSReadBackRunner{haveSSL: "DISABLED"}
+	engine := newScriptedTestMariaDBEngine(t, runner.run)
+
+	_, err := engine.ApplyParameters(context.Background(), requireSecureTransport("1"))
+	if err == nil || !strings.Contains(err.Error(), "serving without TLS") {
+		t.Fatalf("ApplyParameters error = %v, want a refusal naming the engine's own TLS state", err)
+	}
+	for _, c := range runner.calls {
+		if strings.Contains(c.Stdin, "SET GLOBAL") {
+			t.Errorf("the set reached the server anyway: %q", c.Stdin)
+		}
+	}
+	if _, statErr := os.Stat(engine.params.installedPath()); !os.IsNotExist(statErr) {
+		t.Errorf("the refused set is still installed (stat err = %v)", statErr)
+	}
+}
+
+// An absent key reads as enforce here, as it does on the PostgreSQL side. The
+// server's own default for it is off, so this decides only whether the engine
+// may be asked to enforce — never whether it does.
+func TestMariaDBEngine_ApplyParametersReadsAnAbsentEnforcementKeyAsOn(t *testing.T) {
+	runner := &mariadbTLSReadBackRunner{haveSSL: "DISABLED"}
+	engine := newScriptedTestMariaDBEngine(t, runner.run)
+
+	_, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "max_connections", Value: "200"}})
+	if err == nil || !strings.Contains(err.Error(), "serving without TLS") {
+		t.Fatalf("ApplyParameters error = %v, want a set predating the parameter to read as enforcing", err)
+	}
+}
+
+// A set that predates the parameter reads as enforce, and the server's own
+// default for it is off — so the apply has to issue the statement rather than
+// leave the server accepting plaintext under an API reporting enforcement.
+func TestMariaDBEngine_ApplyParametersEnforcesOnAnAbsentKey(t *testing.T) {
+	runner := &mariadbTLSReadBackRunner{haveSSL: "YES", liveEnforcement: "0"}
+	engine := newScriptedTestMariaDBEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "max_connections", Value: "200"}}); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	sql := runner.calls[len(runner.calls)-1].Stdin
+	if !strings.Contains(sql, "SET GLOBAL require_secure_transport = ON;") {
+		t.Errorf("SQL %q left a server that predates the parameter accepting plaintext", sql)
+	}
+}
+
+// The batch of SET GLOBAL statements is not a transaction, so enforcement is
+// issued after it rather than sorted into it: a refusal partway through must not
+// leave the server's TLS posture already moved.
+func TestMariaDBEngine_ApplyParametersLeavesTLSAloneWhenTheBatchFails(t *testing.T) {
+	runner := &mariadbTLSReadBackRunner{haveSSL: "YES", liveEnforcement: "1"}
+	engine := newScriptedTestMariaDBEngine(t, runner.run)
+	runner.err = errors.New("ERROR 1231 (42000): Variable 'max_connections' can't be set to the value of '0'")
+
+	if _, err := engine.ApplyParameters(context.Background(), []handlers_rds.Parameter{
+		{Name: "max_connections", Value: "0"},
+		{Name: "require_secure_transport", Value: "0"},
+	}); err == nil {
+		t.Fatal("ApplyParameters succeeded against a value the server refused")
+	}
+	for _, c := range runner.calls {
+		if strings.Contains(c.Stdin, "SET GLOBAL require_secure_transport") {
+			t.Errorf("a failed batch moved the TLS posture anyway: %q", c.Stdin)
+		}
+	}
+}
+
+// The client can fail without saying whether the server ran the statement, so
+// the posture is read back rather than assumed unchanged. Plaintext nobody asked
+// for is the one outcome that must not survive a failed apply.
+func TestMariaDBEngine_ApplyParametersRestoresTLSAfterAnAmbiguousFailure(t *testing.T) {
+	runner := &mariadbTLSReadBackRunner{haveSSL: "YES", liveEnforcement: "1"}
+	engine := newScriptedTestMariaDBEngine(t, runner.run)
+	// The statement ran and the client still reported a failure, which is what
+	// leaves the live value where the caller cannot infer it.
+	runner.failOn = "SET GLOBAL require_secure_transport"
+	runner.failErr = errors.New("ERROR 2013 (HY000): Lost connection to server during query")
+	runner.liveEnforcementAfter = "0"
+
+	if _, err := engine.ApplyParameters(context.Background(), requireSecureTransport("0")); err == nil {
+		t.Fatal("ApplyParameters succeeded against a client that failed")
+	}
+	restored := 0
+	for _, c := range runner.calls {
+		if strings.Contains(c.Stdin, "SET GLOBAL require_secure_transport = ON;") {
+			restored++
+		}
+	}
+	if restored != 1 {
+		t.Errorf("issued %d statements putting enforcement back, want 1", restored)
+	}
+}
+
+// The resolver canonicalises every boolean, so a value that is neither means the
+// file was written by something other than the platform. Reading an unparsable
+// security setting as off is the one choice not open here.
+func TestMariaDBEngine_ApplyParametersRefusesAnUnparsableEnforcementValue(t *testing.T) {
+	runner := &mariadbTLSReadBackRunner{haveSSL: "YES"}
+	engine := newScriptedTestMariaDBEngine(t, runner.run)
+
+	_, err := engine.ApplyParameters(context.Background(), requireSecureTransport("yes"))
+	if err == nil || !strings.Contains(err.Error(), "neither 1 nor 0") {
+		t.Fatalf("ApplyParameters error = %v, want a refusal naming the unreadable value", err)
+	}
+	if len(runner.calls) != 0 {
+		t.Errorf("ran %d commands against a set that could not be read, want 0", len(runner.calls))
+	}
+	if _, statErr := os.Stat(engine.params.installedPath()); !os.IsNotExist(statErr) {
+		t.Errorf("the refused set is still installed (stat err = %v)", statErr)
+	}
+}
+
 // The whole of the pending-restart answer, since MariaDB has none of its own:
 // the installed drop-in against the one the server actually started on, over the
 // static keys only.
@@ -511,13 +827,14 @@ func writePid(t *testing.T, path string, pid int) {
 // distinguishes a server replaying its redo log from no server at all.
 func TestMariaDBProbe_SeparatesRecoveringFromAbsent(t *testing.T) {
 	tests := []struct {
-		name     string
-		writePid bool
-		alive    bool
-		pingCode int
-		pingErr  error
-		execCode int
-		want     engineState
+		name       string
+		writePid   bool
+		alive      bool
+		pingCode   int
+		pingStderr string
+		pingErr    error
+		execCode   int
+		want       engineState
 	}{
 		{name: "no pidfile at all", want: engineAbsent},
 		{name: "pidfile names a dead process", writePid: true, want: engineAbsent},
@@ -533,14 +850,14 @@ func TestMariaDBProbe_SeparatesRecoveringFromAbsent(t *testing.T) {
 			if tt.writePid {
 				writePid(t, cfg.EnginePidFile, 4242)
 			}
-			run := func(_ context.Context, name string, args ...string) (int, error) {
+			run := func(_ context.Context, name string, args ...string) (int, string, error) {
 				if strings.HasSuffix(name, mariadbAdminBinary) {
-					return tt.pingCode, tt.pingErr
+					return tt.pingCode, tt.pingStderr, tt.pingErr
 				}
 				if !slices.Contains(args, "--execute=SELECT 1") {
 					t.Errorf("the second stage ran %s %v, want a statement the server has to execute", name, args)
 				}
-				return tt.execCode, nil
+				return tt.execCode, "", nil
 			}
 			stateFn := mariadbProbeState(cfg, run, func(int) bool { return tt.alive })
 
@@ -565,12 +882,17 @@ func TestMariaDBProbe_BoundsAStalledClient(t *testing.T) {
 
 	started := time.Now()
 	state, message := mariadbProbeState(cfg,
-		func(ctx context.Context, _ string, args ...string) (int, error) {
+		func(ctx context.Context, _ string, args ...string) (int, string, error) {
 			if !slices.Contains(args, "--connect-timeout=3") {
 				t.Errorf("probe args = %v, want a client connection timeout", args)
 			}
+			// The platform drop-in offers TLS the serving cert cannot prove for a
+			// local socket, so a verifying client would be refused every time.
+			if !slices.Contains(args, "--skip-ssl") {
+				t.Errorf("probe args = %v, want TLS declined on the unix socket", args)
+			}
 			<-ctx.Done()
-			return -1, ctx.Err()
+			return -1, "", ctx.Err()
 		},
 		func(int) bool { return true })(t.Context(), int64(cfg.EnginePort))
 
@@ -591,11 +913,11 @@ func TestMariaDBProbe_MapsToTheHealthTheControlPlaneReads(t *testing.T) {
 	alive := true
 	serving := true
 	probe := newEngineProbe(cfg.EnginePort, mariadbProbeState(cfg,
-		func(context.Context, string, ...string) (int, error) {
+		func(context.Context, string, ...string) (int, string, error) {
 			if serving {
-				return 0, nil
+				return 0, "", nil
 			}
-			return 1, nil
+			return 1, "", nil
 		},
 		func(int) bool { return alive }))
 
@@ -620,7 +942,7 @@ func TestMariaDBProbe_LiveButUnreachableEngineResetsTheRollbackDeadline(t *testi
 	cfg := mariadbTestProbeConfig(t)
 	writePid(t, cfg.EnginePidFile, 4242)
 	probe := newEngineProbe(cfg.EnginePort, mariadbProbeState(cfg,
-		func(context.Context, string, ...string) (int, error) { return 1, nil },
+		func(context.Context, string, ...string) (int, string, error) { return 1, "", nil },
 		func(int) bool { return true }))
 
 	recovery := &fakeRecovery{restored: true}
@@ -794,3 +1116,30 @@ func TestMariaDBEngine_PendingRestartReportsCatalogNames(t *testing.T) {
 		t.Errorf("pending = %v, want only the static setting under its catalog name", pending)
 	}
 }
+
+// Every dynamic value the control plane can resolve has to render, at every
+// class: a catalog entry the SET GLOBAL path cannot express would otherwise
+// surface as a failed apply on a live instance rather than as a test failure on
+// the change that introduced it.
+func TestMariaDBEngine_EveryResolvedDynamicValueRenders(t *testing.T) {
+	meta, err := handlers_rds.LookupEngine("mariadb")
+	if err != nil {
+		t.Fatalf("LookupEngine: %v", err)
+	}
+	rules := controlPlaneRulesFrom(meta)
+
+	for _, class := range handlers_rds.SupportedInstanceClasses() {
+		params, err := meta.ResolveEffectiveParameters(class, nil)
+		if err != nil {
+			t.Fatalf("ResolveEffectiveParameters(%s): %v", class, err)
+		}
+		for _, p := range params {
+			if rules.isStatic(p.Name) {
+				continue
+			}
+			if _, err := mariadbSetValue(rules.dataType(p.Name), p.Value); err != nil {
+				t.Errorf("%s at %s: %v", p.Name, class, err)
+			}
+		}
+	}
+}

cmd/rds-agent/engine_postgres.go

@@ -5,11 +5,10 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
-	"sync"
-	"sync/atomic"
 	"time"
 
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
@@ -19,6 +18,7 @@ import (
 // agent drops to the postgres OS user rather than holding a password of its own.
 type postgresEngine struct {
 	quiesceState
+	parameterManager
 
 	// The control plane's own metadata for this engine, resolved once at
 	// startup: the live password apply runs as the cluster superuser, so it
@@ -32,35 +32,28 @@ type postgresEngine struct {
 	pgData    string
 	socketDir string
 	osUser    string
-	probe     *engineProbe
-	// Set from the bootstrap config, on a different goroutine than the commands
-	// that read it.
-	port atomic.Int64
-
-	params parameterStore
-	// Serializes parameter installs, serving snapshots and rollback restores so
-	// none can copy or replace an intermediate configuration.
-	paramMu       sync.Mutex
-	repairTimeout time.Duration
-	repairPoll    time.Duration
 }
 
 var _ engine = (*postgresEngine)(nil)
 
-// pg_isready is what gives PostgreSQL three states for free: exit 0 is serving,
-// exit 1 is a postmaster up and rejecting connections, and anything else is an
-// engine that is not there at all. Resolved on PATH, where the client package
-// puts it.
+// pg_isready gives PostgreSQL three states for free: exit 0 is serving, exit 1
+// is a postmaster up and rejecting connections, anything else is an engine that
+// is not there. Resolved on PATH, where the client package puts it.
 const postgresProbeBinary = "pg_isready"
 
+// The socket directory rather than loopback: the engine binds the customer ENI
+// and nothing else, so there is no TCP path for the probe to take. It also keeps
+// liveness independent of the network path, of the pg_hba scope and of the TLS
+// enforcement rule — libpq reads a host beginning with / as a socket directory,
+// and the generated `local ... peer` line covers it.
 func newPostgresProbe(cfg config, run probeRunner) *engineProbe {
-	return newEngineProbe(cfg.EnginePort, postgresProbeState(cfg.EngineHost, run))
+	return newEngineProbe(cfg.EnginePort, postgresProbeState(cfg.SocketDir, run))
 }
 
 func postgresProbeState(host string, run probeRunner) probeStateFn {
 	return func(ctx context.Context, port int64) (engineState, string) {
 		portArg := strconv.FormatInt(port, 10)
-		code, err := run(ctx, postgresProbeBinary, "-h", host, "-p", portArg, "-q")
+		code, _, err := run(ctx, postgresProbeBinary, "-h", host, "-p", portArg, "-q")
 		switch {
 		case err != nil:
 			// A missing binary or broken image. Reporting healthy on the strength of
@@ -77,23 +70,9 @@ func postgresProbeState(host string, run probeRunner) probeStateFn {
 	}
 }
 
-// Resolved once, under the name the control plane knows this implementation by.
-// A lookup failure is a mismatch between the agent and the control plane it was
-// built against, so it fails at startup rather than at the first rotation.
-var postgresEngineMeta = mustLookupEngine(enginePostgres)
-
-func mustLookupEngine(name string) handlers_rds.Engine {
-	engine, err := handlers_rds.LookupEngine(name)
-	if err != nil {
-		panic("rds-agent: " + err.Error())
-	}
-	return engine
-}
-
-// The include the resolved parameter set is rendered to, and the copy of the
-// last one the engine accepted. Both live beside the data rather than in /etc,
-// matching rds-init: a class change boots a fresh root volume, which would
-// otherwise revert them.
+// The include the resolved set is rendered to, and the copy of the last one the
+// engine accepted. Both live beside the data rather than in /etc, matching
+// rds-init: a class change boots a fresh root volume and would revert them.
 const (
 	postgresParametersFile = "10-rds-parameters.conf"
 	// Deliberately not a .conf name: include_dir globs *.conf, so the rollback
@@ -101,9 +80,39 @@ const (
 	postgresLastGoodFile = "10-rds-parameters.last-good"
 )
 
-func newPostgresEngine(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) *postgresEngine {
-	e := &postgresEngine{
-		meta:      postgresEngineMeta,
+// The pg_hba rule that implements TLS enforcement, and the directory the
+// generated pg_hba includes it from. Its content is a constant, so the copy
+// rds-init writes in shell and the one written here have no template to drift
+// apart on: enforcement is the presence of the file and nothing more.
+const (
+	postgresHBADir           = "hba.d"
+	postgresForceSSLRuleFile = "20-rds-force-ssl.conf"
+	// An explicit reject rather than an omitted permit: both refuse the
+	// connection, and only this one puts the reason in the server log and in the
+	// client's error.
+	postgresForceSSLRules = `# Managed by the platform: the rule that requires TLS of client connections.
+# Written by rds-init at boot and by rds-agent on a parameter apply.
+hostnossl all all 0.0.0.0/0 reject
+hostnossl all all ::/0 reject
+`
+	// What the read-back must count once the file is in place. Asserting the shape
+	// rather than only the absence of parse errors is what catches the failure
+	// that will actually happen: the derivation above writing the wrong thing.
+	postgresForceSSLRuleCount = 2
+)
+
+// The layout's factory resolves the control plane metadata during startup.
+func newPostgresEngineFromCatalog(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+	meta, err := handlers_rds.LookupEngine(enginePostgres)
+	if err != nil {
+		return nil, fmt.Errorf("this image bakes %s, which this build's control plane does not offer: %w", enginePostgres, err)
+	}
+	return newPostgresEngine(cfg, meta, run, startSess, probe), nil
+}
+
+func newPostgresEngine(cfg config, meta handlers_rds.Engine, run commandRunner, startSess sessionRunner, probe *engineProbe) *postgresEngine {
+	return &postgresEngine{
+		meta:      meta,
 		run:       run,
 		startSess: startSess,
 		psql:      filepath.Join(cfg.EngineBinDir, "psql"),
@@ -112,23 +121,19 @@ func newPostgresEngine(cfg config, run commandRunner, startSess sessionRunner, p
 		pgData:    cfg.EngineDataDir,
 		socketDir: cfg.SocketDir,
 		osUser:    cfg.EngineUser,
-		probe:     probe,
-		params: parameterStore{
-			dir:       filepath.Join(cfg.EngineDataDir, "conf.d"),
-			installed: postgresParametersFile,
-			lastGood:  postgresLastGoodFile,
-			osUser:    cfg.EngineUser,
-			engine:    enginePostgres,
+		parameterManager: parameterManager{
+			probe: probe,
+			params: parameterStore{
+				dir:       filepath.Join(cfg.EngineDataDir, "conf.d"),
+				installed: postgresParametersFile,
+				lastGood:  postgresLastGoodFile,
+				osUser:    cfg.EngineUser,
+				engine:    enginePostgres,
+			},
+			repairTimeout: parameterRepairTimeout,
+			repairPoll:    parameterRepairPoll,
 		},
-		repairTimeout: parameterRepairTimeout,
-		repairPoll:    parameterRepairPoll,
 	}
-	e.port.Store(int64(cfg.EnginePort))
-	return e
-}
-
-func (e *postgresEngine) setPort(port int) {
-	e.port.Store(int64(port))
 }
 
 // The role name and the password ride the environment and are re-quoted by
@@ -166,10 +171,9 @@ ALTER ROLE :"master" WITH LOGIN PASSWORD :'password';
 	return nil
 }
 
-// Installs the resolved set, validates it with the engine's own config parser
-// before the engine ever adopts it, and reloads. A value the engine refuses is
-// rolled back here rather than left on the data volume, where it would survive
-// every VM replace and turn the next restart into a boot loop.
+// Installs the resolved set, validates it with the engine's own parser before
+// the engine adopts it, and reloads. A refused value is rolled back here rather
+// than left on the data volume, where it would survive every VM replace.
 func (e *postgresEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
 	e.paramMu.Lock()
 	defer e.paramMu.Unlock()
@@ -186,16 +190,15 @@ func (e *postgresEngine) ApplyParameters(ctx context.Context, params []handlers_
 		}
 	}
 
-	// The check has to run against the file in place, because the engine parses
-	// the datadir's own include_dir. The window that leaves is closed from both
-	// ends: the rollback below, and the last-known-good restore the agent runs at
-	// boot when the engine does not come up.
-	restore, err := e.params.install(params)
+	// The check runs against the file in place, because the engine parses the
+	// datadir's own include_dir. The window that leaves is closed by the rollback
+	// below and the last-known-good restore the agent runs at boot.
+	restoreParameters, err := e.params.install(params)
 	if err != nil {
 		return nil, err
 	}
 	if err := e.checkConfig(ctx); err != nil {
-		return nil, errors.Join(fmt.Errorf("the engine rejected the parameter set: %w", err), restore())
+		return nil, errors.Join(fmt.Errorf("the engine rejected the parameter set: %w", err), restoreParameters())
 	}
 
 	state, _ := e.probe.state(ctx)
@@ -203,25 +206,62 @@ func (e *postgresEngine) ApplyParameters(ctx context.Context, params []handlers_
 	case engineAbsent:
 		return e.restartOnRepairSetLocked(ctx)
 	case engineRecovering:
-		return nil, errors.Join(errors.New("the engine entered startup or recovery during the parameter apply"), restore())
+		return nil, errors.Join(errors.New("the engine entered startup or recovery during the parameter apply"), restoreParameters())
 	}
 
-	if _, err := e.psqlRun(ctx, "SELECT pg_reload_conf();\n"); err != nil {
+	// After the engine's own parser has passed the set and before the reload that
+	// adopts both files at once.
+	enforce, restoreEnforcement, err := e.syncTLSEnforcement(ctx, state)
+	if err != nil {
+		return nil, errors.Join(fmt.Errorf("apply the TLS enforcement of the parameter set: %w", err), restoreParameters())
+	}
+	// The two files carry one apply from here: a rejected set withdrawn without
+	// its enforcement rule would leave the instance serving the old parameters
+	// under the new set's TLS posture.
+	restore := func() error { return errors.Join(restoreEnforcement(), restoreParameters()) }
+
+	if err := e.reload(ctx); err != nil {
 		// The engine may have gone down after the first probe. In that case start
 		// it on the new, parser-checked repair set rather than restoring the set
 		// that already left it unable to serve.
 		if state, _ := e.probe.state(ctx); state == engineAbsent {
 			return e.restartOnRepairSetLocked(ctx)
 		}
-		return nil, errors.Join(fmt.Errorf("reload the engine configuration: %w", err), restore())
+		return nil, errors.Join(err, restore())
+	}
+	if err := e.verifyTLSEnforcement(ctx, enforce); err != nil {
+		// The reload already happened, so putting the files back is not enough: a
+		// pg_hba that parsed but says the wrong thing has been adopted, and only a
+		// second reload takes the engine off it.
+		return nil, errors.Join(err, restore(), e.reload(ctx))
 	}
 	return e.pendingRestartParameters(ctx)
 }
 
+// Re-derived before the restart: the engine reads pg_hba when it starts, so a
+// repair set started under the previous set's enforcement would serve one TLS
+// posture while the API reported the other.
 func (e *postgresEngine) restartOnRepairSetLocked(ctx context.Context) ([]string, error) {
+	if _, _, err := e.syncTLSEnforcement(ctx, engineAbsent); err != nil {
+		return nil, fmt.Errorf("apply the TLS enforcement of the repair set: %w", err)
+	}
 	return awaitRepairedEngine(ctx, e.probe, e.Restart, e.pendingRestartParameters, e.repairTimeout, e.repairPoll)
 }
 
+// The function's own answer is the reload, not psql's exit status: it returns f
+// when it could not signal the postmaster, and the read-back that follows parses
+// pg_hba from disk, so an unsignalled reload would otherwise verify clean.
+func (e *postgresEngine) reload(ctx context.Context) error {
+	out, err := e.psqlRun(ctx, "SELECT pg_reload_conf();\n")
+	if err != nil {
+		return fmt.Errorf("reload the engine configuration: %w", err)
+	}
+	if result := strings.TrimSpace(out); result != "t" {
+		return fmt.Errorf("reload the engine configuration: pg_reload_conf() returned %q", result)
+	}
+	return nil
+}
+
 // The postmaster's own answer: a static setting it has stored but not adopted is
 // what it reports here, so nothing in the guest has to classify one.
 func (e *postgresEngine) pendingRestartParameters(ctx context.Context) ([]string, error) {
@@ -256,13 +296,160 @@ func (e *postgresEngine) recordServingParametersLocked(ctx context.Context) erro
 func (e *postgresEngine) RestoreLastKnownGoodParameters(ctx context.Context) (bool, error) {
 	e.paramMu.Lock()
 	defer e.paramMu.Unlock()
-	return restoreLastGood(ctx, e.params, e.probe)
+	restored, err := restoreLastGood(ctx, e.params, e.probe)
+	if err != nil || !restored {
+		return restored, err
+	}
+	// A service restart does not re-run rds-init, so a restore that stopped at the
+	// parameter file would bring the last known good set up under the enforcement
+	// of the set that was just rejected. restoreLastGood only replaces the file of
+	// an engine that is down.
+	if _, _, err := e.syncTLSEnforcement(ctx, engineAbsent); err != nil {
+		return true, fmt.Errorf("apply the TLS enforcement of the restored parameter set: %w", err)
+	}
+	return true, nil
+}
+
+// Turns the installed enforcement value into the pg_hba fact that implements it.
+// PostgreSQL has no server setting for this, so the parameter file on its own
+// would report the change applied and enforce nothing.
+//
+// The serving-TLS guard runs only against an engine that is up. The restore and
+// repair paths reach this with the engine down, where the rule is read by the
+// start that follows rather than by a reload.
+func (e *postgresEngine) syncTLSEnforcement(ctx context.Context, state engineState) (enforce bool, restore func() error, err error) {
+	enforce, err = installedTLSEnforcement(e.meta.TLSEnforcementParameter(), e.params.installedPath())
+	if err != nil {
+		return false, nil, err
+	}
+	if enforce && state == engineServing {
+		if err := e.requireServingTLS(ctx); err != nil {
+			return false, nil, err
+		}
+	}
+	restore, err = e.writeTLSEnforcement(enforce)
+	if err != nil {
+		return false, nil, err
+	}
+	return enforce, restore, nil
+}
+
+// Writes or removes the enforcement rule, returning a func putting back whatever
+// was there. Removing matters as much as writing: a restored data volume carries
+// hba.d with it, so a stale rule would keep enforcing under a parameter group
+// that turns enforcement off.
+func (e *postgresEngine) writeTLSEnforcement(enforce bool) (func() error, error) {
+	path := e.forceSSLRulePath()
+	previous, readErr := os.ReadFile(path)
+	if readErr != nil && !os.IsNotExist(readErr) {
+		return nil, fmt.Errorf("read the TLS enforcement rule at %s: %w", path, readErr)
+	}
+	existed := readErr == nil
+	restore := func() error {
+		if !existed {
+			return e.removeTLSEnforcement()
+		}
+		return e.params.write(path, previous)
+	}
+
+	if !enforce {
+		if err := e.removeTLSEnforcement(); err != nil {
+			return nil, err
+		}
+		return restore, nil
+	}
+	// rds-init lays the directory down on every boot, so this covers an agent that
+	// reached an apply before one ran rather than the ordinary case.
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, 0o700); err != nil {
+		return nil, fmt.Errorf("create %s: %w", dir, err)
+	}
+	if err := e.params.chownToEngine(dir); err != nil {
+		return nil, err
+	}
+	if err := e.params.write(path, []byte(postgresForceSSLRules)); err != nil {
+		return nil, err
+	}
+	return restore, nil
+}
+
+func (e *postgresEngine) removeTLSEnforcement() error {
+	path := e.forceSSLRulePath()
+	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("withdraw the TLS enforcement rule at %s: %w", path, err)
+	}
+	return nil
+}
+
+func (e *postgresEngine) forceSSLRulePath() string {
+	return filepath.Join(e.pgData, postgresHBADir, postgresForceSSLRuleFile)
+}
+
+// Refuses to require TLS of clients from an engine that is not serving it, which
+// would reject every connection while the parameter path reported success.
+func (e *postgresEngine) requireServingTLS(ctx context.Context) error {
+	out, err := e.psqlRun(ctx, "SHOW ssl;\n")
+	if err != nil {
+		return fmt.Errorf("read whether the engine is serving TLS: %w", err)
+	}
+	if serving := strings.TrimSpace(out); serving != "on" {
+		return fmt.Errorf("the engine is serving without TLS (ssl = %q), so requiring it of clients would reject every connection", serving)
+	}
+	return nil
+}
+
+// PostgreSQL discards a pg_hba it cannot parse on reload and keeps the rules it
+// is already serving, which presents as an apply that succeeded and enforces
+// nothing. So the file is read back.
+//
+// pg_hba_file_rules re-parses from disk, so this proves the file would be
+// adopted rather than that the postmaster has adopted it. The inference holds
+// because a reloaded pg_hba is discarded only on a parse error.
+const postgresHBAReadBack = `SELECT count(*) FILTER (WHERE type = 'hostnossl' AND auth_method = 'reject'),
+       count(*) FILTER (WHERE error IS NOT NULL) FROM pg_hba_file_rules;
+`
+
+func (e *postgresEngine) verifyTLSEnforcement(ctx context.Context, enforce bool) error {
+	out, err := e.psqlRun(ctx, postgresHBAReadBack)
+	if err != nil {
+		return fmt.Errorf("read back the client authentication rules: %w", err)
+	}
+	rules, unparsable, err := parseHBAReadBack(out)
+	if err != nil {
+		return err
+	}
+	if unparsable > 0 {
+		return fmt.Errorf("the client authentication file carries %d rules the engine could not parse, so it would keep the rules it is already serving", unparsable)
+	}
+	want := 0
+	if enforce {
+		want = postgresForceSSLRuleCount
+	}
+	if rules != want {
+		return fmt.Errorf("the client authentication file carries %d TLS enforcement rules, want %d", rules, want)
+	}
+	return nil
+}
+
+func parseHBAReadBack(out string) (rules, unparsable int, err error) {
+	answer := strings.TrimSpace(out)
+	malformed := fmt.Errorf("the client authentication read-back returned %q, want two counts", answer)
+	counts := strings.Split(answer, "|")
+	if len(counts) != 2 {
+		return 0, 0, malformed
+	}
+	parsed := make([]int, len(counts))
+	for i, count := range counts {
+		if parsed[i], err = strconv.Atoi(strings.TrimSpace(count)); err != nil {
+			return 0, 0, malformed
+		}
+	}
+	return parsed[0], parsed[1], nil
 }
 
 // The engine's own parser, run offline against the datadir. Reading one setting
-// back is enough: postgres parses postgresql.conf and every include first, and
-// exits non-zero naming the file and line of an unknown parameter or a value
-// outside its range.
+// back is enough: postgres parses every include first and exits non-zero naming
+// the file and line of an unknown parameter or an out-of-range value.
 func (e *postgresEngine) checkConfig(ctx context.Context) error {
 	if _, err := e.run(ctx, command{
 		Name: filepath.Join(filepath.Dir(e.psql), "postgres"),
@@ -304,7 +491,7 @@ func (e *postgresEngine) psqlArgs() []string {
 		"--no-psqlrc", "--quiet", "--no-align", "--tuples-only",
 		"-v", "ON_ERROR_STOP=1",
 		"-h", e.socketDir,
-		"-p", strconv.FormatInt(e.port.Load(), 10),
+		"-p", strconv.FormatInt(e.probe.port.Load(), 10),
 		"-U", e.osUser,
 		"-d", "postgres",
 		"-f", "-",
@@ -356,25 +543,6 @@ SELECT pg_backup_start(:'label', fast => true);
 	return nil
 }
 
-// Ends the backup cleanly. A missing hold is an error rather than a silent
-// success: it means the deadline fired first, so the snapshot the control plane
-// just took was not taken against a held checkpoint.
 func (e *postgresEngine) Unquiesce(ctx context.Context) error {
-	held := e.takeHold()
-	if held == nil {
-		return errors.New("the engine is not quiesced; the backup hold had already expired")
-	}
-
-	// The stop has to run on the session that started the backup, and the session
-	// is closed either way — the engine aborts an unstopped backup with it.
-	execErr := held.session.Exec(ctx, "SELECT pg_backup_stop();\n")
-	closeErr := held.session.Close()
-	if execErr != nil {
-		return fmt.Errorf("take the engine out of backup mode: %w", errors.Join(execErr, closeErr))
-	}
-	if closeErr != nil {
-		return fmt.Errorf("close the backup session: %w", closeErr)
-	}
-	slog.Info("rds-agent: engine released from backup mode", "label", held.label)
-	return nil
+	return e.releaseHold(ctx, "SELECT pg_backup_stop();\n")
 }

cmd/rds-agent/engine_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"errors"
+	"fmt"
 	"os"
 	"os/user"
 	"path/filepath"
@@ -139,7 +140,31 @@ func (r *recordingRunner) run(_ context.Context, c command) (string, error) {
 	return r.out, r.err
 }
 
+func testPostgresEngineMeta(t *testing.T) handlers_rds.Engine {
+	t.Helper()
+	meta, err := handlers_rds.LookupEngine(enginePostgres)
+	if err != nil {
+		t.Fatalf("LookupEngine(%s): %v", enginePostgres, err)
+	}
+	return meta
+}
+
 func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
+	t.Helper()
+	cfg := testEngineConfig(t)
+	return newPostgresEngine(cfg, testPostgresEngineMeta(t), withPostgresReadBacks(cfg.EngineDataDir, run), nil, newPostgresProbe(cfg, staticProbe(0)))
+}
+
+// The same engine with the read-backs left to the runner, for the cases that are
+// about what the engine does with each answer rather than about the rest of an
+// apply.
+func newScriptedTestEngine(t *testing.T, run commandRunner) *postgresEngine {
+	t.Helper()
+	cfg := testEngineConfig(t)
+	return newPostgresEngine(cfg, testPostgresEngineMeta(t), run, nil, newPostgresProbe(cfg, staticProbe(0)))
+}
+
+func testEngineConfig(t *testing.T) config {
 	t.Helper()
 	cfg := testLoadConfig(t, enginePostgres)
 	cfg.EngineDataDir = t.TempDir()
@@ -153,7 +178,35 @@ func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
 		t.Fatalf("resolve the current user: %v", err)
 	}
 	cfg.EngineUser = current.Username
-	return newPostgresEngine(cfg, run, nil, newPostgresProbe(cfg, staticProbe(0)))
+	return cfg
+}
+
+// Answers the read-backs every apply runs against a live postmaster, from the
+// datadir the case is working in: an engine serving TLS, a reload the postmaster
+// took, and a pg_hba carrying whatever the agent last wrote. The call still
+// reaches the runner, so a case counting commands sees them; one that is about
+// the read-backs themselves builds its own engine.
+func withPostgresReadBacks(dataDir string, run commandRunner) commandRunner {
+	rule := filepath.Join(dataDir, postgresHBADir, postgresForceSSLRuleFile)
+	return func(ctx context.Context, c command) (string, error) {
+		out, err := run(ctx, c)
+		if err != nil {
+			return out, err
+		}
+		switch {
+		case strings.Contains(c.Stdin, "SHOW ssl"):
+			return "on\n", nil
+		case strings.Contains(c.Stdin, "pg_reload_conf"):
+			return "t\n", nil
+		case strings.Contains(c.Stdin, "pg_hba_file_rules"):
+			rules := 0
+			if _, statErr := os.Stat(rule); statErr == nil {
+				rules = postgresForceSSLRuleCount
+			}
+			return fmt.Sprintf("%d|0\n", rules), nil
+		}
+		return out, err
+	}
 }
 
 // The password reaches psql through the environment and is re-quoted there,
@@ -288,19 +341,27 @@ func TestPostgresEngine_ApplyParametersInstallsAndReloads(t *testing.T) {
 	if !strings.Contains(string(installed), "shared_buffers = '256MB'") {
 		t.Errorf("installed parameters = %q, want the resolved value", installed)
 	}
-	if len(runner.calls) != 3 {
-		t.Fatalf("ran %d commands, want a config check, a reload and a pending-restart read", len(runner.calls))
+	if len(runner.calls) != 5 {
+		t.Fatalf("ran %d commands, want a config check, the TLS guard, a reload, the rule read-back and a pending-restart read", len(runner.calls))
 	}
-	// The check runs before the reload, so a value the engine would refuse never
-	// reaches a running cluster.
+	// The check and the TLS guard both run before the reload, so neither a value
+	// the engine would refuse nor a rule rejecting every client reaches a running
+	// cluster. The read-back of the rules comes after, since the reload is what
+	// puts them in force.
 	if !slices.Contains(runner.calls[0].Args, "-C") {
 		t.Errorf("first command = %v, want the offline config check", runner.calls[0].Args)
 	}
-	if !strings.Contains(runner.calls[1].Stdin, "pg_reload_conf") {
-		t.Errorf("second statement = %q, want the reload", runner.calls[1].Stdin)
+	if !strings.Contains(runner.calls[1].Stdin, "SHOW ssl") {
+		t.Errorf("second statement = %q, want the serving-TLS guard", runner.calls[1].Stdin)
 	}
-	if !strings.Contains(runner.calls[2].Stdin, "pending_restart") {
-		t.Errorf("third statement = %q, want the pending-restart read", runner.calls[2].Stdin)
+	if !strings.Contains(runner.calls[2].Stdin, "pg_reload_conf") {
+		t.Errorf("third statement = %q, want the reload", runner.calls[2].Stdin)
+	}
+	if !strings.Contains(runner.calls[3].Stdin, "pg_hba_file_rules") {
+		t.Errorf("fourth statement = %q, want the client authentication read-back", runner.calls[3].Stdin)
+	}
+	if !strings.Contains(runner.calls[4].Stdin, "pending_restart") {
+		t.Errorf("fifth statement = %q, want the pending-restart read", runner.calls[4].Stdin)
 	}
 	if len(pending) != 2 || pending[0] != "shared_buffers" || pending[1] != "max_connections" {
 		t.Errorf("pending = %v, want both settings the engine reported", pending)
@@ -319,6 +380,21 @@ func TestPostgresEngine_ApplyParametersSurfacesAFailedReload(t *testing.T) {
 	}
 }
 
+// pg_reload_conf() returns f when it could not signal the postmaster, and psql
+// exits 0 all the same. Every read-back after it parses files from disk, so an
+// apply that trusted the exit status would verify clean against a server still
+// serving the old rules.
+func TestPostgresEngine_ApplyParametersSurfacesAnUnsignalledReload(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount, reload: "f"}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	_, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "work_mem", Value: "8MB"}})
+	if err == nil || !strings.Contains(err.Error(), `pg_reload_conf() returned "f"`) {
+		t.Fatalf("ApplyParameters error = %v, want a refusal naming the answer the postmaster gave", err)
+	}
+}
+
 type reloadFailingRunner struct {
 	recordingRunner
 
@@ -341,10 +417,10 @@ func TestPostgresEngine_ApplyParametersRestartsOnARepairSetWhileEngineIsDown(t *
 	codes := []int{2, 2, 0}
 	probeCall := 0
 	cfg := testProbeConfig()
-	engine.probe = newPostgresProbe(cfg, func(context.Context, string, ...string) (int, error) {
+	engine.probe = newPostgresProbe(cfg, func(context.Context, string, ...string) (int, string, error) {
 		code := codes[min(probeCall, len(codes)-1)]
 		probeCall++
-		return code, nil
+		return code, "", nil
 	})
 	engine.repairTimeout = 100 * time.Millisecond
 	engine.repairPoll = time.Millisecond
@@ -381,6 +457,225 @@ func TestPostgresEngine_ApplyParametersRestartsOnARepairSetWhileEngineIsDown(t *
 	}
 }
 
+// Answers the two read-backs an apply runs from fields the case sets, so what
+// the engine does with each answer is what is under test.
+type tlsReadBackRunner struct {
+	recordingRunner
+
+	ssl        string
+	ruleRows   int
+	brokenRows int
+	// What pg_reload_conf() answered. Empty is the postmaster having taken the
+	// signal, which is every case but the one about it not having.
+	reload string
+}
+
+func (r *tlsReadBackRunner) run(ctx context.Context, c command) (string, error) {
+	switch {
+	case strings.Contains(c.Stdin, "SHOW ssl"):
+		r.calls = append(r.calls, c)
+		return r.ssl + "\n", nil
+	case strings.Contains(c.Stdin, "pg_reload_conf"):
+		r.calls = append(r.calls, c)
+		if r.reload == "" {
+			return "t\n", nil
+		}
+		return r.reload + "\n", nil
+	case strings.Contains(c.Stdin, "pg_hba_file_rules"):
+		r.calls = append(r.calls, c)
+		return fmt.Sprintf("%d|%d\n", r.ruleRows, r.brokenRows), nil
+	}
+	return r.recordingRunner.run(ctx, c)
+}
+
+func forceSSLParameter(t *testing.T, value string) []handlers_rds.Parameter {
+	t.Helper()
+	return []handlers_rds.Parameter{
+		{Name: "work_mem", Value: "4096"},
+		{Name: testPostgresEngineMeta(t).TLSEnforcementParameter(), Value: value},
+	}
+}
+
+// PostgreSQL has no server setting for this, so the value in the parameter file
+// is inert: the rule the generated pg_hba includes is the whole of it.
+func TestPostgresEngine_ApplyParametersWritesTheEnforcementRule(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1")); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	rule, err := os.ReadFile(engine.forceSSLRulePath())
+	if err != nil {
+		t.Fatalf("read the enforcement rule: %v", err)
+	}
+	if string(rule) != postgresForceSSLRules {
+		t.Errorf("enforcement rule = %q, want the constant rds-init also writes", rule)
+	}
+}
+
+// A snapshot carries hba.d with it, so restoring one taken while enforcing into
+// a group that turns enforcement off has to take the rule with it.
+func TestPostgresEngine_ApplyParametersRemovesAStaleEnforcementRule(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on"}
+	engine := newScriptedTestEngine(t, runner.run)
+	if err := os.MkdirAll(filepath.Dir(engine.forceSSLRulePath()), 0o700); err != nil {
+		t.Fatalf("create hba.d: %v", err)
+	}
+	if err := os.WriteFile(engine.forceSSLRulePath(), []byte(postgresForceSSLRules), 0o600); err != nil {
+		t.Fatalf("write the rule the restored volume carried: %v", err)
+	}
+
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "0")); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if _, err := os.Stat(engine.forceSSLRulePath()); !os.IsNotExist(err) {
+		t.Errorf("the stale enforcement rule is still in place (stat err = %v)", err)
+	}
+}
+
+// The ordinary state of an instance whose resolved set predates the parameter,
+// and the whole of the migration: it begins requiring TLS with no control-plane
+// work at all.
+func TestPostgresEngine_ApplyParametersEnforcesOnAnAbsentKey(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(t.Context(),
+		[]handlers_rds.Parameter{{Name: "work_mem", Value: "4096"}}); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if _, err := os.Stat(engine.forceSSLRulePath()); err != nil {
+		t.Errorf("a set predating the parameter did not enforce (stat err = %v)", err)
+	}
+}
+
+// The resolver canonicalises every boolean, so a value that is neither means the
+// file was written by something other than the platform. Reading an unparsable
+// security setting as off is the one choice not open here.
+func TestPostgresEngine_ApplyParametersRefusesAnUnparsableEnforcementValue(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on"}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "yes"))
+	if err == nil || !strings.Contains(err.Error(), "neither 1 nor 0") {
+		t.Fatalf("ApplyParameters error = %v, want a refusal naming the unreadable value", err)
+	}
+	if _, statErr := os.Stat(engine.params.installedPath()); !os.IsNotExist(statErr) {
+		t.Errorf("the refused set is still installed (stat err = %v)", statErr)
+	}
+}
+
+// Without this, a live enable against an engine started with no certificate
+// would reject every subsequent connection and report the parameter applied.
+func TestPostgresEngine_ApplyParametersRefusesToEnforceWithoutTLS(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "off"}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1"))
+	if err == nil || !strings.Contains(err.Error(), "serving without TLS") {
+		t.Fatalf("ApplyParameters error = %v, want a refusal naming the engine's own TLS state", err)
+	}
+	if _, statErr := os.Stat(engine.forceSSLRulePath()); !os.IsNotExist(statErr) {
+		t.Errorf("the enforcement rule was written anyway (stat err = %v)", statErr)
+	}
+}
+
+// PostgreSQL discards a pg_hba it cannot parse and keeps the rules it is already
+// serving, which presents as an apply that succeeded and enforces nothing. The
+// shape is asserted as well as the errors, because our own derivation writing
+// the wrong thing is the failure that will actually happen.
+func TestPostgresEngine_ApplyParametersFailsAnUnverifiableEnforcement(t *testing.T) {
+	tests := []struct {
+		name       string
+		value      string
+		ruleRows   int
+		brokenRows int
+		want       string
+	}{
+		{name: "rule missing", value: "1", ruleRows: 0, want: "carries 0 TLS enforcement rules, want 2"},
+		{name: "rule present when it should not be", value: "0", ruleRows: 2, want: "carries 2 TLS enforcement rules, want 0"},
+		{name: "a rule the engine could not parse", value: "1", ruleRows: 2, brokenRows: 1, want: "could not parse"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			runner := &tlsReadBackRunner{ssl: "on", ruleRows: tc.ruleRows, brokenRows: tc.brokenRows}
+			engine := newScriptedTestEngine(t, runner.run)
+
+			_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, tc.value))
+			if err == nil || !strings.Contains(err.Error(), tc.want) {
+				t.Fatalf("ApplyParameters error = %v, want %q", err, tc.want)
+			}
+		})
+	}
+}
+
+// A rejected set has to take its enforcement with it. Withdrawing one without
+// the other would leave the instance serving the previous parameters under the
+// rejected set's TLS posture.
+func TestPostgresEngine_ApplyParametersWithdrawsBothFilesTogether(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
+	engine := newScriptedTestEngine(t, runner.run)
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1")); err != nil {
+		t.Fatalf("first ApplyParameters: %v", err)
+	}
+
+	// Turning enforcement off, verified against a pg_hba that still carries the
+	// rule — so the apply fails after both files have already been replaced.
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "0")); err == nil {
+		t.Fatal("ApplyParameters succeeded against an unverifiable enforcement")
+	}
+	installed, err := os.ReadFile(engine.params.installedPath())
+	if err != nil {
+		t.Fatalf("read the installed parameters: %v", err)
+	}
+	if !strings.Contains(string(installed), "rds.force_ssl = '1'") {
+		t.Errorf("installed parameters = %q, want the previous accepted set restored", installed)
+	}
+	if _, err := os.Stat(engine.forceSSLRulePath()); err != nil {
+		t.Errorf("the enforcement of the previous set was not restored with it (stat err = %v)", err)
+	}
+	// The reload already adopted the rejected files, so putting them back is not
+	// enough on its own.
+	if !strings.Contains(runner.calls[len(runner.calls)-1].Stdin, "pg_reload_conf") {
+		t.Errorf("last statement = %q, want a reload taking the engine off the withdrawn set", runner.calls[len(runner.calls)-1].Stdin)
+	}
+}
+
+// A service restart does not re-run rds-init, so a restore stopping at the
+// parameter file would bring the last known good set up under the enforcement of
+// the set that was just rejected.
+func TestPostgresEngine_RestoreLastKnownGoodReDerivesEnforcement(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1")); err != nil {
+		t.Fatalf("ApplyParameters(enforcing): %v", err)
+	}
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("RecordServingParameters: %v", err)
+	}
+	runner.ruleRows = 0
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "0")); err != nil {
+		t.Fatalf("ApplyParameters(not enforcing): %v", err)
+	}
+	if _, err := os.Stat(engine.forceSSLRulePath()); !os.IsNotExist(err) {
+		t.Fatalf("the enforcement rule survived the set that turned it off (stat err = %v)", err)
+	}
+
+	engine.probe = stubProbe(t, 2, nil)
+	restored, err := engine.RestoreLastKnownGoodParameters(t.Context())
+	if err != nil {
+		t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
+	}
+	if !restored {
+		t.Fatal("did not restore a set that differs from the last known good")
+	}
+	if _, err := os.Stat(engine.forceSSLRulePath()); err != nil {
+		t.Errorf("the restored set's enforcement was not re-derived (stat err = %v)", err)
+	}
+}
+
 func TestPostgresEngine_ApplyParametersKeepsRepairSetWhenRestartTimesOut(t *testing.T) {
 	runner := &recordingRunner{}
 	engine := newTestEngine(t, runner.run)

cmd/rds-agent/health.go

@@ -1,31 +1,37 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"os/exec"
+	"strings"
 	"sync/atomic"
 
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
 
 // A non-zero exit is a result, not a fault, so it comes back as a code; err is
-// reserved for the probe failing to run at all.
-type probeRunner func(ctx context.Context, name string, args ...string) (int, error)
+// reserved for the probe failing to run at all. The client's stderr comes back
+// with it: on a refusal it is the only place the reason is ever stated.
+type probeRunner func(ctx context.Context, name string, args ...string) (int, string, error)
 
-func execProbeRunner(ctx context.Context, name string, args ...string) (int, error) {
-	err := exec.CommandContext(ctx, name, args...).Run()
+func execProbeRunner(ctx context.Context, name string, args ...string) (int, string, error) {
+	var stderr bytes.Buffer
+	cmd := exec.CommandContext(ctx, name, args...)
+	cmd.Stderr = &stderr
+	err := cmd.Run()
+	out := strings.TrimSpace(stderr.String())
 	if err == nil {
-		return 0, nil
+		return 0, out, nil
 	}
 	if ctxErr := ctx.Err(); ctxErr != nil {
-		return -1, ctxErr
+		return -1, out, ctxErr
 	}
-	var exitErr *exec.ExitError
-	if errors.As(err, &exitErr) {
-		return exitErr.ExitCode(), nil
+	if exitErr, ok := errors.AsType[*exec.ExitError](err); ok {
+		return exitErr.ExitCode(), out, nil
 	}
-	return -1, err
+	return -1, out, err
 }
 
 // What the engine reports about itself, which is the only part of the probe
@@ -57,9 +63,8 @@ func (p *engineProbe) setPort(port int) {
 }
 
 // What the engine is doing, before the seenHealthy latch collapses "never up"
-// and "was up and went away" into one health. The parameter rollback needs them
-// apart: a postmaster that is up and replaying WAL will come back on its own, one
-// that is not running at all after a parameter change will not.
+// and "went away" into one health. The rollback needs them apart: an engine
+// replaying WAL comes back on its own, one that is not running will not.
 type engineState int
 
 const (

cmd/rds-agent/health_test.go

@@ -29,8 +29,8 @@ func TestEngineProbe_MapsExitCodesToHealth(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			probe := newPostgresProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
-				return tt.code, tt.runErr
+			probe := newPostgresProbe(testProbeConfig(), func(context.Context, string, ...string) (int, string, error) {
+				return tt.code, "", tt.runErr
 			})
 			probe.seenHealthy = tt.seenHealthy
 
@@ -52,8 +52,8 @@ func TestEngineProbe_MapsExitCodesToHealth(t *testing.T) {
 // has to flip the first time the engine answers.
 func TestEngineProbe_LatchesAfterFirstHealthy(t *testing.T) {
 	code := 0
-	probe := newPostgresProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
-		return code, nil
+	probe := newPostgresProbe(testProbeConfig(), func(context.Context, string, ...string) (int, string, error) {
+		return code, "", nil
 	})
 
 	if got, _ := probe.Check(context.Background()); got != handlers_rds.EngineHealthHealthy {
@@ -69,9 +69,9 @@ func TestEngineProbe_LatchesAfterFirstHealthy(t *testing.T) {
 // must follow it rather than keep asking the default.
 func TestEngineProbe_ProbesTheAssignedPort(t *testing.T) {
 	var gotArgs []string
-	probe := newPostgresProbe(testProbeConfig(), func(_ context.Context, _ string, args ...string) (int, error) {
+	probe := newPostgresProbe(testProbeConfig(), func(_ context.Context, _ string, args ...string) (int, string, error) {
 		gotArgs = args
-		return 0, nil
+		return 0, "", nil
 	})
 
 	probe.setPort(6543)
@@ -86,7 +86,7 @@ func TestExecProbeRunner_PreservesTheProbeDeadline(t *testing.T) {
 	ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
 	defer cancel()
 
-	code, err := execProbeRunner(ctx, "sh", "-c", "while :; do :; done")
+	code, _, err := execProbeRunner(ctx, "sh", "-c", "while :; do :; done")
 	if code != -1 {
 		t.Errorf("exit code = %d, want -1 for a timed-out probe", code)
 	}
@@ -95,6 +95,52 @@ func TestExecProbeRunner_PreservesTheProbeDeadline(t *testing.T) {
 	}
 }
 
-func testProbeConfig() config {
-	return config{EngineHost: defaultEngineHost, EnginePort: engineLayouts[enginePostgres].port}
+// The client's account of a refusal is the reason a probe failure is legible at
+// all, so a runner that dropped it would leave the caller nothing to report.
+func TestExecProbeRunner_CapturesTheClientsStderr(t *testing.T) {
+	code, stderr, err := execProbeRunner(t.Context(), "sh", "-c", "echo 'TLS handshake failed' >&2; exit 1")
+	if err != nil {
+		t.Fatalf("execProbeRunner: %v", err)
+	}
+	if code != 1 {
+		t.Errorf("exit code = %d, want 1", code)
+	}
+	if stderr != "TLS handshake failed" {
+		t.Errorf("stderr = %q, want the client's own message", stderr)
+	}
+}
+
+// The engine binds the customer ENI and nothing else, so a loopback probe would
+// report a serving engine as absent. libpq reads a host beginning with / as a
+// socket directory, which is also what keeps liveness off the network path.
+func TestPostgresProbe_ProbesTheUnixSocket(t *testing.T) {
+	cfg := testProbeConfig()
+	var probed []string
+	probe := newPostgresProbe(cfg, func(_ context.Context, name string, args ...string) (int, string, error) {
+		probed = append([]string{name}, args...)
+		return 0, "", nil
+	})
+
+	if state, _ := probe.state(context.Background()); state != engineServing {
+		t.Fatalf("probe state = %v, want engineServing", state)
+	}
+	host := ""
+	for i, arg := range probed {
+		if arg == "-h" && i+1 < len(probed) {
+			host = probed[i+1]
+		}
+	}
+	if host != cfg.SocketDir {
+		t.Errorf("probe host = %q, want the socket directory %q", host, cfg.SocketDir)
+	}
+	if !strings.HasPrefix(host, "/") {
+		t.Errorf("probe host = %q, which libpq reads as a TCP host rather than a socket directory", host)
+	}
+}
+
+func testProbeConfig() config {
+	return config{
+		SocketDir:  engineLayouts[enginePostgres].socketDir,
+		EnginePort: engineLayouts[enginePostgres].port,
+	}
 }

cmd/rds-agent/heartbeat_test.go

@@ -60,8 +60,8 @@ func TestHeartbeater_BoundsBootstrapFailure(t *testing.T) {
 func TestHeartbeater_ChecksServingParametersOnEveryHealthyProbe(t *testing.T) {
 	code := 0
 	cfg := testProbeConfig()
-	probe := newPostgresProbe(cfg, func(context.Context, string, ...string) (int, error) {
-		return code, nil
+	probe := newPostgresProbe(cfg, func(context.Context, string, ...string) (int, string, error) {
+		return code, "", nil
 	})
 	recorder := &countingServingRecorder{}
 	h := newHeartbeater(newFakeControlPlane(), probe, recorder, 0)

cmd/rds-agent/parameters.go

@@ -6,16 +6,15 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sync"
 	"time"
 
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
 
 // The generated parameter files and what it takes to install one. They live on
-// the data volume rather than under /etc, at the same paths rds-init writes them
-// to: a class change is a VM replacement, so a set under /etc would be discarded
-// with the boot volume and the instance would silently revert to catalog
-// defaults while the API still reported the customer's values.
+// the data volume at the paths rds-init writes: a class change replaces the VM,
+// so a set under /etc would revert to defaults while the API reported otherwise.
 type parameterStore struct {
 	dir string
 	// The include the resolved set is rendered to.
@@ -113,9 +112,8 @@ func (s parameterStore) chownToEngine(path string) error {
 }
 
 // Snapshots the installed set as the rollback target, but only once the engine
-// is running all of it. A set with a setting still pending a restart has not
-// been shown to serve, and promoting it would leave the rollback pointing at the
-// configuration that is about to fail.
+// is running all of it. A set still pending a restart has not been shown to
+// serve, so promoting it would point the rollback at a config about to fail.
 func recordLastGood(ctx context.Context, store parameterStore, pending pendingRestartFn) error {
 	if _, err := os.Stat(store.installedPath()); err != nil {
 		if os.IsNotExist(err) {
@@ -177,15 +175,27 @@ func restoreLastGood(ctx context.Context, store parameterStore, probe *enginePro
 // snapshot are gated on the same answer.
 type pendingRestartFn func(ctx context.Context) ([]string, error)
 
+// The parameter-file state every engine keeps. The operations over it are free
+// functions above rather than methods here, so each implementation names its own
+// restart and pending answers at the call site where the compiler checks them.
+type parameterManager struct {
+	params parameterStore
+	probe  *engineProbe
+	// Serializes parameter installs, serving snapshots and rollback restores so
+	// none can copy or replace an intermediate configuration.
+	paramMu       sync.Mutex
+	repairTimeout time.Duration
+	repairPoll    time.Duration
+}
+
 const (
 	parameterRepairTimeout = 90 * time.Second
 	parameterRepairPoll    = time.Second
 )
 
-// Starts a down engine on the set that was just installed and waits for it to
-// serve. Reached when the engine went away during an apply: the installed set is
-// the newer of the two, so starting on it beats restoring one that already left
-// the engine unable to serve.
+// Starts a down engine on the set just installed and waits for it to serve.
+// Reached when the engine went away during an apply: the installed set is the
+// newer, so starting on it beats restoring one that already failed.
 func awaitRepairedEngine(ctx context.Context, probe *engineProbe, restart func(context.Context) error,
 	pending pendingRestartFn, timeout, poll time.Duration) ([]string, error) {
 	repairCtx, cancel := context.WithTimeout(ctx, timeout)

cmd/rds-agent/paramrecovery.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"log/slog"
 	"time"
 
@@ -49,10 +50,16 @@ func (g *paramGuard) Run(ctx context.Context) {
 		return
 	}
 
+	// A restore that replaced the file and then failed has still moved the
+	// instance onto the last accepted set, so the restart and the report below
+	// are what it needs; returning here would leave it down and unaccounted for.
 	restored, err := g.engine.RestoreLastKnownGoodParameters(ctx)
 	if err != nil {
-		slog.Error("rds-agent: could not restore the last known good parameters", "err", err)
-		return
+		slog.Error("rds-agent: restoring the last known good parameters failed",
+			"err", err, "parametersReplaced", restored)
+		if !restored {
+			return
+		}
 	}
 	if !restored {
 		slog.Warn("rds-agent: engine is down but no different last accepted parameter set is available; not rolling back")
@@ -60,7 +67,7 @@ func (g *paramGuard) Run(ctx context.Context) {
 	}
 
 	message := handlers_rds.ParameterRollbackMessage
-	slog.Error("rds-agent: "+message, "after", g.after)
+	slog.Error("rds-agent: "+message, "after_ms", otelsetup.Millis(g.after))
 	if g.cp != nil {
 		if _, err := g.cp.SubmitState(ctx, g.id, handlers_rds.EngineHealthUnhealthy, message); err != nil {
 			slog.Error("rds-agent: reporting the parameter rollback failed", "err", err)

cmd/rds-agent/paramrecovery_test.go

@@ -369,8 +369,8 @@ func (f *fakeRecovery) Restart(context.Context) error {
 func stubProbe(t *testing.T, code int, err error) *engineProbe {
 	t.Helper()
 	cfg := testLoadConfig(t, enginePostgres)
-	return newPostgresProbe(cfg, func(context.Context, string, ...string) (int, error) {
-		return code, err
+	return newPostgresProbe(cfg, func(context.Context, string, ...string) (int, string, error) {
+		return code, "", err
 	})
 }
 
@@ -404,6 +404,47 @@ func TestParamGuard_RollsBackReportsAndRestartsWhenTheEngineNeverComesUp(t *test
 	}
 }
 
+// PostgreSQL re-derives TLS enforcement from the set it has just put back, so
+// the file can be replaced and the call still fail. The rollback has happened
+// either way, and the worse outcome is an instance left down and unreported.
+func TestParamGuard_ReportsAndRestartsWhenOnlyTheEnforcementFailed(t *testing.T) {
+	engine := &fakeRecovery{restored: true, restoreErr: errors.New("write the TLS enforcement rule: read-only file system")}
+	cp := newFakeControlPlane()
+	guard := newParamGuard(engine, stubProbe(t, 2, nil), cp)
+	guard.id = identity{DBInstanceIdentifier: "db-1"}
+	guard.after, guard.poll = 20*time.Millisecond, time.Millisecond
+	guard.Run(t.Context())
+
+	if engine.restarts != 1 {
+		t.Errorf("restarts = %d, want the engine restarted on the set already put back", engine.restarts)
+	}
+	states := cp.snapshotStates()
+	if len(states) != 1 {
+		t.Fatalf("submitted states = %d, want the rollback still reported", len(states))
+	}
+	if states[0].health != handlers_rds.EngineHealthUnhealthy || states[0].message != handlers_rds.ParameterRollbackMessage {
+		t.Errorf("rollback state = %+v, want the unhealthy rollback report", states[0])
+	}
+}
+
+// The other half of the same contract: a restore that replaced nothing leaves
+// the instance on the set it is already failing under, so restarting it would
+// only churn a cluster that is down for a reason the rollback cannot fix.
+func TestParamGuard_DoesNotRestartWhenTheRestoreReplacedNothing(t *testing.T) {
+	engine := &fakeRecovery{restored: false, restoreErr: errors.New("read the last known good parameters: input/output error")}
+	cp := newFakeControlPlane()
+	guard := newParamGuard(engine, stubProbe(t, 2, nil), cp)
+	guard.after, guard.poll = 20*time.Millisecond, time.Millisecond
+	guard.Run(t.Context())
+
+	if engine.restarts != 0 {
+		t.Errorf("restarts = %d, want none when nothing was rolled back", engine.restarts)
+	}
+	if states := cp.snapshotStates(); len(states) != 0 {
+		t.Errorf("submitted states = %d, want no rollback report", len(states))
+	}
+}
+
 func TestParamGuard_DoesNothingWhenTheEngineIsServing(t *testing.T) {
 	engine := &fakeRecovery{restored: true}
 	newTestGuard(engine, stubProbe(t, 0, nil)).Run(t.Context())

cmd/rds-agent/quiesce.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"io"
 	"log/slog"
 	"os/exec"
@@ -16,10 +17,8 @@ import (
 )
 
 // Both engines tie a backup hold to the session that took it: PostgreSQL aborts
-// a non-exclusive backup when its session ends, and MariaDB releases a BACKUP
-// STAGE with the connection that started it. That is what makes the hold
-// self-expiring — a control plane that dies mid snapshot cannot leave the engine
-// in backup mode forever.
+// a non-exclusive backup when its session ends, MariaDB releases a BACKUP STAGE
+// with its connection. That is what makes the hold self-expiring.
 type engineSession interface {
 	// Runs sql and waits for the engine to finish it.
 	Exec(ctx context.Context, sql string) error
@@ -74,7 +73,7 @@ func (q *quiesceState) beginHoldLocked(label string, session engineSession, hold
 		session: session,
 		expiry:  time.AfterFunc(hold, func() { q.expire(label) }),
 	}
-	slog.Info("rds-agent: engine quiesced for backup", "label", label, "hold", hold)
+	slog.Info("rds-agent: engine quiesced for backup", "label", label, "hold_ms", otelsetup.Millis(hold))
 }
 
 // Takes the hold off the engine, if one is still there, and disarms its
@@ -92,6 +91,29 @@ func (q *quiesceState) takeHold() *quiesceHold {
 	return held
 }
 
+// Ends the backup cleanly, running releaseSQL on the session that took the hold.
+// A missing hold is an error rather than a silent success: it means the deadline
+// fired first, so what the control plane just snapshotted was not held.
+func (q *quiesceState) releaseHold(ctx context.Context, releaseSQL string) error {
+	held := q.takeHold()
+	if held == nil {
+		return errors.New("the engine is not quiesced; the backup hold had already expired")
+	}
+
+	// The release has to run on the session that took the hold, and the session is
+	// closed either way — both engines end an unreleased backup with it.
+	execErr := held.session.Exec(ctx, releaseSQL)
+	closeErr := held.session.Close()
+	if execErr != nil {
+		return fmt.Errorf("take the engine out of backup mode: %w", errors.Join(execErr, closeErr))
+	}
+	if closeErr != nil {
+		return fmt.Errorf("close the backup session: %w", closeErr)
+	}
+	slog.Info("rds-agent: engine released from backup mode", "label", held.label)
+	return nil
+}
+
 // The deadline. Ending the session is enough — both engines release the hold
 // with it — so this never has to reach the engine itself.
 func (q *quiesceState) expire(label string) {

cmd/rds-agent/register.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"log/slog"
 	"time"
 )
@@ -20,7 +21,7 @@ func (a *Agent) register(ctx context.Context) error {
 		a.hb.setInterval(time.Duration(out.HeartbeatIntervalSeconds) * time.Second)
 
 		slog.Info("rds-agent: registered",
-			"dbInstanceIdentifier", a.id.DBInstanceIdentifier, "heartbeat", a.hb.interval)
+			"dbInstanceIdentifier", a.id.DBInstanceIdentifier, "heartbeat_ms", otelsetup.Millis(a.hb.interval))
 		return nil
 	})
 }

cmd/rds-agent/storage.go

@@ -10,10 +10,9 @@ import (
 	"strings"
 )
 
-// The in-guest half of a storage grow. The control plane has already
-// grown the block device by the time this runs; all that is left is to put the
-// filesystem on the capacity that is now there. Both ext4 and XFS grow while
-// mounted, so this needs no ordering against the engine start.
+// The in-guest half of a storage grow: the control plane has already grown the
+// block device, so all that is left is the filesystem. Both ext4 and XFS grow
+// while mounted, so this needs no ordering against the engine start.
 type storageOps interface {
 	// Extends the filesystem at the data mount onto its whole device, and
 	// reports what it did for the command reply.

cmd/spinifex/cmd/admin.go

@@ -25,23 +25,23 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"uuid"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/iam"
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/ebsmetadata"
+	"github.com/mulgadc/spinifex/spinifex/ebsprovider"
 	"github.com/mulgadc/spinifex/spinifex/formation"
 	"github.com/mulgadc/spinifex/spinifex/gpu"
+	handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot"
 	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
 	"github.com/mulgadc/spinifex/spinifex/hostdns"
 	"github.com/mulgadc/spinifex/spinifex/network/host"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/utils"
-	"github.com/mulgadc/viperblock/viperblock"
-	"github.com/mulgadc/viperblock/viperblock/backends/s3"
-	"github.com/mulgadc/viperblock/viperblock/v_utils"
 	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/pterm/pterm"
 	"github.com/spf13/cobra"
 )
@@ -118,8 +118,9 @@ var nodeDrainCmd = &cobra.Command{
 its guests via QMP and unmount their volumes (flushing the viperblock WAL) while
 every service is still running. STORAGE/PERSIST/INFRA are left to systemd's
 ordered unit teardown. This is what spinifex-shutdown.service's ExecStop runs,
-with --only-if-host-stopping so it only drains on a genuine host
-shutdown/reboot -- not on a plain "systemctl restart/stop spinifex.target".
+with --unless-restarting so it drains on a genuine host shutdown/reboot and on
+a plain "systemctl stop spinifex.target", but skips the drain when a restart
+of the stack (e.g. "systemctl restart spinifex.target") is already queued.
 Run by hand without that flag, it always drains unconditionally.`,
 	Run: runNodeDrainLocal,
 }
@@ -197,6 +198,25 @@ takes effect immediately. Prompts for confirmation unless --yes is passed.`,
 	Run: runimagesPromoteCmd,
 }
 
+var volumesCmd = &cobra.Command{
+	Use:   "volumes",
+	Short: "Inspect block storage volumes",
+	Long:  `Inspect the volumes the storage provider holds, independently of the EC2 API.`,
+}
+
+var volumesOrphansCmd = &cobra.Command{
+	Use:   "orphans",
+	Short: "Report volumes the provider holds with no control-plane record",
+	Long: `List volumes the storage provider holds that the control plane has no record
+of. Their blocks are consuming space but they have no API handle, so they
+cannot be described or deleted through the EC2 API.
+
+This command only reports. It never deletes: the evidence that an orphan is
+still wanted is the record that went missing, so removal is an operator
+decision made per volume.`,
+	Run: runVolumesOrphansCmd,
+}
+
 var accountCmd = &cobra.Command{
 	Use:   "account",
 	Short: "Manage Spinifex accounts",
@@ -283,7 +303,9 @@ func init() {
 	nodeCmd.AddCommand(nodeDrainCmd)
 	nodeDrainCmd.Flags().Bool("local", false, "Drain the local node only (required)")
 	nodeDrainCmd.Flags().Duration("timeout", 120*time.Second, "Maximum time to wait per phase")
-	nodeDrainCmd.Flags().Bool("only-if-host-stopping", false, "Skip the drain unless systemd is unwinding into shutdown.target (real reboot/poweroff); unset, always drains")
+	nodeDrainCmd.Flags().Bool("unless-restarting", false, "Drain on a real host shutdown or a plain target stop; skip only when a restart of the stack is already queued. Unset, always drains")
+	nodeDrainCmd.Flags().Bool("only-if-host-stopping", false, "Deprecated: use --unless-restarting")
+	_ = nodeDrainCmd.Flags().MarkDeprecated("only-if-host-stopping", "use --unless-restarting instead")
 	nodeCmd.AddCommand(nodeJSProbeCmd)
 	nodeJSProbeCmd.Flags().Duration("timeout", 10*time.Second, "Maximum time to wait for the canary round-trip")
 
@@ -293,6 +315,9 @@ func init() {
 	imagesCmd.AddCommand(imagesRemoveCmd)
 	imagesCmd.AddCommand(imagesPromoteCmd)
 
+	adminCmd.AddCommand(volumesCmd)
+	volumesCmd.AddCommand(volumesOrphansCmd)
+
 	adminCmd.AddCommand(accountCmd)
 	accountCmd.AddCommand(accountCreateCmd)
 	accountCmd.AddCommand(accountListCmd)
@@ -305,6 +330,9 @@ func init() {
 
 	adminCmd.AddCommand(upgradeCmd)
 	upgradeCmd.Flags().Bool("yes", false, "Apply migrations without prompting")
+	upgradeCmd.Flags().Bool("dry-run", false, "Report pending config and unit changes without applying them")
+	upgradeCmd.Flags().Bool("units-only", false, "Reconcile systemd units only, skip config migrations")
+	upgradeCmd.Flags().Bool("skip-units", false, "Apply config migrations only, skip systemd unit reconciliation")
 	certRenewCmd.Flags().StringSlice("extra-ip", nil, "Additional IP addresses to include in SANs")
 	certRenewCmd.Flags().StringSlice("extra-dns", nil, "Additional DNS names to include in SANs")
 	accountCreateCmd.Flags().String("name", "", "Account name (required)")
@@ -387,19 +415,19 @@ func init() {
 	imagesRemoveCmd.Flags().String("image-id", "", "AMI ID to remove (required)")
 	imagesRemoveCmd.Flags().Bool("force", false, "Bypass dependency, ownership and config-corrupt checks (salvage mode)")
 	imagesRemoveCmd.Flags().Bool("yes", false, "Skip interactive confirmation prompt")
-	if err := imagesRemoveCmd.MarkFlagRequired("image-id"); err != nil {
-		panic(err)
-	}
+	_ = imagesRemoveCmd.MarkFlagRequired("image-id")
 
 	imagesPromoteCmd.Flags().String("image-id", "", "AMI ID to promote to system image (required)")
 	imagesPromoteCmd.Flags().Bool("yes", false, "Skip interactive confirmation prompt")
-	if err := imagesPromoteCmd.MarkFlagRequired("image-id"); err != nil {
-		panic(err)
-	}
+	_ = imagesPromoteCmd.MarkFlagRequired("image-id")
 }
 
 const bytesPerGiB = 1024 * 1024 * 1024
 
+// imageImportTimeout bounds each provider call the import makes. Generous
+// because the snapshot at the end runs over the whole imported volume.
+const imageImportTimeout = 30 * time.Minute
+
 // amiVolumeSizeGiB returns the smallest whole GiB that still holds sizeBytes.
 //
 // Rounding up is load-bearing. The image is copied into a root volume of
@@ -427,16 +455,10 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	var imageStat os.FileInfo
 	var err error
 
-	cfgFile, _ := cmd.Flags().GetString("config")
 	forceCmd, _ := cmd.Flags().GetBool("force")
 	skipVerify, _ := cmd.Flags().GetBool("skip-verify")
 	ostmpDir, _ := cmd.Flags().GetString("tmp-dir")
 
-	// Use default config path
-	if cfgFile == "" {
-		cfgFile = DefaultConfigFile()
-	}
-
 	//configDir, _ := cmd.Flags().GetString("config-dir")
 	baseDir, _ := cmd.Flags().GetString("spinifex-dir")
 
@@ -543,9 +565,11 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	// Create the specified image directory
 	imagePath := fmt.Sprintf("%s/%s/%s/%s", imageDir, image.Distro, image.Version, image.Arch)
 
-	// Create config directory
-	if err := os.MkdirAll(imagePath, 0700); err != nil {
-		fmt.Fprintf(os.Stderr, "Error creating config directory: %v\n", err)
+	// 0770, matching the root:spinifex convention setup.sh gives the shared data
+	// dirs. At 0700 the first sudo import leaves a directory no group member can
+	// descend into, so every later import without sudo fails on mkdir.
+	if err := os.MkdirAll(imagePath, 0770); err != nil {
+		fmt.Fprintf(os.Stderr, "Error creating image directory %s: %v\n", imagePath, err)
 		os.Exit(1)
 	}
 
@@ -607,57 +631,46 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	// Create the specified manifest file to describe the image/AMI
-	manifest := viperblock.VolumeConfig{}
-
-	// Calculate the size
-
-	manifest.AMIMetadata.Name = fmt.Sprintf("ami-%s-%s-%s", image.Distro, image.Version, image.Arch)
-	if amiNameOverride != "" {
-		manifest.AMIMetadata.Name = amiNameOverride
-	}
+	// Describe the image/AMI as the control-plane document it will become.
 	volumeId := utils.GenerateResourceID("ami")
-	manifest.AMIMetadata.ImageID = volumeId
+	amiName := fmt.Sprintf("ami-%s-%s-%s", image.Distro, image.Version, image.Arch)
+	if amiNameOverride != "" {
+		amiName = amiNameOverride
+	}
 
-	manifest.AMIMetadata.Description = fmt.Sprintf("%s cloud image prepared for Spinifex", manifest.AMIMetadata.Name)
-	manifest.AMIMetadata.Architecture = image.Arch
-	manifest.AMIMetadata.PlatformDetails = image.Platform
-	manifest.AMIMetadata.CreationDate = time.Now()
-	manifest.AMIMetadata.RootDeviceType = "ebs"
-	manifest.AMIMetadata.Virtualization = "hvm"
-	manifest.AMIMetadata.ImageOwnerAlias = "system"
-	manifest.AMIMetadata.VolumeSizeGiB = amiVolumeSizeGiB(imageStat.Size())
-	manifest.AMIMetadata.BootMode = image.BootMode
-	manifest.AMIMetadata.Distro = image.Distro
-	manifest.AMIMetadata.DistroFamily = utils.DistroFamily(image.Distro)
+	ami := ebsmetadata.AMI{
+		ImageID:         volumeId,
+		Name:            amiName,
+		Description:     fmt.Sprintf("%s cloud image prepared for Spinifex", amiName),
+		Architecture:    image.Arch,
+		PlatformDetails: image.Platform,
+		CreationDate:    time.Now(),
+		RootDeviceType:  "ebs",
+		Virtualization:  "hvm",
+		ImageOwnerAlias: "system",
+		VolumeSizeGiB:   amiVolumeSizeGiB(imageStat.Size()),
+		SnapshotID:      admin.SnapPrefix(volumeId),
+		BootMode:        image.BootMode,
+		Distro:          image.Distro,
+		DistroFamily:    utils.DistroFamily(image.Distro),
+	}
 
 	// Copy catalog-provided tags (e.g. spinifex:managed-by for system AMIs)
 	// onto the imported AMI so the UI can filter them out.
 	if len(image.Tags) > 0 {
-		manifest.AMIMetadata.Tags = make(map[string]string, len(image.Tags))
-		maps.Copy(manifest.AMIMetadata.Tags, image.Tags)
+		ami.Tags = make(map[string]string, len(image.Tags))
+		maps.Copy(ami.Tags, image.Tags)
 	}
 
-	// Volume Data
-	manifest.VolumeMetadata.VolumeID = volumeId // TODO: Confirm if unique, e.g vol-, if ami- used
-	manifest.VolumeMetadata.VolumeName = manifest.AMIMetadata.Name
-	manifest.VolumeMetadata.TenantID = "system"
-	manifest.VolumeMetadata.SizeGiB = manifest.AMIMetadata.VolumeSizeGiB
-	manifest.VolumeMetadata.State = "available"
-	manifest.VolumeMetadata.AvailabilityZone = "" // TODO: Confirm
-	manifest.VolumeMetadata.CreatedAt = time.Now()
-	manifest.VolumeMetadata.VolumeType = "gp3"
-	manifest.VolumeMetadata.IOPS = 1000
-
 	// Write the manifest to disk
 	// Save as JSON
-	jsonData, err := json.Marshal(manifest)
+	jsonData, err := json.Marshal(ami)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Could not marshal manifest: %v\n", err)
 		os.Exit(1)
 	}
 
-	manifestFilename := fmt.Sprintf("%s/%s.json", imagePath, manifest.AMIMetadata.Name)
+	manifestFilename := fmt.Sprintf("%s/%s.json", imagePath, ami.Name)
 	// Write to file
 	err = os.WriteFile(manifestFilename, jsonData, 0600)
 	if err != nil {
@@ -665,145 +678,118 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	// Upload the image to S3 (predastore)
-
-	appConfig, err := config.LoadConfig(cfgFile)
-
-	if err != nil {
-		fmt.Println("Error loading config file:", err)
-		return
-	}
-
-	s3Config := s3.S3Config{
-		VolumeName: volumeId,
-		VolumeSize: utils.SafeInt64ToUint64(imageStat.Size()),
-		Bucket:     appConfig.Nodes[appConfig.Node].Predastore.Bucket,
-		Region:     appConfig.Nodes[appConfig.Node].Predastore.Region,
-		AccessKey:  appConfig.Nodes[appConfig.Node].Predastore.AccessKey,
-		SecretKey:  appConfig.Nodes[appConfig.Node].Predastore.SecretKey,
-		Host:       appConfig.Nodes[appConfig.Node].Predastore.Host,
-	}
-
-	mkey, err := utils.LoadViperblockMasterKey(appConfig.Nodes[appConfig.Node].Viperblock.EncryptionKeyFile)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not load viperblock encryption key: %v\n", err)
-		os.Exit(1)
-	}
-
-	// Import is an interactive command whose output is a progress bar, so the
-	// volume's routine mount chatter is noise here: a fresh import has no prior
-	// state, making the "no state found" / 404 lines expected rather than
-	// notable. Scoping the logger to this VB (New copies it onto the backend
-	// too) keeps it off the process-wide default. Errors still surface, both
-	// through this logger and as the returned error the caller prints.
-	vbConfig := viperblock.VB{
-		VolumeName: volumeId,
-		VolumeSize: utils.SafeInt64ToUint64(imageStat.Size()),
-		BaseDir:    tmpDir,
-		Cache: viperblock.Cache{
-			Config: viperblock.CacheConfig{
-				Size: 0,
-			},
-		},
-		VolumeConfig:      manifest,
-		MasterKey:         mkey,
-		EncryptionEnabled: mkey != nil,
-		Logger:            slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})),
-	}
-
-	// Bake the deployment CA into the image trust store so a stock cloud image
-	// trusts the gateway from first boot. Best-effort: never fails the import.
-	// The CA path derives from the --spinifex-dir data root (its config/ symlink
-	// holds ca.pem), not NodeBaseDir(): node.BaseDir is unset in the import
-	// command, which would collapse the path to a relative config/ca.pem.
-	bakeCACertIntoImage(extractedImagePath, filepath.Join(baseDir, "config", "ca.pem"))
-
-	// Render the flush bar here rather than inside viperblock, which stays a
-	// pure storage library. The bar is built lazily on the first update so it is
-	// never drawn if the import fails before any bytes flush; viperblock
-	// throttles the callback to ≤101 invocations, so the human-readable title
-	// renders without the per-block render-frequency regression.
-	var flushBar *pterm.ProgressbarPrinter
-	var flushUpdate func(current uint64)
-	progress := func(current, total uint64) {
-		if flushBar == nil {
-			flushBar, flushUpdate = utils.NewByteProgressBar("Flushing image to storage", total)
-		}
-		flushUpdate(current)
-	}
-
-	err = v_utils.ImportDiskImage(&s3Config, &vbConfig, extractedImagePath, progress)
-
-	if flushBar != nil {
-		_, _ = flushBar.Stop()
-	}
-
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not import image to predastore: %v\n", err)
-		os.Exit(1)
-	}
-
 	defer os.RemoveAll(tmpDir)
 
+	appConfig, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not connect to the cluster: %v\n", err)
+		os.Exit(1)
+	}
+	defer nc.Close()
+	node := appConfig.Nodes[appConfig.Node]
+
+	fmt.Println("Writing image to storage ...")
+	provider := ebsprovider.NewNATSProvider(nc, imageImportTimeout)
+	err = admin.ImportImage(context.Background(), provider, admin.ImportOpts{
+		VolumeID:         volumeId,
+		NodeID:           appConfig.Node,
+		SizeBytes:        utils.SafeUint64ToInt64(ami.VolumeSizeGiB * bytesPerGiB),
+		AvailabilityZone: node.AZ,
+		SourcePath:       extractedImagePath,
+		Snapshot:         true,
+		Progress:         os.Stdout,
+	})
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not import image: %v\n", err)
+		os.Exit(1)
+	}
+
+	// admin.ImportImage only wrote the provider's half of the snapshot (its
+	// blocks); this writes the EC2 control plane's, without which
+	// DescribeSnapshots and GetAMISourceVolumeID cannot resolve the import.
+	metaStore := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
+	if err := registerImportedAMISnapshot(metaStore, node.Predastore.Bucket, ami, node.AZ, node.Viperblock.EncryptionKeyFile != ""); err != nil {
+		fmt.Fprintf(os.Stderr, "Imported %s but could not register its snapshot: %v\n", volumeId, err)
+		os.Exit(1)
+	}
+
+	// The document is what DescribeImages enumerates, so it is written last:
+	// until it exists the AMI is not launchable, and an import that failed
+	// partway leaves no half-registered image behind.
+	ami.State = "available"
+	if err := ebsmetadata.NewStore(metaStore, node.Predastore.Bucket).PutAMI(context.Background(), ami); err != nil {
+		fmt.Fprintf(os.Stderr, "Imported %s but could not register it: %v\n", volumeId, err)
+		os.Exit(1)
+	}
+
 	fmt.Printf("✅ Image import complete. Image-ID (AMI): %s\n", volumeId)
 }
 
-// caBakeRunCommand installs the uploaded CA into the guest trust store across
-// the distro families, then removes the staged copy. Each branch is gated on its
-// updater existing so only the matching distro's branch writes anything (install
-// -D creates the anchor dir on minimal images). Debian/Ubuntu use
-// update-ca-certificates, RHEL/Rocky use update-ca-trust, and stock Alpine ships
-// only ca-certificates-bundle (no updater, no anchor dir) so it falls back to
-// appending the PEM to the static bundle. The install chain's exit code is
-// preserved past the cleanup rm so a failure surfaces instead of a false success.
-// The RHEL branch restorecon's only the trust-store paths it touched so the cert
-// is correctly labelled without the image-wide --no-selinux-relabel (below)
-// leaving an unlabelled bundle; restorecon is best-effort and never fails the bake.
-const caBakeRunCommand = `( { command -v update-ca-certificates >/dev/null && install -D -m644 /tmp/spinifex-ca.pem /usr/local/share/ca-certificates/spinifex.crt && update-ca-certificates; } || ` +
-	`{ command -v update-ca-trust >/dev/null && install -D -m644 /tmp/spinifex-ca.pem /etc/pki/ca-trust/source/anchors/spinifex.crt && update-ca-trust && ` +
-	`{ command -v restorecon >/dev/null 2>&1 && restorecon -RF /etc/pki/ca-trust/source/anchors/spinifex.crt /etc/pki/ca-trust/extracted >/dev/null 2>&1; true; }; } || ` +
-	`{ cat /tmp/spinifex-ca.pem >> /etc/ssl/certs/ca-certificates.crt; } ); ` +
-	`rc=$?; rm -f /tmp/spinifex-ca.pem; exit $rc`
-
-// caBakeTimeout bounds the virt-customize run so a stalled libguestfs appliance
-// cannot hang the import indefinitely; on timeout the bake degrades to a skip.
-const caBakeTimeout = 5 * time.Minute
-
-// caBakeCmd builds the virt-customize invocation that uploads the deployment CA
-// into the disk image at imagePath and installs it into the guest trust store.
-// --no-selinux-relabel stops virt-customize flagging the image for a first-boot
-// SELinux autorelabel: that relabel+reboot corrupts XFS roots (RHEL/Rocky) on the
-// reboot, so the run-command relabels only the touched trust paths instead.
-func caBakeCmd(ctx context.Context, imagePath, caCertPath string) *exec.Cmd {
-	return exec.CommandContext(ctx, "virt-customize", "-a", imagePath,
-		"--no-selinux-relabel",
-		"--upload", caCertPath+":/tmp/spinifex-ca.pem",
-		"--run-command", caBakeRunCommand)
+// registerImportedAMISnapshot writes the EC2 control plane's snapshot document
+// for a catalog-imported AMI. The provider-backed import path only writes the
+// storage backend's blocks under ami.SnapshotID, so without this a launch or
+// DescribeSnapshots call cannot resolve it. The volume ID matches ami.ImageID:
+// admin.ImportImage creates the volume under the AMI's own ID.
+func registerImportedAMISnapshot(store objectstore.ObjectStore, bucket string, ami ebsmetadata.AMI, az string, encrypted bool) error {
+	cfg := &handlers_ec2_snapshot.SnapshotConfig{
+		SnapshotID:       ami.SnapshotID,
+		VolumeID:         ami.ImageID,
+		VolumeSize:       utils.SafeUint64ToInt64(ami.VolumeSizeGiB),
+		State:            "completed",
+		Progress:         "100%",
+		StartTime:        time.Now(),
+		Description:      fmt.Sprintf("Imported AMI volume for %s", ami.Name),
+		Encrypted:        encrypted,
+		OwnerID:          utils.GlobalAccountID,
+		AvailabilityZone: az,
+	}
+	if err := handlers_ec2_snapshot.WriteSnapshotConfig(store, bucket, ami.SnapshotID, cfg); err != nil {
+		return fmt.Errorf("register snapshot metadata: %w", err)
+	}
+	return nil
 }
 
-// caBakeRunner resolves virt-customize and runs the CA bake; overridable in tests.
-var caBakeRunner = func(imagePath, caCertPath string) ([]byte, error) {
-	if _, err := exec.LookPath("virt-customize"); err != nil {
-		return nil, fmt.Errorf("virt-customize not found: %w", err)
+func runVolumesOrphansCmd(_ *cobra.Command, _ []string) {
+	appConfig, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not connect to the cluster: %v\n", err)
+		os.Exit(1)
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), caBakeTimeout)
-	defer cancel()
-	return caBakeCmd(ctx, imagePath, caCertPath).CombinedOutput()
-}
+	defer nc.Close()
 
-// bakeCACertIntoImage uploads the deployment CA into the image's trust store via
-// virt-customize so an imported stock image trusts the gateway from first boot.
-// Best-effort: a missing CA, an absent virt-customize, or an image libguestfs
-// cannot inspect logs and continues — image import never fails on the CA.
-func bakeCACertIntoImage(imagePath, caCertPath string) {
-	if _, err := os.Stat(caCertPath); err != nil {
-		slog.Warn("CA bake skipped: deployment CA not found; imported image will not auto-trust the gateway", "ca", caCertPath, "err", err)
+	node := appConfig.Nodes[appConfig.Node]
+	store := objectstore.NewS3ObjectStoreFromConfig(
+		node.Predastore.Host,
+		node.Predastore.Region,
+		node.Predastore.AccessKey,
+		node.Predastore.SecretKey,
+	)
+
+	orphans, err := admin.FindOrphanVolumes(
+		context.Background(),
+		ebsprovider.NewNATSProvider(nc, imageImportTimeout),
+		ebsmetadata.NewStore(store, node.Predastore.Bucket),
+	)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Orphan scan failed:", err)
+		os.Exit(1)
+	}
+	if len(orphans) == 0 {
+		fmt.Println("No orphaned volumes: every volume the provider holds has a control-plane record.")
 		return
 	}
-	if out, err := caBakeRunner(imagePath, caCertPath); err != nil {
-		slog.Warn("CA bake skipped: virt-customize could not customize image; imported image will not auto-trust the gateway", "err", err, "output", string(out))
-		return
+
+	fmt.Printf("%d orphaned volume(s) — held by the provider, unknown to the control plane:\n\n", len(orphans))
+	for _, orphan := range orphans {
+		fmt.Printf("  %s\n", orphan.VolumeID)
+		fmt.Printf("    handle:  %s\n", orphan.Handle)
+		if orphan.Derived {
+			fmt.Println("    note:    derived volume; its base volume is also unknown")
+		}
 	}
+	fmt.Println()
+	fmt.Println("Nothing was deleted. Each of these holds data that cannot be reached")
+	fmt.Println("through the EC2 API; removing one is a per-volume operator decision.")
 }
 
 func runimagesRemoveCmd(cmd *cobra.Command, args []string) {
@@ -1342,18 +1328,32 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	if masterKeyExisted {
 		// Preserve path: reuse the existing identity. The system credentials must
 		// match what seeded the NATS KV `system` secret, so load them rather than
-		// mint new ones. The admin credentials are not recovered: bootstrap.json is
-		// consumed and deleted by awsgw after first boot, and the operator's copy
-		// already lives in ~/.aws/credentials. Leaving them empty makes
-		// finalizeNodeSetup refresh only ~/.aws/config (endpoint/CA for a changed
-		// bind IP), not the credentials.
+		// mint new ones.
 		accessKey, secretKey, err = loadSystemCredentials(configDir)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error loading preserved system credentials: %v\n", err)
 			os.Exit(1)
 		}
-		fmt.Println("\n🔐 Preserved existing identity (master key, credentials and CA unchanged)")
+
+		// The admin pair is minted afresh rather than preserved, because it cannot
+		// be read back: bootstrap.json is consumed and deleted by awsgw on first
+		// start. The half of the identity that survives a re-init is on local disk,
+		// but the IAM store lives in NATS and predastore — so forming a cluster out
+		// of already-initialized nodes rebuilds it and the old access key stops
+		// existing. Carrying an empty pair through instead is worse than useless:
+		// it reaches every joiner's bootstrap.json, where seeding rejects the empty
+		// key ID and awsgw fails to start on every restart, forever.
+		bootstrapResult, err := writeBootstrapFiles(configDir, bootstrapDir, masterKey, accessKey, secretKey, accountID)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error writing bootstrap files: %v\n", err)
+			os.Exit(1)
+		}
+		adminAccessKey = bootstrapResult.AdminAccessKey
+		adminSecretKey = bootstrapResult.AdminSecretKey
+
+		fmt.Println("\n🔐 Preserved existing identity (master key and CA unchanged)")
 		fmt.Printf("   Master key: %s\n", filepath.Join(configDir, "master.key"))
+		fmt.Printf("   Admin credentials reissued in %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
 	} else {
 		// Fresh install: mint system + admin credentials and seed the bootstrap files.
 		accessKey, err = admin.GenerateAWSAccessKey()
@@ -1524,8 +1524,10 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	// is node-local, so it is derived here from this node's own config dirs.
 	northstarConfigPath := filepath.Join(dirs.Northstar, "northstar.toml")
 
-	// Parse multi-node predastore configuration (legacy flag-based approach for single-node)
-	var predastoreHostID int
+	// Parse multi-node predastore configuration. A single-node install is host
+	// 1, the only [[host]] its predastore.toml declares; the multi-node path
+	// below replaces this with the host matching this machine.
+	predastoreHostID := 1
 	if predastoreNodesStr != "" {
 		ips := strings.Split(predastoreNodesStr, ",")
 		if len(ips) < 2 {
@@ -1806,6 +1808,10 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		os.Exit(1)
 	}
 
+	// The host firewall scopes this port to known peers, and a node joining is
+	// not one yet. Closed again on every exit below.
+	closeFormationPort := openFormationPort(port)
+
 	fmt.Printf("\n📡 Formation server started on %s\n", formationAddr)
 	fmt.Printf("   Waiting for %d more node(s) to join...\n", expectedNodes-1)
 	fmt.Printf("   Token expires in %s\n\n", tokenTTL)
@@ -1816,6 +1822,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	if err := fs.WaitForCompletion(formationTimeout); err != nil {
 		fmt.Fprintf(os.Stderr, "❌ %v\n", err)
 		fs.Shutdown(context.Background())
+		closeFormationPort()
 		os.Remove(tokenPath)
 		os.Exit(1)
 	}
@@ -1842,8 +1849,10 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	// is node-local, so it is derived here from this node's own config dirs.
 	northstarConfigPath := filepath.Join(dirs.Northstar, "northstar.toml")
 
-	// Generate multi-node predastore config
-	var predastoreHostID int
+	// Generate multi-node predastore config. A single-node install is host 1,
+	// the only [[host]] its predastore.toml declares; the multi-node path below
+	// replaces this with the host matching this machine.
+	predastoreHostID := 1
 	hasPredastoreConfig := len(predastoreNodes) >= 2
 	if hasPredastoreConfig {
 		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, spxRoot, bindIP, compactionInterval, northstarCreds)
@@ -1858,7 +1867,14 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 			os.Exit(1)
 		}
 
+		// One machine is one predastore host, so the node list entry matching
+		// this bind IP is this node's host ID. Without one the service has no
+		// [[host]] to run and refuses to start.
 		predastoreHostID = admin.FindNodeIDByIP(predastoreNodes, bindIP)
+		if predastoreHostID == 0 {
+			fmt.Fprintf(os.Stderr, "❌ Error: bind IP %s not found in the predastore node list\n", bindIP)
+			os.Exit(1)
+		}
 		fmt.Printf("✅ Created: multi-node predastore.toml (host ID: %d)\n", predastoreHostID)
 	}
 
@@ -1925,6 +1941,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 
 	// Shutdown formation server
 	fs.Shutdown(context.Background())
+	closeFormationPort()
 	if err := os.Remove(tokenPath); err != nil && !errors.Is(err, os.ErrNotExist) {
 		slog.Warn("Failed to remove join token file", "path", tokenPath, "error", err)
 	}
@@ -2392,8 +2409,10 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 
 	northstarCreds, northstarConfigPath := northstarFromFormation(creds, dirs)
 
-	// Generate multi-node predastore config
-	var predastoreHostID int
+	// Generate multi-node predastore config. A single-node install is host 1,
+	// the only [[host]] its predastore.toml declares; the multi-node path below
+	// replaces this with the host matching this machine.
+	predastoreHostID := 1
 	hasPredastoreConfig := len(predastoreNodes) >= 2
 
 	if hasPredastoreConfig {
@@ -2576,13 +2595,24 @@ func initIAMServiceFromConfig() (*handlers_iam.IAMServiceImpl, *config.ClusterCo
 	return svc, cfg, nc, func() { nc.Close() }, nil
 }
 
-// adminAccessPolicyDocument is the AdministratorAccess policy document that
-// grants full access to all actions and resources.
-const adminAccessPolicyDocument = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"*","Resource":"*"}]}`
+// openAccountNameIndex opens the account-name reservation index over an
+// existing NATS connection.
+func openAccountNameIndex(ctx context.Context, nc *nats.Conn) (*handlers_iam.AccountNameIndex, error) {
+	js, err := jetstream.New(nc)
+	if err != nil {
+		return nil, fmt.Errorf("get JetStream context: %w", err)
+	}
+	return handlers_iam.NewAccountNameIndex(ctx, js)
+}
 
 func runAccountCreate(cmd *cobra.Command, args []string) {
 	name, _ := cmd.Flags().GetString("name")
 
+	if remote, _ := cmd.Flags().GetBool("remote"); remote {
+		runAccountCreateRemote(cmd, name)
+		return
+	}
+
 	svc, cfg, nc, cleanup, err := initIAMServiceFromConfig()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
@@ -2590,60 +2620,73 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 	}
 	defer cleanup()
 
-	// 1. Create the account
-	account, err := svc.CreateAccount(name)
+	ctx := context.Background()
+
+	// Reject a duplicate before allocating an account ID. CreateAccount does not
+	// check names, so without this two runs of the same command silently produce
+	// two accounts that differ only by ID.
+	existing, err := handlers_iam.FindAccountByName(svc, name)
 	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error checking existing accounts: %v\n", err)
+		os.Exit(1)
+	}
+	if existing != nil {
+		fmt.Fprintf(os.Stderr, "Error: account %q already exists (account ID %s)\n",
+			existing.AccountName, existing.AccountID)
+		os.Exit(1)
+	}
+
+	// Reserve the name so a concurrent create — including one through
+	// /admin/CreateAccount — cannot take it between the check above and the
+	// create below. The CLI has no client token, so it owns the reservation by
+	// a per-invocation identifier.
+	names, err := openAccountNameIndex(ctx, nc)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	reservationOwner := "spx-cli-" + uuid.NewV4().String()
+	switch err := names.Reserve(ctx, name, reservationOwner); {
+	case errors.Is(err, handlers_iam.ErrNameTaken):
+		fmt.Fprintf(os.Stderr, "Error: account %q already exists\n", name)
+		os.Exit(1)
+	case errors.Is(err, handlers_iam.ErrNameInFlight):
+		fmt.Fprintf(os.Stderr, "Error: another account creation for %q is in progress\n", name)
+		os.Exit(1)
+	case err != nil:
+		fmt.Fprintf(os.Stderr, "Error reserving account name: %v\n", err)
+		os.Exit(1)
+	}
+
+	account, err := svc.CreateAccount(handlers_iam.NormalizeAccountName(name))
+	if err != nil {
+		if relErr := names.Release(ctx, name, reservationOwner); relErr != nil {
+			fmt.Fprintf(os.Stderr, "Warning: could not release name reservation: %v\n", relErr)
+		}
 		fmt.Fprintf(os.Stderr, "Error creating account: %v\n", err)
 		os.Exit(1)
 	}
 	accountID := account.AccountID
 
+	if err := names.Commit(ctx, name, accountID, reservationOwner); err != nil {
+		fmt.Fprintf(os.Stderr, "Warning: could not index account name: %v\n", err)
+	}
+
 	// Create default VPC for the new account (belt-and-suspenders: daemon also
 	// does this via iam.account.created event, but daemon may not be running).
 	nodeConfig := cfg.Nodes[cfg.Node]
-	vpcSvc, vpcErr := handlers_ec2_vpc.NewVPCServiceImplWithNATS(context.Background(), &nodeConfig, nc)
+	vpcSvc, vpcErr := handlers_ec2_vpc.NewVPCServiceImplWithNATS(ctx, &nodeConfig, nc)
 	if vpcErr != nil {
 		fmt.Fprintf(os.Stderr, "Warning: could not create default VPC service: %v\n", vpcErr)
 	} else if _, vpcErr = vpcSvc.EnsureDefaultVPC(accountID); vpcErr != nil {
 		fmt.Fprintf(os.Stderr, "Warning: could not create default VPC: %v\n", vpcErr)
 	}
 
-	// 2. Create admin user
-	_, err = svc.CreateUser(accountID, &iam.CreateUserInput{
-		UserName: aws.String("admin"),
-	})
+	// Shared with /admin/CreateAccount so the CLI and the remote endpoint cannot
+	// drift in what an account is made of.
+	provisioned, err := handlers_iam.ProvisionAccount(svc, accountID, account.AccountName)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error creating admin user: %v\n", err)
-		os.Exit(1)
-	}
-
-	// 3. Create access key for admin user
-	akOut, err := svc.CreateAccessKey(accountID, &iam.CreateAccessKeyInput{
-		UserName: aws.String("admin"),
-	})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error creating access key: %v\n", err)
-		os.Exit(1)
-	}
-
-	// 4. Create AdministratorAccess policy scoped to this account
-	policyARN := fmt.Sprintf("arn:aws:iam::%s:policy/AdministratorAccess", accountID)
-	_, err = svc.CreatePolicy(accountID, &iam.CreatePolicyInput{
-		PolicyName:     aws.String("AdministratorAccess"),
-		PolicyDocument: aws.String(adminAccessPolicyDocument),
-	})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error creating admin policy: %v\n", err)
-		os.Exit(1)
-	}
-
-	// 5. Attach policy to admin user
-	_, err = svc.AttachUserPolicy(accountID, &iam.AttachUserPolicyInput{
-		UserName:  aws.String("admin"),
-		PolicyArn: aws.String(policyARN),
-	})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error attaching policy: %v\n", err)
+		fmt.Fprintf(os.Stderr, "Error provisioning account: %v\n", err)
 		os.Exit(1)
 	}
 
@@ -2665,8 +2708,8 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 	configPath := filepath.Join(homeDir, ".aws", "config")
 
 	if err := admin.UpdateAWSINIFile(credPath, profileName, map[string]string{
-		"aws_access_key_id":     *akOut.AccessKey.AccessKeyId,
-		"aws_secret_access_key": *akOut.AccessKey.SecretAccessKey,
+		"aws_access_key_id":     provisioned.AccessKeyID,
+		"aws_secret_access_key": provisioned.SecretAccessKey,
 	}); err != nil {
 		fmt.Fprintf(os.Stderr, "Warning: could not update AWS credentials: %v\n", err)
 	}
@@ -2686,16 +2729,24 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 	// Print credentials
 	fmt.Println("\nAccount created successfully!")
 	fmt.Printf("  Account ID:        %s\n", accountID)
-	fmt.Printf("  Account Name:      %s\n", name)
-	fmt.Printf("  Admin User:        admin\n")
-	fmt.Printf("  Access Key ID:     %s\n", *akOut.AccessKey.AccessKeyId)
-	fmt.Printf("  Secret Access Key: %s\n", *akOut.AccessKey.SecretAccessKey)
+	fmt.Printf("  Account Name:      %s\n", provisioned.AccountName)
+	fmt.Printf("  Admin User:        %s\n", provisioned.AdminUser)
+	fmt.Printf("  Access Key ID:     %s\n", provisioned.AccessKeyID)
+	fmt.Printf("  Secret Access Key: %s\n", provisioned.SecretAccessKey)
 	fmt.Printf("  AWS Profile:       %s\n", profileName)
 	fmt.Println("\nUse with:")
 	fmt.Printf("  AWS_PROFILE=%s aws ec2 describe-instances\n", profileName)
 }
 
 func runAccountList(cmd *cobra.Command, args []string) {
+	if remote, _ := cmd.Flags().GetBool("remote"); remote {
+		if err := runAccountListRemote(cmd); err != nil {
+			fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+			os.Exit(1)
+		}
+		return
+	}
+
 	svc, _, _, cleanup, err := initIAMServiceFromConfig()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
@@ -2709,20 +2760,7 @@ func runAccountList(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	if len(accounts) == 0 {
-		fmt.Println("No accounts found.")
-		return
-	}
-
-	fmt.Printf("%-14s %-20s %-10s %s\n", "ACCOUNT ID", "NAME", "STATUS", "CREATED")
-	fmt.Printf("%-14s %-20s %-10s %s\n", "----------", "----", "------", "-------")
-	for _, a := range accounts {
-		created := a.CreatedAt
-		if t, err := time.Parse(time.RFC3339, a.CreatedAt); err == nil {
-			created = t.Format("2006-01-02 15:04")
-		}
-		fmt.Printf("%-14s %-20s %-10s %s\n", a.AccountID, a.AccountName, a.Status, created)
-	}
+	printAccountTable(accountSummaries(accounts))
 }
 
 func runCertRenew(cmd *cobra.Command, _ []string) {

cmd/spinifex/cmd/admin_account_delete.go

@@ -0,0 +1,200 @@
+package cmd
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/accountteardown"
+	"github.com/mulgadc/spinifex/spinifex/admin"
+	"github.com/mulgadc/spinifex/spinifex/objectstore"
+	"github.com/spf13/cobra"
+)
+
+// accountDeleteTimeout is the outer bound on the whole teardown. It must stay
+// well above the sum of the per-stage drain budgets, which are what actually
+// decide when to give up on a resource; cutting in before them would abandon a
+// large account midway with no stuck report to show for it.
+const accountDeleteTimeout = 2 * time.Hour
+
+var accountDeleteCmd = &cobra.Command{
+	Use:   "delete <account-id>",
+	Short: "Tear down an account's infrastructure and remove the account",
+	Long: `Remove an account and everything it owns.
+
+The account is marked TERMINATING first, which blocks its credentials, and then
+its resources are deleted in dependency order: compute, attachments, storage,
+network, platform, identity, account. Nothing is removed until the account can
+no longer create more.
+
+The account name must be given to confirm, so a mistyped account id fails
+rather than emptying a live tenant. The system account (000000000000) and the
+super admin (000000000001) can never be deleted.
+
+  spx admin account delete 000000000042 --dry-run
+  spx admin account delete 000000000042 --name tenant@example.com
+
+--force is for resources the ordinary API refuses to delete: a volume attached
+to an instance that will not terminate leaves both stranded and the account can
+never be emptied. It clears the attachment in the control plane without the
+guest's cooperation, so use it only once the guest is being destroyed.`,
+	Args: cobra.ExactArgs(1),
+	RunE: runAccountDelete,
+
+	SilenceUsage: true,
+}
+
+func init() {
+	accountCmd.AddCommand(accountDeleteCmd)
+
+	accountDeleteCmd.Flags().Bool("dry-run", false, "Report what would be deleted and change nothing")
+	accountDeleteCmd.Flags().Bool("force", false, "Escalate past state guards for resources that will not delete")
+	accountDeleteCmd.Flags().Bool("yes", false, "Skip the interactive confirmation (requires --name)")
+	accountDeleteCmd.Flags().String("name", "", "Account name, which must match the stored record")
+	accountDeleteCmd.Flags().Bool("remote", false, "Delete over POST /admin/DeleteAccount instead of connecting to NATS")
+	accountDeleteCmd.Flags().String("endpoint", "", "Gateway endpoint for --remote (default: this node's AWS gateway)")
+	accountDeleteCmd.Flags().String("region", "", "SigV4 region for --remote (default: this node's region)")
+	accountDeleteCmd.Flags().String("ca-bundle", "", "CA certificate for --remote (default: this node's CA)")
+	accountDeleteCmd.Flags().String("client-token", "", "Idempotency token for --remote (default: generated; reuse it to follow the same teardown)")
+}
+
+func runAccountDelete(cmd *cobra.Command, args []string) error {
+	accountID := args[0]
+	if remote, _ := cmd.Flags().GetBool("remote"); remote {
+		return runAccountDeleteRemote(cmd, accountID)
+	}
+
+	dryRun, _ := cmd.Flags().GetBool("dry-run")
+	force, _ := cmd.Flags().GetBool("force")
+	assumeYes, _ := cmd.Flags().GetBool("yes")
+	name, _ := cmd.Flags().GetString("name")
+
+	svc, cfg, nc, cleanup, err := initIAMServiceFromConfig()
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), accountDeleteTimeout)
+	defer cancel()
+
+	node := cfg.Nodes[cfg.Node]
+	buckets := objectstore.NewS3ObjectStoreFromConfig(
+		admin.DialTarget(node.Predastore.Host),
+		node.Predastore.Region,
+		node.Predastore.AccessKey,
+		node.Predastore.SecretKey,
+	)
+
+	engine, err := accountteardown.NewClusterEngine(ctx, nc, len(cfg.Nodes), svc, buckets)
+	if err != nil {
+		return err
+	}
+
+	if dryRun {
+		result, err := engine.Inventory(ctx, accountID)
+		if err != nil {
+			return err
+		}
+		printTeardownPlan(result)
+		fmt.Printf("\nDry run: nothing was deleted. %d resources would be removed.\n", result.DeletedCount())
+		return nil
+	}
+
+	// Always show the inventory before destroying it. An operator who has not
+	// seen what is about to go has not really confirmed anything.
+	plan, err := engine.Inventory(ctx, accountID)
+	if err != nil {
+		return err
+	}
+	printTeardownPlan(plan)
+
+	if name == "" {
+		if assumeYes {
+			return errors.New("--yes requires --name so the account is still confirmed")
+		}
+		name, err = promptAccountName(accountID)
+		if err != nil {
+			return err
+		}
+	}
+
+	result, err := engine.Teardown(ctx, accountteardown.Request{
+		AccountID: accountID, AccountName: name, Force: force,
+	})
+	if result != nil {
+		printTeardownResult(result)
+	}
+	if errors.Is(err, accountteardown.ErrResourcesStuck) {
+		fmt.Fprintf(os.Stderr, "\nThe account is still TERMINATING so its leftovers keep an owner.\n")
+		if !force {
+			fmt.Fprintf(os.Stderr, "Re-run with --force to clear resources the ordinary API refuses to delete.\n")
+		}
+		return err
+	}
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("\nAccount %s deleted. %d resources removed.\n", accountID, result.DeletedCount())
+	return nil
+}
+
+// promptAccountName asks for the confirmation name on a terminal.
+func promptAccountName(accountID string) (string, error) {
+	fmt.Printf("\nThis permanently deletes account %s and everything above.\n", accountID)
+	fmt.Print("Type the account name to confirm: ")
+
+	reader := bufio.NewReader(os.Stdin)
+	entered, err := reader.ReadString('\n')
+	if err != nil {
+		return "", fmt.Errorf("read confirmation: %w", err)
+	}
+	entered = strings.TrimSpace(entered)
+	if entered == "" {
+		return "", errors.New("cancelled: no account name given")
+	}
+	return entered, nil
+}
+
+func printTeardownPlan(result *accountteardown.Result) {
+	fmt.Printf("Account %s (%s)\n\n", result.AccountID, result.AccountName)
+	if result.DeletedCount() == 0 {
+		fmt.Println("The account holds no resources.")
+		return
+	}
+	for _, stage := range result.Stages {
+		if len(stage.Deleted) == 0 {
+			continue
+		}
+		fmt.Printf("  %s\n", stage.Stage)
+		for _, resource := range stage.Deleted {
+			fmt.Printf("    %s\n", resource)
+		}
+	}
+}
+
+func printTeardownResult(result *accountteardown.Result) {
+	fmt.Println()
+	for _, stage := range result.Stages {
+		printTeardownStage(stage)
+	}
+}
+
+// printTeardownStage reports one finished stage. A stage that removed nothing
+// and left nothing behind is not printed: an account with no ECS clusters
+// should not read as seven lines of activity.
+func printTeardownStage(stage accountteardown.StageResult) {
+	if len(stage.Deleted) == 0 && len(stage.Stuck) == 0 {
+		return
+	}
+	fmt.Printf("  %-12s %d deleted, %d stuck (%s)\n",
+		stage.Stage, len(stage.Deleted), len(stage.Stuck), stage.Elapsed)
+	for _, stuck := range stage.Stuck {
+		fmt.Printf("    STUCK %s: %s\n", stuck.Resource, stuck.Reason)
+	}
+}

cmd/spinifex/cmd/admin_account_delete_output_test.go

@@ -0,0 +1,162 @@
+package cmd_test
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+	"github.com/mulgadc/spinifex/spinifex/accountteardown"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// captureOutput redirects os.Stdout for the duration of fn and returns what was
+// written. The teardown printers are the operator's only view of what is about
+// to be destroyed, so what they print is the thing under test.
+func captureOutput(t *testing.T, fn func()) string {
+	t.Helper()
+	r, w, err := os.Pipe()
+	require.NoError(t, err)
+	orig := os.Stdout
+	os.Stdout = w //nolint:reassign // test-local stdout capture, restored below
+
+	fn()
+
+	require.NoError(t, w.Close())
+	os.Stdout = orig //nolint:reassign // restoring the real os.Stdout captured above
+
+	var buf bytes.Buffer
+	_, err = io.Copy(&buf, r)
+	require.NoError(t, err)
+	return buf.String()
+}
+
+// feedStdin swaps os.Stdin for a pipe carrying content, mirroring the stdout
+// capture above.
+func feedStdin(t *testing.T, content string, fn func()) {
+	t.Helper()
+	r, w, err := os.Pipe()
+	require.NoError(t, err)
+	orig := os.Stdin
+	os.Stdin = r                          //nolint:reassign // test-local stdin swap, restored below
+	t.Cleanup(func() { os.Stdin = orig }) //nolint:reassign // restoring the real os.Stdin captured above
+
+	go func() {
+		_, _ = io.WriteString(w, content)
+		_ = w.Close()
+	}()
+
+	fn()
+}
+
+func teardownPlan() *accountteardown.Result {
+	return &accountteardown.Result{
+		AccountID:   "000000000042",
+		AccountName: "tenant@example.com",
+		DryRun:      true,
+		Stages: []accountteardown.StageResult{
+			{
+				Stage:   accountteardown.StageCompute,
+				Deleted: []accountteardown.Resource{{Kind: "instance", ID: "i-1", Detail: "running"}},
+				Elapsed: "0s",
+			},
+			{Stage: accountteardown.StageAttachments, Elapsed: "0s"},
+			{
+				Stage:   accountteardown.StageStorage,
+				Deleted: []accountteardown.Resource{{Kind: "volume", ID: "vol-1"}},
+				Elapsed: "0s",
+			},
+		},
+	}
+}
+
+// The plan is what the operator confirms against, so every resource has to
+// appear under its stage — and an empty stage must not pad the list.
+func TestPrintTeardownPlanListsEveryResource(t *testing.T) {
+	output := captureOutput(t, func() { cmd.PrintTeardownPlan(teardownPlan()) })
+
+	assert.Contains(t, output, "000000000042")
+	assert.Contains(t, output, "tenant@example.com")
+	assert.Contains(t, output, "i-1")
+	assert.Contains(t, output, "vol-1")
+	assert.Contains(t, output, "compute")
+	assert.Contains(t, output, "storage")
+	assert.NotContains(t, output, "attachments", "an empty stage is noise, not information")
+}
+
+// An account with nothing in it must say so. A bare header reads as truncated
+// output, which is exactly when an operator stops and asks.
+func TestPrintTeardownPlanSaysWhenTheAccountIsEmpty(t *testing.T) {
+	output := captureOutput(t, func() {
+		cmd.PrintTeardownPlan(&accountteardown.Result{
+			AccountID:   "000000000042",
+			AccountName: "tenant@example.com",
+			Stages:      []accountteardown.StageResult{{Stage: accountteardown.StageCompute, Elapsed: "0s"}},
+		})
+	})
+
+	assert.Contains(t, output, "holds no resources")
+}
+
+// A stuck resource is the most valuable thing a teardown reports: each one is a
+// delete path that does not work. It must be named, with the reason.
+func TestPrintTeardownResultNamesStuckResources(t *testing.T) {
+	result := &accountteardown.Result{
+		AccountID: "000000000042",
+		Stages: []accountteardown.StageResult{
+			{
+				Stage:   accountteardown.StageCompute,
+				Deleted: []accountteardown.Resource{{Kind: "instance", ID: "i-1"}},
+				Elapsed: "5s",
+			},
+			{Stage: accountteardown.StageAttachments, Elapsed: "0s"},
+			{
+				Stage: accountteardown.StageStorage,
+				Stuck: []accountteardown.Stuck{{
+					Resource: accountteardown.Resource{Kind: "volume", ID: "vol-1"},
+					Reason:   "still attached to i-1",
+				}},
+				Elapsed: "5m0s",
+			},
+		},
+	}
+
+	output := captureOutput(t, func() { cmd.PrintTeardownResult(result) })
+
+	assert.Contains(t, output, "1 deleted, 0 stuck")
+	assert.Contains(t, output, "STUCK")
+	assert.Contains(t, output, "vol-1")
+	assert.Contains(t, output, "still attached to i-1")
+	assert.NotContains(t, output, "attachments")
+}
+
+// The typed name is the confirmation. Surrounding whitespace is a paste
+// artefact rather than a mismatch, but an empty line is a cancellation.
+func TestPromptAccountNameReadsTheConfirmation(t *testing.T) {
+	var name string
+	var err error
+
+	captureOutput(t, func() {
+		feedStdin(t, "  tenant@example.com  \n", func() {
+			name, err = cmd.PromptAccountName("000000000042")
+		})
+	})
+
+	require.NoError(t, err)
+	assert.Equal(t, "tenant@example.com", name)
+}
+
+func TestPromptAccountNameTreatsAnEmptyReplyAsCancelled(t *testing.T) {
+	var err error
+
+	captureOutput(t, func() {
+		feedStdin(t, "\n", func() {
+			_, err = cmd.PromptAccountName("000000000042")
+		})
+	})
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "cancelled")
+}

cmd/spinifex/cmd/admin_account_delete_remote.go

@@ -0,0 +1,142 @@
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/accountteardown"
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	"github.com/spf13/cobra"
+)
+
+// accountDeletePollInterval is how often --remote asks the gateway how far the
+// teardown has got. The work is minutes long, so a tighter poll would only add
+// load without telling the operator anything new.
+var accountDeletePollInterval = 10 * time.Second
+
+// runAccountDeleteRemote drives POST /admin/DeleteAccount and then follows the
+// job to completion, so the remote path reports the same outcome the local one
+// prints rather than only that the teardown started.
+func runAccountDeleteRemote(cmd *cobra.Command, accountID string) error {
+	dryRun, _ := cmd.Flags().GetBool("dry-run")
+	force, _ := cmd.Flags().GetBool("force")
+	assumeYes, _ := cmd.Flags().GetBool("yes")
+	name, _ := cmd.Flags().GetString("name")
+	clientToken, _ := cmd.Flags().GetString("client-token")
+
+	target, err := resolveAdminTarget(cmd)
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), accountDeleteTimeout)
+	defer cancel()
+
+	// The inventory is fetched first even for a real deletion: an operator who
+	// has not seen what is about to go has not really confirmed anything.
+	plan, err := deleteAccountRemote(ctx, target, gateway.DeleteAccountRequest{
+		AccountID: accountID, DryRun: true,
+	})
+	if err != nil {
+		return err
+	}
+	if plan.Inventory != nil {
+		printTeardownPlan(plan.Inventory)
+	}
+	if dryRun {
+		fmt.Printf("\nDry run: nothing was deleted. %d resources would be removed.\n", plan.Inventory.DeletedCount())
+		return nil
+	}
+
+	if name == "" {
+		if assumeYes {
+			return errors.New("--yes requires --name so the account is still confirmed")
+		}
+		if name, err = promptAccountName(accountID); err != nil {
+			return err
+		}
+	}
+	if clientToken == "" {
+		clientToken = newClientToken()
+	}
+
+	started, err := deleteAccountRemote(ctx, target, gateway.DeleteAccountRequest{
+		AccountID: accountID, AccountName: name, ClientToken: clientToken, Force: force,
+	})
+	if err != nil {
+		var adminErr *adminError
+		if errors.As(err, &adminErr) && retryableAdminErrors[adminErr.Code] {
+			fmt.Fprintf(os.Stderr, "Retry with --client-token %s to follow the same teardown.\n", clientToken)
+		}
+		return err
+	}
+	fmt.Printf("\nTeardown started: %s\n", started.DeletionID)
+
+	return followAccountDeletion(ctx, target, accountID)
+}
+
+// followAccountDeletion polls until the job reaches a terminal state, printing
+// each stage as it lands.
+func followAccountDeletion(ctx context.Context, target adminTarget, accountID string) error {
+	reported := 0
+	for {
+		job, err := describeAccountDeletionRemote(ctx, target, accountID)
+		if err != nil {
+			return err
+		}
+
+		for ; reported < len(job.Stages); reported++ {
+			printTeardownStage(job.Stages[reported])
+		}
+
+		switch job.State {
+		case gateway.DeletionStateCompleted:
+			fmt.Printf("\nAccount %s deleted.\n", accountID)
+			return nil
+		case gateway.DeletionStateFailed:
+			fmt.Fprintf(os.Stderr, "\nThe account is still TERMINATING so its leftovers keep an owner.\n")
+			return fmt.Errorf("teardown failed: %s", job.Error)
+		}
+
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("stopped following teardown of %s: %w", accountID, ctx.Err())
+		case <-time.After(accountDeletePollInterval):
+		}
+	}
+}
+
+func deleteAccountRemote(
+	ctx context.Context,
+	target adminTarget,
+	req gateway.DeleteAccountRequest,
+) (*gateway.DeleteAccountResponse, error) {
+	var out gateway.DeleteAccountResponse
+	if err := callAdmin(ctx, target, "DeleteAccount", req, &out); err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+func describeAccountDeletionRemote(ctx context.Context, target adminTarget, accountID string) (*accountDeletionStatus, error) {
+	var out accountDeletionStatus
+	err := callAdmin(ctx, target, "DescribeAccountDeletion",
+		gateway.DescribeAccountDeletionRequest{AccountID: accountID}, &out)
+	if err != nil {
+		return nil, err
+	}
+	return &out, nil
+}
+
+// accountDeletionStatus is the subset of the stored job this command reports.
+// The gateway's own record type is unexported, and the CLI needs no more than
+// the state, the stages and why it stopped.
+type accountDeletionStatus struct {
+	DeletionID string                        `json:"deletionId"`
+	State      string                        `json:"state"`
+	Stages     []accountteardown.StageResult `json:"stages"`
+	Error      string                        `json:"error"`
+}

cmd/spinifex/cmd/admin_account_delete_remote_test.go

@@ -0,0 +1,199 @@
+package cmd_test
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// adminTestCredentials puts a static key in the environment so the SigV4 signer
+// resolves without reaching for IMDS.
+func adminTestCredentials(t *testing.T) {
+	t.Helper()
+	t.Setenv("AWS_ACCESS_KEY_ID", "AKIAEXAMPLE")
+	t.Setenv("AWS_SECRET_ACCESS_KEY", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
+}
+
+// newScriptedAdminServer answers each successive request with the next body,
+// repeating the last one, which is how a job that changes state between polls
+// is reproduced.
+func newScriptedAdminServer(t *testing.T, bodies ...string) (*httptest.Server, *atomic.Int32) {
+	t.Helper()
+	var calls atomic.Int32
+
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		index := int(calls.Add(1)) - 1
+		if index >= len(bodies) {
+			index = len(bodies) - 1
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(bodies[index]))
+	}))
+	t.Cleanup(srv.Close)
+	return srv, &calls
+}
+
+// The delete request must reach /admin/DeleteAccount signed for the spinifex
+// service: a key signed for another service is treated as probing and denied.
+func TestDeleteAccountRemoteSignsAndPosts(t *testing.T) {
+	adminTestCredentials(t)
+
+	var got http.Request
+	srv := newAdminTestServer(t, http.StatusOK,
+		`{"deletionId":"d-1","accountId":"000000000042","state":"RUNNING"}`, &got)
+
+	out, err := cmd.DeleteAccountRemote(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)),
+		gateway.DeleteAccountRequest{
+			AccountID:   "000000000042",
+			AccountName: "tenant@example.com",
+			ClientToken: strings.Repeat("a", 32),
+		})
+	require.NoError(t, err)
+
+	assert.Equal(t, "d-1", out.DeletionID)
+	assert.Equal(t, gateway.DeletionStateRunning, out.State)
+
+	assert.Equal(t, http.MethodPost, got.Method)
+	assert.Equal(t, "/admin/DeleteAccount", got.URL.Path)
+	assert.Contains(t, got.Header.Get("Authorization"), "/us-west-1/spinifex/aws4_request")
+}
+
+// A dry run answers inline with the inventory, which is what the operator
+// confirms against before the real call.
+func TestDeleteAccountRemoteDryRunCarriesTheInventory(t *testing.T) {
+	adminTestCredentials(t)
+
+	var got http.Request
+	srv := newAdminTestServer(t, http.StatusOK,
+		`{"accountId":"000000000042","state":"DRY_RUN","dryRun":true,"inventory":{"accountId":"000000000042",`+
+			`"dryRun":true,"stages":[{"stage":1,"deleted":[{"kind":"instance","id":"i-1"}],"elapsed":"0s"}]}}`, &got)
+
+	out, err := cmd.DeleteAccountRemote(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)),
+		gateway.DeleteAccountRequest{AccountID: "000000000042", DryRun: true})
+	require.NoError(t, err)
+
+	assert.True(t, out.DryRun)
+	require.NotNil(t, out.Inventory)
+	assert.Equal(t, 1, out.Inventory.DeletedCount())
+}
+
+// A refusal must surface as the gateway's code, not as a decode failure, so the
+// operator can tell a protected account from an unreachable cluster.
+func TestDeleteAccountRemoteSurfacesGatewayError(t *testing.T) {
+	adminTestCredentials(t)
+
+	var got http.Request
+	srv := newAdminTestServer(t, http.StatusForbidden,
+		`{"error":{"code":"AccessDenied","message":"denied"},"requestId":"req-9"}`, &got)
+
+	_, err := cmd.DeleteAccountRemote(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)),
+		gateway.DeleteAccountRequest{
+			AccountID:   "000000000001",
+			AccountName: "spinifex",
+			ClientToken: strings.Repeat("a", 32),
+		})
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "AccessDenied")
+	assert.Contains(t, err.Error(), "req-9")
+	assert.False(t, cmd.RetryableAdminError(err), "a protected account is not fixed by retrying")
+}
+
+func TestDescribeAccountDeletionRemoteReportsProgress(t *testing.T) {
+	adminTestCredentials(t)
+
+	var got http.Request
+	srv := newAdminTestServer(t, http.StatusOK,
+		`{"deletionId":"d-1","state":"RUNNING","stages":[{"stage":1,"elapsed":"5s"}]}`, &got)
+
+	out, err := cmd.DescribeAccountDeletionRemote(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042")
+	require.NoError(t, err)
+
+	assert.Equal(t, "/admin/DescribeAccountDeletion", got.URL.Path)
+	assert.Equal(t, gateway.DeletionStateRunning, out.State)
+	assert.Len(t, out.Stages, 1)
+}
+
+// The follow loop is what makes --remote report the same outcome the local
+// path prints. It must keep polling while the job runs and stop on the first
+// terminal state.
+func TestFollowAccountDeletionPollsUntilComplete(t *testing.T) {
+	adminTestCredentials(t)
+	defer cmd.SetAccountDeletePollInterval(time.Millisecond)()
+
+	srv, calls := newScriptedAdminServer(t,
+		`{"deletionId":"d-1","state":"RUNNING","stages":[{"stage":1,"elapsed":"5s"}]}`,
+		`{"deletionId":"d-1","state":"RUNNING","stages":[{"stage":1,"elapsed":"5s"},{"stage":2,"elapsed":"1s"}]}`,
+		`{"deletionId":"d-1","state":"COMPLETED","stages":[{"stage":1,"elapsed":"5s"},{"stage":2,"elapsed":"1s"}]}`,
+	)
+
+	err := cmd.FollowAccountDeletion(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042")
+
+	require.NoError(t, err)
+	assert.Equal(t, int32(3), calls.Load(), "the loop must keep polling while the job runs")
+}
+
+// A failed teardown must exit non-zero with the stored reason: the account is
+// left TERMINATING, and an operator who reads "done" would not go looking.
+func TestFollowAccountDeletionReportsFailure(t *testing.T) {
+	adminTestCredentials(t)
+	defer cmd.SetAccountDeletePollInterval(time.Millisecond)()
+
+	srv, _ := newScriptedAdminServer(t,
+		`{"deletionId":"d-1","state":"FAILED","stages":[],"error":"2 resources left"}`)
+
+	err := cmd.FollowAccountDeletion(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042")
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "2 resources left")
+}
+
+// Giving up on following is not the same as the teardown stopping, so the
+// message has to name the account the caller must go back to.
+func TestFollowAccountDeletionStopsWhenTheContextEnds(t *testing.T) {
+	adminTestCredentials(t)
+	defer cmd.SetAccountDeletePollInterval(50 * time.Millisecond)()
+
+	srv, _ := newScriptedAdminServer(t, `{"deletionId":"d-1","state":"RUNNING","stages":[]}`)
+
+	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+	defer cancel()
+
+	err := cmd.FollowAccountDeletion(ctx,
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042")
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "000000000042")
+}
+
+// An unreadable job is not a finished one. Reporting it as complete would hide
+// a teardown still holding an account's resources.
+func TestFollowAccountDeletionFailsOnAnUnreadableJob(t *testing.T) {
+	adminTestCredentials(t)
+	defer cmd.SetAccountDeletePollInterval(time.Millisecond)()
+
+	var got http.Request
+	srv := newAdminTestServer(t, http.StatusNotFound,
+		`{"error":{"code":"NoSuchEntity","message":"no job"},"requestId":"req-2"}`, &got)
+
+	err := cmd.FollowAccountDeletion(t.Context(),
+		cmd.AdminTarget(srv.URL, "us-west-1", writeServerCA(t, srv)), "000000000042")
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "NoSuchEntity")
+}

cmd/spinifex/cmd/admin_account_list_remote.go

@@ -0,0 +1,67 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	"github.com/spf13/cobra"
+)
+
+// runAccountListRemote reads the listing over POST /admin/ListAccounts, which
+// is how an off-cluster operator or the load-test harness sees the tenants it
+// created without SSH access to a node.
+func runAccountListRemote(cmd *cobra.Command) error {
+	target, err := resolveAdminTarget(cmd)
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), adminRequestTimeout)
+	defer cancel()
+
+	var out gateway.ListAccountsResponse
+	if err := callAdmin(ctx, target, "ListAccounts", struct{}{}, &out); err != nil {
+		return err
+	}
+
+	printAccountTable(out.Accounts)
+	return nil
+}
+
+// accountSummaries adapts the local listing to the shape the remote one
+// returns, so both paths print through one function and cannot drift.
+func accountSummaries(accounts []*handlers_iam.Account) []gateway.AccountSummary {
+	summaries := make([]gateway.AccountSummary, 0, len(accounts))
+	for _, account := range accounts {
+		if account == nil {
+			continue
+		}
+		summaries = append(summaries, gateway.AccountSummary{
+			AccountID:   account.AccountID,
+			AccountName: account.AccountName,
+			Status:      account.Status,
+			CreatedAt:   account.CreatedAt,
+		})
+	}
+	return summaries
+}
+
+func printAccountTable(accounts []gateway.AccountSummary) {
+	if len(accounts) == 0 {
+		fmt.Println("No accounts found.")
+		return
+	}
+
+	fmt.Printf("%-14s %-30s %-12s %s\n", "ACCOUNT ID", "NAME", "STATUS", "CREATED")
+	fmt.Printf("%-14s %-30s %-12s %s\n", "----------", "----", "------", "-------")
+	for _, account := range accounts {
+		created := account.CreatedAt
+		if parsed, err := time.Parse(time.RFC3339, account.CreatedAt); err == nil {
+			created = parsed.Format("2006-01-02 15:04")
+		}
+		fmt.Printf("%-14s %-30s %-12s %s\n", account.AccountID, account.AccountName, account.Status, created)
+	}
+}

cmd/spinifex/cmd/admin_account_quota.go

@@ -0,0 +1,185 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strconv"
+
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	handlers_quota "github.com/mulgadc/spinifex/spinifex/handlers/quota"
+	"github.com/spf13/cobra"
+)
+
+// quotaDimensions maps each CLI flag to the override field it sets. One table
+// drives the flag registration and the parse, so a new dimension cannot be
+// added to one and forgotten in the other.
+var quotaDimensions = []struct {
+	flag  string
+	usage string
+	field func(*handlers_quota.Overrides) **int
+}{
+	{"vcpus", "Total vCPUs across running and stopped instances", func(o *handlers_quota.Overrides) **int { return &o.VCPUs }},
+	{"vpcs", "VPCs", func(o *handlers_quota.Overrides) **int { return &o.VPCs }},
+	{"subnets", "Subnets, counted per account rather than per VPC", func(o *handlers_quota.Overrides) **int { return &o.Subnets }},
+	{"eips", "Elastic IP addresses", func(o *handlers_quota.Overrides) **int { return &o.EIPs }},
+	{"volumes", "EBS volumes", func(o *handlers_quota.Overrides) **int { return &o.Volumes }},
+	{"volumes-gib", "Total EBS capacity in GiB", func(o *handlers_quota.Overrides) **int { return &o.VolumesGiB }},
+	{"rds-instances", "RDS database instances", func(o *handlers_quota.Overrides) **int { return &o.RDSInstances }},
+	{"load-balancers", "Load balancers, ALB and NLB together", func(o *handlers_quota.Overrides) **int { return &o.LoadBalancers }},
+}
+
+var accountQuotaCmd = &cobra.Command{
+	Use:   "quota",
+	Short: "Inspect and adjust an account's service quotas",
+	Long: `Inspect and adjust the service quotas applied to one account.
+
+Every account inherits the [quota] limits from awsgw.toml. An override raises or
+lowers individual dimensions for one account without affecting any other, so a
+customer can be given more vCPUs while every other tenant stays on the baseline.
+
+Only the dimensions passed as flags are changed; the rest keep inheriting. Use
+-1 for no limit on a dimension, and --clear to drop every override.`,
+}
+
+var accountQuotaGetCmd = &cobra.Command{
+	Use:   "get <account-id>",
+	Short: "Show the limits in force for an account",
+	Long: `Show the limits in force for an account and where each came from:
+"config" for an inherited baseline, "override" for one set on this account.`,
+	Args: cobra.ExactArgs(1),
+	RunE: runAccountQuotaGet,
+}
+
+var accountQuotaSetCmd = &cobra.Command{
+	Use:   "set <account-id>",
+	Short: "Override one or more limits for an account",
+	Long: `Override one or more limits for an account. Dimensions not named keep
+inheriting the configured baseline.
+
+  spx admin account quota set 000000000005 --vcpus 32
+  spx admin account quota set 000000000005 --clear`,
+	Args: cobra.ExactArgs(1),
+	RunE: runAccountQuotaSet,
+}
+
+// Both subcommands are served only by the /admin surface, so unlike the account
+// commands there is no NATS path and no --remote to opt into.
+func init() {
+	accountCmd.AddCommand(accountQuotaCmd)
+	accountQuotaCmd.AddCommand(accountQuotaGetCmd)
+	accountQuotaCmd.AddCommand(accountQuotaSetCmd)
+
+	for _, c := range []*cobra.Command{accountQuotaGetCmd, accountQuotaSetCmd} {
+		c.Flags().String("endpoint", "", "Gateway endpoint (default: this node's AWS gateway)")
+		c.Flags().String("region", "", "SigV4 region (default: this node's region)")
+		c.Flags().String("ca-bundle", "", "CA certificate (default: this node's CA)")
+	}
+
+	for _, d := range quotaDimensions {
+		accountQuotaSetCmd.Flags().Int(d.flag, 0, d.usage+" (-1 for no limit)")
+	}
+	accountQuotaSetCmd.Flags().Bool("clear", false, "Remove every override, returning the account to the configured limits")
+}
+
+func runAccountQuotaGet(cmd *cobra.Command, args []string) error {
+	target, err := resolveAdminTarget(cmd)
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), adminRequestTimeout)
+	defer cancel()
+
+	var out gateway.AccountQuotaResponse
+	req := gateway.AccountQuotaRequest{AccountID: args[0]}
+	if err := callAdmin(ctx, target, "GetAccountQuota", req, &out); err != nil {
+		return err
+	}
+	printAccountQuota(&out)
+	return nil
+}
+
+func runAccountQuotaSet(cmd *cobra.Command, args []string) error {
+	target, err := resolveAdminTarget(cmd)
+	if err != nil {
+		return err
+	}
+
+	over, changed, err := quotaOverridesFromFlags(cmd)
+	if err != nil {
+		return err
+	}
+	clearAll, err := cmd.Flags().GetBool("clear")
+	if err != nil {
+		return err
+	}
+	// Refusing both is not pedantry: --clear silently winning would discard
+	// limits the caller believed it was setting in the same breath.
+	if clearAll && changed {
+		return fmt.Errorf("--clear cannot be combined with a dimension flag")
+	}
+	if !clearAll && !changed {
+		return fmt.Errorf("no dimension given; pass at least one limit flag, or --clear to remove every override")
+	}
+	if clearAll {
+		over = handlers_quota.Overrides{}
+	}
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), adminRequestTimeout)
+	defer cancel()
+
+	var out gateway.AccountQuotaResponse
+	req := gateway.AccountQuotaRequest{AccountID: args[0], Overrides: over}
+	if err := callAdmin(ctx, target, "PutAccountQuota", req, &out); err != nil {
+		return err
+	}
+	printAccountQuota(&out)
+	return nil
+}
+
+// quotaOverridesFromFlags builds the override set from the flags the caller
+// actually passed. An unset flag is left nil so the dimension keeps inheriting,
+// which is why Changed is consulted rather than the flag's value.
+func quotaOverridesFromFlags(cmd *cobra.Command) (handlers_quota.Overrides, bool, error) {
+	var over handlers_quota.Overrides
+	changed := false
+	for _, d := range quotaDimensions {
+		if !cmd.Flags().Changed(d.flag) {
+			continue
+		}
+		value, err := cmd.Flags().GetInt(d.flag)
+		if err != nil {
+			return over, false, err
+		}
+		if value < handlers_quota.Unlimited {
+			return over, false, fmt.Errorf("--%s must be 0 or greater, or %d for no limit", d.flag, handlers_quota.Unlimited)
+		}
+		*d.field(&over) = &value
+		changed = true
+	}
+	return over, changed, nil
+}
+
+func printAccountQuota(resp *gateway.AccountQuotaResponse) {
+	fmt.Printf("Account %s\n\n", resp.AccountID)
+	fmt.Printf("%-16s %10s  %s\n", "DIMENSION", "LIMIT", "SOURCE")
+	fmt.Printf("%-16s %10s  %s\n", "---------", "-----", "------")
+
+	names := make([]string, 0, len(resp.Limits))
+	for name := range resp.Limits {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	for _, name := range names {
+		limit := strconv.Itoa(resp.Limits[name])
+		if resp.Limits[name] == handlers_quota.Unlimited {
+			limit = "unlimited"
+		}
+		fmt.Printf("%-16s %10s  %s\n", name, limit, resp.Source[name])
+	}
+
+	if resp.Overrides.UpdatedAt != "" {
+		fmt.Printf("\nOverride set by %s at %s\n", resp.Overrides.UpdatedBy, resp.Overrides.UpdatedAt)
+	}
+}

cmd/spinifex/cmd/admin_account_quota_test.go

@@ -0,0 +1,140 @@
+//test:in-package — drives the unexported flag table, parser and printer the quota subcommands are built from.
+package cmd
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	handlers_quota "github.com/mulgadc/spinifex/spinifex/handlers/quota"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/require"
+)
+
+// newQuotaSetCmd builds a command carrying the same flags as the real one, so
+// the parse is tested without mutating the package-level command between tests.
+func newQuotaSetCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "set"}
+	for _, d := range quotaDimensions {
+		cmd.Flags().Int(d.flag, 0, d.usage)
+	}
+	cmd.Flags().Bool("clear", false, "")
+	return cmd
+}
+
+// Only the flags actually passed become overrides. Every other dimension must
+// stay nil so it keeps inheriting, which is the whole point of a sparse record.
+func TestQuotaOverridesFromFlagsIsSparse(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--vcpus", "32", "--rds-instances", "8"}))
+
+	over, changed, err := quotaOverridesFromFlags(cmd)
+	require.NoError(t, err)
+	require.True(t, changed)
+
+	require.NotNil(t, over.VCPUs)
+	require.Equal(t, 32, *over.VCPUs)
+	require.NotNil(t, over.RDSInstances)
+	require.Equal(t, 8, *over.RDSInstances)
+
+	require.Nil(t, over.VPCs)
+	require.Nil(t, over.EIPs)
+	require.Nil(t, over.Volumes)
+	require.Nil(t, over.VolumesGiB)
+	require.Nil(t, over.Subnets)
+	require.Nil(t, over.LoadBalancers)
+}
+
+// A flag left unset must not be read as an explicit 0, which would deny every
+// request on that dimension instead of inheriting it.
+func TestQuotaOverridesFromFlagsUnsetIsNotZero(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--vpcs", "8"}))
+
+	over, changed, err := quotaOverridesFromFlags(cmd)
+	require.NoError(t, err)
+	require.True(t, changed)
+	require.Nil(t, over.VCPUs, "an unset flag must not become an explicit zero")
+}
+
+// Passing 0 deliberately is a real limit and must be preserved.
+func TestQuotaOverridesFromFlagsExplicitZero(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--eips", "0"}))
+
+	over, changed, err := quotaOverridesFromFlags(cmd)
+	require.NoError(t, err)
+	require.True(t, changed)
+	require.NotNil(t, over.EIPs)
+	require.Equal(t, 0, *over.EIPs)
+}
+
+func TestQuotaOverridesFromFlagsUnlimited(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--vcpus", "-1"}))
+
+	over, _, err := quotaOverridesFromFlags(cmd)
+	require.NoError(t, err)
+	require.NotNil(t, over.VCPUs)
+	require.Equal(t, handlers_quota.Unlimited, *over.VCPUs)
+}
+
+// Anything below the Unlimited sentinel is a typo, not a smaller limit.
+func TestQuotaOverridesFromFlagsRejectsBelowUnlimited(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--vcpus", "-2"}))
+
+	_, _, err := quotaOverridesFromFlags(cmd)
+	require.Error(t, err)
+}
+
+// No flags at all means nothing changed, which the caller turns into an error
+// rather than a write that silently does nothing.
+func TestQuotaOverridesFromFlagsNoneChanged(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags(nil))
+
+	over, changed, err := quotaOverridesFromFlags(cmd)
+	require.NoError(t, err)
+	require.False(t, changed)
+	require.True(t, over.Empty())
+}
+
+// The table must name every dimension, label its source, and render the
+// Unlimited sentinel as a word rather than as -1, which reads like a bug.
+func TestPrintAccountQuota(t *testing.T) {
+	resp := &gateway.AccountQuotaResponse{
+		AccountID: "000000000042",
+		Limits:    map[string]int{"vcpus": 32, "vpcs": handlers_quota.Unlimited, "eips": 4},
+		Source:    map[string]string{"vcpus": "override", "vpcs": "override", "eips": "config"},
+		Overrides: handlers_quota.Overrides{UpdatedBy: "operator", UpdatedAt: "2026-08-20T01:00:00Z"},
+	}
+
+	out := captureStdout(t, func() { printAccountQuota(resp) })
+
+	require.Contains(t, out, "000000000042")
+	require.Contains(t, out, "vcpus")
+	require.Contains(t, out, "32")
+	require.Contains(t, out, "override")
+	require.Contains(t, out, "config")
+	require.Contains(t, out, "unlimited")
+	require.NotContains(t, out, "-1")
+	require.Contains(t, out, "operator")
+}
+
+// Every dimension the override record carries must be reachable from the CLI,
+// or a limit exists that an operator cannot set.
+func TestQuotaDimensionsCoverEveryOverrideField(t *testing.T) {
+	var over handlers_quota.Overrides
+	for _, d := range quotaDimensions {
+		v := 1
+		*d.field(&over) = &v
+	}
+	require.False(t, over.Empty())
+
+	seen := map[string]bool{}
+	for _, d := range quotaDimensions {
+		require.False(t, seen[d.flag], "duplicate flag %q", d.flag)
+		seen[d.flag] = true
+	}
+	require.Len(t, quotaDimensions, 8)
+}

cmd/spinifex/cmd/admin_acm.go

@@ -60,16 +60,12 @@ func init() {
 	certCreateTenantCACmd.Flags().StringSlice("domain", nil, "Domain the tenant CA is permitted to sign for (repeatable)")
 	certCreateTenantCACmd.Flags().Bool("regenerate", false, "Replace an existing tenant CA with a fresh root (invalidates every device's trust store — requires confirmation)")
 	certCreateTenantCACmd.Flags().Bool("yes", false, "Skip the --regenerate confirmation prompt")
-	if err := certCreateTenantCACmd.MarkFlagRequired("domain"); err != nil {
-		panic(err)
-	}
+	_ = certCreateTenantCACmd.MarkFlagRequired("domain")
 
 	certCmd.AddCommand(certForceRenewCmd)
 	certForceRenewCmd.Flags().String("arn", "", "Certificate ARN to force-renew (required)")
 	certForceRenewCmd.Flags().String("account", "", "Account ID that owns the certificate; defaults to the bootstrap account")
-	if err := certForceRenewCmd.MarkFlagRequired("arn"); err != nil {
-		panic(err)
-	}
+	_ = certForceRenewCmd.MarkFlagRequired("arn")
 }
 
 func runCertForceRenew(cmd *cobra.Command, _ []string) {

cmd/spinifex/cmd/admin_cabake_test.go

@@ -1,67 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"errors"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestCABakeCmd_Args(t *testing.T) {
-	cmd := caBakeCmd(context.Background(), "/img/disk.raw", "/data/config/ca.pem")
-	args := cmd.Args
-
-	assert.Equal(t, "virt-customize", filepath.Base(args[0]))
-	assert.Contains(t, args, "-a")
-	assert.Contains(t, args, "/img/disk.raw")
-	assert.Contains(t, args, "--upload")
-	assert.Contains(t, args, "/data/config/ca.pem:/tmp/spinifex-ca.pem")
-	assert.Contains(t, args, "--run-command")
-
-	// --no-selinux-relabel stops virt-customize forcing a first-boot autorelabel,
-	// whose relabel+reboot corrupts XFS roots on RHEL/Rocky.
-	assert.Contains(t, args, "--no-selinux-relabel")
-
-	// The run-command covers Debian/Ubuntu (update-ca-certificates), RHEL/Rocky
-	// (update-ca-trust), and the stock-Alpine fallback that appends to the static
-	// ca-certificates bundle when no updater is present. The RHEL branch relabels
-	// only the touched trust paths so the cert is labelled without a full relabel.
-	joined := strings.Join(args, " ")
-	assert.Contains(t, joined, "update-ca-certificates")
-	assert.Contains(t, joined, "update-ca-trust")
-	assert.Contains(t, joined, "restorecon")
-	assert.Contains(t, joined, "/etc/ssl/certs/ca-certificates.crt")
-}
-
-func TestBakeCACertIntoImage_SkipsWhenCAMissing(t *testing.T) {
-	called := false
-	orig := caBakeRunner
-	caBakeRunner = func(string, string) ([]byte, error) { called = true; return nil, nil }
-	defer func() { caBakeRunner = orig }()
-
-	// A missing CA must not invoke virt-customize and must not panic.
-	bakeCACertIntoImage("/img/disk.raw", filepath.Join(t.TempDir(), "absent-ca.pem"))
-	assert.False(t, called, "runner must not be called when the CA is absent")
-}
-
-func TestBakeCACertIntoImage_ContinuesOnRunnerError(t *testing.T) {
-	caPath := filepath.Join(t.TempDir(), "ca.pem")
-	require.NoError(t, os.WriteFile(caPath, []byte("ca"), 0600))
-
-	called := false
-	orig := caBakeRunner
-	caBakeRunner = func(string, string) ([]byte, error) {
-		called = true
-		return []byte("libguestfs: cannot inspect image"), errors.New("exit status 1")
-	}
-	defer func() { caBakeRunner = orig }()
-
-	// An image libguestfs cannot inspect must skip-and-continue, never panic/fail.
-	bakeCACertIntoImage("/img/disk.raw", caPath)
-	assert.True(t, called, "runner must be invoked when the CA is present")
-}

cmd/spinifex/cmd/admin_firewall.go

@@ -0,0 +1,54 @@
+package cmd
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+	"os/exec"
+	"strconv"
+)
+
+// firewallApplyHelper is the fixed-verb root helper setup.sh installs. Absent on
+// a developer machine and on any node installed with --firewall=off, where there
+// is no policy to punch a hole in.
+var firewallApplyHelper = "/usr/local/lib/spinifex/spinifex-firewall-apply"
+
+// openFormationPort opens the formation port to any source and returns the
+// function that closes it again.
+//
+// The port is peer-scoped in the steady-state policy, where it carries the
+// daemon cluster manager. A node dialling in to join is not a peer yet, so
+// without this the join handshake is dropped rather than refused and the joiner
+// retries in silence until it times out. The window is bounded by formation
+// itself, and the handshake behind it is TLS 1.3 with a bearer token.
+//
+// Best effort: a node with no policy installed needs nothing done, and a
+// failure to open must not stop an operator forming a cluster.
+func openFormationPort(port int) func() {
+	if _, err := os.Stat(firewallApplyHelper); err != nil {
+		return func() {}
+	}
+
+	if err := runFirewallHelper("open-port", port); err != nil {
+		fmt.Printf("⚠️  Could not open port %d in the host firewall: %v\n", port, err)
+		fmt.Printf("   Joining nodes may be unable to reach the formation server.\n")
+		return func() {}
+	}
+
+	return func() {
+		if err := runFirewallHelper("close-port", port); err != nil {
+			// Loud, not fatal: formation has already succeeded by here, and the
+			// port stays open until the next reconcile or setup.sh run.
+			slog.Error("Failed to close the formation port in the host firewall",
+				"port", port, "helper", firewallApplyHelper, "err", err)
+		}
+	}
+}
+
+func runFirewallHelper(verb string, port int) error {
+	cmd := exec.Command(firewallApplyHelper, verb, strconv.Itoa(port))
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("%s: %w: %s", verb, err, out)
+	}
+	return nil
+}

cmd/spinifex/cmd/admin_firewall_test.go

@@ -0,0 +1,70 @@
+package cmd_test
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// stubFirewallHelper points the firewall helper at a script that logs its
+// arguments, and returns the path of that log.
+func stubFirewallHelper(t *testing.T, body string) string {
+	t.Helper()
+	dir := t.TempDir()
+	log := filepath.Join(dir, "calls")
+	helper := filepath.Join(dir, "spinifex-firewall-apply")
+	script := "#!/bin/sh\necho \"$*\" >> " + log + "\n" + body
+	require.NoError(t, os.WriteFile(helper, []byte(script), 0o755))
+	t.Cleanup(cmd.SetFirewallApplyHelper(helper))
+	return log
+}
+
+func helperCalls(t *testing.T, log string) []string {
+	t.Helper()
+	data, err := os.ReadFile(log)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	require.NoError(t, err)
+	return strings.Fields(strings.TrimSpace(strings.ReplaceAll(string(data), "\n", " ")))
+}
+
+func TestOpenFormationPort(t *testing.T) {
+	log := stubFirewallHelper(t, "exit 0\n")
+
+	closePort := cmd.OpenFormationPort(4432)
+	assert.Equal(t, []string{"open-port", "4432"}, helperCalls(t, log))
+
+	closePort()
+	assert.Equal(t, []string{"open-port", "4432", "close-port", "4432"}, helperCalls(t, log),
+		"the window must be closed again once formation ends")
+}
+
+// A machine with no policy installed has nothing to open, and formation must not
+// be held up by looking for a helper that was never installed.
+func TestOpenFormationPort_NoHelperInstalled(t *testing.T) {
+	t.Cleanup(cmd.SetFirewallApplyHelper(filepath.Join(t.TempDir(), "absent")))
+
+	closePort := cmd.OpenFormationPort(4432)
+	require.NotNil(t, closePort)
+	closePort()
+}
+
+// Opening is best effort: an operator forming a cluster must not be blocked by a
+// firewall helper that fails. The close must then not run either, since there is
+// nothing open to close.
+func TestOpenFormationPort_FailureDoesNotBlockFormation(t *testing.T) {
+	log := stubFirewallHelper(t, "exit 1\n")
+
+	closePort := cmd.OpenFormationPort(4432)
+	require.NotNil(t, closePort)
+	closePort()
+
+	assert.Equal(t, []string{"open-port", "4432"}, helperCalls(t, log),
+		"a failed open must not be followed by a close")
+}

cmd/spinifex/cmd/admin_jsprobe.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"time"
@@ -17,6 +18,12 @@ import (
 // other consumer reads.
 const canaryKey = "_watchdog.canary"
 
+// errProbeInconclusive marks a failure that happened before the probe ever
+// reached JetStream: config load, connect, or bucket open. The watchdog must
+// treat this as "learned nothing", not as JetStream refusing a write — during
+// recovery the bucket may simply not exist yet.
+var errProbeInconclusive = errors.New("probe did not reach JetStream")
+
 var nodeJSProbeCmd = &cobra.Command{
 	Use:   "js-probe",
 	Short: "Verify JetStream still accepts writes on this node",
@@ -42,6 +49,9 @@ func runNodeJSProbe(cmd *cobra.Command, _ []string) {
 
 	if err := probeJetStreamWrite(ctx); err != nil {
 		fmt.Fprintf(os.Stderr, "js-probe: %v\n", err)
+		if errors.Is(err, errProbeInconclusive) {
+			os.Exit(2)
+		}
 		os.Exit(1)
 	}
 	fmt.Println("js-probe: ok")
@@ -52,21 +62,26 @@ func runNodeJSProbe(cmd *cobra.Command, _ []string) {
 // than a bucket of its own: a dedicated bucket is a separate stream that can be
 // placed on a different node, so it could stay writable while the bucket the
 // control plane actually depends on is wedged.
+//
+// Every stage before canaryRoundTrip is wrapped in errProbeInconclusive: the
+// probe has observed nothing about JetStream's write path yet, so a caller
+// must not read config, connect, or bucket-open failures as a JetStream
+// failure.
 func probeJetStreamWrite(ctx context.Context) error {
 	_, nc, err := loadConfigAndConnect()
 	if err != nil {
-		return fmt.Errorf("connect: %w", err)
+		return fmt.Errorf("%w: connect: %w", errProbeInconclusive, err)
 	}
 	defer nc.Close()
 
 	js, err := jetstream.New(nc)
 	if err != nil {
-		return fmt.Errorf("jetstream: %w", err)
+		return fmt.Errorf("%w: jetstream: %w", errProbeInconclusive, err)
 	}
 
 	kv, err := js.KeyValue(ctx, daemon.ClusterStateBucket)
 	if err != nil {
-		return fmt.Errorf("open bucket %s: %w", daemon.ClusterStateBucket, err)
+		return fmt.Errorf("%w: open bucket %s: %w", errProbeInconclusive, daemon.ClusterStateBucket, err)
 	}
 	return canaryRoundTrip(ctx, kv)
 }

cmd/spinifex/cmd/admin_jsprobe_test.go

@@ -2,12 +2,17 @@ package cmd
 
 import (
 	"context"
+	"fmt"
+	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
 	"github.com/nats-io/nats.go/jetstream"
+	"github.com/spf13/viper"
 	"github.com/stretchr/testify/require"
 
+	"github.com/mulgadc/spinifex/spinifex/daemon"
 	"github.com/mulgadc/spinifex/spinifex/testutil"
 )
 
@@ -73,3 +78,82 @@ func TestCanaryRoundTrip_FailsWhenBucketIsGone(t *testing.T) {
 func TestCanaryKeyIsNamespaced(t *testing.T) {
 	require.True(t, len(canaryKey) > 0 && canaryKey[0] == '_')
 }
+
+// nodeTomlWithNATSHost writes a minimal single-node spinifex.toml pointing at
+// the given NATS host, the shape probeJetStreamWrite's loadConfigAndConnect
+// call needs to resolve node1's NATS endpoint.
+func nodeTomlWithNATSHost(t *testing.T, natsHost string) string {
+	t.Helper()
+	return writeSpinifexToml(t, fmt.Sprintf(`
+version = "1.0"
+epoch = 1
+node = "node1"
+
+[nodes.node1]
+node = "node1"
+region = "us-east-1"
+az = "us-east-1a"
+
+[nodes.node1.nats]
+host = %q
+`, natsHost))
+}
+
+// A config file viper cannot parse must never read as a JetStream failure:
+// the probe has not connected to anything yet.
+func TestProbeJetStreamWrite_ConfigLoadFailureIsInconclusive(t *testing.T) {
+	resetGlobalViper(t)
+
+	dir := t.TempDir()
+	path := filepath.Join(dir, "spinifex.toml")
+	require.NoError(t, os.WriteFile(path, []byte("not [[[ valid toml"), 0o600))
+	viper.Set("config", path)
+
+	err := probeJetStreamWrite(context.Background())
+	require.Error(t, err)
+	require.ErrorIs(t, err, errProbeInconclusive, "config-load failure must be inconclusive")
+}
+
+// A NATS endpoint that refuses the connection must also read as inconclusive,
+// not as JetStream refusing a write — this is the class of failure that
+// turned a broken spx binary into a cluster-wide NATS restart.
+func TestProbeJetStreamWrite_ConnectFailureIsInconclusive(t *testing.T) {
+	resetGlobalViper(t)
+
+	viper.Set("config", nodeTomlWithNATSHost(t, "nats://127.0.0.1:1"))
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	err := probeJetStreamWrite(ctx)
+	require.Error(t, err)
+	require.ErrorIs(t, err, errProbeInconclusive, "connect failure must be inconclusive")
+}
+
+// A reachable NATS with no cluster-state bucket yet (recovery, or a bucket
+// that has not been created) must not read as a JetStream write failure.
+func TestProbeJetStreamWrite_BucketOpenFailureIsInconclusive(t *testing.T) {
+	resetGlobalViper(t)
+
+	ns, _, _ := testutil.StartTestJetStream(t)
+	viper.Set("config", nodeTomlWithNATSHost(t, ns.ClientURL()))
+
+	err := probeJetStreamWrite(context.Background())
+	require.Error(t, err)
+	require.ErrorIs(t, err, errProbeInconclusive, "missing bucket must be inconclusive")
+}
+
+// The positive end-to-end path: once the bucket exists and NATS accepts
+// writes, the probe succeeds through every stage rather than just at the
+// canaryRoundTrip unit level covered above.
+func TestProbeJetStreamWrite_SucceedsAgainstRealClusterStateBucket(t *testing.T) {
+	resetGlobalViper(t)
+
+	ns, _, js := testutil.StartTestJetStream(t)
+	_, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: daemon.ClusterStateBucket})
+	require.NoError(t, err)
+
+	viper.Set("config", nodeTomlWithNATSHost(t, ns.ClientURL()))
+
+	require.NoError(t, probeJetStreamWrite(context.Background()))
+}

cmd/spinifex/cmd/admin_ochre.go

@@ -1,10 +1,11 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
-	"log/slog"
 	"os"
 	"os/exec"
 	"path"
@@ -15,16 +16,16 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/mulgadc/predastore/pkg/masterkey"
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/ebsprovider"
 	gateway_bedrock "github.com/mulgadc/spinifex/spinifex/gateway/bedrock"
+	"github.com/mulgadc/spinifex/spinifex/gateway/bedrock/hfhub"
 	handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/utils"
-	"github.com/mulgadc/viperblock/viperblock"
-	vbs3 "github.com/mulgadc/viperblock/viperblock/backends/s3"
-	"github.com/mulgadc/viperblock/viperblock/v_utils"
+	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/pterm/pterm"
 	"github.com/spf13/cobra"
@@ -77,12 +78,51 @@ explicit act.`,
 	Run: runOchreWeightsRemove,
 }
 
+var ochreWeightsPullCmd = &cobra.Command{
+	Use:   "pull",
+	Short: "Fetch a self-host model's weights from Hugging Face into predastore",
+	Long: `pull resolves --revision to an immutable commit SHA, lists the repo's file
+tree, downloads only *.safetensors shards plus the fixed config/tokenizer
+set, and streams each into predastore under --s3-uri (or a keyed default
+when omitted). It never touches the offline 'stage' path or the weights KV
+store itself; it prints the resulting s3:// URI for 'weights stage --s3-uri'.
+
+Refuses before any object lands if --model-id is not a self-host catalog
+entry, the repo has no safetensors, or the hub returns 401/403 (a gated repo
+with no usable token). A failure partway through removes every object
+already written so 'stage' never sees a half-model.`,
+	Run: runOchreWeightsPull,
+}
+
+// ochreCredentialsCmd groups vendor credential management for pull's
+// optional stored-token fallback (D2); it holds no Run of its own.
+var ochreCredentialsCmd = &cobra.Command{
+	Use:   "credentials",
+	Short: "Manage stored provider credentials for self-host model pulls",
+}
+
+var ochreCredentialsSetCmd = &cobra.Command{
+	Use:   "set",
+	Short: "Store a vendor API token/licence credential",
+	Long: `set stores --vendor's token, encrypted at rest under the cluster's IAM master
+key, in the bedrock-credentials KV bucket. --account defaults to the
+platform account, since self-host models are platform-level, not per-tenant.
+
+The token itself is never a command-line argument: --token must be '-', and
+the secret is read from stdin, so it never appears in shell history or a
+process listing.`,
+	Run: runOchreCredentialsSet,
+}
+
 func init() {
 	adminCmd.AddCommand(ochreCmd)
 	ochreCmd.AddCommand(ochreWeightsCmd)
+	ochreCmd.AddCommand(ochreCredentialsCmd)
 	ochreWeightsCmd.AddCommand(ochreWeightsStageCmd)
 	ochreWeightsCmd.AddCommand(ochreWeightsListCmd)
 	ochreWeightsCmd.AddCommand(ochreWeightsRemoveCmd)
+	ochreWeightsCmd.AddCommand(ochreWeightsPullCmd)
+	ochreCredentialsCmd.AddCommand(ochreCredentialsSetCmd)
 
 	ochreWeightsStageCmd.Flags().String("model-id", "", "Catalog model ID to stage weights for (required)")
 	ochreWeightsStageCmd.Flags().String("s3-uri", "", "predastore S3 URI holding the model's Hugging Face files, e.g. s3://bucket/prefix/ (required)")
@@ -92,6 +132,19 @@ func init() {
 
 	ochreWeightsRemoveCmd.Flags().String("model-id", "", "Model ID to remove from staged weights (required)")
 	_ = ochreWeightsRemoveCmd.MarkFlagRequired("model-id")
+
+	ochreWeightsPullCmd.Flags().String("model-id", "", "Self-host catalog model ID this pull is for (required)")
+	ochreWeightsPullCmd.Flags().String("hf-repo", "", "Hugging Face repo, e.g. meta-llama/Llama-3.2-1B-Instruct (default: the catalog entry's repo)")
+	ochreWeightsPullCmd.Flags().String("revision", "main", "Hugging Face branch, tag or commit SHA to resolve and pin (default: the catalog entry's pinned revision)")
+	ochreWeightsPullCmd.Flags().String("s3-uri", "", "predastore destination prefix, e.g. s3://bucket/prefix/ (default: s3://ochre-weights/<repo>/<sha>/)")
+	ochreWeightsPullCmd.Flags().String("hf-token", "", "Hugging Face token for a gated repo (falls back to HF_TOKEN, then a stored platform credential)")
+	_ = ochreWeightsPullCmd.MarkFlagRequired("model-id")
+
+	ochreCredentialsSetCmd.Flags().String("vendor", "", "Vendor to store a credential for, e.g. huggingface (required)")
+	ochreCredentialsSetCmd.Flags().String("account", "", "Account ID to store the credential under (default: platform account)")
+	ochreCredentialsSetCmd.Flags().String("token", "", "Must be '-': reads the secret from stdin (required)")
+	_ = ochreCredentialsSetCmd.MarkFlagRequired("vendor")
+	_ = ochreCredentialsSetCmd.MarkFlagRequired("token")
 }
 
 // requiredWeightsFiles are the fixed-name Hugging Face artefacts stage
@@ -109,6 +162,264 @@ var requiredWeightsFiles = []string{
 // tokenizer.model under original/, not at the prefix root. Either is enough.
 var tokenizerFileNames = []string{"tokenizer.json", "tokenizer.model"}
 
+// vendorHuggingFace is the CredentialStore vendor key 'ochre credentials
+// set' and pull's stored-credential fallback share for the Hugging Face
+// licence token (D2).
+const vendorHuggingFace = "huggingface"
+
+// ochrePullManifestFile is the fixed name 'weights pull' writes into the
+// destination prefix and 'weights stage' reads back (D3), recording which
+// upstream commit a staged model's weights came from.
+const ochrePullManifestFile = "ochre-pull.json"
+
+// ochrePullManifest is the JSON body of ochrePullManifestFile.
+type ochrePullManifest struct {
+	HFRepo      string    `json:"hf_repo"`
+	RevisionSHA string    `json:"revision_sha"`
+	PulledAt    time.Time `json:"pulled_at"`
+}
+
+// rootOnlyPullFileNames are the fixed-name config artefacts pull fetches only
+// at the repo root. A same-named file in a subdir is a component config, not
+// the model's -- e.g. sentence-transformers' 1_Pooling/config.json -- and must
+// never flatten onto the root config.json, so nested matches are dropped.
+var rootOnlyPullFileNames = map[string]bool{
+	"config.json":                  true,
+	"tokenizer_config.json":        true,
+	"generation_config.json":       true,
+	"special_tokens_map.json":      true,
+	"model.safetensors.index.json": true,
+}
+
+// nestablePullFileNames are the tokenizer artefacts matched by basename at any
+// depth: some repos keep them in a subdir (e.g. Llama's original/tokenizer.model).
+var nestablePullFileNames = map[string]bool{
+	"tokenizer.json":  true,
+	"tokenizer.model": true,
+}
+
+// selectPullFiles filters a Hugging Face tree listing down to the
+// safetensors-only set pull downloads (D4): every *.safetensors file plus
+// the fixed config/tokenizer set. Anything else -- notably .bin/.pt pickle
+// checkpoints, which can execute code on load -- is dropped silently rather
+// than aborting the whole pull.
+func selectPullFiles(entries []hfhub.TreeEntry) []hfhub.TreeEntry {
+	var selected []hfhub.TreeEntry
+	for _, e := range entries {
+		if e.Type != "file" {
+			continue
+		}
+		base := path.Base(e.Path)
+		nested := strings.Contains(e.Path, "/")
+		switch {
+		case strings.HasSuffix(e.Path, ".safetensors"):
+			selected = append(selected, e)
+		case nestablePullFileNames[base]:
+			selected = append(selected, e)
+		case rootOnlyPullFileNames[base] && !nested:
+			selected = append(selected, e)
+		}
+	}
+	return selected
+}
+
+// anySafetensors reports whether entries contains at least one *.safetensors
+// file. selectPullFiles's config/tokenizer set alone is never enough --
+// D4 aborts a pull with no actual weights to serve.
+func anySafetensors(entries []hfhub.TreeEntry) bool {
+	for _, e := range entries {
+		if strings.HasSuffix(e.Path, ".safetensors") {
+			return true
+		}
+	}
+	return false
+}
+
+// defaultPullPrefix derives D7's fallback destination when --s3-uri is
+// omitted: keyed by the exact repo and resolved commit SHA, so re-pulling
+// the same commit lands at the same, self-deduping URI.
+func defaultPullPrefix(hfRepo, sha string) string {
+	return fmt.Sprintf("s3://ochre-weights/%s/%s/", strings.Trim(hfRepo, "/"), sha)
+}
+
+// putPullManifest writes ochrePullManifestFile into bucket/prefix, the last
+// step of a successful pull (D3).
+func putPullManifest(ctx context.Context, store objectstore.ObjectStore, bucket, prefix string, manifest ochrePullManifest) error {
+	data, err := json.Marshal(manifest)
+	if err != nil {
+		return fmt.Errorf("encode pull manifest: %w", err)
+	}
+	key := prefix + ochrePullManifestFile
+	if _, err := store.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String(key), Body: bytes.NewReader(data)}); err != nil {
+		return fmt.Errorf("put pull manifest s3://%s/%s: %w", bucket, key, err)
+	}
+	return nil
+}
+
+// readPullManifest reads and decodes bucket/prefix's ochre-pull.json, if
+// present. A missing manifest is not an error: stage's offline path (an
+// operator-supplied prefix with no pull manifest) must keep working
+// unchanged, so ok is simply false.
+func readPullManifest(ctx context.Context, store objectstore.ObjectStore, bucket, prefix string) (ochrePullManifest, bool, error) {
+	key := prefix + ochrePullManifestFile
+	out, err := store.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
+	if err != nil {
+		if objectstore.IsNoSuchKeyError(err) {
+			return ochrePullManifest{}, false, nil
+		}
+		return ochrePullManifest{}, false, fmt.Errorf("get pull manifest s3://%s/%s: %w", bucket, key, err)
+	}
+	defer out.Body.Close()
+
+	var manifest ochrePullManifest
+	if err := json.NewDecoder(out.Body).Decode(&manifest); err != nil {
+		return ochrePullManifest{}, false, fmt.Errorf("decode pull manifest s3://%s/%s: %w", bucket, key, err)
+	}
+	return manifest, true, nil
+}
+
+// cleanupPulledObjects best-effort deletes every key already written by a
+// pull that failed partway through (D5): a half-uploaded prefix must never
+// be left for 'stage' to mistake for a complete model. A delete failure is
+// logged, not returned -- the original pull error is what the operator needs.
+func cleanupPulledObjects(ctx context.Context, store objectstore.ObjectStore, bucket string, keys []string) {
+	for _, key := range keys {
+		if _, err := store.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)}); err != nil {
+			fmt.Fprintf(os.Stderr, "warning: could not clean up s3://%s/%s after a failed pull: %v\n", bucket, key, err)
+		}
+	}
+}
+
+// pullOneFile downloads hfPath from the hub to a local temp file and PUTs it
+// to bucket/key. A temp file is required, not a direct HTTP-to-S3 pipe:
+// predastore's PutObject signs the request over an io.ReadSeeker, which an
+// HTTP response body is not. tmpDir is cleaned up by the caller.
+func pullOneFile(ctx context.Context, hf *hfhub.Client, store objectstore.ObjectStore, hfRepo, sha, hfPath, bucket, key, tmpDir string) error {
+	body, _, err := hf.DownloadFile(ctx, hfRepo, sha, hfPath)
+	if err != nil {
+		return err
+	}
+	defer body.Close()
+
+	tmpFile, err := os.CreateTemp(tmpDir, "part-*")
+	if err != nil {
+		return fmt.Errorf("create temp file: %w", err)
+	}
+	defer os.Remove(tmpFile.Name())
+	defer tmpFile.Close()
+
+	if _, err := io.Copy(tmpFile, body); err != nil {
+		return fmt.Errorf("download to temp file: %w", err)
+	}
+	if _, err := tmpFile.Seek(0, io.SeekStart); err != nil {
+		return fmt.Errorf("seek temp file: %w", err)
+	}
+
+	if _, err := store.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(bucket), Key: aws.String(key), Body: tmpFile}); err != nil {
+		return fmt.Errorf("put s3://%s/%s: %w", bucket, key, err)
+	}
+	return nil
+}
+
+// pullFilesToPrefix downloads and uploads every selected file, one at a time
+// so at most one file is ever held on local disk (D6). Keys are flattened to
+// each file's basename to match the flat layout 'stage' validates -- e.g.
+// Llama's original/tokenizer.model lands at <prefix>tokenizer.model. It
+// returns the keys written so far even on error, so the caller can clean up
+// a partial prefix (D5).
+func pullFilesToPrefix(ctx context.Context, hf *hfhub.Client, store objectstore.ObjectStore, hfRepo, sha, bucket, prefix string, files []hfhub.TreeEntry) ([]string, error) {
+	tmpDir, err := os.MkdirTemp("", "spinifex-ochre-pull-*")
+	if err != nil {
+		return nil, fmt.Errorf("create temp dir: %w", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	var uploaded []string
+	seen := make(map[string]string, len(files))
+	for _, f := range files {
+		key := prefix + path.Base(f.Path)
+		if prev, dup := seen[key]; dup {
+			return uploaded, fmt.Errorf("refusing to overwrite %s: both %s and %s flatten to it", key, prev, f.Path)
+		}
+		seen[key] = f.Path
+		if err := pullOneFile(ctx, hf, store, hfRepo, sha, f.Path, bucket, key, tmpDir); err != nil {
+			return uploaded, fmt.Errorf("pull %s: %w", f.Path, err)
+		}
+		uploaded = append(uploaded, key)
+	}
+	return uploaded, nil
+}
+
+// runPullWeights holds 'ochre weights pull' decision and side-effect logic:
+// catalog validation, ref resolution to an immutable commit SHA (D3), tree
+// listing and safetensors-only filtering (D4), streaming each selected file
+// into predastore (D6), and finally the ochre-pull.json manifest. It never
+// touches the weights KV store; 'stage' consumes the printed s3:// URI
+// separately. A failure removes every object already written (D5).
+func runPullWeights(ctx context.Context, hf *hfhub.Client, store objectstore.ObjectStore, modelID, hfRepo, revision string, revisionExplicit bool, s3URIFlag string) (string, error) {
+	spec, found, selfHost := gateway_bedrock.LookupServingSpec(modelID)
+	if !found || !selfHost {
+		if !found {
+			return "", fmt.Errorf("unknown model ID %q: not present in the Ochre catalog", modelID)
+		}
+		return "", fmt.Errorf("%q is a provider-served model, not self-host; weights pull does not apply", modelID)
+	}
+
+	// The catalog carries the canonical repo and a pinned revision so a bare
+	// pull cannot silently grab a regressed upstream default; an explicit flag
+	// still overrides either.
+	if hfRepo == "" {
+		hfRepo = spec.HFRepo
+	}
+	if hfRepo == "" {
+		return "", fmt.Errorf("no --hf-repo given and %q has no catalog HF repo", modelID)
+	}
+	if !revisionExplicit && spec.HFRevision != "" {
+		revision = spec.HFRevision
+	}
+
+	fmt.Printf("Resolving %s@%s ...\n", hfRepo, revision)
+	sha, err := hf.ResolveRevision(ctx, hfRepo, revision)
+	if err != nil {
+		return "", err
+	}
+
+	fmt.Printf("Listing files at %s@%s ...\n", hfRepo, sha)
+	tree, err := hf.ListTree(ctx, hfRepo, sha)
+	if err != nil {
+		return "", err
+	}
+
+	selected := selectPullFiles(tree)
+	if !anySafetensors(selected) {
+		return "", fmt.Errorf("%s@%s has no *.safetensors files; refusing to pull pickle-format (.bin) weights", hfRepo, sha)
+	}
+
+	s3URI := s3URIFlag
+	if s3URI == "" {
+		s3URI = defaultPullPrefix(hfRepo, sha)
+	}
+	bucket, prefix, err := parseWeightsS3URI(s3URI)
+	if err != nil {
+		return "", err
+	}
+
+	fmt.Printf("Pulling %d file(s) into s3://%s/%s ...\n", len(selected), bucket, prefix)
+	uploaded, err := pullFilesToPrefix(ctx, hf, store, hfRepo, sha, bucket, prefix, selected)
+	if err != nil {
+		cleanupPulledObjects(ctx, store, bucket, uploaded)
+		return "", err
+	}
+
+	manifest := ochrePullManifest{HFRepo: hfRepo, RevisionSHA: sha, PulledAt: time.Now().UTC()}
+	if err := putPullManifest(ctx, store, bucket, prefix, manifest); err != nil {
+		cleanupPulledObjects(ctx, store, bucket, uploaded)
+		return "", err
+	}
+
+	return fmt.Sprintf("s3://%s/%s", bucket, prefix), nil
+}
+
 // parseWeightsS3URI splits an s3://bucket/prefix URI into its bucket and
 // prefix. The prefix is normalised to end with '/' so downstream listing and
 // validation scope to the directory rather than any key merely sharing the
@@ -307,63 +618,6 @@ func buildWeightsImage(srcDir, imagePath string, contentBytes int64) error {
 	return mkfsExt4Runner(srcDir, imagePath)
 }
 
-// snapshotImportedWeightsVolume snapshots a viperblock volume that
-// v_utils.ImportDiskImage just wrote and closed -- never attached, never
-// nbdkit-served, so there is no live checkpoint to load. This mirrors the
-// offline snapshot sequence handlers/ec2/image/service_impl.go's
-// snapshotStoppedVolume uses for a stopped instance's root volume: reopen
-// read-only, load the numbered checkpoint Close() wrote, then CreateSnapshot.
-//
-// az names the zone recorded on the snapshot for DescribeSnapshots; it is not
-// used to resolve the clone.
-func snapshotImportedWeightsVolume(s3Config *vbs3.S3Config, volumeID string, volumeSize uint64, walDir, az string, mkey *masterkey.Key) (string, error) {
-	vbConfig := viperblock.VB{
-		VolumeName:        volumeID,
-		VolumeSize:        volumeSize,
-		BaseDir:           walDir,
-		Cache:             viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}},
-		MasterKey:         mkey,
-		EncryptionEnabled: mkey != nil,
-		Logger:            slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})),
-	}
-
-	vb, err := viperblock.New(&vbConfig, "s3", *s3Config)
-	if err != nil {
-		return "", fmt.Errorf("new viperblock: %w", err)
-	}
-	defer func() {
-		vb.StopChunkUploader()
-		vb.StopWALSyncer()
-	}()
-
-	if err := vb.Backend.Init(); err != nil {
-		return "", fmt.Errorf("backend init: %w", err)
-	}
-	if err := vb.LoadState(); err != nil {
-		return "", fmt.Errorf("load state: %w", err)
-	}
-	if err := vb.LoadBlockState(); err != nil {
-		return "", fmt.Errorf("load block state: %w", err)
-	}
-	defer func() {
-		if err := vb.RemoveLocalFiles(); err != nil {
-			slog.Warn("snapshotImportedWeightsVolume: failed to remove local files", "volumeId", volumeID, "err", err)
-		}
-	}()
-
-	snapshotID := admin.SnapPrefix(volumeID)
-	if _, err := vb.CreateSnapshot(snapshotID); err != nil {
-		return "", fmt.Errorf("create snapshot: %w", err)
-	}
-
-	store := objectstore.NewS3ObjectStoreFromConfig(s3Config.Host, s3Config.Region, s3Config.AccessKey, s3Config.SecretKey)
-	if err := registerWeightsSnapshot(store, s3Config.Bucket, snapshotID, volumeID, volumeSize, az, mkey != nil); err != nil {
-		return "", err
-	}
-
-	return snapshotID, nil
-}
-
 // registerWeightsSnapshot writes the EC2 control plane's half of the snapshot
 // prefix. CreateSnapshot above writes only viperblock's half -- the block
 // checkpoint and config.json -- but CreateVolume resolves a SnapshotId through
@@ -447,6 +701,16 @@ func runStageWeights(ctx context.Context, store objectstore.ObjectStore, weights
 		return "", err
 	}
 
+	// A pull manifest is optional: absent means an operator-supplied prefix
+	// from the offline path, which stage has always supported and must keep
+	// supporting unchanged.
+	var sourceRevision string
+	if manifest, ok, err := readPullManifest(ctx, store, bucket, prefix); err != nil {
+		return "", err
+	} else if ok {
+		sourceRevision = manifest.RevisionSHA
+	}
+
 	tmpDir, err := os.MkdirTemp(tmpDirFlag, "spinifex-weights-tmp-*")
 	if err != nil {
 		return "", fmt.Errorf("could not create temp dir: %w", err)
@@ -469,7 +733,7 @@ func runStageWeights(ctx context.Context, store objectstore.ObjectStore, weights
 		return "", err
 	}
 
-	if err := weightsStore.PutWeights(ctx, modelID, sourceURI, snapshotID); err != nil {
+	if err := weightsStore.PutWeightsWithRevision(ctx, modelID, sourceURI, snapshotID, sourceRevision); err != nil {
 		return "", err
 	}
 
@@ -485,12 +749,11 @@ func runStageWeights(ctx context.Context, store objectstore.ObjectStore, weights
 }
 
 // materializeWeightsVolume is the real weightsMaterializer: it packages
-// downloadDir into a filesystem image, imports it into a new viperblock
-// volume backed by node's predastore, and snapshots it. This is the
-// expensive, environment-dependent half of stage -- it shells out to
-// mkfs.ext4 and talks to a live S3-shaped backend -- so it is exercised live
-// (see the plan's Testing section) rather than in unit tests.
-func materializeWeightsVolume(node config.Config, downloadDir string, contentBytes int64, mkey *masterkey.Key) (string, error) {
+// downloadDir into a filesystem image, imports it into a new volume through
+// the EBS provider, and snapshots it. This is the expensive,
+// environment-dependent half of stage -- it shells out to mkfs.ext4 and needs
+// a live provider -- so it is exercised live rather than in unit tests.
+func materializeWeightsVolume(ctx context.Context, provider ebsprovider.EBSProvider, nodeID string, node config.Config, downloadDir string, contentBytes int64) (string, error) {
 	tmpDir := filepath.Dir(downloadDir)
 	volumeId := utils.GenerateResourceID("vol")
 	imagePath := filepath.Join(tmpDir, volumeId+".img")
@@ -505,74 +768,32 @@ func materializeWeightsVolume(node config.Config, downloadDir string, contentByt
 		return "", fmt.Errorf("could not stat image: %w", err)
 	}
 
-	// Round the volume up to a whole GiB rather than using the raw image size.
-	// viperblock rejects a tail write whose block overruns VolumeSize, so an
-	// unaligned size fails at the last block. This also keeps the byte size and
-	// the manifest's SizeGiB describing the same volume.
+	// Round the volume up to a whole GiB rather than using the raw image size,
+	// so the byte size and the recorded SizeGiB describe the same volume.
 	sizeGiB := amiVolumeSizeGiB(imageStat.Size())
 	volumeBytes := sizeGiB * bytesPerGiB
 
-	s3Config := vbs3.S3Config{
-		VolumeName: volumeId,
-		VolumeSize: volumeBytes,
-		Bucket:     node.Predastore.Bucket,
-		Region:     node.Predastore.Region,
-		AccessKey:  node.Predastore.AccessKey,
-		SecretKey:  node.Predastore.SecretKey,
-		Host:       node.Predastore.Host,
-	}
-
-	// AMIMetadata is deliberately left zero-valued: a weights volume is not
-	// bootable and must never be registered as a launchable AMI. Leaving it
-	// unset also means ImportDiskImage does not attempt its own automatic
-	// snapshot -- that happens explicitly below, after the volume is closed.
-	manifest := viperblock.VolumeConfig{}
-	manifest.VolumeMetadata.VolumeID = volumeId
-	manifest.VolumeMetadata.VolumeName = volumeId
-	manifest.VolumeMetadata.TenantID = "system"
-	manifest.VolumeMetadata.SizeGiB = sizeGiB
-	manifest.VolumeMetadata.State = "available"
-	manifest.VolumeMetadata.AvailabilityZone = node.Predastore.Region
-	manifest.VolumeMetadata.CreatedAt = time.Now()
-	manifest.VolumeMetadata.VolumeType = "gp3"
-	manifest.VolumeMetadata.IOPS = 1000
-
-	vbConfig := viperblock.VB{
-		VolumeName:        volumeId,
-		VolumeSize:        volumeBytes,
-		BaseDir:           tmpDir,
-		Cache:             viperblock.Cache{Config: viperblock.CacheConfig{Size: 0}},
-		VolumeConfig:      manifest,
-		MasterKey:         mkey,
-		EncryptionEnabled: mkey != nil,
-		Logger:            slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})),
-	}
-
-	var flushBar *pterm.ProgressbarPrinter
-	var flushUpdate func(current uint64)
-	progress := func(current, total uint64) {
-		if flushBar == nil {
-			flushBar, flushUpdate = utils.NewByteProgressBar("Flushing weights to storage", total)
-		}
-		flushUpdate(current)
-	}
-
 	// Name the volume in the failure path: a part-written import leaves blocks
 	// in predastore, and without the ID an operator cannot find them to reclaim.
-	if err := v_utils.ImportDiskImage(&s3Config, &vbConfig, imagePath, progress); err != nil {
-		if flushBar != nil {
-			_, _ = flushBar.Stop()
-		}
+	fmt.Println("Writing weights to storage ...")
+	if err := admin.ImportImage(ctx, provider, admin.ImportOpts{
+		VolumeID:         volumeId,
+		NodeID:           nodeID,
+		SizeBytes:        utils.SafeUint64ToInt64(volumeBytes),
+		AvailabilityZone: node.AZ,
+		SourcePath:       imagePath,
+		Snapshot:         true,
+		Progress:         os.Stdout,
+	}); err != nil {
 		return "", fmt.Errorf("could not import weights volume %s: %w", volumeId, err)
 	}
-	if flushBar != nil {
-		_, _ = flushBar.Stop()
-	}
 
-	fmt.Println("Snapshotting weights volume ...")
-	snapshotID, err := snapshotImportedWeightsVolume(&s3Config, volumeId, volumeBytes, tmpDir, node.AZ, mkey)
-	if err != nil {
-		return "", fmt.Errorf("could not snapshot weights volume: %w", err)
+	// The provider wrote viperblock's half of the snapshot prefix; this writes
+	// the EC2 control plane's, without which a launch cannot resolve it.
+	snapshotID := admin.SnapPrefix(volumeId)
+	store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
+	if err := registerWeightsSnapshot(store, node.Predastore.Bucket, snapshotID, volumeId, volumeBytes, node.AZ, node.Viperblock.EncryptionKeyFile != ""); err != nil {
+		return "", err
 	}
 	return snapshotID, nil
 }
@@ -583,8 +804,8 @@ func materializeWeightsVolume(node config.Config, downloadDir string, contentByt
 // stage time -- and treats a missing object as gone. Any other read failure
 // is a real error, not a signal to guess the snapshot's existence.
 func newWeightsSnapshotChecker(store objectstore.ObjectStore, bucket string) weightsSnapshotChecker {
-	return func(_ context.Context, snapshotID string) (bool, error) {
-		if _, err := handlers_ec2_snapshot.ReadSnapshotConfig(store, bucket, snapshotID); err != nil {
+	return func(ctx context.Context, snapshotID string) (bool, error) {
+		if _, err := handlers_ec2_snapshot.ReadSnapshotConfig(ctx, store, bucket, snapshotID); err != nil {
 			if objectstore.IsNoSuchKeyError(err) {
 				return false, nil
 			}
@@ -629,15 +850,9 @@ func runOchreWeightsStage(cmd *cobra.Command, _ []string) {
 	weightsStore := gateway_bedrock.NewWeightsStore(js, len(appConfig.Nodes))
 	store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
 
-	mkey, err := utils.LoadViperblockMasterKey(node.Viperblock.EncryptionKeyFile)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not load viperblock encryption key: %v\n", err)
-		ochreExit(1)
-		return
-	}
-
-	materialize := func(_ context.Context, downloadDir string, contentBytes int64) (string, error) {
-		return materializeWeightsVolume(node, downloadDir, contentBytes, mkey)
+	provider := ebsprovider.NewNATSProvider(nc, imageImportTimeout)
+	materialize := func(ctx context.Context, downloadDir string, contentBytes int64) (string, error) {
+		return materializeWeightsVolume(ctx, provider, appConfig.Node, node, downloadDir, contentBytes)
 	}
 	checkSnapshotLive := newWeightsSnapshotChecker(store, node.Predastore.Bucket)
 
@@ -673,7 +888,7 @@ func listWeightsOutput(ctx context.Context, weightsStore *gateway_bedrock.Weight
 		return "No models staged.", nil
 	}
 
-	tableData := pterm.TableData{{"MODEL ID", "SOURCE URI", "SNAPSHOT ID", "STATUS"}}
+	tableData := pterm.TableData{{"MODEL ID", "SOURCE URI", "REVISION", "SNAPSHOT ID", "STATUS"}}
 	for _, e := range entries {
 		live, err := checkSnapshotLive(ctx, e.SnapshotID)
 		if err != nil {
@@ -683,7 +898,11 @@ func listWeightsOutput(ctx context.Context, weightsStore *gateway_bedrock.Weight
 		if !live {
 			status = weightsStatusDangling
 		}
-		tableData = append(tableData, []string{e.ModelID, e.SourceURI, e.SnapshotID, status})
+		revision := e.SourceRevision
+		if revision == "" {
+			revision = "-"
+		}
+		tableData = append(tableData, []string{e.ModelID, e.SourceURI, revision, e.SnapshotID, status})
 	}
 	return pterm.DefaultTable.WithHasHeader().WithLeftAlignment().WithData(tableData).Srender()
 }
@@ -766,6 +985,142 @@ func runOchreWeightsRemove(cmd *cobra.Command, _ []string) {
 	fmt.Printf("✅ Removed staged-weights entry for %s (snapshot %s and source objects untouched).\n", modelID, entry.SnapshotID)
 }
 
+// resolveHFToken applies pull's token resolution order (D2): the --hf-token
+// flag, then HF_TOKEN env (the must-have path, self-sufficient on its own),
+// then a stored platform credential for vendor "huggingface". The stored
+// credential step is a best-effort fallback: a cluster with no IAM master
+// key provisioned yet simply yields no token rather than failing the pull.
+func resolveHFToken(ctx context.Context, cmd *cobra.Command, cfg *config.ClusterConfig, nc *nats.Conn) string {
+	if token, _ := cmd.Flags().GetString("hf-token"); token != "" {
+		return token
+	}
+	if token := os.Getenv("HF_TOKEN"); token != "" {
+		return token
+	}
+
+	masterKeyPath := filepath.Join(cfg.NodeBaseDir(), "config", "master.key")
+	masterKey, err := handlers_iam.LoadMasterKey(masterKeyPath)
+	if err != nil {
+		return ""
+	}
+	js, err := jetstream.New(nc)
+	if err != nil {
+		return ""
+	}
+	credStore := gateway_bedrock.NewCredentialStore(js, masterKey, len(cfg.Nodes), nil)
+	token, ok, err := credStore.Resolve(ctx, utils.GlobalAccountID, vendorHuggingFace)
+	if err != nil || !ok {
+		return ""
+	}
+	return token
+}
+
+func runOchreWeightsPull(cmd *cobra.Command, _ []string) {
+	modelID, _ := cmd.Flags().GetString("model-id")
+	hfRepo, _ := cmd.Flags().GetString("hf-repo")
+	revision, _ := cmd.Flags().GetString("revision")
+	revisionExplicit := cmd.Flags().Changed("revision")
+	s3URI, _ := cmd.Flags().GetString("s3-uri")
+
+	appConfig, nc, err := loadConfigAndConnectFn()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+	defer nc.Close()
+	node := appConfig.Nodes[appConfig.Node]
+
+	ctx := context.Background()
+	token := resolveHFToken(ctx, cmd, appConfig, nc)
+
+	hf := hfhub.NewClient(token)
+	store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
+
+	finalURI, err := runPullWeights(ctx, hf, store, modelID, hfRepo, revision, revisionExplicit, s3URI)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		ochreExit(1)
+		return
+	}
+	fmt.Printf("✅ Pulled into %s\n", finalURI)
+	fmt.Println(finalURI)
+}
+
+// readTokenFromStdin reads a credential from r, trimming exactly one
+// trailing newline (matching 'printf "%s" "$TOKEN" | cmd --token -' or a
+// piped 'echo'). The token is never logged.
+func readTokenFromStdin(r io.Reader) (string, error) {
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return "", fmt.Errorf("read token from stdin: %w", err)
+	}
+	token := strings.TrimSuffix(strings.TrimSuffix(string(data), "\n"), "\r")
+	if token == "" {
+		return "", fmt.Errorf("no token read from stdin")
+	}
+	return token, nil
+}
+
+// ochreCredentialsStore connects to the cluster, loads the node's IAM master
+// key from disk -- the same on-disk key initIAMServiceFromConfig uses -- and
+// returns a CredentialStore ready for PutCredential/Resolve.
+func ochreCredentialsStore() (*gateway_bedrock.CredentialStore, func(), error) {
+	cfg, nc, err := loadConfigAndConnectFn()
+	if err != nil {
+		return nil, nil, fmt.Errorf("connect to cluster: %w", err)
+	}
+	masterKeyPath := filepath.Join(cfg.NodeBaseDir(), "config", "master.key")
+	masterKey, err := handlers_iam.LoadMasterKey(masterKeyPath)
+	if err != nil {
+		nc.Close()
+		return nil, nil, fmt.Errorf("load master key: %w", err)
+	}
+	js, err := jetstream.New(nc)
+	if err != nil {
+		nc.Close()
+		return nil, nil, fmt.Errorf("jetstream context: %w", err)
+	}
+	return gateway_bedrock.NewCredentialStore(js, masterKey, len(cfg.Nodes), nil), func() { nc.Close() }, nil
+}
+
+func runOchreCredentialsSet(cmd *cobra.Command, _ []string) {
+	vendor, _ := cmd.Flags().GetString("vendor")
+	accountID, _ := cmd.Flags().GetString("account")
+	tokenFlag, _ := cmd.Flags().GetString("token")
+
+	if tokenFlag != "-" {
+		fmt.Fprintln(os.Stderr, "Error: --token must be '-' (the secret is read from stdin, never a command-line argument)")
+		ochreExit(1)
+		return
+	}
+	if accountID == "" {
+		accountID = utils.GlobalAccountID
+	}
+
+	token, err := readTokenFromStdin(os.Stdin)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+
+	store, cleanup, err := ochreCredentialsStore()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+	defer cleanup()
+
+	if err := store.PutCredential(context.Background(), accountID, vendor, token); err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		ochreExit(1)
+		return
+	}
+	fmt.Printf("✅ Stored %s credential for account %s\n", vendor, accountID)
+}
+
 var adminOchreAccessCmd = &cobra.Command{
 	Use:   "access",
 	Short: "Manage per-account model access grants",
@@ -814,15 +1169,11 @@ func init() {
 		c.Flags().String("account-id", "", "12-digit account ID to change (required)")
 		c.Flags().String("model-id", "", "Model ID to change (e.g. meta.llama3-2-1b-instruct-v1:0)")
 		c.Flags().Bool("all-models", false, "Apply to every model in the platform catalog")
-		if err := c.MarkFlagRequired("account-id"); err != nil {
-			panic(err)
-		}
+		_ = c.MarkFlagRequired("account-id")
 	}
 
 	adminOchreAccessListCmd.Flags().String("account-id", "", "12-digit account ID to inspect (required)")
-	if err := adminOchreAccessListCmd.MarkFlagRequired("account-id"); err != nil {
-		panic(err)
-	}
+	_ = adminOchreAccessListCmd.MarkFlagRequired("account-id")
 }
 
 // ochreAccessStore connects to the cluster and returns the grant store along

cmd/spinifex/cmd/admin_ochre_appliance.go

@@ -0,0 +1,97 @@
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	handlers_ochrevector "github.com/mulgadc/spinifex/spinifex/handlers/ochrevector"
+	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/spf13/cobra"
+)
+
+// applianceTeardownTimeout bounds the teardown NATS round trip: it covers an
+// RDS DeleteDBInstance call plus a KV delete, not a VM boot, so it is far
+// shorter than the appliance's own launch timeout.
+const applianceTeardownTimeout = 5 * time.Minute
+
+var ochreApplianceCmd = &cobra.Command{
+	Use:   "appliance",
+	Short: "Manage the shared platform Postgres appliance backing every Ochre vector index",
+}
+
+var ochreApplianceTeardownCmd = &cobra.Command{
+	Use:   "teardown",
+	Short: "Destroy the shared platform Postgres appliance",
+	Long: `teardown destroys the platform appliance singleton: its backing RDS DB
+instance, this node's VPC host port, and its KV record.
+
+This is a destructive, cluster-wide operator action -- every account's vector
+index data stored in the appliance is lost with it. By default the index
+registry and KB/DataSource metadata survive, exactly as they survive a
+restart: a future daemon startup re-provisions a fresh appliance (a new
+instance, a new password) and reconciles each surviving index by re-creating
+it and re-ingesting from its recorded source, with no operator action and no
+change to its id. --purge-metadata additionally wipes that metadata for an
+intentional full destroy, after which nothing is auto-restored.
+
+Requires --confirm.`,
+	Run: runOchreApplianceTeardown,
+}
+
+func init() {
+	ochreCmd.AddCommand(ochreApplianceCmd)
+	ochreApplianceCmd.AddCommand(ochreApplianceTeardownCmd)
+
+	ochreApplianceTeardownCmd.Flags().Bool("confirm", false,
+		"Required: confirms you intend to destroy the shared platform appliance and every account's vector index data with it")
+	ochreApplianceTeardownCmd.Flags().Bool("purge-metadata", false,
+		"Also wipe the index registry and KB/DataSource records, disabling auto-restore on the next re-provision")
+}
+
+// applianceTeardownFn indirects the NATS call so runApplianceTeardownGuarded
+// is testable without a live daemon, mirroring vectorServiceFn/
+// endpointServiceFn.
+var applianceTeardownFn = func(ctx context.Context, purgeMetadata bool) error {
+	_, nc, err := loadConfigAndConnectFn()
+	if err != nil {
+		return err
+	}
+	defer nc.Close()
+	_, err = utils.NATSRequest[handlers_ochrevector.TeardownApplianceResponse](ctx, nc,
+		handlers_ochrevector.SubjectTeardownAppliance, &handlers_ochrevector.TeardownApplianceRequest{PurgeMetadata: purgeMetadata},
+		applianceTeardownTimeout, utils.GlobalAccountID)
+	return err
+}
+
+// runApplianceTeardownGuarded is the testable core of 'ochre appliance
+// teardown': it refuses without --confirm before making any NATS call, so a
+// missing flag never even opens a connection to the cluster.
+func runApplianceTeardownGuarded(ctx context.Context, confirmed, purgeMetadata bool, teardown func(context.Context, bool) error) (string, error) {
+	if !confirmed {
+		return "", errors.New("refusing to tear down the shared platform appliance without --confirm: " +
+			"this destroys every account's vector index data and is not automatically rebuilt")
+	}
+	if err := teardown(ctx, purgeMetadata); err != nil {
+		return "", err
+	}
+	if purgeMetadata {
+		return "✅ Platform appliance torn down, index/KB metadata purged.", nil
+	}
+	return "✅ Platform appliance torn down.", nil
+}
+
+func runOchreApplianceTeardown(cmd *cobra.Command, _ []string) {
+	confirmed, _ := cmd.Flags().GetBool("confirm")
+	purgeMetadata, _ := cmd.Flags().GetBool("purge-metadata")
+
+	msg, err := runApplianceTeardownGuarded(context.Background(), confirmed, purgeMetadata, applianceTeardownFn)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		ochreExit(1)
+		return
+	}
+	fmt.Println(msg)
+}