release: update go.mod for v1.18.0

The local helper masked x-amz-security-token by line prefix, which matched the
header form and missed the presigned query form entirely, leaking the token into
the exported logs on every signature mismatch. Defer to bluebottle's
RedactedCanonicalRequest, which knows the structure and masks both forms.

.github/actions/junit-to-es/action.yml

@@ -0,0 +1,43 @@
+name: JUnit to Elasticsearch
+description: >-
+  Bulk-index one doc per JUnit test case into the mulga Elasticsearch sink,
+  so per-test pass rates outlive the 90-day artifact retention. Call it once
+  per job right after the JUnit XML is produced (with `if: always()`), next to
+  the junit-report action that renders the same files into the job summary.
+
+inputs:
+  junit-glob:
+    description: Shell glob matching the JUnit XML files to ship.
+    required: true
+  cell:
+    description: >-
+      Cell label stored as labels.ci_cell — the same value the job puts in
+      ci.cell for its OTel attributes (single, multi, gpu, ddil, 20, ...).
+      Without it, results from different topologies collapse into one bucket.
+    required: true
+  es-sink:
+    description: host:port of the Elasticsearch sink.
+    required: false
+    default: "192.168.1.13:9200"
+  index:
+    description: Target index.
+    required: false
+    default: mulga-ci-test-results
+
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      env:
+        JUNIT_GLOB: ${{ inputs.junit-glob }}
+        JUNIT_CELL: ${{ inputs.cell }}
+        MULGA_ES_SINK: ${{ inputs.es-sink }}
+        ES_INDEX: ${{ inputs.index }}
+        RUN_ID: ${{ github.run_id }}
+        RUN_ATTEMPT: ${{ github.run_attempt }}
+        WORKFLOW: ${{ github.workflow }}
+        BRANCH: ${{ github.head_ref || github.ref_name }}
+        SHA: ${{ github.sha }}
+        ARTIFACT_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+      run: |
+        python3 "${{ github.action_path }}/ship.py"

.github/actions/junit-to-es/ship.py

@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+"""Ship per-test JUnit results to the Elasticsearch sink.
+
+One doc per leaf test case, bulk-indexed into mulga-ci-test-results. The
+mulga-ci-runs summary says which suite failed; this says which test, with the
+failure text, so a per-test pass-rate trend outlives artifact retention.
+
+Inputs (env):
+    JUNIT_GLOB      Glob matching JUnit XML files. No matches is not an error:
+                    a leg that never ran produces none.
+    JUNIT_CELL      Cell label for these results (single, multi, gpu, 20, ...).
+    MULGA_ES_SINK   host:port of the sink. Anonymous requests are superuser
+                    there, so no credentials are involved.
+    ES_INDEX        Target index; defaults to mulga-ci-test-results.
+    RUN_ID, RUN_ATTEMPT, WORKFLOW, BRANCH, SHA, ARTIFACT_URL
+"""
+
+from __future__ import annotations
+
+import glob
+import json
+import os
+import re
+import sys
+import urllib.error
+import urllib.request
+import xml.etree.ElementTree as ET
+from datetime import datetime, timezone
+
+# Go test output carries colour codes into the JUnit CDATA (harness.Phase
+# emits them), which are noise in a stored failure message.
+_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
+
+# Enough failure text to recognise a failure; the artifact log stays the full
+# record.
+_MAX_MESSAGE = 4000
+
+# go-junit-report emits this when a `=== RUN` line has no matching result line,
+# which means the log it was fed was truncated or filtered. It is a parse gap,
+# not a test outcome, so it is stored under its own status and left out of every
+# pass rate rather than counted as a failure.
+_NO_RESULT = "No test result found"
+
+
+def _strip_ansi(s: str) -> str:
+    return _ANSI_RE.sub("", s)
+
+
+def _truncate(s: str) -> str:
+    s = s.strip()
+    return s if len(s) <= _MAX_MESSAGE else s[: _MAX_MESSAGE - 1] + "…"
+
+
+def _suite_from_path(path: str) -> str:
+    base = os.path.basename(path)
+    if base.endswith(".xml"):
+        base = base[:-4]
+    if base.startswith("junit-"):
+        base = base[len("junit-") :]
+    return base
+
+
+def _cases(path: str) -> list[dict]:
+    """Parse one JUnit file into leaf test cases."""
+    try:
+        root = ET.parse(path).getroot()
+    except ET.ParseError as exc:
+        print(f"::warning::junit-to-es: {path} is not parseable ({exc}), skipped")
+        return []
+
+    # JUnit XML has two shapes: <testsuite> as root, or <testsuites> wrapping
+    # several. .iter() handles both.
+    out: list[dict] = []
+    for s in root.iter("testsuite"):
+        for tc in s.findall("testcase"):
+            status = "pass"
+            fail_type = ""
+            message = ""
+            for tag in ("failure", "error"):
+                node = tc.find(tag)
+                if node is not None:
+                    status = "fail" if tag == "failure" else "error"
+                    fail_type = tag
+                    message = _strip_ansi(
+                        ((node.get("message") or "") + "\n" + (node.text or "")).strip()
+                    )
+                    if (node.get("message") or "").strip() == _NO_RESULT:
+                        status = "unknown"
+                    break
+            if status == "pass" and tc.find("skipped") is not None:
+                status = "skip"
+            out.append(
+                {
+                    "name": tc.get("name", ""),
+                    "classname": tc.get("classname", ""),
+                    "time": float(tc.get("time", 0) or 0),
+                    "status": status,
+                    "fail_type": fail_type,
+                    "message": message,
+                }
+            )
+
+    # Drop container cases: Go reports a parent t.Run as a rollup of its
+    # children, so keeping both double-counts one outcome. Same rule the job
+    # summary renderer applies.
+    names = {c["name"] for c in out}
+    containers = {n.rsplit("/", 1)[0] for n in names if "/" in n}
+    return [c for c in out if c["name"] not in containers]
+
+
+def _doc(cell: str, suite: str, case: dict, ts: str, ci: dict) -> dict:
+    full = case["name"]
+    top, _, sub = full.partition("/")
+    doc = {
+        "@timestamp": ts,
+        "suite": suite,
+        "status": case["status"],
+        "duration_seconds": case["time"],
+        "artifact_url": ci["artifact_url"],
+        "labels": {"ci_run_id": ci["run_id"], "ci_cell": cell},
+        "test": {
+            "name": top,
+            "subtest": sub,
+            "full_name": full,
+            "classname": case["classname"],
+        },
+        "ci": {
+            "workflow": ci["workflow"],
+            "branch": ci["branch"],
+            "sha": ci["sha"],
+            "run_attempt": ci["run_attempt"],
+        },
+    }
+    if case["fail_type"]:
+        doc["failure"] = {
+            "type": case["fail_type"],
+            "message": _truncate(case["message"]),
+        }
+    return doc
+
+
+def _doc_id(ci: dict, cell: str, suite: str, full_name: str) -> str:
+    # Deterministic so a replayed post overwrites instead of double-counting.
+    # run_attempt is part of it because a re-run is a distinct result.
+    return f"{ci['run_id']}-{ci['run_attempt']}-{cell}-{suite}-{full_name}"
+
+
+def _bulk(sink: str, index: str, body: str) -> int:
+    req = urllib.request.Request(
+        f"http://{sink}/{index}/_bulk",
+        data=body.encode("utf-8"),
+        headers={"Content-Type": "application/x-ndjson"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            payload = json.loads(resp.read().decode("utf-8"))
+    except (urllib.error.URLError, OSError) as exc:
+        print(f"::error::junit-to-es: bulk POST to {sink}/{index} failed: {exc}")
+        return 1
+
+    if payload.get("errors"):
+        for item in payload.get("items", []):
+            err = next(iter(item.values())).get("error")
+            if err:
+                print(f"::error::junit-to-es: bulk rejected a doc: {json.dumps(err)}")
+                return 1
+    return 0
+
+
+def main() -> int:
+    pattern = os.environ.get("JUNIT_GLOB", "").strip()
+    if not pattern:
+        print("::error::junit-to-es: JUNIT_GLOB is empty")
+        return 1
+    cell = os.environ.get("JUNIT_CELL", "").strip()
+    if not cell:
+        print("::error::junit-to-es: JUNIT_CELL is empty")
+        return 1
+    sink = os.environ.get("MULGA_ES_SINK", "").strip()
+    if not sink:
+        print("::error::junit-to-es: MULGA_ES_SINK is empty")
+        return 1
+    index = os.environ.get("ES_INDEX", "").strip() or "mulga-ci-test-results"
+
+    ci = {
+        "run_id": os.environ.get("RUN_ID", ""),
+        "run_attempt": os.environ.get("RUN_ATTEMPT", ""),
+        "workflow": os.environ.get("WORKFLOW", ""),
+        "branch": os.environ.get("BRANCH", ""),
+        "sha": os.environ.get("SHA", ""),
+        "artifact_url": os.environ.get("ARTIFACT_URL", ""),
+    }
+    ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    lines: list[str] = []
+    counts: dict[str, int] = {"pass": 0, "fail": 0, "error": 0, "skip": 0, "unknown": 0}
+    for path in sorted(glob.glob(pattern)):
+        suite = _suite_from_path(path)
+        for case in _cases(path):
+            doc = _doc(cell, suite, case, ts, ci)
+            counts[case["status"]] = counts.get(case["status"], 0) + 1
+            meta = {"index": {"_id": _doc_id(ci, cell, suite, doc["test"]["full_name"])}}
+            lines.append(json.dumps(meta))
+            lines.append(json.dumps(doc))
+
+    if not lines:
+        # Not an error on its own: the leg may have been skipped, or died
+        # before any test ran. The run summary doc records that separately.
+        print(f"junit-to-es: no JUnit files matched {pattern}, nothing to post")
+        return 0
+
+    rc = _bulk(sink, index, "\n".join(lines) + "\n")
+    if rc == 0:
+        print(
+            f"junit-to-es: posted {sum(counts.values())} test docs "
+            f"(cell {cell}) to {sink}/{index} — {counts['pass']} pass, "
+            f"{counts['fail']} fail, {counts['error']} error, "
+            f"{counts['skip']} skip, {counts['unknown']} unknown"
+        )
+    return rc
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/dependabot.yml

@@ -5,7 +5,7 @@ updates:
     directory: /
     schedule:
       interval: weekly
-      day: monday
+      day: sunday
       timezone: Australia/Sydney
     cooldown:
       default-days: 3
@@ -19,7 +19,7 @@ updates:
     directory: /
     schedule:
       interval: weekly
-      day: monday
+      day: sunday
       timezone: Australia/Sydney
     cooldown:
       default-days: 3
@@ -38,7 +38,7 @@ updates:
     directory: /spinifex/services/spinifexui/frontend
     schedule:
       interval: weekly
-      day: monday
+      day: sunday
       timezone: Australia/Sydney
     cooldown:
       default-days: 3

.github/resolve-sibling-refs.sh

@@ -1,20 +1,21 @@
 #!/usr/bin/env bash
-# Resolve which branch of each sibling repo (viperblock, predastore) the CI
-# workspace should build against. A cross-repo change is developed on a branch
+# Resolve which branch of each sibling repo (viperblock, predastore, northstar,
+# bluebottle) the CI workspace should build against. A cross-repo change is developed on a branch
 # of the SAME name in each repo; build against that branch when it exists so the
 # set can be tested together, and fall back to dev otherwise. This mirrors the
 # resolution the e2e path already does in
 # mulga/scripts/tofu-cluster/build-install-artifacts.sh, so unit CI and e2e CI
 # agree on how sibling branches are selected.
 #
-# Reads $BRANCH (the triggering head/ref name) and writes viperblock=<ref> and
-# predastore=<ref> to $GITHUB_OUTPUT for the checkout steps to consume.
+# Reads $BRANCH (the triggering head/ref name) and writes viperblock=<ref>,
+# predastore=<ref>, northstar=<ref> and bluebottle=<ref> to $GITHUB_OUTPUT for
+# the checkout steps to consume.
 set -euo pipefail
 
 : "${BRANCH:?BRANCH must be set to the triggering branch name}"
 : "${GITHUB_OUTPUT:?GITHUB_OUTPUT must be set}"
 
-for repo in viperblock predastore; do
+for repo in viperblock predastore northstar bluebottle; do
   if git ls-remote --exit-code --heads \
       "https://github.com/mulgadc/${repo}.git" "$BRANCH" >/dev/null 2>&1; then
     ref="$BRANCH"

.github/scripts/suite-sets.sh

@@ -24,10 +24,14 @@
 # behaviour, so a second node adds no assertion. Its DB VMs are capped at four
 # concurrent by the suite's own semaphore, which is what keeps it inside the
 # job's 50-minute budget alongside the suites above.
-E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds"
+# quota belongs to both topologies because the gate it asserts is gateway code
+# that runs identically on either, and because a quota left switched on would
+# cap every suite sharing the environment — so it must be proved to restore
+# itself wherever it runs, not only on one node.
+E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota"
 
 # Suites runnable against a multi-node environment.
-E2E_SUITES_MULTI="multinode lb cert"
+E2E_SUITES_MULTI="multinode lb cert quota"
 
 # grep -xE alternation form, for narrowing a requested set to the eligible one.
 E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"

.github/workflows/e2e-baremetal.yml

@@ -143,6 +143,16 @@ jobs:
           [ -f "$LOG" ] || exit 0
           go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-baremetal.xml" || true
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: "20"
+
       - name: Analyze e2e failures
         if: always()
         uses: ./.github/actions/e2e-analyze

.github/workflows/e2e-ddil.yml

@@ -180,6 +180,16 @@ jobs:
           junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
           title: "DDIL scenario results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
+          cell: ddil
+
       - name: Collect Cluster Logs
         if: failure()
         working-directory: mulga/scripts/tofu-cluster

.github/workflows/e2e-nightly.yml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
     inputs:
       cells:
-        description: "Comma-separated cell numbers to run (blank = all)"
+        description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-19 tofu matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection"
         required: false
 
 permissions:
@@ -46,7 +46,7 @@ jobs:
           - { cell: 19, install: binary, topo: single,     net: nat,    host: debian-13,    arch: x86_64, runner: ci-single, extra: "nat-single" }
 
     runs-on: [ self-hosted, "${{ matrix.runner }}" ]
-    timeout-minutes: 35
+    timeout-minutes: ${{ matrix.extra == 'tofu-examples' && 45 || 35 }}
     env:
       TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}
@@ -251,6 +251,18 @@ jobs:
                if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
              done; exit \$FAIL"
 
+      - name: Import RDS PostgreSQL AMI
+        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples'
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          INVENTORY=$(tofu output -json inventory)
+          WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
+            "spx admin images import --name spinifex-rds-postgres"
+
       - name: Run Tofu Workbooks
         if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples'
         working-directory: mulga/scripts/tofu-cluster
@@ -330,6 +342,14 @@ jobs:
               | tar -C "${ARTIFACT_DIR}/vm-console-node${idx}" -xf - 2>/dev/null
             ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
               > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
+            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \
+              > "${ARTIFACT_DIR}/openvswitch-ipsec-node${idx}.log" 2>/dev/null
+            # ovn-controller and ovs-vswitchd log to files, never to journald:
+            # without these a cross-chassis black-hole cannot be attributed to
+            # IPsec versus SB port binding.
+            mkdir -p "${ARTIFACT_DIR}/ovs-logs-node${idx}"
+            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo tar -C /var/log -cf - ovn openvswitch 2>/dev/null || true' \
+              | tar -C "${ARTIFACT_DIR}/ovs-logs-node${idx}" -xf - 2>/dev/null
           done
 
           cat "${ARTIFACT_DIR}"/spinifex-journal-node*.log \
@@ -351,6 +371,16 @@ jobs:
             go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
           done
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always() && steps.gate.outputs.selected == 'true'
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: ${{ matrix.cell }}
+
       - name: Analyze e2e failures
         if: always() && steps.gate.outputs.selected == 'true'
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -367,8 +397,11 @@ jobs:
           if-no-files-found: warn
           retention-days: 14
 
+      # Gated like every other step, not just always(): a cell the dispatch
+      # deselected never checked out or ran tofu init, so destroying would fail
+      # on missing plugins and report the cell red for correctly doing nothing.
       - name: Teardown
-        if: always()
+        if: always() && steps.gate.outputs.selected == 'true'
         working-directory: mulga/scripts/tofu-cluster
         env:
           MATRIX_HOST: ${{ matrix.host }}
@@ -385,7 +418,7 @@ jobs:
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
 
       - name: Upload tofu log
-        if: always()
+        if: always() && steps.gate.outputs.selected == 'true'
         uses: actions/upload-artifact@v7
         with:
           name: tofu-log-cell-${{ matrix.cell }}-${{ github.run_id }}
@@ -414,3 +447,615 @@ jobs:
       # There's no matrix leg here, just the one fixed baremetal cell.
       otel-attrs: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=20,ci.matrix=baremetal/pxe/real-hw
     secrets: inherit
+
+  # ISO cells. Unlike 1-19 these need no tofu: the harnesses run QEMU directly
+  # on the runner. The nodes are therefore L1, which keeps the instances they
+  # launch at L2 and their benchmark numbers comparable between the two cells.
+  #
+  # Serialised against each other because the guest MACs are fixed
+  # (52:54:00:12:...): stable MACs are what make DHCP discovery work, and two
+  # concurrent runs would put duplicates on the rack LAN, which is a confusing
+  # failure for everyone on it, not just CI.
+  iso-single:
+    name: cell-21 (iso/dhcp/single)
+    if: >-
+      github.event_name == 'schedule' ||
+      inputs.cells == '' ||
+      contains(format(',{0},', inputs.cells), ',21,')
+    runs-on: [self-hosted, ci-single]
+    # ISO build, install, boot and pool re-form is ~35 of these on its own,
+    # before the workload step imports an AMI and launches an instance.
+    timeout-minutes: 90
+    concurrency:
+      group: iso-qemu-rack
+      cancel-in-progress: false
+    env:
+      CI_WAN_MODE: dhcp
+      CI_PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
+      # What a real node gets. The harness default of 4G is a dev-box size, and
+      # importing an 820 MiB AMI on top of the running stack does not fit in it.
+      MEM: 8G
+      SMP: 8
+      # .230 is consumed by the OVN router's SNAT, leaving .231-.234 for
+      # instances. Disjoint from cell-22's pool so a run that dies holding its
+      # SNAT address cannot collide with the next cell.
+      EXT_POOL_START: 192.168.0.230
+      EXT_POOL_END: 192.168.0.234
+      EXT_GATEWAY: 192.168.1.1
+      EXT_PREFIX_LEN: 23
+      # Pins where the benchmark writes, so collecting and removing it later are
+      # both exact. Left to itself the harness picks a timestamped name.
+      BENCH_OUT: /tmp/cell21-bench-${{ github.run_id }}
+    steps:
+      - name: Clean Workspace
+        run: |
+          sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          path: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ env.GIT_BRANCH }}
+          path: mulga/spinifex
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ env.GIT_BRANCH }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: mulga/spinifex
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: mulga/viperblock
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: mulga/predastore
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: mulga/northstar
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: mulga/bluebottle
+
+      # Assert the host can do what the cell assumes, before spending 40
+      # minutes finding out it cannot. A checklist in a plan doc goes stale;
+      # this does not.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          ip link show "${BRIDGE:-br-wan}" >/dev/null ||
+            { echo "::error::bridge br-wan not found on this runner"; exit 1; }
+          grep -qE "^allow +${BRIDGE:-br-wan}$" /etc/qemu/bridge.conf ||
+            { echo "::error::br-wan not permitted in /etc/qemu/bridge.conf"; exit 1; }
+          command -v sshpass >/dev/null ||
+            { echo "::error::sshpass not installed"; exit 1; }
+          free_mb=$(free -m | awk '/^Mem:/ {print $7}')
+          [ "$free_mb" -ge 8192 ] ||
+            { echo "::error::need 8 GiB available, have ${free_mb} MiB"; exit 1; }
+          # A 40G image on tmpfs competes with the 8 GiB the VM itself needs.
+          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
+          [ "$fstype" != tmpfs ] ||
+            { echo "::error::$TMPDIR is tmpfs — the node disk would be in RAM"; exit 1; }
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          [ "$avail_gb" -ge 60 ] ||
+            { echo "::error::need 60 GiB on $TMPDIR, have ${avail_gb} GiB"; exit 1; }
+
+      - name: Build artefacts
+        working-directory: mulga/scripts/iso-builder
+        run: make build-rootfs build-squashfs build-initramfs
+
+      # One key for both halves of the cell. test-workload.sh SSHes to the node
+      # AND registers the same key as the EC2 key pair it uses to reach the
+      # instances, so the key test-smoke.sh installs has to be that one — its
+      # default ~/.ssh/id_*.pub would leave the workload step locked out.
+      - name: Prepare SSH key
+        run: |
+          set -euo pipefail
+          mkdir -p "$HOME/.ssh" && chmod 700 "$HOME/.ssh"
+          # Two keys, because the guest key must not share a name with the one
+          # smoke-test.sh registers from the node: the node holds that private
+          # half, and IMDS caches material by key name for five minutes, so
+          # re-importing over it is invisible to the guests.
+          for k in spinifex-key spinifex-workload-key; do
+            [ -f "$HOME/.ssh/$k" ] ||
+              ssh-keygen -t ed25519 -N "" -f "$HOME/.ssh/$k" -C "$k-ci"
+          done
+
+      # Boots the ISO, installs unattended, reads the lease off the serial
+      # console, runs the in-guest smoke test, and installs the key for the
+      # next step.
+      - name: Install and smoke test
+        id: smoke
+        working-directory: mulga/scripts/iso-builder
+        run: |
+          set -euo pipefail
+          # KEEP_NODE: the harness owns the only VM there is and tears it down
+          # on exit, so without this the workload step below has no node left.
+          KEEP_NODE=1 SSH_IDENTITY="$HOME/.ssh/spinifex-key.pub" \
+            ./test/test-smoke.sh | tee /tmp/smoke-$GITHUB_RUN_ID.log
+          ip=$(sed -n 's/^\[smoke\] node address: //p' /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
+          [ -n "$ip" ] || { echo "::error::smoke test did not report a node address"; exit 1; }
+          echo "node_ip=$ip" >> "$GITHUB_OUTPUT"
+
+      # The customer path: AMI, key, VPC, instance, SSH in. --benchmark records
+      # host and guest disk numbers so a regression shows up as a number rather
+      # than a story.
+      - name: Workload and benchmark
+        working-directory: mulga/scripts/iso-builder
+        run: |
+          set -euo pipefail
+          cat > /tmp/cell21.env <<EOF
+          NODES=(${{ steps.smoke.outputs.node_ip }})
+          NODE_NAMES=(spinifex-ci)
+          SSH_USER=spinifex
+          SSH_KEY_PATH=$HOME/.ssh/spinifex-workload-key
+          # SSH_KEY_PATH reaches the guests, SSH_IDENTITY reaches the node, and
+          # they are deliberately different keys. Without the second, node SSH
+          # falls back to the runner's default identity, which the node has
+          # never seen.
+          SSH_IDENTITY=$HOME/.ssh/spinifex-key
+          EXT_POOL_START=$EXT_POOL_START
+          EXT_POOL_END=$EXT_POOL_END
+          EXT_GATEWAY=$EXT_GATEWAY
+          EXT_PREFIX_LEN=$EXT_PREFIX_LEN
+          EOF
+          ./test/test-workload.sh --env /tmp/cell21.env --benchmark
+
+      - name: Collect logs
+        if: always()
+        run: |
+          mkdir -p /tmp/cell21-artifacts-$GITHUB_RUN_ID
+          cp /tmp/smoke-$GITHUB_RUN_ID.log /tmp/cell21-artifacts-$GITHUB_RUN_ID/ ||
+            echo "::warning::no smoke log to collect"
+          # fio JSON and the summary table. Warned about rather than swallowed:
+          # a benchmark that ran and produced nothing is worth knowing about.
+          if [ -d "$BENCH_OUT" ]; then
+            cp -r "$BENCH_OUT" /tmp/cell21-artifacts-$GITHUB_RUN_ID/benchmark
+          else
+            echo "::warning::no benchmark results at $BENCH_OUT"
+          fi
+          # The guest's own view. Without it a node that dies mid-test is only
+          # visible as the harness command that stopped producing output.
+          serial=$(sed -n 's/^\[smoke\] serial log preserved: //p' \
+                   /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
+          [ -n "$serial" ] && cp "$serial" /tmp/cell21-artifacts-$GITHUB_RUN_ID/serial.log || true
+          node=$(sed -n 's/^\[smoke\] node logs preserved: //p' \
+                 /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
+          [ -n "$node" ] && cp "$node" /tmp/cell21-artifacts-$GITHUB_RUN_ID/node.log || true
+          # QEMU's stderr no longer reaches the step log: it is redirected so a
+          # kept-alive VM cannot hold the step's stdout open.
+          qemu=$(sed -n 's/^\[smoke\] qemu log preserved: //p' \
+                 /tmp/smoke-$GITHUB_RUN_ID.log | tail -1)
+          [ -n "$qemu" ] && cp "$qemu" /tmp/cell21-artifacts-$GITHUB_RUN_ID/qemu.log || true
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: e2e-artifacts-cell-21-${{ github.run_id }}
+          path: /tmp/cell21-artifacts-${{ github.run_id }}
+          if-no-files-found: warn
+          # Matches the predastore benchmark in cell-23: a night's numbers are
+          # only useful against the nights before them.
+          retention-days: 90
+
+      - name: Teardown
+        if: always()
+        working-directory: mulga/scripts/iso-builder
+        run: |
+          # No env file means the workload step never ran, so there is nothing
+          # of its to clean up.
+          if [ -f /tmp/cell21.env ]; then
+            ./test/test-workload.sh --env /tmp/cell21.env --cleanup || true
+          fi
+          sudo pkill -f spinifex-smoke-test || true
+          rm -rf /tmp/cell21-artifacts-$GITHUB_RUN_ID /tmp/cell21.env \
+                 /tmp/smoke-$GITHUB_RUN_ID.log "$BENCH_OUT"
+          # Runs before BENCH_OUT existed left a timestamped directory behind
+          # every night. Sweep anything older than a day.
+          find /tmp -maxdepth 1 -name 'spinifex-benchmark-*' -type d -mmin +1440 \
+               -exec rm -rf {} + 2>/dev/null || true
+          # KEEP_NODE leaves the node's disk and serial log behind for us.
+          sleep 2
+          rm -f "$TMPDIR"/tmp.*.qcow2 "$TMPDIR"/tmp.*.log "$TMPDIR"/tmp.*-node.log
+
+  iso-multi:
+    name: cell-22 (iso/dhcp/three-node)
+    # After cell-21 rather than beside it: same concurrency group, and running
+    # them together would need 32 GiB on one box.
+    needs: [iso-single]
+    if: >-
+      always() && (
+      github.event_name == 'schedule' ||
+      inputs.cells == '' ||
+      contains(format(',{0},', inputs.cells), ',22,') )
+    runs-on: [self-hosted, ci-multi]
+    timeout-minutes: 90
+    concurrency:
+      group: iso-qemu-rack
+      cancel-in-progress: false
+    env:
+      MEM: 8G
+      SMP: 8
+      PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
+      EXT_POOL_START: 192.168.0.235
+      EXT_POOL_END: 192.168.0.239
+      EXT_GATEWAY: 192.168.1.1
+      EXT_PREFIX_LEN: 23
+    steps:
+      - name: Clean Workspace
+        run: |
+          sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+          # Pinned so the per-node serial logs can be collected afterwards, and
+          # kept beside TMPDIR rather than in /tmp: it also holds nine 40G node
+          # disks, which is what the tmpfs assertion below is about.
+          echo "WORK_DIR=$E2E_ROOT/work/cell22-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          path: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ env.GIT_BRANCH }}
+          path: mulga/spinifex
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ env.GIT_BRANCH }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: mulga/spinifex
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/viperblock
+          ref: ${{ steps.siblings.outputs.viperblock }}
+          path: mulga/viperblock
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: mulga/predastore
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: mulga/northstar
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: mulga/bluebottle
+
+      # 3 x 8 GiB plus 9 sparse 40G disks, before the instances the cluster
+      # launches on top.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          free_mb=$(free -m | awk '/^Mem:/ {print $7}')
+          [ "$free_mb" -ge 26624 ] ||
+            { echo "::error::need 26 GiB available, have ${free_mb} MiB"; exit 1; }
+          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
+          [ "$fstype" != tmpfs ] ||
+            { echo "::error::$TMPDIR is tmpfs — nine node disks would be in RAM"; exit 1; }
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          [ "$avail_gb" -ge 120 ] ||
+            { echo "::error::need 120 GiB on $TMPDIR ($fstype), have ${avail_gb} GiB"; exit 1; }
+
+      - name: Build artefacts
+        working-directory: mulga/scripts/iso-builder
+        run: make build-rootfs build-squashfs build-initramfs
+
+      # Four phases: install three nodes, convert to one cluster, assert it
+      # formed (including the external pool), then pull node 1's power and
+      # assert the survivors hold quorum.
+      #
+      # Phase 5 (test-workload.sh, as cell-21 runs it) is not wired yet: the
+      # nodes are still on user-mode NAT and reachable only through the
+      # harness's own port forwards, and test-cluster.sh does not open the
+      # formation port. Both land with mulga-yxtda and the bridged WAN mode;
+      # until then this cell proves the cluster forms and survives, not that a
+      # customer can use it.
+      - name: Install, form and verify the cluster
+        working-directory: mulga/scripts/iso-builder
+        run: |
+          set -euo pipefail
+          ./test/test-cluster.sh 2>&1 | tee /tmp/cell22-$GITHUB_RUN_ID.log
+
+      - name: Collect logs
+        if: always()
+        run: |
+          mkdir -p /tmp/cell22-artifacts-$GITHUB_RUN_ID
+          cp /tmp/cell22-$GITHUB_RUN_ID.log /tmp/cell22-artifacts-$GITHUB_RUN_ID/ ||
+            echo "::warning::no cluster log to collect"
+          # Serial only. The same directory holds the nodes' disk images, which
+          # are gigabytes and must not go into an artifact.
+          cp "$WORK_DIR"/*-serial.log /tmp/cell22-artifacts-$GITHUB_RUN_ID/ 2>/dev/null ||
+            echo "no per-node serial logs — the run passed and cleaned up after itself"
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: e2e-artifacts-cell-22-${{ github.run_id }}
+          path: /tmp/cell22-artifacts-${{ github.run_id }}
+          if-no-files-found: warn
+          retention-days: 90
+
+      - name: Teardown
+        if: always()
+        run: |
+          sudo pkill -f spinifex-cluster-test || true
+          rm -rf /tmp/cell22-artifacts-$GITHUB_RUN_ID /tmp/cell22-$GITHUB_RUN_ID.log \
+                 "$WORK_DIR"
+
+  # Predastore throughput ("cell 23"). Predastore has no runners of its own and
+  # spinifex depends on it, so the benchmark lives here and checks predastore
+  # out as a sibling.
+  #
+  # needs: is the whole point. A throughput benchmark sharing a host with a cell
+  # measures the neighbour, and ci-single and ci-multi both resolve to
+  # bottlebrush, so "a different label" would not have separated them. needs: on
+  # a matrix job waits for every leg, so this starts on a quiet machine.
+  #
+  # always() because a failed cell must not silently skip the benchmark and
+  # leave a green-looking night with no number in it.
+  performance:
+    name: cell-23 (predastore/throughput)
+    needs: [matrix, baremetal, iso-single, iso-multi]
+    if: >-
+      always() && (
+      github.event_name == 'schedule' ||
+      inputs.cells == '' ||
+      contains(format(',{0},', inputs.cells), ',23,') )
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 60
+    env:
+      # Two-minute samples. The 30s smoke preset is too short to compare nights.
+      PERF_PRESET: compare
+    steps:
+      - name: Clean Workspace
+        run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+          echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
+
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      # Which predastore branch to measure: the same-named branch when one
+      # exists, dev otherwise. Matches how the e2e path already picks siblings,
+      # so a cross-repo change is benchmarked as a set.
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      # The benchmark works in $TMPDIR and predastore refuses writes below a 5%
+      # free-space watermark. Without this the symptom is several hundred
+      # "storage pool is at capacity" lines from warp and no statement of how
+      # much space there actually was.
+      #
+      # TMPDIR is pointed at the runner's disk above, and the tmpfs assertion is
+      # the one that matters: a storage benchmark on a RAM-backed filesystem
+      # measures memcpy. It would report a number that looks excellent, sit in
+      # the baseline, and never notice a disk regression.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
+          case "$fstype" in
+            tmpfs|ramfs)
+              echo "::error::$TMPDIR is $fstype — a storage benchmark there measures RAM"
+              exit 1 ;;
+          esac
+          # Work dirs from runs that were killed before their cleanup trap ran.
+          # -mtime +0 so a concurrent run's directory is never in scope.
+          find "$TMPDIR" -maxdepth 1 -name 'predastore-e2e-performance.*' \
+               -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          pct_used=$(df --output=pcent "$TMPDIR" | tail -1 | tr -dc '0-9')
+          if [ "$pct_used" -ge 90 ] || [ "$avail_gb" -lt 20 ]; then
+            echo "::error::$TMPDIR ($fstype) is ${pct_used}% full with ${avail_gb} GiB free."
+            echo "::error::The benchmark needs headroom above predastore's 5% watermark."
+            exit 1
+          fi
+
+      # PERF_PORT_OFFSET defaults to 10000, so the benchmark's own nodes do not
+      # collide with a cluster left on the default ports.
+      - name: Run throughput benchmark
+        working-directory: predastore
+        run: |
+          set -euo pipefail
+          # Into the results directory, so the artifact reads on its own rather
+          # than needing the job log it was printed into. The benchmark only
+          # ever creates this directory, so a log placed here first survives.
+          mkdir -p "$PERF_RESULTS_ROOT"
+          make e2e-performance 2>&1 | tee "$PERF_RESULTS_ROOT/e2e-performance.log"
+
+      # Report-only. The baseline comparison and the -10% gate are mulga-wpjsx;
+      # until the run-to-run spread is known from real nights, a threshold would
+      # be a guess, and a gate that cries wolf gets muted.
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: predastore-perf-${{ github.run_id }}
+          path: ${{ env.PERF_RESULTS_ROOT }}
+          if-no-files-found: warn
+          retention-days: 90
+
+      - name: Clean up results
+        if: always()
+        run: rm -rf "$PERF_RESULTS_ROOT"
+
+  # Fault injection, not throughput. Cell-23 asks how fast predastore is when
+  # every node is healthy; this asks whether it survives one that is not.
+  #
+  # After cell-23 rather than beside it: both want a quiet single runner, and a
+  # benchmark sharing a machine with a fault injection run measures the wrong
+  # thing. The port offset keeps them apart even so.
+  stress:
+    name: cell-24 (predastore/fault-injection)
+    needs: [matrix, baremetal, iso-single, iso-multi, performance]
+    if: >-
+      always() && (
+      github.event_name == 'schedule' ||
+      inputs.cells == '' ||
+      contains(format(',{0},', inputs.cells), ',24,') )
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: 30
+    env:
+      # Freezes a follower. STRESS_HOST=leader is a known failure: a gate keeps
+      # the frozen node first in its meta read order and pays the full client
+      # timeout per key, so listing slows in proportion to object count.
+      STRESS_HOST: follower
+      # Clear of cell-23's default 10000, so the two never collide on a runner
+      # that lets them overlap.
+      STRESS_PORT_OFFSET: "20000"
+    steps:
+      - name: Clean Workspace
+        run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+          echo "STRESS_RESULTS_ROOT=$E2E_ROOT/predastore-stress-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
+
+      - name: Checkout Spinifex
+        uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - name: Checkout Predastore
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      # A frozen node is indistinguishable from a slow one, so a run on a
+      # loaded or nearly full disk fails as a timeout and reads as a real
+      # fault. The checks are the cheap way to tell those apart afterwards.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          find "$TMPDIR" -maxdepth 1 -name 'predastore-e2e-stress.*' \
+               -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          if [ "$avail_gb" -lt 10 ]; then
+            echo "::error::$TMPDIR has ${avail_gb} GiB free; the run needs headroom above predastore's 5% watermark."
+            exit 1
+          fi
+
+      # Leftovers from a run killed before its cleanup trap: a surviving node
+      # holds its ports and fails this run somewhere unrelated.
+      - name: Clear stale nodes
+        run: pkill -KILL -f '/bin/s3d -config' || true
+
+      - name: Run fault injection
+        working-directory: predastore
+        run: |
+          set -euo pipefail
+          # events.txt carries the timeline and the FAIL line, but not the build
+          # output or anything Warp wrote before the harness took over. The
+          # harness only creates this directory, so a log placed here survives.
+          mkdir -p "$STRESS_RESULTS_ROOT"
+          make e2e-stress 2>&1 | tee "$STRESS_RESULTS_ROOT/e2e-stress.log"
+
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: predastore-stress-${{ github.run_id }}
+          path: ${{ env.STRESS_RESULTS_ROOT }}
+          if-no-files-found: warn
+          retention-days: 90
+
+      - name: Clean up
+        if: always()
+        run: |
+          pkill -KILL -f '/bin/s3d -config' || true
+          rm -rf "$STRESS_RESULTS_ROOT"

.github/workflows/e2e-suites.yml

@@ -394,6 +394,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "E2E ${{ matrix.suite }} results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: single
+
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -796,6 +806,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "Multi-Node E2E ${{ matrix.suite }} results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: multi
+
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze

.github/workflows/e2e.yml

@@ -52,7 +52,7 @@ jobs:
     # inputs.suites.
     if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-iam') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-ecs') || contains(github.event.inputs.suites, 'e2e-storagegrowth') || contains(github.event.inputs.suites, 'e2e-partialblock') || contains(github.event.inputs.suites, 'e2e-rds')
     runs-on: [self-hosted, ci-single]
-    timeout-minutes: 50
+    timeout-minutes: 60
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
@@ -250,53 +250,58 @@ jobs:
           EOF
           ok "kubectl installed on VM"
 
-      # CreateDBInstance boots the DB VM from the rds-postgres AMI, so that image
+      # CreateDBInstance boots the DB VM from an RDS AMI, so both engines' images
       # must be in the VM's image store before the suite runs — same split as the
       # eks-node pair above: build on the runner (needs source + network), import
-      # on the VM (needs predastore/NATS). The suite's psql client needs no step
-      # here: it runs inside a client VM that installs postgresql-client from its
-      # own user-data, because a private-only endpoint is unreachable from the
-      # runner and from the cluster host alike.
-      - name: Build rds-postgres AMI (libguestfs, runner)
+      # on the VM (needs predastore/NATS). The suite's psql and mariadb clients
+      # need no step here: they run inside a client VM that installs them from
+      # its own user-data, because a private-only endpoint is unreachable from
+      # the runner and from the cluster host alike.
+      - name: Build RDS AMIs (libguestfs, runner)
         if: contains(env.SUITES, 'e2e-rds')
         working-directory: spinifex
         run: |
           set -euo pipefail
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          banner "build rds-postgres AMI (libguestfs — no host devices)"
           sudo apt-get update -qq
           sudo apt-get install -y --no-install-recommends libguestfs-tools qemu-utils
           export LIBGUESTFS_BACKEND=direct
           sudo chmod 0644 /boot/vmlinuz-* || true
-          group_open "make build-rds-postgres-image"
-          make build-rds-postgres-image
-          group_close
-          ok "raw: $(du -h /tmp/rds-postgres-image-build/rds-postgres-alpine.raw | cut -f1)"
+          for IMAGE in rds-postgres rds-mariadb; do
+            banner "build ${IMAGE} AMI (libguestfs — no host devices)"
+            group_open "make build-${IMAGE}-image"
+            make "build-${IMAGE}-image"
+            group_close
+            ok "raw: $(du -h "/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" | cut -f1)"
+          done
 
-      - name: Import rds-postgres AMI on VM
+      - name: Import RDS AMIs on VM
         if: contains(env.SUITES, 'e2e-rds')
         run: |
           set -euo pipefail
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
-            spinifex/scripts/images/rds-postgres/manifest.conf)"
-          # SYSTEM_TAG carries three tags, not one: managed-by marks it a system
-          # image and engine/engine-version are what an EngineVersion request
-          # resolves against. --tag is repeatable, so expand rather than pass the
-          # whole string as a single value.
-          TAG_ARGS=""
-          for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
           SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
           VM=tf-user@${{ steps.vm.outputs.wan_ip }}
-          banner "import rds-postgres AMI to VM image store"
-          scp "${SSH_OPTS[@]}" \
-            /tmp/rds-postgres-image-build/rds-postgres-alpine.raw \
-            "$VM":/tmp/rds-postgres-alpine.raw
-          ssh "${SSH_OPTS[@]}" "$VM" \
-            "spx admin images import --file /tmp/rds-postgres-alpine.raw \
-               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
-               --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
-          ok "rds-postgres AMI imported"
+          for IMAGE in rds-postgres rds-mariadb; do
+            eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
+              "spinifex/scripts/images/${IMAGE}/manifest.conf")"
+            # SYSTEM_TAG carries four tags, not one: managed-by marks it a system
+            # image, engine/engine-version are what an EngineVersion request
+            # resolves against, and the contract tag gates the data-volume format
+            # grant. --tag is repeatable, so expand rather than pass the whole
+            # string as a single value.
+            TAG_ARGS=""
+            for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
+            banner "import ${IMAGE} AMI to VM image store"
+            scp "${SSH_OPTS[@]}" \
+              "/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" \
+              "$VM":"/tmp/${IMAGE}-alpine.raw"
+            ssh "${SSH_OPTS[@]}" "$VM" \
+              "spx admin images import --file /tmp/${IMAGE}-alpine.raw \
+                 --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
+                 --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
+            ok "${IMAGE} AMI imported"
+          done
 
       - name: Build E2E test binaries on runner
         working-directory: spinifex/tests/e2e
@@ -383,9 +388,10 @@ jobs:
                echo \"::group::suite \$SUITE\"; \
                printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
+               case \$SUITE in rds) TMO=50m ;; *) TMO=30m ;; esac; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
-                 ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
+                 ./\$SUITE.test -test.v -test.timeout=\$TMO ${TEST_RUN_FLAG} \
                  2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \
                       | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
                rc=\${PIPESTATUS[0]}; \
@@ -458,6 +464,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "E2E results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: single
+
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -897,6 +913,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "Multi-Node E2E results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: multi
+
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze

.github/workflows/gpu-e2e.yml

@@ -287,6 +287,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "GPU E2E results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always() && steps.gate.outputs.paused != 'true'
+        continue-on-error: true
+        uses: ./.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: gpu
+
       - name: Analyze e2e failures
         if: always() && steps.gate.outputs.paused != 'true'
         uses: ./.github/actions/e2e-analyze

.github/workflows/publish-system-ami.yml

@@ -1,22 +1,27 @@
-name: Publish EKS + ECS node AMIs
+name: Publish System AMIs
 
 # Builds the system AMIs (libguestfs, on the runner) and publishes each
 # compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket that backs
-# iso.mulgadc.com/system-ami/. Four images ship:
+# iso.mulgadc.com/system-ami/. Six images ship:
 #   spinifex-eks-node       Alpine EKS K3s node (server|agent, role at boot)
 #   spinifex-ecs-node       Alpine ECS container instance
 #   spinifex-eks-node-gpu   Ubuntu EKS K3s GPU worker (NVIDIA driver + toolkit)
 #   spinifex-ecs-node-gpu   Ubuntu ECS GPU container instance
+#   spinifex-rds-postgres   Alpine RDS PostgreSQL instance
+#   spinifex-rds-mariadb    Alpine RDS MariaDB instance
 # Operators then install them on any cluster with:
 #   spx admin images import --name spinifex-eks-node
 #   spx admin images import --name spinifex-ecs-node
 #   spx admin images import --name spinifex-eks-node-gpu
 #   spx admin images import --name spinifex-ecs-node-gpu
+#   spx admin images import --name spinifex-rds-postgres
+#   spx admin images import --name spinifex-rds-mariadb
 #
-# Manual-only: publishing overwrites the mirror artifact, so it is a deliberate
-# action, not something every push should trigger. The `variant` input scopes
-# which images build — the Ubuntu GPU legs are 16G and much slower (NVIDIA
-# driver download + DKMS build), so publishing only the CPU AMIs stays cheap.
+# Publishing overwrites the mirror artifact, so it is a deliberate action, not
+# something every push should trigger: it runs manually, or from release.yml
+# once a tag has passed its gates. The `variant` input scopes which images
+# build — the Ubuntu GPU legs are 16G and much slower (NVIDIA driver download +
+# DKMS build), so publishing only the CPU AMIs stays cheap.
 
 on:
   workflow_dispatch:
@@ -29,6 +34,17 @@ on:
           - all
           - cpu
           - gpu
+          - rds
+      dry_run:
+        description: "Build + convert only; skip the R2 upload"
+        type: boolean
+        default: false
+  workflow_call:
+    inputs:
+      variant:
+        description: "Which images to build (all|cpu|gpu|rds)"
+        type: string
+        default: all
       dry_run:
         description: "Build + convert only; skip the R2 upload"
         type: boolean
@@ -56,10 +72,12 @@ jobs:
           set -euo pipefail
           cpu='{"name":"spinifex-eks-node","manifest":"scripts/images/eks-node/manifest.conf"},{"name":"spinifex-ecs-node","manifest":"scripts/images/ecs-agent/manifest.conf"}'
           gpu='{"name":"spinifex-eks-node-gpu","manifest":"scripts/images/eks-node-gpu/manifest.conf"},{"name":"spinifex-ecs-node-gpu","manifest":"scripts/images/ecs-agent-gpu/manifest.conf"}'
+          rds='{"name":"spinifex-rds-postgres","manifest":"scripts/images/rds-postgres/manifest.conf"},{"name":"spinifex-rds-mariadb","manifest":"scripts/images/rds-mariadb/manifest.conf"}'
           case "${VARIANT}" in
             cpu) items="${cpu}" ;;
             gpu) items="${gpu}" ;;
-            *)   items="${cpu},${gpu}" ;;
+            rds) items="${rds}" ;;
+            *)   items="${cpu},${gpu},${rds}" ;;
           esac
           echo "images={\"include\":[${items}]}" >> "$GITHUB_OUTPUT"
 
@@ -94,6 +112,20 @@ jobs:
           ref: dev
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: dev
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: dev
+          path: bluebottle
+
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
@@ -111,7 +143,7 @@ jobs:
         # "go.work already exists".
         run: |
           rm -f go.work go.work.sum
-          go work init ./spinifex ./viperblock ./predastore
+          go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Install build + upload tooling
         run: |

.github/workflows/release.yml

@@ -28,13 +28,25 @@ jobs:
           repository: mulgadc/viperblock
           path: viperblock
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          path: bluebottle
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
 
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
-          go-version: "1.26.5"
+          go-version: "1.27.0"
 
       - name: Build microVM artifacts (lb-agent + kernel + initramfs)
         run: cd spinifex && make build-microvm-image
@@ -73,6 +85,18 @@ jobs:
           repository: mulgadc/viperblock
           path: viperblock
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          path: bluebottle
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
 
@@ -207,6 +231,17 @@ jobs:
             echo "Release ${TAG} promoted to latest"
           fi
 
+  # Refresh the system AMIs on the R2 mirror only once the release is out of
+  # draft. Shares the ci-single runner with `iso`, so it queues behind it.
+  system-images:
+    needs: promote
+    permissions:
+      contents: read
+    uses: ./.github/workflows/publish-system-ami.yml
+    with:
+      variant: all
+    secrets: inherit
+
   # Annotate the (still-draft) release if either gate failed.
   mark-failed:
     needs: [ verify, baremetal ]
@@ -268,6 +303,18 @@ jobs:
           repository: mulgadc/viperblock
           path: viperblock
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          path: bluebottle
+
       - name: Checkout Mulga iso-builder
         uses: actions/checkout@v7
         with:
@@ -286,7 +333,7 @@ jobs:
           cache: false
 
       - name: Configure Go workspace
-        run: go work init ./spinifex ./predastore ./viperblock
+        run: go work init ./spinifex ./predastore ./viperblock ./northstar ./bluebottle
 
       - name: Install ISO builder dependencies
         run: |

.github/workflows/segscan-oracle.yml

@@ -51,6 +51,20 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
       - uses: actions/create-github-app-token@v3
         id: app-token
         with:
@@ -73,7 +87,7 @@ jobs:
           go-version-file: "spinifex/go.mod"
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Run Segscan Oracle Test
         env:

.github/workflows/spinifex.yml

@@ -53,14 +53,65 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
+      # the last go.sum change and never refreshed it. Both caches are managed
+      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
-          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
+          cache: false
+
+      # Module contents follow the five go.sum files, so a content key is exact.
+      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
+      # dependencies out of the key and re-downloading on every run.
+      - name: Cache Go modules
+        uses: actions/cache@v6
+        with:
+          path: ~/go/pkg/mod
+          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
+          restore-keys: gomod-
+
+      # Build artifacts follow source, so the key rolls per run and falls back to
+      # this job's newest entry. Keyed per job because -race, -covermode=atomic
+      # and plain builds produce disjoint entries and would evict each other.
+      - name: Cache Go build
+        uses: actions/cache@v6
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
+
+      # The EBS provider contract suite needs a real implementation on each
+      # side of the boundary: qemu-img/qemu-nbd/qemu-io back qemunbdd, which
+      # shares no code with viperblock, and nbdkit plus viperblock's plugin
+      # back viperblockd, the provider production actually uses. Without them
+      # the suite runs against MemoryProvider alone. nbdkit-plugin-dev carries
+      # the headers and nbdkit.pc that cgo needs to build that plugin, and
+      # libnbd-bin supplies the third-party NBD client (nbdinfo/nbdcopy) that
+      # judges the published export by something outside this codebase.
+      - name: Install conformance tools
+        run: sudo apt-get update && sudo apt-get install -y --no-install-recommends qemu-utils nbdkit nbdkit-plugin-dev libnbd-bin
+
+      - name: Build viperblock nbdkit plugin
+        run: make -C viperblock go_build_nbd
 
       - name: Build Spinifex
         run: make -C spinifex build
@@ -69,10 +120,13 @@ jobs:
         run: make -C spinifex manifest-check manifest-lint
 
       - name: Run Tests with Coverage
-        run: make -C spinifex test-cover
+        env:
+          SPINIFEX_REQUIRE_CONFORMANCE_TOOLS: "1"
+          VIPERBLOCK_NBDKIT_PLUGIN: ${{ github.workspace }}/viperblock/lib/nbdkit-viperblock-plugin.so
+        run: make -C spinifex test-cover GOTESTFLAGS=-count=1
 
       - name: Run Action Tests
-        run: make -C spinifex test-actions
+        run: make -C spinifex test-actions GOTESTFLAGS=-count=1
 
       - name: Check Diff Coverage
         run: cd spinifex && scripts/diff-coverage.sh coverage.out --base HEAD~1
@@ -114,14 +168,51 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
+      # the last go.sum change and never refreshed it. Both caches are managed
+      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
-          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
+          cache: false
+
+      # Module contents follow the five go.sum files, so a content key is exact.
+      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
+      # dependencies out of the key and re-downloading on every run.
+      - name: Cache Go modules
+        uses: actions/cache@v6
+        with:
+          path: ~/go/pkg/mod
+          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
+          restore-keys: gomod-
+
+      # Build artifacts follow source, so the key rolls per run and falls back to
+      # this job's newest entry. Keyed per job because -race, -covermode=atomic
+      # and plain builds produce disjoint entries and would evict each other.
+      - name: Cache Go build
+        uses: actions/cache@v6
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Run Integration Tests
         run: make -C spinifex test-integration
@@ -157,17 +248,54 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
+      # the last go.sum change and never refreshed it. Both caches are managed
+      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
-          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
+          cache: false
+
+      # Module contents follow the five go.sum files, so a content key is exact.
+      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
+      # dependencies out of the key and re-downloading on every run.
+      - name: Cache Go modules
+        uses: actions/cache@v6
+        with:
+          path: ~/go/pkg/mod
+          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
+          restore-keys: gomod-
+
+      # Build artifacts follow source, so the key rolls per run and falls back to
+      # this job's newest entry. Keyed per job because -race, -covermode=atomic
+      # and plain builds produce disjoint entries and would evict each other.
+      - name: Cache Go build
+        uses: actions/cache@v6
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Run Tests with Race Detector
-        run: make -C spinifex test-race
+        run: make -C spinifex test-race GOTESTFLAGS=-count=1
 
   lint_and_security:
     name: Lint and Security
@@ -200,19 +328,56 @@ jobs:
           ref: ${{ steps.siblings.outputs.predastore }}
           path: predastore
 
+      - name: Checkout Northstar
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/northstar
+          ref: ${{ steps.siblings.outputs.northstar }}
+          path: northstar
+
+      - name: Checkout Bluebottle
+        uses: actions/checkout@v7
+        with:
+          repository: mulgadc/bluebottle
+          ref: ${{ steps.siblings.outputs.bluebottle }}
+          path: bluebottle
+
+      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
+      # the last go.sum change and never refreshed it. Both caches are managed
+      # below instead.
       - name: Setup Go
         uses: actions/setup-go@v7
         with:
-          cache-dependency-path: "spinifex/go.sum"
           go-version-file: "spinifex/go.mod"
+          cache: false
+
+      # Module contents follow the five go.sum files, so a content key is exact.
+      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
+      # dependencies out of the key and re-downloading on every run.
+      - name: Cache Go modules
+        uses: actions/cache@v6
+        with:
+          path: ~/go/pkg/mod
+          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
+          restore-keys: gomod-
+
+      # Build artifacts follow source, so the key rolls per run and falls back to
+      # this job's newest entry. Keyed per job because -race, -covermode=atomic
+      # and plain builds produce disjoint entries and would evict each other.
+      - name: Cache Go build
+        uses: actions/cache@v6
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          restore-keys: gobuild-${{ github.job }}-
 
       - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore
+        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Lint
         uses: golangci/golangci-lint-action@v9
         with:
-          version: v2.12.2
+          version: v2.13.0
           working-directory: spinifex
 
       - name: Govulncheck

.gitignore

@@ -34,9 +34,10 @@ Thumbs.db
 # Environment
 .env
 .venv/
+.cache/
+
+# VScode
+.vscode/
 
 # Test reports
 *-report.txt
-
-# Per-worktree golangci-lint cache
-/.cache/

.golangci.yml

@@ -26,7 +26,7 @@ linters:
     - errorlint # catches ==/!= on errors, type assertions on errors, and %v instead of %w — all break on wrapped errors
     - exptostd # detects golang.org/x/exp/ functions replaceable by stdlib
     - fatcontext # detects nested contexts in loops
-    - forbidigo # blocks the legacy JetStream/KV symbols
+    - forbidigo # blocks forbidden identifiers
     - gocheckcompilerdirectives # validates go compiler directive comments (//go:)
     - godot # checks that declaration comments end in a period
     - goprintffuncname # checks that printf-like functions are named with f at the end
@@ -69,6 +69,18 @@ linters:
             - pkg: math/rand$
               desc: Use math/rand/v2 instead, see https://go.dev/blog/randv2
 
+        # The EC2 control plane reaches block storage through ebsprovider, not
+        # by importing the engine. viperblock.New starts a chunk uploader, so
+        # every construction is a writer against a volume viperblockd serves.
+        # Nothing under handlers/ may import viperblock, in any form, tests
+        # included. A types-only import counts: one became an engine import.
+        ec2-control-plane:
+          files:
+            - '**/spinifex/handlers/**'
+          deny:
+            - pkg: github.com/mulgadc/viperblock
+              desc: Use the ebsprovider.EBSProvider interface; the control plane must not construct or name viperblock types.
+
     embeddedstructfieldcheck:
       forbid-mutex: true
 
@@ -80,6 +92,8 @@ linters:
     forbidigo:
       analyze-types: true
       forbid:
+        - pattern: '^panic$'
+          msg: panic terminates the process; return an error or explicitly justify the panic.
         - pattern: '\.JetStream$'
           pkg: '^github\.com/nats-io/nats\.go$'
           msg: legacy nc.JetStream() is superseded; construct with jetstream.New(nc).
@@ -104,6 +118,7 @@ linters:
 
     modernize:
       disable:
+        - embedlit # its fix promotes embedded fields into the outer literal, which does not compile
         - newexpr # don't replace aws.String() with new() — AWS SDK idiom
         - stringsbuilder # don't replace small string concatenations with strings.Builder
 
@@ -154,6 +169,7 @@ linters:
           - bodyclose
           - errcheck
           - errchkjson
+          - forbidigo
           - gosec
 
       # spinifex-specific: gosec exclusions

Makefile

@@ -1,10 +1,10 @@
 GO_PROJECT_NAME := spx
 SHELL := /bin/bash
 
-# golangci-lint's default cache (~/.cache/golangci-lint) is keyed by module
-# path, not worktree path, so parallel worktrees of this module collide and
-# surface each other's issues. Scope it per worktree.
-export GOLANGCI_LINT_CACHE := $(CURDIR)/.cache/golangci-lint
+# GOFIPS140 is a global build-config change: it alters every package's action ID,
+# stdlib included, so a target that omits it shares no build-cache entry with one
+# that sets it. Exported once here rather than per-recipe so the two can't drift.
+export GOFIPS140 := v1.0.0
 
 # Detect architecture for cross-platform support
 ARCH := $(shell uname -m)
@@ -49,7 +49,7 @@ else
   _SECQ  = 2>&1 | tee
 endif
 
-build: go_build build-installer build-lb-agent
+build: go_build build-installer build-lb-agent generate-aws-model-coverage
 
 # Build spinifex-ui frontend (requires pnpm)
 build-ui:
@@ -63,19 +63,23 @@ LDFLAGS := -s -w -X github.com/mulgadc/spinifex/cmd/spinifex/cmd.Version=$(VERSI
 
 go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
-	GOFIPS140=v1.0.0 go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
+	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
 
 build-installer:
 	@echo -e "\n....Building spinifex-installer"
-	GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
+	go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
 
 build-lb-agent:
 	@echo -e "\n....Building lb-agent (static)"
-	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
+	CGO_ENABLED=0 go build -ldflags "-s -w" -o ./bin/lb-agent cmd/lb-agent/main.go
 
 build-ecs-agent: ## Build the ecs-agent (ships in the ECS-AMI guest; not in host `build`)
 	@echo -e "\n....Building ecs-agent (static)"
-	CGO_ENABLED=0 GOFIPS140=v1.0.0 go build -ldflags "-s -w -X main.version=$(VERSION)" -o ./bin/ecs-agent ./cmd/ecs-agent
+	CGO_ENABLED=0 go build -ldflags "-s -w -X main.version=$(VERSION)" -o ./bin/ecs-agent ./cmd/ecs-agent
+
+build-loadgen: ## Build spx-loadgen (drives the stress harness's measured load; not shipped to nodes)
+	@echo -e "\n....Building spx-loadgen"
+	GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/spx-loadgen ./cmd/spx-loadgen
 
 build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-node)
 ifndef IMAGE
@@ -111,6 +115,12 @@ build-rds-postgres-image: ## Build the spinifex-rds-postgres AMI (Alpine + Postg
 import-rds-postgres-image: ## Build + register the rds-postgres AMI (requires a running cluster)
 	$(MAKE) build-system-image IMAGE=rds-postgres IMPORT=1
 
+build-rds-mariadb-image: ## Build the spinifex-rds-mariadb AMI (Alpine + MariaDB 11.8 + rds-init; IMPORT=1 to register)
+	$(MAKE) build-system-image IMAGE=rds-mariadb
+
+import-rds-mariadb-image: ## Build + register the rds-mariadb AMI (requires a running cluster)
+	$(MAKE) build-system-image IMAGE=rds-mariadb IMPORT=1
+
 MICROVM_OUT_DIR := build/microvm
 MICROVM_ARTIFACTS := $(MICROVM_OUT_DIR)/vmlinuz $(MICROVM_OUT_DIR)/initramfs.cpio.gz
 MICROVM_INPUTS := scripts/build-microvm-image.sh $(MICROVM_OUT_DIR)/init.sh $(MICROVM_OUT_DIR)/inittab bin/lb-agent
@@ -134,9 +144,22 @@ install-microvm: $(MICROVM_ARTIFACTS) ## Install microVM artifacts to /usr/share
 # the pre-commit gate: manifest checks, lint, vuln, and the unit and e2e-harness tiers.
 # integration and race tests skipped to keep quick, they run in CI.
 preflight:
-	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-harness
+	@$(MAKE) --no-print-directory QUIET=1 manifest-check manifest-lint lint govulncheck test-cover diff-coverage test-package-check test-harness test-build-scripts
 	@echo -e "\n ✅ Preflight passed — safe to commit."
 
+# Shell suites + shellcheck for build/scripts/, the systemd-unit helpers that
+# ship on every node (unlike scripts/images/, kept in preflight: a wrong
+# restart decision here is an outage, not asset churn).
+test-build-scripts:
+	@echo -e "\n....Running build/scripts/**/*_test.sh...."
+	@for t in $$(find build/scripts -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		sh "$$t" || exit 1; \
+	done
+	@echo -e "\n....Running shellcheck over build/scripts/**/*.sh...."
+	shellcheck -S warning $$(find build/scripts -name '*.sh' | sort)
+	@echo "  test-build-scripts ok"
+
 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
 # default `go test ./spinifex/...`. Runs with mocked AWS clients — no
 # infrastructure required, safe to run in CI without a cluster.
@@ -146,9 +169,26 @@ test-harness:
 
 # In-process integration tier: the real gateway router against embedded NATS
 # JetStream, with only the daemon-side NATS subjects stubbed.
+AWS_MODEL_CONFORMANCE_REPORT ?= $(CURDIR)/.cache/aws-model-conformance-report.txt
+AWS_MODEL_CONFORMANCE_MODE ?= fail
+AWS_MODEL_OPERATION_COVERAGE_REPORT ?= $(CURDIR)/docs/aws-model-operation-coverage.md
+# -count=1 is load-bearing: the conformance report is written by the test binary,
+# so a cached pass skips the run, leaves no report, and the cat below fails the
+# target. It also keeps the conformance gate honest — a cached result would mean
+# the check never ran against this commit.
 test-integration:
 	@echo -e "\n....Running in-process integration tests...."
-	$(_Q)LOG_IGNORE=1 go test -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -count=1 -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
+	@cat $(AWS_MODEL_CONFORMANCE_REPORT)
+	@$(MAKE) --no-print-directory generate-aws-model-coverage
+	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_REPORT)"
+
+generate-aws-model-coverage:
+	@mkdir -p $(dir $(AWS_MODEL_OPERATION_COVERAGE_REPORT))
+	@go run ./cmd/aws-model-coverage > $(AWS_MODEL_OPERATION_COVERAGE_REPORT)
+
+aws-model-coverage: generate-aws-model-coverage
+	@cat $(AWS_MODEL_OPERATION_COVERAGE_REPORT)
 
 # Segscan storage oracle: needs the mulga umbrella repo's scripts/segscan
 # checked out alongside spinifex (see spinifex/testutil/segscanoracle), which
@@ -179,44 +219,72 @@ manifest-lint-update:
 # Run unit tests
 test:
 	@echo -e "\n....Running tests for $(GO_PROJECT_NAME)...."
-	GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/...
+	LOG_IGNORE=1 go test -timeout 180s ./spinifex/... ./cmd/... ./internal/...
+
+# Empty locally, where reusing a cached result between runs is the point.
+# CI passes -count=1 so a green run means the suite executed against that commit
+# rather than replaying results the persisted build cache carried over.
+GOTESTFLAGS ?=
 
 # Run unit tests with coverage profile
 COVERPROFILE ?= coverage.out
 test-cover:
 	@echo -e "\n....Running tests with coverage for $(GO_PROJECT_NAME)...."
-	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 180s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... ./cmd/... ./internal/... $(_COVQ)
 	@scripts/check-coverage.sh $(COVERPROFILE) $(QUIET)
 
 # Run unit tests with race detector
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
-	$(_Q)GOFIPS140=v1.0.0 LOG_IGNORE=1 go test -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... $(_RACEQ)
 
 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
 test-actions:
 	@echo -e "\n....Running action tests...."
-	LOG_IGNORE=1 go test -timeout 60s ./.github/actions/...
+	LOG_IGNORE=1 go test $(GOTESTFLAGS) -timeout 60s ./.github/actions/...
 
-# Shell suites + shellcheck for scripts/images/ helpers baked into system
-# images. Kept out of `preflight` (a dedicated CI job gates it on
-# scripts/images/** changes instead) so image-asset churn doesn't run on
-# every Go contributor's commit.
+# Shell suites + shellcheck for scripts/images/ and images/mkosi.profiles/
+# helpers baked into system images. Kept out of `preflight` (a dedicated CI
+# job gates it on scripts/images/**+images/mkosi.profiles/** changes instead)
+# so image-asset churn doesn't run on every Go contributor's commit.
+#
+# images/mkosi.profiles/ ships scripts with no .sh suffix (mkosi's own
+# mkosi.*.chroot lifecycle hooks, and wrapper binaries like vllm-serve under
+# mkosi.extra/) so a plain `-name '*.sh'` glob silently skips exactly the
+# files most worth linting. Sources there are found by shebang instead of
+# name; this also means a profile that ships no scripts at all yields an
+# empty match rather than an error.
 test-images:
 	@echo -e "\n....Running scripts/images/**/*_test.sh...."
 	@for t in $$(find scripts/images -name '*_test.sh' | sort); do \
 		echo "-- $$t"; \
 		bash "$$t" || exit 1; \
 	done
+	@echo -e "\n....Running images/mkosi.profiles/**/*_test.sh...."
+	@for t in $$(find images/mkosi.profiles -name '*_test.sh' | sort); do \
+		echo "-- $$t"; \
+		bash "$$t" || exit 1; \
+	done
 	@echo -e "\n....Running shellcheck over scripts/images/**/*.sh...."
 	shellcheck -S warning $$(find scripts/images -name '*.sh' | sort)
+	@echo -e "\n....Running shellcheck over images/mkosi.profiles/** shell sources...."
+	@srcs="$$(grep -rlIE '^#!/bin/(bash|sh)' images/mkosi.profiles 2>/dev/null | sort)"; \
+	if [ -n "$$srcs" ]; then \
+		shellcheck -S warning $$srcs; \
+	else \
+		echo "  (no shell sources found under images/mkosi.profiles)"; \
+	fi
 	@echo "  test-images ok"
 
 # Check that new/changed code meets coverage threshold (runs tests first)
 diff-coverage: test-cover
 	@QUIET=$(QUIET) scripts/diff-coverage.sh $(COVERPROFILE)
 
+# Check that newly added test files use an external test package
+test-package-check:
+	@QUIET=$(QUIET) scripts/check-test-package.sh
+
 bench:
 	@echo -e "\n....Running benchmarks for $(GO_PROJECT_NAME)...."
 	LOG_IGNORE=1 go test -benchmem -run=. -bench=. ./...
@@ -228,6 +296,20 @@ bench:
 # `make install-microvm` explicitly the first time.
 deploy: build
 	sudo install -m 755 bin/spx /usr/local/bin/spx
+	@if [ "$${SKIP_ASSET_PREFLIGHT:-}" != "1" ]; then \
+		if ! sudo /usr/local/bin/spx admin preflight; then \
+			echo ""; \
+			echo "[deploy] host-asset preflight failed: this node's helper scripts or sudoers"; \
+			echo "  grants don't match the binary just installed. Remediate with:"; \
+			echo "    scripts/update-nodes.sh   (from the mulga root)"; \
+			echo "  or:"; \
+			echo "    make reinstall"; \
+			echo "  Set SKIP_ASSET_PREFLIGHT=1 to bypass for a deliberate binary-only push."; \
+			exit 1; \
+		fi; \
+	else \
+		echo "[deploy] SKIP_ASSET_PREFLIGHT=1 — skipping host-asset preflight"; \
+	fi
 	@if [ -f $(MICROVM_OUT_DIR)/vmlinuz ]; then \
 		$(MAKE) install-microvm; \
 	else \
@@ -257,9 +339,9 @@ install-system:
 		ovn-central ovn-host openvswitch-switch dhcpcd-base
 
 install-go:
-	@echo -e "\n....Installing Go 1.26.5 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.27.0 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.26.5.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.27.0.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi
@@ -342,7 +424,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-harness test-integration test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
+.PHONY: test-package-check build build-ui build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image install-microvm go_build preflight test test-cover test-race diff-coverage bench test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle manifest-check manifest-lint manifest-lint-update \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

README.md

@@ -3,19 +3,20 @@
 </p>
 
 <p align="center">
-  <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.26+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
+  <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.27+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
-  <a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/Docs-docs.mulgadc.com-DC2626?style=flat-square" alt="Docs"></a>
+  <a href="https://mulgadc.com"><img src="https://img.shields.io/badge/home-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
 </p>
 
 <p align="center">
   <a href="#what-is-spinifex">What is Spinifex?</a> ·
   <a href="#the-platform">Platform</a> ·
   <a href="#three-ways-to-deploy">Deploy</a> ·
-  <a href="#aws-compatibility">AWS services</a> ·
+  <a href="#aws-compatibility">AWS Compatibility</a> ·
+  <a href="#live-demo">Live Demo</a> ·
   <a href="#core-components">Components</a> ·
   <a href="#architecture-at-a-glance">Architecture</a> ·
-  <a href="#installation">Installation</a> ·
+  <a href="#self-hosted-installation">Installation</a> ·
   <a href="https://docs.mulgadc.com">Docs</a>
 </p>
 
@@ -23,26 +24,15 @@
 
 # Spinifex: the open, AWS-compatible cloud you run yourself
 
-Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your
-software already uses (EC2, EBS, S3, VPC, IAM) on infrastructure you control.
-Move workloads off the hyperscalers, run AI at a fraction of the cost, or take
-the cloud to the edge, without rewriting a line. The exact build you ship to AWS,
-Spinifex serves on your own hardware: same APIs, same Terraform, same SDKs.
+Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your software already uses (EC2, EBS, S3, VPC, IAM) on infrastructure you control. Move workloads off the hyperscalers, run AI at a fraction of the cost, or take the cloud to the edge, without rewriting a line. The exact build you ship to AWS, Spinifex serves on your own hardware: same APIs, same Terraform, same SDKs.
 
 ## What is Spinifex?
 
-Spinifex is an open-source infrastructure platform that recreates the AWS service
-surface (EC2, EBS, S3, VPC, IAM) on bare-metal, edge, and on-premises
-environments. Run cloud-native software without a hyperscaler.
+Spinifex is an open-source infrastructure platform that recreates the AWS service surface (EC2, EBS, S3, VPC, IAM) on bare-metal, edge, and on-premises environments. Run cloud-native software without a hyperscaler.
 
-Most AWS alternatives wrap an API and call it a cloud. We rebuilt the engineering
-underneath: distributed object storage with erasure coding, block storage built
-to survive failure, and compute on bare metal. That depth is the reason your
-software runs unchanged, instead of rewritten.
+Built for teams that need:
 
-It's built for teams that need:
-
-- The AWS tooling they already use (AWS CLI, SDKs, Terraform), with nothing to rewrite
+- The AWS tooling you already use (AWS CLI, SDKs, Terraform), with nothing to rewrite
 - Full control of the stack: your hardware, your network, your data, your keys
 - A cloud that keeps running offline, through disconnection, with no external control plane
 - An open core (AGPL-3.0), auditable and yours to keep, with no lock-in
@@ -57,63 +47,69 @@ You change *where* your software runs, not *what* it is.
 
 From commodity hardware up to unmodified AWS tooling, every layer is replaceable and yours to own.
 
-## Three ways to deploy
+## Three Ways To Deploy
 
-One AWS-compatible surface, three deployment shapes. Pick the one that matches
-your reality; the platform on top is the same.
+The same AWS-compatible platform runs in three shapes. Pick the one that matches your hardware.
 
 ### Neocloud
 
-Lift and shift onto a partner Neocloud. Move workloads off the hyperscalers and
-onto our Neocloud partner ecosystem without rewriting them. GPU capacity
-(H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the
-software.
+Lift and shift onto a partner Neocloud without rewriting workloads. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
 
 ### On-premise
 
-Bring your own hardware and host Spinifex in your own data centre. A real
-multi-node HA cluster integrated with your storage and networking, with full
-control of stack, data, and jurisdiction. A predictable bill, no egress
-surprises, an auditable open-source core.
+Host Spinifex on your own hardware in your own data centre. You get a multi-node HA cluster integrated with your storage and networking, so the stack, the data, and the jurisdiction stay yours.
 
 ### Edge
 
-Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels,
-factories, and clinics. Compute next to the data, running through disconnection,
-on hardware you own. The same AWS APIs your software already uses.
+Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels, factories, and clinics. Compute next to the data, running through disconnection, on hardware you own. The same AWS APIs your software already uses.
 
-## AWS compatibility
+## AWS Compatibility
 
-Speak the AWS API surface, natively. The AWS SDKs, AWS CLI, and Terraform:
-everything you deploy on AWS deploys on Spinifex unchanged. At the edge,
-on-premise, or on a partner Neocloud.
+Spinifex speaks the AWS API surface. The AWS SDKs, AWS CLI, and Terraform all work unchanged, whether you deploy at the edge, on-premise, or on a partner Neocloud.
 
 | Service | What it is | Status |
 | --- | --- | --- |
 | **EC2** | Compute | Available |
-| **EBS** | Block storage | Available |
-| **S3** | Object storage | Available |
+| **EBS** | Block Storage | Available |
+| **S3** | Object Storage | Available |
 | **VPC** | Networking | Available |
-| **IAM** | Identity & auth | Available |
-| **ALB / NLB** | Load balancers | Available |
+| **IAM** | Identity & Auth | Available |
+| **ALB / NLB** | Load Balancers | Available |
 | **EKS** | Kubernetes | Available |
-| **ECR** | Container registry | Available |
-| **ECS** | Container service | Available |
+| **ECR** | Container Registry | Available |
+| **ECS** | Container Service | Available |
 | **RDS** | Databases | Available |
 | **Bedrock** | AI Deployment | Q3 2026 |
 
-Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land.
-[Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
+Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
+
+## Live Demo
+
+Create a live sandbox in under a minute. The fastest way to experience the stack. Try Spinifex against one of your own workloads on infrastructure we run for you.
+
+* Live in under a minute. No card, no install.
+* Yours for 72 hours to test and experiment.
+* Real AWS-compatible endpoint, access keys and console.
+* EC2, EBS, S3, VPC, IAM, EKS and RDS, ready to call.
+* Point your existing AWS CLI or Terraform straight at it.
+
+Sign up at [https://mulgadc.com/signup](https://mulgadc.com/signup)
 
 ## Core Components
 
-### Spinifex (Compute Service – EC2 Alternative)
+### Spinifex (Compute and Platform Services – EC2, EKS, ECS, RDS)
 
-Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs similar to EC2. It manages lifecycle operations like start, stop, and terminate, using QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs using the AWS CLI, SDKs, or Terraform—without needing Kubernetes or heavyweight orchestrators. Keep in mind, you can also setup a Kubernetes environment using Spinifex with underlying instances.
+Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing an EC2 compatible API. It manages lifecycle operations like start, stop, and terminate through QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs with the AWS CLI, SDKs, or Terraform, without Kubernetes or a heavyweight orchestrator.
 
-- EC2-like VM management on bare metal
-- Launches with cloud-init metadata support
-- Works with standard AWS tooling
+The managed services build on that same instance layer. An EKS node group, an ECS container instance, and an RDS database are Spinifex VMs on your own hardware, each driven through its own AWS API.
+
+- **EC2** – VM lifecycle on bare metal, cloud-init and IMDS metadata, key pairs, tags, spot instances, capacity reservations, and GPU passthrough
+- **EKS** – Managed Kubernetes on k3s, with HA control planes, managed node groups, cluster add-ons, an OIDC provider, and IAM access entries
+- **ECS** – Task and service scheduling across container instances, with awsvpc ENIs, capacity providers, rolling deployments, and GPU-aware placement
+- **ECR** – Private container registry backed by Predastore, with content-addressed blobs deduplicated per account and lifecycle policies
+- **RDS** – Managed PostgreSQL and MariaDB, with parameter groups, automated backups, and DNS endpoints served by Northstar
+- **ALB / NLB** – Layer 7 load balancing on HAProxy and layer 4 on nginx `stream`, with health checks and ACM-issued listener certificates
+- **VPC and IAM** – OVN-backed networking and security groups, with roles, policies, and SigV4 auth applied across every service
 
 ### Viperblock (Block Storage – EBS Alternative)
 
@@ -122,41 +118,36 @@ Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs
 - Fast, durable virtual disks
 - Replication for resilience
 - Exposed over NBD or embedded in VMs
-- Supports high performance WAL logs using local NVMe drives to reduce IO traffic to S3.
-- In memory read/write block cache for blazing performance.
+- High performance WAL logs on local NVMe drives, reducing IO traffic to S3
+- In-memory read/write block cache
 
 ### Predastore (Object Storage – S3-Compatible)
 
-[Predastore](https://github.com/mulgadc/predastore) is a fully S3-compatible object storage system. It supports the AWS S3 API, including Signature V4 authentication, multipart uploads, and Terraform provisioning. Data is chunked and distributed across nodes using Reed-Solomon erasure coding, making it fault-tolerant and ideal for large-scale or low-bandwidth scenarios.
+[Predastore](https://github.com/mulgadc/predastore) is a fully S3-compatible object storage system. It supports the AWS S3 API, including Signature V4 authentication, multipart uploads, and Terraform provisioning. Predastore chunks data and distributes it across nodes using Reed-Solomon erasure coding, so it survives node loss and suits large-scale or low-bandwidth sites.
 
 - S3-compatible API and auth
 - Multipart uploads, streaming reads/writes
 - Data redundancy with Reed-Solomon encoding
 
+### Northstar (Authoritative DNS)
+
+[Northstar](https://github.com/mulgadc/northstar) is a lightweight authoritative DNS server that handles both internal service discovery and public-facing DNS for a Spinifex cluster. Zones are human-readable TOML files, served from the local filesystem or synced from Predastore, keeping DNS configuration in the same object storage as the rest of your infrastructure.
+
+- Authoritative DNS over UDP, TCP, and DNS-over-TLS
+- Service discovery for cluster components via SRV records
+- Zone files stored locally or in S3-compatible storage
+
 ## Architecture at a Glance
 
 <p align="center">
   <img src=".github/assets/architecture.svg" alt="Spinifex message-driven architecture: standard AWS clients call the AWS Gateway over HTTPS with SigV4, which publishes to a NATS bus answered by stateless ec2/ebs/s3/vpc/iam daemons over local backends." width="900">
 </p>
 
-Every AWS API call is authenticated at the gateway, published to a NATS subject,
-and answered by whichever daemon claims it. Daemons are stateless: scale
-horizontally by starting more. No etcd, no Kubernetes, no external control plane.
-Just systemd units, a NATS cluster, and your hardware. Deep dive:
-**[`docs/DESIGN.md`](docs/DESIGN.md)**.
+Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless: scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Just systemd units, a NATS cluster, and your hardware. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
 
-## Key Features
+## Self-hosted Installation
 
-- **AWS-compatible APIs.** Use the AWS CLI, SDKs, and Terraform you already know. Repoint your endpoint and ship; nothing to rewrite.
-- **Zero cloud dependency.** Runs entirely on your hardware. No phone-home, no control plane, no external authority. Works fully offline.
-- **Bare-metal compute.** QEMU-based with direct hardware access. No hypervisor tax, no abstraction overhead.
-- **Built-in storage.** Block and object storage included: NVMe caching, Reed-Solomon erasure coding, and replication out of the box.
-- **Edge-first architecture.** Designed for disconnected, contested, and resource-constrained environments from day one.
-- **Open core, no lock-in.** AGPL-3.0 with a commercial option. Inspect it, modify it, deploy it. The platform is yours to keep.
-
-## Installation
-
-Installation requires an Ubuntu / Debian system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for maintaining and installing Spinifex.
+Installation requires an Ubuntu 26.04 or Debian 13 system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for installing Spinifex.
 
 ### Bare Metal ISO
 
@@ -166,16 +157,11 @@ The recommended installation is a [bootable x86 installer](https://iso.mulgadc.c
 curl -fLO https://iso.mulgadc.com/spinifex.iso
 ```
 
-Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware. The install guide walks through the full process.
-
+Follow the [USB install guide](https://docs.mulgadc.com/docs/install-usb) to write the ISO to USB and install on your hardware.
 
 ### Single Node Install
 
-The installation is straightforward to set up and running on a single node for testing purposes. Debian 13 is currently supported, additional Linux distributions are on the immediate roadmap.
-
->*Prerequisite:* Linux bridge for networking.
-
-Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) prerequisites for setup details.
+>*Prerequisite:* Spinifex requires a Linux bridge configured on the host for VM networking. See the [single-node install guide](https://docs.mulgadc.com/docs/install#prerequisites) for setup details.
 
 ```bash
 curl -fsSL https://install.mulgadc.com | bash
@@ -193,36 +179,23 @@ aws ec2 describe-instance-types
 
 ### Development Setup
 
-For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation
-
-### Component Repositories
-
-Spinifex coordinates these independent components:
-
-- **[Predastore](https://github.com/mulgadc/predastore)** - S3-compatible object storage
-- **[Viperblock](https://github.com/mulgadc/viperblock)** - EBS-compatible block storage
-
-Each component can be developed independently. See component-specific documentation for focused development guides.
+For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation.
 
 ## Spinifex UI
 
-Spinifex ships with a built-in web console — an optional alternative to the AWS CLI, SDKs, and Terraform. If you're familiar with the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
+Spinifex ships with a built-in web console, an optional alternative to the AWS CLI, SDKs, and Terraform. If you know the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
 
 <p align="center">
-  <img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console — dashboard view" width="900">
+  <img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console dashboard" width="900">
 </p>
 
-The console is served by each node on port `3000` over TLS, and becomes available as soon as `spinifex.target` is up:
+Each node serves the console on port `3000` over TLS, and it comes up with `spinifex.target`:
 
-```bash
-open https://YOUR_NODE_IP:3000
-```
+- **Same API, different surface.** The UI issues the same AWS SigV4 calls the CLI does, so RBAC, audit trails, and IAM policies apply the same way.
+- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed. There is no separate user database.
+- **Self-hosted, works offline.** The Spinifex binary embeds the UI and each node serves it directly, with no external CDN or analytics calls.
 
-- **Same API, different surface.** Every action in the UI is the same AWS SigV4 call the CLI makes — so RBAC, audit trails, and IAM policies apply uniformly.
-- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed — no separate user database.
-- **Self-hosted, works offline.** The UI is embedded in the Spinifex binary and served from the node itself. No external CDN, no analytics calls, no cloud dependency.
-
-For the full walkthrough — first-time TLS certificate trust, login, and feature tour — see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
+For the full walkthrough, covering first-time TLS certificate trust, login, and a feature tour, see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
 
 ## Development Philosophy
 
@@ -230,16 +203,9 @@ For the full walkthrough — first-time TLS certificate trust, login, and featur
 
 Spinifex is developed by experienced infrastructure engineers with deep AWS expertise, including former AWS team members who understand the intricacies of building production-grade cloud services. Our team brings decades of combined experience from AWS, enterprise infrastructure, and edge computing environments.
 
-**Real-World Experience:**
-
-- Production AWS service development and operations
-- Large-scale infrastructure deployment and management
-- Edge computing and resource-constrained environments
-- Enterprise security and compliance requirements
-
 ### AI-Assisted Development
 
-While Spinifex is architected and implemented by experienced engineers, we leverage **Claude Code** (Anthropic's AI coding assistant) to accelerate certain development tasks. This approach combines human expertise with AI efficiency:
+Spinifex is architected and implemented by experienced engineers, with **Claude Code** accelerating certain development tasks. The split keeps the quality and reliability standards that production infrastructure demands:
 
 **How We Use Claude Code:**
 
@@ -255,12 +221,10 @@ While Spinifex is architected and implemented by experienced engineers, we lever
 - **Performance Optimization**: Real-world performance tuning and benchmarking
 - **Production Operations**: Deployment strategies and operational procedures
 
-This hybrid approach ensures Spinifex benefits from both proven engineering expertise and modern development acceleration, while maintaining the quality and reliability standards required for production infrastructure.
-
 ## Trademarks
 
 "AWS", "Amazon Web Services", and all related service names (EC2, EBS, S3, VPC, IAM, and others) are trademarks of Amazon.com, Inc. or its affiliates. Mulga Defense Corporation and Spinifex are independent and not affiliated with, endorsed by, or sponsored by Amazon.com, Inc. or Amazon Web Services, Inc. References to AWS services describe interoperability and compatibility only.
 
 ## License
 
-Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE). You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.
+Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE), with a commercial license available. You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.

build/Dockerfile.distro

@@ -1,6 +1,6 @@
 ARG VERSION=dev
 
-FROM golang:1.26-bookworm AS builder
+FROM golang:1.27-bookworm AS builder
 
 ARG VERSION
 
@@ -16,9 +16,11 @@ COPY predastore/go.mod predastore/go.sum ./predastore/
 COPY viperblock/go.mod viperblock/go.sum ./viperblock/
 COPY viperblock/nbd/go.mod viperblock/nbd/go.sum ./viperblock/nbd/
 COPY viperblock/nbd/libguestfs.org/nbdkit/go.mod ./viperblock/nbd/libguestfs.org/nbdkit/
+COPY northstar/go.mod northstar/go.sum ./northstar/
+COPY bluebottle/go.mod bluebottle/go.sum ./bluebottle/
 
 # Generate go.work (gitignored, so not in checkout)
-RUN cd spinifex && go work init . ../predastore ../viperblock ../viperblock/nbd ../viperblock/nbd/libguestfs.org/nbdkit
+RUN cd spinifex && go work init . ../predastore ../viperblock ../viperblock/nbd ../viperblock/nbd/libguestfs.org/nbdkit ../northstar ../bluebottle
 
 # Download dependencies (cached layer)
 RUN cd spinifex && go mod download
@@ -27,6 +29,8 @@ RUN cd spinifex && go mod download
 COPY spinifex/ ./spinifex/
 COPY predastore/ ./predastore/
 COPY viperblock/ ./viperblock/
+COPY northstar/ ./northstar/
+COPY bluebottle/ ./bluebottle/
 
 # Build spinifex binary (spx)
 RUN COMMIT=$(cd spinifex && git rev-parse --short HEAD 2>/dev/null || echo "unknown") && \

build/scripts/nats-js-watchdog.sh

@@ -18,22 +18,97 @@
 # So a health check alone misses case 2 entirely. Both probes run here, and a
 # sustained failure of either restarts the service. The multi-sample window
 # avoids restarting during transient startup/quorum 503s.
+#
+# Three more guards sit around that decision, because "restart when the
+# probes fail" is not safe on its own:
+#
+#   - Age gate: spinifex-nats.service is Type=simple, so it reports active the
+#     instant the process forks, long before JetStream finishes replaying its
+#     state. Acting before a recovery budget has elapsed restarts a server
+#     that is still coming up, not one that has failed.
+#   - Cooldown: a restart that did not fix things must not be retried by the
+#     very next run, seconds later.
+#   - Escalation: a restart that has not fixed things after several attempts
+#     is not going to fix it on the next one either, and an unrunnable spx
+#     must never be read as proof of a JetStream failure at all.
 
 set -eu
 
 HEALTH_URL="http://127.0.0.1:8222/healthz?js-enabled-only=true"
 SERVICE="spinifex-nats.service"
 SPX="${SPX_BIN:-/usr/local/bin/spx}"
-SAMPLES=3        # consecutive failing samples required before acting
-SAMPLE_GAP=2     # seconds between samples
-PROBE_TIMEOUT=5  # seconds for one canary round-trip
+SAMPLES="${SAMPLES:-3}"           # consecutive failing samples required before acting
+SAMPLE_GAP="${SAMPLE_GAP:-2}"     # seconds between samples
+PROBE_TIMEOUT="${PROBE_TIMEOUT:-5}"  # seconds for one canary round-trip
 
-# Only act on a running service: a stopped/activating unit is systemd's to manage,
-# and try-restart on it would either no-op or fight an in-flight (re)start.
-if ! systemctl is-active --quiet "$SERVICE"; then
+# Recovery budget: on casuarina NATS needed 3m49s between "Started" and
+# JetStream actually accepting writes, and the bead recorded ~6min elsewhere.
+# 600s clears both with margin.
+NATS_MIN_AGE="${NATS_MIN_AGE:-600}"
+
+# Minimum time between restarts this watchdog performs, independent of the
+# age gate: closes the case where a restart did not fix things and the very
+# next run pulls the trigger again before the new process could recover.
+RESTART_COOLDOWN="${RESTART_COOLDOWN:-600}"
+
+# A restart that has not fixed things after this many attempts is not going
+# to fix it on the next one either, and looping costs more than stopping.
+RESTART_ESCALATE_AT="${RESTART_ESCALATE_AT:-3}"
+
+STATE_DIR="${STATE_DIR:-/run/spinifex}"
+COOLDOWN_STAMP="$STATE_DIR/nats-js-watchdog.last-restart"
+RESTART_COUNT_FILE="$STATE_DIR/nats-js-watchdog.restart-count"
+# Marks that the escalation message has already been printed for the current
+# latch, so a node that stays broken says so once instead of once per tick.
+ESCALATED_STAMP="$STATE_DIR/nats-js-watchdog.escalated"
+
+# Monotonic clock source, so a wall-clock step during boot (NTP correcting an
+# unset RTC) cannot manufacture a false age or cooldown. Overridable so tests
+# can stub a clock without faking /proc.
+UPTIME_FILE="${UPTIME_FILE:-/proc/uptime}"
+
+uptime_seconds() {
+    awk '{printf "%d", $1}' "$UPTIME_FILE"
+}
+
+# nats_age_seconds prints how long $SERVICE has been running. It reads
+# ExecMainStartTimestampMonotonic rather than is-active: that property goes
+# true the instant Type=simple forks, long before JetStream has replayed its
+# state, so it cannot express the recovery window this gate exists to
+# protect. A unit systemd has never started prints 0, which also covers a
+# stopped/activating unit the way the old is-active guard did.
+nats_age_seconds() {
+    start_usec=$(systemctl show --property=ExecMainStartTimestampMonotonic --value "$SERVICE" 2>/dev/null) || start_usec=""
+    case "$start_usec" in
+        ''|*[!0-9]*) echo 0; return ;;
+    esac
+    if [ "$start_usec" -eq 0 ]; then
+        echo 0
+        return
+    fi
+    echo $(( $(uptime_seconds) - start_usec / 1000000 ))
+}
+
+age=$(nats_age_seconds)
+if [ "$age" -lt "$NATS_MIN_AGE" ]; then
     exit 0
 fi
 
+if [ -f "$COOLDOWN_STAMP" ]; then
+    last=$(cat "$COOLDOWN_STAMP" 2>/dev/null) || last=""
+    case "$last" in ''|*[!0-9]*) last=0 ;; esac
+    if [ $(( $(uptime_seconds) - last )) -lt "$RESTART_COOLDOWN" ]; then
+        exit 0
+    fi
+fi
+
+restart_count=$(cat "$RESTART_COUNT_FILE" 2>/dev/null) || restart_count=""
+case "$restart_count" in ''|*[!0-9]*) restart_count=0 ;; esac
+
+# The escalation count is read here but acted on only after the probes, below.
+# Checking it up front would return before the one line that clears it, so a
+# latched watchdog could never observe recovery and never re-arm itself.
+
 # curl is the probe transport; if it is unavailable, do nothing rather than
 # false-restart a healthy server.
 if ! command -v curl >/dev/null 2>&1; then
@@ -45,13 +120,22 @@ js_healthy() {
     curl -fsS -o /dev/null --max-time 5 "$HEALTH_URL" 2>/dev/null
 }
 
-# Round-trips a canary key through the cluster-state bucket. Absent spx this
-# degrades to the health check alone rather than restarting on a missing tool.
+# Round-trips a canary key through the cluster-state bucket. Exit 0 means the
+# write succeeded and exit 1 means JetStream refused it; anything else
+# (spx's own exit 2, a missing binary, or one that cannot run at all) means
+# the probe observed nothing about JetStream and must never be read as
+# evidence of a failure.
 js_writable() {
     if [ ! -x "$SPX" ]; then
-        return 0
+        return 2
     fi
     "$SPX" admin node js-probe --timeout "${PROBE_TIMEOUT}s" >/dev/null 2>&1
+    rc=$?
+    case "$rc" in
+        0) return 0 ;;
+        1) return 1 ;;
+        *) return 2 ;;
+    esac
 }
 
 failure=""
@@ -59,6 +143,18 @@ i=1
 while [ "$i" -le "$SAMPLES" ]; do
     if js_healthy; then
         if js_writable; then
+            wrc=0
+        else
+            wrc=$?
+        fi
+        if [ "$wrc" -eq 0 ]; then
+            # Recovery: drop the history so a later failure gets the full
+            # restart budget again, and re-arm the escalation message.
+            rm -f "$RESTART_COUNT_FILE" "$ESCALATED_STAMP"
+            exit 0
+        fi
+        if [ "$wrc" -eq 2 ]; then
+            echo "nats-js-watchdog: js-probe inconclusive (config, connect, bucket-open, or spx itself failed); not evidence of a JetStream failure, taking no action" >&2
             exit 0
         fi
         failure="JetStream accepts no writes (canary round-trip failed) while $HEALTH_URL reports healthy"
@@ -71,5 +167,24 @@ while [ "$i" -le "$SAMPLES" ]; do
     i=$((i + 1))
 done
 
+# The failure is real and current, so the restart history now means what it
+# says. Above the threshold another restart will not help, so stop acting —
+# but keep probing on later runs, because that is what notices a recovery.
+if [ "$restart_count" -ge "$RESTART_ESCALATE_AT" ]; then
+    if [ ! -f "$ESCALATED_STAMP" ]; then
+        mkdir -p "$STATE_DIR"
+        : > "$ESCALATED_STAMP"
+        echo "nats-js-watchdog: already restarted $SERVICE $restart_count time(s) without recovery; giving up rather than looping (clear $RESTART_COUNT_FILE to re-arm)" >&2
+    fi
+    exit 0
+fi
+
 echo "nats-js-watchdog: $failure on $SAMPLES consecutive probes; restarting $SERVICE" >&2
+mkdir -p "$STATE_DIR"
+uptime_seconds > "$COOLDOWN_STAMP"
+restart_count=$((restart_count + 1))
+echo "$restart_count" > "$RESTART_COUNT_FILE"
+if [ "$restart_count" -ge "$RESTART_ESCALATE_AT" ]; then
+    echo "nats-js-watchdog: restart $restart_count of $RESTART_ESCALATE_AT before escalation; if $SERVICE is still failing after this, the watchdog will stop acting until $RESTART_COUNT_FILE is cleared" >&2
+fi
 systemctl try-restart "$SERVICE"

build/scripts/nats-js-watchdog_test.sh

@@ -0,0 +1,197 @@
+#!/bin/sh
+# Self-contained POSIX test for nats-js-watchdog.sh's restart-decision logic:
+# the age gate, cooldown, escalation, and js-probe exit-2 handling. No real
+# NATS or spx: systemctl, curl, and spx are all stubbed on PATH ahead of the
+# real ones, driven entirely by env vars this test controls.
+#
+# Run: sh build/scripts/nats-js-watchdog_test.sh
+set -eu
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+SCRIPT="${SCRIPT_DIR}/nats-js-watchdog.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "${WORK}"' EXIT
+
+FAKE_BIN="${WORK}/bin"
+mkdir -p "${FAKE_BIN}"
+
+FAILS=0
+fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); }
+pass() { echo "ok: $*"; }
+
+# Fake systemctl: `show` reports a controllable ExecMainStartTimestampMonotonic,
+# and `try-restart` just records that it was called rather than touching a
+# real unit.
+cat > "${FAKE_BIN}/systemctl" <<'EOF'
+#!/bin/sh
+case "$1" in
+    show)
+        echo "${FAKE_NATS_START_USEC:-0}"
+        ;;
+    try-restart)
+        echo "try-restart $2" >> "${FAKE_RESTART_LOG:-/dev/null}"
+        ;;
+esac
+EOF
+chmod +x "${FAKE_BIN}/systemctl"
+
+# Fake curl: js_healthy() only cares about the exit status.
+cat > "${FAKE_BIN}/curl" <<'EOF'
+#!/bin/sh
+[ "${FAKE_HEALTHY:-1}" = "1" ]
+EOF
+chmod +x "${FAKE_BIN}/curl"
+
+# Fake spx: js-probe's whole contract from the watchdog's side is its exit
+# code (0 ok, 1 JetStream refused, 2 inconclusive).
+cat > "${FAKE_BIN}/spx" <<'EOF'
+#!/bin/sh
+exit "${FAKE_SPX_EXIT:-0}"
+EOF
+chmod +x "${FAKE_BIN}/spx"
+
+PATH="${FAKE_BIN}:${PATH}"
+export PATH
+
+# reset_env <name>: fresh per-case dirs and a default env that already clears
+# the age, cooldown, and escalation gates (old enough NATS, no cooldown
+# stamp, no restart history) and a healthy, single-sample probe, so a case
+# only has to override what it is actually testing.
+reset_env() {
+    CASE="${WORK}/$1"
+    mkdir -p "${CASE}/state"
+    UPTIME_FILE="${CASE}/uptime"
+    printf '700.00 0.00\n' > "${UPTIME_FILE}"
+    STATE_DIR="${CASE}/state"
+    FAKE_RESTART_LOG="${CASE}/restart.log"
+    FAKE_NATS_START_USEC=1000000   # 1s: 700s uptime - 1s start = 699s age, clears the 600s default gate
+    FAKE_HEALTHY=1
+    FAKE_SPX_EXIT=0
+    SAMPLES=1
+    SAMPLE_GAP=0
+    NATS_MIN_AGE=600
+    RESTART_COOLDOWN=600
+    RESTART_ESCALATE_AT=3
+    SPX_BIN="${FAKE_BIN}/spx"
+    STDOUT="${CASE}/stdout"
+    STDERR="${CASE}/stderr"
+    export UPTIME_FILE STATE_DIR FAKE_RESTART_LOG FAKE_NATS_START_USEC \
+        FAKE_HEALTHY FAKE_SPX_EXIT SAMPLES SAMPLE_GAP NATS_MIN_AGE \
+        RESTART_COOLDOWN RESTART_ESCALATE_AT SPX_BIN
+}
+
+# Run the watchdog, capturing its exit code (without tripping set -e) + streams.
+invoke() { rc=0; sh "${SCRIPT}" >"${STDOUT}" 2>"${STDERR}" || rc=$?; }
+
+restarted() { [ -s "${FAKE_RESTART_LOG}" ]; }
+
+# --- Case 1: NATS younger than the age gate -> no restart ---
+reset_env young
+FAKE_NATS_START_USEC=690000000  # 690s: 700s uptime - 690s start = 10s age, well under 600s
+export FAKE_NATS_START_USEC
+invoke
+[ "${rc}" -eq 0 ] && pass "young: exit 0" || fail "young: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "young: must not restart while still in the recovery window" || pass "young: no restart"
+
+# --- Case 2: cooldown stamp fresh -> no restart ---
+reset_env cooldown
+echo 650 > "${STATE_DIR}/nats-js-watchdog.last-restart"
+invoke
+[ "${rc}" -eq 0 ] && pass "cooldown: exit 0" || fail "cooldown: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "cooldown: must not restart again inside the cooldown window" || pass "cooldown: no restart"
+
+# --- Case 3: escalation threshold reached while still failing -> no restart,
+# loud log. The probe must fail: a healthy probe at the threshold is case 3b,
+# and it re-arms rather than giving up. ---
+reset_env escalated
+FAKE_SPX_EXIT=1
+export FAKE_SPX_EXIT
+echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count"
+invoke
+[ "${rc}" -eq 0 ] && pass "escalated: exit 0" || fail "escalated: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "escalated: must not restart past the escalation threshold" || pass "escalated: no restart"
+grep -q 'giving up rather than looping' "${STDERR}" \
+    && pass "escalated: loud log emitted" || fail "escalated: escalation log missing: $(cat "${STDERR}")"
+
+# Second identical run: the node is still broken, so the decision stands, but
+# repeating the message every timer tick is what buried the real signal for 42
+# hours on node1.
+invoke
+grep -q 'giving up rather than looping' "${STDERR}" \
+    && fail "escalated: message repeated on every run instead of once per latch" \
+    || pass "escalated: message not repeated"
+
+# --- Case 3b: escalation threshold reached but JetStream has recovered ->
+# the watchdog must re-arm itself. This is the production bug: the escalation
+# check used to run before the probe, so the clear below was unreachable and
+# the node stayed disarmed until reboot. ---
+reset_env escalated_recovered
+echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count"
+: > "${STATE_DIR}/nats-js-watchdog.escalated"
+invoke
+[ "${rc}" -eq 0 ] && pass "escalated-recovered: exit 0" || fail "escalated-recovered: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "escalated-recovered: must not restart a recovered server" || pass "escalated-recovered: no restart"
+[ -f "${STATE_DIR}/nats-js-watchdog.restart-count" ] \
+    && fail "escalated-recovered: restart count must clear once JetStream is writable again" \
+    || pass "escalated-recovered: restart count cleared"
+[ -f "${STATE_DIR}/nats-js-watchdog.escalated" ] \
+    && fail "escalated-recovered: escalation marker must clear so a later failure is reported" \
+    || pass "escalated-recovered: escalation marker cleared"
+[ -s "${STDERR}" ] \
+    && fail "escalated-recovered: a recovered node must be silent: $(cat "${STDERR}")" \
+    || pass "escalated-recovered: silent"
+
+# --- Case 3c: re-armed watchdog acts again. One healthy run clears the latch,
+# and the next genuine failure must restart -- proving it is armed, not just
+# quiet. ---
+reset_env escalated_rearm
+echo 3 > "${STATE_DIR}/nats-js-watchdog.restart-count"
+invoke
+FAKE_SPX_EXIT=1
+export FAKE_SPX_EXIT
+invoke
+restarted && pass "rearm: restarts again after recovery cleared the latch" \
+    || fail "rearm: expected a restart once re-armed: $(cat "${STDERR}")"
+[ "$(cat "${STATE_DIR}/nats-js-watchdog.restart-count" 2>/dev/null)" = "1" ] \
+    && pass "rearm: restart count restarts from 1" \
+    || fail "rearm: expected count 1, got $(cat "${STATE_DIR}/nats-js-watchdog.restart-count" 2>/dev/null)"
+
+# --- Case 4: js-probe exits 2 (inconclusive) -> no restart ---
+reset_env inconclusive
+FAKE_SPX_EXIT=2
+export FAKE_SPX_EXIT
+invoke
+[ "${rc}" -eq 0 ] && pass "inconclusive: exit 0" || fail "inconclusive: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "inconclusive: exit 2 must never be read as a JetStream failure" || pass "inconclusive: no restart"
+grep -q 'inconclusive' "${STDERR}" \
+    && pass "inconclusive: reason logged" || fail "inconclusive: log missing: $(cat "${STDERR}")"
+
+# --- Case 5: genuine JetStream write failure -> restart happens ---
+reset_env genuine_failure
+FAKE_SPX_EXIT=1
+export FAKE_SPX_EXIT
+echo 2 > "${STATE_DIR}/nats-js-watchdog.restart-count"
+invoke
+[ "${rc}" -eq 0 ] && pass "genuine-failure: exit 0" || fail "genuine-failure: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && pass "genuine-failure: restart triggered" || fail "genuine-failure: expected a restart: $(cat "${STDERR}")"
+[ "$(cat "${STATE_DIR}/nats-js-watchdog.restart-count" 2>/dev/null)" = "3" ] \
+    && pass "genuine-failure: restart count incremented" || fail "genuine-failure: restart count not incremented"
+[ -f "${STATE_DIR}/nats-js-watchdog.last-restart" ] \
+    && pass "genuine-failure: cooldown stamp written" || fail "genuine-failure: cooldown stamp missing"
+
+# --- Case 6: healthy NATS -> no restart, and prior restart history clears ---
+reset_env healthy
+echo 2 > "${STATE_DIR}/nats-js-watchdog.restart-count"
+invoke
+[ "${rc}" -eq 0 ] && pass "healthy: exit 0" || fail "healthy: expected exit 0 (rc=${rc}): $(cat "${STDERR}")"
+restarted && fail "healthy: must not restart a healthy server" || pass "healthy: no restart"
+[ -f "${STATE_DIR}/nats-js-watchdog.restart-count" ] \
+    && fail "healthy: restart count must clear once JetStream is confirmed writable" \
+    || pass "healthy: restart count cleared"
+
+if [ "${FAILS}" -eq 0 ]; then
+    echo "PASS: all nats-js-watchdog cases"
+    exit 0
+fi
+echo "FAILED: ${FAILS} case(s)"
+exit 1

build/scripts/wait-for-nats.sh

@@ -26,6 +26,7 @@ fi
 # Final fallback
 NATS_HOST="${NATS_HOST:-127.0.0.1}"
 
+# shellcheck disable=SC2034  # loop count only, index unused
 for i in $(seq 1 "$TIMEOUT"); do
     if nc -z "$NATS_HOST" "$NATS_PORT" 2>/dev/null; then
         exit 0

build/systemd/regenerate-ssh-host-keys.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Regenerate SSH host keys
 ConditionPathExists=!/etc/ssh/ssh_host_ed25519_key

build/systemd/spinifex-awsgw.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex AWS Gateway
 PartOf=spinifex.target

build/systemd/spinifex-daemon.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 3
 [Unit]
 Description=Spinifex Daemon (VM Orchestration)
 PartOf=spinifex.target
@@ -38,12 +39,13 @@ PrivateDevices=no
 DevicePolicy=closed
 DeviceAllow=/dev/kvm rw
 DeviceAllow=/dev/net/tun rw
+DeviceAllow=/dev/vhost-net rw
 DeviceAllow=char-vfio rw
 DeviceAllow=/dev/vfio/vfio rw
 ReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
-# /proc/sys/net/ipv4/conf: per-tap IMDS endpoints set rp_filter/accept_local, which
-# ProtectKernelTunables=yes would otherwise keep read-only.
-ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf
+# /proc/sys/net/ipv4/conf: ProtectKernelTunables would keep the per-tap IMDS
+# rp_filter/accept_local read-only; /etc/spinifex/firewall: the root helper writes peers.nft.
+ReadWritePaths=/var/lib/spinifex/spinifex /var/log/spinifex /run/spinifex /run/spinifex/nbd /proc/sys/net/ipv4/conf -/etc/spinifex/firewall
 
 # RG-9: hardening baseline
 ProtectHome=yes

build/systemd/spinifex-firewall.service

@@ -0,0 +1,25 @@
+# spinifex-unit-version: 1
+[Unit]
+Description=Spinifex host firewall (nftables input policy)
+Documentation=file:/etc/spinifex/firewall/spinifex.nft
+Wants=network-pre.target
+Before=network-pre.target shutdown.target
+DefaultDependencies=no
+ConditionPathExists=/etc/spinifex/firewall/peers.nft
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+StandardInput=null
+ProtectSystem=full
+ProtectHome=true
+ExecStart=/usr/local/lib/spinifex/spinifex-firewall-apply
+ExecReload=/usr/local/lib/spinifex/spinifex-firewall-apply
+TimeoutStopSec=10s
+
+# No ExecStop: flushing on stop would take vpcd's MASQUERADE and per-EIP FORWARD
+# rules with it. Remove the policy deliberately with
+# `nft delete table inet spinifex_filter`.
+
+[Install]
+WantedBy=sysinit.target

build/systemd/spinifex-guests.slice

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex guest VM tier
 Before=slices.target

build/systemd/spinifex-nats-watchdog.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 2
 [Unit]
 Description=Spinifex NATS JetStream liveness watchdog
 PartOf=spinifex.target
@@ -15,6 +16,10 @@ ExecStart=/var/lib/spinifex/nats-js-watchdog.sh
 # cutting it off. Only the failing path runs that long; a healthy node exits on
 # the first sample.
 TimeoutStopSec=45
+# Same 34s worst case bounds the start side: without this the unit inherits
+# oneshot's infinite default, so a genuinely hung run would sit in
+# activating(start) forever and could not be retriggered by its timer.
+TimeoutStartSec=120
 
 # Hardening (kept compatible with talking to PID1 over /run/systemd).
 NoNewPrivileges=yes

build/systemd/spinifex-nats-watchdog.timer

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex NATS JetStream liveness watchdog timer
 PartOf=spinifex.target

build/systemd/spinifex-nats.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex NATS Message Bus
 PartOf=spinifex.target

build/systemd/spinifex-northstar.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Northstar (Authoritative DNS)
 PartOf=spinifex.target

build/systemd/spinifex-predastore.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 3
 [Unit]
 Description=Spinifex Predastore (S3-compatible Object Storage)
 PartOf=spinifex.target
@@ -6,9 +7,9 @@ Wants=spinifex-nats.service
 
 [Service]
 EnvironmentFile=-/etc/spinifex/telemetry.env
-# Guest block I/O arrives with no caller trace; sample those roots at 5% so
-# they don't drown request traces. Request-linked spans are always kept.
-Environment=MULGA_ROOT_TRACE_RATIO=0.05
+# No MULGA_ROOT_TRACE_RATIO here: the sink's tail sampler sheds guest block I/O
+# instead, and it can see whether a trace errored. A head ratio decides before
+# the error exists, so it discarded failures at the same rate as successes.
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-storage
@@ -17,9 +18,9 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
 ExecStart=/usr/local/bin/spx service predastore start
 Restart=on-failure
 RestartSec=5
-# Matches viperblock's drain budget: viperblock's WAL flush writes into
-# predastore, and After= already stops viperblock first, but this is the
-# backstop if that ordering is ever bypassed (direct stop, future edit).
+# Bounds predastore's own graceful stop. viperblock's WAL flush writes into
+# predastore, but After= already stops viperblock first (within its own 150s
+# budget); this is the backstop if that ordering is ever bypassed.
 TimeoutStopSec=30
 OOMScoreAdjust=-500
 # Legitimately the most memory-hungry infra service (S3 buffering/caching);

build/systemd/spinifex-qmp-collector.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex QMP Telemetry Collector (guest metrics)
 PartOf=spinifex.target

build/systemd/spinifex-shutdown.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex graceful guest drain
 PartOf=spinifex.target
@@ -11,11 +12,11 @@ RemainAfterExit=yes
 ExecStart=/bin/true
 # ExecStop fires on ANY target stop, including the stop half of `systemctl
 # restart spinifex.target` (a rolling deploy) -- PartOf propagates regardless
-# of whether the host is actually going down. --only-if-host-stopping drains
-# only when systemd itself is unwinding into shutdown.target; a plain restart
-# or `systemctl stop spinifex.target` falls through so QEMU/nbdkit survive
-# (KillMode=process) and the daemon reconnects to them on the way back up.
-ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --only-if-host-stopping
+# of whether the host is actually going down. --unless-restarting drains on a
+# real shutdown/reboot and on a plain `systemctl stop spinifex.target`; it
+# skips only when a restart of the stack is already queued, so QEMU/nbdkit
+# survive (KillMode=process) and the daemon reconnects on the way back up.
+ExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --unless-restarting
 TimeoutStopSec=180
 
 [Install]

build/systemd/spinifex-system.slice

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex control/infra tier (NATS, predastore, viperblock, daemon, awsgw, vpcd, ui)
 Before=slices.target

build/systemd/spinifex-ui.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Web UI
 PartOf=spinifex.target

build/systemd/spinifex-viperblock.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 2
 [Unit]
 Description=Spinifex Viperblock (Block Storage)
 PartOf=spinifex.target
@@ -23,8 +24,10 @@ MemoryMax=25%
 # way guest QEMU survives a daemon restart. Tearing one out mid-write corrupts
 # that guest's filesystem; idle nbdkit is reaped explicitly on shutdown.
 KillMode=process
-# Bounds this process's own WAL drain, which the plugin caps at 20s.
-TimeoutStopSec=30
+# One worst-case nbdkit reap: killProcessGracePeriod (120s, utils.KillProcess)
+# plus its own shutdownDrainTimeout (20s, viperblock/nbd) and headroom — must
+# strictly exceed 120s now shutdownVolumes reaps concurrently, not serially.
+TimeoutStopSec=150
 EnvironmentFile=/etc/spinifex/systemd.env
 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml
 Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/
@@ -48,7 +51,8 @@ ProtectKernelTunables=yes
 ProtectKernelModules=yes
 ProtectKernelLogs=yes
 ProtectControlGroups=yes
-ProtectProc=invisible
+# ProtectProc must stay default: hidepid=invisible hides spinifex-daemon's
+# QEMU /proc entries (different uid) orphan nbdkit reaping must see.
 SystemCallArchitectures=native
 MemoryDenyWriteExecute=yes
 RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK

build/systemd/spinifex-vpcd.service

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex VPC Daemon (OVN Networking)
 PartOf=spinifex.target

build/systemd/spinifex.slice

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex resource-bounded slice (host ceiling)
 Before=slices.target

build/systemd/spinifex.target

@@ -1,3 +1,4 @@
+# spinifex-unit-version: 1
 [Unit]
 Description=Spinifex Infrastructure Platform
 After=network-online.target

cmd/aws-model-coverage/main.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/mulgadc/spinifex/internal/awsmodel"
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+)
+
+func main() {
+	dispatch := gateway.AWSOperationInventory()
+	coverages := make([]awsmodel.OperationCoverage, 0, len(awsmodel.Services()))
+	for _, service := range awsmodel.Services() {
+		inventory, ok := dispatch[string(service)]
+		var modelInventory awsmodel.DispatchInventory
+		switch {
+		case ok:
+			modelInventory = awsmodel.DispatchInventory{
+				Registered:  inventory.Registered,
+				Stubbed:     inventory.Stubbed,
+				Unsupported: inventory.Unsupported,
+			}
+		case service == awsmodel.S3:
+			modelInventory = awsmodel.DispatchInventory{
+				Opaque: true,
+				Note:   "Spinifex delegates the S3 REST surface to Predastore, which has no operation-name dispatch table to compare mechanically.",
+			}
+		default:
+			fmt.Fprintf(os.Stderr, "aws-model-coverage: no gateway inventory for %s\n", service)
+			os.Exit(1)
+		}
+
+		coverage, err := awsmodel.CompareOperations(service, modelInventory)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "aws-model-coverage: %v\n", err)
+			os.Exit(1)
+		}
+		coverages = append(coverages, coverage)
+	}
+
+	fmt.Print(awsmodel.RenderCoverageMarkdown(coverages))
+}

cmd/ecs-agent/ecrresolver.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"net/http"
+	"net/url"
 	"strings"
 	"sync"
 
@@ -60,12 +61,25 @@ func (e *ecrResolver) client() (*http.Client, error) {
 	return c, nil
 }
 
-// isECRRef reports whether ref targets an ECR registry, i.e. its registry host
-// is "<acct>.dkr.ecr.<region>.<domain>". Only ECR pulls need a minted token;
-// public registries (docker.io and friends) pull anonymously, so refs without
-// an ECR host must not drag in IMDS host credentials.
-func isECRRef(ref string) bool {
+// gatewayRegistryHost returns the host[:port] of the co-located ECR registry
+// from the gateway URL; empty/unparseable yields "" (matches no ref). The
+// registry shares the gateway listener, so its port is the gateway port.
+func gatewayRegistryHost(gatewayURL string) string {
+	u, err := url.Parse(gatewayURL)
+	if err != nil {
+		return ""
+	}
+	return u.Host
+}
+
+// isECRRef reports whether ref targets an ECR registry: an "<acct>.dkr.ecr."
+// host, or one equal to gatewayHost (the agent's own co-located registry
+// advertised as a bare host/IP). Other refs pull anonymously, no token.
+func isECRRef(ref, gatewayHost string) bool {
 	host, _, _ := strings.Cut(ref, "/")
+	if gatewayHost != "" && host == gatewayHost {
+		return true
+	}
 	if !strings.Contains(host, ".") && !strings.Contains(host, ":") {
 		return false // no registry host -> docker.io default
 	}
@@ -75,7 +89,7 @@ func isECRRef(ref string) bool {
 // Authorize returns the ECR token's user/password and proxy endpoint for ref.
 // Non-ECR refs resolve to anonymous pull (empty credentials).
 func (e *ecrResolver) Authorize(ctx context.Context, ref string) (user, pass, endpoint string, err error) {
-	if !isECRRef(ref) {
+	if !isECRRef(ref, gatewayRegistryHost(e.gatewayURL)) {
 		return "", "", "", nil
 	}
 	c, err := e.creds.Retrieve(ctx)

cmd/ecs-agent/ecrresolver_test.go

@@ -115,17 +115,44 @@ func TestECRResolver_NonECRAnonymous(t *testing.T) {
 }
 
 func TestIsECRRef(t *testing.T) {
-	cases := map[string]bool{
-		"nginx:alpine":            false,
-		"docker.io/library/nginx": false,
-		"ghcr.io/org/app:1":       false,
-		"registry:5000/app:1":     false,
-		"123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1": true,
-		"123456789012.dkr.ecr.ap-southeast-2.amazonaws.com/x":    true,
+	cases := []struct {
+		ref, gatewayHost string
+		want             bool
+	}{
+		{"nginx:alpine", "192.168.1.13:9999", false},
+		{"docker.io/library/nginx", "192.168.1.13:9999", false},
+		{"ghcr.io/org/app:1", "192.168.1.13:9999", false},
+		{"registry:5000/app:1", "192.168.1.13:9999", false},
+		{"123456789012.dkr.ecr.us-east-1.spinifex.internal/app:1", "192.168.1.13:9999", true},
+		{"123456789012.dkr.ecr.ap-southeast-2.amazonaws.com/x", "192.168.1.13:9999", true},
+		// Co-located gateway registry advertised as a bare host/IP.
+		{"192.168.1.13:9999/bunyip:latest", "192.168.1.13:9999", true},
+		{"192.168.1.13:9999/bunyip:latest", "", false},
 	}
-	for ref, want := range cases {
-		if got := isECRRef(ref); got != want {
-			t.Errorf("isECRRef(%q) = %v, want %v", ref, got, want)
+	for _, c := range cases {
+		if got := isECRRef(c.ref, c.gatewayHost); got != c.want {
+			t.Errorf("isECRRef(%q, %q) = %v, want %v", c.ref, c.gatewayHost, got, c.want)
 		}
 	}
 }
+
+// TestECRResolver_Authorize_BareHostGateway asserts that a ref whose host
+// equals the agent's own gateway host authenticates via the co-located ECR
+// registry, even though it lacks a ".dkr.ecr." name.
+func TestECRResolver_Authorize_BareHostGateway(t *testing.T) {
+	srv := gatewayStub(t, "AWS:jwt", "https://192.168.1.13:9999", http.StatusOK)
+	creds := stubCreds{c: credentials.Credentials{
+		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "sess",
+		Expiration: time.Now().Add(time.Hour),
+	}}
+	r := newECRResolver(creds, "us-east-1", srv.URL, trusting(srv))
+
+	host := strings.TrimPrefix(srv.URL, "https://")
+	user, pass, _, err := r.Authorize(context.Background(), host+"/bunyip:latest")
+	if err != nil {
+		t.Fatalf("Authorize: %v", err)
+	}
+	if user != "AWS" || pass != "jwt" {
+		t.Errorf("got %q:%q, want AWS:jwt", user, pass)
+	}
+}

cmd/ecs-agent/runtime/runtime_test.go

@@ -1,71 +1,9 @@
 package runtime
 
 import (
-	"context"
-	"errors"
 	"testing"
 )
 
-// staticResolver is a test Resolver returning fixed creds or an error.
-type staticResolver struct {
-	user, pass, endpoint string
-	err                  error
-}
-
-func (s staticResolver) Authorize(_ context.Context, _ string) (string, string, string, error) {
-	return s.user, s.pass, s.endpoint, s.err
-}
-
-func TestFakePuller_RecordsAndDefaultsRef(t *testing.T) {
-	f := &FakePuller{}
-	img, err := f.Pull(context.Background(), PullSpec{Ref: "repo/app:1"}, staticResolver{user: "AWS"})
-	if err != nil {
-		t.Fatalf("Pull: %v", err)
-	}
-	if img.Ref != "repo/app:1" {
-		t.Errorf("Ref defaulted wrong: %q", img.Ref)
-	}
-	if len(f.Pulls) != 1 || f.Pulls[0].Ref != "repo/app:1" {
-		t.Errorf("Pulls = %+v", f.Pulls)
-	}
-	if len(f.Authzd) != 1 || f.Authzd[0] != "AWS" {
-		t.Errorf("Authzd = %+v", f.Authzd)
-	}
-}
-
-func TestFakePuller_ResolverError(t *testing.T) {
-	f := &FakePuller{}
-	_, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, staticResolver{err: errors.New("boom")})
-	if err == nil {
-		t.Fatal("expected resolver error to propagate")
-	}
-}
-
-func TestFakePuller_ProgrammedError(t *testing.T) {
-	f := &FakePuller{Err: errors.New("pull failed")}
-	if _, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil); err == nil {
-		t.Fatal("expected programmed error")
-	}
-}
-
-func TestFakePuller_OnPullOverrides(t *testing.T) {
-	want := Image{Ref: "x", Digest: "sha256:abc", Size: 42}
-	f := &FakePuller{Err: errors.New("ignored"), OnPull: func(PullSpec) (Image, error) {
-		return want, nil
-	}}
-	got, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil)
-	if err != nil || got != want {
-		t.Fatalf("got %+v, %v; want %+v", got, err, want)
-	}
-}
-
-func TestFakePuller_Close(t *testing.T) {
-	f := &FakePuller{}
-	if err := f.Close(); err != nil || !f.Closed {
-		t.Fatalf("Close: err=%v closed=%v", err, f.Closed)
-	}
-}
-
 func TestNormalizeRef(t *testing.T) {
 	cases := map[string]string{
 		"nginx:alpine":      "docker.io/library/nginx:alpine",

cmd/installer/autoinstall/autoinstall.go

@@ -138,9 +138,11 @@ func buildConfig() (*install.Config, error) {
 
 // resolveStorage builds the disk configuration from the kernel cmdline.
 //
-//	SPINIFEX_FS     ext4 (default) or one of the zfs-* topologies
-//	SPINIFEX_DISKS  explicit ordered member list, required for multi-disk pools
-//	SPINIFEX_DISK   single-disk selector, as before
+//	SPINIFEX_FS               ext4 (default) or one of the zfs-* topologies
+//	SPINIFEX_DISKS            explicit ordered member list, required for multi-disk pools
+//	SPINIFEX_DISK             the OS disk, as before
+//	SPINIFEX_DISK_SPINIFEX    ext4 only: dedicated disk for /var/lib/spinifex
+//	SPINIFEX_DISK_PREDASTORE  ext4 only: dedicated disk for /var/lib/spinifex/predastore
 func resolveStorage() (install.DiskConfig, error) {
 	cfg := install.DiskConfig{FS: install.FSExt4}
 	if v := strings.TrimSpace(os.Getenv("SPINIFEX_FS")); v != "" {
@@ -171,6 +173,9 @@ func resolveStorage() (install.DiskConfig, error) {
 		cfg.Disks = []install.Disk{d}
 	}
 
+	if err := applyDiskRoles(&cfg); err != nil {
+		return cfg, err
+	}
 	if cfg.ZFS, err = parseZFSOpts(); err != nil {
 		return cfg, err
 	}
@@ -181,6 +186,51 @@ func resolveStorage() (install.DiskConfig, error) {
 	return cfg, nil
 }
 
+// roleVars maps each optional data role to the variable that assigns it. The OS
+// role is not here: it always comes from SPINIFEX_DISK.
+var roleVars = []struct {
+	role install.DiskRole
+	env  string
+}{
+	{install.RoleSpinifex, "SPINIFEX_DISK_SPINIFEX"},
+	{install.RolePredastore, "SPINIFEX_DISK_PREDASTORE"},
+}
+
+// applyDiskRoles attaches dedicated data drives to an ext4 install. Roles are
+// always set on ext4, even with no extra drives, so the OS assignment is
+// explicit rather than implied by position.
+func applyDiskRoles(cfg *install.DiskConfig) error {
+	if cfg.FS.IsZFS() {
+		for _, rv := range roleVars {
+			if strings.TrimSpace(os.Getenv(rv.env)) != "" {
+				return fmt.Errorf("%s applies to ext4 only — %s spans every member already", rv.env, cfg.FS.Label())
+			}
+		}
+		return nil
+	}
+	if len(cfg.Disks) != 1 {
+		return fmt.Errorf("ext4 takes one OS disk, %d given — set SPINIFEX_DISK, and SPINIFEX_DISK_SPINIFEX or SPINIFEX_DISK_PREDASTORE for dedicated drives",
+			len(cfg.Disks))
+	}
+
+	roles := []install.RoleMount{{Role: install.RoleOS, Disk: cfg.Disks[0]}}
+	for _, rv := range roleVars {
+		spec := strings.TrimSpace(os.Getenv(rv.env))
+		if spec == "" {
+			continue
+		}
+		// Resolved with the same fail-closed selector as the OS disk, so an
+		// ambiguous value lists the candidates rather than picking one.
+		d, err := resolveDisk(spec)
+		if err != nil {
+			return fmt.Errorf("%s: %w", rv.env, err)
+		}
+		roles = append(roles, install.RoleMount{Role: rv.role, Disk: d})
+	}
+	*cfg = cfg.WithRoles(roles)
+	return nil
+}
+
 // parseZFSOpts reads the advanced pool tunables. Every one is optional; unset
 // values are computed from the hardware at install time.
 func parseZFSOpts() (install.ZFSOpts, error) {

cmd/installer/autoinstall/autoinstall_test.go

@@ -436,10 +436,53 @@ func TestResolveStorage(t *testing.T) {
 			wantErr: `"vdz" is not an available disk`,
 		},
 		{
-			name:    "member list that fails validation",
+			name:    "member list is for pools, not ext4 roles",
 			disks:   four,
 			env:     map[string]string{"SPINIFEX_DISKS": "vdb,vdc"},
-			wantErr: "supports a single disk only",
+			wantErr: "ext4 takes one OS disk",
+		},
+		{
+			name:  "dedicated role drives, os first",
+			disks: four,
+			env: map[string]string{
+				"SPINIFEX_DISK":            "vdb",
+				"SPINIFEX_DISK_SPINIFEX":   "vdc",
+				"SPINIFEX_DISK_PREDASTORE": "vdd",
+			},
+			wantFS:    install.FSExt4,
+			wantPaths: []string{"/dev/vdb", "/dev/vdc", "/dev/vdd"},
+		},
+		{
+			// The two-drive layout: predastore gets its own disk while
+			// /var/lib/spinifex stays on the root filesystem.
+			name:      "predastore alone",
+			disks:     four,
+			env:       map[string]string{"SPINIFEX_DISK": "vdb", "SPINIFEX_DISK_PREDASTORE": "vdc"},
+			wantFS:    install.FSExt4,
+			wantPaths: []string{"/dev/vdb", "/dev/vdc"},
+		},
+		{
+			name:  "role drive on a pool",
+			disks: four,
+			env: map[string]string{
+				"SPINIFEX_FS": "zfs-raid1", "SPINIFEX_DISKS": "vdb,vdc",
+				"SPINIFEX_DISK_SPINIFEX": "vdd",
+			},
+			wantErr: "SPINIFEX_DISK_SPINIFEX applies to ext4 only",
+		},
+		{
+			name:    "role drive that is not present",
+			disks:   four[:1],
+			env:     map[string]string{"SPINIFEX_DISK": "vdb", "SPINIFEX_DISK_PREDASTORE": "vdz"},
+			wantErr: "SPINIFEX_DISK_PREDASTORE",
+		},
+		{
+			// Two roles cannot share a drive; the disk is erased once and the
+			// second mount would land on top of the first.
+			name:    "role drive repeats the os disk",
+			disks:   four,
+			env:     map[string]string{"SPINIFEX_DISK": "vdb", "SPINIFEX_DISK_SPINIFEX": "vdb"},
+			wantErr: "selected more than once",
 		},
 		{
 			name:    "bad zfs tunable",

cmd/installer/firstboot/firstboot.go

@@ -146,6 +146,21 @@ hostnamectl set-hostname %s
 printf "nameserver 1.1.1.1\nnameserver 8.8.8.8\n" > /etc/resolvconf/resolv.conf.d/base
 resolvconf -u
 
+# The RTC on a fresh machine can be badly wrong — a flat CMOS battery is routine
+# on refurbished hardware. Credentials and TLS certificates are minted further
+# down, and a certificate issued against a wrong clock carries a NotBefore that
+# outlives the correction, so the clock has to be right before that happens.
+# chronyd is already running; makestep forces an immediate correction instead of
+# waiting out the slew. Bounded and non-fatal: an air-gapped install must still
+# complete, and rtcsync writes the corrected time back so later boots start close.
+echo "[firstboot] syncing clock before credentials are minted"
+chronyc -a makestep >/dev/null 2>&1 || true
+if chronyc waitsync 12 0.1 0 5 >/dev/null 2>&1; then
+    echo "[firstboot] clock synced: $(date -uIs)"
+else
+    echo "[firstboot] WARNING: clock not synced, continuing with $(date -uIs)"
+fi
+
 # Configure OVN networking.
 # br-wan (and br-lan if present) are Linux bridges managed by systemd-networkd
 # (declared in /etc/systemd/network/ by the installer). setup-ovn.sh
@@ -156,7 +171,8 @@ resolvconf -u
 # Write the banner (first time, spinifex-banner service will do this on reboot)
 /usr/local/bin/spx admin banner --boot-check
 
-# Cluster formation — capture credentials to file for display on console.
+# Cluster formation. Output goes to the journal via the systemd unit; spx
+# admin init writes credentials to disk and never prints them.
 %s 2>&1
 
 # Fix ownership of files spx admin init wrote. spx runs as root under
@@ -206,6 +222,12 @@ fi
 
 # Enable services to start, on reboot
 systemctl enable spinifex.target spinifex-banner.service
+# setup.sh enables this on the package install path, but the ISO chroot returns
+# before that line (no systemd to talk to), so the ISO needs it here. An nft
+# ruleset is runtime state and does not survive a reboot; without this the unit
+# never reapplies the policy and the node comes back up open. Enabled, not
+# started: ConditionPathExists holds it inert until the daemon writes a peer file.
+systemctl enable spinifex-firewall.service
 # WantedBy=timers.target only self-activates once enabled — without this the
 # JetStream ENOSPC-latch watchdog never runs and a full disk needs a manual restart.
 systemctl enable --now spinifex-nats-watchdog.timer

cmd/installer/install/blockdev.go

@@ -45,6 +45,7 @@ var (
 	sysClassBlockDir = "/sys/class/block"
 	devByIDDir       = "/dev/disk/by-id"
 	procMountsPath   = "/proc/mounts"
+	procSwapsPath    = "/proc/swaps"
 )
 
 // ListDisks enumerates install-candidate disks from /sys/block. It reads sysfs
@@ -132,6 +133,144 @@ func partitionPath(disk string, n int) string {
 	return fmt.Sprintf("%s%d", disk, n)
 }
 
+// readKernelPartitions returns the partition device names the kernel currently
+// publishes for a disk, sorted.
+//
+// It reads sysfs rather than /dev deliberately. A partition device node
+// outlives the table that created it, so a stale node is indistinguishable
+// from a fresh one by existence alone — which is how a disk whose old table
+// the kernel never dropped passes every check and then gets formatted at the
+// previous layout's offsets.
+func readKernelPartitions(disk string) ([]string, error) {
+	name := filepath.Base(disk)
+	entries, err := os.ReadDir(filepath.Join(sysBlockDir, name))
+	if err != nil {
+		return nil, fmt.Errorf("read the kernel's partition list for %s: %w", disk, err)
+	}
+	var out []string
+	for _, e := range entries {
+		// Partitions are the child directories carrying a "partition" attribute.
+		// "queue", "device" and the plain attribute files sit alongside them.
+		if _, err := os.Stat(filepath.Join(sysBlockDir, name, e.Name(), "partition")); err != nil {
+			continue
+		}
+		out = append(out, e.Name())
+	}
+	slices.Sort(out)
+	return out, nil
+}
+
+// kernelPartitions is indirected so tests can drive the partition-table
+// assertions without a real block device.
+var kernelPartitions = readKernelPartitions
+
+// partitionDevice maps a kernel partition name back to its device node,
+// resolved alongside the disk it belongs to rather than assuming /dev.
+func partitionDevice(disk, part string) string {
+	return filepath.Join(filepath.Dir(disk), part)
+}
+
+// diskHolders names the md, LVM or device-mapper devices claiming this disk or
+// one of its partitions. A holder is the usual reason the kernel refuses to
+// re-read a partition table, so it is what the operator needs told when the
+// installer gives up on a disk.
+func diskHolders(disk string) []string {
+	name := filepath.Base(disk)
+	dirs := []string{filepath.Join(sysBlockDir, name, "holders")}
+	parts, _ := kernelPartitions(disk)
+	for _, p := range parts {
+		dirs = append(dirs, filepath.Join(sysBlockDir, name, p, "holders"))
+	}
+
+	seen := map[string]bool{}
+	var out []string
+	for _, dir := range dirs {
+		entries, err := os.ReadDir(dir)
+		if err != nil {
+			continue
+		}
+		for _, e := range entries {
+			if seen[e.Name()] {
+				continue
+			}
+			seen[e.Name()] = true
+			out = append(out, e.Name())
+		}
+	}
+	slices.Sort(out)
+	return out
+}
+
+// diskMount is one filesystem mounted from a disk the installer is about to
+// erase.
+type diskMount struct {
+	Device     string
+	Mountpoint string
+}
+
+// devicesOnDisk is the set of device paths belonging to a disk: the whole disk
+// and every partition the kernel currently publishes. Matching /proc tables
+// against a set rather than a string prefix keeps /dev/nvme0n1 from claiming
+// /dev/nvme0n11.
+func devicesOnDisk(disk string) map[string]bool {
+	out := map[string]bool{disk: true}
+	parts, _ := kernelPartitions(disk)
+	for _, p := range parts {
+		out[partitionDevice(disk, p)] = true
+	}
+	return out
+}
+
+// mountsOnDisk returns the filesystems mounted from a disk, deepest mountpoint
+// first so a nested mount is released before the one it sits inside.
+func mountsOnDisk(disk string) []diskMount {
+	devs := devicesOnDisk(disk)
+	var out []diskMount
+	for _, fields := range procTable(procMountsPath, 0) {
+		if len(fields) >= 2 && devs[fields[0]] {
+			out = append(out, diskMount{Device: fields[0], Mountpoint: fields[1]})
+		}
+	}
+	slices.SortFunc(out, func(a, b diskMount) int {
+		return strings.Count(b.Mountpoint, "/") - strings.Count(a.Mountpoint, "/")
+	})
+	return out
+}
+
+// swapsOnDisk returns the active swap areas backed by a disk.
+func swapsOnDisk(disk string) []string {
+	devs := devicesOnDisk(disk)
+	var out []string
+	// The first line of /proc/swaps is a column header, not an entry.
+	for _, fields := range procTable(procSwapsPath, 1) {
+		if len(fields) >= 1 && devs[fields[0]] {
+			out = append(out, fields[0])
+		}
+	}
+	return out
+}
+
+// procTable splits a whitespace-delimited /proc table into fields per line,
+// dropping the first skip lines and any blank ones.
+func procTable(path string, skip int) [][]string {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil
+	}
+	var out [][]string
+	n := 0
+	for line := range strings.SplitSeq(string(data), "\n") {
+		n++
+		if n <= skip {
+			continue
+		}
+		if fields := strings.Fields(line); len(fields) > 0 {
+			out = append(out, fields)
+		}
+	}
+	return out
+}
+
 // byIDPreference orders the /dev/disk/by-id/ namespaces we prefer to name a
 // pool member by. wwn- ids are last: they are stable but opaque, so a degraded
 // pool reports a number nobody can map to a drive bay.

cmd/installer/install/blockdev_test.go

@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 	"slices"
 	"strconv"
+	"strings"
 	"testing"
 )
 
@@ -40,6 +41,7 @@ func newFakeSysfs(t *testing.T, disks ...fakeDisk) *fakeSysfs {
 	swap(&sysClassBlockDir, filepath.Join(f.root, "sys/class/block"))
 	swap(&devByIDDir, filepath.Join(f.root, "dev/disk/by-id"))
 	swap(&procMountsPath, filepath.Join(f.root, "proc/mounts"))
+	swap(&procSwapsPath, filepath.Join(f.root, "proc/swaps"))
 
 	f.mkdirAll(sysBlockDir)
 	f.mkdirAll(sysClassBlockDir)
@@ -113,6 +115,18 @@ func (f *fakeSysfs) setMounts(content string) {
 	f.write(procMountsPath, content)
 }
 
+func (f *fakeSysfs) setSwaps(content string) {
+	f.t.Helper()
+	f.write(procSwapsPath, content)
+}
+
+// addHolder marks a partition as claimed by md, LVM or device-mapper — the
+// state that makes the kernel refuse to re-read the disk's partition table.
+func (f *fakeSysfs) addHolder(disk, part, holder string) {
+	f.t.Helper()
+	f.mkdirAll(filepath.Join(sysBlockDir, disk, part, "holders", holder))
+}
+
 func boolAttr(b bool) string {
 	if b {
 		return "1\n"
@@ -352,3 +366,119 @@ func TestHasAnyPrefix(t *testing.T) {
 		t.Error("sda should not match")
 	}
 }
+
+func TestReadKernelPartitionsListsOnlyPartitions(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{
+		name: "nvme0n1", sectors: 1 << 22,
+		partitions: []string{"nvme0n1p3", "nvme0n1p1", "nvme0n1p2"},
+	})
+
+	got, err := readKernelPartitions("/dev/nvme0n1")
+	if err != nil {
+		t.Fatalf("readKernelPartitions: %v", err)
+	}
+	// Sorted, and "queue" and "device" alongside them are not partitions.
+	want := []string{"nvme0n1p1", "nvme0n1p2", "nvme0n1p3"}
+	if !slices.Equal(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+}
+
+func TestReadKernelPartitionsIsEmptyForAnUnpartitionedDisk(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{name: "nvme2n1", sectors: 1 << 22})
+
+	got, err := readKernelPartitions("/dev/nvme2n1")
+	if err != nil {
+		t.Fatalf("readKernelPartitions: %v", err)
+	}
+	if len(got) != 0 {
+		t.Errorf("got %v, want none", got)
+	}
+}
+
+func TestReadKernelPartitionsFailsOnAnUnknownDisk(t *testing.T) {
+	newFakeSysfs(t)
+
+	if _, err := readKernelPartitions("/dev/nvme9n1"); err == nil {
+		t.Fatal("expected an error for a disk with no sysfs entry")
+	}
+}
+
+func TestDiskHoldersNamesEveryClaimant(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{
+		name: "sda", sectors: 1 << 22, partitions: []string{"sda1", "sda2"},
+	})
+	f.addHolder("sda", "sda1", "md0")
+	f.addHolder("sda", "sda2", "dm-0")
+
+	if got, want := diskHolders("/dev/sda"), []string{"dm-0", "md0"}; !slices.Equal(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+}
+
+func TestDiskHoldersIsEmptyWhenNothingClaimsTheDisk(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1"}})
+
+	if got := diskHolders("/dev/sda"); len(got) != 0 {
+		t.Errorf("got %v, want none", got)
+	}
+}
+
+func TestMountsOnDiskFindsOnlyThisDiskDeepestFirst(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1", "sda2"}},
+		fakeDisk{name: "sdb", sectors: 1 << 22, partitions: []string{"sdb1"}},
+	)
+	f.setMounts(strings.Join([]string{
+		"/dev/sda1 /oldroot ext4 rw 0 0",
+		"/dev/sda2 /oldroot/boot/efi vfat rw 0 0",
+		"/dev/sdb1 /other ext4 rw 0 0",
+		"proc /proc proc rw 0 0",
+	}, "\n") + "\n")
+
+	got := mountsOnDisk("/dev/sda")
+	want := []diskMount{
+		{Device: "/dev/sda2", Mountpoint: "/oldroot/boot/efi"},
+		{Device: "/dev/sda1", Mountpoint: "/oldroot"},
+	}
+	if !slices.Equal(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+}
+
+// /dev/nvme0n1 must not claim /dev/nvme0n11, which a string-prefix match would.
+func TestMountsOnDiskDoesNotMatchASiblingNamespace(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "nvme0n1", sectors: 1 << 22, partitions: []string{"nvme0n1p1"}},
+		fakeDisk{name: "nvme0n11", sectors: 1 << 22, partitions: []string{"nvme0n11p1"}},
+	)
+	f.setMounts("/dev/nvme0n11p1 /elsewhere ext4 rw 0 0\n")
+
+	if got := mountsOnDisk("/dev/nvme0n1"); len(got) != 0 {
+		t.Errorf("got %v, want none", got)
+	}
+}
+
+func TestSwapsOnDiskSkipsTheHeaderAndOtherDisks(t *testing.T) {
+	f := newFakeSysfs(t,
+		fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1"}},
+		fakeDisk{name: "sdb", sectors: 1 << 22, partitions: []string{"sdb1"}},
+	)
+	f.setSwaps(strings.Join([]string{
+		"Filename\t\t\t\tType\t\tSize\tUsed\tPriority",
+		"/dev/sda1 partition 8388604 0 -2",
+		"/dev/sdb1 partition 8388604 0 -3",
+	}, "\n") + "\n")
+
+	if got, want := swapsOnDisk("/dev/sda"), []string{"/dev/sda1"}; !slices.Equal(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+}
+
+func TestSwapsOnDiskIsEmptyWithoutASwapsTable(t *testing.T) {
+	newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 22, partitions: []string{"sda1"}})
+
+	if got := swapsOnDisk("/dev/sda"); len(got) != 0 {
+		t.Errorf("got %v, want none", got)
+	}
+}

cmd/installer/install/disk.go

@@ -4,6 +4,9 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	"path/filepath"
+	"slices"
+	"strings"
 	"time"
 )
 
@@ -14,6 +17,10 @@ const (
 	biosPartNum = 1
 	espPartNum  = 2
 	rootPartNum = 3
+
+	// dataPartNum is the sole partition on a role drive. Data drives carry no
+	// bootloader, so they start at 1 rather than mirroring the boot layout.
+	dataPartNum = 1
 )
 
 // GPT partition type GUIDs, by their sgdisk short code.
@@ -43,6 +50,95 @@ const (
 	tailReserveMiB = 1024
 )
 
+// partitionSettleTimeout bounds the wait for the kernel and udev to publish a
+// disk's new partition table. A var so tests do not pay it.
+var partitionSettleTimeout = 15 * time.Second
+
+// releaseDisk detaches whatever the live environment has attached to a disk
+// before it is erased. A mounted filesystem or an active swap area makes
+// BLKRRPART return EBUSY, and the kernel then keeps serving the disk's old
+// partition table while sgdisk writes the new one and reports success.
+func releaseDisk(d Disk) {
+	for _, m := range mountsOnDisk(d.Path) {
+		err := runQuiet("umount", m.Device)
+		if err != nil {
+			// Lazy detach: a busy mount must not stall an install that is about to
+			// erase the filesystem underneath it anyway.
+			err = runQuiet("umount", "-l", m.Device)
+		}
+		if err != nil {
+			slog.Warn("could not unmount before erase", "device", m.Device, "mountpoint", m.Mountpoint, "err", err)
+			continue
+		}
+		slog.Info("released mount before erase", "device", m.Device, "mountpoint", m.Mountpoint)
+	}
+	for _, s := range swapsOnDisk(d.Path) {
+		if err := runQuiet("swapoff", s); err != nil {
+			slog.Warn("could not disable swap before erase", "device", s, "err", err)
+			continue
+		}
+		slog.Info("disabled swap before erase", "device", s)
+	}
+}
+
+// settlePartitions forces the kernel to re-read a disk's partition table and
+// blocks until its view matches want, naming the partitions by number.
+//
+// An empty want asserts the disk has no partitions at all. That is the only
+// state no leftover can satisfy: a disk whose previous table had the same
+// partition numbers as the one being written — an old OS install is p1, p2, p3,
+// exactly what the boot layout writes — passes an "are the expected partitions
+// there" check on stale nodes alone.
+func settlePartitions(d Disk, want ...int) error {
+	if err := runQuiet("partprobe", d.Path); err != nil {
+		// partprobe rescans the whole disk and fails if it cannot open any part of
+		// it; the ioctl on its own sometimes still gets through.
+		if err := runQuiet("blockdev", "--rereadpt", d.Path); err != nil {
+			slog.Warn("could not force a partition table re-read", "disk", d.Path, "err", err)
+		}
+	}
+	_ = runQuiet("udevadm", "settle", "--timeout=10")
+
+	wanted := make([]string, 0, len(want))
+	for _, n := range want {
+		wanted = append(wanted, filepath.Base(d.PartitionPath(n)))
+	}
+	slices.Sort(wanted)
+
+	deadline := time.Now().Add(partitionSettleTimeout)
+	for {
+		got, err := kernelPartitions(d.Path)
+		if err != nil {
+			return err
+		}
+		if slices.Equal(got, wanted) {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			return staleTableError(d, wanted, got)
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+// staleTableError explains a kernel partition view that never caught up. The
+// erase case gets its own wording because it is the dangerous one — proceeding
+// would format the previous layout — and because the holder is usually the
+// whole diagnosis.
+func staleTableError(d Disk, wanted, got []string) error {
+	if len(wanted) == 0 {
+		msg := fmt.Sprintf("%s still shows partitions [%s] after its table was erased — the kernel is using the old table, so formatting would write at the previous layout's offsets",
+			d.Path, strings.Join(got, ", "))
+		if h := diskHolders(d.Path); len(h) > 0 {
+			return fmt.Errorf("%s; it is held by %s, which must be stopped before this disk can be installed to",
+				msg, strings.Join(h, ", "))
+		}
+		return fmt.Errorf("%s; something in the live environment is holding it open", msg)
+	}
+	return fmt.Errorf("%s: the kernel shows partitions [%s] but the new table has [%s] — the table was written but never picked up",
+		d.Path, strings.Join(got, ", "), strings.Join(wanted, ", "))
+}
+
 // espSizeMiB picks the ESP size for a disk.
 func espSizeMiB(d Disk) int {
 	if d.Bytes >= espLargeDiskGiB<<30 {
@@ -60,14 +156,43 @@ func partitionDisks(cfg DiskConfig) error {
 	if cfg.FS.IsZFS() {
 		ptype = typeZFS
 	}
-	for _, d := range cfg.Disks {
+	for _, d := range cfg.bootDisks() {
 		if err := partitionOne(d, ptype); err != nil {
 			return fmt.Errorf("partition %s: %w", d.Path, err)
 		}
 	}
+	for _, rm := range cfg.DataMounts() {
+		if err := partitionDataDisk(rm); err != nil {
+			return fmt.Errorf("partition %s (%s): %w", rm.Disk.Path, rm.Role, err)
+		}
+	}
 	return waitForPartitions(cfg)
 }
 
+// partitionDataDisk writes the single-partition GPT for a role drive. It gets
+// no bios_boot and no ESP: nothing installs a bootloader there, and a formatted
+// ESP no firmware entry points at only misleads whoever reads lsblk in an
+// incident.
+func partitionDataDisk(rm RoleMount) error {
+	if err := clearDisk(rm.Disk); err != nil {
+		return fmt.Errorf("clear: %w", err)
+	}
+	// Same tail reserve as a boot drive, for the same reason: a replacement of
+	// the "same" size is routinely a few thousand sectors smaller.
+	if err := run("sgdisk",
+		"-n", fmt.Sprintf("%d:1M:-%dM", dataPartNum, tailReserveMiB),
+		"-t", fmt.Sprintf("%d:%s", dataPartNum, typeLinuxFS),
+		"-c", fmt.Sprintf("%d:%s", dataPartNum, rm.Role.Label()),
+		rm.Disk.Path,
+	); err != nil {
+		return err
+	}
+	if err := run("sgdisk", "-G", rm.Disk.Path); err != nil {
+		slog.Warn("could not randomise GPT GUIDs", "disk", rm.Disk.Path, "err", err)
+	}
+	return settlePartitions(rm.Disk, dataPartNum)
+}
+
 // partitionOne writes the GPT for a single disk.
 func partitionOne(d Disk, rootType string) error {
 	// Stale ZFS labels and filesystem signatures survive a new partition table
@@ -96,26 +221,16 @@ func partitionOne(d Disk, rootType string) error {
 	if err := run("sgdisk", "-G", d.Path); err != nil {
 		slog.Warn("could not randomise GPT GUIDs", "disk", d.Path, "err", err)
 	}
-	return nil
+	return settlePartitions(d, biosPartNum, espPartNum, rootPartNum)
 }
 
 // waitForPartitions ensures every partition device node the installer is about
-// to use exists. Trixie's udev is slow enough after a table rewrite that mkfs
-// races it and fails with ENOENT on a device the kernel has already accepted.
+// to use exists. The kernel's own view was already asserted per disk as each
+// table was written; udev publishes the /dev nodes and by-id symlinks after
+// that, and Trixie is slow enough about it that mkfs races them.
 func waitForPartitions(cfg DiskConfig) error {
-	for _, d := range cfg.Disks {
-		// Best-effort: partprobe failing is not fatal, since udev may still act
-		// on the BLKRRPART ioctl sgdisk issued itself.
-		if err := run("partprobe", d.Path); err != nil {
-			slog.Warn("partprobe failed, continuing", "disk", d.Path, "err", err)
-		}
-	}
-	if err := run("udevadm", "settle", "--timeout=10"); err != nil {
-		slog.Warn("udevadm settle failed, continuing", "err", err)
-	}
-
-	deadline := time.Now().Add(15 * time.Second)
-	for _, d := range cfg.Disks {
+	deadline := time.Now().Add(partitionSettleTimeout)
+	for _, d := range cfg.bootDisks() {
 		wanted := []string{d.PartitionPath(espPartNum), d.PartitionPath(rootPartNum)}
 		// The pool is built from by-id paths, so their symlinks have to exist
 		// too — udev publishes them later than the kernel device node.
@@ -128,6 +243,11 @@ func waitForPartitions(cfg DiskConfig) error {
 			}
 		}
 	}
+	for _, rm := range cfg.DataMounts() {
+		if err := waitForPath(rm.Disk.PartitionPath(dataPartNum), deadline); err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
@@ -146,7 +266,7 @@ func waitForPath(path string, deadline time.Time) error {
 // formatESPs makes a FAT32 filesystem on every disk's ESP. Each one is
 // independent — there is no RAID here, they are kept in step by content.
 func formatESPs(cfg DiskConfig) error {
-	for _, d := range cfg.Disks {
+	for _, d := range cfg.bootDisks() {
 		esp := d.PartitionPath(espPartNum)
 		// A distinct label per member so `lsblk -f` on a degraded machine tells
 		// the operator which disk each ESP belongs to.

cmd/installer/install/diskconfig.go

@@ -82,6 +82,45 @@ func ParseFSType(s string) (FSType, error) {
 	return f, nil
 }
 
+// DiskRole is the purpose a selected drive serves on an ext4 install. A ZFS
+// pool carries every workload on one set of members, so roles do not apply
+// there — the topology already spans all the disks.
+type DiskRole string
+
+const (
+	RoleOS         DiskRole = "os"
+	RoleSpinifex   DiskRole = "spinifex"
+	RolePredastore DiskRole = "predastore"
+)
+
+// AllDiskRoles is canonical order, outermost mountpoint first. Mounting, fstab
+// emission and rsync protect filters all walk it forwards; teardown walks it
+// back, because predastore nests inside spinifex.
+var AllDiskRoles = []DiskRole{RoleOS, RoleSpinifex, RolePredastore}
+
+// roleMountpoints is where each role lands on the installed system. A role left
+// unassigned simply stays on the filesystem above it.
+var roleMountpoints = map[DiskRole]string{
+	RoleOS:         "/",
+	RoleSpinifex:   "/var/lib/spinifex",
+	RolePredastore: "/var/lib/spinifex/predastore",
+}
+
+// Label is the role name as shown in the UI and written to partition labels.
+func (r DiskRole) Label() string { return string(r) }
+
+// Mountpoint is the role's path on the installed system.
+func (r DiskRole) Mountpoint() string { return roleMountpoints[r] }
+
+// RoleMount binds one drive to one role.
+type RoleMount struct {
+	Role DiskRole
+	Disk Disk
+}
+
+// Mountpoint is where this drive's filesystem is mounted on the target.
+func (r RoleMount) Mountpoint() string { return r.Role.Mountpoint() }
+
 // ZFSOpts are the advanced tunables. A zero value means "use the computed
 // default", which is why Ashift and ARCMaxMiB are plain ints — 0 is not legal
 // for either, so it is unambiguous as a sentinel.
@@ -94,14 +133,81 @@ type ZFSOpts struct {
 }
 
 // DiskConfig is the storage half of the installer's configuration. Disks is
-// ordered and the order is significant: RAID10 pairs members two at a time.
+// ordered and the order is significant: RAID10 pairs members two at a time, and
+// on ext4 the first entry is the OS drive.
 type DiskConfig struct {
 	FS    FSType
 	Disks []Disk
 	ZFS   ZFSOpts
+
+	// Roles assigns ext4 drives to mountpoints, in AllDiskRoles order. Empty
+	// means a single-disk install with everything on the root filesystem.
+	Roles []RoleMount
 }
 
-// Primary is the first selected disk — the whole target in ext4 mode, and the
+// WithRoles returns a copy with ext4 drive roles assigned and Disks rebuilt
+// from them in canonical order, so the list of drives to erase and the list of
+// roles cannot drift apart — there is one place a drive can be named, not two.
+func (d DiskConfig) WithRoles(roles []RoleMount) DiskConfig {
+	d.Roles, d.Disks = nil, nil
+	add := func(rm RoleMount) {
+		d.Roles = append(d.Roles, rm)
+		d.Disks = append(d.Disks, rm.Disk)
+	}
+	for _, want := range AllDiskRoles {
+		for _, rm := range roles {
+			if rm.Role == want {
+				add(rm)
+			}
+		}
+	}
+	// Anything with an unknown or duplicate role is kept so Validate can name
+	// it, rather than being dropped into a silently smaller selection.
+	for _, rm := range roles {
+		if _, known := roleMountpoints[rm.Role]; !known {
+			add(rm)
+		}
+	}
+	return d
+}
+
+// RolesForDisks assigns roles positionally, which is the default the UI offers:
+// first disk picked is the OS, second is spinifex, third is predastore.
+func RolesForDisks(disks []Disk) []RoleMount {
+	out := make([]RoleMount, 0, len(disks))
+	for i, disk := range disks {
+		var role DiskRole
+		if i < len(AllDiskRoles) {
+			role = AllDiskRoles[i]
+		}
+		out = append(out, RoleMount{Role: role, Disk: disk})
+	}
+	return out
+}
+
+// DataMounts is every role except the OS drive, shallowest mountpoint first.
+// This is the traversal every stage of the install shares.
+func (d DiskConfig) DataMounts() []RoleMount {
+	var out []RoleMount
+	for _, rm := range d.Roles {
+		if rm.Role != RoleOS {
+			out = append(out, rm)
+		}
+	}
+	return out
+}
+
+// bootDisks are the drives a bootloader is installed to. Every ZFS member gets
+// its own ESP; on ext4 only the OS drive does, because a data drive holding a
+// bootloader that nothing points at is a trap during recovery.
+func (d DiskConfig) bootDisks() []Disk {
+	if d.FS.IsZFS() || len(d.Disks) == 0 {
+		return d.Disks
+	}
+	return d.Disks[:1]
+}
+
+// Primary is the first selected disk — the OS drive in ext4 mode, and the
 // disk shown in single-disk summaries.
 func (d DiskConfig) Primary() Disk {
 	if len(d.Disks) == 0 {
@@ -222,9 +328,8 @@ func (d DiskConfig) Validate() error {
 	if len(d.Disks) == 0 {
 		return fmt.Errorf("%s: select at least one disk", d.FS.Label())
 	}
-	if !d.FS.IsZFS() && len(d.Disks) > 1 {
-		return fmt.Errorf("%s: supports a single disk only, %d selected — choose a zfs mode to use them all",
-			d.FS.Label(), len(d.Disks))
+	if err := d.validateRoles(); err != nil {
+		return err
 	}
 	if n := len(d.Disks); n < d.FS.MinDisks() {
 		return fmt.Errorf("%s: needs at least %d disks, %d selected", d.FS.Label(), d.FS.MinDisks(), n)
@@ -243,21 +348,66 @@ func (d DiskConfig) Validate() error {
 		if disk.LiveMedia {
 			return fmt.Errorf("%s is the installer's own boot media and cannot be erased", disk.Path)
 		}
-		// GRUB's i386-pc target cannot read a 4Kn drive, so on legacy BIOS the
-		// installed system would have no reachable bootloader.
-		if !bootedEFI() && disk.LogicalBlockSize == 4096 {
-			return fmt.Errorf("%s is a 4Kn drive (4096-byte sectors) and this machine booted in legacy BIOS mode — booting from it is not supported; enable UEFI in firmware",
-				disk.Path)
-		}
 		if disk.Bytes < minRootBytes {
 			return fmt.Errorf("%s is too small (%s, need at least %dGiB)",
 				disk.Path, disk.SizeHuman(), minRootBytes>>30)
 		}
 	}
 
+	// GRUB's i386-pc target cannot read a 4Kn drive, so on legacy BIOS a drive
+	// holding a bootloader would leave the installed system unreachable. Only
+	// boot drives are constrained: a data role is never booted from.
+	for _, disk := range d.bootDisks() {
+		if !bootedEFI() && disk.LogicalBlockSize == 4096 {
+			return fmt.Errorf("%s is a 4Kn drive (4096-byte sectors) and this machine booted in legacy BIOS mode — booting from it is not supported; enable UEFI in firmware",
+				disk.Path)
+		}
+	}
+
 	return d.validateSizes()
 }
 
+// validateRoles checks the ext4 role assignment. Roles are the only way to use
+// more than one disk without ZFS, so a multi-disk selection without them is
+// rejected rather than quietly installed onto the first drive.
+func (d DiskConfig) validateRoles() error {
+	if d.FS.IsZFS() {
+		if len(d.Roles) > 0 {
+			return fmt.Errorf("%s: drive roles apply to ext4 only — a pool spans every member already", d.FS.Label())
+		}
+		return nil
+	}
+	if len(d.Roles) == 0 {
+		if len(d.Disks) > 1 {
+			return fmt.Errorf("%s: %d disks selected but no roles assigned — assign os, spinifex and predastore, or choose a zfs mode",
+				d.FS.Label(), len(d.Disks))
+		}
+		return nil
+	}
+	if len(d.Roles) > len(AllDiskRoles) {
+		return fmt.Errorf("%s: at most %d disks, %d selected — the roles are os, spinifex and predastore",
+			d.FS.Label(), len(AllDiskRoles), len(d.Roles))
+	}
+
+	seen := map[DiskRole]bool{}
+	for _, rm := range d.Roles {
+		if _, ok := roleMountpoints[rm.Role]; !ok {
+			return fmt.Errorf("%s has no role assigned — every selected disk needs one", rm.Disk.Path)
+		}
+		if seen[rm.Role] {
+			return fmt.Errorf("two disks are assigned the %s role", rm.Role)
+		}
+		seen[rm.Role] = true
+	}
+	if !seen[RoleOS] {
+		return fmt.Errorf("%s: no disk assigned the os role — one disk must hold the operating system", d.FS.Label())
+	}
+	if len(d.Roles) != len(d.Disks) {
+		return fmt.Errorf("internal: %d roles for %d disks", len(d.Roles), len(d.Disks))
+	}
+	return nil
+}
+
 // minRootBytes matches Proxmox's hard floor for a root disk.
 const minRootBytes = 2 << 30
 
@@ -314,5 +464,20 @@ func (d DiskConfig) Warnings() []string {
 	if d.FS.IsZFS() && d.FS != FSZFSRAID0 && d.Buildable() && d.Tolerated() == 0 {
 		out = append(out, "this topology stores no redundancy")
 	}
+	// A three-drive role layout looks like an array and is not one: each drive
+	// is an independent point of failure, so the node loses whatever sat on it.
+	if len(d.DataMounts()) > 0 {
+		out = append(out, "role-assigned drives store no redundancy — losing any one drive loses what it held")
+	}
+	for _, rm := range d.DataMounts() {
+		if rm.Role == RolePredastore && rm.Disk.Bytes < smallPredastoreBytes {
+			out = append(out, fmt.Sprintf("%s is small for object storage (%s) — predastore will fill quickly",
+				rm.Disk.Path, rm.Disk.SizeHuman()))
+		}
+	}
 	return out
 }
+
+// smallPredastoreBytes is where a dedicated object-store drive stops being
+// worth the drive. Advisory only: the install is legal below it.
+const smallPredastoreBytes = 64 << 30

cmd/installer/install/diskconfig_test.go

@@ -20,7 +20,7 @@ func TestDiskConfigValidate(t *testing.T) {
 
 		{"no filesystem chosen", DiskConfig{Disks: disks(1, 40)}, "no filesystem selected"},
 		{"no disks", DiskConfig{FS: FSZFSRAIDZ1}, "select at least one disk"},
-		{"ext4 cannot span disks", DiskConfig{FS: FSExt4, Disks: disks(2, 40)}, "single disk only"},
+		{"ext4 spans disks only with roles", DiskConfig{FS: FSExt4, Disks: disks(2, 40)}, "no roles assigned"},
 		{"raidz1 needs three", DiskConfig{FS: FSZFSRAIDZ1, Disks: disks(2, 40)}, "at least 3 disks"},
 		{"raidz2 needs four", DiskConfig{FS: FSZFSRAIDZ2, Disks: disks(3, 40)}, "at least 4 disks"},
 		{"raidz3 needs five", DiskConfig{FS: FSZFSRAIDZ3, Disks: disks(4, 40)}, "at least 5 disks"},

cmd/installer/install/install.go

@@ -61,6 +61,7 @@ func Run(cfg *Config) error {
 	}
 	steps = append(steps,
 		step{"copy rootfs", func() error { return copyRootfs(cfg.Storage) }},
+		step{"verify role layout", func() error { return verifyRoleLayout(cfg.Storage) }},
 		step{"create swap", func() error { return createSwap(cfg.Storage) }},
 		step{"write fstab", func() error { return writeFstab(cfg.Storage) }},
 		step{"install spinifex", func() error { return installSpinifex(cfg) }},
@@ -84,7 +85,7 @@ func Run(cfg *Config) error {
 	return reboot()
 }
 
-// ext4RootSteps formats and mounts a single-disk ext4 root.
+// ext4RootSteps formats and mounts the ext4 root and any role filesystems.
 func ext4RootSteps(cfg *Config) []step {
 	return []step{
 		{"format partitions", func() error { return formatPartitions(cfg.Storage) }},
@@ -124,15 +125,36 @@ func cleanupTarget(cfg DiskConfig) {
 		exportPool()
 		return
 	}
+	// Deepest first: predastore nests inside spinifex, and unmounting the outer
+	// filesystem fails with EBUSY while the inner one is still attached.
+	for _, rm := range slices.Backward(cfg.DataMounts()) {
+		_ = runQuiet("umount", targetPath(rm.Mountpoint()))
+	}
 	_ = runQuiet("umount", efiPart())
 	_ = runQuiet("umount", mountRoot)
 }
 
+// targetPath maps a path on the installed system to its location under the
+// install mount root.
+func targetPath(mountpoint string) string {
+	return filepath.Join(mountRoot, strings.TrimPrefix(mountpoint, "/"))
+}
+
 func formatPartitions(cfg DiskConfig) error {
 	if err := formatESPs(cfg); err != nil {
 		return err
 	}
-	return run("mkfs.ext4", "-F", cfg.Primary().PartitionPath(rootPartNum))
+	if err := run("mkfs.ext4", "-F", cfg.Primary().PartitionPath(rootPartNum)); err != nil {
+		return err
+	}
+	for _, rm := range cfg.DataMounts() {
+		// Labelled with the role so `lsblk -f` identifies each drive on a
+		// console that has no access to the installer log.
+		if err := run("mkfs.ext4", "-F", "-L", rm.Role.Label(), rm.Disk.PartitionPath(dataPartNum)); err != nil {
+			return fmt.Errorf("format %s (%s): %w", rm.Disk.Path, rm.Role, err)
+		}
+	}
+	return nil
 }
 
 func mountPartitions(cfg DiskConfig) error {
@@ -146,7 +168,36 @@ func mountPartitions(cfg DiskConfig) error {
 	if err := os.MkdirAll(efiPart(), 0o755); err != nil {
 		return err
 	}
-	return run("mount", d.PartitionPath(espPartNum), efiPart())
+	if err := run("mount", d.PartitionPath(espPartNum), efiPart()); err != nil {
+		return err
+	}
+	// Role filesystems are mounted here, before the rootfs copy, so rsync lays
+	// each tree down on its own drive with the ownership setup.sh gave it at
+	// ISO build time. Mounting afterwards hides those trees and loses it.
+	for _, rm := range cfg.DataMounts() {
+		target := targetPath(rm.Mountpoint())
+		if err := os.MkdirAll(target, 0o755); err != nil {
+			return err
+		}
+		if err := run("mount", rm.Disk.PartitionPath(dataPartNum), target); err != nil {
+			return fmt.Errorf("mount %s at %s: %w", rm.Disk.Path, rm.Mountpoint(), err)
+		}
+	}
+	return nil
+}
+
+// mountProtectFilters stops rsync's --delete from acting across a mount
+// boundary. Every mountpoint the installer created before the copy needs one,
+// whether it is a ZFS dataset or a role filesystem.
+func mountProtectFilters(cfg DiskConfig) []string {
+	if cfg.FS.IsZFS() {
+		return datasetProtectFilters(cfg)
+	}
+	var out []string
+	for _, rm := range cfg.DataMounts() {
+		out = append(out, "--filter=P "+rm.Mountpoint())
+	}
+	return out
 }
 
 // copyRootfs copies the live squashfs environment onto the target disk using
@@ -171,7 +222,7 @@ func copyRootfs(cfg DiskConfig) error {
 		"--exclude=/lost+found",
 		"--exclude=/boot/efi",
 	}
-	args = append(args, datasetProtectFilters(cfg)...)
+	args = append(args, mountProtectFilters(cfg)...)
 	args = append(args, "/", mountRoot+"/")
 	if err := run("rsync", args...); err != nil {
 		return err
@@ -213,6 +264,71 @@ func copyRootfs(cfg DiskConfig) error {
 	return nil
 }
 
+// verifyRoleLayout asserts the rootfs copy landed on the role drives rather
+// than on top of them. A mount that happened in the wrong order produces a node
+// that boots, starts every service, and has each one writing to an empty
+// directory on the wrong drive — healthy-looking and silently wrong.
+func verifyRoleLayout(cfg DiskConfig) error {
+	for _, rm := range cfg.DataMounts() {
+		target := targetPath(rm.Mountpoint())
+		dev, err := deviceOf(target)
+		if err != nil {
+			return fmt.Errorf("verify %s: %w", rm.Mountpoint(), err)
+		}
+		parentDev, err := deviceOf(filepath.Dir(target))
+		if err != nil {
+			return fmt.Errorf("verify %s: %w", rm.Mountpoint(), err)
+		}
+		if dev == parentDev {
+			return fmt.Errorf("%s is not a separate filesystem — %s was not mounted before the rootfs copy",
+				rm.Mountpoint(), rm.Disk.Path)
+		}
+		if err := matchesLiveOwnership(target, rm.Mountpoint()); err != nil {
+			return err
+		}
+		// Logged so the layout is recoverable from the install log alone, which
+		// on a headless node is the only record of where each workload landed.
+		slog.Info("role mounted", "role", rm.Role, "mountpoint", rm.Mountpoint(), "disk", rm.Disk.Path)
+	}
+	return nil
+}
+
+// matchesLiveOwnership compares a copied directory against the live rootfs,
+// which is where setup.sh set every service owner at ISO build time. Comparing
+// rather than hardcoding uids keeps this correct as services are added.
+func matchesLiveOwnership(target, source string) error {
+	src, err := os.Stat(source)
+	if err != nil {
+		// Nothing to compare against; the mount check above still holds.
+		slog.Warn("skipping ownership check, path absent in live environment", "path", source, "err", err)
+		return nil
+	}
+	dst, err := os.Stat(target)
+	if err != nil {
+		return fmt.Errorf("verify %s: %w", source, err)
+	}
+	srcSt, dstSt := src.Sys().(*syscall.Stat_t), dst.Sys().(*syscall.Stat_t)
+	if srcSt.Uid != dstSt.Uid || srcSt.Gid != dstSt.Gid || src.Mode().Perm() != dst.Mode().Perm() {
+		return fmt.Errorf("%s copied as %d:%d %o, expected %d:%d %o — the drive was mounted after the rootfs copy",
+			source, dstSt.Uid, dstSt.Gid, dst.Mode().Perm(), srcSt.Uid, srcSt.Gid, src.Mode().Perm())
+	}
+	return nil
+}
+
+// deviceOf returns the filesystem a path lives on, for distinguishing a real
+// mount from a directory of the same name.
+func deviceOf(path string) (uint64, error) {
+	fi, err := os.Stat(path)
+	if err != nil {
+		return 0, err
+	}
+	st, ok := fi.Sys().(*syscall.Stat_t)
+	if !ok {
+		return 0, fmt.Errorf("%s: no stat information", path)
+	}
+	return st.Dev, nil
+}
+
 func installSpinifex(cfg *Config) error {
 	// The rootfs copy already contains spx and spinifex-installer at
 	// /usr/local/bin/ — no binary copy needed. Regenerate machine-specific
@@ -808,6 +924,18 @@ func writeFstab(cfg DiskConfig) error {
 		}
 		fstab += fmt.Sprintf("UUID=%s / ext4 errors=remount-ro 0 1\nUUID=%s /boot/efi vfat umask=0077 0 1\n",
 			rootUUID, efiUUID)
+
+		// Shallowest first, so `mount -a` in a rescue shell works top-down.
+		// Deliberately no nofail: a missing data drive must stop the boot for
+		// an admin, not let predastore write objects onto the OS drive while
+		// every service reports healthy. fsck pass 2 keeps root first.
+		for _, rm := range cfg.DataMounts() {
+			uuid, err := partUUID(rm.Disk.PartitionPath(dataPartNum))
+			if err != nil {
+				return fmt.Errorf("get %s UUID: %w", rm.Role, err)
+			}
+			fstab += fmt.Sprintf("UUID=%s %s ext4 errors=remount-ro 0 2\n", uuid, rm.Mountpoint())
+		}
 	}
 
 	// Only claim the swap the previous step actually produced; systemd fails the
@@ -1014,7 +1142,9 @@ func parseSize(s string) (int64, error) {
 	return n * mult, nil
 }
 
-func partUUID(dev string) (string, error) {
+// partUUID reads a partition's filesystem UUID. A variable so fstab generation
+// can be tested without real block devices.
+var partUUID = func(dev string) (string, error) {
 	out, err := exec.Command("blkid", "-s", "UUID", "-o", "value", dev).Output()
 	if err != nil {
 		return "", fmt.Errorf("blkid %s: %w", dev, err)

cmd/installer/install/roles_test.go

@@ -0,0 +1,391 @@
+package install
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+)
+
+// roleCfg builds an ext4 configuration from role/disk-name pairs, in whatever
+// order they are given — WithRoles is responsible for normalising them.
+func roleCfg(pairs ...any) DiskConfig {
+	var roles []RoleMount
+	for i := 0; i+1 < len(pairs); i += 2 {
+		roles = append(roles, RoleMount{
+			Role: pairs[i].(DiskRole),
+			Disk: disk(pairs[i+1].(string), 100),
+		})
+	}
+	return DiskConfig{FS: FSExt4}.WithRoles(roles)
+}
+
+// fakeUUIDs makes partUUID return a value derived from the device path, so a
+// generated fstab can be checked without touching a real disk.
+func fakeUUIDs(t *testing.T) {
+	t.Helper()
+	prev := partUUID
+	partUUID = func(dev string) (string, error) { return "uuid" + strings.ReplaceAll(dev, "/", "-"), nil }
+	t.Cleanup(func() { partUUID = prev })
+}
+
+func TestWithRolesNormalisesOrderAndDerivesDisks(t *testing.T) {
+	// Given deepest-first, which is the order that breaks mounting.
+	cfg := roleCfg(RolePredastore, "c", RoleSpinifex, "b", RoleOS, "a")
+
+	wantRoles := []DiskRole{RoleOS, RoleSpinifex, RolePredastore}
+	for i, rm := range cfg.Roles {
+		if rm.Role != wantRoles[i] {
+			t.Fatalf("role %d = %s, want %s", i, rm.Role, wantRoles[i])
+		}
+	}
+	// Disks is derived, so the OS drive is first and every role drive appears
+	// exactly once in the list the installer erases.
+	if got := cfg.Paths(); !slices.Equal(got, []string{"/dev/a", "/dev/b", "/dev/c"}) {
+		t.Fatalf("Paths() = %v, want the roles in canonical order", got)
+	}
+	if cfg.Primary().Path != "/dev/a" {
+		t.Errorf("Primary() = %s, want the os drive", cfg.Primary().Path)
+	}
+}
+
+func TestDataMountsExcludeTheOSDriveAndNestShallowestFirst(t *testing.T) {
+	got := roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c").DataMounts()
+	want := []string{"/var/lib/spinifex", "/var/lib/spinifex/predastore"}
+	for i, rm := range got {
+		if rm.Mountpoint() != want[i] {
+			t.Fatalf("mount %d = %s, want %s", i, rm.Mountpoint(), want[i])
+		}
+	}
+	if len(got) != len(want) {
+		t.Fatalf("got %d data mounts, want %d", len(got), len(want))
+	}
+}
+
+// A predastore drive without a spinifex drive is the two-drive layout, and it
+// nests a mount inside the root filesystem rather than inside another mount.
+func TestPredastoreRoleWithoutSpinifexIsValid(t *testing.T) {
+	cfg := roleCfg(RoleOS, "a", RolePredastore, "b")
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("Validate() = %v, want nil", err)
+	}
+	if got := cfg.DataMounts(); len(got) != 1 || got[0].Mountpoint() != "/var/lib/spinifex/predastore" {
+		t.Fatalf("DataMounts() = %v, want predastore only", got)
+	}
+}
+
+func TestValidateRoles(t *testing.T) {
+	twoRoles := func(a, b DiskRole) DiskConfig { return roleCfg(a, "a", b, "b") }
+
+	tests := []struct {
+		name    string
+		cfg     DiskConfig
+		wantErr string
+	}{
+		{"single disk needs no roles", DiskConfig{FS: FSExt4, Disks: disks(1, 40)}, ""},
+		{"os alone", roleCfg(RoleOS, "a"), ""},
+		{"all three", roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c"), ""},
+		{"no os role", twoRoles(RoleSpinifex, RolePredastore), "no disk assigned the os role"},
+		{"duplicate role", twoRoles(RoleOS, RoleOS), "two disks are assigned the os role"},
+		{"unknown role", roleCfg(RoleOS, "a", DiskRole("cache"), "b"), "no role assigned"},
+		{"more disks than roles", roleCfg(
+			RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c", DiskRole("cache"), "d"), "at most 3 disks"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.cfg.Validate()
+			switch {
+			case tt.wantErr == "" && err != nil:
+				t.Fatalf("Validate() = %v, want nil", err)
+			case tt.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tt.wantErr)):
+				t.Fatalf("Validate() = %v, want error containing %q", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+// Roles describe where a workload lives on a plain filesystem. A pool already
+// spans every member, so accepting them there would silently do nothing.
+func TestValidateRejectsRolesOnZFS(t *testing.T) {
+	cfg := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+	cfg.Roles = []RoleMount{{Role: RoleOS, Disk: cfg.Disks[0]}}
+	if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "ext4 only") {
+		t.Fatalf("Validate() = %v, want a roles-are-ext4-only error", err)
+	}
+}
+
+func TestBootDisksAreOnlyTheOSDriveOnExt4(t *testing.T) {
+	cfg := roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c")
+	if got := cfg.bootDisks(); len(got) != 1 || got[0].Path != "/dev/a" {
+		t.Fatalf("bootDisks() = %v, want the os drive only", got)
+	}
+	// Every ZFS member carries its own ESP, so all of them stay boot disks.
+	zfs := DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}
+	if got := zfs.bootDisks(); len(got) != 2 {
+		t.Fatalf("bootDisks() = %v, want every pool member", got)
+	}
+}
+
+func TestWriteFstabEmitsRoleMountsShallowestFirst(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+	fakeUUIDs(t)
+
+	cfg := roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c")
+	if err := writeFstab(cfg); err != nil {
+		t.Fatalf("writeFstab: %v", err)
+	}
+	got := readFile(t, root, "etc/fstab")
+
+	// Order matters: `mount -a` in a rescue shell walks the file top-down, and
+	// predastore cannot be mounted before the filesystem it sits inside.
+	wantOrder := []string{" / ", "/boot/efi", " /var/lib/spinifex ", " /var/lib/spinifex/predastore "}
+	at := -1
+	for _, want := range wantOrder {
+		i := strings.Index(got, want)
+		if i < 0 {
+			t.Fatalf("fstab is missing %q:\n%s", want, got)
+		}
+		if i < at {
+			t.Fatalf("%q is out of order:\n%s", want, got)
+		}
+		at = i
+	}
+
+	for _, dev := range []string{"/dev/b1", "/dev/c1"} {
+		if want := "UUID=uuid-dev-" + strings.TrimPrefix(dev, "/dev/"); !strings.Contains(got, want) {
+			t.Errorf("fstab does not mount %s by UUID:\n%s", dev, got)
+		}
+	}
+	// Data mounts are fsck'd after root, never alongside it.
+	for line := range strings.SplitSeq(got, "\n") {
+		if strings.Contains(line, "/var/lib/spinifex") && !strings.HasSuffix(line, "0 2") {
+			t.Errorf("data mount should be fsck pass 2: %q", line)
+		}
+	}
+}
+
+// Fail-closed by design: a missing data drive must stop the boot for an admin,
+// not let predastore write objects onto the OS drive while everything reports
+// healthy. nofail would silently produce exactly that.
+func TestWriteFstabNeverMarksRoleMountsNofail(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+	fakeUUIDs(t)
+
+	if err := writeFstab(roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c")); err != nil {
+		t.Fatalf("writeFstab: %v", err)
+	}
+	if got := readFile(t, root, "etc/fstab"); strings.Contains(got, "nofail") {
+		t.Errorf("role mounts must not be nofail:\n%s", got)
+	}
+}
+
+func TestWriteFstabSingleDiskIsUnchanged(t *testing.T) {
+	root := withMountRoot(t)
+	targetSkeleton(t, root)
+	fakeUUIDs(t)
+
+	if err := writeFstab(roleCfg(RoleOS, "a")); err != nil {
+		t.Fatalf("writeFstab: %v", err)
+	}
+	if got := readFile(t, root, "etc/fstab"); strings.Contains(got, "/var/lib/spinifex") {
+		t.Errorf("a single-disk install should mount nothing extra:\n%s", got)
+	}
+}
+
+// rsync runs with --delete, so every mountpoint that exists before the copy
+// needs protecting or the copy can delete across a mount boundary on a retry.
+func TestMountProtectFiltersCoverEveryRoleMount(t *testing.T) {
+	got := mountProtectFilters(roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c"))
+	want := []string{"--filter=P /var/lib/spinifex", "--filter=P /var/lib/spinifex/predastore"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("mountProtectFilters() = %v, want %v", got, want)
+	}
+	if got := mountProtectFilters(roleCfg(RoleOS, "a")); len(got) != 0 {
+		t.Errorf("a single-disk install has no mounts to protect, got %v", got)
+	}
+	// ZFS keeps its dataset filters — the union is by mode, not additive.
+	if got := mountProtectFilters(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)}); len(got) == 0 {
+		t.Error("zfs mode should still protect its dataset mountpoints")
+	}
+}
+
+func TestMountPartitionsMountsRolesAfterRootAndBeforeTheCopy(t *testing.T) {
+	f := fakeCommands(t)
+	root := withMountRoot(t)
+
+	cfg := roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c")
+	if err := mountPartitions(cfg); err != nil {
+		t.Fatalf("mountPartitions: %v", err)
+	}
+
+	// The nested mount cannot precede the one it sits inside, or it is masked.
+	want := []string{
+		"mount /dev/a3 " + root,
+		"mount /dev/a2 " + root + "/boot/efi",
+		"mount /dev/b1 " + root + "/var/lib/spinifex",
+		"mount /dev/c1 " + root + "/var/lib/spinifex/predastore",
+	}
+	var mounts []string
+	for _, c := range f.calls {
+		if strings.HasPrefix(c, "mount ") {
+			mounts = append(mounts, c)
+		}
+	}
+	if !slices.Equal(mounts, want) {
+		t.Fatalf("mount order:\n got %v\nwant %v", mounts, want)
+	}
+}
+
+func TestCleanupTargetUnmountsDeepestFirst(t *testing.T) {
+	f := fakeCommands(t)
+	root := withMountRoot(t)
+
+	cleanupTarget(roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c"))
+
+	// Unmounting the outer filesystem first fails with EBUSY and leaves a
+	// half-mounted target behind for the next attempt.
+	want := []string{
+		"umount " + root + "/var/lib/spinifex/predastore",
+		"umount " + root + "/var/lib/spinifex",
+		"umount " + root + "/boot/efi",
+		"umount " + root,
+	}
+	// The chroot binds are released first and are not part of the target order.
+	var got []string
+	for _, c := range f.calls {
+		if strings.HasPrefix(c, "umount ") && slices.Contains(want, c) {
+			got = append(got, c)
+		}
+	}
+	if !slices.Equal(got, want) {
+		t.Fatalf("unmount order:\n got %v\nwant %v", got, want)
+	}
+}
+
+func TestPartitionDisksGivesDataDrivesOnePartitionAndNoESP(t *testing.T) {
+	f := fakeCommands(t)
+	nodes := tempDisks(t, 2, 200)
+	cfg := DiskConfig{FS: FSExt4}.WithRoles(
+		[]RoleMount{{Role: RoleOS, Disk: nodes[0]}, {Role: RoleSpinifex, Disk: nodes[1]}})
+	osDisk, data := nodes[0].Path, nodes[1].Path
+
+	if err := partitionDisks(cfg); err != nil {
+		t.Fatalf("partitionDisks: %v", err)
+	}
+
+	f.mustRun(t, "sgdisk", osDisk, "-n 3:0:-1024M", "-t 3:8300")
+	f.mustRun(t, "sgdisk", osDisk, "-t 2:EF00")
+	// A data drive is never booted from, so a bios_boot or an ESP there would
+	// only mislead whoever reads lsblk during an incident.
+	for _, c := range f.calls {
+		if strings.Contains(c, data) && (strings.Contains(c, "EF02") || strings.Contains(c, "EF00")) {
+			t.Errorf("data drive must get no bios_boot or ESP: %q", c)
+		}
+	}
+	f.mustRun(t, "sgdisk", data, "-n 1:1M:-1024M", "-t 1:8300", "-c 1:spinifex")
+}
+
+func TestFormatPartitionsLabelsEachRoleFilesystem(t *testing.T) {
+	f := fakeCommands(t)
+	cfg := roleCfg(RoleOS, "a", RoleSpinifex, "b", RolePredastore, "c")
+
+	if err := formatPartitions(cfg); err != nil {
+		t.Fatalf("formatPartitions: %v", err)
+	}
+	// One ESP only: a data drive holding a bootloader nothing points at would
+	// mislead whoever reads lsblk during an incident.
+	esps := 0
+	for _, c := range f.calls {
+		if strings.HasPrefix(c, "mkfs.fat") {
+			esps++
+		}
+	}
+	if esps != 1 {
+		t.Errorf("want one ESP, got %d:\n%v", esps, f.calls)
+	}
+	f.mustRun(t, "mkfs.ext4", "-L", "spinifex", "/dev/b1")
+	f.mustRun(t, "mkfs.ext4", "-L", "predastore", "/dev/c1")
+}
+
+func TestWarningsFlagTheAbsenceOfRedundancy(t *testing.T) {
+	got := strings.Join(roleCfg(RoleOS, "a", RoleSpinifex, "b").Warnings(), "\n")
+	if !strings.Contains(got, "no redundancy") {
+		t.Errorf("a multi-drive role layout must warn it is not an array, got %q", got)
+	}
+	if got := roleCfg(RoleOS, "a").Warnings(); len(got) != 0 {
+		t.Errorf("a single-disk install has nothing to warn about, got %v", got)
+	}
+}
+
+func TestWarningsFlagASmallPredastoreDrive(t *testing.T) {
+	cfg := DiskConfig{FS: FSExt4}.WithRoles([]RoleMount{
+		{Role: RoleOS, Disk: disk("a", 500)},
+		{Role: RolePredastore, Disk: disk("b", 8)},
+	})
+	got := strings.Join(cfg.Warnings(), "\n")
+	if !strings.Contains(got, "small for object storage") {
+		t.Errorf("an 8GiB predastore drive should be flagged, got %q", got)
+	}
+	if err := cfg.Validate(); err != nil {
+		t.Errorf("the warning must not block the install: %v", err)
+	}
+}
+
+func TestRolesForDisksAssignsPositionally(t *testing.T) {
+	got := RolesForDisks(disks(3, 40))
+	for i, rm := range got {
+		if want := AllDiskRoles[i]; rm.Role != want {
+			t.Errorf("disk %d got role %s, want %s", i, rm.Role, want)
+		}
+	}
+	// A fourth disk has no role, which Validate is left to reject by name.
+	if extra := RolesForDisks(disks(4, 40)); extra[3].Role != "" {
+		t.Errorf("disk 4 got role %q, want none", extra[3].Role)
+	}
+}
+
+func TestFourKnDataDriveIsAllowedWhereABootDriveIsNot(t *testing.T) {
+	if bootedEFI() {
+		t.Skip("machine booted in EFI mode; the 4Kn rule does not apply")
+	}
+	fourKn := func(name string) Disk {
+		d := disk(name, 100)
+		d.LogicalBlockSize = 4096
+		return d
+	}
+	// GRUB's i386-pc target cannot read a 4Kn drive, but nothing boots from a
+	// data drive, so only the OS role is constrained.
+	ok := DiskConfig{FS: FSExt4}.WithRoles(
+		[]RoleMount{{Role: RoleOS, Disk: disk("a", 100)}, {Role: RoleSpinifex, Disk: fourKn("b")}})
+	if err := ok.Validate(); err != nil {
+		t.Errorf("a 4Kn data drive should be accepted: %v", err)
+	}
+
+	bad := DiskConfig{FS: FSExt4}.WithRoles(
+		[]RoleMount{{Role: RoleOS, Disk: fourKn("a")}, {Role: RoleSpinifex, Disk: disk("b", 100)}})
+	if err := bad.Validate(); err == nil || !strings.Contains(err.Error(), "4Kn") {
+		t.Errorf("Validate() = %v, want a 4Kn rejection for the os drive", err)
+	}
+}
+
+func TestVerifyRoleLayoutCatchesAMountThatNeverHappened(t *testing.T) {
+	root := withMountRoot(t)
+	// Plain directories on one filesystem: exactly what a mount performed after
+	// the rootfs copy, or not at all, leaves behind.
+	if err := os.MkdirAll(filepath.Join(root, "var/lib/spinifex/predastore"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+
+	err := verifyRoleLayout(roleCfg(RoleOS, "a", RoleSpinifex, "b"))
+	if err == nil || !strings.Contains(err.Error(), "not a separate filesystem") {
+		t.Fatalf("verifyRoleLayout() = %v, want a missing-mount error", err)
+	}
+	if !strings.Contains(fmt.Sprint(err), "/dev/b") {
+		t.Errorf("the error must name the drive that was not mounted: %v", err)
+	}
+}

cmd/installer/install/steps_test.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"slices"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -16,17 +18,69 @@ import (
 type fakeExec struct {
 	calls []string
 	fail  func(name string, args []string) error
+
+	// parts is the kernel's partition view per disk. Seeded by the test and then
+	// moved by the sgdisk calls the installer makes, so the partition-table
+	// assertions can run without a block device.
+	parts map[string][]string
+
+	// stale marks disks whose view never moves however many tables are written
+	// to them: something holds the disk open, BLKRRPART returns EBUSY, and the
+	// kernel keeps serving the old table while sgdisk reports success.
+	stale map[string]bool
+}
+
+// holdOpen makes a disk keep its current partition view forever.
+func (f *fakeExec) holdOpen(disk string) { f.stale[disk] = true }
+
+// seedPartitions gives a disk a pre-existing partition table — the state a
+// drive is in when the node has been installed on before.
+func (f *fakeExec) seedPartitions(disk string, nums ...int) {
+	for _, n := range nums {
+		f.parts[disk] = append(f.parts[disk], filepath.Base(partitionPath(disk, n)))
+	}
+}
+
+// applySgdisk moves the fake kernel view in step with a table rewrite, the way
+// a kernel that honoured every BLKRRPART would: -Z empties it, each -n adds the
+// partition it creates.
+func (f *fakeExec) applySgdisk(args []string) {
+	if len(args) == 0 {
+		return
+	}
+	disk := args[len(args)-1]
+	if f.stale[disk] {
+		return
+	}
+	for i, a := range args {
+		switch {
+		case a == "-Z":
+			f.parts[disk] = nil
+		case a == "-n" && i+1 < len(args):
+			num, _, _ := strings.Cut(args[i+1], ":")
+			n, err := strconv.Atoi(num)
+			if err != nil {
+				continue
+			}
+			f.parts[disk] = append(f.parts[disk], filepath.Base(partitionPath(disk, n)))
+		}
+	}
 }
 
 func fakeCommands(t *testing.T) *fakeExec {
 	t.Helper()
-	f := &fakeExec{}
-	prevRun, prevQuiet, prevEnv := run, runQuiet, runEnv
+	f := &fakeExec{parts: map[string][]string{}, stale: map[string]bool{}}
+	prevRun, prevQuiet, prevEnv, prevParts := run, runQuiet, runEnv, kernelPartitions
 
 	record := func(name string, args ...string) error {
 		f.calls = append(f.calls, strings.TrimSpace(name+" "+strings.Join(args, " ")))
 		if f.fail != nil {
-			return f.fail(name, args)
+			if err := f.fail(name, args); err != nil {
+				return err
+			}
+		}
+		if name == "sgdisk" {
+			f.applySgdisk(args)
 		}
 		return nil
 	}
@@ -35,8 +89,21 @@ func fakeCommands(t *testing.T) *fakeExec {
 	runEnv = func(env []string, name string, args ...string) error {
 		return record(name, append(env, args...)...)
 	}
+	kernelPartitions = func(disk string) ([]string, error) {
+		got := slices.Clone(f.parts[disk])
+		slices.Sort(got)
+		return got, nil
+	}
 
-	t.Cleanup(func() { run, runQuiet, runEnv = prevRun, prevQuiet, prevEnv })
+	// A short settle so a test asserting the stale-table failure does not wait
+	// out the install's real budget.
+	prevTimeout := partitionSettleTimeout
+	partitionSettleTimeout = 10 * time.Millisecond
+
+	t.Cleanup(func() {
+		run, runQuiet, runEnv, kernelPartitions = prevRun, prevQuiet, prevEnv, prevParts
+		partitionSettleTimeout = prevTimeout
+	})
 	return f
 }
 
@@ -114,6 +181,24 @@ func targetSkeleton(t *testing.T, root string) {
 
 // tempDisks builds disks whose device nodes are real files, so the steps that
 // wait for partition nodes to appear can complete.
+// fakeProcTables points the mount and swap readers at fixture tables for the
+// duration of a test.
+func fakeProcTables(t *testing.T, mounts, swaps string) {
+	t.Helper()
+	dir := t.TempDir()
+	point := func(dst *string, name, content string) {
+		prev := *dst
+		p := filepath.Join(dir, name)
+		if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
+			t.Fatalf("write %s: %v", p, err)
+		}
+		*dst = p
+		t.Cleanup(func() { *dst = prev })
+	}
+	point(&procMountsPath, "mounts", mounts)
+	point(&procSwapsPath, "swaps", swaps)
+}
+
 func tempDisks(t *testing.T, n int, gib int64) []Disk {
 	t.Helper()
 	dir := t.TempDir()
@@ -237,21 +322,154 @@ func TestEspSizeMiB(t *testing.T) {
 	}
 }
 
-func TestClearDiskOnlyLabelclearsAnExistingPartition(t *testing.T) {
+// A drive arriving with a previous install's layout must have every partition
+// cleared, not just the one the root filesystem used to be on: a data drive's
+// partition is p1, so a p3-only labelclear leaves a former pool member's ZFS
+// label in place at both ends of the device.
+func TestClearDiskWipesEveryPartitionItFinds(t *testing.T) {
 	f := fakeCommands(t)
-	disks := tempDisks(t, 1, 40)
-	if err := clearDisk(disks[0]); err != nil {
+	d := tempDisks(t, 1, 40)[0]
+	f.seedPartitions(d.Path, biosPartNum, espPartNum, rootPartNum)
+
+	if err := clearDisk(d); err != nil {
 		t.Fatalf("clearDisk: %v", err)
 	}
-	f.mustRun(t, "zpool labelclear -f "+disks[0].PartitionPath(rootPartNum))
+	for _, n := range []int{biosPartNum, espPartNum, rootPartNum} {
+		f.mustRun(t, "zpool labelclear -f "+d.PartitionPath(n))
+		f.mustRun(t, "wipefs -a "+d.PartitionPath(n))
+	}
+	f.mustRun(t, "wipefs -a "+d.Path)
+	f.mustRun(t, "sgdisk -Z "+d.Path)
+}
 
-	f2 := fakeCommands(t)
+func TestClearDiskProbesNoPartitionsOnABlankDrive(t *testing.T) {
+	f := fakeCommands(t)
 	blank := Disk{Path: filepath.Join(t.TempDir(), "sdb")}
+
 	if err := clearDisk(blank); err != nil {
 		t.Fatalf("clearDisk: %v", err)
 	}
-	f2.mustNotRun(t, "labelclear")
-	f2.mustRun(t, "wipefs -a "+blank.Path)
+	f.mustNotRun(t, "labelclear")
+	f.mustRun(t, "wipefs -a "+blank.Path)
+}
+
+// The reported field failure: a drive whose old table the kernel never dropped.
+// Proceeding would format the previous layout's offsets, so the install must
+// stop, and the message must say the table is stale rather than blaming the
+// device node that did not appear.
+func TestClearDiskFailsWhenTheKernelKeepsTheOldTable(t *testing.T) {
+	f := fakeCommands(t)
+	d := tempDisks(t, 1, 40)[0]
+	f.seedPartitions(d.Path, biosPartNum, espPartNum, rootPartNum)
+	// sgdisk writes to the platters and exits 0, but BLKRRPART returned EBUSY,
+	// so the kernel's view never changes.
+	f.holdOpen(d.Path)
+
+	err := clearDisk(d)
+	if err == nil {
+		t.Fatal("clearDisk succeeded against a stale partition table")
+	}
+	for _, want := range []string{d.Path, "still shows partitions", "previous layout"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("err = %q, want it to mention %q", err, want)
+		}
+	}
+}
+
+func TestClearDiskReleasesMountsAndSwapFirst(t *testing.T) {
+	f := fakeCommands(t)
+	d := tempDisks(t, 1, 40)[0]
+	f.seedPartitions(d.Path, biosPartNum, espPartNum, rootPartNum)
+
+	root, esp := d.PartitionPath(rootPartNum), d.PartitionPath(espPartNum)
+	swapDev := d.PartitionPath(biosPartNum)
+	fakeProcTables(t,
+		fmt.Sprintf("%s /oldroot ext4 rw 0 0\n%s /oldroot/boot/efi vfat rw 0 0\n", root, esp),
+		fmt.Sprintf("Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n%s partition 1000 0 -2\n", swapDev),
+	)
+
+	if err := clearDisk(d); err != nil {
+		t.Fatalf("clearDisk: %v", err)
+	}
+	f.mustRun(t, "swapoff "+swapDev)
+
+	// Deepest mountpoint first: unmounting /oldroot while /oldroot/boot/efi is
+	// still attached fails with EBUSY and leaves the disk held.
+	var order []string
+	for _, c := range f.calls {
+		if c == "umount "+root || c == "umount "+esp {
+			order = append(order, c)
+		}
+	}
+	if want := []string{"umount " + esp, "umount " + root}; !slices.Equal(order, want) {
+		t.Errorf("unmount order:\n got %v\nwant %v", order, want)
+	}
+}
+
+// A drive held by md or LVM cannot be released — the ISO ships neither tool —
+// so the operator has to be told what is holding it.
+func TestStaleTableErrorNamesTheHolder(t *testing.T) {
+	f := newFakeSysfs(t, fakeDisk{name: "sda", sectors: 1 << 20, partitions: []string{"sda1"}})
+	f.addHolder("sda", "sda1", "md0")
+
+	prev := kernelPartitions
+	kernelPartitions = readKernelPartitions
+	t.Cleanup(func() { kernelPartitions = prev })
+
+	err := staleTableError(Disk{Path: "/dev/sda"}, nil, []string{"sda1"})
+	if err == nil || !strings.Contains(err.Error(), "held by md0") {
+		t.Fatalf("err = %v, want it to name the md0 holder", err)
+	}
+}
+
+// The reported field failure as a unit test: two drives arrive carrying a
+// previous install's partitions and the third has never been partitioned. All
+// three must be taken over, and the blank one must not be the only one to get
+// a new table.
+func TestPartitionDisksTakesOverDrivesThatAlreadyHavePartitions(t *testing.T) {
+	f := fakeCommands(t)
+	nodes := tempDisks(t, 3, 200)
+	f.seedPartitions(nodes[0].Path, biosPartNum, espPartNum, rootPartNum) // held an OS
+	f.seedPartitions(nodes[1].Path, dataPartNum)                          // held data
+	// nodes[2] arrives blank.
+
+	cfg := DiskConfig{FS: FSExt4}.WithRoles([]RoleMount{
+		{Role: RoleOS, Disk: nodes[0]},
+		{Role: RoleSpinifex, Disk: nodes[1]},
+		{Role: RolePredastore, Disk: nodes[2]},
+	})
+	if err := partitionDisks(cfg); err != nil {
+		t.Fatalf("partitionDisks: %v", err)
+	}
+	for _, d := range nodes {
+		f.mustRun(t, "sgdisk -Z "+d.Path)
+	}
+	f.mustRun(t, "sgdisk", nodes[0].Path, "-n 3:0:-1024M")
+	for _, d := range nodes[1:] {
+		f.mustRun(t, "sgdisk", d.Path, "-n 1:1M:-1024M", "-t 1:8300")
+	}
+}
+
+// A previous OS install leaves p1, p2, p3 — exactly the numbering the boot
+// layout writes. Asserting the expected partitions exist would pass on those
+// stale nodes and the install would format the old offsets, so the erase-stage
+// assertion on an empty table is what has to catch it.
+func TestPartitionOneStopsWhenAnOldTableHasTheSameLayout(t *testing.T) {
+	f := fakeCommands(t)
+	d := tempDisks(t, 1, 200)[0]
+	f.seedPartitions(d.Path, biosPartNum, espPartNum, rootPartNum)
+	f.holdOpen(d.Path)
+
+	err := partitionOne(d, typeLinuxFS)
+	if err == nil {
+		t.Fatal("partitionOne succeeded against a stale partition table")
+	}
+	// Specifically the erase-stage message: reaching the post-write check would
+	// mean the stale nodes had already been accepted.
+	if !strings.Contains(err.Error(), "still shows partitions") {
+		t.Errorf("err = %q, want the erase-stage stale-table message", err)
+	}
+	f.mustNotRun(t, "sgdisk -n")
 }
 
 func TestFormatESPsLabelsEveryMember(t *testing.T) {
@@ -669,13 +887,6 @@ func TestInstallBootToolZFSStopsWhenADiskFails(t *testing.T) {
 	f.mustNotRun(t, "SPINIFEX_BOOT_TOOL_NVRAM=1")
 }
 
-func TestRefreshBootToolNoteOnlyAppliesToZFS(t *testing.T) {
-	// Log-only, but it is the operator's only record of the disk-replacement
-	// command, so both branches are walked.
-	refreshBootToolNote(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)})
-	refreshBootToolNote(DiskConfig{FS: FSExt4, Disks: disks(1, 40)})
-}
-
 func TestMaskSystemdUnitIsIdempotent(t *testing.T) {
 	root := t.TempDir()
 	if err := os.MkdirAll(filepath.Join(root, "etc/systemd/system"), 0o755); err != nil {
@@ -695,14 +906,6 @@ func TestMaskSystemdUnitIsIdempotent(t *testing.T) {
 	}
 }
 
-func TestCopyAssetsAreBestEffort(t *testing.T) {
-	root := t.TempDir()
-	// Neither asset exists outside the live ISO; a missing one must not fail
-	// the install, only the branding.
-	copyGrubFont(root)
-	copySplashImage(root)
-}
-
 func TestWriteFstabOnZFSHoldsNothingButSwap(t *testing.T) {
 	root := withMountRoot(t)
 	targetSkeleton(t, root)

cmd/installer/install/zfs.go

@@ -182,8 +182,7 @@ func ImportablePools() ([]string, error) {
 		// zpool exits non-zero when there is nothing to import, which is the
 		// normal case. Anything else — no binary, no /dev/zfs — must surface, or
 		// a machine with an existing pool looks empty to the caller.
-		var exitErr *exec.ExitError
-		if errors.As(err, &exitErr) {
+		if _, ok := errors.AsType[*exec.ExitError](err); ok {
 			return nil, nil
 		}
 		return nil, fmt.Errorf("zpool import: %w", err)
@@ -340,23 +339,35 @@ func exportPool() {
 	}
 }
 
-// clearDisk removes any prior filesystem, partition table and ZFS label from a
-// disk so a retry after a partial install is not rejected for stale metadata —
-// a failure that otherwise reads as mysterious rather than as leftover debris.
+// clearDisk takes a disk over unconditionally, whatever it holds: everything
+// attached to it is released, every signature on it is wiped, its partition
+// table is erased, and the kernel is made to agree the disk is now empty.
+//
+// The last part is the point. sgdisk writes to the platters and exits 0 even
+// when the kernel refuses to drop the old table, so without the assertion an
+// install proceeds against partition nodes describing the previous layout.
 //
 // The probes are quiet because a blank disk makes all of them fail, and their
 // stderr on the install console reads like the install is going wrong.
 func clearDisk(d Disk) error {
-	// ZFS labels live at both ends of the device and wipefs does not know about
-	// them, so an old pool member has to be cleared before its partition goes.
-	if old := d.PartitionPath(rootPartNum); exists(old) {
-		_ = runQuiet("zpool", "labelclear", "-f", old)
-	}
-	_ = runQuiet("wipefs", "-a", d.Path)
-	return runQuiet("sgdisk", "-Z", d.Path)
-}
+	releaseDisk(d)
 
-func exists(path string) bool {
-	_, err := os.Stat(path)
-	return err == nil
+	// Every partition, not just the root slice. ZFS labels live at both ends of
+	// a device and wipefs does not know about them, so a data drive that was
+	// once a pool member keeps its label and fails a later zpool create.
+	parts, err := kernelPartitions(d.Path)
+	if err != nil {
+		return err
+	}
+	for _, p := range parts {
+		dev := partitionDevice(d.Path, p)
+		_ = runQuiet("zpool", "labelclear", "-f", dev)
+		_ = runQuiet("wipefs", "-a", dev)
+	}
+
+	_ = runQuiet("wipefs", "-a", d.Path)
+	if err := runQuiet("sgdisk", "-Z", d.Path); err != nil {
+		return fmt.Errorf("erase partition table: %w", err)
+	}
+	return settlePartitions(d)
 }

cmd/installer/systemd/systemd.go

@@ -68,6 +68,8 @@ ExecStopPost=/bin/sh -c 'if [ "$EXIT_STATUS" != "0" ]; then printf "\n*** Spinif
 # ovn-central startup cannot wedge multi-user.target and keep getty from
 # ever reaching the login prompt.
 TimeoutStartSec=300s
+# Install diagnostics land in the journal, which persists for the life of the
+# node — nothing this script runs may print secrets to stdout or stderr.
 StandardOutput=journal
 StandardError=journal
 

cmd/installer/ui/storage_test.go

@@ -0,0 +1,139 @@
+package ui
+
+import (
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/cmd/installer/install"
+)
+
+// diskModel is a storage screen with n unselected disks and ext4 chosen, which
+// is where the wizard starts.
+func diskModel(n int) model {
+	m := model{fsCursor: slices.Index(install.AllFSTypes, install.FSExt4)}
+	for i := range n {
+		m.disks = append(m.disks, install.Disk{
+			Path:              string(rune('a'+i)) + "-disk",
+			Bytes:             200 << 30,
+			LogicalBlockSize:  512,
+			PhysicalBlockSize: 512,
+		})
+	}
+	return m
+}
+
+func rolePaths(cfg install.DiskConfig) map[install.DiskRole]string {
+	out := map[install.DiskRole]string{}
+	for _, rm := range cfg.Roles {
+		out[rm.Role] = rm.Disk.Path
+	}
+	return out
+}
+
+// A single-disk machine should need no storage input at all, so the disk that
+// newModel preselects has to arrive already holding the os role.
+func TestPreselectedDiskIsAValidConfiguration(t *testing.T) {
+	m := newModel([]install.Disk{{Path: "/dev/sda", Bytes: 200 << 30, LogicalBlockSize: 512}}, nil)
+	cfg := m.storage()
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("the preselected disk must be installable as-is: %v", err)
+	}
+	if cfg.Primary().Path != "/dev/sda" {
+		t.Errorf("Primary() = %q, want the preselected disk", cfg.Primary().Path)
+	}
+}
+
+// Picking disks in order gives the layout an operator almost always wants, so
+// the role key is only needed for the exceptions.
+func TestSelectingDisksAssignsRolesInOrder(t *testing.T) {
+	m := diskModel(3)
+	for i := range 3 {
+		m = m.toggleDisk(i)
+	}
+	got := rolePaths(m.storage())
+	for role, want := range map[install.DiskRole]string{
+		install.RoleOS: "a-disk", install.RoleSpinifex: "b-disk", install.RolePredastore: "c-disk",
+	} {
+		if got[role] != want {
+			t.Errorf("%s = %q, want %q", role, got[role], want)
+		}
+	}
+	if err := m.storage().Validate(); err != nil {
+		t.Errorf("the default assignment must be valid: %v", err)
+	}
+}
+
+// Deselecting has to drop the role with the disk, or every later role shifts
+// onto the wrong drive.
+func TestDeselectingADiskReleasesItsRole(t *testing.T) {
+	m := diskModel(3)
+	for i := range 3 {
+		m = m.toggleDisk(i)
+	}
+	m = m.toggleDisk(1) // drop the spinifex drive
+
+	got := rolePaths(m.storage())
+	if got[install.RoleOS] != "a-disk" || got[install.RolePredastore] != "c-disk" {
+		t.Fatalf("remaining roles moved: %v", got)
+	}
+	if _, ok := got[install.RoleSpinifex]; ok {
+		t.Errorf("the spinifex role should be free again: %v", got)
+	}
+}
+
+// The two-drive predastore layout is only reachable by reassigning, so cycling
+// swaps rather than refusing a role another disk already holds.
+func TestCycleRoleSwapsWithTheDiskThatHoldsIt(t *testing.T) {
+	m := diskModel(2)
+	m = m.toggleDisk(0).toggleDisk(1)
+	// b-disk: spinifex → predastore, which no disk holds, so no swap.
+	m = m.cycleRole(1)
+
+	got := rolePaths(m.storage())
+	if got[install.RoleOS] != "a-disk" || got[install.RolePredastore] != "b-disk" {
+		t.Fatalf("roles = %v, want os on a-disk and predastore on b-disk", got)
+	}
+
+	// Cycling past the end wraps onto os, which a-disk holds, so they trade.
+	m = m.cycleRole(1)
+	got = rolePaths(m.storage())
+	if got[install.RoleOS] != "b-disk" || got[install.RolePredastore] != "a-disk" {
+		t.Fatalf("roles = %v, want the two disks to have swapped", got)
+	}
+	if err := m.storage().Validate(); err != nil {
+		t.Errorf("a swap must always leave a valid assignment: %v", err)
+	}
+}
+
+// ZFS pairs mirrors by selection order and has no roles to assign.
+func TestStorageLeavesRolesUnsetOnZFS(t *testing.T) {
+	m := diskModel(2)
+	m.fsCursor = slices.Index(install.AllFSTypes, install.FSZFSRAID1)
+	m = m.toggleDisk(0).toggleDisk(1)
+
+	cfg := m.storage()
+	if len(cfg.Roles) != 0 {
+		t.Errorf("Roles = %v, want none on a pool", cfg.Roles)
+	}
+	if !slices.Equal(cfg.Paths(), []string{"a-disk", "b-disk"}) {
+		t.Errorf("Paths() = %v, want the selection order preserved", cfg.Paths())
+	}
+}
+
+// A three-drive selection looks like an array and is not one, so the mapping
+// and the absence of redundancy both have to be on screen before the erase.
+func TestConfirmScreenStatesTheLayoutAndTheRisk(t *testing.T) {
+	m := diskModel(3)
+	m.height = 40
+	for i := range 3 {
+		m = m.toggleDisk(i)
+	}
+	got := m.viewDiskConfirm(100)
+
+	for _, want := range []string{"/var/lib/spinifex/predastore", "c-disk", "losing one loses"} {
+		if !strings.Contains(got, want) {
+			t.Errorf("confirm screen is missing %q:\n%s", want, got)
+		}
+	}
+}

cmd/installer/ui/ui.go

@@ -47,9 +47,13 @@ type model struct {
 	//
 	// selected holds indexes into disks in the order they were picked, which is
 	// what pairs members for RAID10 — so it is a slice, not a set.
+	//
+	// roleOf runs parallel to selected and holds the ext4 drive role, so the
+	// two are edited together and cannot drift.
 	disks         []install.Disk
 	storageCursor int
 	selected      []int
+	roleOf        []install.DiskRole
 	fsCursor      int
 	zfsCursor     int
 	zfs           install.ZFSOpts
@@ -173,11 +177,14 @@ func newModel(disks []install.Disk, nics []netprobe.NIC) model {
 	}
 
 	// The first non-live, non-removable disk is preselected so a single-disk
-	// machine needs no storage input at all.
+	// machine needs no storage input at all. It takes the os role, since on ext4
+	// a selected disk with no role is not a valid configuration.
 	var preselected []int
+	var preselectedRoles []install.DiskRole
 	for i, d := range disks {
 		if !d.LiveMedia && !d.Removable {
 			preselected = []int{i}
+			preselectedRoles = []install.DiskRole{install.RoleOS}
 			break
 		}
 	}
@@ -186,6 +193,7 @@ func newModel(disks []install.Disk, nics []netprobe.NIC) model {
 		screen:               screenWelcome,
 		disks:                disks,
 		selected:             preselected,
+		roleOf:               preselectedRoles,
 		nics:                 nics,
 		eraseInput:           eraseIn,
 		roles:                roles,
@@ -439,11 +447,22 @@ func (m model) fsType() install.FSType { return install.AllFSTypes[m.fsCursor] }
 // storage assembles the disk configuration from the current selection.
 func (m model) storage() install.DiskConfig {
 	cfg := install.DiskConfig{FS: m.fsType(), ZFS: m.zfs}
-	for _, i := range m.selected {
-		if i < len(m.disks) {
-			cfg.Disks = append(cfg.Disks, m.disks[i])
+	if cfg.FS.IsZFS() {
+		for _, i := range m.selected {
+			if i < len(m.disks) {
+				cfg.Disks = append(cfg.Disks, m.disks[i])
+			}
 		}
+		return cfg
 	}
+	roles := make([]install.RoleMount, 0, len(m.selected))
+	for si, i := range m.selected {
+		if i >= len(m.disks) || si >= len(m.roleOf) {
+			continue
+		}
+		roles = append(roles, install.RoleMount{Role: m.roleOf[si], Disk: m.disks[i]})
+	}
+	cfg = cfg.WithRoles(roles)
 	return cfg
 }
 
@@ -479,6 +498,10 @@ func (m model) handleDiskKey(key string) (tea.Model, tea.Cmd) {
 			m.zfsCursor = 0
 			m.screen = screenZFSOptions
 		}
+	case "r":
+		if !m.fsType().IsZFS() && m.storageCursor != fsRow {
+			m = m.cycleRole(slices.Index(m.selected, m.storageCursor-1))
+		}
 	case "esc":
 		m.validationErr = ""
 		m.screen = screenWelcome
@@ -502,12 +525,45 @@ func (m model) handleDiskKey(key string) (tea.Model, tea.Cmd) {
 // array with the version bubbletea still holds.
 func (m model) toggleDisk(i int) model {
 	sel := slices.Clone(m.selected)
+	roles := slices.Clone(m.roleOf)
 	if at := slices.Index(sel, i); at >= 0 {
 		sel = slices.Delete(sel, at, at+1)
+		roles = slices.Delete(roles, at, at+1)
 	} else {
 		sel = append(sel, i)
+		roles = append(roles, freeRole(roles))
 	}
 	m.selected = sel
+	m.roleOf = roles
+	m.validationErr = ""
+	return m
+}
+
+// freeRole is the first role no selected disk holds yet, so picking disks in
+// order produces the usual layout without touching the role key at all.
+func freeRole(taken []install.DiskRole) install.DiskRole {
+	for _, r := range install.AllDiskRoles {
+		if !slices.Contains(taken, r) {
+			return r
+		}
+	}
+	return ""
+}
+
+// cycleRole moves the disk at selection index si to the next role, swapping
+// with whichever disk already holds it. Swapping keeps every press legal, so
+// the operator never has to unassign one drive before reassigning another.
+func (m model) cycleRole(si int) model {
+	if si < 0 || si >= len(m.roleOf) {
+		return m
+	}
+	roles := slices.Clone(m.roleOf)
+	next := install.AllDiskRoles[(slices.Index(install.AllDiskRoles, roles[si])+1)%len(install.AllDiskRoles)]
+	if other := slices.Index(roles, next); other >= 0 {
+		roles[other] = roles[si]
+	}
+	roles[si] = next
+	m.roleOf = roles
 	m.validationErr = ""
 	return m
 }
@@ -685,7 +741,7 @@ func (m model) viewDisk(w int) string {
 	subtitle := styleMuted.Render("All data on the selected disks will be permanently erased.")
 
 	fs := m.fsType()
-	req := "single disk"
+	req := "1-3 disks, one per role"
 	if n := fs.MinDisks(); n > 1 {
 		req = fmt.Sprintf("%d+ disks, same size", n)
 	} else if fs.IsZFS() {
@@ -702,8 +758,12 @@ func (m model) viewDisk(w int) string {
 	for i, d := range m.disks {
 		marker := "[ ]"
 		if at := slices.Index(m.selected, i); at >= 0 {
-			// Numbered, not ticked: the order is what pairs mirrors.
+			// Numbered, not ticked: the order is what pairs mirrors. On ext4 the
+			// role is what matters instead, so it takes the same column.
 			marker = fmt.Sprintf("[%d]", at+1)
+			if !fs.IsZFS() && at < len(m.roleOf) {
+				marker = fmt.Sprintf("[%s]", m.roleOf[at])
+			}
 		}
 		note := d.Content
 		switch {
@@ -712,7 +772,7 @@ func (m model) viewDisk(w int) string {
 		case d.Removable:
 			note += ", removable"
 		}
-		line := fmt.Sprintf("  %s %-16s %-8s %-20s %s", marker, d.Path, d.SizeHuman(), truncate(d.Model, 20), note)
+		line := fmt.Sprintf("  %-12s %-16s %-8s %-20s %s", marker, d.Path, d.SizeHuman(), truncate(d.Model, 20), note)
 		if m.storageCursor == i+1 {
 			line = styleSelected.Render("> " + strings.TrimPrefix(line, "  "))
 		} else {
@@ -736,7 +796,7 @@ func (m model) viewDisk(w int) string {
 		rows = append(rows, styleError.Render("  "+m.validationErr))
 	}
 
-	keys := "↑/↓ to move • ←/→ filesystem • Space to select disk • Enter to continue"
+	keys := "↑/↓ to move • ←/→ filesystem • Space to select disk • R to change its role • Enter to continue"
 	if fs.IsZFS() {
 		keys = "↑/↓ to move • ←/→ filesystem • Space to select disk • A for ZFS options • Enter to continue"
 	}
@@ -761,6 +821,11 @@ func (m model) geometryPreview() string {
 		return styleMuted.Render(fmt.Sprintf("  %s %s — %d selected.",
 			cfg.FS.Label(), cfg.Requirement(), len(cfg.Disks)))
 	}
+	// A role layout has no single capacity worth quoting — what matters is
+	// which drive each workload lands on.
+	if mounts := cfg.DataMounts(); len(mounts) > 0 {
+		return styleLabel.Render("  " + roleLayoutLine(cfg))
+	}
 	line := fmt.Sprintf("  %s across %d disk(s) — %s usable",
 		cfg.FS.Label(), len(cfg.Disks), humanBytes(cfg.UsableBytes()))
 	if n := cfg.Tolerated(); n > 0 {
@@ -769,6 +834,16 @@ func (m model) geometryPreview() string {
 	return styleLabel.Render(line)
 }
 
+// roleLayoutLine renders the mountpoint each drive will carry, which is the
+// only summary of a role install an operator can check at a glance.
+func roleLayoutLine(cfg install.DiskConfig) string {
+	parts := make([]string, 0, len(cfg.Roles))
+	for _, rm := range cfg.Roles {
+		parts = append(parts, fmt.Sprintf("%s → %s (%s)", rm.Mountpoint(), rm.Disk.Path, rm.Disk.SizeHuman()))
+	}
+	return strings.Join(parts, "   ")
+}
+
 func (m model) viewZFSOptions(w int) string {
 	title := styleTitle.Render("ZFS Options")
 	subtitle := styleMuted.Render("Defaults are computed from the selected disks and this machine's memory.")
@@ -793,12 +868,20 @@ func (m model) viewZFSOptions(w int) string {
 }
 
 func (m model) viewDiskConfirm(w int) string {
+	cfg := m.storage()
 	title := styleTitle.Render("Confirm Disk Erasure")
-	disk := styleLabel.Render(strings.Join(m.storage().Paths(), ", "))
+	disk := styleLabel.Render(strings.Join(cfg.Paths(), ", "))
 	msg := fmt.Sprintf("All data on %s will be permanently erased.\nType 'yes' to confirm:", disk)
 
 	var lines []string
-	lines = append(lines, title, msg, "", m.eraseInput.View())
+	lines = append(lines, title)
+	// A multi-drive ext4 layout resembles an array and is not one, so the
+	// mapping and the absence of redundancy are both stated before the erase.
+	if len(cfg.DataMounts()) > 0 {
+		lines = append(lines, styleLabel.Render(roleLayoutLine(cfg)),
+			styleWarning.Render("Each drive is independent — losing one loses everything it held."), "")
+	}
+	lines = append(lines, msg, "", m.eraseInput.View())
 	if m.validationErr != "" {
 		lines = append(lines, "", styleError.Render(m.validationErr))
 	}

cmd/lb-agent/main.go

@@ -18,8 +18,8 @@ import (
 	"os/signal"
 	"syscall"
 
+	"github.com/mulgadc/bluebottle/pkg/otelsetup"
 	"github.com/mulgadc/spinifex/spinifex/lbagent"
-	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 
 	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
@@ -28,7 +28,7 @@ func main() {
 	// Without this the agent falls back to slog's default text handler, which folds
 	// attributes into the message so the console scraper recovers a level and nothing
 	// else. No OTLP export: the agent's only egress is the gateway it is watching.
-	otelsetup.SetDefaultJSONLogger(slog.LevelInfo)
+	otelsetup.SetDefaultJSONLogger("lb-agent", slog.LevelInfo)
 
 	var (
 		lbID       string

cmd/rds-agent/acknowledge.go

@@ -0,0 +1,155 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mulgadc/spinifex/internal/rdsgw"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
+)
+
+// The completion receipt rds-init writes once the master role is durably
+// applied. Deliberately outside PGDATA: clear_unfinished_datadir does an rm -rf
+// of PGDATA, so keeping the receipt out of it means no initdb-adjacent path can
+// touch it and the datadir stays a byte-for-byte stock one.
+const (
+	receiptRelDir  = ".spinifex-rds/bootstrap"
+	receiptFile    = "receipt.env"
+	receiptPayload = "RDS_RECEIPT_PAYLOAD_ID"
+	receiptDBID    = "RDS_RECEIPT_DB_INSTANCE_IDENTIFIER"
+)
+
+func (a *Agent) receiptPath() string {
+	return filepath.Join(a.cfg.DataMount, receiptRelDir, receiptFile)
+}
+
+// Confirms the initial bootstrap, which is what destroys the staged ciphertext.
+// A no-op on the attach path, and deliberately synchronous: it inherits the
+// cancellation semantics register, bootstrap and the data-mount wait already
+// share, and blocking the stateful loops behind it is correct because on a first
+// boot the engine is not bootstrapped until rds-init has finished anyway.
+//
+// It does not wait for the engine to be healthy. The receipt already proves the
+// master role was durably applied, and acknowledging early shortens the window
+// in which the ciphertext exists.
+func (a *Agent) acknowledgeBootstrap(ctx context.Context) error {
+	pending := a.pending
+	if pending == nil {
+		return nil
+	}
+
+	// Polled rather than waited on: rds-init runs after this agent registered,
+	// so the file appears some time into the boot. There is no give-up deadline
+	// here — the control plane's bootstrap timeout already fails the instance,
+	// and a second one in the guest would only give the two a way to disagree.
+	if err := retryObserved(ctx, "bootstrap receipt", func(context.Context) error {
+		return a.checkBootstrapReceipt(pending)
+	}, func(err error) {
+		// An absent receipt is the expected state for the whole initdb window,
+		// so reporting it would put a failure on every healthy create's beat and
+		// then into the reason a timed-out create records.
+		if errors.Is(err, fs.ErrNotExist) {
+			return
+		}
+		a.hb.setBootstrapFailure("bootstrap receipt", err)
+	}); err != nil {
+		return err
+	}
+	a.hb.clearBootstrapFailure()
+
+	if err := retryObserved(ctx, "bootstrap acknowledgement", func(ctx context.Context) error {
+		return terminalOnDenial(a.cp.AcknowledgeBootstrap(ctx, a.id, pending))
+	}, func(err error) {
+		a.hb.setBootstrapFailure("bootstrap acknowledgement", err)
+	}); err != nil {
+		if !errors.Is(err, errRetryTerminal) {
+			return err
+		}
+		// The staged payload will never be delivered to this VM, and blocking the
+		// command poller and the parameter guard on that would leave a serving
+		// database no one can administer. The reason rides the heartbeat instead.
+		slog.ErrorContext(ctx, "rds-agent: the initial bootstrap cannot be acknowledged",
+			"payloadId", pending.payloadID, "err", err)
+		return nil
+	}
+	a.hb.clearBootstrapFailure()
+
+	// AcknowledgedAt on the record is the durable audit trail, so the receipt is
+	// removed rather than archived: keeping it would only grow the surface of
+	// stale receipts riding along in data-volume snapshots. A removal failure is
+	// harmless, since a later boot fetches attach and never consults one.
+	if err := os.Remove(a.receiptPath()); err != nil && !os.IsNotExist(err) {
+		slog.WarnContext(ctx, "rds-agent: could not remove the bootstrap receipt",
+			"receipt", a.receiptPath(), "err", err)
+	}
+	slog.InfoContext(ctx, "rds-agent: initial bootstrap acknowledged", "payloadId", pending.payloadID)
+	return nil
+}
+
+// The control plane resolved these from its own record — a superseded
+// generation, a payload it never staged, a data volume that is not this
+// instance's. Asking again cannot change any of them.
+func terminalOnDenial(err error) error {
+	var apiErr *rdsgw.APIError
+	if !errors.As(err, &apiErr) {
+		return err
+	}
+	switch apiErr.Code {
+	case awserrors.ErrorAccessDenied, awserrors.ErrorInvalidParameterValue:
+		return fmt.Errorf("%w: %w", errRetryTerminal, err)
+	}
+	return err
+}
+
+// A receipt naming another payload or another DB instance is treated as absent.
+// The receipt lives on the data volume, so it rides along in every snapshot of
+// it and a restored instance's volume carries the source instance's receipt.
+func (a *Agent) checkBootstrapReceipt(pending *pendingBootstrap) error {
+	path := a.receiptPath()
+	values, err := readReceipt(path)
+	if err != nil {
+		return err
+	}
+	if got := values[receiptPayload]; got != pending.payloadID {
+		return fmt.Errorf("%s names bootstrap payload %q, not %q", path, got, pending.payloadID)
+	}
+	if got := values[receiptDBID]; got != a.id.DBInstanceIdentifier {
+		return fmt.Errorf("%s names DB instance %q, not %q", path, got, a.id.DBInstanceIdentifier)
+	}
+	return nil
+}
+
+// KEY=value, the one format both readers can parse: rds-init in POSIX shell
+// with no guaranteed jq, and this.
+func readReceipt(path string) (map[string]string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	values := make(map[string]string)
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		key, value, ok := strings.Cut(line, "=")
+		if !ok {
+			continue
+		}
+		values[strings.TrimSpace(key)] = strings.TrimSpace(value)
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("read %s: %w", path, err)
+	}
+	return values, nil
+}

cmd/rds-agent/agent.go

@@ -1,16 +1,21 @@
 package main
 
 import (
+	"bufio"
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
 	"net/url"
+	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/mulgadc/spinifex/internal/gwsign"
 	"github.com/mulgadc/spinifex/internal/rdsgw"
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 )
 
 // Overridable via -ldflags "-X main.version=...".
@@ -30,9 +35,12 @@ type identity struct {
 type controlPlane interface {
 	Register(ctx context.Context, id identity) (*handlers_rds.RegisterDBInstanceOutput, error)
 	SubmitState(ctx context.Context, id identity, health handlers_rds.EngineHealth, message string) (*handlers_rds.SubmitDBStateChangeOutput, error)
-	// The first call of an instance's life returns Mode=initialize with the
-	// master password; every later call returns Mode=attach without it.
+	// Returns Mode=initialize with the master password for as long as a payload
+	// staged for this VM generation is unacknowledged, and Mode=attach after.
 	GetBootstrapConfig(ctx context.Context, id identity) (*handlers_rds.GetDBBootstrapConfigOutput, error)
+	// Reports that PostgreSQL durably applied the staged password, which is what
+	// lets the control plane destroy it.
+	AcknowledgeBootstrap(ctx context.Context, id identity, pending *pendingBootstrap) error
 	PollCommands(ctx context.Context, id identity, replies []handlers_rds.CommandReply, wait time.Duration) ([]handlers_rds.Command, error)
 }
 
@@ -112,6 +120,27 @@ func (g *gatewayControlPlane) GetBootstrapConfig(ctx context.Context, id identit
 	return &out, nil
 }
 
+func (g *gatewayControlPlane) AcknowledgeBootstrap(ctx context.Context, id identity, pending *pendingBootstrap) error {
+	params := url.Values{
+		"PayloadId":    {pending.payloadID},
+		"VMGeneration": {strconv.FormatInt(pending.vmGeneration, 10)},
+	}
+	setIfPresent(params, "DBInstanceIdentifier", id.DBInstanceIdentifier)
+	setIfPresent(params, "DataVolumeId", pending.dataVolumeID)
+
+	ctx, cancel := context.WithTimeout(ctx, callTimeout)
+	defer cancel()
+
+	var out handlers_rds.AcknowledgeDBBootstrapOutput
+	if err := g.client.Call(ctx, "AcknowledgeDBBootstrap", params, &out); err != nil {
+		return err
+	}
+	if !out.Acknowledged {
+		return fmt.Errorf("the control plane did not acknowledge bootstrap payload %s", pending.payloadID)
+	}
+	return nil
+}
+
 // Declared here so the in-guest binary does not link the server side of the API.
 type pollDBCommandsOutput struct {
 	Commands []handlers_rds.Command `xml:"Commands>member"`
@@ -148,12 +177,17 @@ func setIfPresent(params url.Values, key, value string) {
 }
 
 type Agent struct {
-	cfg           config
-	id            identity
-	cp            controlPlane
-	probe         *engineProbe
-	engine        *postgresEngine
-	handoffWriter func(string, *handlers_rds.GetDBBootstrapConfigOutput) error
+	cfg             config
+	id              identity
+	cp              controlPlane
+	probe           *engineProbe
+	engine          engine
+	handoffWriter   func(string, *handlers_rds.GetDBBootstrapConfigOutput) error
+	dataMountWaiter func(context.Context, string, string) error
+
+	// Set by bootstrap when the fetch replayed a staged payload; nil leaves the
+	// acknowledgement step a no-op, which is every attach boot.
+	pending *pendingBootstrap
 
 	hb    *heartbeater
 	cmd   *commander
@@ -162,18 +196,28 @@ type Agent struct {
 
 // Assembles from already-built parts, so tests can pass fakes; New builds the
 // production control plane and delegates here.
-func newAgent(cfg config, cp controlPlane, probe *engineProbe) *Agent {
+func newAgent(cfg config, cp controlPlane, probe *engineProbe) (*Agent, error) {
+	if cfg.MountsFile == "" {
+		cfg.MountsFile = defaultMountsFile
+	}
 	id := identity{
 		DBInstanceIdentifier: cfg.DBInstanceIdentifier,
 		AgentVersion:         version,
 		EngineVersion:        cfg.EngineVersion,
 	}
-	a := &Agent{cfg: cfg, id: id, cp: cp, probe: probe, handoffWriter: writeHandoff}
-	a.engine = newPostgresEngine(cfg, execCommandRunner, execSessionRunner)
-	a.hb = newHeartbeater(cp, probe, handlers_rds.HeartbeatInterval)
+	a := &Agent{
+		cfg: cfg, id: id, cp: cp, probe: probe,
+		handoffWriter: writeHandoff, dataMountWaiter: waitForDataMount,
+	}
+	eng, err := newEngine(cfg, execCommandRunner, execSessionRunner, probe)
+	if err != nil {
+		return nil, err
+	}
+	a.engine = eng
+	a.hb = newHeartbeater(cp, probe, a.engine, handlers_rds.HeartbeatInterval)
 	a.cmd = newCommander(cp, newCommandRegistry(a.engine, newGuestStorage(cfg, execCommandRunner)), cfg.PollWait)
-	a.guard = newParamGuard(a.engine, probe)
-	return a
+	a.guard = newParamGuard(a.engine, probe, cp)
+	return a, nil
 }
 
 // Does not wait for IMDS: the register loop rides out a datapath still coming up.
@@ -185,14 +229,17 @@ func New(cfg config) (*Agent, error) {
 	if err != nil {
 		return nil, err
 	}
-	return newAgent(cfg, cp, newEngineProbe(cfg, execProbeRunner)), nil
+	probe, err := newProbe(cfg, execProbeRunner)
+	if err != nil {
+		return nil, err
+	}
+	return newAgent(cfg, cp, probe)
 }
 
 func (a *Agent) Run(ctx context.Context) error {
-	// Named before the first dial. When register cannot reach the gateway the
-	// retry loop is all that survives, and the address it is dialing is what
-	// separates a mgmt NIC that came up without an address from a control plane
-	// that is genuinely down.
+	// Named before the first dial: when register cannot reach the gateway, the
+	// address it is dialing is what separates a mgmt NIC that came up without one
+	// from a control plane that is genuinely down.
 	slog.Info("rds-agent: starting",
 		"gatewayURL", a.cfg.GatewayURL, "dbInstanceIdentifier", a.id.DBInstanceIdentifier,
 		"agentVersion", a.id.AgentVersion)
@@ -201,7 +248,14 @@ func (a *Agent) Run(ctx context.Context) error {
 	if err := a.register(ctx); err != nil {
 		return err
 	}
-	a.hb.id, a.cmd.id = a.id, a.id
+	a.hb.id, a.cmd.id, a.guard.id = a.id, a.id, a.id
+
+	// Ahead of the handoff, so rds-init never unblocks and no datadir is touched.
+	// An agent running on the wrong image would initialise one engine over
+	// another's data, coming up healthy and empty beside data nothing references.
+	if err := a.refuseEngineMismatch(ctx); err != nil {
+		return err
+	}
 
 	// Beating before the bootstrap keeps a stuck boot visible as a live VM with
 	// a down engine rather than as silence.
@@ -211,7 +265,22 @@ func (a *Agent) Run(ctx context.Context) error {
 		return err
 	}
 
-	// Directives are only meaningful against a bootstrapped engine.
+	// The handoff must land before this wait: rds-datadir needs it to identify
+	// and authorize the volume. Commands and the parameter guard wait because
+	// their durable state belongs on that mount, never on the disposable boot disk.
+	if err := a.dataMountWaiter(ctx, a.cfg.MountsFile, a.cfg.DataMount); err != nil {
+		// A shutdown cancels this wait; main treats the wrapped context.Canceled
+		// as a clean stop, exactly as it does for register and bootstrap above.
+		return fmt.Errorf("wait for data mount %s: %w", a.cfg.DataMount, err)
+	}
+
+	// The receipt it waits on lives on that mount, so this cannot run any earlier.
+	// A cancelled wait is a clean stop here too.
+	if err := a.acknowledgeBootstrap(ctx); err != nil {
+		return err
+	}
+
+	// Directives are only meaningful against a bootstrapped, mounted engine.
 	go a.cmd.Run(ctx)
 
 	// Started after the handoff, so the window it measures covers the engine's
@@ -222,17 +291,104 @@ func (a *Agent) Run(ctx context.Context) error {
 	return nil
 }
 
+// The engine cloud-init says this VM was launched as, against the engine the
+// image bakes. An empty expectation is no assertion rather than a disagreement:
+// a VM launched before the control plane emitted RDS_ENGINE carries none.
+func (a *Agent) refuseEngineMismatch(ctx context.Context) error {
+	expected := strings.ToLower(strings.TrimSpace(a.cfg.Engine))
+	if expected == "" || expected == a.cfg.BakedEngine {
+		return nil
+	}
+
+	message := fmt.Sprintf("this VM was launched as engine %q but its image bakes %q; refusing to bootstrap",
+		expected, a.cfg.BakedEngine)
+	slog.ErrorContext(ctx, "rds-agent: "+message)
+	// Reported before returning, so the control plane learns why the instance
+	// never came up rather than only that it never did.
+	if _, err := a.cp.SubmitState(ctx, a.id, handlers_rds.EngineHealthUnhealthy, message); err != nil {
+		slog.ErrorContext(ctx, "rds-agent: reporting the engine mismatch failed", "err", err)
+	}
+	return errors.New(message)
+}
+
+const dataMountPollInterval = time.Second
+
+// waitForDataMount blocks the stateful agent loops until the kernel reports the
+// configured data mount. Registration, heartbeat, and bootstrap run before it.
+func waitForDataMount(ctx context.Context, mountsFile, target string) error {
+	ticker := time.NewTicker(dataMountPollInterval)
+	defer ticker.Stop()
+
+	for {
+		mounted, err := mountTableContains(mountsFile, target)
+		if err != nil {
+			slog.WarnContext(ctx, "rds-agent: could not read mount table while waiting for data volume",
+				"mountsFile", mountsFile, "dataMount", target, "err", err)
+		} else if mounted {
+			slog.InfoContext(ctx, "rds-agent: data volume mount is ready", "dataMount", target)
+			return nil
+		}
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+		}
+	}
+}
+
+func mountTableContains(path, target string) (bool, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		fields := strings.Fields(scanner.Text())
+		if len(fields) >= 2 && unescapeMountField(fields[1]) == target {
+			return true, nil
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return false, err
+	}
+	return false, nil
+}
+
+// Linux mount tables encode whitespace and backslashes as octal escapes.
+func unescapeMountField(field string) string {
+	replacer := strings.NewReplacer(`\134`, `\`, `\040`, " ", `\011`, "\t", `\012`, "\n")
+	return replacer.Replace(field)
+}
+
 // Boot-retry bounds. Tight to start, since the usual cause is a control plane
 // still creating this instance's record, and capped so an outage does not turn
 // a booting fleet into a retry storm.
 const (
 	retryMin = 1 * time.Second
 	retryMax = 30 * time.Second
+
+	// With the production backoff, attempt five is about 15 seconds into the
+	// failure and still precedes the first 30-second heartbeat.
+	retryErrorAttempt = 5
 )
 
+// Wrapping an error in this stops retryObserved. For the failures the control
+// plane decides from its own record, no retry can change the answer, and looping
+// on one only keeps the caller blocked.
+var errRetryTerminal = errors.New("terminal failure")
+
 // Never gives up on its own: an agent that stopped trying would leave the
 // control plane with no signal at all.
 func retry(ctx context.Context, what string, fn func(context.Context) error) error {
+	return retryObserved(ctx, what, fn, nil)
+}
+
+// onFailure lets boot-critical callers publish the latest error while the retry
+// remains blocked. It must not block because it runs on the retry path itself.
+func retryObserved(ctx context.Context, what string, fn func(context.Context) error, onFailure func(error)) error {
 	delay := retryMin
 	for attempt := 1; ; attempt++ {
 		err := fn(ctx)
@@ -242,7 +398,20 @@ func retry(ctx context.Context, what string, fn func(context.Context) error) err
 		if ctx.Err() != nil {
 			return fmt.Errorf("%s: %w", what, ctx.Err())
 		}
-		slog.Warn("rds-agent: "+what+" failed, retrying", "attempt", attempt, "retryIn", delay, "err", err)
+		if onFailure != nil {
+			onFailure(err)
+		}
+		if errors.Is(err, errRetryTerminal) {
+			slog.ErrorContext(ctx, "rds-agent: "+what+" failed terminally, not retrying", "err", err)
+			return fmt.Errorf("%s: %w", what, err)
+		}
+		if attempt >= retryErrorAttempt {
+			slog.ErrorContext(ctx, "rds-agent: "+what+" still failing, retrying",
+				"attempt", attempt, "retry_in_ms", otelsetup.Millis(delay), "err", err)
+		} else {
+			slog.WarnContext(ctx, "rds-agent: "+what+" failed, retrying",
+				"attempt", attempt, "retry_in_ms", otelsetup.Millis(delay), "err", err)
+		}
 
 		select {
 		case <-ctx.Done():

cmd/rds-agent/agent_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"errors"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -15,9 +16,12 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/gwsign"
 	"github.com/mulgadc/spinifex/internal/rdsgw"
+	"github.com/mulgadc/spinifex/spinifex/awserrors"
 	gateway_rds "github.com/mulgadc/spinifex/spinifex/gateway/rds"
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 	"github.com/mulgadc/spinifex/spinifex/utils"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 // fakeControlPlane is a scriptable controlPlane. Every call records what the
@@ -32,6 +36,12 @@ type fakeControlPlane struct {
 	bootstrapOut  *handlers_rds.GetDBBootstrapConfigOutput
 	bootstrapErrs []error
 	bootstrapReqs []identity
+	// Scripted responses ahead of bootstrapOut, so a test can answer the first
+	// fetch differently from the ones the agent retries with.
+	bootstrapQueue []*handlers_rds.GetDBBootstrapConfigOutput
+
+	ackErrs []error
+	ackReqs []pendingBootstrap
 
 	states []submittedState
 
@@ -95,9 +105,21 @@ func (f *fakeControlPlane) GetBootstrapConfig(_ context.Context, id identity) (*
 	if err := nextErr(&f.bootstrapErrs); err != nil {
 		return nil, err
 	}
+	if len(f.bootstrapQueue) > 0 {
+		out := f.bootstrapQueue[0]
+		f.bootstrapQueue = f.bootstrapQueue[1:]
+		return out, nil
+	}
 	return f.bootstrapOut, nil
 }
 
+func (f *fakeControlPlane) AcknowledgeBootstrap(_ context.Context, _ identity, pending *pendingBootstrap) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.ackReqs = append(f.ackReqs, *pending)
+	return nextErr(&f.ackErrs)
+}
+
 func (f *fakeControlPlane) PollCommands(ctx context.Context, _ identity, replies []handlers_rds.CommandReply, _ time.Duration) ([]handlers_rds.Command, error) {
 	f.mu.Lock()
 	f.pollReplies = append(f.pollReplies, slices.Clone(replies))
@@ -124,6 +146,12 @@ func (f *fakeControlPlane) PollCommands(ctx context.Context, _ identity, replies
 	return nil, ctx.Err()
 }
 
+func (f *fakeControlPlane) ackRequests() []pendingBootstrap {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return slices.Clone(f.ackReqs)
+}
+
 func (f *fakeControlPlane) snapshotStates() []submittedState {
 	f.mu.Lock()
 	defer f.mu.Unlock()
@@ -140,23 +168,31 @@ func (f *fakeControlPlane) snapshotReplies() [][]handlers_rds.CommandReply {
 	return out
 }
 
-// testConfig points the agent's file output at a temp dir and its probe at a
-// name no real binary answers to.
+// testConfig stamps the engine a real rds-postgres image bakes, so the agent
+// resolves the same guest layout it would in a guest, and points its file output
+// at a temp dir.
 func testConfig(t *testing.T) config {
 	t.Helper()
-	return config{
-		GatewayURL: "https://gw.test",
-		HandoffDir: filepath.Join(t.TempDir(), "spinifex-rds"),
-		EngineHost: defaultEngineHost,
-		EnginePort: defaultEnginePort,
-		PGIsReady:  "pg_isready",
-		PollWait:   time.Second,
+	cfg := testLoadConfig(t, enginePostgres)
+	cfg.GatewayURL = "https://gw.test"
+	cfg.HandoffDir = filepath.Join(t.TempDir(), "spinifex-rds")
+	cfg.PollWait = time.Second
+	return cfg
+}
+
+// newTestAgent builds the agent over a probe the test drives directly.
+func newTestAgent(t *testing.T, cfg config, cp controlPlane, run probeRunner) *Agent {
+	t.Helper()
+	a, err := newAgent(cfg, cp, newPostgresProbe(cfg, run))
+	if err != nil {
+		t.Fatalf("newAgent: %v", err)
 	}
+	return a
 }
 
 // staticProbe is a probeRunner that always reports the same exit code.
 func staticProbe(code int) probeRunner {
-	return func(context.Context, string, ...string) (int, error) { return code, nil }
+	return func(context.Context, string, ...string) (int, string, error) { return code, "", nil }
 }
 
 func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput {
@@ -171,6 +207,36 @@ func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput
 		MasterUsername:       "master",
 		MasterUserPassword:   password,
 		Port:                 5432,
+		ListenAddress:        testListenAddress,
+		ClientCIDR:           testClientCIDR,
+		DataVolumeID:         "vol-data-01",
+		DataVolumeSerial:     "voldata01",
+		VMGeneration:         1,
+		PayloadID:            testPayloadID,
+		BootstrapPending:     password != nil,
+	}
+}
+
+const testPayloadID = "bp-0123456789abcdef"
+
+// The customer ENI address the engine binds, and the VPC range its client
+// authentication admits. writeHandoff refuses a config without either.
+const (
+	testListenAddress = "10.0.1.20"
+	testClientCIDR    = "10.0.0.0/16"
+)
+
+// Writes the completion receipt rds-init leaves behind, so the acknowledgement
+// step finds what a finished initialization would have produced.
+func writeReceipt(t *testing.T, a *Agent, payloadID, dbInstanceIdentifier string) {
+	t.Helper()
+	path := a.receiptPath()
+	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		t.Fatalf("mkdir receipt dir: %v", err)
+	}
+	body := fmt.Sprintf("%s=%s\n%s=%s\n", receiptPayload, payloadID, receiptDBID, dbInstanceIdentifier)
+	if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
+		t.Fatalf("write receipt: %v", err)
 	}
 }
 
@@ -192,6 +258,11 @@ func runAgent(t *testing.T, a *Agent) error {
 	cancel()
 	select {
 	case err := <-done:
+		// The cancellation above is this helper's own, so Run reporting it is the
+		// expected clean stop rather than a failure, exactly as main treats it.
+		if errors.Is(err, context.Canceled) {
+			return nil
+		}
 		return err
 	case <-time.After(5 * time.Second):
 		t.Fatal("Run did not exit after cancellation")
@@ -218,7 +289,7 @@ func TestRun_RegistersThenDeliversBootstrap(t *testing.T) {
 	cp.bootstrapOut = bootstrapOutput(&password)
 
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	if err := runAgent(t, a); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
@@ -249,7 +320,7 @@ func TestRun_AttachModeNeedsNoPassword(t *testing.T) {
 	cp.bootstrapOut = bootstrapOutput(nil)
 
 	cfg := testConfig(t)
-	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
 
@@ -271,7 +342,7 @@ func TestRun_RetriesRegisterAndBootstrap(t *testing.T) {
 	cp.bootstrapErrs = []error{errors.New("ServerInternal")}
 
 	cfg := testConfig(t)
-	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
 
@@ -283,12 +354,14 @@ func TestRun_RetriesRegisterAndBootstrap(t *testing.T) {
 	}
 }
 
-func TestBootstrap_RetriesHandoffWithoutRefetchingConsumedPassword(t *testing.T) {
+// The fetch mutates nothing, so a handoff that cannot be written is retried
+// against the payload already in hand rather than costing a second fetch.
+func TestBootstrap_RetriesHandoffWithoutRefetching(t *testing.T) {
 	cp := newFakeControlPlane()
-	password := "one-shot-secret"
+	password := "staged-secret"
 	cp.bootstrapOut = bootstrapOutput(&password)
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 
 	writes := 0
 	a.handoffWriter = func(_ string, got *handlers_rds.GetDBBootstrapConfigOutput) error {
@@ -311,6 +384,163 @@ func TestBootstrap_RetriesHandoffWithoutRefetchingConsumedPassword(t *testing.T)
 	if writes != 2 {
 		t.Errorf("handoff writes = %d, want 2", writes)
 	}
+	if a.pending == nil || a.pending.payloadID != testPayloadID {
+		t.Errorf("pending bootstrap = %+v, want the staged payload %s", a.pending, testPayloadID)
+	}
+}
+
+// A daemon that predates the encrypted payload answers initialize with nothing
+// to initialise with. Writing that handoff would let rds-init run initdb with an
+// empty master password, so it is retried until an upgraded node answers.
+func TestBootstrap_RetriesInitializeWithNoPassword(t *testing.T) {
+	cp := newFakeControlPlane()
+	empty := ""
+	password := "staged-secret"
+	cp.bootstrapQueue = []*handlers_rds.GetDBBootstrapConfigOutput{{
+		Mode: handlers_rds.BootstrapModeInitialize, DBInstanceIdentifier: "db-resolved",
+		MasterUsername: "master", MasterUserPassword: &empty,
+	}}
+	cp.bootstrapOut = bootstrapOutput(&password)
+	cfg := testConfig(t)
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+
+	var handed []*handlers_rds.GetDBBootstrapConfigOutput
+	a.handoffWriter = func(_ string, got *handlers_rds.GetDBBootstrapConfigOutput) error {
+		handed = append(handed, got)
+		return nil
+	}
+
+	if err := a.bootstrap(t.Context()); err != nil {
+		t.Fatalf("bootstrap: %v", err)
+	}
+	if len(handed) != 1 {
+		t.Fatalf("handoff writes = %d, want 1 once a usable payload arrived", len(handed))
+	}
+	if handed[0].MasterUserPassword == nil || *handed[0].MasterUserPassword != password {
+		t.Error("the agent wrote a handoff for a fetch that carried no master password")
+	}
+}
+
+func TestAcknowledgeBootstrap_WaitsForTheReceiptThenRetiresIt(t *testing.T) {
+	cp := newFakeControlPlane()
+	cfg := testConfig(t)
+	cfg.DataMount = t.TempDir()
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+	a.id = identity{DBInstanceIdentifier: "db-resolved"}
+	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1, dataVolumeID: "vol-data-01"}
+
+	// rds-init runs after the agent registers, so the receipt is absent when the
+	// acknowledgement step first looks and appears some way into the boot.
+	if err := a.checkBootstrapReceipt(a.pending); err == nil {
+		t.Fatal("a missing receipt must not read as a completed bootstrap")
+	}
+	writeReceipt(t, a, testPayloadID, "db-resolved")
+
+	if err := a.acknowledgeBootstrap(t.Context()); err != nil {
+		t.Fatalf("acknowledgeBootstrap: %v", err)
+	}
+	if len(cp.ackRequests()) != 1 {
+		t.Fatalf("acknowledgements = %d, want 1", len(cp.ackRequests()))
+	}
+	if got := cp.ackRequests()[0]; got != *a.pending {
+		t.Errorf("acknowledged %+v, want %+v", got, *a.pending)
+	}
+	if _, err := os.Stat(a.receiptPath()); !os.IsNotExist(err) {
+		t.Error("the receipt must be removed once the record holds the audit trail")
+	}
+}
+
+// The receipt rides along in every snapshot of the data volume, so a restored
+// instance's volume carries the source instance's receipt. Acknowledging on one
+// would destroy the ciphertext this instance still needs.
+func TestAcknowledgeBootstrap_IgnoresAForeignReceipt(t *testing.T) {
+	tests := map[string]struct{ payloadID, dbID string }{
+		"another payload":  {payloadID: "bp-somebody-else", dbID: "db-resolved"},
+		"another instance": {payloadID: testPayloadID, dbID: "db-the-snapshot-source"},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			cp := newFakeControlPlane()
+			cfg := testConfig(t)
+			cfg.DataMount = t.TempDir()
+			a := newTestAgent(t, cfg, cp, staticProbe(0))
+			a.id = identity{DBInstanceIdentifier: "db-resolved"}
+			a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
+			writeReceipt(t, a, tc.payloadID, tc.dbID)
+
+			ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
+			defer cancel()
+			if err := a.acknowledgeBootstrap(ctx); err == nil {
+				t.Fatal("acknowledgeBootstrap returned nil, want it still waiting on a real receipt")
+			}
+			if n := len(cp.ackRequests()); n != 0 {
+				t.Errorf("acknowledgements = %d, want 0", n)
+			}
+		})
+	}
+}
+
+// An attach boot has nothing staged, so it must not go looking for a receipt the
+// datadir will never grow.
+func TestAcknowledgeBootstrap_IsANoOpOnAttach(t *testing.T) {
+	cp := newFakeControlPlane()
+	cfg := testConfig(t)
+	cfg.DataMount = t.TempDir()
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+
+	if err := a.acknowledgeBootstrap(t.Context()); err != nil {
+		t.Fatalf("acknowledgeBootstrap: %v", err)
+	}
+	if n := len(cp.ackRequests()); n != 0 {
+		t.Errorf("acknowledgements = %d, want 0 when nothing was staged", n)
+	}
+}
+
+// The image is the authority on which engine the agent runs. A VM launched as
+// another engine has to stop ahead of the handoff: rds-init would otherwise
+// initialise one engine over the other's datadir and come up healthy and empty
+// beside data nothing references.
+func TestRun_RefusesAnEngineTheImageDoesNotBake(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+
+	cfg := testConfig(t)
+	cfg.Engine = "mariadb"
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+
+	err := a.Run(t.Context())
+	if err == nil {
+		t.Fatal("Run returned nil for a VM launched as an engine this image does not bake")
+	}
+	if !strings.Contains(err.Error(), "mariadb") {
+		t.Errorf("error = %v, want it to name the engine the VM was launched as", err)
+	}
+	if _, statErr := os.Stat(filepath.Join(cfg.HandoffDir, handoffEnvFile)); !os.IsNotExist(statErr) {
+		t.Error("the agent wrote a bootstrap handoff for an engine it refused to run")
+	}
+	if n := len(cp.bootstrapReqs); n != 0 {
+		t.Errorf("bootstrap fetches = %d, want none before the refusal", n)
+	}
+	// Reported rather than only logged, so the control plane learns why the
+	// instance never came up rather than only that it never did.
+	states := cp.snapshotStates()
+	if len(states) != 1 || states[0].health != handlers_rds.EngineHealthUnhealthy {
+		t.Fatalf("submitted states = %+v, want a single unhealthy report", states)
+	}
+}
+
+// The same assertion when it agrees is not a gate: the boot goes on exactly as
+// it does for a VM launched before the control plane sent one at all.
+func TestRun_ProceedsWhenTheDeliveredEngineMatchesTheImage(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+
+	cfg := testConfig(t)
+	cfg.Engine = enginePostgres
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
 }
 
 // A configured identifier is asserted on the wire so a mis-provisioned VM is
@@ -322,7 +552,7 @@ func TestRun_SendsConfiguredIdentifier(t *testing.T) {
 	cfg := testConfig(t)
 	cfg.DBInstanceIdentifier = "db-configured"
 	cfg.EngineVersion = "18.1"
-	if err := runAgent(t, newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))); err != nil {
+	if err := runAgent(t, newTestAgent(t, cfg, cp, staticProbe(0))); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
 
@@ -332,6 +562,108 @@ func TestRun_SendsConfiguredIdentifier(t *testing.T) {
 	}
 }
 
+func TestRun_WaitsForDataMountBeforePollingCommands(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapOut = bootstrapOutput(nil)
+	cfg := testConfig(t)
+	a := newTestAgent(t, cfg, cp, staticProbe(2))
+
+	waiting := make(chan struct{})
+	release := make(chan struct{})
+	a.dataMountWaiter = func(ctx context.Context, mountsFile, target string) error {
+		if mountsFile != defaultMountsFile || target != cfg.DataMount {
+			t.Errorf("mount waiter got %q/%q, want configured defaults", mountsFile, target)
+		}
+		close(waiting)
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-release:
+			return nil
+		}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	select {
+	case <-waiting:
+	case <-ctx.Done():
+		t.Fatal("agent did not reach the data-mount wait after writing the handoff")
+	}
+	if _, err := os.Stat(filepath.Join(cfg.HandoffDir, handoffEnvFile)); err != nil {
+		t.Fatalf("handoff was not present before mount wait: %v", err)
+	}
+	select {
+	case <-cp.polled:
+		t.Fatal("command poll started before the data volume was mounted")
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	close(release)
+	select {
+	case <-cp.polled:
+	case <-ctx.Done():
+		t.Fatal("command poll did not start after the data mount became ready")
+	}
+	cancel()
+	if err := <-done; err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+}
+
+func TestMountTableContains(t *testing.T) {
+	mounts := filepath.Join(t.TempDir(), "mounts")
+	require.NoError(t, os.WriteFile(mounts, []byte(
+		"/dev/vda1 / ext4 rw 0 0\n/dev/vdb /var/lib/postgresql ext4 rw 0 0\n"+
+			"/dev/vdc /path\\040with\\040spaces xfs rw 0 0\nmalformed\n"), 0o600))
+
+	tests := []struct {
+		target string
+		want   bool
+	}{
+		{target: "/var/lib/postgresql", want: true},
+		{target: "/path with spaces", want: true},
+		{target: "/missing", want: false},
+	}
+	for _, tc := range tests {
+		t.Run(tc.target, func(t *testing.T) {
+			got, err := mountTableContains(mounts, tc.target)
+			require.NoError(t, err)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}
+
+func TestRun_HeartbeatIncludesBootstrapFailure(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.bootstrapErrs = []error{errors.New("403 AccessDenied: instance profile is not authorised")}
+	cp.registerOut.HeartbeatIntervalSeconds = 0
+
+	cfg := testConfig(t)
+	a := newTestAgent(t, cfg, cp, staticProbe(2))
+	a.hb.interval = 10 * time.Millisecond
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- a.Run(ctx) }()
+
+	waitFor(t, func() bool {
+		for _, state := range cp.snapshotStates() {
+			if strings.Contains(state.message, "bootstrap fetch failed") &&
+				strings.Contains(state.message, "403 AccessDenied") {
+				return true
+			}
+		}
+		return false
+	}, "the bootstrap failure to reach a heartbeat")
+	cancel()
+	<-done
+}
+
 func TestRun_HeartbeatsEngineHealth(t *testing.T) {
 	cp := newFakeControlPlane()
 	cp.bootstrapOut = bootstrapOutput(nil)
@@ -340,7 +672,7 @@ func TestRun_HeartbeatsEngineHealth(t *testing.T) {
 	cp.registerOut.HeartbeatIntervalSeconds = 0
 
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	a.hb.interval = 10 * time.Millisecond
 
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
@@ -370,7 +702,7 @@ func TestRegister_AdoptsControlPlaneCadence(t *testing.T) {
 	cp.registerOut.HeartbeatIntervalSeconds = 45
 
 	cfg := testConfig(t)
-	a := newAgent(cfg, cp, newEngineProbe(cfg, staticProbe(0)))
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
 	if err := runAgent(t, a); err != nil {
 		t.Fatalf("Run: %v", err)
 	}
@@ -387,12 +719,15 @@ func TestRun_HeartbeatReportsDownEngine(t *testing.T) {
 	cp.registerOut.HeartbeatIntervalSeconds = 0
 
 	cfg := testConfig(t)
-	probe := newEngineProbe(cfg, staticProbe(2))
+	probe := newPostgresProbe(cfg, staticProbe(2))
 	// The engine has served before, so a silent one now is a failure rather
 	// than a boot still in progress.
 	probe.seenHealthy = true
 
-	a := newAgent(cfg, cp, probe)
+	a, err := newAgent(cfg, cp, probe)
+	if err != nil {
+		t.Fatalf("newAgent: %v", err)
+	}
 	a.hb.interval = 10 * time.Millisecond
 
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
@@ -470,3 +805,69 @@ func readFile(t *testing.T, path string) string {
 	}
 	return string(body)
 }
+
+// AccessDenied is decided by the control plane's own record, so no retry can
+// change it. Looping on one would leave the command poller and the parameter
+// guard unstarted for the life of a VM whose engine is serving normally.
+func TestAcknowledgeBootstrap_StopsOnATerminalDenial(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.ackErrs = []error{&rdsgw.APIError{
+		Action: "AcknowledgeDBBootstrap", StatusCode: 403,
+		Code:    awserrors.ErrorAccessDenied,
+		Message: "bootstrap payload bp-x at generation 1 does not match the current state",
+	}}
+	cfg := testConfig(t)
+	cfg.DataMount = t.TempDir()
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+	a.id = identity{DBInstanceIdentifier: "db-resolved"}
+	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
+	writeReceipt(t, a, testPayloadID, "db-resolved")
+
+	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+	defer cancel()
+	if err := a.acknowledgeBootstrap(ctx); err != nil {
+		t.Fatalf("acknowledgeBootstrap = %v, want nil so the stateful loops still start", err)
+	}
+	if n := len(cp.ackRequests()); n != 1 {
+		t.Errorf("acknowledgements = %d, want 1 attempt and no retry", n)
+	}
+	if a.hb.bootstrapFailure.Load() == nil {
+		t.Error("a terminal denial must still reach the operator through the heartbeat")
+	}
+	if _, err := os.Stat(a.receiptPath()); err != nil {
+		t.Error("a denied acknowledgement must leave the receipt for a later attempt")
+	}
+}
+
+// rds-init runs after this agent, so the receipt is absent for the whole initdb
+// window. Reporting that would put a failure on every healthy create's beat, and
+// then into the reason a create that timed out for some other cause records.
+func TestAcknowledgeBootstrap_DoesNotReportAnAbsentReceiptAsAFailure(t *testing.T) {
+	cp := newFakeControlPlane()
+	cfg := testConfig(t)
+	cfg.DataMount = t.TempDir()
+	a := newTestAgent(t, cfg, cp, staticProbe(0))
+	a.id = identity{DBInstanceIdentifier: "db-resolved"}
+	a.pending = &pendingBootstrap{payloadID: testPayloadID, vmGeneration: 1}
+
+	ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
+	defer cancel()
+	if err := a.acknowledgeBootstrap(ctx); err == nil {
+		t.Fatal("acknowledgeBootstrap returned nil, want it still waiting for rds-init")
+	}
+	if got := a.hb.bootstrapFailure.Load(); got != nil {
+		t.Errorf("bootstrap failure = %q, want none while the receipt has simply not been written", *got)
+	}
+
+	// A receipt that exists but names another instance is a real mismatch and is
+	// reported, which is what separates the two cases.
+	writeReceipt(t, a, testPayloadID, "db-the-snapshot-source")
+	mismatch, cancelMismatch := context.WithTimeout(t.Context(), 50*time.Millisecond)
+	defer cancelMismatch()
+	if err := a.acknowledgeBootstrap(mismatch); err == nil {
+		t.Fatal("a foreign receipt must not be acknowledged")
+	}
+	if a.hb.bootstrapFailure.Load() == nil {
+		t.Error("a receipt naming another instance must be reported")
+	}
+}

cmd/rds-agent/bootstrap.go

@@ -12,11 +12,19 @@ import (
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
 
+// The staged payload this boot has to acknowledge once PostgreSQL has applied
+// it. Nil on an attach boot, which makes the acknowledgement step a no-op.
+type pendingBootstrap struct {
+	payloadID    string
+	vmGeneration int64
+	dataVolumeID string
+}
+
 // Fetches the boot material and writes the handoff rds-init is blocked waiting
 // for, then points the health probe at the assigned port.
 func (a *Agent) bootstrap(ctx context.Context) error {
 	var cfg *handlers_rds.GetDBBootstrapConfigOutput
-	if err := retry(ctx, "bootstrap fetch", func(ctx context.Context) error {
+	if err := retryObserved(ctx, "bootstrap fetch", func(ctx context.Context) error {
 		fetched, err := a.cp.GetBootstrapConfig(ctx, a.id)
 		if err != nil {
 			return err
@@ -24,28 +32,47 @@ func (a *Agent) bootstrap(ctx context.Context) error {
 		if fetched == nil {
 			return fmt.Errorf("bootstrap fetch returned no config")
 		}
+		// A control plane predating the encrypted payload answers initialize with
+		// nothing to initialise with. Retrying re-dials the queue group until an
+		// upgraded node answers, so a mixed-version fleet heals within one boot.
+		if fetched.Mode == handlers_rds.BootstrapModeInitialize &&
+			(fetched.MasterUserPassword == nil || *fetched.MasterUserPassword == "") {
+			return fmt.Errorf("bootstrap fetch returned mode=%s with no master password", fetched.Mode)
+		}
 		cfg = fetched
 		return nil
+	}, func(err error) {
+		a.hb.setBootstrapFailure("bootstrap fetch", err)
 	}); err != nil {
 		return err
 	}
+	a.hb.clearBootstrapFailure()
 
-	// Once an initialize response reaches the guest, keep retrying that same
-	// payload. Re-fetching after a local write failure returns attach because the
-	// control plane has already consumed the one-shot password.
-	if err := retry(ctx, "bootstrap handoff", func(context.Context) error {
+	// The fetch mutates nothing, so a handoff that cannot be written is retried
+	// against the same staged payload rather than against an attach response.
+	if err := retryObserved(ctx, "bootstrap handoff", func(context.Context) error {
 		return a.handoffWriter(a.cfg.HandoffDir, cfg)
+	}, func(err error) {
+		a.hb.setBootstrapFailure("bootstrap handoff", err)
 	}); err != nil {
 		return err
 	}
+	a.hb.clearBootstrapFailure()
 	if cfg.Port > 0 {
 		a.probe.setPort(int(cfg.Port))
-		a.engine.setPort(int(cfg.Port))
+	}
+	if cfg.BootstrapPending && cfg.PayloadID != "" {
+		a.pending = &pendingBootstrap{
+			payloadID:    cfg.PayloadID,
+			vmGeneration: cfg.VMGeneration,
+			dataVolumeID: cfg.DataVolumeID,
+		}
 	}
 	// Mode is logged but never branched on: rds-init decides whether to initdb
 	// from the state of the datadir.
 	slog.Info("rds-agent: bootstrap config delivered",
-		"mode", cfg.Mode, "port", cfg.Port, "parameters", len(cfg.Parameters))
+		"mode", cfg.Mode, "port", cfg.Port, "parameters", len(cfg.Parameters),
+		"bootstrapPending", cfg.BootstrapPending)
 	return nil
 }
 
@@ -69,6 +96,15 @@ func writeHandoff(dir string, cfg *handlers_rds.GetDBBootstrapConfigOutput) erro
 	if cfg.MasterUsername == "" {
 		return fmt.Errorf("bootstrap config carries no master username")
 	}
+	// Half a handoff would leave rds-init deciding what to bind and who to admit,
+	// and the only answers it could reach on its own are the wildcards this
+	// contract exists to remove.
+	if cfg.ListenAddress == "" {
+		return fmt.Errorf("bootstrap config carries no listen address for the engine to bind")
+	}
+	if cfg.ClientCIDR == "" {
+		return fmt.Errorf("bootstrap config carries no client CIDR to scope client authentication to")
+	}
 	if err := os.MkdirAll(dir, handoffDirMode); err != nil {
 		return fmt.Errorf("create handoff dir %s: %w", dir, err)
 	}
@@ -78,7 +114,11 @@ func writeHandoff(dir string, cfg *handlers_rds.GetDBBootstrapConfigOutput) erro
 		return fmt.Errorf("secure handoff dir %s: %w", dir, err)
 	}
 
-	if err := writeHandoffFile(dir, handoffParamsFile, renderParameters(cfg.Parameters)); err != nil {
+	params, err := renderParameters(cfg.Engine, cfg.Parameters)
+	if err != nil {
+		return err
+	}
+	if err := writeHandoffFile(dir, handoffParamsFile, params); err != nil {
 		return err
 	}
 	// Cert and key are written as a pair; half of one would start the engine
@@ -101,7 +141,19 @@ func renderBootstrapEnv(cfg *handlers_rds.GetDBBootstrapConfigOutput) string {
 	var b strings.Builder
 	b.WriteString("# Written by rds-agent. Regenerated on every boot; edits are lost.\n")
 	writeEnvLine(&b, "RDS_MODE", cfg.Mode)
+	writeEnvLine(&b, "RDS_DB_INSTANCE_IDENTIFIER", cfg.DBInstanceIdentifier)
 	writeEnvLine(&b, "RDS_MASTER_USERNAME", cfg.MasterUsername)
+	writeEnvLine(&b, "RDS_LISTEN_ADDRESS", cfg.ListenAddress)
+	writeEnvLine(&b, "RDS_CLIENT_CIDR", cfg.ClientCIDR)
+	writeEnvLine(&b, "RDS_DATA_VOLUME_ID", cfg.DataVolumeID)
+	writeEnvLine(&b, "RDS_DATA_VOLUME_SERIAL", cfg.DataVolumeSerial)
+	writeEnvLine(&b, "RDS_VM_GENERATION", strconv.FormatInt(cfg.VMGeneration, 10))
+	writeEnvLine(&b, "RDS_FORMAT_AUTHORIZED", strconv.FormatBool(cfg.FormatAuthorized))
+	// Neither needs volume access, which is why the assertion is made here: the
+	// agent runs before rds-datadir mounts the volume, so rds-init is the first
+	// component that can compare it against the completion receipt on disk.
+	writeEnvLine(&b, "RDS_BOOTSTRAP_PENDING", boolFlag(cfg.BootstrapPending))
+	writeEnvLine(&b, "RDS_PAYLOAD_ID", cfg.PayloadID)
 	// Present only in initialize mode: an attach fetch has no password to write
 	// and rds-init must not find a stale one.
 	if cfg.MasterUserPassword != nil {
@@ -116,6 +168,15 @@ func renderBootstrapEnv(cfg *handlers_rds.GetDBBootstrapConfigOutput) string {
 	return b.String()
 }
 
+// rds-init compares against 1 rather than sourcing a Go boolean, so the two
+// halves of the guard cannot drift on spelling.
+func boolFlag(v bool) string {
+	if v {
+		return "1"
+	}
+	return "0"
+}
+
 func writeEnvLine(b *strings.Builder, key, value string) {
 	b.WriteString(key)
 	b.WriteString("=")
@@ -129,18 +190,23 @@ func shellQuote(s string) string {
 	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
 }
 
-// postgresql.conf syntax. Values are quoted so a setting with a space or unit
-// suffix survives; the engine accepts quoted numerics and booleans too.
-func renderParameters(params []handlers_rds.Parameter) string {
+// postgresql.conf syntax, which MariaDB's option files also parse. Values are
+// quoted so a space or unit suffix survives. Names are the engine's startup
+// spellings, not the customer's: the wrong one leaves the engine unable to boot.
+func renderParameters(engineName string, params []handlers_rds.Parameter) (string, error) {
+	engine, err := handlers_rds.LookupEngine(engineName)
+	if err != nil {
+		return "", fmt.Errorf("render the resolved parameters: %w", err)
+	}
 	var b strings.Builder
 	b.WriteString("# Resolved parameter group, written by rds-agent.\n")
 	for _, p := range params {
 		if p.Name == "" {
 			continue
 		}
-		fmt.Fprintf(&b, "%s = '%s'\n", p.Name, strings.ReplaceAll(p.Value, "'", "''"))
+		fmt.Fprintf(&b, "%s = '%s'\n", engine.OptionFileName(p.Name), strings.ReplaceAll(p.Value, "'", "''"))
 	}
-	return b.String()
+	return b.String(), nil
 }
 
 // Atomic: a temp file in the same directory, renamed over the target. The mode

cmd/rds-agent/bootstrap_test.go

@@ -16,10 +16,17 @@ func TestWriteHandoff_WritesEveryFileRootOnly(t *testing.T) {
 
 	err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:               handlers_rds.BootstrapModeInitialize,
+		Engine:             "postgres",
 		MasterUsername:     "master",
+		ListenAddress:      "10.0.1.20",
+		ClientCIDR:         "10.0.0.0/16",
 		MasterUserPassword: &password,
 		DBName:             "appdb",
 		Port:               5433,
+		DataVolumeID:       "vol-data-01",
+		DataVolumeSerial:   "voldata01",
+		VMGeneration:       4,
+		FormatAuthorized:   true,
 		Parameters: []handlers_rds.Parameter{
 			{Name: "max_connections", Value: "200"},
 			{Name: "shared_buffers", Value: "256MB"},
@@ -38,6 +45,10 @@ func TestWriteHandoff_WritesEveryFileRootOnly(t *testing.T) {
 		"RDS_MASTER_PASSWORD='s3cret'",
 		"RDS_DB_NAME='appdb'",
 		"RDS_PORT='5433'",
+		"RDS_DATA_VOLUME_ID='vol-data-01'",
+		"RDS_DATA_VOLUME_SERIAL='voldata01'",
+		"RDS_VM_GENERATION='4'",
+		"RDS_FORMAT_AUTHORIZED='true'",
 	} {
 		if !strings.Contains(env, want) {
 			t.Errorf("handoff env missing %s:\n%s", want, env)
@@ -74,15 +85,29 @@ func TestWriteHandoff_AttachOmitsPassword(t *testing.T) {
 	dir := filepath.Join(t.TempDir(), "handoff")
 	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:           handlers_rds.BootstrapModeAttach,
+		Engine:         "postgres",
 		MasterUsername: "master",
+		ListenAddress:  "10.0.1.20",
+		ClientCIDR:     "10.0.0.0/16",
 		Port:           5432,
 	}); err != nil {
 		t.Fatalf("writeHandoff: %v", err)
 	}
 
-	if env := readFile(t, filepath.Join(dir, handoffEnvFile)); strings.Contains(env, "RDS_MASTER_PASSWORD") {
+	env := readFile(t, filepath.Join(dir, handoffEnvFile))
+	if strings.Contains(env, "RDS_MASTER_PASSWORD") {
 		t.Errorf("handoff env = %q, want no RDS_MASTER_PASSWORD key", env)
 	}
+	for _, want := range []string{
+		"RDS_DATA_VOLUME_ID=''",
+		"RDS_DATA_VOLUME_SERIAL=''",
+		"RDS_VM_GENERATION='0'",
+		"RDS_FORMAT_AUTHORIZED='false'",
+	} {
+		if !strings.Contains(env, want) {
+			t.Errorf("handoff env missing fail-closed field %s: %q", want, env)
+		}
+	}
 }
 
 // The env file is sourced by a shell, so a password holding shell metacharacters
@@ -93,7 +118,10 @@ func TestWriteHandoff_QuotesShellMetacharacters(t *testing.T) {
 
 	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:               handlers_rds.BootstrapModeInitialize,
+		Engine:             "postgres",
 		MasterUsername:     "master",
+		ListenAddress:      "10.0.1.20",
+		ClientCIDR:         "10.0.0.0/16",
 		MasterUserPassword: &password,
 		Port:               5432,
 	}); err != nil {
@@ -113,7 +141,10 @@ func TestWriteHandoff_SkipsHalfATLSPair(t *testing.T) {
 	dir := filepath.Join(t.TempDir(), "handoff")
 	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:               handlers_rds.BootstrapModeAttach,
+		Engine:             "postgres",
 		MasterUsername:     "master",
+		ListenAddress:      "10.0.1.20",
+		ClientCIDR:         "10.0.0.0/16",
 		Port:               5432,
 		ServingCertificate: "CERT",
 	}); err != nil {
@@ -130,8 +161,9 @@ func TestWriteHandoff_SkipsHalfATLSPair(t *testing.T) {
 func TestWriteHandoff_RejectsConfigWithNoMasterUser(t *testing.T) {
 	dir := filepath.Join(t.TempDir(), "handoff")
 	err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
-		Mode: handlers_rds.BootstrapModeAttach,
-		Port: 5432,
+		Mode:   handlers_rds.BootstrapModeAttach,
+		Engine: "postgres",
+		Port:   5432,
 	})
 	if err == nil {
 		t.Fatal("writeHandoff accepted a config with no master username, want an error")
@@ -151,7 +183,10 @@ func TestWriteHandoff_TightensExistingDirectory(t *testing.T) {
 
 	if err := writeHandoff(dir, &handlers_rds.GetDBBootstrapConfigOutput{
 		Mode:           handlers_rds.BootstrapModeAttach,
+		Engine:         "postgres",
 		MasterUsername: "master",
+		ListenAddress:  "10.0.1.20",
+		ClientCIDR:     "10.0.0.0/16",
 		Port:           5432,
 	}); err != nil {
 		t.Fatalf("writeHandoff: %v", err)
@@ -200,3 +235,46 @@ func findLine(t *testing.T, body, prefix string) string {
 	t.Fatalf("no line starting %q in:\n%s", prefix, body)
 	return ""
 }
+
+// The include rds-init installs is what the engine starts on, so the spelling
+// written here is the one that has to boot.
+func TestRenderParameters_UsesTheEngineStartupSpelling(t *testing.T) {
+	params := []handlers_rds.Parameter{
+		{Name: "time_zone", Value: "SYSTEM"},
+		{Name: "max_connections", Value: "85"},
+	}
+
+	rendered, err := renderParameters("mariadb", params)
+	if err != nil {
+		t.Fatalf("renderParameters: %v", err)
+	}
+	if !strings.Contains(rendered, "default_time_zone = 'SYSTEM'\n") {
+		t.Errorf("time_zone was not written under its startup spelling:\n%s", rendered)
+	}
+	if strings.Contains(rendered, "\ntime_zone = ") {
+		t.Errorf("time_zone reached the option file, which mariadbd refuses:\n%s", rendered)
+	}
+	if !strings.Contains(rendered, "max_connections = '85'\n") {
+		t.Errorf("an unmapped parameter was not written unchanged:\n%s", rendered)
+	}
+}
+
+func TestRenderParameters_PostgresKeepsEveryName(t *testing.T) {
+	rendered, err := renderParameters("postgres", []handlers_rds.Parameter{
+		{Name: "shared_buffers", Value: "262144"},
+	})
+	if err != nil {
+		t.Fatalf("renderParameters: %v", err)
+	}
+	if !strings.Contains(rendered, "shared_buffers = '262144'\n") {
+		t.Errorf("postgres name was rewritten:\n%s", rendered)
+	}
+}
+
+// Failing closed beats writing a file the engine may refuse: the bad include
+// would already be on the data volume by the time the server rejected it.
+func TestRenderParameters_RefusesAnUnknownEngine(t *testing.T) {
+	if _, err := renderParameters("", nil); err == nil {
+		t.Fatal("renderParameters accepted a config naming no engine, want an error")
+	}
+}

cmd/rds-agent/config.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/mulgadc/spinifex/internal/guestenv"
@@ -11,18 +13,11 @@ const (
 	defaultEnvFile    = "/etc/spinifex-rds/agent.env"
 	defaultGatewayCA  = "/etc/spinifex-rds/gateway-ca.pem"
 	defaultHandoffDir = "/run/spinifex-rds"
-	defaultEngineHost = "127.0.0.1"
-	defaultEnginePort = 5432
-	defaultPGIsReady  = "pg_isready"
-
-	// The guest layout rds-init lays down. Overridable for the same reason it is
-	// there: a sibling engine preset points at its own paths.
-	defaultPGBin         = "/usr/libexec/postgresql18"
-	defaultPGData        = "/var/lib/postgresql/18/data"
-	defaultSocketDir     = "/run/postgresql"
-	defaultPGUser        = "postgres"
-	defaultRCService     = "rc-service"
-	defaultEngineSvcName = "postgresql"
+	// Where setup.sh stamps the engine the image bakes. The agent builds its
+	// engine implementation from this file rather than from anything delivered.
+	defaultEngineFile         = "/etc/spinifex-rds/engine"
+	defaultRCService          = "rc-service"
+	defaultEngineProbeTimeout = 5 * time.Second
 	// The long-poll window the agent asks the gateway to hold a request open
 	// for. The gateway caps it at 20s.
 	defaultPollWait = 20 * time.Second
@@ -42,21 +37,34 @@ type config struct {
 	// version alone rather than clearing it.
 	EngineVersion string
 	HandoffDir    string
-	EngineHost    string
 	EnginePort    int
-	// Overridable so a test or a sibling engine preset can point at its own.
-	PGIsReady string
-	PollWait  time.Duration
+	PollWait      time.Duration
+
+	// The engine the control plane launched this VM as, delivered by cloud-init,
+	// and the engine the image itself bakes. Two independent statements of the
+	// same fact: the agent refuses to bootstrap when they disagree.
+	Engine      string
+	EngineFile  string
+	BakedEngine string
 
 	// Where the command handlers reach the engine: the client binaries, the
 	// datadir the parameter file is installed into, the socket they connect
 	// over, the OS user they drop to, and the service the stop goes through.
-	PGBin         string
-	PGData        string
+	EngineBinDir  string
+	EngineDataDir string
 	SocketDir     string
-	PGUser        string
+	EngineUser    string
 	RCService     string
 	EngineService string
+	// Where the engine records the pid its probe checks for liveness, for an
+	// engine whose own client cannot tell a server still recovering from one that
+	// is not running at all.
+	EnginePidFile      string
+	EngineProbeTimeout time.Duration
+	// Where the engine records why it would not start, which the probe quotes
+	// when the server is up but not serving. Overridable so a test can point it
+	// at a fixture.
+	EngineErrorLog string
 
 	// Where the data volume is mounted, and the two kernel surfaces a storage
 	// grow resolves its device from. Overridable so a test can point them at
@@ -75,19 +83,21 @@ func loadConfig(envFile string) config {
 		GatewayCA:            get("RDS_GATEWAY_CA"),
 		Region:               get("RDS_REGION"),
 		DBInstanceIdentifier: get("RDS_DB_INSTANCE_IDENTIFIER"),
+		Engine:               get("RDS_ENGINE"),
+		EngineFile:           get("RDS_ENGINE_FILE"),
 		EngineVersion:        get("RDS_ENGINE_VERSION"),
 		HandoffDir:           get("RDS_HANDOFF_DIR"),
-		EngineHost:           get("RDS_ENGINE_HOST"),
-		PGIsReady:            get("RDS_PG_ISREADY"),
-		PGBin:                get("RDS_PG_BIN"),
-		PGData:               get("RDS_PGDATA"),
+		EngineBinDir:         get("RDS_ENGINE_BIN"),
+		EngineDataDir:        get("RDS_ENGINE_DATA"),
 		SocketDir:            get("RDS_SOCKET_DIR"),
-		PGUser:               get("RDS_PG_USER"),
+		EngineUser:           get("RDS_ENGINE_USER"),
 		RCService:            get("RDS_RC_SERVICE"),
 		EngineService:        get("RDS_ENGINE_SERVICE"),
+		EnginePidFile:        get("RDS_ENGINE_PIDFILE"),
+		EngineErrorLog:       get("RDS_ENGINE_LOG"),
 		DataMount:            get("RDS_DATA_MOUNT"),
-		EnginePort:           defaultEnginePort,
 		PollWait:             defaultPollWait,
+		EngineProbeTimeout:   defaultEngineProbeTimeout,
 	}
 	if cfg.GatewayCA == "" {
 		cfg.GatewayCA = defaultGatewayCA
@@ -95,35 +105,20 @@ func loadConfig(envFile string) config {
 	if cfg.HandoffDir == "" {
 		cfg.HandoffDir = defaultHandoffDir
 	}
-	if cfg.EngineHost == "" {
-		cfg.EngineHost = defaultEngineHost
-	}
-	if cfg.PGIsReady == "" {
-		cfg.PGIsReady = defaultPGIsReady
-	}
-	if cfg.PGBin == "" {
-		cfg.PGBin = defaultPGBin
-	}
-	if cfg.PGData == "" {
-		cfg.PGData = defaultPGData
-	}
-	if cfg.SocketDir == "" {
-		cfg.SocketDir = defaultSocketDir
-	}
-	if cfg.PGUser == "" {
-		cfg.PGUser = defaultPGUser
+	if cfg.EngineFile == "" {
+		cfg.EngineFile = defaultEngineFile
 	}
 	if cfg.RCService == "" {
 		cfg.RCService = defaultRCService
 	}
-	if cfg.EngineService == "" {
-		cfg.EngineService = defaultEngineSvcName
-	}
+	// The image's own stamp rather than the delivered engine: an agent that took
+	// its layout from the payload could run one engine's implementation against
+	// another engine's datadir, which is the failure the stamp exists to prevent.
+	cfg.BakedEngine = readBakedEngine(cfg.EngineFile)
+	cfg.applyLayout(engineLayouts[cfg.BakedEngine])
+
 	// rds-datadir reads the same RDS_DATA_MOUNT, so the two agree on where the
 	// volume landed without either asserting it to the other.
-	if cfg.DataMount == "" {
-		cfg.DataMount = defaultDataMount
-	}
 	cfg.MountsFile = defaultMountsFile
 	cfg.SysBlock = defaultSysBlock
 	// The authoritative port comes from the bootstrap config; this is only what
@@ -138,5 +133,53 @@ func loadConfig(envFile string) config {
 			cfg.PollWait = d
 		}
 	}
+	if v := get("RDS_ENGINE_PROBE_TIMEOUT"); v != "" {
+		if d, err := time.ParseDuration(v); err == nil && d > 0 {
+			cfg.EngineProbeTimeout = d
+		}
+	}
 	return cfg
 }
+
+// Fills in whatever the delivered configuration left unset. An override always
+// wins, so a test can point any of these at a fixture. An unrecognised engine
+// leaves the layout empty, and New refuses rather than guessing at one.
+func (c *config) applyLayout(layout engineLayout) {
+	if c.EngineBinDir == "" {
+		c.EngineBinDir = layout.binDir
+	}
+	if c.EngineDataDir == "" {
+		c.EngineDataDir = layout.dataDir
+	}
+	if c.SocketDir == "" {
+		c.SocketDir = layout.socketDir
+	}
+	if c.EngineUser == "" {
+		c.EngineUser = layout.osUser
+	}
+	if c.EngineService == "" {
+		c.EngineService = layout.service
+	}
+	if c.EnginePidFile == "" {
+		c.EnginePidFile = layout.pidFile
+	}
+	if c.EngineErrorLog == "" {
+		c.EngineErrorLog = layout.errorLog
+	}
+	if c.DataMount == "" {
+		c.DataMount = layout.dataMount
+	}
+	if c.EnginePort == 0 {
+		c.EnginePort = layout.port
+	}
+}
+
+// The engine the image bakes. An absent or unreadable stamp reads as empty,
+// which New refuses: an agent that guessed would defeat the stamp.
+func readBakedEngine(path string) string {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return ""
+	}
+	return strings.ToLower(strings.TrimSpace(string(raw)))
+}

cmd/rds-agent/config_test.go

@@ -3,21 +3,34 @@ package main
 import (
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 )
 
+// Stamps the engine a real image bakes and loads the configuration against it,
+// which is what selects the guest layout the agent's defaults come from.
+func testLoadConfig(t *testing.T, engine string) config {
+	t.Helper()
+	stamp := filepath.Join(t.TempDir(), "engine")
+	if err := os.WriteFile(stamp, []byte(engine+"\n"), 0o444); err != nil {
+		t.Fatalf("write the engine stamp: %v", err)
+	}
+	t.Setenv("RDS_ENGINE_FILE", stamp)
+	return loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+}
+
 func TestLoadConfig_Defaults(t *testing.T) {
-	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+	cfg := testLoadConfig(t, enginePostgres)
 
 	if cfg.GatewayCA != defaultGatewayCA || cfg.HandoffDir != defaultHandoffDir {
 		t.Errorf("CA/handoff = %q/%q, want the built-in defaults", cfg.GatewayCA, cfg.HandoffDir)
 	}
-	if cfg.EngineHost != defaultEngineHost || cfg.EnginePort != defaultEnginePort {
-		t.Errorf("engine endpoint = %s:%d, want the local default", cfg.EngineHost, cfg.EnginePort)
+	if cfg.PollWait != defaultPollWait {
+		t.Errorf("pollWait = %v, want the built-in default", cfg.PollWait)
 	}
-	if cfg.PGIsReady != defaultPGIsReady || cfg.PollWait != defaultPollWait {
-		t.Errorf("probe/pollWait = %q/%v, want the built-in defaults", cfg.PGIsReady, cfg.PollWait)
+	if cfg.EngineProbeTimeout != defaultEngineProbeTimeout {
+		t.Errorf("engine probe timeout = %v, want %v", cfg.EngineProbeTimeout, defaultEngineProbeTimeout)
 	}
 	// Identity is optional: the gateway resolves it from the caller's
 	// credentials, so an agent with no configured identifier is normal.
@@ -26,11 +39,94 @@ func TestLoadConfig_Defaults(t *testing.T) {
 	}
 }
 
+// The guest layout is the image's to state, not the control plane's: everything
+// the agent reaches the engine through comes from the stamp it was built with.
+func TestLoadConfig_TakesTheLayoutFromTheBakedEngine(t *testing.T) {
+	cfg := testLoadConfig(t, enginePostgres)
+
+	if cfg.BakedEngine != enginePostgres {
+		t.Fatalf("baked engine = %q, want %q", cfg.BakedEngine, enginePostgres)
+	}
+	layout := engineLayouts[enginePostgres]
+	if cfg.EngineBinDir != layout.binDir || cfg.EngineDataDir != layout.dataDir {
+		t.Errorf("bin/data = %q/%q, want the PostgreSQL layout", cfg.EngineBinDir, cfg.EngineDataDir)
+	}
+	if cfg.EngineUser != layout.osUser || cfg.EngineService != layout.service {
+		t.Errorf("user/service = %q/%q, want the PostgreSQL layout", cfg.EngineUser, cfg.EngineService)
+	}
+	if cfg.SocketDir != layout.socketDir || cfg.DataMount != layout.dataMount {
+		t.Errorf("socket/mount = %q/%q, want the PostgreSQL layout", cfg.SocketDir, cfg.DataMount)
+	}
+	if cfg.EnginePort != layout.port {
+		t.Errorf("port = %d, want the PostgreSQL default %d", cfg.EnginePort, layout.port)
+	}
+}
+
+// An image with no stamp cannot say which engine it is, and an agent that
+// guessed would be the failure the stamp exists to prevent.
+func TestNewEngine_RefusesAnImageWithNoEngineStamp(t *testing.T) {
+	t.Setenv("RDS_ENGINE_FILE", filepath.Join(t.TempDir(), "absent-engine"))
+	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
+
+	if _, err := newProbe(cfg, staticProbe(0)); err == nil {
+		t.Error("newProbe built a probe for an image carrying no engine stamp")
+	}
+	if _, err := newEngine(cfg, nil, nil, nil); err == nil {
+		t.Error("newEngine built an engine for an image carrying no engine stamp")
+	}
+}
+
+func TestNewEngine_RefusesAnEngineItDoesNotImplement(t *testing.T) {
+	cfg := testLoadConfig(t, "oracle")
+
+	_, err := newEngine(cfg, nil, nil, nil)
+	if err == nil {
+		t.Fatal("newEngine built an engine this agent does not implement")
+	}
+	if !strings.Contains(err.Error(), "oracle") {
+		t.Errorf("error = %v, want it to name the engine the image bakes", err)
+	}
+}
+
+// The implementation is the agent's, but the definition it validates names and
+// classifies parameters against is the control plane's, which now offers MariaDB.
+func TestNewEngine_BuildsMariaDBForAMariaDBImage(t *testing.T) {
+	cfg := testLoadConfig(t, engineMariaDB)
+	probe, err := newProbe(cfg, staticProbe(0))
+	if err != nil {
+		t.Fatalf("newProbe: %v", err)
+	}
+
+	built, err := newEngine(cfg, nil, nil, probe)
+	if err != nil {
+		t.Fatalf("newEngine: %v", err)
+	}
+	if _, ok := built.(*mariadbEngine); !ok {
+		t.Errorf("newEngine built %T for a MariaDB image", built)
+	}
+}
+
+// The probe reads the pidfile Alpine's packaged service starts mariadbd with,
+// because nothing in the generated configuration can move it: the service passes
+// --pid-file on the command line, which beats an option file.
+func TestLoadConfig_TakesTheMariaDBPidfileFromTheLayout(t *testing.T) {
+	cfg := testLoadConfig(t, engineMariaDB)
+
+	if cfg.EnginePidFile != engineLayouts[engineMariaDB].pidFile {
+		t.Errorf("pidfile = %q, want the MariaDB layout's", cfg.EnginePidFile)
+	}
+	if cfg.EnginePidFile == "" {
+		t.Error("an engine whose probe needs a pidfile was given none")
+	}
+}
+
 func TestLoadConfig_ReadsEnvFile(t *testing.T) {
+	t.Setenv("RDS_ENGINE_FILE", filepath.Join(t.TempDir(), "absent-engine"))
 	path := filepath.Join(t.TempDir(), "agent.env")
 	body := "RDS_GATEWAY_URL=https://gw.internal:9999\n" +
 		"RDS_REGION=ap-southeast-2\n" +
 		"RDS_DB_INSTANCE_IDENTIFIER=db-1\n" +
+		"RDS_ENGINE=postgres\n" +
 		"RDS_ENGINE_VERSION=18.1\n" +
 		"RDS_ENGINE_PORT=6543\n" +
 		"RDS_POLL_WAIT=5s\n"
@@ -45,19 +141,22 @@ func TestLoadConfig_ReadsEnvFile(t *testing.T) {
 	if cfg.DBInstanceIdentifier != "db-1" || cfg.EngineVersion != "18.1" {
 		t.Errorf("identity = %q/%q, want db-1/18.1", cfg.DBInstanceIdentifier, cfg.EngineVersion)
 	}
+	if cfg.Engine != enginePostgres {
+		t.Errorf("delivered engine = %q, want the one cloud-init carried", cfg.Engine)
+	}
 	if cfg.EnginePort != 6543 || cfg.PollWait != 5*time.Second {
 		t.Errorf("port/pollWait = %d/%v, want 6543/5s", cfg.EnginePort, cfg.PollWait)
 	}
 }
 
-// An unparseable override falls back to the default rather than leaving the
-// agent with a zero port or a zero-length long poll.
+// An unparseable override falls back to the layout's own default rather than
+// leaving the agent with a zero port or a zero-length long poll.
 func TestLoadConfig_IgnoresUnusableOverrides(t *testing.T) {
 	t.Setenv("RDS_ENGINE_PORT", "not-a-port")
 	t.Setenv("RDS_POLL_WAIT", "forever")
 
-	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
-	if cfg.EnginePort != defaultEnginePort || cfg.PollWait != defaultPollWait {
+	cfg := testLoadConfig(t, enginePostgres)
+	if cfg.EnginePort != engineLayouts[enginePostgres].port || cfg.PollWait != defaultPollWait {
 		t.Errorf("port/pollWait = %d/%v, want the defaults", cfg.EnginePort, cfg.PollWait)
 	}
 }

cmd/rds-agent/defaultparams_test.go

@@ -0,0 +1,73 @@
+package main
+
+//test:in-package — the agent is a main package, which has no external test
+// package to import it from, and this covers the unexported default parameter
+// set the image bakes.
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The include the image build validates against a real mariadbd. It is checked
+// in rather than generated during the build because the build runs inside the
+// libguestfs appliance, where there is no Go toolchain to resolve the catalog.
+const mariadbDefaultParametersFixture = "../../scripts/images/rds-mariadb/default-parameters.cnf"
+
+// Byte-for-byte what rds-init installs on a smallest-class instance that
+// overrides nothing. Regenerate with RDS_UPDATE_FIXTURES=1 go test ./cmd/rds-agent/.
+//
+// Keeping it in step matters because setup.sh feeds exactly this file to
+// mariadbd: a catalog that grew a name the server refuses at startup would
+// otherwise reach a customer as an instance that never boots.
+func TestMariaDBDefaultParametersFixture_MatchesTheCatalog(t *testing.T) {
+	engine, err := handlers_rds.LookupEngine("mariadb")
+	if err != nil {
+		t.Fatalf("LookupEngine: %v", err)
+	}
+	class := handlers_rds.SmallestInstanceClass()
+	params, err := engine.ResolveEffectiveParameters(class, nil)
+	if err != nil {
+		t.Fatalf("ResolveEffectiveParameters(%s): %v", class, err)
+	}
+	body, err := renderParameters("mariadb", params)
+	if err != nil {
+		t.Fatalf("renderParameters: %v", err)
+	}
+	want := mariadbParametersHeader + body
+
+	path := filepath.FromSlash(mariadbDefaultParametersFixture)
+	if os.Getenv("RDS_UPDATE_FIXTURES") == "1" {
+		if err := os.WriteFile(path, []byte(want), 0o644); err != nil {
+			t.Fatalf("update %s: %v", path, err)
+		}
+		return
+	}
+
+	got, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read %s: %v", path, err)
+	}
+	if string(got) != want {
+		t.Errorf("%s is stale for class %s.\nRegenerate with RDS_UPDATE_FIXTURES=1 go test ./cmd/rds-agent/\n\ngot:\n%s\nwant:\n%s",
+			path, class, got, want)
+	}
+}
+
+// The one thing the fixture exists to prove cannot regress without the image
+// build catching it, asserted here too so a plain preflight names it.
+func TestMariaDBDefaultParametersFixture_CarriesNoSetOnlyName(t *testing.T) {
+	got, err := os.ReadFile(filepath.FromSlash(mariadbDefaultParametersFixture))
+	if err != nil {
+		t.Fatalf("read the fixture: %v", err)
+	}
+	for _, name := range []string{"\ntime_zone ", "\ntx_isolation ", "\ninnodb_buffer_pool_instances "} {
+		if strings.Contains(string(got), name) {
+			t.Errorf("the default include carries %q, which mariadbd refuses at startup", name)
+		}
+	}
+}

cmd/rds-agent/engine.go

@@ -3,7 +3,6 @@ package main
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"os"
 	"os/exec"
@@ -11,17 +10,15 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-	"sync"
-	"sync/atomic"
 	"syscall"
 	"time"
 
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
 
-// The in-guest half of the engine seam. Everything PostgreSQL-shaped that a
-// control-plane command needs lives behind this, so the command registry above
-// stays engine-agnostic.
+// The in-guest half of the engine seam. Everything engine-shaped that a
+// control-plane command needs lives behind this, so the command registry stays
+// engine-agnostic.
 type engineOps interface {
 	// Rotates the master role's password live. Never persisted anywhere in the
 	// guest: it exists only for the length of this call.
@@ -44,11 +41,127 @@ type engineOps interface {
 // a control-plane directive: nothing issues it, the agent runs it when the
 // engine fails to come up after a parameter change.
 type parameterRecovery interface {
-	// Reports whether the installed set differed and was replaced.
-	RestoreLastKnownGoodParameters() (bool, error)
+	// Reports whether the installed set differed and was replaced. True with a
+	// non-nil error is a set already on disk whose re-derived TLS enforcement did
+	// not land: the caller still has to restart and report.
+	RestoreLastKnownGoodParameters(ctx context.Context) (bool, error)
 	Restart(ctx context.Context) error
 }
 
+// Records the parameter file only after the probe observes the engine serving
+// it. This is separate from apply, where static values are installed but not
+// adopted until a later restart.
+type servingParameterRecorder interface {
+	RecordServingParameters(ctx context.Context) error
+}
+
+// Everything one engine implementation owns. The narrow interfaces above are
+// what the heartbeat, the commander and the rollback guard each hold; this is
+// only what the agent has to construct.
+type engine interface {
+	engineOps
+	parameterRecovery
+	servingParameterRecorder
+}
+
+// The guest layout an engine preset lays down, keyed by the engine the image
+// bakes. Every default that belongs to one engine lives here, so nothing
+// outside an implementation names its paths, service, port or probe.
+type engineLayout struct {
+	binDir    string
+	dataDir   string
+	socketDir string
+	osUser    string
+	service   string
+	dataMount string
+	port      int
+	// Where the engine records the pid the probe checks for liveness. Empty for
+	// an engine whose probe answers without one.
+	pidFile string
+	// Where the engine records why it would not start. Empty for an engine whose
+	// image does not direct one, which leaves the probe reporting its own reason
+	// alone rather than guessing at a path.
+	errorLog string
+	// The engine's own liveness signal, which is the only part of engineProbe
+	// that is not shared.
+	newProbe func(cfg config, run probeRunner) *engineProbe
+	// The implementation every control-plane directive is served by. It fails
+	// when the control plane this agent was built against does not offer the
+	// engine the image bakes, rather than serving one it has no definition for.
+	newEngine func(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error)
+}
+
+// The names the control plane knows these implementations by, which are also
+// what setup.sh stamps into each image.
+const (
+	enginePostgres = "postgres"
+	engineMariaDB  = "mariadb"
+)
+
+var engineLayouts = map[string]engineLayout{
+	enginePostgres: {
+		binDir:    "/usr/libexec/postgresql18",
+		dataDir:   "/var/lib/postgresql/18/data",
+		socketDir: "/run/postgresql",
+		osUser:    "postgres",
+		service:   "postgresql",
+		dataMount: "/var/lib/postgresql",
+		port:      5432,
+		newProbe:  newPostgresProbe,
+		newEngine: newPostgresEngineFromCatalog,
+	},
+	engineMariaDB: {
+		binDir:    "/usr/bin",
+		dataDir:   "/var/lib/mysql/data",
+		socketDir: "/run/mysqld",
+		osUser:    "mysql",
+		service:   "mariadb",
+		dataMount: "/var/lib/mysql",
+		port:      3306,
+		// Alpine's service passes --pid-file=/run/mysqld/$RC_SVCNAME.pid, and the
+		// command line beats the option file, so this is the path rather than
+		// anything the drop-in names. setup.sh asserts it at build time.
+		pidFile: "/run/mysqld/mariadb.pid",
+		// log_error in the platform drop-in rds-init writes. Without it
+		// mysqld_safe would send the server's errors to syslog, which nothing
+		// collects off a guest that has no SSH.
+		errorLog:  "/var/log/mysql/error.log",
+		newProbe:  newMariaDBProbe,
+		newEngine: newMariaDBEngineFromCatalog,
+	},
+}
+
+// Builds the health probe the image's engine supplies.
+func newProbe(cfg config, run probeRunner) (*engineProbe, error) {
+	layout, err := layoutFor(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return layout.newProbe(cfg, run), nil
+}
+
+// Builds the implementation the image bakes. Nothing here consults the control
+// plane for which engine to be: the agent runs the engine it is made of, and the
+// delivered engine is only ever checked against it.
+func newEngine(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+	layout, err := layoutFor(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return layout.newEngine(cfg, run, startSess, probe)
+}
+
+func layoutFor(cfg config) (engineLayout, error) {
+	if cfg.BakedEngine == "" {
+		return engineLayout{}, fmt.Errorf("this image carries no engine stamp at %s", cfg.EngineFile)
+	}
+	layout, ok := engineLayouts[cfg.BakedEngine]
+	if !ok {
+		return engineLayout{}, fmt.Errorf("this image bakes engine %q, which this agent does not implement", cfg.BakedEngine)
+	}
+	return layout, nil
+}
+
 // One child process. Env replaces the agent's own environment rather than
 // extending it, so a secret placed here reaches only the process that needs it.
 type command struct {
@@ -58,10 +171,14 @@ type command struct {
 	Stdin string
 	// The OS user to drop to before exec. Empty runs as the agent itself.
 	User string
+	// The statement that makes this client print the session sentinel. Read only
+	// when the command is started as a held session, whose reader has to know
+	// when the engine has finished each script it was fed.
+	SentinelStatement string
 }
 
-// Returns stdout. stderr is folded into the error, because psql reports the
-// actual SQL failure there while exiting with a bare status.
+// Returns stdout. stderr is folded into the error, because both clients report
+// the actual SQL failure there while exiting with a bare status.
 type commandRunner func(ctx context.Context, c command) (string, error)
 
 func execCommandRunner(ctx context.Context, c command) (string, error) {
@@ -106,303 +223,51 @@ func lookupCredential(username string) (*syscall.Credential, error) {
 	return &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)}, nil
 }
 
-// The engine is reached over its unix socket under peer authentication, so the
-// agent drops to the postgres OS user rather than holding a password of its own.
-type postgresEngine struct {
-	run       commandRunner
-	startSess sessionRunner
-	psql      string
-	rcService string
-	service   string
-	pgData    string
-	socketDir string
-	osUser    string
-	// Set from the bootstrap config, on a different goroutine than the commands
-	// that read it.
-	port atomic.Int64
-
-	// The backup mode currently held, if any. Guarded because the expiry timer
-	// releases it on its own goroutine.
-	mu   sync.Mutex
-	held *quiesceHold
-}
-
-var _ engineOps = (*postgresEngine)(nil)
-
-func newPostgresEngine(cfg config, run commandRunner, startSess sessionRunner) *postgresEngine {
-	e := &postgresEngine{
-		run:       run,
-		startSess: startSess,
-		psql:      filepath.Join(cfg.PGBin, "psql"),
-		rcService: cfg.RCService,
-		service:   cfg.EngineService,
-		pgData:    cfg.PGData,
-		socketDir: cfg.SocketDir,
-		osUser:    cfg.PGUser,
-	}
-	e.port.Store(int64(cfg.EnginePort))
-	return e
-}
-
-func (e *postgresEngine) setPort(port int) {
-	e.port.Store(int64(port))
-}
-
-// The role name and the password ride the environment and are re-quoted by
-// psql, so neither reaches a shell word or an argv another process can read.
-func (e *postgresEngine) SetPassword(ctx context.Context, username, password string) error {
-	if username == "" || password == "" {
-		return fmt.Errorf("set-password requires both %s and %s",
-			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
-	}
-	// The statement below runs as the cluster superuser over the socket under
-	// peer auth, so a reserved name in the command payload would hand the
-	// customer the bootstrap superuser rather than rotate their own role.
-	if err := handlers_rds.EnginePostgres().ValidateUsernameNotReserved(username); err != nil {
-		return fmt.Errorf("refusing to set the password of a role the engine reserves: %w", err)
-	}
-	const sql = `\getenv master RDS_MASTER_USERNAME
-\getenv password RDS_MASTER_PASSWORD
-ALTER ROLE :"master" WITH LOGIN PASSWORD :'password';
-`
-	_, err := e.psqlRun(ctx, sql,
-		"RDS_MASTER_USERNAME="+username,
-		"RDS_MASTER_PASSWORD="+password,
-	)
-	if err != nil {
-		// psql echoes the failing statement, which here would carry the new
-		// password back to the control plane and into the event ring.
-		return fmt.Errorf("apply the master password: %s", redact(err.Error(), password))
+// Through the service manager rather than the engine's own control tool, so the
+// supervisor records the state and does not restart the engine underneath a VM
+// that is going down.
+func serviceAction(ctx context.Context, run commandRunner, rcService, service, action string) error {
+	if _, err := run(ctx, command{
+		Name: rcService,
+		Args: []string{service, action},
+		Env:  []string{"PATH=" + defaultGuestPath},
+	}); err != nil {
+		return fmt.Errorf("%s the %s service: %w", action, service, err)
 	}
 	return nil
 }
 
-// The include the resolved parameter set is rendered to, and the copy of the
-// last one the engine accepted. Both live beside the data rather than in /etc,
-// matching rds-init: a class change boots a fresh root volume, which would
-// otherwise revert them when a fresh root volume is used.
-const (
-	parametersFileName = "10-rds-parameters.conf"
-	// Deliberately not a .conf name: include_dir globs *.conf, so the rollback
-	// copy must not be read as a second set of settings.
-	lastGoodFileName = "10-rds-parameters.last-good"
-)
-
-// Installs the resolved set, validates it with the engine's own config parser
-// before the engine ever adopts it, and reloads. A value the engine refuses is
-// rolled back here rather than left on the data volume, where it would survive
-// every VM replace and turn the next restart into a boot loop.
-func (e *postgresEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
-	// The check has to run against the file in place, because the engine parses
-	// the datadir's own include_dir. The window that leaves is closed from both
-	// ends: the rollback below, and the last-known-good restore the agent runs at
-	// boot when the engine does not come up.
-	previous, restore, err := e.installParameters(params)
-	if err != nil {
-		return nil, err
-	}
-	if err := e.checkConfig(ctx); err != nil {
-		return nil, errors.Join(fmt.Errorf("the engine rejected the parameter set: %w", err), restore())
-	}
-	if _, err := e.psqlRun(ctx, "SELECT pg_reload_conf();\n"); err != nil {
-		return nil, errors.Join(fmt.Errorf("reload the engine configuration: %w", err), restore())
-	}
-
-	// Recorded only once the engine has both parsed and adopted it, so the
-	// rollback target is a set that is known to work rather than the last one
-	// written.
-	if err := e.recordLastKnownGood(previous); err != nil {
-		return nil, err
-	}
-
-	// Read after the reload: a setting only becomes pending_restart once the
-	// engine has seen the new value and declined to adopt it live.
-	out, err := e.psqlRun(ctx, "SELECT name FROM pg_settings WHERE pending_restart ORDER BY name;\n")
-	if err != nil {
-		return nil, fmt.Errorf("read the settings pending a restart: %w", err)
-	}
-	var pending []string
-	for line := range strings.SplitSeq(out, "\n") {
-		if name := strings.TrimSpace(line); name != "" {
-			pending = append(pending, name)
-		}
-	}
-	return pending, nil
-}
-
-func (e *postgresEngine) parametersPath() string {
-	return filepath.Join(e.pgData, "conf.d", parametersFileName)
-}
-
-func (e *postgresEngine) lastGoodPath() string {
-	return filepath.Join(e.pgData, "conf.d", lastGoodFileName)
-}
-
-// Written as root and handed to the engine's user, at the same path and mode
-// rds-init installs it at, so a later boot overwrites rather than shadows it.
-// Returns the content that was there and a func putting it back, so the caller
-// can undo the install without re-rendering anything.
-func (e *postgresEngine) installParameters(params []handlers_rds.Parameter) (previous []byte, restore func() error, err error) {
-	path := e.parametersPath()
-	// A missing file is the first-ever apply, and its rollback is a removal.
-	previous, readErr := os.ReadFile(path)
-	if readErr != nil && !os.IsNotExist(readErr) {
-		return nil, nil, fmt.Errorf("read the installed parameters at %s: %w", path, readErr)
-	}
-	existed := readErr == nil
-
-	if err := e.writeEngineFile(path, []byte(renderParameters(params))); err != nil {
-		return nil, nil, err
-	}
-	return previous, func() error {
-		if !existed {
-			if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
-				return fmt.Errorf("withdraw the rejected parameters at %s: %w", path, err)
-			}
-			return nil
-		}
-		return e.writeEngineFile(path, previous)
-	}, nil
-}
-
-// Keeps the rollback target in step with what the engine is actually running.
-// The first apply of an instance's life has no predecessor, so the freshly
-// accepted set becomes the target instead.
-func (e *postgresEngine) recordLastKnownGood(previous []byte) error {
-	content := previous
-	if content == nil {
-		var err error
-		if content, err = os.ReadFile(e.parametersPath()); err != nil {
-			return fmt.Errorf("read the accepted parameters: %w", err)
-		}
-	}
-	return e.writeEngineFile(e.lastGoodPath(), content)
-}
-
-// Puts the last set the engine accepted back in place, for a restart that failed
-// after a parameter change: the instance comes back on the old parameters rather
-// than boot-looping into recovery. Reports whether anything was restored.
-func (e *postgresEngine) RestoreLastKnownGoodParameters() (bool, error) {
-	lastGood, err := os.ReadFile(e.lastGoodPath())
-	if err != nil {
-		if os.IsNotExist(err) {
-			return false, nil
-		}
-		return false, fmt.Errorf("read the last known good parameters: %w", err)
-	}
-	current, err := os.ReadFile(e.parametersPath())
+// Whether the installed set requires TLS of client connections. Read back from
+// the file rather than taken from the set being applied, because the restore and
+// repair paths put a file in place without holding one. Both engines' generated
+// parameter files are written in the same syntax, so one reader serves them.
+func installedTLSEnforcement(name, installedPath string) (bool, error) {
+	values, err := readOptionFile(installedPath)
 	if err != nil && !os.IsNotExist(err) {
 		return false, fmt.Errorf("read the installed parameters: %w", err)
 	}
-	if bytes.Equal(current, lastGood) {
+	value, ok := values[name]
+	if !ok {
+		// The ordinary state of an instance whose resolved set predates the
+		// parameter, and the whole of the migration story: it begins enforcing at
+		// its next boot with no control-plane work at all.
+		return true, nil
+	}
+	switch value {
+	case "1":
+		return true, nil
+	case "0":
 		return false, nil
 	}
-	if err := e.writeEngineFile(e.parametersPath(), lastGood); err != nil {
-		return false, err
-	}
-	return true, nil
-}
-
-// Atomic: a temp file in the same directory, renamed over the target. The temp
-// name deliberately does not end in .conf, so a crash between write and rename
-// cannot leave the engine reading a half-written include.
-func (e *postgresEngine) writeEngineFile(path string, content []byte) error {
-	tmp := path + ".new"
-	if err := os.WriteFile(tmp, content, 0o600); err != nil {
-		return fmt.Errorf("write %s: %w", tmp, err)
-	}
-	if err := e.chownToEngine(tmp); err != nil {
-		os.Remove(tmp)
-		return err
-	}
-	if err := os.Rename(tmp, path); err != nil {
-		os.Remove(tmp)
-		return fmt.Errorf("install %s: %w", path, err)
-	}
-	return nil
-}
-
-// The engine's own parser, run offline against the datadir. Reading one setting
-// back is enough: postgres parses postgresql.conf and every include first, and
-// exits non-zero naming the file and line of an unknown parameter or a value
-// outside its range.
-func (e *postgresEngine) checkConfig(ctx context.Context) error {
-	if _, err := e.run(ctx, command{
-		Name: filepath.Join(filepath.Dir(e.psql), "postgres"),
-		Args: []string{"-D", e.pgData, "-C", "shared_buffers"},
-		Env:  []string{"PATH=" + defaultGuestPath},
-		User: e.osUser,
-	}); err != nil {
-		return err
-	}
-	return nil
-}
-
-// The engine reads its own config, so a root-owned file would be unreadable to
-// it. A guest with no such user is a broken image, not something to work around.
-func (e *postgresEngine) chownToEngine(path string) error {
-	credential, err := lookupCredential(e.osUser)
-	if err != nil {
-		return err
-	}
-	if err := os.Chown(path, int(credential.Uid), int(credential.Gid)); err != nil {
-		return fmt.Errorf("hand %s to %s: %w", path, e.osUser, err)
-	}
-	return nil
-}
-
-// Through the service manager rather than pg_ctl, so the supervisor records the
-// engine as stopped and does not restart it underneath a VM that is going down.
-func (e *postgresEngine) Stop(ctx context.Context) error {
-	return e.serviceAction(ctx, "stop")
-}
-
-// Only the parameter rollback calls this: a restart the control plane wants goes
-// through RebootDBInstance, which cycles the VM.
-func (e *postgresEngine) Restart(ctx context.Context) error {
-	return e.serviceAction(ctx, "restart")
-}
-
-func (e *postgresEngine) serviceAction(ctx context.Context, action string) error {
-	if _, err := e.run(ctx, command{
-		Name: e.rcService,
-		Args: []string{e.service, action},
-		Env:  []string{"PATH=" + defaultGuestPath},
-	}); err != nil {
-		return fmt.Errorf("%s the %s service: %w", action, e.service, err)
-	}
-	return nil
+	// The resolver canonicalises every boolean, so anything else means the file
+	// was written by something other than the platform. The permissive reading is
+	// the one that must not be chosen for a setting that turns TLS off.
+	return false, fmt.Errorf("the installed parameters set %s to %q, which is neither 1 nor 0", name, value)
 }
 
 // A minimal environment for a child that is not carrying secrets.
 const defaultGuestPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 
-// Feeds sql on stdin rather than as an argument, so a statement is never
-// visible in the process table.
-func (e *postgresEngine) psqlRun(ctx context.Context, sql string, env ...string) (string, error) {
-	return e.run(ctx, command{
-		Name:  e.psql,
-		Args:  e.psqlArgs(),
-		Env:   append([]string{"PATH=" + defaultGuestPath}, env...),
-		Stdin: sql,
-		User:  e.osUser,
-	})
-}
-
-// Reading the script from stdin is what lets one invocation serve both a
-// one-shot run and a session held open across several statements.
-func (e *postgresEngine) psqlArgs() []string {
-	return []string{
-		"--no-psqlrc", "--quiet", "--no-align", "--tuples-only",
-		"-v", "ON_ERROR_STOP=1",
-		"-h", e.socketDir,
-		"-p", strconv.FormatInt(e.port.Load(), 10),
-		"-U", e.osUser,
-		"-d", "postgres",
-		"-f", "-",
-	}
-}
-
 func redact(s, secret string) string {
 	if secret == "" {
 		return s

cmd/rds-agent/engine_contract_test.go

@@ -0,0 +1,333 @@
+package main
+
+//test:in-package — the agent is a main package, which has no external test
+// package to import it from, and these cases are built on the unexported
+// engine interface and each implementation's own constructor.
+
+import (
+	"context"
+	"errors"
+	"os"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// A probe the test drives directly, in place of whichever binaries the engine's
+// own state function would have run. Each implementation's state function is
+// covered on its own; what these cases exercise is what the engine does with the
+// answer.
+func drivenProbe(state *engineState) *engineProbe {
+	return newEngineProbe(0, func(context.Context, int64) (engineState, string) {
+		return *state, "driven by the test"
+	})
+}
+
+// One engine behind the interfaces the heartbeat, the commander and the rollback
+// guard hold, plus the few facts a shared case needs to say what it means.
+type engineHarness struct {
+	engine  engine
+	runner  *recordingRunner
+	params  parameterStore
+	service string
+	// The state the probe reports, moved by the test rather than by the engine.
+	state *engineState
+	// Points the next quiesce at a session the test can inspect, and returns the
+	// commands those sessions were opened with.
+	useSession func(*fakeSession) *[]command
+	// A setting this engine's catalog really carries, so a shared case installs
+	// something the implementation would genuinely be handed.
+	parameter handlers_rds.Parameter
+	// What the engine's own include directive globs. Neither the rollback copy
+	// nor the serving copy may end in it.
+	includeSuffix string
+}
+
+func engineHarnesses() []struct {
+	name  string
+	build func(*testing.T) *engineHarness
+} {
+	return []struct {
+		name  string
+		build func(*testing.T) *engineHarness
+	}{
+		{name: enginePostgres, build: buildPostgresHarness},
+		{name: engineMariaDB, build: buildMariaDBHarness},
+	}
+}
+
+func forEachEngine(t *testing.T, run func(*testing.T, *engineHarness)) {
+	t.Helper()
+	for _, fixture := range engineHarnesses() {
+		t.Run(fixture.name, func(t *testing.T) { run(t, fixture.build(t)) })
+	}
+}
+
+func buildPostgresHarness(t *testing.T) *engineHarness {
+	t.Helper()
+	runner := &recordingRunner{}
+	e := newTestEngine(t, runner.run)
+	state := engineServing
+	e.probe = drivenProbe(&state)
+
+	return &engineHarness{
+		engine: e, runner: runner, params: e.params, service: e.service, state: &state,
+		useSession: func(session *fakeSession) *[]command {
+			started := &[]command{}
+			e.startSess = func(_ context.Context, c command) (engineSession, error) {
+				*started = append(*started, c)
+				return session, nil
+			}
+			return started
+		},
+		parameter:     handlers_rds.Parameter{Name: "work_mem", Value: "4096"},
+		includeSuffix: ".conf",
+	}
+}
+
+func buildMariaDBHarness(t *testing.T) *engineHarness {
+	t.Helper()
+	runner := &recordingRunner{}
+	e := newTestMariaDBEngine(t, runner.run)
+	state := engineServing
+	e.probe = drivenProbe(&state)
+
+	return &engineHarness{
+		engine: e, runner: runner, params: e.params, service: e.service, state: &state,
+		useSession: func(session *fakeSession) *[]command {
+			started := &[]command{}
+			e.startSess = func(_ context.Context, c command) (engineSession, error) {
+				*started = append(*started, c)
+				return session, nil
+			}
+			return started
+		},
+		parameter:     handlers_rds.Parameter{Name: "max_connections", Value: "200"},
+		includeSuffix: ".cnf",
+	}
+}
+
+// The parameters land where rds-init installs them, so the next boot overwrites
+// the file rather than shadowing it with a second copy.
+func TestEngineContract_ApplyParametersInstallsWhereRdsInitDoes(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		if _, err := h.engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{h.parameter}); err != nil {
+			t.Fatalf("ApplyParameters: %v", err)
+		}
+
+		installed, err := os.ReadFile(h.params.installedPath())
+		if err != nil {
+			t.Fatalf("read the installed parameters: %v", err)
+		}
+		if !strings.Contains(string(installed), h.parameter.Name+" = '"+h.parameter.Value+"'") {
+			t.Errorf("installed parameters = %q, want the resolved value", installed)
+		}
+		if !strings.HasSuffix(h.params.installedPath(), h.includeSuffix) {
+			t.Errorf("the installed set at %s is not a file the engine includes", h.params.installedPath())
+		}
+		// Comparison material, not a second set of settings: an include glob that
+		// read either of these would apply a withdrawn configuration.
+		copies := []string{h.params.lastGoodPath()}
+		if h.params.serving != "" {
+			copies = append(copies, h.params.servingPath())
+		}
+		for _, path := range copies {
+			if strings.HasSuffix(path, h.includeSuffix) {
+				t.Errorf("the copy at %s would be included by the engine", path)
+			}
+		}
+	})
+}
+
+// An engine still starting or replaying has not finished adopting the set it is
+// on, so replacing that set underneath it would be installing over a moving
+// target.
+func TestEngineContract_ApplyParametersRefusesWhileTheEngineIsRecovering(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		*h.state = engineRecovering
+
+		if _, err := h.engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{h.parameter}); err == nil {
+			t.Fatal("ApplyParameters succeeded against an engine that was still coming up")
+		}
+		if _, err := os.Stat(h.params.installedPath()); !os.IsNotExist(err) {
+			t.Errorf("the refused set was installed anyway (stat err = %v)", err)
+		}
+	})
+}
+
+// The boot-time half of the parameter safety net: the last set the engine
+// accepted goes back only when the engine is genuinely down, and only once.
+func TestEngineContract_RestoreLastKnownGoodOnlyRollsBackADownEngine(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		accepted := h.parameter
+		if _, err := h.engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{accepted}); err != nil {
+			t.Fatalf("ApplyParameters: %v", err)
+		}
+		if err := h.engine.RecordServingParameters(t.Context()); err != nil {
+			t.Fatalf("RecordServingParameters: %v", err)
+		}
+		if _, err := h.engine.ApplyParameters(t.Context(),
+			[]handlers_rds.Parameter{{Name: accepted.Name, Value: accepted.Value + "0"}}); err != nil {
+			t.Fatalf("second ApplyParameters: %v", err)
+		}
+
+		// A guard whose deadline expired while an apply brought the engine back must
+		// not reverse that repair.
+		restored, err := h.engine.RestoreLastKnownGoodParameters(t.Context())
+		if err != nil {
+			t.Fatalf("RestoreLastKnownGoodParameters against a serving engine: %v", err)
+		}
+		if restored {
+			t.Fatal("rolled the parameters back while the engine was serving")
+		}
+
+		*h.state = engineAbsent
+		restored, err = h.engine.RestoreLastKnownGoodParameters(t.Context())
+		if err != nil {
+			t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
+		}
+		if !restored {
+			t.Fatal("did not roll back to a set that differs from the last accepted one")
+		}
+		installed, err := os.ReadFile(h.params.installedPath())
+		if err != nil {
+			t.Fatalf("read the installed parameters: %v", err)
+		}
+		if !strings.Contains(string(installed), accepted.Name+" = '"+accepted.Value+"'") {
+			t.Errorf("installed = %q, want the last accepted set", installed)
+		}
+
+		// Repeating a rollback would only churn a cluster that is failing for
+		// another reason.
+		restored, err = h.engine.RestoreLastKnownGoodParameters(t.Context())
+		if err != nil {
+			t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
+		}
+		if restored {
+			t.Error("rolled back again when the installed set already matched")
+		}
+	})
+}
+
+// Through the service manager rather than the engine's own control tool, so the
+// supervisor records the engine as stopped and does not restart it underneath a
+// VM that is going down.
+func TestEngineContract_StopGoesThroughTheServiceManager(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		if err := h.engine.Stop(t.Context()); err != nil {
+			t.Fatalf("Stop: %v", err)
+		}
+		if len(h.runner.calls) != 1 {
+			t.Fatalf("ran %d commands, want the service stop", len(h.runner.calls))
+		}
+		call := h.runner.calls[0]
+		if !slices.Equal(call.Args, []string{h.service, "stop"}) {
+			t.Errorf("stop ran %s %v, want the service manager", call.Name, call.Args)
+		}
+	})
+}
+
+// One backup at a time, released by Unquiesce or by its own deadline, whichever
+// comes first. The deadline is what keeps a control plane that dies mid-snapshot
+// from leaving the engine held.
+func TestEngineContract_QuiesceHoldsUntilReleased(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		session := &fakeSession{}
+		started := h.useSession(session)
+
+		if err := h.engine.Quiesce(t.Context(), "orders-db-pre-upgrade", time.Minute); err != nil {
+			t.Fatalf("Quiesce: %v", err)
+		}
+		if len(*started) != 1 {
+			t.Fatalf("started %d sessions, want 1", len(*started))
+		}
+		if session.closes() != 0 {
+			t.Fatal("the session was closed while the backup was meant to be held")
+		}
+
+		err := h.engine.Quiesce(t.Context(), "second", time.Minute)
+		if err == nil {
+			t.Fatal("a second quiesce was accepted while the first was held")
+		}
+		if !strings.Contains(err.Error(), "orders-db-pre-upgrade") {
+			t.Errorf("error %q does not name the backup already holding the engine", err)
+		}
+		if len(*started) != 1 {
+			t.Errorf("started %d sessions, want the second refused before opening one", len(*started))
+		}
+
+		if err := h.engine.Unquiesce(t.Context()); err != nil {
+			t.Fatalf("Unquiesce: %v", err)
+		}
+		if session.closes() != 1 {
+			t.Errorf("closed %d times, want the session ended exactly once", session.closes())
+		}
+	})
+}
+
+// A hold that expired mid-snapshot means the snapshot was not taken against a
+// held checkpoint, so the release reports it rather than succeeding silently.
+func TestEngineContract_QuiesceReleasesItselfOnItsDeadline(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		session := &fakeSession{}
+		h.useSession(session)
+
+		if err := h.engine.Quiesce(t.Context(), "orders-db-pre-upgrade", 10*time.Millisecond); err != nil {
+			t.Fatalf("Quiesce: %v", err)
+		}
+		deadline := time.Now().Add(2 * time.Second)
+		for session.closes() == 0 && time.Now().Before(deadline) {
+			time.Sleep(time.Millisecond)
+		}
+		if session.closes() != 1 {
+			t.Fatal("the backup session outlived its deadline")
+		}
+
+		err := h.engine.Unquiesce(t.Context())
+		if err == nil {
+			t.Fatal("Unquiesce succeeded after the hold expired")
+		}
+		if !strings.Contains(err.Error(), "expired") {
+			t.Errorf("error %q does not say the hold had expired", err)
+		}
+	})
+}
+
+// A failed start leaves nothing held, so a retry is not refused by a hold that
+// never took effect.
+func TestEngineContract_QuiesceEndsTheSessionWhenTheBackupWillNotStart(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		session := &fakeSession{execErr: errors.New("the engine refused the backup")}
+		h.useSession(session)
+
+		if err := h.engine.Quiesce(t.Context(), "orders-db-pre-upgrade", time.Minute); err == nil {
+			t.Fatal("Quiesce succeeded against an engine that refused the backup")
+		}
+		if session.closes() != 1 {
+			t.Errorf("closed %d times, want the failed session ended", session.closes())
+		}
+		if err := h.engine.Unquiesce(t.Context()); err == nil {
+			t.Error("a failed quiesce left a hold behind")
+		}
+	})
+}
+
+// Every command the control plane can issue is served by both implementations,
+// so a directive cannot be answered as unsupported on one engine and honoured on
+// the other.
+func TestEngineContract_ServesEveryRegisteredCommand(t *testing.T) {
+	forEachEngine(t, func(t *testing.T, h *engineHarness) {
+		registry := newCommandRegistry(h.engine, &fakeStorage{})
+		for _, name := range []string{
+			handlers_rds.CommandSetPassword, handlers_rds.CommandApplyParams, handlers_rds.CommandStopEngine,
+			handlers_rds.CommandGrowFilesystem, handlers_rds.CommandQuiesce, handlers_rds.CommandUnquiesce,
+		} {
+			if _, ok := registry[name]; !ok {
+				t.Errorf("command %q has no handler", name)
+			}
+		}
+	})
+}

cmd/rds-agent/engine_mariadb.go

@@ -0,0 +1,814 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The control plane's own statements about this engine, resolved once at
+// startup. Held as functions rather than as the Engine value so nothing here
+// re-derives a rule the API already owns.
+type controlPlaneRules struct {
+	// The whole master-username rule rather than only the reserved set. MariaDB's
+	// client has no identifier quoting, so the name is interpolated into SQL by
+	// this process and every part of the rule is load-bearing here.
+	validateUsername func(username string) error
+	// Whether a setting takes effect only at a restart. Already customer-facing
+	// and authoritative, since the API refuses ApplyMethod=immediate on one.
+	isStatic func(name string) bool
+	// The catalog name behind a spelling read back out of an option file. Only a
+	// setting whose startup spelling differs from the customer's moves.
+	catalogName func(optionFileName string) string
+	// The catalog's data type for a setting, which decides how its value is
+	// rendered into SQL. Empty for a name the catalog does not carry.
+	dataType func(name string) string
+	// The engine's own name for the setting that requires TLS of a client
+	// connection. Held rather than spelled out here, so the guest derives
+	// enforcement from the same key the control plane resolves it under.
+	tlsEnforcementParameter string
+}
+
+func controlPlaneRulesFrom(meta handlers_rds.Engine) controlPlaneRules {
+	// Built once per engine: everything read back out of a generated file has to
+	// return to the catalog's namespace before it is classified or reported, or a
+	// startup spelling would read as an unknown name.
+	catalogNames := map[string]string{}
+	for _, name := range meta.CatalogParameterNames() {
+		if optionFileName := meta.OptionFileName(name); optionFileName != name {
+			catalogNames[optionFileName] = name
+		}
+	}
+
+	return controlPlaneRules{
+		validateUsername: meta.ValidateMasterUsername,
+		isStatic: func(name string) bool {
+			spec, ok := meta.LookupParameter(name)
+			// A name the catalog does not carry cannot be shown to have been adopted
+			// without a restart, and is never issued as a live SET GLOBAL either.
+			return !ok || spec.ApplyType == handlers_rds.ApplyTypeStatic
+		},
+		catalogName: func(optionFileName string) string {
+			if name, ok := catalogNames[optionFileName]; ok {
+				return name
+			}
+			return optionFileName
+		},
+		dataType: func(name string) string {
+			spec, ok := meta.LookupParameter(name)
+			if !ok {
+				return ""
+			}
+			return spec.DataType
+		},
+		tlsEnforcementParameter: meta.TLSEnforcementParameter(),
+	}
+}
+
+// Reached over its unix socket as root, which the datadir's unix_socket plugin
+// authenticates from the connecting process' own uid. That is why the agent
+// holds no password of its own.
+type mariadbEngine struct {
+	quiesceState
+	parameterManager
+
+	rules     controlPlaneRules
+	run       commandRunner
+	startSess sessionRunner
+	client    string
+	socket    string
+	rcService string
+	service   string
+}
+
+var _ engine = (*mariadbEngine)(nil)
+
+const (
+	mariadbClientBinary = "mariadb"
+	mariadbAdminBinary  = "mariadb-admin"
+	// The account mariadb-install-db creates for the unix_socket plugin, which
+	// rds-init keeps for the platform and never hands to the customer.
+	mariadbSuperuser                  = "root"
+	mariadbProbeConnectTimeoutSeconds = 3
+	// How much of the engine's error log a probe reason carries. The server names
+	// what it refused on in its last few lines, and the reason reaches a customer
+	// through StatusInfos, so this quotes the tail rather than the whole file.
+	mariadbErrorLogTailBytes = 4096
+	mariadbErrorLogTailLines = 4
+	// How much of the probe client's stderr a reason carries, on the same grounds
+	// as the log tail above.
+	mariadbProbeStderrMaxBytes = 512
+	// Named by the platform drop-in rds-init writes, so both halves reach the
+	// same socket without either asserting it to the other.
+	mariadbSocketFile = "mysqld.sock"
+	// The customer reaches the instance over its customer ENI, so the master is
+	// created and rotated on the wildcard host rather than on localhost.
+	mariadbMasterHost = "%"
+)
+
+// The generated drop-ins on the data volume. The rollback and serving copies
+// deliberately do not end in .cnf: !includedir reads *.cnf, and neither is a
+// second set of settings.
+const (
+	mariadbParametersFile = "10-rds-parameters.cnf"
+	mariadbLastGoodFile   = "10-rds-parameters.last-good"
+	mariadbServingFile    = "10-rds-parameters.serving"
+	// MariaDB treats a setting before any group as a fatal parsing error, so
+	// every generated file carries the header rds-init writes.
+	mariadbParametersHeader = "[mysqld]\n"
+)
+
+// The layout's factory. The rules come from the control plane's own definition
+// of this engine, so a build whose control plane does not offer MariaDB refuses
+// to run this implementation rather than inventing a definition for it.
+func newMariaDBEngineFromCatalog(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+	meta, err := handlers_rds.LookupEngine(engineMariaDB)
+	if err != nil {
+		return nil, fmt.Errorf("this image bakes %s, which this build's control plane does not offer: %w", engineMariaDB, err)
+	}
+	return newMariaDBEngine(cfg, controlPlaneRulesFrom(meta), run, startSess, probe), nil
+}
+
+func newMariaDBEngine(cfg config, rules controlPlaneRules, run commandRunner, startSess sessionRunner, probe *engineProbe) *mariadbEngine {
+	return &mariadbEngine{
+		rules:     rules,
+		run:       run,
+		startSess: startSess,
+		client:    filepath.Join(cfg.EngineBinDir, mariadbClientBinary),
+		socket:    mariadbSocketPath(cfg),
+		rcService: cfg.RCService,
+		service:   cfg.EngineService,
+		parameterManager: parameterManager{
+			probe: probe,
+			params: parameterStore{
+				// The mount point rather than the datadir one level inside it: the
+				// include directory has to outlive the sweep a failed bootstrap runs.
+				dir:       filepath.Join(cfg.DataMount, "conf.d"),
+				installed: mariadbParametersFile,
+				lastGood:  mariadbLastGoodFile,
+				serving:   mariadbServingFile,
+				header:    mariadbParametersHeader,
+				osUser:    cfg.EngineUser,
+				engine:    engineMariaDB,
+			},
+			repairTimeout: parameterRepairTimeout,
+			repairPoll:    parameterRepairPoll,
+		},
+	}
+}
+
+func mariadbSocketPath(cfg config) string {
+	return filepath.Join(cfg.SocketDir, mariadbSocketFile)
+}
+
+// Every local connection, from the probe's ping to a held quiesce session.
+// --no-defaults so no option file can move it; --skip-ssl because the serving
+// cert names the endpoint rather than this socket, so a verifying client fails.
+func mariadbSocketConnectArgs(socket string) []string {
+	return []string{
+		"--no-defaults", "--protocol=socket", "--socket=" + socket,
+		"--user=" + mariadbSuperuser, "--skip-ssl",
+	}
+}
+
+// Reports whether a pid is a live process. EPERM is a process this agent may not
+// signal rather than one that is gone, which matters because a stale pidfile
+// read as "alive" is far less dangerous than a live engine read as "absent".
+type processLivenessFn func(pid int) bool
+
+func processAlive(pid int) bool {
+	if pid <= 0 {
+		return false
+	}
+	err := syscall.Kill(pid, 0)
+	return err == nil || errors.Is(err, syscall.EPERM)
+}
+
+func newMariaDBProbe(cfg config, run probeRunner) *engineProbe {
+	return newEngineProbe(cfg.EnginePort, mariadbProbeState(cfg, run, processAlive))
+}
+
+// Three stages: during InnoDB crash recovery mariadbd opens neither socket nor
+// port, so a ping cannot tell a recovering engine from an absent one — and
+// reading recovery as absent has the rollback guard restart a server mid-replay.
+func mariadbProbeState(cfg config, run probeRunner, alive processLivenessFn) probeStateFn {
+	pidFile, socket := cfg.EnginePidFile, mariadbSocketPath(cfg)
+	admin, client := filepath.Join(cfg.EngineBinDir, mariadbAdminBinary), filepath.Join(cfg.EngineBinDir, mariadbClientBinary)
+	// The probe alone bounds the connect: it must come back and report, where a
+	// statement the agent issues is allowed to take as long as the engine needs.
+	connect := append(mariadbSocketConnectArgs(socket),
+		fmt.Sprintf("--connect-timeout=%d", mariadbProbeConnectTimeoutSeconds))
+	probeTimeout := cfg.EngineProbeTimeout
+	if probeTimeout <= 0 {
+		probeTimeout = defaultEngineProbeTimeout
+	}
+	runBounded := func(ctx context.Context, name string, args ...string) (int, string, error) {
+		probeCtx, cancel := context.WithTimeout(ctx, probeTimeout)
+		defer cancel()
+		return run(probeCtx, name, args...)
+	}
+
+	return func(ctx context.Context, _ int64) (engineState, string) {
+		pid, err := readPidFile(pidFile)
+		switch {
+		case os.IsNotExist(err):
+			return engineAbsent, fmt.Sprintf("the engine has written no pidfile at %s", pidFile)
+		case err != nil:
+			return engineAbsent, fmt.Sprintf("the engine pidfile %s could not be read: %v", pidFile, err)
+		case !alive(pid):
+			return engineAbsent, fmt.Sprintf("the engine pidfile %s names pid %d, which is not running", pidFile, pid)
+		}
+
+		// The process is up from here on, so every remaining failure is an engine
+		// not serving yet rather than one that is gone. Reporting absent against a
+		// live server would have the rollback guard restart one making progress.
+		switch code, stderr, err := runBounded(ctx, admin, append(slices.Clone(connect), "ping")...); {
+		case err != nil:
+			return engineRecovering, fmt.Sprintf("engine probe could not run: %v", err)
+		case code != 0:
+			// A server that is genuinely recovering and one that refused the client
+			// are indistinguishable from the socket. The server states its side in
+			// its own log, and the client states its side on stderr.
+			return engineRecovering, withProbeStderr(withEngineLogTail(
+				"engine is not answering on its socket yet (startup or crash recovery)",
+				cfg.EngineErrorLog), stderr)
+		}
+
+		// ping answers even on ER_ACCESS_DENIED, so on its own it certifies a server
+		// that may be unable to execute anything. The statement rides argv rather
+		// than stdin: it carries nothing secret and a probe reads no result back.
+		query := append(slices.Clone(connect), "--batch", "--skip-column-names", "--execute=SELECT 1")
+		switch code, stderr, err := runBounded(ctx, client, query...); {
+		case err != nil:
+			return engineRecovering, fmt.Sprintf("engine probe could not run: %v", err)
+		case code != 0:
+			return engineRecovering, withProbeStderr(
+				"engine answered its socket but would not execute a statement", stderr)
+		}
+		return engineServing, ""
+	}
+}
+
+// Appends what the probe client said on stderr. A client that fails during the
+// handshake is refused by the server without the server ever logging why, so
+// this is the only account of that half of the exchange.
+func withProbeStderr(reason, stderr string) string {
+	stderr = strings.TrimSpace(stderr)
+	if stderr == "" {
+		return reason
+	}
+	return reason + "; the probe client reported: " + collapseProbeStderr(stderr)
+}
+
+// One line, bounded: the reason reaches a customer through StatusInfos, and a
+// client that repeats itself per attempt would otherwise crowd out the rest.
+func collapseProbeStderr(stderr string) string {
+	line := strings.Join(strings.Fields(strings.ReplaceAll(stderr, "\n", " ")), " ")
+	if len(line) > mariadbProbeStderrMaxBytes {
+		return line[:mariadbProbeStderrMaxBytes] + "..."
+	}
+	return line
+}
+
+// Appends the last few lines of the engine's error log to reason. An unset path,
+// an unreadable file or an empty one leaves reason alone: the probe's own
+// statement is still true, and a guessed cause would be worse than none.
+func withEngineLogTail(reason, path string) string {
+	tail := engineLogTail(path)
+	if tail == "" {
+		return reason
+	}
+	return reason + "; the engine's log ends: " + tail
+}
+
+func engineLogTail(path string) string {
+	if path == "" {
+		return ""
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return ""
+	}
+	defer func() { _ = f.Close() }()
+
+	info, err := f.Stat()
+	if err != nil {
+		return ""
+	}
+	offset := max(info.Size()-mariadbErrorLogTailBytes, 0)
+	buf := make([]byte, info.Size()-offset)
+	if _, err := f.ReadAt(buf, offset); err != nil {
+		return ""
+	}
+
+	lines := strings.Split(string(buf), "\n")
+	// A read that started mid-file almost certainly started mid-line, and half a
+	// line is more misleading than one fewer.
+	if offset > 0 && len(lines) > 0 {
+		lines = lines[1:]
+	}
+	kept := make([]string, 0, mariadbErrorLogTailLines)
+	for i := len(lines) - 1; i >= 0 && len(kept) < mariadbErrorLogTailLines; i-- {
+		if line := strings.TrimSpace(lines[i]); line != "" {
+			kept = append(kept, line)
+		}
+	}
+	slices.Reverse(kept)
+	return strings.Join(kept, " | ")
+}
+
+func readPidFile(path string) (int, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return 0, err
+	}
+	pid, err := strconv.Atoi(strings.TrimSpace(string(raw)))
+	if err != nil {
+		return 0, fmt.Errorf("%s does not hold a pid: %w", path, err)
+	}
+	return pid, nil
+}
+
+func (e *mariadbEngine) Stop(ctx context.Context) error {
+	return serviceAction(ctx, e.run, e.rcService, e.service, "stop")
+}
+
+// Only the parameter rollback and a failed apply call this: a restart the
+// control plane wants goes through RebootDBInstance, which cycles the VM. The
+// serving copy is refreshed first, since the server starts on what is installed.
+func (e *mariadbEngine) Restart(ctx context.Context) error {
+	if err := e.recordServingCopy(); err != nil {
+		return err
+	}
+	return serviceAction(ctx, e.run, e.rcService, e.service, "restart")
+}
+
+// The installed set copied to the name the pending-restart comparison reads. A
+// live SET GLOBAL apply deliberately does not touch it: that value was adopted
+// without a restart, which is what "not pending" means.
+func (e *mariadbEngine) recordServingCopy() error {
+	content, err := os.ReadFile(e.params.installedPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("read the installed parameters: %w", err)
+	}
+	return e.params.write(e.params.servingPath(), content)
+}
+
+// A value safe to interpolate into a single-quoted SQL literal. The quote is
+// doubled, which holds in every sql_mode, and the backslash is doubled, which is
+// why every statement built here pins a mode with backslash escapes on.
+func sqlLiteral(value string) string {
+	return strings.ReplaceAll(strings.ReplaceAll(value, `\`, `\\`), `'`, `''`)
+}
+
+// The spellings MariaDB parses for a boolean system variable, and the only ones
+// the catalog's own enum narrows a boolean to.
+var mariadbBooleanValues = map[string]string{
+	"on": "ON", "off": "OFF", "true": "ON", "false": "OFF", "1": "ON", "0": "OFF",
+}
+
+// One SET GLOBAL right-hand side. MariaDB refuses a quoted literal for a numeric
+// or boolean with ER_WRONG_TYPE_FOR_VAR, so the catalog's data type picks the
+// form; re-rendering from the parsed value is what keeps it safe to interpolate.
+func mariadbSetValue(dataType, value string) (string, error) {
+	switch dataType {
+	case handlers_rds.ParamTypeInteger:
+		n, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64)
+		if err != nil {
+			return "", fmt.Errorf("value %q is not an integer", value)
+		}
+		return strconv.FormatInt(n, 10), nil
+	case handlers_rds.ParamTypeReal:
+		f, err := strconv.ParseFloat(strings.TrimSpace(value), 64)
+		if err != nil {
+			return "", fmt.Errorf("value %q is not a real number", value)
+		}
+		return strconv.FormatFloat(f, 'g', -1, 64), nil
+	case handlers_rds.ParamTypeBoolean:
+		if literal, ok := mariadbBooleanValues[strings.ToLower(strings.TrimSpace(value))]; ok {
+			return literal, nil
+		}
+		return "", fmt.Errorf("value %q is not a boolean", value)
+	default:
+		return "'" + sqlLiteral(value) + "'", nil
+	}
+}
+
+// Turns off the two logs that would otherwise copy a statement's text, and pins
+// the mode the escaping above assumes. All three are session-scoped: a rotation
+// must not leave the customer's own general log switched off behind it.
+const mariadbSessionGuard = `SET SESSION sql_log_off = 1;
+SET SESSION slow_query_log = 0;
+SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';
+`
+
+// Rotates the master's password. The statement is built here rather than by the
+// client, which has neither psql's identifier interpolation nor its parameter
+// quoting, and it rides stdin so it is never visible in the process table.
+func (e *mariadbEngine) SetPassword(ctx context.Context, username, password string) error {
+	if username == "" || password == "" {
+		return fmt.Errorf("set-password requires both %s and %s",
+			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
+	}
+	// This runs as the server's own superuser, so a reserved name in the payload
+	// would rotate the platform's account. The whole rule matters here, not just
+	// the reserved set: the name is interpolated below rather than client-quoted.
+	if err := e.rules.validateUsername(username); err != nil {
+		return fmt.Errorf("refusing to set the password of role %q: %w", username, err)
+	}
+
+	escaped := sqlLiteral(password)
+	sql := fmt.Sprintf("%sALTER USER '%s'@'%s' IDENTIFIED BY '%s';\n",
+		mariadbSessionGuard, username, mariadbMasterHost, escaped)
+	if _, err := e.clientRun(ctx, sql); err != nil {
+		// The client echoes the failing statement, which carries the escaped form
+		// of the password rather than the raw one — so redacting only the raw one
+		// would leak exactly the passwords that needed escaping.
+		return fmt.Errorf("apply the master password: %s", redact(redact(err.Error(), password), escaped))
+	}
+	return nil
+}
+
+// Installs the resolved set and applies the half a running server can adopt.
+// MariaDB re-reads no file while up, so SET GLOBAL is the only immediate path.
+// It has no offline parser, so the safety net is the boot-time rollback.
+func (e *mariadbEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+
+	initialState, _ := e.probe.state(ctx)
+	if initialState == engineRecovering {
+		return nil, errors.New("apply parameters while the engine is still starting or recovering")
+	}
+	if initialState == engineServing {
+		// A command can arrive before the first heartbeat. Seed the configuration
+		// currently being served before replacing its file in that window.
+		if err := e.recordServingParametersLocked(ctx); err != nil {
+			return nil, fmt.Errorf("record the parameters serving before the apply: %w", err)
+		}
+	}
+
+	restore, err := e.params.install(params)
+	if err != nil {
+		return nil, err
+	}
+
+	state, _ := e.probe.state(ctx)
+	switch state {
+	case engineAbsent:
+		return e.restartOnRepairSetLocked(ctx)
+	case engineRecovering:
+		return nil, errors.Join(errors.New("the engine entered startup or recovery during the parameter apply"), restore())
+	}
+
+	// Before the set reaches the server: enforcement is one of the values below,
+	// and a server told to require a transport it cannot offer would refuse every
+	// client the statement after.
+	if err := e.requireServingTLSForEnforcement(ctx); err != nil {
+		return nil, errors.Join(err, restore())
+	}
+
+	if err := e.applyDynamicParameters(ctx, params); err != nil {
+		// The engine may have gone down after the first probe. In that case start it
+		// on the newly installed set rather than restoring one that already left it
+		// unable to serve.
+		if state, _ := e.probe.state(ctx); state == engineAbsent {
+			return e.restartOnRepairSetLocked(ctx)
+		}
+		return nil, errors.Join(err, restore())
+	}
+	// Last, and on its own: the batch above is not a transaction, so moving the
+	// TLS posture inside it would leave enforcement changed on a set the server
+	// went on to refuse, with only the parameter file rolled back.
+	if err := e.applyTLSEnforcement(ctx); err != nil {
+		if state, _ := e.probe.state(ctx); state == engineAbsent {
+			return e.restartOnRepairSetLocked(ctx)
+		}
+		return nil, errors.Join(err, restore())
+	}
+	return e.pendingRestartParameters(ctx)
+}
+
+// The dynamic half, in one invocation that stops at the first refusal so the
+// server names the setting it would not take. Only catalog names are emitted,
+// which is what makes the identifier safe to build into the statement.
+//
+// TLS enforcement is excluded: applyTLSEnforcement issues it after this batch
+// has succeeded.
+func (e *mariadbEngine) applyDynamicParameters(ctx context.Context, params []handlers_rds.Parameter) error {
+	var b strings.Builder
+	b.WriteString("SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';\n")
+	applied := 0
+	for _, p := range params {
+		if p.Name == "" || p.Name == e.rules.tlsEnforcementParameter || e.rules.isStatic(p.Name) {
+			continue
+		}
+		value, err := mariadbSetValue(e.rules.dataType(p.Name), p.Value)
+		if err != nil {
+			return fmt.Errorf("apply the dynamic parameters: %s: %w", p.Name, err)
+		}
+		fmt.Fprintf(&b, "SET GLOBAL %s = %s;\n", p.Name, value)
+		applied++
+	}
+	if applied == 0 {
+		return nil
+	}
+	if _, err := e.clientRun(ctx, b.String()); err != nil {
+		return fmt.Errorf("apply the dynamic parameters: %w", err)
+	}
+	return nil
+}
+
+// Puts the installed set's enforcement on the running server. The value comes
+// from the installed file rather than the set in hand, so a set that predates
+// the parameter enforces here rather than leaving the server on mariadbd's own
+// default of off while the API reports enforcement.
+func (e *mariadbEngine) applyTLSEnforcement(ctx context.Context) error {
+	enforce, err := installedTLSEnforcement(e.rules.tlsEnforcementParameter, e.params.installedPath())
+	if err != nil {
+		return err
+	}
+	previous, err := e.readTLSEnforcement(ctx)
+	if err != nil {
+		return err
+	}
+	if previous == enforce {
+		return nil
+	}
+	applyErr := e.setTLSEnforcement(ctx, enforce)
+	if applyErr == nil {
+		return nil
+	}
+	// The client can fail without saying whether the server ran the statement, so
+	// the posture it left behind is read rather than assumed unchanged: the one
+	// outcome that must not survive a failed apply is plaintext nobody asked for.
+	live, readErr := e.readTLSEnforcement(ctx)
+	if readErr != nil {
+		return errors.Join(applyErr, readErr)
+	}
+	if live == previous {
+		return applyErr
+	}
+	return errors.Join(applyErr, e.setTLSEnforcement(ctx, previous))
+}
+
+func (e *mariadbEngine) readTLSEnforcement(ctx context.Context) (bool, error) {
+	name := e.rules.tlsEnforcementParameter
+	out, err := e.clientRun(ctx, "SELECT @@global."+name+";\n")
+	if err != nil {
+		return false, fmt.Errorf("read whether the engine requires TLS of clients: %w", err)
+	}
+	switch value := strings.TrimSpace(out); value {
+	case "1":
+		return true, nil
+	case "0":
+		return false, nil
+	default:
+		return false, fmt.Errorf("the engine reports %s as %q, which is neither 1 nor 0", name, value)
+	}
+}
+
+func (e *mariadbEngine) setTLSEnforcement(ctx context.Context, enforce bool) error {
+	name, value := e.rules.tlsEnforcementParameter, "0"
+	if enforce {
+		value = "1"
+	}
+	// Rendered the way the batch renders every other boolean, so the statement the
+	// server sees does not depend on which path issued it.
+	literal, err := mariadbSetValue(handlers_rds.ParamTypeBoolean, value)
+	if err != nil {
+		return fmt.Errorf("render %s: %w", name, err)
+	}
+	sql := fmt.Sprintf("SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';\nSET GLOBAL %s = %s;\n", name, literal)
+	if _, err := e.clientRun(ctx, sql); err != nil {
+		return fmt.Errorf("set %s to %s: %w", name, literal, err)
+	}
+	return nil
+}
+
+// Refuses to require TLS of clients from a server that is not serving it, which
+// would reject every TCP connection while the parameter path reported success.
+// Read back from the installed file rather than taken from the set in hand, so
+// what is checked is also what the server reads at its next start.
+func (e *mariadbEngine) requireServingTLSForEnforcement(ctx context.Context) error {
+	enforce, err := installedTLSEnforcement(e.rules.tlsEnforcementParameter, e.params.installedPath())
+	if err != nil {
+		return err
+	}
+	if !enforce {
+		return nil
+	}
+	// DISABLED is a server built with TLS and started without a certificate,
+	// which is what a boot that was handed none leaves behind. The agent's own
+	// connections are unaffected either way: MariaDB counts a unix socket as a
+	// secure transport.
+	out, err := e.clientRun(ctx, "SELECT @@global.have_ssl;\n")
+	if err != nil {
+		return fmt.Errorf("read whether the engine is serving TLS: %w", err)
+	}
+	if serving := strings.TrimSpace(out); serving != "YES" {
+		return fmt.Errorf("the engine is serving without TLS (have_ssl = %q), so requiring it of clients would reject every connection", serving)
+	}
+	return nil
+}
+
+func (e *mariadbEngine) restartOnRepairSetLocked(ctx context.Context) ([]string, error) {
+	return awaitRepairedEngine(ctx, e.probe, e.Restart, e.pendingRestartParameters, e.repairTimeout, e.repairPoll)
+}
+
+// MariaDB has no pending_restart, so this is computed from the files: the
+// catalog-static keys whose installed value differs from the one the server
+// started on. Live values are not compared — mysqld silently rewrites them.
+func (e *mariadbEngine) pendingRestartParameters(context.Context) ([]string, error) {
+	installed, err := readOptionFile(e.params.installedPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			// Nothing is installed, so nothing is waiting to be adopted.
+			return nil, nil
+		}
+		return nil, fmt.Errorf("read the installed parameters: %w", err)
+	}
+	serving, err := readOptionFile(e.params.servingPath())
+	if err != nil && !os.IsNotExist(err) {
+		return nil, fmt.Errorf("read the parameters the engine started on: %w", err)
+	}
+	// Back to the catalog's names before anything is classified or reported: the
+	// files are written in the engine's startup spellings, and the answer is a
+	// customer-facing list the API refuses ApplyMethod=immediate against.
+	installed = e.catalogKeyed(installed)
+	serving = e.catalogKeyed(serving)
+
+	// An absent serving copy is a set the engine has not started on: rds-init
+	// writes the two together on every boot, so the whole static half counts as
+	// pending rather than being promoted on the strength of a missing file.
+	var pending []string
+	for name, value := range installed {
+		if !e.rules.isStatic(name) {
+			continue
+		}
+		if served, ok := serving[name]; !ok || served != value {
+			pending = append(pending, name)
+		}
+	}
+	// A static setting the group stopped naming reverts to its default at the next
+	// start, which the running server has not adopted either.
+	for name := range serving {
+		if _, ok := installed[name]; !ok && e.rules.isStatic(name) {
+			pending = append(pending, name)
+		}
+	}
+	slices.Sort(pending)
+	return pending, nil
+}
+
+// Snapshots the installed set as the rollback target, but only when nothing in
+// it is still waiting for a restart. The parameter mutex keeps an apply from
+// replacing the file between that check and the copy.
+func (e *mariadbEngine) RecordServingParameters(ctx context.Context) error {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return e.recordServingParametersLocked(ctx)
+}
+
+func (e *mariadbEngine) recordServingParametersLocked(ctx context.Context) error {
+	return recordLastGood(ctx, e.params, e.pendingRestartParameters)
+}
+
+// Puts the last set the engine accepted back in place, for a start that failed
+// after a parameter change.
+func (e *mariadbEngine) RestoreLastKnownGoodParameters(ctx context.Context) (bool, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return restoreLastGood(ctx, e.params, e.probe)
+}
+
+// The same settings under the names the catalog and the API know them by, since
+// a file is written in the engine's startup spellings and those are not always
+// the customer's.
+func (e *mariadbEngine) catalogKeyed(values map[string]string) map[string]string {
+	out := make(map[string]string, len(values))
+	for name, value := range values {
+		out[e.rules.catalogName(name)] = value
+	}
+	return out
+}
+
+// The subset of MariaDB's option-file syntax the generated drop-ins are written
+// in: a group header, comments, and one `name = value` per line. Nothing else
+// reaches these two files, because only rds-init and this agent write them.
+func readOptionFile(path string) (map[string]string, error) {
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	values := make(map[string]string)
+	for line := range strings.SplitSeq(string(raw), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") || strings.HasPrefix(line, "[") {
+			continue
+		}
+		// A bare option such as skip-log-bin carries no value; it is still a setting
+		// whose presence or absence differs.
+		name, value, _ := strings.Cut(line, "=")
+		values[normaliseOptionName(name)] = strings.Trim(strings.TrimSpace(value), `"'`)
+	}
+	return values, nil
+}
+
+// MariaDB reads - and _ as the same character in an option name, so the two
+// spellings of one setting must not compare as two settings.
+func normaliseOptionName(name string) string {
+	return strings.ReplaceAll(strings.TrimSpace(name), "-", "_")
+}
+
+// How long any one BACKUP STAGE waits for its metadata locks. Well under the
+// control plane's quiesce timeout, so a stage that cannot take its lock is
+// reported rather than left queued in front of live traffic.
+const mariadbQuiesceLockWait = 20 * time.Second
+
+// MariaDB's own backup API rather than FLUSH TABLES WITH READ LOCK: FTWRL makes
+// the whole database read-only for the full hold, and its acquisition phase is
+// unbounded. BACKUP STAGE blocks commits instead, and covers Aria and MyISAM.
+func (e *mariadbEngine) Quiesce(ctx context.Context, label string, hold time.Duration) error {
+	if err := validateQuiesceRequest(label, hold); err != nil {
+		return err
+	}
+
+	// Held across the whole start, so a second quiesce waits and then finds the
+	// first one's hold rather than opening a concurrent backup alongside it.
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.held != nil {
+		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)
+	}
+
+	// Deliberately not the command's context: the session has to outlive the call
+	// that started it, and its own deadline is what bounds it instead.
+	session, err := e.startSess(context.WithoutCancel(ctx), command{
+		Name:              e.client,
+		Args:              e.clientArgs(),
+		Env:               []string{"PATH=" + defaultGuestPath},
+		SentinelStatement: "SELECT '" + sessionSentinel + "';\n",
+	})
+	if err != nil {
+		return fmt.Errorf("open a backup session: %w", err)
+	}
+
+	// In order and on one connection: MariaDB releases the hold with the session
+	// that took it, which bounds a control plane that dies mid-snapshot. Fed one
+	// at a time so a stage that cannot take its locks is named.
+	stages := []string{
+		fmt.Sprintf("SET SESSION lock_wait_timeout = %d;\n", int(mariadbQuiesceLockWait.Seconds())),
+		"BACKUP STAGE START;\n",
+		"BACKUP STAGE FLUSH;\n",
+		"BACKUP STAGE BLOCK_DDL;\n",
+		"BACKUP STAGE BLOCK_COMMIT;\n",
+	}
+	for _, sql := range stages {
+		if err := session.Exec(ctx, sql); err != nil {
+			if closeErr := session.Close(); closeErr != nil {
+				slog.Warn("rds-agent: closing a failed backup session", "err", closeErr)
+			}
+			return fmt.Errorf("put the engine into backup mode at %q: %w", strings.TrimSpace(sql), err)
+		}
+	}
+
+	e.beginHoldLocked(label, session, hold)
+	return nil
+}
+
+func (e *mariadbEngine) Unquiesce(ctx context.Context) error {
+	return e.releaseHold(ctx, "BACKUP STAGE END;\n")
+}
+
+// Feeds sql on stdin rather than as an argument, so a statement is never visible
+// in the process table.
+func (e *mariadbEngine) clientRun(ctx context.Context, sql string) (string, error) {
+	return e.run(ctx, command{
+		Name:  e.client,
+		Args:  e.clientArgs(),
+		Env:   []string{"PATH=" + defaultGuestPath},
+		Stdin: sql,
+	})
+}
+
+// --unbuffered so a held session's output reaches the reader as each statement
+// finishes rather than when a pipe buffer fills. No User on the command: the
+// connection args carry it, and the agent runs as the account they name.
+func (e *mariadbEngine) clientArgs() []string {
+	return append(mariadbSocketConnectArgs(e.socket),
+		"--batch", "--skip-column-names", "--unbuffered")
+}

cmd/rds-agent/engine_mariadb_log_test.go

@@ -0,0 +1,211 @@
+package main
+
+//test:in-package — the agent is a main package, which has no external test
+// package to import it from, and this covers the unexported engineLogTail and
+// probe-reason helpers directly.
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+)
+
+const mariadbRDSInitScript = "../../scripts/images/rds-mariadb/rds-init"
+
+func writeEngineLog(t *testing.T, body string) string {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "error.log")
+	if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
+		t.Fatalf("write engine log: %v", err)
+	}
+	return path
+}
+
+// The whole point of the log: "not answering on its socket" is the same message
+// for a server still recovering and for one that refused to start, and only the
+// engine's own log separates them.
+func TestMariaDBProbe_QuotesTheEngineLogWhenTheSocketIsSilent(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	cfg.EngineErrorLog = writeEngineLog(t, strings.Join([]string{
+		"2026-08-12  1:49:26 0 [Note] Starting MariaDB 11.8.0",
+		"2026-08-12  1:49:26 0 [ERROR] InnoDB: Cannot allocate memory for the buffer pool",
+		"2026-08-12  1:49:26 0 [ERROR] Plugin 'InnoDB' registration as a STORAGE ENGINE failed.",
+		"",
+	}, "\n"))
+	writePid(t, cfg.EnginePidFile, os.Getpid())
+
+	state, message := mariadbProbeState(cfg,
+		func(context.Context, string, ...string) (int, string, error) { return 1, "", nil },
+		func(int) bool { return true })(t.Context(), int64(cfg.EnginePort))
+
+	if state != engineRecovering {
+		t.Errorf("state = %v, want recovering", state)
+	}
+	if !strings.Contains(message, "Cannot allocate memory for the buffer pool") {
+		t.Errorf("message = %q, want the engine's own refusal quoted", message)
+	}
+}
+
+func TestMariaDBProbe_SurvivesAnAbsentEngineLog(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	cfg.EngineErrorLog = filepath.Join(t.TempDir(), "does-not-exist.log")
+	writePid(t, cfg.EnginePidFile, os.Getpid())
+
+	state, message := mariadbProbeState(cfg,
+		func(context.Context, string, ...string) (int, string, error) { return 1, "", nil },
+		func(int) bool { return true })(t.Context(), int64(cfg.EnginePort))
+
+	if state != engineRecovering {
+		t.Errorf("state = %v, want recovering", state)
+	}
+	if message != "engine is not answering on its socket yet (startup or crash recovery)" {
+		t.Errorf("message = %q, want the bare reason when there is no log to quote", message)
+	}
+}
+
+// A client refused during the handshake is closed by the server without the
+// server ever logging a cause, so the engine log alone leaves the failure
+// unexplained and only the client's own stderr names it.
+func TestMariaDBProbe_QuotesTheProbeClientWhenItIsRefused(t *testing.T) {
+	cfg := mariadbTestProbeConfig(t)
+	cfg.EngineErrorLog = writeEngineLog(t, "2026-08-12  4:32:39 148 [Warning] Aborted connection 148\n")
+	writePid(t, cfg.EnginePidFile, os.Getpid())
+
+	state, message := mariadbProbeState(cfg,
+		func(context.Context, string, ...string) (int, string, error) {
+			return 1, "ERROR 2026 (HY000): TLS/SSL error: self-signed certificate\n", nil
+		},
+		func(int) bool { return true })(t.Context(), int64(cfg.EnginePort))
+
+	if state != engineRecovering {
+		t.Errorf("state = %v, want recovering", state)
+	}
+	if !strings.Contains(message, "self-signed certificate") {
+		t.Errorf("message = %q, want the probe client's own error quoted", message)
+	}
+	if !strings.Contains(message, "Aborted connection") {
+		t.Errorf("message = %q, want the engine log kept alongside the client's error", message)
+	}
+}
+
+func TestCollapseProbeStderr(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{name: "folds to one line", in: "ERROR 2026:\n  TLS error\n", want: "ERROR 2026: TLS error"},
+		{name: "empty stays empty", in: "   \n", want: ""},
+		{
+			name: "bounded",
+			in:   strings.Repeat("y", mariadbProbeStderrMaxBytes+50),
+			want: strings.Repeat("y", mariadbProbeStderrMaxBytes) + "...",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := collapseProbeStderr(tt.in); got != tt.want {
+				t.Errorf("collapseProbeStderr() = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+// A probe that says nothing must not append an empty clause: the reason reaches
+// a customer, and a dangling "reported:" would read as truncated output.
+func TestWithProbeStderrLeavesReasonAloneWhenTheClientIsSilent(t *testing.T) {
+	const reason = "engine is not answering on its socket yet"
+	if got := withProbeStderr(reason, "  \n "); got != reason {
+		t.Errorf("withProbeStderr() = %q, want the reason unchanged", got)
+	}
+}
+
+func TestEngineLogTail(t *testing.T) {
+	tests := []struct {
+		name        string
+		body        string
+		want        string
+		wantMissing string
+	}{
+		{
+			name: "keeps only the last lines",
+			body: "one\ntwo\nthree\nfour\nfive\nsix\n",
+			want: "three | four | five | six",
+		},
+		{
+			name: "skips blank lines",
+			body: "alpha\n\n\nbeta\n\n",
+			want: "alpha | beta",
+		},
+		{
+			name: "empty file yields nothing",
+			body: "",
+			want: "",
+		},
+		{
+			name:        "drops a line the byte cap cut in half",
+			body:        strings.Repeat("x", mariadbErrorLogTailBytes) + "\nlast\n",
+			want:        "last",
+			wantMissing: "x",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := engineLogTail(writeEngineLog(t, tt.body))
+			if got != tt.want {
+				t.Errorf("engineLogTail() = %q, want %q", got, tt.want)
+			}
+			if tt.wantMissing != "" && strings.Contains(got, tt.wantMissing) {
+				t.Errorf("engineLogTail() = %q, want no partial line", got)
+			}
+		})
+	}
+}
+
+func TestEngineLogTailUnsetPath(t *testing.T) {
+	if got := engineLogTail(""); got != "" {
+		t.Errorf("engineLogTail(\"\") = %q, want empty", got)
+	}
+}
+
+// The agent reads a path the image writes, and nothing at runtime would notice
+// the two drifting apart: the log would simply always be missing and every
+// refusal would go back unexplained again.
+func TestMariaDBErrorLogPathMatchesTheImage(t *testing.T) {
+	raw, err := os.ReadFile(mariadbRDSInitScript)
+	if err != nil {
+		t.Fatalf("read rds-init: %v", err)
+	}
+	script := string(raw)
+
+	logDir := shellDefault(t, script, "LOG_DIR", "RDS_LOG_DIR")
+	engineLog := shellDefault(t, script, "ENGINE_LOG", "RDS_ENGINE_LOG")
+	want := strings.Replace(engineLog, "${LOG_DIR}", logDir, 1)
+
+	if got := engineLayouts[engineMariaDB].errorLog; got != want {
+		t.Errorf("layout errorLog = %q, but rds-init writes log_error = %q", got, want)
+	}
+}
+
+// Pulls the fallback out of a `NAME="${OVERRIDE:-fallback}"` assignment.
+func shellDefault(t *testing.T, script, name, override string) string {
+	t.Helper()
+	re := regexp.MustCompile(`(?m)^` + name + `="\$\{` + override + `:-(.*)\}"$`)
+	m := re.FindStringSubmatch(script)
+	if m == nil {
+		t.Fatalf("rds-init has no %s assignment defaulting from %s", name, override)
+	}
+	return m[1]
+}
+
+func TestWithEngineLogTailLeavesReasonAloneWithoutALog(t *testing.T) {
+	const reason = "engine is not answering on its socket yet"
+	if got := withEngineLogTail(reason, ""); got != reason {
+		t.Errorf("withEngineLogTail() = %q, want the reason unchanged", got)
+	}
+}

cmd/rds-agent/engine_postgres.go

@@ -0,0 +1,548 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The engine is reached over its unix socket under peer authentication, so the
+// agent drops to the postgres OS user rather than holding a password of its own.
+type postgresEngine struct {
+	quiesceState
+	parameterManager
+
+	// The control plane's own metadata for this engine, resolved once at
+	// startup: the live password apply runs as the cluster superuser, so it
+	// re-checks the role name against the same reserved set the API validates.
+	meta      handlers_rds.Engine
+	run       commandRunner
+	startSess sessionRunner
+	psql      string
+	rcService string
+	service   string
+	pgData    string
+	socketDir string
+	osUser    string
+}
+
+var _ engine = (*postgresEngine)(nil)
+
+// pg_isready gives PostgreSQL three states for free: exit 0 is serving, exit 1
+// is a postmaster up and rejecting connections, anything else is an engine that
+// is not there. Resolved on PATH, where the client package puts it.
+const postgresProbeBinary = "pg_isready"
+
+// The socket directory rather than loopback: the engine binds the customer ENI
+// and nothing else, so there is no TCP path for the probe to take. It also keeps
+// liveness independent of the network path, of the pg_hba scope and of the TLS
+// enforcement rule — libpq reads a host beginning with / as a socket directory,
+// and the generated `local ... peer` line covers it.
+func newPostgresProbe(cfg config, run probeRunner) *engineProbe {
+	return newEngineProbe(cfg.EnginePort, postgresProbeState(cfg.SocketDir, run))
+}
+
+func postgresProbeState(host string, run probeRunner) probeStateFn {
+	return func(ctx context.Context, port int64) (engineState, string) {
+		portArg := strconv.FormatInt(port, 10)
+		code, _, err := run(ctx, postgresProbeBinary, "-h", host, "-p", portArg, "-q")
+		switch {
+		case err != nil:
+			// A missing binary or broken image. Reporting healthy on the strength of
+			// nothing would hide it, so this reads as absent like an engine that did
+			// not answer.
+			return engineAbsent, fmt.Sprintf("engine probe could not run: %v", err)
+		case code == 0:
+			return engineServing, ""
+		case code == 1:
+			return engineRecovering, "engine is rejecting connections (startup or recovery)"
+		default:
+			return engineAbsent, fmt.Sprintf("engine did not respond on %s:%s", host, portArg)
+		}
+	}
+}
+
+// The include the resolved set is rendered to, and the copy of the last one the
+// engine accepted. Both live beside the data rather than in /etc, matching
+// rds-init: a class change boots a fresh root volume and would revert them.
+const (
+	postgresParametersFile = "10-rds-parameters.conf"
+	// Deliberately not a .conf name: include_dir globs *.conf, so the rollback
+	// copy must not be read as a second set of settings.
+	postgresLastGoodFile = "10-rds-parameters.last-good"
+)
+
+// The pg_hba rule that implements TLS enforcement, and the directory the
+// generated pg_hba includes it from. Its content is a constant, so the copy
+// rds-init writes in shell and the one written here have no template to drift
+// apart on: enforcement is the presence of the file and nothing more.
+const (
+	postgresHBADir           = "hba.d"
+	postgresForceSSLRuleFile = "20-rds-force-ssl.conf"
+	// An explicit reject rather than an omitted permit: both refuse the
+	// connection, and only this one puts the reason in the server log and in the
+	// client's error.
+	postgresForceSSLRules = `# Managed by the platform: the rule that requires TLS of client connections.
+# Written by rds-init at boot and by rds-agent on a parameter apply.
+hostnossl all all 0.0.0.0/0 reject
+hostnossl all all ::/0 reject
+`
+	// What the read-back must count once the file is in place. Asserting the shape
+	// rather than only the absence of parse errors is what catches the failure
+	// that will actually happen: the derivation above writing the wrong thing.
+	postgresForceSSLRuleCount = 2
+)
+
+// The layout's factory resolves the control plane metadata during startup.
+func newPostgresEngineFromCatalog(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+	meta, err := handlers_rds.LookupEngine(enginePostgres)
+	if err != nil {
+		return nil, fmt.Errorf("this image bakes %s, which this build's control plane does not offer: %w", enginePostgres, err)
+	}
+	return newPostgresEngine(cfg, meta, run, startSess, probe), nil
+}
+
+func newPostgresEngine(cfg config, meta handlers_rds.Engine, run commandRunner, startSess sessionRunner, probe *engineProbe) *postgresEngine {
+	return &postgresEngine{
+		meta:      meta,
+		run:       run,
+		startSess: startSess,
+		psql:      filepath.Join(cfg.EngineBinDir, "psql"),
+		rcService: cfg.RCService,
+		service:   cfg.EngineService,
+		pgData:    cfg.EngineDataDir,
+		socketDir: cfg.SocketDir,
+		osUser:    cfg.EngineUser,
+		parameterManager: parameterManager{
+			probe: probe,
+			params: parameterStore{
+				dir:       filepath.Join(cfg.EngineDataDir, "conf.d"),
+				installed: postgresParametersFile,
+				lastGood:  postgresLastGoodFile,
+				osUser:    cfg.EngineUser,
+				engine:    enginePostgres,
+			},
+			repairTimeout: parameterRepairTimeout,
+			repairPoll:    parameterRepairPoll,
+		},
+	}
+}
+
+// The role name and the password ride the environment and are re-quoted by
+// psql, so neither reaches a shell word or an argv another process can read.
+func (e *postgresEngine) SetPassword(ctx context.Context, username, password string) error {
+	if username == "" || password == "" {
+		return fmt.Errorf("set-password requires both %s and %s",
+			handlers_rds.CommandParamMasterUsername, handlers_rds.CommandParamMasterUserPassword)
+	}
+	// The statement below runs as the cluster superuser over the socket under
+	// peer auth, so a reserved name in the command payload would hand the
+	// customer the bootstrap superuser rather than rotate their own role.
+	if err := e.meta.ValidateUsernameNotReserved(username); err != nil {
+		return fmt.Errorf("refusing to set the password of a role the engine reserves: %w", err)
+	}
+	// psql interpolates the password into the ALTER ROLE before the server sees
+	// it, and these three are what would write it to the log — the last on any
+	// failure at its own default. SUSET, so the parameter group cannot win.
+	const sql = `SET log_statement = 'none';
+SET log_min_duration_statement = -1;
+SET log_min_error_statement = 'panic';
+\getenv master RDS_MASTER_USERNAME
+\getenv password RDS_MASTER_PASSWORD
+ALTER ROLE :"master" WITH LOGIN PASSWORD :'password';
+`
+	_, err := e.psqlRun(ctx, sql,
+		"RDS_MASTER_USERNAME="+username,
+		"RDS_MASTER_PASSWORD="+password,
+	)
+	if err != nil {
+		// psql echoes the failing statement, which here would carry the new
+		// password back to the control plane and into the event ring.
+		return fmt.Errorf("apply the master password: %s", redact(err.Error(), password))
+	}
+	return nil
+}
+
+// Installs the resolved set, validates it with the engine's own parser before
+// the engine adopts it, and reloads. A refused value is rolled back here rather
+// than left on the data volume, where it would survive every VM replace.
+func (e *postgresEngine) ApplyParameters(ctx context.Context, params []handlers_rds.Parameter) ([]string, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+
+	initialState, _ := e.probe.state(ctx)
+	if initialState == engineRecovering {
+		return nil, errors.New("apply parameters while the engine is still starting or recovering")
+	}
+	if initialState == engineServing {
+		// A command can arrive before the first heartbeat. Seed the configuration
+		// currently being served before replacing its file in that window.
+		if err := e.recordServingParametersLocked(ctx); err != nil {
+			return nil, fmt.Errorf("record the parameters serving before the apply: %w", err)
+		}
+	}
+
+	// The check runs against the file in place, because the engine parses the
+	// datadir's own include_dir. The window that leaves is closed by the rollback
+	// below and the last-known-good restore the agent runs at boot.
+	restoreParameters, err := e.params.install(params)
+	if err != nil {
+		return nil, err
+	}
+	if err := e.checkConfig(ctx); err != nil {
+		return nil, errors.Join(fmt.Errorf("the engine rejected the parameter set: %w", err), restoreParameters())
+	}
+
+	state, _ := e.probe.state(ctx)
+	switch state {
+	case engineAbsent:
+		return e.restartOnRepairSetLocked(ctx)
+	case engineRecovering:
+		return nil, errors.Join(errors.New("the engine entered startup or recovery during the parameter apply"), restoreParameters())
+	}
+
+	// After the engine's own parser has passed the set and before the reload that
+	// adopts both files at once.
+	enforce, restoreEnforcement, err := e.syncTLSEnforcement(ctx, state)
+	if err != nil {
+		return nil, errors.Join(fmt.Errorf("apply the TLS enforcement of the parameter set: %w", err), restoreParameters())
+	}
+	// The two files carry one apply from here: a rejected set withdrawn without
+	// its enforcement rule would leave the instance serving the old parameters
+	// under the new set's TLS posture.
+	restore := func() error { return errors.Join(restoreEnforcement(), restoreParameters()) }
+
+	if err := e.reload(ctx); err != nil {
+		// The engine may have gone down after the first probe. In that case start
+		// it on the new, parser-checked repair set rather than restoring the set
+		// that already left it unable to serve.
+		if state, _ := e.probe.state(ctx); state == engineAbsent {
+			return e.restartOnRepairSetLocked(ctx)
+		}
+		return nil, errors.Join(err, restore())
+	}
+	if err := e.verifyTLSEnforcement(ctx, enforce); err != nil {
+		// The reload already happened, so putting the files back is not enough: a
+		// pg_hba that parsed but says the wrong thing has been adopted, and only a
+		// second reload takes the engine off it.
+		return nil, errors.Join(err, restore(), e.reload(ctx))
+	}
+	return e.pendingRestartParameters(ctx)
+}
+
+// Re-derived before the restart: the engine reads pg_hba when it starts, so a
+// repair set started under the previous set's enforcement would serve one TLS
+// posture while the API reported the other.
+func (e *postgresEngine) restartOnRepairSetLocked(ctx context.Context) ([]string, error) {
+	if _, _, err := e.syncTLSEnforcement(ctx, engineAbsent); err != nil {
+		return nil, fmt.Errorf("apply the TLS enforcement of the repair set: %w", err)
+	}
+	return awaitRepairedEngine(ctx, e.probe, e.Restart, e.pendingRestartParameters, e.repairTimeout, e.repairPoll)
+}
+
+// The function's own answer is the reload, not psql's exit status: it returns f
+// when it could not signal the postmaster, and the read-back that follows parses
+// pg_hba from disk, so an unsignalled reload would otherwise verify clean.
+func (e *postgresEngine) reload(ctx context.Context) error {
+	out, err := e.psqlRun(ctx, "SELECT pg_reload_conf();\n")
+	if err != nil {
+		return fmt.Errorf("reload the engine configuration: %w", err)
+	}
+	if result := strings.TrimSpace(out); result != "t" {
+		return fmt.Errorf("reload the engine configuration: pg_reload_conf() returned %q", result)
+	}
+	return nil
+}
+
+// The postmaster's own answer: a static setting it has stored but not adopted is
+// what it reports here, so nothing in the guest has to classify one.
+func (e *postgresEngine) pendingRestartParameters(ctx context.Context) ([]string, error) {
+	out, err := e.psqlRun(ctx, "SELECT name FROM pg_settings WHERE pending_restart ORDER BY name;\n")
+	if err != nil {
+		return nil, fmt.Errorf("read the settings pending a restart: %w", err)
+	}
+	var pending []string
+	for line := range strings.SplitSeq(out, "\n") {
+		if name := strings.TrimSpace(line); name != "" {
+			pending = append(pending, name)
+		}
+	}
+	return pending, nil
+}
+
+// Snapshots the include only when the running postmaster has no settings still
+// pending a restart. The parameter mutex keeps an apply from replacing the file
+// between that check and the copy.
+func (e *postgresEngine) RecordServingParameters(ctx context.Context) error {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	return e.recordServingParametersLocked(ctx)
+}
+
+func (e *postgresEngine) recordServingParametersLocked(ctx context.Context) error {
+	return recordLastGood(ctx, e.params, e.pendingRestartParameters)
+}
+
+// Puts the last set the engine accepted back in place, for a restart that failed
+// after a parameter change.
+func (e *postgresEngine) RestoreLastKnownGoodParameters(ctx context.Context) (bool, error) {
+	e.paramMu.Lock()
+	defer e.paramMu.Unlock()
+	restored, err := restoreLastGood(ctx, e.params, e.probe)
+	if err != nil || !restored {
+		return restored, err
+	}
+	// A service restart does not re-run rds-init, so a restore that stopped at the
+	// parameter file would bring the last known good set up under the enforcement
+	// of the set that was just rejected. restoreLastGood only replaces the file of
+	// an engine that is down.
+	if _, _, err := e.syncTLSEnforcement(ctx, engineAbsent); err != nil {
+		return true, fmt.Errorf("apply the TLS enforcement of the restored parameter set: %w", err)
+	}
+	return true, nil
+}
+
+// Turns the installed enforcement value into the pg_hba fact that implements it.
+// PostgreSQL has no server setting for this, so the parameter file on its own
+// would report the change applied and enforce nothing.
+//
+// The serving-TLS guard runs only against an engine that is up. The restore and
+// repair paths reach this with the engine down, where the rule is read by the
+// start that follows rather than by a reload.
+func (e *postgresEngine) syncTLSEnforcement(ctx context.Context, state engineState) (enforce bool, restore func() error, err error) {
+	enforce, err = installedTLSEnforcement(e.meta.TLSEnforcementParameter(), e.params.installedPath())
+	if err != nil {
+		return false, nil, err
+	}
+	if enforce && state == engineServing {
+		if err := e.requireServingTLS(ctx); err != nil {
+			return false, nil, err
+		}
+	}
+	restore, err = e.writeTLSEnforcement(enforce)
+	if err != nil {
+		return false, nil, err
+	}
+	return enforce, restore, nil
+}
+
+// Writes or removes the enforcement rule, returning a func putting back whatever
+// was there. Removing matters as much as writing: a restored data volume carries
+// hba.d with it, so a stale rule would keep enforcing under a parameter group
+// that turns enforcement off.
+func (e *postgresEngine) writeTLSEnforcement(enforce bool) (func() error, error) {
+	path := e.forceSSLRulePath()
+	previous, readErr := os.ReadFile(path)
+	if readErr != nil && !os.IsNotExist(readErr) {
+		return nil, fmt.Errorf("read the TLS enforcement rule at %s: %w", path, readErr)
+	}
+	existed := readErr == nil
+	restore := func() error {
+		if !existed {
+			return e.removeTLSEnforcement()
+		}
+		return e.params.write(path, previous)
+	}
+
+	if !enforce {
+		if err := e.removeTLSEnforcement(); err != nil {
+			return nil, err
+		}
+		return restore, nil
+	}
+	// rds-init lays the directory down on every boot, so this covers an agent that
+	// reached an apply before one ran rather than the ordinary case.
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, 0o700); err != nil {
+		return nil, fmt.Errorf("create %s: %w", dir, err)
+	}
+	if err := e.params.chownToEngine(dir); err != nil {
+		return nil, err
+	}
+	if err := e.params.write(path, []byte(postgresForceSSLRules)); err != nil {
+		return nil, err
+	}
+	return restore, nil
+}
+
+func (e *postgresEngine) removeTLSEnforcement() error {
+	path := e.forceSSLRulePath()
+	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("withdraw the TLS enforcement rule at %s: %w", path, err)
+	}
+	return nil
+}
+
+func (e *postgresEngine) forceSSLRulePath() string {
+	return filepath.Join(e.pgData, postgresHBADir, postgresForceSSLRuleFile)
+}
+
+// Refuses to require TLS of clients from an engine that is not serving it, which
+// would reject every connection while the parameter path reported success.
+func (e *postgresEngine) requireServingTLS(ctx context.Context) error {
+	out, err := e.psqlRun(ctx, "SHOW ssl;\n")
+	if err != nil {
+		return fmt.Errorf("read whether the engine is serving TLS: %w", err)
+	}
+	if serving := strings.TrimSpace(out); serving != "on" {
+		return fmt.Errorf("the engine is serving without TLS (ssl = %q), so requiring it of clients would reject every connection", serving)
+	}
+	return nil
+}
+
+// PostgreSQL discards a pg_hba it cannot parse on reload and keeps the rules it
+// is already serving, which presents as an apply that succeeded and enforces
+// nothing. So the file is read back.
+//
+// pg_hba_file_rules re-parses from disk, so this proves the file would be
+// adopted rather than that the postmaster has adopted it. The inference holds
+// because a reloaded pg_hba is discarded only on a parse error.
+const postgresHBAReadBack = `SELECT count(*) FILTER (WHERE type = 'hostnossl' AND auth_method = 'reject'),
+       count(*) FILTER (WHERE error IS NOT NULL) FROM pg_hba_file_rules;
+`
+
+func (e *postgresEngine) verifyTLSEnforcement(ctx context.Context, enforce bool) error {
+	out, err := e.psqlRun(ctx, postgresHBAReadBack)
+	if err != nil {
+		return fmt.Errorf("read back the client authentication rules: %w", err)
+	}
+	rules, unparsable, err := parseHBAReadBack(out)
+	if err != nil {
+		return err
+	}
+	if unparsable > 0 {
+		return fmt.Errorf("the client authentication file carries %d rules the engine could not parse, so it would keep the rules it is already serving", unparsable)
+	}
+	want := 0
+	if enforce {
+		want = postgresForceSSLRuleCount
+	}
+	if rules != want {
+		return fmt.Errorf("the client authentication file carries %d TLS enforcement rules, want %d", rules, want)
+	}
+	return nil
+}
+
+func parseHBAReadBack(out string) (rules, unparsable int, err error) {
+	answer := strings.TrimSpace(out)
+	malformed := fmt.Errorf("the client authentication read-back returned %q, want two counts", answer)
+	counts := strings.Split(answer, "|")
+	if len(counts) != 2 {
+		return 0, 0, malformed
+	}
+	parsed := make([]int, len(counts))
+	for i, count := range counts {
+		if parsed[i], err = strconv.Atoi(strings.TrimSpace(count)); err != nil {
+			return 0, 0, malformed
+		}
+	}
+	return parsed[0], parsed[1], nil
+}
+
+// The engine's own parser, run offline against the datadir. Reading one setting
+// back is enough: postgres parses every include first and exits non-zero naming
+// the file and line of an unknown parameter or an out-of-range value.
+func (e *postgresEngine) checkConfig(ctx context.Context) error {
+	if _, err := e.run(ctx, command{
+		Name: filepath.Join(filepath.Dir(e.psql), "postgres"),
+		Args: []string{"-D", e.pgData, "-C", "shared_buffers"},
+		Env:  []string{"PATH=" + defaultGuestPath},
+		User: e.osUser,
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (e *postgresEngine) Stop(ctx context.Context) error {
+	return serviceAction(ctx, e.run, e.rcService, e.service, "stop")
+}
+
+// Only the parameter rollback calls this: a restart the control plane wants goes
+// through RebootDBInstance, which cycles the VM.
+func (e *postgresEngine) Restart(ctx context.Context) error {
+	return serviceAction(ctx, e.run, e.rcService, e.service, "restart")
+}
+
+// Feeds sql on stdin rather than as an argument, so a statement is never
+// visible in the process table.
+func (e *postgresEngine) psqlRun(ctx context.Context, sql string, env ...string) (string, error) {
+	return e.run(ctx, command{
+		Name:  e.psql,
+		Args:  e.psqlArgs(),
+		Env:   append([]string{"PATH=" + defaultGuestPath}, env...),
+		Stdin: sql,
+		User:  e.osUser,
+	})
+}
+
+// Reading the script from stdin is what lets one invocation serve both a
+// one-shot run and a session held open across several statements.
+func (e *postgresEngine) psqlArgs() []string {
+	return []string{
+		"--no-psqlrc", "--quiet", "--no-align", "--tuples-only",
+		"-v", "ON_ERROR_STOP=1",
+		"-h", e.socketDir,
+		"-p", strconv.FormatInt(e.probe.port.Load(), 10),
+		"-U", e.osUser,
+		"-d", "postgres",
+		"-f", "-",
+	}
+}
+
+// Puts the engine into backup mode: the datadir is checkpointed and the engine
+// stops writing over the pages a snapshot is about to read. The hold is released
+// by Unquiesce, or by its own deadline, whichever comes first.
+func (e *postgresEngine) Quiesce(ctx context.Context, label string, hold time.Duration) error {
+	if err := validateQuiesceRequest(label, hold); err != nil {
+		return err
+	}
+
+	// Held across the whole start, so a second quiesce waits and then finds the
+	// first one's hold rather than opening a concurrent backup alongside it.
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.held != nil {
+		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)
+	}
+
+	// Deliberately not the command's context: the session has to outlive the call
+	// that started it, and its own deadline is what bounds it instead.
+	session, err := e.startSess(context.WithoutCancel(ctx), command{
+		Name:              e.psql,
+		Args:              e.psqlArgs(),
+		Env:               []string{"PATH=" + defaultGuestPath, "RDS_BACKUP_LABEL=" + label},
+		User:              e.osUser,
+		SentinelStatement: `\echo ` + sessionSentinel + "\n",
+	})
+	if err != nil {
+		return fmt.Errorf("open a backup session: %w", err)
+	}
+
+	// fast forces an immediate checkpoint rather than spreading it over the
+	// checkpoint interval, which would hold the snapshot open for minutes.
+	const sql = `\getenv label RDS_BACKUP_LABEL
+SELECT pg_backup_start(:'label', fast => true);
+`
+	if err := session.Exec(ctx, sql); err != nil {
+		if closeErr := session.Close(); closeErr != nil {
+			slog.Warn("rds-agent: closing a failed backup session", "err", closeErr)
+		}
+		return fmt.Errorf("put the engine into backup mode: %w", err)
+	}
+
+	e.beginHoldLocked(label, session, hold)
+	return nil
+}
+
+func (e *postgresEngine) Unquiesce(ctx context.Context) error {
+	return e.releaseHold(ctx, "SELECT pg_backup_stop();\n")
+}

cmd/rds-agent/engine_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"errors"
+	"fmt"
 	"os"
 	"os/user"
 	"path/filepath"
@@ -139,11 +140,35 @@ func (r *recordingRunner) run(_ context.Context, c command) (string, error) {
 	return r.out, r.err
 }
 
+func testPostgresEngineMeta(t *testing.T) handlers_rds.Engine {
+	t.Helper()
+	meta, err := handlers_rds.LookupEngine(enginePostgres)
+	if err != nil {
+		t.Fatalf("LookupEngine(%s): %v", enginePostgres, err)
+	}
+	return meta
+}
+
 func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
 	t.Helper()
-	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
-	cfg.PGData = t.TempDir()
-	if err := os.MkdirAll(filepath.Join(cfg.PGData, "conf.d"), 0o700); err != nil {
+	cfg := testEngineConfig(t)
+	return newPostgresEngine(cfg, testPostgresEngineMeta(t), withPostgresReadBacks(cfg.EngineDataDir, run), nil, newPostgresProbe(cfg, staticProbe(0)))
+}
+
+// The same engine with the read-backs left to the runner, for the cases that are
+// about what the engine does with each answer rather than about the rest of an
+// apply.
+func newScriptedTestEngine(t *testing.T, run commandRunner) *postgresEngine {
+	t.Helper()
+	cfg := testEngineConfig(t)
+	return newPostgresEngine(cfg, testPostgresEngineMeta(t), run, nil, newPostgresProbe(cfg, staticProbe(0)))
+}
+
+func testEngineConfig(t *testing.T) config {
+	t.Helper()
+	cfg := testLoadConfig(t, enginePostgres)
+	cfg.EngineDataDir = t.TempDir()
+	if err := os.MkdirAll(filepath.Join(cfg.EngineDataDir, "conf.d"), 0o700); err != nil {
 		t.Fatalf("create conf.d: %v", err)
 	}
 	// The guest's postgres user does not exist here, and the chown of the
@@ -152,8 +177,36 @@ func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
 	if err != nil {
 		t.Fatalf("resolve the current user: %v", err)
 	}
-	cfg.PGUser = current.Username
-	return newPostgresEngine(cfg, run, nil)
+	cfg.EngineUser = current.Username
+	return cfg
+}
+
+// Answers the read-backs every apply runs against a live postmaster, from the
+// datadir the case is working in: an engine serving TLS, a reload the postmaster
+// took, and a pg_hba carrying whatever the agent last wrote. The call still
+// reaches the runner, so a case counting commands sees them; one that is about
+// the read-backs themselves builds its own engine.
+func withPostgresReadBacks(dataDir string, run commandRunner) commandRunner {
+	rule := filepath.Join(dataDir, postgresHBADir, postgresForceSSLRuleFile)
+	return func(ctx context.Context, c command) (string, error) {
+		out, err := run(ctx, c)
+		if err != nil {
+			return out, err
+		}
+		switch {
+		case strings.Contains(c.Stdin, "SHOW ssl"):
+			return "on\n", nil
+		case strings.Contains(c.Stdin, "pg_reload_conf"):
+			return "t\n", nil
+		case strings.Contains(c.Stdin, "pg_hba_file_rules"):
+			rules := 0
+			if _, statErr := os.Stat(rule); statErr == nil {
+				rules = postgresForceSSLRuleCount
+			}
+			return fmt.Sprintf("%d|0\n", rules), nil
+		}
+		return out, err
+	}
 }
 
 // The password reaches psql through the environment and is re-quoted there,
@@ -185,6 +238,40 @@ func TestPostgresEngine_SetPasswordKeepsTheSecretOutOfArgv(t *testing.T) {
 	}
 }
 
+// psql interpolates the password into the ALTER ROLE before the server sees it,
+// so a parameter group that turns statement logging on would write the rotated
+// password into a postmaster.log every snapshot then carries.
+func TestPostgresEngine_SetPasswordSilencesStatementLogging(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", "n3w-pw"); err != nil {
+		t.Fatalf("SetPassword: %v", err)
+	}
+	if len(runner.calls) != 1 {
+		t.Fatalf("ran %d commands, want 1", len(runner.calls))
+	}
+	sql := runner.calls[0].Stdin
+	alter := strings.Index(sql, "ALTER ROLE")
+	if alter < 0 {
+		t.Fatalf("SQL %q does not alter the role", sql)
+	}
+	for _, guard := range []string{
+		"SET log_statement = 'none';",
+		"SET log_min_duration_statement = -1;",
+		"SET log_min_error_statement = 'panic';",
+	} {
+		at := strings.Index(sql, guard)
+		if at < 0 {
+			t.Errorf("SQL %q is missing %q", sql, guard)
+			continue
+		}
+		if at > alter {
+			t.Errorf("SQL %q sets %q only after the ALTER ROLE has already been logged", sql, guard)
+		}
+	}
+}
+
 // psql echoes the failing statement, which here would carry the new password
 // back into a reply the control plane records.
 func TestPostgresEngine_SetPasswordRedactsTheSecretFromAFailure(t *testing.T) {
@@ -254,19 +341,27 @@ func TestPostgresEngine_ApplyParametersInstallsAndReloads(t *testing.T) {
 	if !strings.Contains(string(installed), "shared_buffers = '256MB'") {
 		t.Errorf("installed parameters = %q, want the resolved value", installed)
 	}
-	if len(runner.calls) != 3 {
-		t.Fatalf("ran %d commands, want a config check, a reload and a pending-restart read", len(runner.calls))
+	if len(runner.calls) != 5 {
+		t.Fatalf("ran %d commands, want a config check, the TLS guard, a reload, the rule read-back and a pending-restart read", len(runner.calls))
 	}
-	// The check runs before the reload, so a value the engine would refuse never
-	// reaches a running cluster.
+	// The check and the TLS guard both run before the reload, so neither a value
+	// the engine would refuse nor a rule rejecting every client reaches a running
+	// cluster. The read-back of the rules comes after, since the reload is what
+	// puts them in force.
 	if !slices.Contains(runner.calls[0].Args, "-C") {
 		t.Errorf("first command = %v, want the offline config check", runner.calls[0].Args)
 	}
-	if !strings.Contains(runner.calls[1].Stdin, "pg_reload_conf") {
-		t.Errorf("second statement = %q, want the reload", runner.calls[1].Stdin)
+	if !strings.Contains(runner.calls[1].Stdin, "SHOW ssl") {
+		t.Errorf("second statement = %q, want the serving-TLS guard", runner.calls[1].Stdin)
 	}
-	if !strings.Contains(runner.calls[2].Stdin, "pending_restart") {
-		t.Errorf("third statement = %q, want the pending-restart read", runner.calls[2].Stdin)
+	if !strings.Contains(runner.calls[2].Stdin, "pg_reload_conf") {
+		t.Errorf("third statement = %q, want the reload", runner.calls[2].Stdin)
+	}
+	if !strings.Contains(runner.calls[3].Stdin, "pg_hba_file_rules") {
+		t.Errorf("fourth statement = %q, want the client authentication read-back", runner.calls[3].Stdin)
+	}
+	if !strings.Contains(runner.calls[4].Stdin, "pending_restart") {
+		t.Errorf("fifth statement = %q, want the pending-restart read", runner.calls[4].Stdin)
 	}
 	if len(pending) != 2 || pending[0] != "shared_buffers" || pending[1] != "max_connections" {
 		t.Errorf("pending = %v, want both settings the engine reported", pending)
@@ -276,7 +371,7 @@ func TestPostgresEngine_ApplyParametersInstallsAndReloads(t *testing.T) {
 // A reload that failed leaves the engine on its old values, so reporting the
 // apply as successful would tell the customer a change took effect that did not.
 func TestPostgresEngine_ApplyParametersSurfacesAFailedReload(t *testing.T) {
-	runner := &recordingRunner{err: errors.New("could not connect to server")}
+	runner := &reloadFailingRunner{reloadErr: errors.New("reload was rejected")}
 	engine := newTestEngine(t, runner.run)
 
 	if _, err := engine.ApplyParameters(context.Background(),
@@ -285,23 +380,318 @@ func TestPostgresEngine_ApplyParametersSurfacesAFailedReload(t *testing.T) {
 	}
 }
 
-// Through the service manager, so the supervisor records the engine as stopped
-// and does not restart it underneath a VM that is going down.
-func TestPostgresEngine_StopGoesThroughTheServiceManager(t *testing.T) {
-	runner := &recordingRunner{}
-	engine := newTestEngine(t, runner.run)
+// pg_reload_conf() returns f when it could not signal the postmaster, and psql
+// exits 0 all the same. Every read-back after it parses files from disk, so an
+// apply that trusted the exit status would verify clean against a server still
+// serving the old rules.
+func TestPostgresEngine_ApplyParametersSurfacesAnUnsignalledReload(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount, reload: "f"}
+	engine := newScriptedTestEngine(t, runner.run)
 
-	if err := engine.Stop(context.Background()); err != nil {
-		t.Fatalf("Stop: %v", err)
-	}
-	if len(runner.calls) != 1 {
-		t.Fatalf("ran %d commands, want 1", len(runner.calls))
-	}
-	call := runner.calls[0]
-	if filepath.Base(call.Name) != "rc-service" {
-		t.Errorf("ran %q, want rc-service", call.Name)
-	}
-	if len(call.Args) != 2 || call.Args[0] != "postgresql" || call.Args[1] != "stop" {
-		t.Errorf("args = %v, want [postgresql stop]", call.Args)
+	_, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "work_mem", Value: "8MB"}})
+	if err == nil || !strings.Contains(err.Error(), `pg_reload_conf() returned "f"`) {
+		t.Fatalf("ApplyParameters error = %v, want a refusal naming the answer the postmaster gave", err)
+	}
+}
+
+type reloadFailingRunner struct {
+	recordingRunner
+
+	reloadErr error
+}
+
+func (r *reloadFailingRunner) run(ctx context.Context, c command) (string, error) {
+	if strings.Contains(c.Stdin, "pg_reload_conf") {
+		r.calls = append(r.calls, c)
+		return "", r.reloadErr
+	}
+	return r.recordingRunner.run(ctx, c)
+}
+
+// A failed instance cannot reload through psql. Its corrected, parser-checked
+// include must start the service and remain installed as the serving set.
+func TestPostgresEngine_ApplyParametersRestartsOnARepairSetWhileEngineIsDown(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+	codes := []int{2, 2, 0}
+	probeCall := 0
+	cfg := testProbeConfig()
+	engine.probe = newPostgresProbe(cfg, func(context.Context, string, ...string) (int, string, error) {
+		code := codes[min(probeCall, len(codes)-1)]
+		probeCall++
+		return code, "", nil
+	})
+	engine.repairTimeout = 100 * time.Millisecond
+	engine.repairPoll = time.Millisecond
+	params := []handlers_rds.Parameter{
+		{Name: "shared_buffers", Value: "32768"},
+		{Name: "work_mem", Value: "8192"},
+	}
+
+	pending, err := engine.ApplyParameters(t.Context(), params)
+	if err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if len(pending) != 0 {
+		t.Errorf("pending = %v, want the restarted engine's actual pending set", pending)
+	}
+	if len(runner.calls) != 3 {
+		t.Fatalf("calls = %+v, want config check, service restart and pending read", runner.calls)
+	}
+	if !slices.Contains(runner.calls[0].Args, "-C") {
+		t.Errorf("first call = %+v, want the offline config check", runner.calls[0])
+	}
+	if !slices.Equal(runner.calls[1].Args, []string{"postgresql", "restart"}) {
+		t.Errorf("second call args = %v, want the service restart", runner.calls[1].Args)
+	}
+	if !strings.Contains(runner.calls[2].Stdin, "pending_restart") {
+		t.Errorf("third call = %+v, want the pending-restart read", runner.calls[2])
+	}
+	installed, err := os.ReadFile(engine.params.installedPath())
+	if err != nil {
+		t.Fatalf("read installed parameters: %v", err)
+	}
+	if !strings.Contains(string(installed), "work_mem = '8192'") {
+		t.Errorf("installed parameters = %q, want the repair set retained", installed)
+	}
+}
+
+// Answers the two read-backs an apply runs from fields the case sets, so what
+// the engine does with each answer is what is under test.
+type tlsReadBackRunner struct {
+	recordingRunner
+
+	ssl        string
+	ruleRows   int
+	brokenRows int
+	// What pg_reload_conf() answered. Empty is the postmaster having taken the
+	// signal, which is every case but the one about it not having.
+	reload string
+}
+
+func (r *tlsReadBackRunner) run(ctx context.Context, c command) (string, error) {
+	switch {
+	case strings.Contains(c.Stdin, "SHOW ssl"):
+		r.calls = append(r.calls, c)
+		return r.ssl + "\n", nil
+	case strings.Contains(c.Stdin, "pg_reload_conf"):
+		r.calls = append(r.calls, c)
+		if r.reload == "" {
+			return "t\n", nil
+		}
+		return r.reload + "\n", nil
+	case strings.Contains(c.Stdin, "pg_hba_file_rules"):
+		r.calls = append(r.calls, c)
+		return fmt.Sprintf("%d|%d\n", r.ruleRows, r.brokenRows), nil
+	}
+	return r.recordingRunner.run(ctx, c)
+}
+
+func forceSSLParameter(t *testing.T, value string) []handlers_rds.Parameter {
+	t.Helper()
+	return []handlers_rds.Parameter{
+		{Name: "work_mem", Value: "4096"},
+		{Name: testPostgresEngineMeta(t).TLSEnforcementParameter(), Value: value},
+	}
+}
+
+// PostgreSQL has no server setting for this, so the value in the parameter file
+// is inert: the rule the generated pg_hba includes is the whole of it.
+func TestPostgresEngine_ApplyParametersWritesTheEnforcementRule(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1")); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	rule, err := os.ReadFile(engine.forceSSLRulePath())
+	if err != nil {
+		t.Fatalf("read the enforcement rule: %v", err)
+	}
+	if string(rule) != postgresForceSSLRules {
+		t.Errorf("enforcement rule = %q, want the constant rds-init also writes", rule)
+	}
+}
+
+// A snapshot carries hba.d with it, so restoring one taken while enforcing into
+// a group that turns enforcement off has to take the rule with it.
+func TestPostgresEngine_ApplyParametersRemovesAStaleEnforcementRule(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on"}
+	engine := newScriptedTestEngine(t, runner.run)
+	if err := os.MkdirAll(filepath.Dir(engine.forceSSLRulePath()), 0o700); err != nil {
+		t.Fatalf("create hba.d: %v", err)
+	}
+	if err := os.WriteFile(engine.forceSSLRulePath(), []byte(postgresForceSSLRules), 0o600); err != nil {
+		t.Fatalf("write the rule the restored volume carried: %v", err)
+	}
+
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "0")); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if _, err := os.Stat(engine.forceSSLRulePath()); !os.IsNotExist(err) {
+		t.Errorf("the stale enforcement rule is still in place (stat err = %v)", err)
+	}
+}
+
+// The ordinary state of an instance whose resolved set predates the parameter,
+// and the whole of the migration: it begins requiring TLS with no control-plane
+// work at all.
+func TestPostgresEngine_ApplyParametersEnforcesOnAnAbsentKey(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(t.Context(),
+		[]handlers_rds.Parameter{{Name: "work_mem", Value: "4096"}}); err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if _, err := os.Stat(engine.forceSSLRulePath()); err != nil {
+		t.Errorf("a set predating the parameter did not enforce (stat err = %v)", err)
+	}
+}
+
+// The resolver canonicalises every boolean, so a value that is neither means the
+// file was written by something other than the platform. Reading an unparsable
+// security setting as off is the one choice not open here.
+func TestPostgresEngine_ApplyParametersRefusesAnUnparsableEnforcementValue(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on"}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "yes"))
+	if err == nil || !strings.Contains(err.Error(), "neither 1 nor 0") {
+		t.Fatalf("ApplyParameters error = %v, want a refusal naming the unreadable value", err)
+	}
+	if _, statErr := os.Stat(engine.params.installedPath()); !os.IsNotExist(statErr) {
+		t.Errorf("the refused set is still installed (stat err = %v)", statErr)
+	}
+}
+
+// Without this, a live enable against an engine started with no certificate
+// would reject every subsequent connection and report the parameter applied.
+func TestPostgresEngine_ApplyParametersRefusesToEnforceWithoutTLS(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "off"}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1"))
+	if err == nil || !strings.Contains(err.Error(), "serving without TLS") {
+		t.Fatalf("ApplyParameters error = %v, want a refusal naming the engine's own TLS state", err)
+	}
+	if _, statErr := os.Stat(engine.forceSSLRulePath()); !os.IsNotExist(statErr) {
+		t.Errorf("the enforcement rule was written anyway (stat err = %v)", statErr)
+	}
+}
+
+// PostgreSQL discards a pg_hba it cannot parse and keeps the rules it is already
+// serving, which presents as an apply that succeeded and enforces nothing. The
+// shape is asserted as well as the errors, because our own derivation writing
+// the wrong thing is the failure that will actually happen.
+func TestPostgresEngine_ApplyParametersFailsAnUnverifiableEnforcement(t *testing.T) {
+	tests := []struct {
+		name       string
+		value      string
+		ruleRows   int
+		brokenRows int
+		want       string
+	}{
+		{name: "rule missing", value: "1", ruleRows: 0, want: "carries 0 TLS enforcement rules, want 2"},
+		{name: "rule present when it should not be", value: "0", ruleRows: 2, want: "carries 2 TLS enforcement rules, want 0"},
+		{name: "a rule the engine could not parse", value: "1", ruleRows: 2, brokenRows: 1, want: "could not parse"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			runner := &tlsReadBackRunner{ssl: "on", ruleRows: tc.ruleRows, brokenRows: tc.brokenRows}
+			engine := newScriptedTestEngine(t, runner.run)
+
+			_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, tc.value))
+			if err == nil || !strings.Contains(err.Error(), tc.want) {
+				t.Fatalf("ApplyParameters error = %v, want %q", err, tc.want)
+			}
+		})
+	}
+}
+
+// A rejected set has to take its enforcement with it. Withdrawing one without
+// the other would leave the instance serving the previous parameters under the
+// rejected set's TLS posture.
+func TestPostgresEngine_ApplyParametersWithdrawsBothFilesTogether(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
+	engine := newScriptedTestEngine(t, runner.run)
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1")); err != nil {
+		t.Fatalf("first ApplyParameters: %v", err)
+	}
+
+	// Turning enforcement off, verified against a pg_hba that still carries the
+	// rule — so the apply fails after both files have already been replaced.
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "0")); err == nil {
+		t.Fatal("ApplyParameters succeeded against an unverifiable enforcement")
+	}
+	installed, err := os.ReadFile(engine.params.installedPath())
+	if err != nil {
+		t.Fatalf("read the installed parameters: %v", err)
+	}
+	if !strings.Contains(string(installed), "rds.force_ssl = '1'") {
+		t.Errorf("installed parameters = %q, want the previous accepted set restored", installed)
+	}
+	if _, err := os.Stat(engine.forceSSLRulePath()); err != nil {
+		t.Errorf("the enforcement of the previous set was not restored with it (stat err = %v)", err)
+	}
+	// The reload already adopted the rejected files, so putting them back is not
+	// enough on its own.
+	if !strings.Contains(runner.calls[len(runner.calls)-1].Stdin, "pg_reload_conf") {
+		t.Errorf("last statement = %q, want a reload taking the engine off the withdrawn set", runner.calls[len(runner.calls)-1].Stdin)
+	}
+}
+
+// A service restart does not re-run rds-init, so a restore stopping at the
+// parameter file would bring the last known good set up under the enforcement of
+// the set that was just rejected.
+func TestPostgresEngine_RestoreLastKnownGoodReDerivesEnforcement(t *testing.T) {
+	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
+	engine := newScriptedTestEngine(t, runner.run)
+
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1")); err != nil {
+		t.Fatalf("ApplyParameters(enforcing): %v", err)
+	}
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("RecordServingParameters: %v", err)
+	}
+	runner.ruleRows = 0
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "0")); err != nil {
+		t.Fatalf("ApplyParameters(not enforcing): %v", err)
+	}
+	if _, err := os.Stat(engine.forceSSLRulePath()); !os.IsNotExist(err) {
+		t.Fatalf("the enforcement rule survived the set that turned it off (stat err = %v)", err)
+	}
+
+	engine.probe = stubProbe(t, 2, nil)
+	restored, err := engine.RestoreLastKnownGoodParameters(t.Context())
+	if err != nil {
+		t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
+	}
+	if !restored {
+		t.Fatal("did not restore a set that differs from the last known good")
+	}
+	if _, err := os.Stat(engine.forceSSLRulePath()); err != nil {
+		t.Errorf("the restored set's enforcement was not re-derived (stat err = %v)", err)
+	}
+}
+
+func TestPostgresEngine_ApplyParametersKeepsRepairSetWhenRestartTimesOut(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+	engine.probe = stubProbe(t, 2, nil)
+	engine.repairTimeout = 10 * time.Millisecond
+	engine.repairPoll = time.Millisecond
+
+	_, err := engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{{Name: "work_mem", Value: "8192"}})
+	if err == nil || !strings.Contains(err.Error(), "wait for the engine") {
+		t.Fatalf("ApplyParameters error = %v, want the repair wait failure", err)
+	}
+	installed, readErr := os.ReadFile(engine.params.installedPath())
+	if readErr != nil {
+		t.Fatalf("read installed parameters: %v", readErr)
+	}
+	if !strings.Contains(string(installed), "work_mem = '8192'") {
+		t.Errorf("installed parameters = %q, want the checked repair set retained", installed)
 	}
 }

cmd/rds-agent/health.go

@@ -1,38 +1,49 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"errors"
-	"fmt"
 	"os/exec"
-	"strconv"
+	"strings"
 	"sync/atomic"
 
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
 
 // A non-zero exit is a result, not a fault, so it comes back as a code; err is
-// reserved for the probe failing to run at all.
-type probeRunner func(ctx context.Context, name string, args ...string) (int, error)
+// reserved for the probe failing to run at all. The client's stderr comes back
+// with it: on a refusal it is the only place the reason is ever stated.
+type probeRunner func(ctx context.Context, name string, args ...string) (int, string, error)
 
-func execProbeRunner(ctx context.Context, name string, args ...string) (int, error) {
-	err := exec.CommandContext(ctx, name, args...).Run()
+func execProbeRunner(ctx context.Context, name string, args ...string) (int, string, error) {
+	var stderr bytes.Buffer
+	cmd := exec.CommandContext(ctx, name, args...)
+	cmd.Stderr = &stderr
+	err := cmd.Run()
+	out := strings.TrimSpace(stderr.String())
 	if err == nil {
-		return 0, nil
+		return 0, out, nil
 	}
-	var exitErr *exec.ExitError
-	if errors.As(err, &exitErr) {
-		return exitErr.ExitCode(), nil
+	if ctxErr := ctx.Err(); ctxErr != nil {
+		return -1, out, ctxErr
 	}
-	return -1, err
+	if exitErr, ok := errors.AsType[*exec.ExitError](err); ok {
+		return exitErr.ExitCode(), out, nil
+	}
+	return -1, out, err
 }
 
-// Shells out to pg_isready rather than dialling the port, since only a startup
-// exchange separates an engine that is serving from one still in recovery.
+// What the engine reports about itself, which is the only part of the probe
+// that differs between engines. The port is passed in rather than read from the
+// engine, so the atomic below stays the one place it is followed from.
+type probeStateFn func(ctx context.Context, port int64) (engineState, string)
+
+// Engine-neutral: the latch, the port, the degraded mapping and the translation
+// to a reported health are shared, and only the state function is supplied by
+// the engine the image bakes.
 type engineProbe struct {
-	run    probeRunner
-	binary string
-	host   string
+	stateFn probeStateFn
 	// Set from the bootstrap config once it lands, from a different goroutine
 	// than the heartbeat that reads it.
 	port atomic.Int64
@@ -41,9 +52,9 @@ type engineProbe struct {
 	seenHealthy bool
 }
 
-func newEngineProbe(cfg config, run probeRunner) *engineProbe {
-	p := &engineProbe{run: run, binary: cfg.PGIsReady, host: cfg.EngineHost}
-	p.port.Store(int64(cfg.EnginePort))
+func newEngineProbe(port int, stateFn probeStateFn) *engineProbe {
+	p := &engineProbe{stateFn: stateFn}
+	p.port.Store(int64(port))
 	return p
 }
 
@@ -52,15 +63,14 @@ func (p *engineProbe) setPort(port int) {
 }
 
 // What the engine is doing, before the seenHealthy latch collapses "never up"
-// and "was up and went away" into one health. The parameter rollback needs them
-// apart: a postmaster that is up and replaying WAL will come back on its own, one
-// that is not running at all after a parameter change will not.
+// and "went away" into one health. The rollback needs them apart: an engine
+// replaying WAL comes back on its own, one that is not running will not.
 type engineState int
 
 const (
 	// Not answering at all, or the probe could not run.
 	engineAbsent engineState = iota
-	// The postmaster is up and rejecting connections: startup or crash recovery.
+	// The process is up but not yet serving: startup or crash recovery.
 	engineRecovering
 	engineServing
 )
@@ -83,21 +93,7 @@ func (p *engineProbe) Check(ctx context.Context) (handlers_rds.EngineHealth, str
 // The raw probe result. Safe to call from a goroutine other than the
 // heartbeat's: it touches no latched state.
 func (p *engineProbe) state(ctx context.Context) (engineState, string) {
-	port := strconv.FormatInt(p.port.Load(), 10)
-	code, err := p.run(ctx, p.binary, "-h", p.host, "-p", port, "-q")
-	switch {
-	case err != nil:
-		// A missing binary or broken image. Reporting healthy on the strength of
-		// nothing would hide it, so this reads as absent like an engine that did
-		// not answer.
-		return engineAbsent, fmt.Sprintf("engine probe could not run: %v", err)
-	case code == 0:
-		return engineServing, ""
-	case code == 1:
-		return engineRecovering, "engine is rejecting connections (startup or recovery)"
-	default:
-		return engineAbsent, fmt.Sprintf("engine did not respond on %s:%s", p.host, port)
-	}
+	return p.stateFn(ctx, p.port.Load())
 }
 
 // A non-answering engine: starting until it has answered once, unhealthy after.

cmd/rds-agent/health_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"strings"
 	"testing"
+	"time"
 
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
@@ -28,8 +29,8 @@ func TestEngineProbe_MapsExitCodesToHealth(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			probe := newEngineProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
-				return tt.code, tt.runErr
+			probe := newPostgresProbe(testProbeConfig(), func(context.Context, string, ...string) (int, string, error) {
+				return tt.code, "", tt.runErr
 			})
 			probe.seenHealthy = tt.seenHealthy
 
@@ -51,8 +52,8 @@ func TestEngineProbe_MapsExitCodesToHealth(t *testing.T) {
 // has to flip the first time the engine answers.
 func TestEngineProbe_LatchesAfterFirstHealthy(t *testing.T) {
 	code := 0
-	probe := newEngineProbe(testProbeConfig(), func(context.Context, string, ...string) (int, error) {
-		return code, nil
+	probe := newPostgresProbe(testProbeConfig(), func(context.Context, string, ...string) (int, string, error) {
+		return code, "", nil
 	})
 
 	if got, _ := probe.Check(context.Background()); got != handlers_rds.EngineHealthHealthy {
@@ -68,9 +69,9 @@ func TestEngineProbe_LatchesAfterFirstHealthy(t *testing.T) {
 // must follow it rather than keep asking the default.
 func TestEngineProbe_ProbesTheAssignedPort(t *testing.T) {
 	var gotArgs []string
-	probe := newEngineProbe(testProbeConfig(), func(_ context.Context, _ string, args ...string) (int, error) {
+	probe := newPostgresProbe(testProbeConfig(), func(_ context.Context, _ string, args ...string) (int, string, error) {
 		gotArgs = args
-		return 0, nil
+		return 0, "", nil
 	})
 
 	probe.setPort(6543)
@@ -81,6 +82,65 @@ func TestEngineProbe_ProbesTheAssignedPort(t *testing.T) {
 	}
 }
 
-func testProbeConfig() config {
-	return config{EngineHost: defaultEngineHost, EnginePort: defaultEnginePort, PGIsReady: defaultPGIsReady}
+func TestExecProbeRunner_PreservesTheProbeDeadline(t *testing.T) {
+	ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
+	defer cancel()
+
+	code, _, err := execProbeRunner(ctx, "sh", "-c", "while :; do :; done")
+	if code != -1 {
+		t.Errorf("exit code = %d, want -1 for a timed-out probe", code)
+	}
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Errorf("error = %v, want the context deadline", err)
+	}
+}
+
+// The client's account of a refusal is the reason a probe failure is legible at
+// all, so a runner that dropped it would leave the caller nothing to report.
+func TestExecProbeRunner_CapturesTheClientsStderr(t *testing.T) {
+	code, stderr, err := execProbeRunner(t.Context(), "sh", "-c", "echo 'TLS handshake failed' >&2; exit 1")
+	if err != nil {
+		t.Fatalf("execProbeRunner: %v", err)
+	}
+	if code != 1 {
+		t.Errorf("exit code = %d, want 1", code)
+	}
+	if stderr != "TLS handshake failed" {
+		t.Errorf("stderr = %q, want the client's own message", stderr)
+	}
+}
+
+// The engine binds the customer ENI and nothing else, so a loopback probe would
+// report a serving engine as absent. libpq reads a host beginning with / as a
+// socket directory, which is also what keeps liveness off the network path.
+func TestPostgresProbe_ProbesTheUnixSocket(t *testing.T) {
+	cfg := testProbeConfig()
+	var probed []string
+	probe := newPostgresProbe(cfg, func(_ context.Context, name string, args ...string) (int, string, error) {
+		probed = append([]string{name}, args...)
+		return 0, "", nil
+	})
+
+	if state, _ := probe.state(context.Background()); state != engineServing {
+		t.Fatalf("probe state = %v, want engineServing", state)
+	}
+	host := ""
+	for i, arg := range probed {
+		if arg == "-h" && i+1 < len(probed) {
+			host = probed[i+1]
+		}
+	}
+	if host != cfg.SocketDir {
+		t.Errorf("probe host = %q, want the socket directory %q", host, cfg.SocketDir)
+	}
+	if !strings.HasPrefix(host, "/") {
+		t.Errorf("probe host = %q, which libpq reads as a TCP host rather than a socket directory", host)
+	}
+}
+
+func testProbeConfig() config {
+	return config{
+		SocketDir:  engineLayouts[enginePostgres].socketDir,
+		EnginePort: engineLayouts[enginePostgres].port,
+	}
 }

cmd/rds-agent/heartbeat.go

@@ -3,25 +3,37 @@ package main
 import (
 	"context"
 	"log/slog"
+	"sync/atomic"
 	"time"
+	"unicode/utf8"
 
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
 
+const (
+	servingParameterRecordTimeout = 15 * time.Second
+	maxBootstrapFailureBytes      = 512
+)
+
 // The beat carries the probe's result, not the agent's own liveness: an agent
 // up while the engine is down is what the recovery reconciler must see.
 type heartbeater struct {
 	cp       controlPlane
 	probe    *engineProbe
+	recorder servingParameterRecorder
 	id       identity
 	interval time.Duration
+
+	// Bootstrap and heartbeat run concurrently. Keeping the latest failure here
+	// makes a VM stuck before engine startup explain why through its next beat.
+	bootstrapFailure atomic.Pointer[string]
 }
 
-func newHeartbeater(cp controlPlane, probe *engineProbe, interval time.Duration) *heartbeater {
+func newHeartbeater(cp controlPlane, probe *engineProbe, recorder servingParameterRecorder, interval time.Duration) *heartbeater {
 	if interval <= 0 {
 		interval = handlers_rds.HeartbeatInterval
 	}
-	return &heartbeater{cp: cp, probe: probe, interval: interval}
+	return &heartbeater{cp: cp, probe: probe, recorder: recorder, interval: interval}
 }
 
 // Called before Run starts and from Run's own goroutine after, so the interval
@@ -32,8 +44,40 @@ func (h *heartbeater) setInterval(d time.Duration) {
 	}
 }
 
+func (h *heartbeater) setBootstrapFailure(what string, err error) {
+	message := what + " failed: " + err.Error()
+	if len(message) > maxBootstrapFailureBytes {
+		limit := maxBootstrapFailureBytes - len("...")
+		for limit > 0 && !utf8.RuneStart(message[limit]) {
+			limit--
+		}
+		message = message[:limit] + "..."
+	}
+	h.bootstrapFailure.Store(&message)
+}
+
+func (h *heartbeater) clearBootstrapFailure() {
+	h.bootstrapFailure.Store(nil)
+}
+
 func (h *heartbeater) beat(ctx context.Context) {
 	health, message := h.probe.Check(ctx)
+	if failure := h.bootstrapFailure.Load(); failure != nil {
+		if message != "" {
+			message += "; "
+		}
+		message += *failure
+	}
+	if health == handlers_rds.EngineHealthHealthy && h.recorder != nil {
+		// Checking every healthy beat also observes a restart that completed
+		// between probes. The recorder skips unchanged and pending-restart sets.
+		recordCtx, cancel := context.WithTimeout(ctx, servingParameterRecordTimeout)
+		err := h.recorder.RecordServingParameters(recordCtx)
+		cancel()
+		if err != nil {
+			slog.Warn("rds-agent: recording the serving parameters failed", "err", err)
+		}
+	}
 
 	out, err := h.cp.SubmitState(ctx, h.id, health, message)
 	if err != nil {

cmd/rds-agent/heartbeat_test.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+	"strings"
+	"testing"
+	"unicode/utf8"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The first healthy probe is the first proof that the installed include can
+// start PostgreSQL, so it must seed recovery without an apply command.
+func TestHeartbeater_FirstServingProbeSeedsLastKnownGood(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+	content := []byte("work_mem = '4096'\n")
+	if err := os.WriteFile(engine.params.installedPath(), content, 0o600); err != nil {
+		t.Fatalf("write installed parameters: %v", err)
+	}
+
+	cp := newFakeControlPlane()
+	h := newHeartbeater(cp, engine.probe, engine, 0)
+	h.beat(context.Background())
+
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
+	if err != nil {
+		t.Fatalf("read last known good: %v", err)
+	}
+	if string(lastGood) != string(content) {
+		t.Errorf("last known good = %q, want %q", lastGood, content)
+	}
+	states := cp.snapshotStates()
+	if len(states) != 1 || states[0].health != handlers_rds.EngineHealthHealthy {
+		t.Errorf("states = %+v, want one healthy heartbeat", states)
+	}
+}
+
+func TestHeartbeater_BoundsBootstrapFailure(t *testing.T) {
+	h := newHeartbeater(newFakeControlPlane(), newPostgresProbe(testProbeConfig(), staticProbe(2)), nil, 0)
+	h.setBootstrapFailure("bootstrap fetch", errors.New(strings.Repeat("界", 1000)))
+
+	failure := h.bootstrapFailure.Load()
+	if failure == nil {
+		t.Fatal("bootstrap failure was not stored")
+	}
+	if len(*failure) > maxBootstrapFailureBytes {
+		t.Errorf("bootstrap failure is %d bytes, want at most %d", len(*failure), maxBootstrapFailureBytes)
+	}
+	if !utf8.ValidString(*failure) {
+		t.Error("bootstrap failure was truncated inside a UTF-8 encoding")
+	}
+	if !strings.HasSuffix(*failure, "...") {
+		t.Errorf("bootstrap failure %q does not show that it was truncated", *failure)
+	}
+}
+
+func TestHeartbeater_ChecksServingParametersOnEveryHealthyProbe(t *testing.T) {
+	code := 0
+	cfg := testProbeConfig()
+	probe := newPostgresProbe(cfg, func(context.Context, string, ...string) (int, string, error) {
+		return code, "", nil
+	})
+	recorder := &countingServingRecorder{}
+	h := newHeartbeater(newFakeControlPlane(), probe, recorder, 0)
+
+	h.beat(context.Background())
+	h.beat(context.Background())
+	code = 2
+	h.beat(context.Background())
+	code = 0
+	h.beat(context.Background())
+
+	if recorder.calls != 3 {
+		t.Errorf("record calls = %d, want one per healthy probe", recorder.calls)
+	}
+}
+
+type countingServingRecorder struct {
+	calls int
+}
+
+var _ servingParameterRecorder = (*countingServingRecorder)(nil)
+
+func (r *countingServingRecorder) RecordServingParameters(context.Context) error {
+	r.calls++
+	return nil
+}

cmd/rds-agent/parameters.go

@@ -0,0 +1,233 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+)
+
+// The generated parameter files and what it takes to install one. They live on
+// the data volume at the paths rds-init writes: a class change replaces the VM,
+// so a set under /etc would revert to defaults while the API reported otherwise.
+type parameterStore struct {
+	dir string
+	// The include the resolved set is rendered to.
+	installed string
+	// The copy of the last set the engine accepted, which a boot loop is rolled
+	// back to. Deliberately not a name the engine's own include glob reads, so it
+	// is never parsed as a second set of settings.
+	lastGood string
+	// What the engine actually started on, for an engine that cannot be asked
+	// which of its settings are still pending a restart. Empty for one that can.
+	serving string
+	// Prepended to every file written here, for a configuration format that has
+	// no setting outside a group.
+	header string
+	// The engine parses its own configuration, so a root-owned file would be
+	// unreadable to it.
+	osUser string
+	// Names the engine whose option-file spellings these files are written in.
+	engine string
+}
+
+func (s parameterStore) installedPath() string { return filepath.Join(s.dir, s.installed) }
+func (s parameterStore) lastGoodPath() string  { return filepath.Join(s.dir, s.lastGood) }
+func (s parameterStore) servingPath() string   { return filepath.Join(s.dir, s.serving) }
+
+// Byte-for-byte what rds-init writes for the same set, so the copies compared
+// against it are comparing values rather than formatting.
+func (s parameterStore) render(params []handlers_rds.Parameter) ([]byte, error) {
+	body, err := renderParameters(s.engine, params)
+	if err != nil {
+		return nil, err
+	}
+	return []byte(s.header + body), nil
+}
+
+// Installs the resolved set at the path rds-init uses, so a later boot
+// overwrites rather than shadows it. Returns a func putting back whatever was
+// there, so the caller can undo the install without re-rendering anything.
+func (s parameterStore) install(params []handlers_rds.Parameter) (restore func() error, err error) {
+	path := s.installedPath()
+	// A missing file is the first-ever apply, and its rollback is a removal.
+	previous, readErr := os.ReadFile(path)
+	if readErr != nil && !os.IsNotExist(readErr) {
+		return nil, fmt.Errorf("read the installed parameters at %s: %w", path, readErr)
+	}
+	existed := readErr == nil
+
+	rendered, err := s.render(params)
+	if err != nil {
+		return nil, err
+	}
+	if err := s.write(path, rendered); err != nil {
+		return nil, err
+	}
+	return func() error {
+		if !existed {
+			if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+				return fmt.Errorf("withdraw the rejected parameters at %s: %w", path, err)
+			}
+			return nil
+		}
+		return s.write(path, previous)
+	}, nil
+}
+
+// Atomic: a temp file in the same directory, renamed over the target. The temp
+// name deliberately does not end in the engine's include suffix, so a crash
+// between write and rename cannot leave the engine reading a half-written file.
+func (s parameterStore) write(path string, content []byte) error {
+	tmp := path + ".new"
+	if err := os.WriteFile(tmp, content, 0o600); err != nil {
+		return fmt.Errorf("write %s: %w", tmp, err)
+	}
+	if err := s.chownToEngine(tmp); err != nil {
+		os.Remove(tmp)
+		return err
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		os.Remove(tmp)
+		return fmt.Errorf("install %s: %w", path, err)
+	}
+	return nil
+}
+
+// A guest with no engine user is a broken image, not something to work around.
+func (s parameterStore) chownToEngine(path string) error {
+	credential, err := lookupCredential(s.osUser)
+	if err != nil {
+		return err
+	}
+	if err := os.Chown(path, int(credential.Uid), int(credential.Gid)); err != nil {
+		return fmt.Errorf("hand %s to %s: %w", path, s.osUser, err)
+	}
+	return nil
+}
+
+// Snapshots the installed set as the rollback target, but only once the engine
+// is running all of it. A set still pending a restart has not been shown to
+// serve, so promoting it would point the rollback at a config about to fail.
+func recordLastGood(ctx context.Context, store parameterStore, pending pendingRestartFn) error {
+	if _, err := os.Stat(store.installedPath()); err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("inspect the serving parameters: %w", err)
+	}
+	names, err := pending(ctx)
+	if err != nil {
+		return err
+	}
+	if len(names) > 0 {
+		return nil
+	}
+
+	content, err := os.ReadFile(store.installedPath())
+	if err != nil {
+		return fmt.Errorf("read the serving parameters: %w", err)
+	}
+	lastGood, err := os.ReadFile(store.lastGoodPath())
+	if err == nil && bytes.Equal(content, lastGood) {
+		return nil
+	}
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("read the last known good parameters: %w", err)
+	}
+	return store.write(store.lastGoodPath(), content)
+}
+
+// Puts the last set the engine accepted back in place, for a restart that failed
+// after a parameter change. The probe is checked again under the caller's
+// parameter lock, so a repair that just brought the engine back is not reversed.
+func restoreLastGood(ctx context.Context, store parameterStore, probe *engineProbe) (bool, error) {
+	if state, _ := probe.state(ctx); state != engineAbsent {
+		return false, nil
+	}
+	lastGood, err := os.ReadFile(store.lastGoodPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, fmt.Errorf("read the last known good parameters: %w", err)
+	}
+	current, err := os.ReadFile(store.installedPath())
+	if err != nil && !os.IsNotExist(err) {
+		return false, fmt.Errorf("read the installed parameters: %w", err)
+	}
+	if bytes.Equal(current, lastGood) {
+		return false, nil
+	}
+	if err := store.write(store.installedPath(), lastGood); err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+// The settings the engine has accepted but will not honour until it restarts.
+// Each implementation answers it its own way, and both the apply and the serving
+// snapshot are gated on the same answer.
+type pendingRestartFn func(ctx context.Context) ([]string, error)
+
+// The parameter-file state every engine keeps. The operations over it are free
+// functions above rather than methods here, so each implementation names its own
+// restart and pending answers at the call site where the compiler checks them.
+type parameterManager struct {
+	params parameterStore
+	probe  *engineProbe
+	// Serializes parameter installs, serving snapshots and rollback restores so
+	// none can copy or replace an intermediate configuration.
+	paramMu       sync.Mutex
+	repairTimeout time.Duration
+	repairPoll    time.Duration
+}
+
+const (
+	parameterRepairTimeout = 90 * time.Second
+	parameterRepairPoll    = time.Second
+)
+
+// Starts a down engine on the set just installed and waits for it to serve.
+// Reached when the engine went away during an apply: the installed set is the
+// newer, so starting on it beats restoring one that already failed.
+func awaitRepairedEngine(ctx context.Context, probe *engineProbe, restart func(context.Context) error,
+	pending pendingRestartFn, timeout, poll time.Duration) ([]string, error) {
+	repairCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	if err := restart(repairCtx); err != nil {
+		return nil, fmt.Errorf("start the engine on the repaired parameter set: %w", err)
+	}
+
+	timer := time.NewTimer(0)
+	defer timer.Stop()
+	lastMessage := "engine did not respond"
+	for {
+		select {
+		case <-repairCtx.Done():
+			if ctx.Err() != nil {
+				return nil, fmt.Errorf("wait for the engine on the repaired parameter set: %w", ctx.Err())
+			}
+			return nil, fmt.Errorf("wait for the engine on the repaired parameter set: %s: %w", lastMessage, repairCtx.Err())
+		case <-timer.C:
+		}
+
+		state, message := probe.state(repairCtx)
+		if state == engineServing {
+			names, err := pending(repairCtx)
+			if err == nil {
+				return names, nil
+			}
+			lastMessage = err.Error()
+		} else if message != "" {
+			lastMessage = message
+		}
+		timer.Reset(poll)
+	}
+}

cmd/rds-agent/paramrecovery.go

@@ -2,8 +2,11 @@ package main
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"log/slog"
 	"time"
+
+	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
 )
 
 // How long the engine is given to come up after a boot before the installed
@@ -29,12 +32,14 @@ const parameterRollbackPoll = 10 * time.Second
 type paramGuard struct {
 	engine parameterRecovery
 	probe  *engineProbe
+	cp     controlPlane
+	id     identity
 	after  time.Duration
 	poll   time.Duration
 }
 
-func newParamGuard(engine parameterRecovery, probe *engineProbe) *paramGuard {
-	return &paramGuard{engine: engine, probe: probe, after: parameterRollbackAfter, poll: parameterRollbackPoll}
+func newParamGuard(engine parameterRecovery, probe *engineProbe, cp controlPlane) *paramGuard {
+	return &paramGuard{engine: engine, probe: probe, cp: cp, after: parameterRollbackAfter, poll: parameterRollbackPoll}
 }
 
 // Runs once per agent lifetime: a rollback that did not bring the engine back
@@ -45,20 +50,29 @@ func (g *paramGuard) Run(ctx context.Context) {
 		return
 	}
 
-	restored, err := g.engine.RestoreLastKnownGoodParameters()
+	// A restore that replaced the file and then failed has still moved the
+	// instance onto the last accepted set, so the restart and the report below
+	// are what it needs; returning here would leave it down and unaccounted for.
+	restored, err := g.engine.RestoreLastKnownGoodParameters(ctx)
 	if err != nil {
-		slog.Error("rds-agent: could not restore the last known good parameters", "err", err)
-		return
+		slog.Error("rds-agent: restoring the last known good parameters failed",
+			"err", err, "parametersReplaced", restored)
+		if !restored {
+			return
+		}
 	}
 	if !restored {
-		// The engine is already running the set it last accepted, so whatever is
-		// keeping it down is not the parameters.
-		slog.Warn("rds-agent: engine is down but its parameters are the last accepted set; not rolling back")
+		slog.Warn("rds-agent: engine is down but no different last accepted parameter set is available; not rolling back")
 		return
 	}
 
-	slog.Error("rds-agent: engine did not start after a parameter change; rolled back to the last accepted set",
-		"after", g.after)
+	message := handlers_rds.ParameterRollbackMessage
+	slog.Error("rds-agent: "+message, "after_ms", otelsetup.Millis(g.after))
+	if g.cp != nil {
+		if _, err := g.cp.SubmitState(ctx, g.id, handlers_rds.EngineHealthUnhealthy, message); err != nil {
+			slog.Error("rds-agent: reporting the parameter rollback failed", "err", err)
+		}
+	}
 	if err := g.engine.Restart(ctx); err != nil {
 		slog.Error("rds-agent: restart after the parameter rollback failed", "err", err)
 	}

cmd/rds-agent/paramrecovery_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"os"
-	"path/filepath"
 	"slices"
 	"strings"
 	"testing"
@@ -50,7 +49,7 @@ func TestPostgresEngine_ApplyParametersRollsBackAValueTheEngineRejects(t *testin
 		t.Fatal("ApplyParameters succeeded against a config the engine rejected")
 	}
 
-	installed, err := os.ReadFile(engine.parametersPath())
+	installed, err := os.ReadFile(engine.params.installedPath())
 	if err != nil {
 		t.Fatalf("read the installed parameters: %v", err)
 	}
@@ -69,36 +68,221 @@ func TestPostgresEngine_ApplyParametersWithdrawsTheFirstRejectedSet(t *testing.T
 		[]handlers_rds.Parameter{{Name: "work_mem", Value: "0"}}); err == nil {
 		t.Fatal("ApplyParameters succeeded against a config the engine rejected")
 	}
-	if _, err := os.Stat(engine.parametersPath()); !os.IsNotExist(err) {
+	if _, err := os.Stat(engine.params.installedPath()); !os.IsNotExist(err) {
 		t.Errorf("the rejected include is still installed (stat err = %v)", err)
 	}
 }
 
-// The rollback target has to be a set the engine actually adopted, so a second
-// accepted apply leaves the first one as what a failed restart falls back to.
-func TestPostgresEngine_LastKnownGoodTracksTheAcceptedSet(t *testing.T) {
+type pendingAfterReloadRunner struct {
+	recordingRunner
+
+	pendingReads int
+}
+
+func (r *pendingAfterReloadRunner) run(ctx context.Context, c command) (string, error) {
+	if strings.Contains(c.Stdin, "pending_restart") {
+		r.pendingReads++
+		if r.pendingReads > 1 {
+			r.calls = append(r.calls, c)
+			return "shared_buffers\n", nil
+		}
+	}
+	return r.recordingRunner.run(ctx, c)
+}
+
+// A command can beat the first heartbeat. The apply must preserve the file the
+// current postmaster started with before installing a static replacement.
+func TestPostgresEngine_ApplyBeforeFirstHeartbeatSeedsLastKnownGood(t *testing.T) {
+	runner := &pendingAfterReloadRunner{}
+	engine := newTestEngine(t, runner.run)
+	first := []byte("shared_buffers = '32768'\n")
+	if err := os.WriteFile(engine.params.installedPath(), first, 0o600); err != nil {
+		t.Fatalf("write serving parameter set: %v", err)
+	}
+
+	pending, err := engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{{Name: "shared_buffers", Value: "65536"}})
+	if err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if !slices.Equal(pending, []string{"shared_buffers"}) {
+		t.Errorf("pending = %v, want shared_buffers", pending)
+	}
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
+	if err != nil {
+		t.Fatalf("read last known good: %v", err)
+	}
+	if !slices.Equal(lastGood, first) {
+		t.Errorf("last known good = %q, want the pre-apply set %q", lastGood, first)
+	}
+}
+
+type blockingParameterRunner struct {
+	recordingRunner
+
+	checkStarted  chan struct{}
+	continueCheck chan struct{}
+	pendingReads  int
+}
+
+func (r *blockingParameterRunner) run(ctx context.Context, c command) (string, error) {
+	if slices.Contains(c.Args, "-C") {
+		r.calls = append(r.calls, c)
+		close(r.checkStarted)
+		select {
+		case <-ctx.Done():
+			return "", ctx.Err()
+		case <-r.continueCheck:
+			return "", nil
+		}
+	}
+	if strings.Contains(c.Stdin, "pending_restart") {
+		r.pendingReads++
+		if r.pendingReads > 1 {
+			r.calls = append(r.calls, c)
+			return "shared_buffers\n", nil
+		}
+	}
+	return r.recordingRunner.run(ctx, c)
+}
+
+func TestPostgresEngine_ServingSnapshotWaitsForParameterApply(t *testing.T) {
+	runner := &blockingParameterRunner{
+		checkStarted:  make(chan struct{}),
+		continueCheck: make(chan struct{}),
+	}
+	engine := newTestEngine(t, runner.run)
+	first := []byte("shared_buffers = '32768'\n")
+	if err := os.WriteFile(engine.params.installedPath(), first, 0o600); err != nil {
+		t.Fatalf("write serving parameter set: %v", err)
+	}
+	if err := os.WriteFile(engine.params.lastGoodPath(), first, 0o600); err != nil {
+		t.Fatalf("write last known good: %v", err)
+	}
+
+	applyDone := make(chan error, 1)
+	go func() {
+		_, err := engine.ApplyParameters(t.Context(), []handlers_rds.Parameter{{Name: "shared_buffers", Value: "65536"}})
+		applyDone <- err
+	}()
+	<-runner.checkStarted
+
+	recordDone := make(chan error, 1)
+	go func() {
+		recordDone <- engine.RecordServingParameters(t.Context())
+	}()
+	var earlyRecordErr error
+	recordedEarly := false
+	select {
+	case earlyRecordErr = <-recordDone:
+		recordedEarly = true
+	case <-time.After(10 * time.Millisecond):
+	}
+	close(runner.continueCheck)
+	if err := <-applyDone; err != nil {
+		t.Fatalf("ApplyParameters: %v", err)
+	}
+	if !recordedEarly {
+		earlyRecordErr = <-recordDone
+	}
+	if earlyRecordErr != nil {
+		t.Fatalf("RecordServingParameters: %v", earlyRecordErr)
+	}
+	if recordedEarly {
+		t.Error("serving snapshot completed while the parameter apply held its transaction")
+	}
+
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
+	if err != nil {
+		t.Fatalf("read last known good: %v", err)
+	}
+	if !slices.Equal(lastGood, first) {
+		t.Errorf("last known good = %q, want the set served before the apply %q", lastGood, first)
+	}
+}
+
+// The rollback target moves only when a restart proves the installed static
+// values can serve, not when ApplyParameters merely reloads them.
+func TestPostgresEngine_LastKnownGoodTracksTheServingSet(t *testing.T) {
 	runner := &recordingRunner{}
 	engine := newTestEngine(t, runner.run)
 
-	for _, value := range []string{"4096", "8192"} {
+	apply := func(value string) {
+		t.Helper()
 		if _, err := engine.ApplyParameters(context.Background(),
 			[]handlers_rds.Parameter{{Name: "work_mem", Value: value}}); err != nil {
 			t.Fatalf("ApplyParameters(%s): %v", value, err)
 		}
 	}
+	assertLastGood := func(value string) {
+		t.Helper()
+		lastGood, err := os.ReadFile(engine.params.lastGoodPath())
+		if err != nil {
+			t.Fatalf("read the last known good parameters: %v", err)
+		}
+		if !strings.Contains(string(lastGood), "work_mem = '"+value+"'") {
+			t.Errorf("last known good = %q, want work_mem %s", lastGood, value)
+		}
+	}
 
-	lastGood, err := os.ReadFile(engine.lastGoodPath())
+	apply("4096")
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("RecordServingParameters: %v", err)
+	}
+	apply("8192")
+	assertLastGood("4096")
+
+	// Simulate the restart that adopts 8192, then another apply before the next
+	// restart. The target remains the set the postmaster actually started with.
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("RecordServingParameters after restart: %v", err)
+	}
+	apply("16384")
+	assertLastGood("8192")
+
+	if strings.HasSuffix(engine.params.lastGoodPath(), ".conf") {
+		t.Errorf("the rollback copy at %s would be included by the engine", engine.params.lastGoodPath())
+	}
+}
+
+func TestPostgresEngine_RecordServingParametersSkipsPendingRestart(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestEngine(t, runner.run)
+	first := []byte("shared_buffers = '32768'\n")
+	if err := os.WriteFile(engine.params.installedPath(), first, 0o600); err != nil {
+		t.Fatalf("write first parameter set: %v", err)
+	}
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("record first parameter set: %v", err)
+	}
+
+	second := []byte("shared_buffers = '65536'\n")
+	if err := os.WriteFile(engine.params.installedPath(), second, 0o600); err != nil {
+		t.Fatalf("write pending parameter set: %v", err)
+	}
+	runner.out = "shared_buffers\n"
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("record with a pending restart: %v", err)
+	}
+	lastGood, err := os.ReadFile(engine.params.lastGoodPath())
 	if err != nil {
-		t.Fatalf("read the last known good parameters: %v", err)
+		t.Fatalf("read last known good: %v", err)
 	}
-	if !strings.Contains(string(lastGood), "work_mem = '4096'") {
-		t.Errorf("last known good = %q, want the set the engine ran before the latest apply", lastGood)
+	if !slices.Equal(lastGood, first) {
+		t.Errorf("last known good = %q, want the previously served set %q", lastGood, first)
 	}
 
-	// include_dir globs *.conf, so the rollback copy must not read as a second
-	// set of settings alongside the live one.
-	if strings.HasSuffix(engine.lastGoodPath(), ".conf") {
-		t.Errorf("the rollback copy at %s would be included by the engine", engine.lastGoodPath())
+	// Once a restart has adopted the installed set, pending_restart clears and
+	// the same healthy-heartbeat path may advance the rollback target.
+	runner.out = ""
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("record the restarted parameter set: %v", err)
+	}
+	lastGood, err = os.ReadFile(engine.params.lastGoodPath())
+	if err != nil {
+		t.Fatalf("read advanced last known good: %v", err)
+	}
+	if !slices.Equal(lastGood, second) {
+		t.Errorf("last known good = %q, want the restarted set %q", lastGood, second)
 	}
 }
 
@@ -107,7 +291,7 @@ func TestPostgresEngine_RestoreLastKnownGoodParameters(t *testing.T) {
 	engine := newTestEngine(t, runner.run)
 
 	// Nothing to restore before anything has ever been applied.
-	restored, err := engine.RestoreLastKnownGoodParameters()
+	restored, err := engine.RestoreLastKnownGoodParameters(t.Context())
 	if err != nil {
 		t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
 	}
@@ -115,21 +299,37 @@ func TestPostgresEngine_RestoreLastKnownGoodParameters(t *testing.T) {
 		t.Error("restored a set with no last known good on disk")
 	}
 
-	for _, value := range []string{"4096", "8192"} {
-		if _, err := engine.ApplyParameters(context.Background(),
-			[]handlers_rds.Parameter{{Name: "work_mem", Value: value}}); err != nil {
-			t.Fatalf("ApplyParameters(%s): %v", value, err)
-		}
+	if _, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "work_mem", Value: "4096"}}); err != nil {
+		t.Fatalf("ApplyParameters(4096): %v", err)
+	}
+	if err := engine.RecordServingParameters(t.Context()); err != nil {
+		t.Fatalf("RecordServingParameters: %v", err)
+	}
+	if _, err := engine.ApplyParameters(context.Background(),
+		[]handlers_rds.Parameter{{Name: "work_mem", Value: "8192"}}); err != nil {
+		t.Fatalf("ApplyParameters(8192): %v", err)
 	}
 
-	restored, err = engine.RestoreLastKnownGoodParameters()
+	// A guard whose deadline expired while an apply restarted the engine must
+	// recheck health under the parameter lock rather than reverse that repair.
+	restored, err = engine.RestoreLastKnownGoodParameters(t.Context())
+	if err != nil {
+		t.Fatalf("RestoreLastKnownGoodParameters against serving engine: %v", err)
+	}
+	if restored {
+		t.Fatal("restored the old set while the engine was serving")
+	}
+
+	engine.probe = stubProbe(t, 2, nil)
+	restored, err = engine.RestoreLastKnownGoodParameters(t.Context())
 	if err != nil {
 		t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
 	}
 	if !restored {
 		t.Fatal("did not restore a set that differs from the last known good")
 	}
-	installed, err := os.ReadFile(engine.parametersPath())
+	installed, err := os.ReadFile(engine.params.installedPath())
 	if err != nil {
 		t.Fatalf("read the installed parameters: %v", err)
 	}
@@ -139,7 +339,7 @@ func TestPostgresEngine_RestoreLastKnownGoodParameters(t *testing.T) {
 
 	// Idempotent: the second call finds them already equal and does nothing, so a
 	// repeated rollback cannot churn a cluster that is failing for another reason.
-	restored, err = engine.RestoreLastKnownGoodParameters()
+	restored, err = engine.RestoreLastKnownGoodParameters(t.Context())
 	if err != nil {
 		t.Fatalf("RestoreLastKnownGoodParameters: %v", err)
 	}
@@ -156,7 +356,7 @@ type fakeRecovery struct {
 	restarts   int
 }
 
-func (f *fakeRecovery) RestoreLastKnownGoodParameters() (bool, error) {
+func (f *fakeRecovery) RestoreLastKnownGoodParameters(context.Context) (bool, error) {
 	return f.restored, f.restoreErr
 }
 
@@ -168,14 +368,14 @@ func (f *fakeRecovery) Restart(context.Context) error {
 // A probe whose result the test drives directly, standing in for pg_isready.
 func stubProbe(t *testing.T, code int, err error) *engineProbe {
 	t.Helper()
-	cfg := loadConfig(filepath.Join(t.TempDir(), "absent.env"))
-	return newEngineProbe(cfg, func(context.Context, string, ...string) (int, error) {
-		return code, err
+	cfg := testLoadConfig(t, enginePostgres)
+	return newPostgresProbe(cfg, func(context.Context, string, ...string) (int, string, error) {
+		return code, "", err
 	})
 }
 
 func newTestGuard(engine parameterRecovery, probe *engineProbe) *paramGuard {
-	g := newParamGuard(engine, probe)
+	g := newParamGuard(engine, probe, nil)
 	// Real budgets are minutes, which is the point of them; the behaviour under
 	// test is the decision, not the wait.
 	g.after, g.poll = 20*time.Millisecond, time.Millisecond
@@ -184,13 +384,65 @@ func newTestGuard(engine parameterRecovery, probe *engineProbe) *paramGuard {
 
 // pg_isready exit 2 is an engine that is not there at all, which after a
 // parameter change is the boot-loop shape this breaks.
-func TestParamGuard_RollsBackAndRestartsWhenTheEngineNeverComesUp(t *testing.T) {
+func TestParamGuard_RollsBackReportsAndRestartsWhenTheEngineNeverComesUp(t *testing.T) {
 	engine := &fakeRecovery{restored: true}
-	newTestGuard(engine, stubProbe(t, 2, nil)).Run(t.Context())
+	cp := newFakeControlPlane()
+	guard := newParamGuard(engine, stubProbe(t, 2, nil), cp)
+	guard.id = identity{DBInstanceIdentifier: "db-1"}
+	guard.after, guard.poll = 20*time.Millisecond, time.Millisecond
+	guard.Run(t.Context())
 
 	if engine.restarts != 1 {
 		t.Errorf("restarts = %d, want the engine restarted on the rolled-back set", engine.restarts)
 	}
+	states := cp.snapshotStates()
+	if len(states) != 1 {
+		t.Fatalf("submitted states = %d, want the rollback report", len(states))
+	}
+	if states[0].health != handlers_rds.EngineHealthUnhealthy || states[0].message != handlers_rds.ParameterRollbackMessage {
+		t.Errorf("rollback state = %+v, want the unhealthy rollback report", states[0])
+	}
+}
+
+// PostgreSQL re-derives TLS enforcement from the set it has just put back, so
+// the file can be replaced and the call still fail. The rollback has happened
+// either way, and the worse outcome is an instance left down and unreported.
+func TestParamGuard_ReportsAndRestartsWhenOnlyTheEnforcementFailed(t *testing.T) {
+	engine := &fakeRecovery{restored: true, restoreErr: errors.New("write the TLS enforcement rule: read-only file system")}
+	cp := newFakeControlPlane()
+	guard := newParamGuard(engine, stubProbe(t, 2, nil), cp)
+	guard.id = identity{DBInstanceIdentifier: "db-1"}
+	guard.after, guard.poll = 20*time.Millisecond, time.Millisecond
+	guard.Run(t.Context())
+
+	if engine.restarts != 1 {
+		t.Errorf("restarts = %d, want the engine restarted on the set already put back", engine.restarts)
+	}
+	states := cp.snapshotStates()
+	if len(states) != 1 {
+		t.Fatalf("submitted states = %d, want the rollback still reported", len(states))
+	}
+	if states[0].health != handlers_rds.EngineHealthUnhealthy || states[0].message != handlers_rds.ParameterRollbackMessage {
+		t.Errorf("rollback state = %+v, want the unhealthy rollback report", states[0])
+	}
+}
+
+// The other half of the same contract: a restore that replaced nothing leaves
+// the instance on the set it is already failing under, so restarting it would
+// only churn a cluster that is down for a reason the rollback cannot fix.
+func TestParamGuard_DoesNotRestartWhenTheRestoreReplacedNothing(t *testing.T) {
+	engine := &fakeRecovery{restored: false, restoreErr: errors.New("read the last known good parameters: input/output error")}
+	cp := newFakeControlPlane()
+	guard := newParamGuard(engine, stubProbe(t, 2, nil), cp)
+	guard.after, guard.poll = 20*time.Millisecond, time.Millisecond
+	guard.Run(t.Context())
+
+	if engine.restarts != 0 {
+		t.Errorf("restarts = %d, want none when nothing was rolled back", engine.restarts)
+	}
+	if states := cp.snapshotStates(); len(states) != 0 {
+		t.Errorf("submitted states = %d, want no rollback report", len(states))
+	}
 }
 
 func TestParamGuard_DoesNothingWhenTheEngineIsServing(t *testing.T) {

cmd/rds-agent/quiesce.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"io"
 	"log/slog"
 	"os/exec"
@@ -15,10 +16,9 @@ import (
 	"time"
 )
 
-// PostgreSQL's non-exclusive backup API ties the backup to the session that
-// started it: when that session ends, for any reason, the backup is aborted.
-// That is what makes the hold self-expiring — a control plane that dies mid
-// snapshot cannot leave the engine in backup mode forever.
+// Both engines tie a backup hold to the session that took it: PostgreSQL aborts
+// a non-exclusive backup when its session ends, MariaDB releases a BACKUP STAGE
+// with its connection. That is what makes the hold self-expiring.
 type engineSession interface {
 	// Runs sql and waits for the engine to finish it.
 	Exec(ctx context.Context, sql string) error
@@ -31,7 +31,8 @@ type engineSession interface {
 type sessionRunner func(ctx context.Context, c command) (engineSession, error)
 
 // Written after every statement so the reader knows the engine is done with it.
-// The statements this carries return LSNs, so it cannot appear in a result.
+// The statements this carries return LSNs and backup stages, so it cannot appear
+// in a result.
 const sessionSentinel = "--rds-agent-ready--"
 
 // The backup mode currently held, and the timer that ends it regardless of what
@@ -42,76 +43,66 @@ type quiesceHold struct {
 	expiry  *time.Timer
 }
 
-// Puts the engine into backup mode: the datadir is checkpointed and the engine
-// stops writing over the pages a snapshot is about to read. The hold is released
-// by Unquiesce, or by its own deadline, whichever comes first.
-func (e *postgresEngine) Quiesce(ctx context.Context, label string, hold time.Duration) error {
+// The hold bookkeeping, which is the same for both engines: only the statements
+// that take and release it differ, not the single-hold rule or the deadline that
+// releases it whatever happens to the caller.
+type quiesceState struct {
+	// Guarded because the expiry timer releases the hold on its own goroutine.
+	mu   sync.Mutex
+	held *quiesceHold
+}
+
+// The two things every quiesce needs, checked before a session is opened so a
+// malformed command costs the engine nothing.
+func validateQuiesceRequest(label string, hold time.Duration) error {
 	if label == "" {
 		return errors.New("quiesce requires a backup label")
 	}
 	if hold <= 0 {
 		return errors.New("quiesce requires a positive hold deadline")
 	}
-
-	// Held across the whole start, so a second quiesce waits and then finds the
-	// first one's hold rather than opening a concurrent backup alongside it.
-	e.mu.Lock()
-	defer e.mu.Unlock()
-	if e.held != nil {
-		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)
-	}
-
-	// Deliberately not the command's context: the session has to outlive the call
-	// that started it, and its own deadline is what bounds it instead.
-	session, err := e.startSess(context.WithoutCancel(ctx), command{
-		Name:  e.psql,
-		Args:  e.psqlArgs(),
-		Env:   []string{"PATH=" + defaultGuestPath, "RDS_BACKUP_LABEL=" + label},
-		Stdin: "",
-		User:  e.osUser,
-	})
-	if err != nil {
-		return fmt.Errorf("open a backup session: %w", err)
-	}
-
-	// fast forces an immediate checkpoint rather than spreading it over the
-	// checkpoint interval, which would hold the snapshot open for minutes.
-	const sql = `\getenv label RDS_BACKUP_LABEL
-SELECT pg_backup_start(:'label', fast => true);
-`
-	if err := session.Exec(ctx, sql); err != nil {
-		if closeErr := session.Close(); closeErr != nil {
-			slog.Warn("rds-agent: closing a failed backup session", "err", closeErr)
-		}
-		return fmt.Errorf("put the engine into backup mode: %w", err)
-	}
-
-	e.held = &quiesceHold{
-		label:   label,
-		session: session,
-		expiry:  time.AfterFunc(hold, func() { e.expireQuiesce(label) }),
-	}
-	slog.Info("rds-agent: engine quiesced for backup", "label", label, "hold", hold)
 	return nil
 }
 
-// Ends the backup cleanly. A missing hold is an error rather than a silent
-// success: it means the deadline fired first, so the snapshot the control plane
-// just took was not taken against a held checkpoint.
-func (e *postgresEngine) Unquiesce(ctx context.Context) error {
-	e.mu.Lock()
-	held := e.held
-	e.held = nil
-	e.mu.Unlock()
+// Records the hold and arms its deadline. Called with mu held, so a second
+// quiesce waits and then finds this hold rather than opening a concurrent
+// backup alongside it.
+func (q *quiesceState) beginHoldLocked(label string, session engineSession, hold time.Duration) {
+	q.held = &quiesceHold{
+		label:   label,
+		session: session,
+		expiry:  time.AfterFunc(hold, func() { q.expire(label) }),
+	}
+	slog.Info("rds-agent: engine quiesced for backup", "label", label, "hold_ms", otelsetup.Millis(hold))
+}
 
+// Takes the hold off the engine, if one is still there, and disarms its
+// deadline. The caller ends the session, because the statement that releases the
+// backup has to run on it first.
+func (q *quiesceState) takeHold() *quiesceHold {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	held := q.held
+	q.held = nil
+	if held != nil {
+		held.expiry.Stop()
+	}
+	return held
+}
+
+// Ends the backup cleanly, running releaseSQL on the session that took the hold.
+// A missing hold is an error rather than a silent success: it means the deadline
+// fired first, so what the control plane just snapshotted was not held.
+func (q *quiesceState) releaseHold(ctx context.Context, releaseSQL string) error {
+	held := q.takeHold()
 	if held == nil {
 		return errors.New("the engine is not quiesced; the backup hold had already expired")
 	}
-	held.expiry.Stop()
 
-	// The stop has to run on the session that started the backup, and the session
-	// is closed either way — the engine aborts an unstopped backup with it.
-	execErr := held.session.Exec(ctx, "SELECT pg_backup_stop();\n")
+	// The release has to run on the session that took the hold, and the session is
+	// closed either way — both engines end an unreleased backup with it.
+	execErr := held.session.Exec(ctx, releaseSQL)
 	closeErr := held.session.Close()
 	if execErr != nil {
 		return fmt.Errorf("take the engine out of backup mode: %w", errors.Join(execErr, closeErr))
@@ -123,17 +114,17 @@ func (e *postgresEngine) Unquiesce(ctx context.Context) error {
 	return nil
 }
 
-// The deadline. Ending the session is enough — the engine aborts the backup it
-// was holding — so this never has to reach the engine itself.
-func (e *postgresEngine) expireQuiesce(label string) {
-	e.mu.Lock()
-	held := e.held
+// The deadline. Ending the session is enough — both engines release the hold
+// with it — so this never has to reach the engine itself.
+func (q *quiesceState) expire(label string) {
+	q.mu.Lock()
+	held := q.held
 	if held == nil || held.label != label {
-		e.mu.Unlock()
+		q.mu.Unlock()
 		return
 	}
-	e.held = nil
-	e.mu.Unlock()
+	q.held = nil
+	q.mu.Unlock()
 
 	slog.Warn("rds-agent: backup hold expired; releasing the engine", "label", label)
 	if err := held.session.Close(); err != nil {
@@ -141,21 +132,27 @@ func (e *postgresEngine) expireQuiesce(label string) {
 	}
 }
 
-// A psql child with its stdin held open, so statements can be fed to it one at a
-// time and the engine keeps seeing a single session.
-type psqlSession struct {
+// A client child with its stdin held open, so statements can be fed to it one at
+// a time and the engine keeps seeing a single session.
+type clientSession struct {
 	cmd    *exec.Cmd
 	stdin  io.WriteCloser
 	stdout *bufio.Reader
 	stderr *bytes.Buffer
+	// The statement that makes this client print the sentinel, which is the only
+	// part of a held session that differs between engines.
+	sentinel string
 
 	mu     sync.Mutex
 	closed bool
 }
 
-var _ engineSession = (*psqlSession)(nil)
+var _ engineSession = (*clientSession)(nil)
 
 func execSessionRunner(ctx context.Context, c command) (engineSession, error) {
+	if c.SentinelStatement == "" {
+		return nil, fmt.Errorf("%s was started as a session with no sentinel statement", c.Name)
+	}
 	cmd := exec.CommandContext(ctx, c.Name, c.Args...)
 	cmd.Env = c.Env
 	if c.User != "" {
@@ -180,24 +177,30 @@ func execSessionRunner(ctx context.Context, c command) (engineSession, error) {
 	if err := cmd.Start(); err != nil {
 		return nil, fmt.Errorf("start %s: %w", c.Name, err)
 	}
-	return &psqlSession{cmd: cmd, stdin: stdin, stdout: bufio.NewReader(stdout), stderr: &stderr}, nil
+	return &clientSession{
+		cmd:      cmd,
+		stdin:    stdin,
+		stdout:   bufio.NewReader(stdout),
+		stderr:   &stderr,
+		sentinel: c.SentinelStatement,
+	}, nil
 }
 
-// Feeds sql and waits for the sentinel that follows it. ON_ERROR_STOP makes psql
-// exit on a failed statement, so a sentinel that never arrives is the failure
-// itself and the engine's own message is on stderr.
-func (s *psqlSession) Exec(ctx context.Context, sql string) error {
+// Feeds sql and waits for the sentinel that follows it. Both clients are run in
+// a mode that exits on a failed statement, so a sentinel that never arrives is
+// the failure itself and the engine's own message is on stderr.
+func (s *clientSession) Exec(ctx context.Context, sql string) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.closed {
 		return errors.New("the backup session is closed")
 	}
-	if _, err := io.WriteString(s.stdin, sql+"\\echo "+sessionSentinel+"\n"); err != nil {
+	if _, err := io.WriteString(s.stdin, sql+s.sentinel); err != nil {
 		return fmt.Errorf("%w: %s", err, strings.TrimSpace(s.stderr.String()))
 	}
 
-	// Read on its own goroutine so a psql that never answers costs the caller its
-	// deadline rather than blocking it forever.
+	// Read on its own goroutine so a client that never answers costs the caller
+	// its deadline rather than blocking it forever.
 	done := make(chan error, 1)
 	go func() { done <- s.awaitSentinel() }()
 	select {
@@ -208,7 +211,7 @@ func (s *psqlSession) Exec(ctx context.Context, sql string) error {
 	}
 }
 
-func (s *psqlSession) awaitSentinel() error {
+func (s *clientSession) awaitSentinel() error {
 	for {
 		line, err := s.stdout.ReadString('\n')
 		if strings.TrimSpace(line) == sessionSentinel {
@@ -226,7 +229,7 @@ func (s *psqlSession) awaitSentinel() error {
 // Closing stdin ends the script, and the wait reaps the child. A session that
 // will not end is killed, because leaving it is what would keep the engine in
 // backup mode.
-func (s *psqlSession) Close() error {
+func (s *clientSession) Close() error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	if s.closed {

cmd/rds-agent/quiesce_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"errors"
 	"slices"
 	"strings"
 	"sync"
@@ -113,81 +112,6 @@ func TestPostgresEngine_UnquiesceStopsTheBackupAndEndsTheSession(t *testing.T) {
 	}
 }
 
-// A hold that expired mid-snapshot means the snapshot was not taken against a
-// held checkpoint, so the release reports it rather than succeeding silently.
-func TestPostgresEngine_UnquiesceWithoutAHoldIsAnError(t *testing.T) {
-	engine, _ := newQuiesceEngine(t, &fakeSession{})
-
-	err := engine.Unquiesce(context.Background())
-	if err == nil {
-		t.Fatal("Unquiesce succeeded with no hold to release")
-	}
-	if !strings.Contains(err.Error(), "expired") {
-		t.Errorf("error %q does not say the hold had expired", err)
-	}
-}
-
-// The engine aborts the backup when the session ends, which is what makes the
-// hold self-expiring: a control plane that dies cannot leave it held forever.
-func TestPostgresEngine_QuiesceReleasesItselfOnItsDeadline(t *testing.T) {
-	session := &fakeSession{}
-	engine, _ := newQuiesceEngine(t, session)
-
-	if err := engine.Quiesce(context.Background(), "orders-db-pre-upgrade", 10*time.Millisecond); err != nil {
-		t.Fatalf("Quiesce: %v", err)
-	}
-	deadline := time.Now().Add(2 * time.Second)
-	for session.closes() == 0 && time.Now().Before(deadline) {
-		time.Sleep(time.Millisecond)
-	}
-
-	if session.closes() != 1 {
-		t.Fatal("the backup session outlived its deadline")
-	}
-	// The hold is gone with it, so a later release is honest about that.
-	if err := engine.Unquiesce(context.Background()); err == nil {
-		t.Error("Unquiesce succeeded after the hold expired")
-	}
-}
-
-// One backup at a time: a second start would open a concurrent backup alongside
-// the first rather than extending it.
-func TestPostgresEngine_QuiesceRefusesASecondHold(t *testing.T) {
-	session := &fakeSession{}
-	engine, started := newQuiesceEngine(t, session)
-	if err := engine.Quiesce(context.Background(), "first", time.Minute); err != nil {
-		t.Fatalf("Quiesce: %v", err)
-	}
-
-	err := engine.Quiesce(context.Background(), "second", time.Minute)
-	if err == nil {
-		t.Fatal("a second quiesce was accepted while the first was held")
-	}
-	if !strings.Contains(err.Error(), "first") {
-		t.Errorf("error %q does not name the backup already holding the engine", err)
-	}
-	if len(*started) != 1 {
-		t.Errorf("started %d sessions, want the second refused before opening one", len(*started))
-	}
-}
-
-// A failed start leaves nothing held, so a retry is not refused by a hold that
-// never took effect.
-func TestPostgresEngine_QuiesceEndsTheSessionWhenTheBackupWillNotStart(t *testing.T) {
-	session := &fakeSession{execErr: errors.New(`ERROR:  a backup is already in progress`)}
-	engine, _ := newQuiesceEngine(t, session)
-
-	if err := engine.Quiesce(context.Background(), "orders-db-pre-upgrade", time.Minute); err == nil {
-		t.Fatal("Quiesce succeeded against an engine that refused the backup")
-	}
-	if session.closes() != 1 {
-		t.Errorf("closed %d times, want the failed session ended", session.closes())
-	}
-	if engine.held != nil {
-		t.Error("a failed quiesce left a hold behind")
-	}
-}
-
 func TestCommandRegistry_QuiesceCarriesTheLabelAndDeadline(t *testing.T) {
 	engine := &fakeEngine{}
 	reply := newCommander(nil, newCommandRegistry(engine, &fakeStorage{}), 0).execute(context.Background(),

cmd/rds-agent/register.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"log/slog"
 	"time"
 )
@@ -20,7 +21,7 @@ func (a *Agent) register(ctx context.Context) error {
 		a.hb.setInterval(time.Duration(out.HeartbeatIntervalSeconds) * time.Second)
 
 		slog.Info("rds-agent: registered",
-			"dbInstanceIdentifier", a.id.DBInstanceIdentifier, "heartbeat", a.hb.interval)
+			"dbInstanceIdentifier", a.id.DBInstanceIdentifier, "heartbeat_ms", otelsetup.Millis(a.hb.interval))
 		return nil
 	})
 }

cmd/rds-agent/storage.go

@@ -10,10 +10,9 @@ import (
 	"strings"
 )
 
-// The in-guest half of a storage grow. The control plane has already
-// grown the block device by the time this runs; all that is left is to put the
-// filesystem on the capacity that is now there. Both ext4 and XFS grow while
-// mounted, so this needs no ordering against the engine start.
+// The in-guest half of a storage grow: the control plane has already grown the
+// block device, so all that is left is the filesystem. Both ext4 and XFS grow
+// while mounted, so this needs no ordering against the engine start.
 type storageOps interface {
 	// Extends the filesystem at the data mount onto its whole device, and
 	// reports what it did for the command reply.
@@ -28,8 +27,6 @@ const (
 	defaultXFSGrowfs  = "xfs_growfs"
 	defaultMountsFile = "/proc/mounts"
 	defaultSysBlock   = "/sys/class/block"
-	// Where rds-datadir mounts the data volume; the datadir sits one level in.
-	defaultDataMount = "/var/lib/postgresql"
 	// growpart says this when the partition already fills the disk, which a
 	// resumed grow is the normal way to reach.
 	growpartNoChange = "NOCHANGE"

cmd/rds-agent/storage_test.go

@@ -61,12 +61,12 @@ func newTestStorage(t *testing.T, device, fstype string, run commandRunner) *gue
 	mounts := filepath.Join(dir, "mounts")
 	content := "proc /proc proc rw 0 0\n"
 	if device != "" {
-		content += device + " " + defaultDataMount + " " + fstype + " rw,relatime 0 0\n"
+		content += device + " " + testDataMount + " " + fstype + " rw,relatime 0 0\n"
 	}
 	if err := os.WriteFile(mounts, []byte(content), 0o600); err != nil {
 		t.Fatalf("write the mount table: %v", err)
 	}
-	cfg := config{DataMount: defaultDataMount, MountsFile: mounts, SysBlock: filepath.Join(dir, "block")}
+	cfg := config{DataMount: testDataMount, MountsFile: mounts, SysBlock: filepath.Join(dir, "block")}
 	if err := os.MkdirAll(cfg.SysBlock, 0o755); err != nil {
 		t.Fatalf("create the sysfs tree: %v", err)
 	}
@@ -93,6 +93,9 @@ func seedPartition(t *testing.T, g *guestStorage, disk, name, number string) {
 
 // The data volume is formatted whole, so the usual grow is resize2fs against
 // the device with no partition to push out first.
+// Where rds-datadir mounts the data volume in the PostgreSQL preset.
+const testDataMount = "/var/lib/postgresql"
+
 func TestGrowFilesystem_ResizesAnExt4DeviceOnline(t *testing.T) {
 	log := &runLog{}
 	g := newTestStorage(t, "/dev/vdb", "ext4", log.run)
@@ -108,7 +111,7 @@ func TestGrowFilesystem_ResizesAnExt4DeviceOnline(t *testing.T) {
 	if args := log.calls[0].Args; len(args) != 1 || args[0] != "/dev/vdb" {
 		t.Errorf("resize2fs args = %v, want the device", args)
 	}
-	if !strings.Contains(message, "/dev/vdb") || !strings.Contains(message, defaultDataMount) {
+	if !strings.Contains(message, "/dev/vdb") || !strings.Contains(message, testDataMount) {
 		t.Errorf("reply message = %q, want the device and mount it grew", message)
 	}
 }
@@ -125,7 +128,7 @@ func TestGrowFilesystem_GrowsXFSByMountPoint(t *testing.T) {
 	if got := log.names(); len(got) != 1 || got[0] != defaultXFSGrowfs {
 		t.Fatalf("ran %v, want just %s", got, defaultXFSGrowfs)
 	}
-	if args := log.calls[0].Args; len(args) != 1 || args[0] != defaultDataMount {
+	if args := log.calls[0].Args; len(args) != 1 || args[0] != testDataMount {
 		t.Errorf("xfs_growfs args = %v, want the mount point", args)
 	}
 }

cmd/spinifex/cmd/admin.go

@@ -25,23 +25,23 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"uuid"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/iam"
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/ebsmetadata"
+	"github.com/mulgadc/spinifex/spinifex/ebsprovider"
 	"github.com/mulgadc/spinifex/spinifex/formation"
 	"github.com/mulgadc/spinifex/spinifex/gpu"
+	handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot"
 	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
 	"github.com/mulgadc/spinifex/spinifex/hostdns"
 	"github.com/mulgadc/spinifex/spinifex/network/host"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/utils"
-	"github.com/mulgadc/viperblock/viperblock"
-	"github.com/mulgadc/viperblock/viperblock/backends/s3"
-	"github.com/mulgadc/viperblock/viperblock/v_utils"
 	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/pterm/pterm"
 	"github.com/spf13/cobra"
 )
@@ -118,8 +118,9 @@ var nodeDrainCmd = &cobra.Command{
 its guests via QMP and unmount their volumes (flushing the viperblock WAL) while
 every service is still running. STORAGE/PERSIST/INFRA are left to systemd's
 ordered unit teardown. This is what spinifex-shutdown.service's ExecStop runs,
-with --only-if-host-stopping so it only drains on a genuine host
-shutdown/reboot -- not on a plain "systemctl restart/stop spinifex.target".
+with --unless-restarting so it drains on a genuine host shutdown/reboot and on
+a plain "systemctl stop spinifex.target", but skips the drain when a restart
+of the stack (e.g. "systemctl restart spinifex.target") is already queued.
 Run by hand without that flag, it always drains unconditionally.`,
 	Run: runNodeDrainLocal,
 }
@@ -197,6 +198,25 @@ takes effect immediately. Prompts for confirmation unless --yes is passed.`,
 	Run: runimagesPromoteCmd,
 }
 
+var volumesCmd = &cobra.Command{
+	Use:   "volumes",
+	Short: "Inspect block storage volumes",
+	Long:  `Inspect the volumes the storage provider holds, independently of the EC2 API.`,
+}
+
+var volumesOrphansCmd = &cobra.Command{
+	Use:   "orphans",
+	Short: "Report volumes the provider holds with no control-plane record",
+	Long: `List volumes the storage provider holds that the control plane has no record
+of. Their blocks are consuming space but they have no API handle, so they
+cannot be described or deleted through the EC2 API.
+
+This command only reports. It never deletes: the evidence that an orphan is
+still wanted is the record that went missing, so removal is an operator
+decision made per volume.`,
+	Run: runVolumesOrphansCmd,
+}
+
 var accountCmd = &cobra.Command{
 	Use:   "account",
 	Short: "Manage Spinifex accounts",
@@ -283,7 +303,9 @@ func init() {
 	nodeCmd.AddCommand(nodeDrainCmd)
 	nodeDrainCmd.Flags().Bool("local", false, "Drain the local node only (required)")
 	nodeDrainCmd.Flags().Duration("timeout", 120*time.Second, "Maximum time to wait per phase")
-	nodeDrainCmd.Flags().Bool("only-if-host-stopping", false, "Skip the drain unless systemd is unwinding into shutdown.target (real reboot/poweroff); unset, always drains")
+	nodeDrainCmd.Flags().Bool("unless-restarting", false, "Drain on a real host shutdown or a plain target stop; skip only when a restart of the stack is already queued. Unset, always drains")
+	nodeDrainCmd.Flags().Bool("only-if-host-stopping", false, "Deprecated: use --unless-restarting")
+	_ = nodeDrainCmd.Flags().MarkDeprecated("only-if-host-stopping", "use --unless-restarting instead")
 	nodeCmd.AddCommand(nodeJSProbeCmd)
 	nodeJSProbeCmd.Flags().Duration("timeout", 10*time.Second, "Maximum time to wait for the canary round-trip")
 
@@ -293,6 +315,9 @@ func init() {
 	imagesCmd.AddCommand(imagesRemoveCmd)
 	imagesCmd.AddCommand(imagesPromoteCmd)
 
+	adminCmd.AddCommand(volumesCmd)
+	volumesCmd.AddCommand(volumesOrphansCmd)
+
 	adminCmd.AddCommand(accountCmd)
 	accountCmd.AddCommand(accountCreateCmd)
 	accountCmd.AddCommand(accountListCmd)
@@ -305,6 +330,9 @@ func init() {
 
 	adminCmd.AddCommand(upgradeCmd)
 	upgradeCmd.Flags().Bool("yes", false, "Apply migrations without prompting")
+	upgradeCmd.Flags().Bool("dry-run", false, "Report pending config and unit changes without applying them")
+	upgradeCmd.Flags().Bool("units-only", false, "Reconcile systemd units only, skip config migrations")
+	upgradeCmd.Flags().Bool("skip-units", false, "Apply config migrations only, skip systemd unit reconciliation")
 	certRenewCmd.Flags().StringSlice("extra-ip", nil, "Additional IP addresses to include in SANs")
 	certRenewCmd.Flags().StringSlice("extra-dns", nil, "Additional DNS names to include in SANs")
 	accountCreateCmd.Flags().String("name", "", "Account name (required)")
@@ -387,19 +415,19 @@ func init() {
 	imagesRemoveCmd.Flags().String("image-id", "", "AMI ID to remove (required)")
 	imagesRemoveCmd.Flags().Bool("force", false, "Bypass dependency, ownership and config-corrupt checks (salvage mode)")
 	imagesRemoveCmd.Flags().Bool("yes", false, "Skip interactive confirmation prompt")
-	if err := imagesRemoveCmd.MarkFlagRequired("image-id"); err != nil {
-		panic(err)
-	}
+	_ = imagesRemoveCmd.MarkFlagRequired("image-id")
 
 	imagesPromoteCmd.Flags().String("image-id", "", "AMI ID to promote to system image (required)")
 	imagesPromoteCmd.Flags().Bool("yes", false, "Skip interactive confirmation prompt")
-	if err := imagesPromoteCmd.MarkFlagRequired("image-id"); err != nil {
-		panic(err)
-	}
+	_ = imagesPromoteCmd.MarkFlagRequired("image-id")
 }
 
 const bytesPerGiB = 1024 * 1024 * 1024
 
+// imageImportTimeout bounds each provider call the import makes. Generous
+// because the snapshot at the end runs over the whole imported volume.
+const imageImportTimeout = 30 * time.Minute
+
 // amiVolumeSizeGiB returns the smallest whole GiB that still holds sizeBytes.
 //
 // Rounding up is load-bearing. The image is copied into a root volume of
@@ -427,16 +455,10 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	var imageStat os.FileInfo
 	var err error
 
-	cfgFile, _ := cmd.Flags().GetString("config")
 	forceCmd, _ := cmd.Flags().GetBool("force")
 	skipVerify, _ := cmd.Flags().GetBool("skip-verify")
 	ostmpDir, _ := cmd.Flags().GetString("tmp-dir")
 
-	// Use default config path
-	if cfgFile == "" {
-		cfgFile = DefaultConfigFile()
-	}
-
 	//configDir, _ := cmd.Flags().GetString("config-dir")
 	baseDir, _ := cmd.Flags().GetString("spinifex-dir")
 
@@ -543,9 +565,11 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	// Create the specified image directory
 	imagePath := fmt.Sprintf("%s/%s/%s/%s", imageDir, image.Distro, image.Version, image.Arch)
 
-	// Create config directory
-	if err := os.MkdirAll(imagePath, 0700); err != nil {
-		fmt.Fprintf(os.Stderr, "Error creating config directory: %v\n", err)
+	// 0770, matching the root:spinifex convention setup.sh gives the shared data
+	// dirs. At 0700 the first sudo import leaves a directory no group member can
+	// descend into, so every later import without sudo fails on mkdir.
+	if err := os.MkdirAll(imagePath, 0770); err != nil {
+		fmt.Fprintf(os.Stderr, "Error creating image directory %s: %v\n", imagePath, err)
 		os.Exit(1)
 	}
 
@@ -607,57 +631,46 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	// Create the specified manifest file to describe the image/AMI
-	manifest := viperblock.VolumeConfig{}
-
-	// Calculate the size
-
-	manifest.AMIMetadata.Name = fmt.Sprintf("ami-%s-%s-%s", image.Distro, image.Version, image.Arch)
-	if amiNameOverride != "" {
-		manifest.AMIMetadata.Name = amiNameOverride
-	}
+	// Describe the image/AMI as the control-plane document it will become.
 	volumeId := utils.GenerateResourceID("ami")
-	manifest.AMIMetadata.ImageID = volumeId
+	amiName := fmt.Sprintf("ami-%s-%s-%s", image.Distro, image.Version, image.Arch)
+	if amiNameOverride != "" {
+		amiName = amiNameOverride
+	}
 
-	manifest.AMIMetadata.Description = fmt.Sprintf("%s cloud image prepared for Spinifex", manifest.AMIMetadata.Name)
-	manifest.AMIMetadata.Architecture = image.Arch
-	manifest.AMIMetadata.PlatformDetails = image.Platform
-	manifest.AMIMetadata.CreationDate = time.Now()
-	manifest.AMIMetadata.RootDeviceType = "ebs"
-	manifest.AMIMetadata.Virtualization = "hvm"
-	manifest.AMIMetadata.ImageOwnerAlias = "system"
-	manifest.AMIMetadata.VolumeSizeGiB = amiVolumeSizeGiB(imageStat.Size())
-	manifest.AMIMetadata.BootMode = image.BootMode
-	manifest.AMIMetadata.Distro = image.Distro
-	manifest.AMIMetadata.DistroFamily = utils.DistroFamily(image.Distro)
+	ami := ebsmetadata.AMI{
+		ImageID:         volumeId,
+		Name:            amiName,
+		Description:     fmt.Sprintf("%s cloud image prepared for Spinifex", amiName),
+		Architecture:    image.Arch,
+		PlatformDetails: image.Platform,
+		CreationDate:    time.Now(),
+		RootDeviceType:  "ebs",
+		Virtualization:  "hvm",
+		ImageOwnerAlias: "system",
+		VolumeSizeGiB:   amiVolumeSizeGiB(imageStat.Size()),
+		SnapshotID:      admin.SnapPrefix(volumeId),
+		BootMode:        image.BootMode,
+		Distro:          image.Distro,
+		DistroFamily:    utils.DistroFamily(image.Distro),
+	}
 
 	// Copy catalog-provided tags (e.g. spinifex:managed-by for system AMIs)
 	// onto the imported AMI so the UI can filter them out.
 	if len(image.Tags) > 0 {
-		manifest.AMIMetadata.Tags = make(map[string]string, len(image.Tags))
-		maps.Copy(manifest.AMIMetadata.Tags, image.Tags)
+		ami.Tags = make(map[string]string, len(image.Tags))
+		maps.Copy(ami.Tags, image.Tags)
 	}
 
-	// Volume Data
-	manifest.VolumeMetadata.VolumeID = volumeId // TODO: Confirm if unique, e.g vol-, if ami- used
-	manifest.VolumeMetadata.VolumeName = manifest.AMIMetadata.Name
-	manifest.VolumeMetadata.TenantID = "system"
-	manifest.VolumeMetadata.SizeGiB = manifest.AMIMetadata.VolumeSizeGiB
-	manifest.VolumeMetadata.State = "available"
-	manifest.VolumeMetadata.AvailabilityZone = "" // TODO: Confirm
-	manifest.VolumeMetadata.CreatedAt = time.Now()
-	manifest.VolumeMetadata.VolumeType = "gp3"
-	manifest.VolumeMetadata.IOPS = 1000
-
 	// Write the manifest to disk
 	// Save as JSON
-	jsonData, err := json.Marshal(manifest)
+	jsonData, err := json.Marshal(ami)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Could not marshal manifest: %v\n", err)
 		os.Exit(1)
 	}
 
-	manifestFilename := fmt.Sprintf("%s/%s.json", imagePath, manifest.AMIMetadata.Name)
+	manifestFilename := fmt.Sprintf("%s/%s.json", imagePath, ami.Name)
 	// Write to file
 	err = os.WriteFile(manifestFilename, jsonData, 0600)
 	if err != nil {
@@ -665,145 +678,118 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	// Upload the image to S3 (predastore)
-
-	appConfig, err := config.LoadConfig(cfgFile)
-
-	if err != nil {
-		fmt.Println("Error loading config file:", err)
-		return
-	}
-
-	s3Config := s3.S3Config{
-		VolumeName: volumeId,
-		VolumeSize: utils.SafeInt64ToUint64(imageStat.Size()),
-		Bucket:     appConfig.Nodes[appConfig.Node].Predastore.Bucket,
-		Region:     appConfig.Nodes[appConfig.Node].Predastore.Region,
-		AccessKey:  appConfig.Nodes[appConfig.Node].Predastore.AccessKey,
-		SecretKey:  appConfig.Nodes[appConfig.Node].Predastore.SecretKey,
-		Host:       appConfig.Nodes[appConfig.Node].Predastore.Host,
-	}
-
-	mkey, err := utils.LoadViperblockMasterKey(appConfig.Nodes[appConfig.Node].Viperblock.EncryptionKeyFile)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not load viperblock encryption key: %v\n", err)
-		os.Exit(1)
-	}
-
-	// Import is an interactive command whose output is a progress bar, so the
-	// volume's routine mount chatter is noise here: a fresh import has no prior
-	// state, making the "no state found" / 404 lines expected rather than
-	// notable. Scoping the logger to this VB (New copies it onto the backend
-	// too) keeps it off the process-wide default. Errors still surface, both
-	// through this logger and as the returned error the caller prints.
-	vbConfig := viperblock.VB{
-		VolumeName: volumeId,
-		VolumeSize: utils.SafeInt64ToUint64(imageStat.Size()),
-		BaseDir:    tmpDir,
-		Cache: viperblock.Cache{
-			Config: viperblock.CacheConfig{
-				Size: 0,
-			},
-		},
-		VolumeConfig:      manifest,
-		MasterKey:         mkey,
-		EncryptionEnabled: mkey != nil,
-		Logger:            slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError})),
-	}
-
-	// Bake the deployment CA into the image trust store so a stock cloud image
-	// trusts the gateway from first boot. Best-effort: never fails the import.
-	// The CA path derives from the --spinifex-dir data root (its config/ symlink
-	// holds ca.pem), not NodeBaseDir(): node.BaseDir is unset in the import
-	// command, which would collapse the path to a relative config/ca.pem.
-	bakeCACertIntoImage(extractedImagePath, filepath.Join(baseDir, "config", "ca.pem"))
-
-	// Render the flush bar here rather than inside viperblock, which stays a
-	// pure storage library. The bar is built lazily on the first update so it is
-	// never drawn if the import fails before any bytes flush; viperblock
-	// throttles the callback to ≤101 invocations, so the human-readable title
-	// renders without the per-block render-frequency regression.
-	var flushBar *pterm.ProgressbarPrinter
-	var flushUpdate func(current uint64)
-	progress := func(current, total uint64) {
-		if flushBar == nil {
-			flushBar, flushUpdate = utils.NewByteProgressBar("Flushing image to storage", total)
-		}
-		flushUpdate(current)
-	}
-
-	err = v_utils.ImportDiskImage(&s3Config, &vbConfig, extractedImagePath, progress)
-
-	if flushBar != nil {
-		_, _ = flushBar.Stop()
-	}
-
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Could not import image to predastore: %v\n", err)
-		os.Exit(1)
-	}
-
 	defer os.RemoveAll(tmpDir)
 
+	appConfig, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not connect to the cluster: %v\n", err)
+		os.Exit(1)
+	}
+	defer nc.Close()
+	node := appConfig.Nodes[appConfig.Node]
+
+	fmt.Println("Writing image to storage ...")
+	provider := ebsprovider.NewNATSProvider(nc, imageImportTimeout)
+	err = admin.ImportImage(context.Background(), provider, admin.ImportOpts{
+		VolumeID:         volumeId,
+		NodeID:           appConfig.Node,
+		SizeBytes:        utils.SafeUint64ToInt64(ami.VolumeSizeGiB * bytesPerGiB),
+		AvailabilityZone: node.AZ,
+		SourcePath:       extractedImagePath,
+		Snapshot:         true,
+		Progress:         os.Stdout,
+	})
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not import image: %v\n", err)
+		os.Exit(1)
+	}
+
+	// admin.ImportImage only wrote the provider's half of the snapshot (its
+	// blocks); this writes the EC2 control plane's, without which
+	// DescribeSnapshots and GetAMISourceVolumeID cannot resolve the import.
+	metaStore := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
+	if err := registerImportedAMISnapshot(metaStore, node.Predastore.Bucket, ami, node.AZ, node.Viperblock.EncryptionKeyFile != ""); err != nil {
+		fmt.Fprintf(os.Stderr, "Imported %s but could not register its snapshot: %v\n", volumeId, err)
+		os.Exit(1)
+	}
+
+	// The document is what DescribeImages enumerates, so it is written last:
+	// until it exists the AMI is not launchable, and an import that failed
+	// partway leaves no half-registered image behind.
+	ami.State = "available"
+	if err := ebsmetadata.NewStore(metaStore, node.Predastore.Bucket).PutAMI(context.Background(), ami); err != nil {
+		fmt.Fprintf(os.Stderr, "Imported %s but could not register it: %v\n", volumeId, err)
+		os.Exit(1)
+	}
+
 	fmt.Printf("✅ Image import complete. Image-ID (AMI): %s\n", volumeId)
 }
 
-// caBakeRunCommand installs the uploaded CA into the guest trust store across
-// the distro families, then removes the staged copy. Each branch is gated on its
-// updater existing so only the matching distro's branch writes anything (install
-// -D creates the anchor dir on minimal images). Debian/Ubuntu use
-// update-ca-certificates, RHEL/Rocky use update-ca-trust, and stock Alpine ships
-// only ca-certificates-bundle (no updater, no anchor dir) so it falls back to
-// appending the PEM to the static bundle. The install chain's exit code is
-// preserved past the cleanup rm so a failure surfaces instead of a false success.
-// The RHEL branch restorecon's only the trust-store paths it touched so the cert
-// is correctly labelled without the image-wide --no-selinux-relabel (below)
-// leaving an unlabelled bundle; restorecon is best-effort and never fails the bake.
-const caBakeRunCommand = `( { command -v update-ca-certificates >/dev/null && install -D -m644 /tmp/spinifex-ca.pem /usr/local/share/ca-certificates/spinifex.crt && update-ca-certificates; } || ` +
-	`{ command -v update-ca-trust >/dev/null && install -D -m644 /tmp/spinifex-ca.pem /etc/pki/ca-trust/source/anchors/spinifex.crt && update-ca-trust && ` +
-	`{ command -v restorecon >/dev/null 2>&1 && restorecon -RF /etc/pki/ca-trust/source/anchors/spinifex.crt /etc/pki/ca-trust/extracted >/dev/null 2>&1; true; }; } || ` +
-	`{ cat /tmp/spinifex-ca.pem >> /etc/ssl/certs/ca-certificates.crt; } ); ` +
-	`rc=$?; rm -f /tmp/spinifex-ca.pem; exit $rc`
-
-// caBakeTimeout bounds the virt-customize run so a stalled libguestfs appliance
-// cannot hang the import indefinitely; on timeout the bake degrades to a skip.
-const caBakeTimeout = 5 * time.Minute
-
-// caBakeCmd builds the virt-customize invocation that uploads the deployment CA
-// into the disk image at imagePath and installs it into the guest trust store.
-// --no-selinux-relabel stops virt-customize flagging the image for a first-boot
-// SELinux autorelabel: that relabel+reboot corrupts XFS roots (RHEL/Rocky) on the
-// reboot, so the run-command relabels only the touched trust paths instead.
-func caBakeCmd(ctx context.Context, imagePath, caCertPath string) *exec.Cmd {
-	return exec.CommandContext(ctx, "virt-customize", "-a", imagePath,
-		"--no-selinux-relabel",
-		"--upload", caCertPath+":/tmp/spinifex-ca.pem",
-		"--run-command", caBakeRunCommand)
+// registerImportedAMISnapshot writes the EC2 control plane's snapshot document
+// for a catalog-imported AMI. The provider-backed import path only writes the
+// storage backend's blocks under ami.SnapshotID, so without this a launch or
+// DescribeSnapshots call cannot resolve it. The volume ID matches ami.ImageID:
+// admin.ImportImage creates the volume under the AMI's own ID.
+func registerImportedAMISnapshot(store objectstore.ObjectStore, bucket string, ami ebsmetadata.AMI, az string, encrypted bool) error {
+	cfg := &handlers_ec2_snapshot.SnapshotConfig{
+		SnapshotID:       ami.SnapshotID,
+		VolumeID:         ami.ImageID,
+		VolumeSize:       utils.SafeUint64ToInt64(ami.VolumeSizeGiB),
+		State:            "completed",
+		Progress:         "100%",
+		StartTime:        time.Now(),
+		Description:      fmt.Sprintf("Imported AMI volume for %s", ami.Name),
+		Encrypted:        encrypted,
+		OwnerID:          utils.GlobalAccountID,
+		AvailabilityZone: az,
+	}
+	if err := handlers_ec2_snapshot.WriteSnapshotConfig(store, bucket, ami.SnapshotID, cfg); err != nil {
+		return fmt.Errorf("register snapshot metadata: %w", err)
+	}
+	return nil
 }
 
-// caBakeRunner resolves virt-customize and runs the CA bake; overridable in tests.
-var caBakeRunner = func(imagePath, caCertPath string) ([]byte, error) {
-	if _, err := exec.LookPath("virt-customize"); err != nil {
-		return nil, fmt.Errorf("virt-customize not found: %w", err)
+func runVolumesOrphansCmd(_ *cobra.Command, _ []string) {
+	appConfig, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not connect to the cluster: %v\n", err)
+		os.Exit(1)
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), caBakeTimeout)
-	defer cancel()
-	return caBakeCmd(ctx, imagePath, caCertPath).CombinedOutput()
-}
+	defer nc.Close()
 
-// bakeCACertIntoImage uploads the deployment CA into the image's trust store via
-// virt-customize so an imported stock image trusts the gateway from first boot.
-// Best-effort: a missing CA, an absent virt-customize, or an image libguestfs
-// cannot inspect logs and continues — image import never fails on the CA.
-func bakeCACertIntoImage(imagePath, caCertPath string) {
-	if _, err := os.Stat(caCertPath); err != nil {
-		slog.Warn("CA bake skipped: deployment CA not found; imported image will not auto-trust the gateway", "ca", caCertPath, "err", err)
+	node := appConfig.Nodes[appConfig.Node]
+	store := objectstore.NewS3ObjectStoreFromConfig(
+		node.Predastore.Host,
+		node.Predastore.Region,
+		node.Predastore.AccessKey,
+		node.Predastore.SecretKey,
+	)
+
+	orphans, err := admin.FindOrphanVolumes(
+		context.Background(),
+		ebsprovider.NewNATSProvider(nc, imageImportTimeout),
+		ebsmetadata.NewStore(store, node.Predastore.Bucket),
+	)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Orphan scan failed:", err)
+		os.Exit(1)
+	}
+	if len(orphans) == 0 {
+		fmt.Println("No orphaned volumes: every volume the provider holds has a control-plane record.")
 		return
 	}
-	if out, err := caBakeRunner(imagePath, caCertPath); err != nil {
-		slog.Warn("CA bake skipped: virt-customize could not customize image; imported image will not auto-trust the gateway", "err", err, "output", string(out))
-		return
+
+	fmt.Printf("%d orphaned volume(s) — held by the provider, unknown to the control plane:\n\n", len(orphans))
+	for _, orphan := range orphans {
+		fmt.Printf("  %s\n", orphan.VolumeID)
+		fmt.Printf("    handle:  %s\n", orphan.Handle)
+		if orphan.Derived {
+			fmt.Println("    note:    derived volume; its base volume is also unknown")
+		}
 	}
+	fmt.Println()
+	fmt.Println("Nothing was deleted. Each of these holds data that cannot be reached")
+	fmt.Println("through the EC2 API; removing one is a per-volume operator decision.")
 }
 
 func runimagesRemoveCmd(cmd *cobra.Command, args []string) {
@@ -1342,18 +1328,32 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	if masterKeyExisted {
 		// Preserve path: reuse the existing identity. The system credentials must
 		// match what seeded the NATS KV `system` secret, so load them rather than
-		// mint new ones. The admin credentials are not recovered: bootstrap.json is
-		// consumed and deleted by awsgw after first boot, and the operator's copy
-		// already lives in ~/.aws/credentials. Leaving them empty makes
-		// finalizeNodeSetup refresh only ~/.aws/config (endpoint/CA for a changed
-		// bind IP), not the credentials.
+		// mint new ones.
 		accessKey, secretKey, err = loadSystemCredentials(configDir)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error loading preserved system credentials: %v\n", err)
 			os.Exit(1)
 		}
-		fmt.Println("\n🔐 Preserved existing identity (master key, credentials and CA unchanged)")
+
+		// The admin pair is minted afresh rather than preserved, because it cannot
+		// be read back: bootstrap.json is consumed and deleted by awsgw on first
+		// start. The half of the identity that survives a re-init is on local disk,
+		// but the IAM store lives in NATS and predastore — so forming a cluster out
+		// of already-initialized nodes rebuilds it and the old access key stops
+		// existing. Carrying an empty pair through instead is worse than useless:
+		// it reaches every joiner's bootstrap.json, where seeding rejects the empty
+		// key ID and awsgw fails to start on every restart, forever.
+		bootstrapResult, err := writeBootstrapFiles(configDir, bootstrapDir, masterKey, accessKey, secretKey, accountID)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error writing bootstrap files: %v\n", err)
+			os.Exit(1)
+		}
+		adminAccessKey = bootstrapResult.AdminAccessKey
+		adminSecretKey = bootstrapResult.AdminSecretKey
+
+		fmt.Println("\n🔐 Preserved existing identity (master key and CA unchanged)")
 		fmt.Printf("   Master key: %s\n", filepath.Join(configDir, "master.key"))
+		fmt.Printf("   Admin credentials reissued in %s\n", filepath.Join(bootstrapDir, "bootstrap.json"))
 	} else {
 		// Fresh install: mint system + admin credentials and seed the bootstrap files.
 		accessKey, err = admin.GenerateAWSAccessKey()
@@ -1405,9 +1405,10 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	fmt.Printf("   Key: %s\n", viperblockKeyPath)
 
 	if !masterKeyExisted {
-		fmt.Printf("\n🔑 Generated admin credentials (save these — they won't be shown again):\n")
-		fmt.Printf("   Access Key:  %s\n", adminAccessKey)
-		fmt.Printf("   Secret Key:  %s\n", adminSecretKey)
+		// Never echo the keys: on an ISO install stdout is the systemd journal,
+		// which persists them for the life of the node. They are recoverable from
+		// bootstrap.json and ~/.aws/credentials.
+		fmt.Printf("\n🔑 Generated admin credentials (written to ~/.aws/credentials)\n")
 		fmt.Printf("   Account:     %s (%s)\n", admin.DefaultAccountName(), admin.DefaultAccountID())
 		fmt.Printf("   AWS Profile: spinifex\n")
 	}
@@ -1523,8 +1524,10 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	// is node-local, so it is derived here from this node's own config dirs.
 	northstarConfigPath := filepath.Join(dirs.Northstar, "northstar.toml")
 
-	// Parse multi-node predastore configuration (legacy flag-based approach for single-node)
-	var predastoreNodeID int
+	// Parse multi-node predastore configuration. A single-node install is host
+	// 1, the only [[host]] its predastore.toml declares; the multi-node path
+	// below replaces this with the host matching this machine.
+	predastoreHostID := 1
 	if predastoreNodesStr != "" {
 		ips := strings.Split(predastoreNodesStr, ",")
 		if len(ips) < 2 {
@@ -1545,14 +1548,16 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 			})
 		}
 
-		predastoreNodeID = admin.FindNodeIDByIP(predastoreNodes, bindIP)
-		if predastoreNodeID == 0 {
+		// One machine is one predastore host, so the node list index that
+		// matches this bind IP is this node's host ID.
+		predastoreHostID = admin.FindNodeIDByIP(predastoreNodes, bindIP)
+		if predastoreHostID == 0 {
 			fmt.Fprintf(os.Stderr, "❌ Error: --bind IP %s not found in --predastore-nodes list\n", bindIP)
 			os.Exit(1)
 		}
 
 		// Generate multi-node predastore.toml
-		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP, compactionInterval, northstarCreds)
+		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, spxRoot, bindIP, compactionInterval, northstarCreds)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
 			os.Exit(1)
@@ -1563,7 +1568,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 			fmt.Fprintf(os.Stderr, "Error writing predastore config: %v\n", err)
 			os.Exit(1)
 		}
-		fmt.Printf("✅ Created: multi-node predastore.toml (node ID: %d)\n", predastoreNodeID)
+		fmt.Printf("✅ Created: multi-node predastore.toml (host ID: %d)\n", predastoreHostID)
 	}
 
 	// Pre-generate default VPC/subnet/IGW IDs for bootstrap config.
@@ -1593,7 +1598,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		ClusterRoutes: clusterRoutes,
 		ClusterName:   clusterName,
 
-		PredastoreNodeID:          predastoreNodeID,
+		PredastoreHostID:          predastoreHostID,
 		CompactionIntervalSeconds: compactionInterval,
 		Services:                  services,
 
@@ -1803,6 +1808,10 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		os.Exit(1)
 	}
 
+	// The host firewall scopes this port to known peers, and a node joining is
+	// not one yet. Closed again on every exit below.
+	closeFormationPort := openFormationPort(port)
+
 	fmt.Printf("\n📡 Formation server started on %s\n", formationAddr)
 	fmt.Printf("   Waiting for %d more node(s) to join...\n", expectedNodes-1)
 	fmt.Printf("   Token expires in %s\n\n", tokenTTL)
@@ -1813,6 +1822,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	if err := fs.WaitForCompletion(formationTimeout); err != nil {
 		fmt.Fprintf(os.Stderr, "❌ %v\n", err)
 		fs.Shutdown(context.Background())
+		closeFormationPort()
 		os.Remove(tokenPath)
 		os.Exit(1)
 	}
@@ -1839,11 +1849,13 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 	// is node-local, so it is derived here from this node's own config dirs.
 	northstarConfigPath := filepath.Join(dirs.Northstar, "northstar.toml")
 
-	// Generate multi-node predastore config
-	var predastoreNodeID int
+	// Generate multi-node predastore config. A single-node install is host 1,
+	// the only [[host]] its predastore.toml declares; the multi-node path below
+	// replaces this with the host matching this machine.
+	predastoreHostID := 1
 	hasPredastoreConfig := len(predastoreNodes) >= 2
 	if hasPredastoreConfig {
-		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP, compactionInterval, northstarCreds)
+		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, spxRoot, bindIP, compactionInterval, northstarCreds)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
 			os.Exit(1)
@@ -1855,8 +1867,15 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 			os.Exit(1)
 		}
 
-		predastoreNodeID = admin.FindNodeIDByIP(predastoreNodes, bindIP)
-		fmt.Printf("✅ Created: multi-node predastore.toml (node ID: %d)\n", predastoreNodeID)
+		// One machine is one predastore host, so the node list entry matching
+		// this bind IP is this node's host ID. Without one the service has no
+		// [[host]] to run and refuses to start.
+		predastoreHostID = admin.FindNodeIDByIP(predastoreNodes, bindIP)
+		if predastoreHostID == 0 {
+			fmt.Fprintf(os.Stderr, "❌ Error: bind IP %s not found in the predastore node list\n", bindIP)
+			os.Exit(1)
+		}
+		fmt.Printf("✅ Created: multi-node predastore.toml (host ID: %d)\n", predastoreHostID)
 	}
 
 	spinifexTomlPath := filepath.Join(configDir, "spinifex.toml")
@@ -1880,7 +1899,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		ClusterRoutes: clusterRoutes,
 		ClusterName:   clusterName,
 
-		PredastoreNodeID:          predastoreNodeID,
+		PredastoreHostID:          predastoreHostID,
 		CompactionIntervalSeconds: compactionInterval,
 		Services:                  services,
 		RemoteNodes:               buildRemoteNodes(allNodes, node, northstarConfigPath),
@@ -1922,6 +1941,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 
 	// Shutdown formation server
 	fs.Shutdown(context.Background())
+	closeFormationPort()
 	if err := os.Remove(tokenPath); err != nil && !errors.Is(err, os.ErrNotExist) {
 		slog.Warn("Failed to remove join token file", "path", tokenPath, "error", err)
 	}
@@ -2389,12 +2409,14 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 
 	northstarCreds, northstarConfigPath := northstarFromFormation(creds, dirs)
 
-	// Generate multi-node predastore config
-	var predastoreNodeID int
+	// Generate multi-node predastore config. A single-node install is host 1,
+	// the only [[host]] its predastore.toml declares; the multi-node path below
+	// replaces this with the host matching this machine.
+	predastoreHostID := 1
 	hasPredastoreConfig := len(predastoreNodes) >= 2
 
 	if hasPredastoreConfig {
-		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, creds.AccessKey, creds.SecretKey, creds.Region, creds.NatsToken, configDir, bindIP, compactionInterval,
+		predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, creds.AccessKey, creds.SecretKey, creds.Region, creds.NatsToken, configDir, dataDir, bindIP, compactionInterval,
 			northstarCreds)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating multi-node predastore config: %v\n", err)
@@ -2407,12 +2429,12 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 			os.Exit(1)
 		}
 
-		predastoreNodeID = admin.FindNodeIDByIP(predastoreNodes, bindIP)
-		if predastoreNodeID == 0 {
+		predastoreHostID = admin.FindNodeIDByIP(predastoreNodes, bindIP)
+		if predastoreHostID == 0 {
 			fmt.Fprintf(os.Stderr, "❌ Error: bind IP %s not found in predastore node list\n", bindIP)
 			os.Exit(1)
 		}
-		fmt.Printf("✅ Created: multi-node predastore.toml (node ID: %d)\n", predastoreNodeID)
+		fmt.Printf("✅ Created: multi-node predastore.toml (host ID: %d)\n", predastoreHostID)
 	}
 
 	spinifexTomlPath := filepath.Join(configDir, "spinifex.toml")
@@ -2436,7 +2458,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		ClusterRoutes: clusterRoutes,
 		ClusterName:   creds.ClusterName,
 
-		PredastoreNodeID:          predastoreNodeID,
+		PredastoreHostID:          predastoreHostID,
 		CompactionIntervalSeconds: compactionInterval,
 		Services:                  services,
 		RemoteNodes:               buildRemoteNodes(statusResp.Nodes, node, northstarConfigPath),
@@ -2573,13 +2595,24 @@ func initIAMServiceFromConfig() (*handlers_iam.IAMServiceImpl, *config.ClusterCo
 	return svc, cfg, nc, func() { nc.Close() }, nil
 }
 
-// adminAccessPolicyDocument is the AdministratorAccess policy document that
-// grants full access to all actions and resources.
-const adminAccessPolicyDocument = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"*","Resource":"*"}]}`
+// openAccountNameIndex opens the account-name reservation index over an
+// existing NATS connection.
+func openAccountNameIndex(ctx context.Context, nc *nats.Conn) (*handlers_iam.AccountNameIndex, error) {
+	js, err := jetstream.New(nc)
+	if err != nil {
+		return nil, fmt.Errorf("get JetStream context: %w", err)
+	}
+	return handlers_iam.NewAccountNameIndex(ctx, js)
+}
 
 func runAccountCreate(cmd *cobra.Command, args []string) {
 	name, _ := cmd.Flags().GetString("name")
 
+	if remote, _ := cmd.Flags().GetBool("remote"); remote {
+		runAccountCreateRemote(cmd, name)
+		return
+	}
+
 	svc, cfg, nc, cleanup, err := initIAMServiceFromConfig()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
@@ -2587,60 +2620,73 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 	}
 	defer cleanup()
 
-	// 1. Create the account
-	account, err := svc.CreateAccount(name)
+	ctx := context.Background()
+
+	// Reject a duplicate before allocating an account ID. CreateAccount does not
+	// check names, so without this two runs of the same command silently produce
+	// two accounts that differ only by ID.
+	existing, err := handlers_iam.FindAccountByName(svc, name)
 	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error checking existing accounts: %v\n", err)
+		os.Exit(1)
+	}
+	if existing != nil {
+		fmt.Fprintf(os.Stderr, "Error: account %q already exists (account ID %s)\n",
+			existing.AccountName, existing.AccountID)
+		os.Exit(1)
+	}
+
+	// Reserve the name so a concurrent create — including one through
+	// /admin/CreateAccount — cannot take it between the check above and the
+	// create below. The CLI has no client token, so it owns the reservation by
+	// a per-invocation identifier.
+	names, err := openAccountNameIndex(ctx, nc)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	reservationOwner := "spx-cli-" + uuid.NewV4().String()
+	switch err := names.Reserve(ctx, name, reservationOwner); {
+	case errors.Is(err, handlers_iam.ErrNameTaken):
+		fmt.Fprintf(os.Stderr, "Error: account %q already exists\n", name)
+		os.Exit(1)
+	case errors.Is(err, handlers_iam.ErrNameInFlight):
+		fmt.Fprintf(os.Stderr, "Error: another account creation for %q is in progress\n", name)
+		os.Exit(1)
+	case err != nil:
+		fmt.Fprintf(os.Stderr, "Error reserving account name: %v\n", err)
+		os.Exit(1)
+	}
+
+	account, err := svc.CreateAccount(handlers_iam.NormalizeAccountName(name))
+	if err != nil {
+		if relErr := names.Release(ctx, name, reservationOwner); relErr != nil {
+			fmt.Fprintf(os.Stderr, "Warning: could not release name reservation: %v\n", relErr)
+		}
 		fmt.Fprintf(os.Stderr, "Error creating account: %v\n", err)
 		os.Exit(1)
 	}
 	accountID := account.AccountID
 
+	if err := names.Commit(ctx, name, accountID, reservationOwner); err != nil {
+		fmt.Fprintf(os.Stderr, "Warning: could not index account name: %v\n", err)
+	}
+
 	// Create default VPC for the new account (belt-and-suspenders: daemon also
 	// does this via iam.account.created event, but daemon may not be running).
 	nodeConfig := cfg.Nodes[cfg.Node]
-	vpcSvc, vpcErr := handlers_ec2_vpc.NewVPCServiceImplWithNATS(context.Background(), &nodeConfig, nc)
+	vpcSvc, vpcErr := handlers_ec2_vpc.NewVPCServiceImplWithNATS(ctx, &nodeConfig, nc)
 	if vpcErr != nil {
 		fmt.Fprintf(os.Stderr, "Warning: could not create default VPC service: %v\n", vpcErr)
 	} else if _, vpcErr = vpcSvc.EnsureDefaultVPC(accountID); vpcErr != nil {
 		fmt.Fprintf(os.Stderr, "Warning: could not create default VPC: %v\n", vpcErr)
 	}
 
-	// 2. Create admin user
-	_, err = svc.CreateUser(accountID, &iam.CreateUserInput{
-		UserName: aws.String("admin"),
-	})
+	// Shared with /admin/CreateAccount so the CLI and the remote endpoint cannot
+	// drift in what an account is made of.
+	provisioned, err := handlers_iam.ProvisionAccount(svc, accountID, account.AccountName)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error creating admin user: %v\n", err)
-		os.Exit(1)
-	}
-
-	// 3. Create access key for admin user
-	akOut, err := svc.CreateAccessKey(accountID, &iam.CreateAccessKeyInput{
-		UserName: aws.String("admin"),
-	})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error creating access key: %v\n", err)
-		os.Exit(1)
-	}
-
-	// 4. Create AdministratorAccess policy scoped to this account
-	policyARN := fmt.Sprintf("arn:aws:iam::%s:policy/AdministratorAccess", accountID)
-	_, err = svc.CreatePolicy(accountID, &iam.CreatePolicyInput{
-		PolicyName:     aws.String("AdministratorAccess"),
-		PolicyDocument: aws.String(adminAccessPolicyDocument),
-	})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error creating admin policy: %v\n", err)
-		os.Exit(1)
-	}
-
-	// 5. Attach policy to admin user
-	_, err = svc.AttachUserPolicy(accountID, &iam.AttachUserPolicyInput{
-		UserName:  aws.String("admin"),
-		PolicyArn: aws.String(policyARN),
-	})
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error attaching policy: %v\n", err)
+		fmt.Fprintf(os.Stderr, "Error provisioning account: %v\n", err)
 		os.Exit(1)
 	}
 
@@ -2662,8 +2708,8 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 	configPath := filepath.Join(homeDir, ".aws", "config")
 
 	if err := admin.UpdateAWSINIFile(credPath, profileName, map[string]string{
-		"aws_access_key_id":     *akOut.AccessKey.AccessKeyId,
-		"aws_secret_access_key": *akOut.AccessKey.SecretAccessKey,
+		"aws_access_key_id":     provisioned.AccessKeyID,
+		"aws_secret_access_key": provisioned.SecretAccessKey,
 	}); err != nil {
 		fmt.Fprintf(os.Stderr, "Warning: could not update AWS credentials: %v\n", err)
 	}
@@ -2683,16 +2729,24 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 	// Print credentials
 	fmt.Println("\nAccount created successfully!")
 	fmt.Printf("  Account ID:        %s\n", accountID)
-	fmt.Printf("  Account Name:      %s\n", name)
-	fmt.Printf("  Admin User:        admin\n")
-	fmt.Printf("  Access Key ID:     %s\n", *akOut.AccessKey.AccessKeyId)
-	fmt.Printf("  Secret Access Key: %s\n", *akOut.AccessKey.SecretAccessKey)
+	fmt.Printf("  Account Name:      %s\n", provisioned.AccountName)
+	fmt.Printf("  Admin User:        %s\n", provisioned.AdminUser)
+	fmt.Printf("  Access Key ID:     %s\n", provisioned.AccessKeyID)
+	fmt.Printf("  Secret Access Key: %s\n", provisioned.SecretAccessKey)
 	fmt.Printf("  AWS Profile:       %s\n", profileName)
 	fmt.Println("\nUse with:")
 	fmt.Printf("  AWS_PROFILE=%s aws ec2 describe-instances\n", profileName)
 }
 
 func runAccountList(cmd *cobra.Command, args []string) {
+	if remote, _ := cmd.Flags().GetBool("remote"); remote {
+		if err := runAccountListRemote(cmd); err != nil {
+			fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+			os.Exit(1)
+		}
+		return
+	}
+
 	svc, _, _, cleanup, err := initIAMServiceFromConfig()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
@@ -2706,20 +2760,7 @@ func runAccountList(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
-	if len(accounts) == 0 {
-		fmt.Println("No accounts found.")
-		return
-	}
-
-	fmt.Printf("%-14s %-20s %-10s %s\n", "ACCOUNT ID", "NAME", "STATUS", "CREATED")
-	fmt.Printf("%-14s %-20s %-10s %s\n", "----------", "----", "------", "-------")
-	for _, a := range accounts {
-		created := a.CreatedAt
-		if t, err := time.Parse(time.RFC3339, a.CreatedAt); err == nil {
-			created = t.Format("2006-01-02 15:04")
-		}
-		fmt.Printf("%-14s %-20s %-10s %s\n", a.AccountID, a.AccountName, a.Status, created)
-	}
+	printAccountTable(accountSummaries(accounts))
 }
 
 func runCertRenew(cmd *cobra.Command, _ []string) {