release: update go.mod for v1.20.0

RegisterTargets read the target group, appended in memory and wrote the whole record back with no revision guard, so two concurrent registrations lost one. Terraform parallelises by default, so any target group with more than one member hit it, and the apply exited 0 with half the fleet not behind the load balancer.

Every writer of the record now goes through the store's existing CAS rather than a wholesale Put, because a CAS loop on one writer is defeated by a non-CAS writer on the same key. Contention stays per target group; no global lock is introduced.

RequestCertificate returned InternalErrorException for a missing tenant CA, which is a client-actionable precondition rather than a server fault. Worse, that code is classified retryable, so the provider backed off and retried instead of failing, and a single certificate sat at Still creating for over nine minutes. It now returns ACM's not-found code, which is terminal, and carries a message naming the missing CA and how to create one.

ListCertificates omitted CertificateSummaryList entirely on an empty account, since the SDK marshaller drops a nil slice unconditionally.

Closes: mulga-c6khe
Closes: mulga-wbjal
Closes: mulga-l6yft
Closes: mulga-61h3h

.github/actions/e2e-analyze/analyze.go

@@ -134,21 +134,22 @@ var cascadeMarkers = []string{
 	"Expected value not to be nil",
 }
 
-// teardownPatterns match the content a test logs from t.Cleanup or from the
-// harness's on-failure diagnostics — i.e. after it has already failed. They are
-// the LAST file:line lines in the failure body, so extractErrorLine's
-// keep-the-last rule picks them over the assertion unless they are skipped.
-// Anchored deliberately: a fragment loose enough to match an assertion would
-// blank out the one line the report exists to show.
-var teardownPatterns = []*regexp.Regexp{
+// nonFailurePatterns match content a test emits AFTER the assertion that failed
+// — t.Cleanup, harness diagnostics, or a t.Logf on a passing branch — which
+// extractErrorLine's keep-the-last rule would otherwise pick over the assertion.
+// Go emits t.Logf and t.Errorf identically, so anchor every entry: a fragment
+// loose enough to match an assertion blanks out the line the report exists to
+// show. `\bas expected\b` unanchored also matches "did not fail as expected".
+var nonFailurePatterns = []*regexp.Regexp{
 	regexp.MustCompile(`^DB instance \S+ is gone$`),
 	regexp.MustCompile(`^db diagnostics \S+:`),
+	regexp.MustCompile(`^\S+: management bridge \S+:\d+ refused as expected$`),
 }
 
-// isTeardownLine reports whether a candidate error line was emitted after the
-// failure it would otherwise be mistaken for.
-func isTeardownLine(content string) bool {
-	for _, re := range teardownPatterns {
+// isNonFailureLine reports whether a candidate error line was emitted after the
+// failure it would otherwise be mistaken for, or reports an expected outcome.
+func isNonFailureLine(content string) bool {
+	for _, re := range nonFailurePatterns {
 		if re.MatchString(content) {
 			return true
 		}
@@ -245,12 +246,12 @@ func extractErrorLine(body string) string {
 			// progress lines from the same file:line pattern before the
 			// failing assertion finally fires. Cleanup runs after it, though,
 			// so its lines are last of all and have to be skipped.
-			if content := strings.TrimSpace(m[1]); !isTeardownLine(content) {
+			if content := strings.TrimSpace(m[1]); !isNonFailureLine(content) {
 				goFileLine = content
 			}
 			continue
 		}
-		if fileHintRe.MatchString(l) && !isTeardownLine(l) {
+		if fileHintRe.MatchString(l) && !isNonFailureLine(l) {
 			lastTestLine = l
 		}
 	}
@@ -296,15 +297,23 @@ func extractFileHint(body string) string {
 	}
 	// Scanned line by line rather than across the body so a cleanup line's own
 	// file:line does not become the site the report points a reader at.
-	hint := ""
+	hint, filtered := "", ""
 	for _, raw := range strings.Split(body, "\n") {
 		l := strings.TrimSpace(raw)
-		if m := goFileLineRe.FindStringSubmatch(l); m != nil && isTeardownLine(strings.TrimSpace(m[1])) {
+		all := fileHintRe.FindAllString(l, -1)
+		if len(all) == 0 {
 			continue
 		}
-		if all := fileHintRe.FindAllString(l, -1); len(all) > 0 {
-			hint = all[len(all)-1]
+		if m := goFileLineRe.FindStringSubmatch(l); m != nil && isNonFailureLine(strings.TrimSpace(m[1])) {
+			filtered = all[len(all)-1]
+			continue
 		}
+		hint = all[len(all)-1]
+	}
+	// A body whose only file:line was allowlisted still gets a pointer: coarse
+	// beats none, and extractErrorLine has the same last-resort shape.
+	if hint == "" {
+		return filtered
 	}
 	return hint
 }

.github/actions/e2e-analyze/analyze_test.go

@@ -141,6 +141,46 @@ func TestExtractErrorLine_SkipsCleanupLoggedAfterTheFailure(t *testing.T) {
 	}
 }
 
+// A subtest looping over two subjects calls t.Errorf for the first and t.Logf
+// for the second, so the success line is last in the body. Nightly run
+// 34383800404 reported that log line as the failure.
+func TestExtractErrorLine_SkipsSuccessLoggedAfterTheFailure(t *testing.T) {
+	body := `    isolation_test.go:80: rds-e2e-iso-a-1788981456: management bridge 10.15.8.10:5432 could not be tested: want connection refused, got dial tcp 10.15.8.10:5432: i/o timeout
+    isolation_test.go:84: rds-e2e-iso-b-1788981456: management bridge 10.15.8.11:5432 refused as expected`
+	got := extractErrorLine(body)
+	if !strings.Contains(got, "could not be tested") {
+		t.Errorf("extractErrorLine = %q, want the assertion at isolation_test.go:80", got)
+	}
+	if hint := extractFileHint(body); hint != "isolation_test.go:80" {
+		t.Errorf("extractFileHint = %q, want isolation_test.go:80", hint)
+	}
+}
+
+// "as expected" reads the same in an affirmation and in its negation, so an
+// unanchored pattern suppresses the assertion it was meant to outrank and
+// promotes a harness progress line in its place.
+func TestExtractErrorLine_KeepsNegatedExpectationAssertions(t *testing.T) {
+	body := `    isolation_test.go:60: waiting for the db instance to become available
+    isolation_test.go:80: rds-e2e-iso-a: management bridge 10.15.8.10:5432 was not refused as expected: got a connection`
+	got := extractErrorLine(body)
+	if !strings.Contains(got, "was not refused as expected") {
+		t.Errorf("extractErrorLine = %q, want the assertion at isolation_test.go:80", got)
+	}
+	if hint := extractFileHint(body); hint != "isolation_test.go:80" {
+		t.Errorf("extractFileHint = %q, want isolation_test.go:80", hint)
+	}
+}
+
+// Every candidate allowlisted leaves extractFileHint with nothing to return,
+// and unlike extractErrorLine it has no last-resort branch. A coarse pointer
+// beats sending the reader to no file at all.
+func TestExtractFileHint_FallsBackWhenEveryCandidateFiltered(t *testing.T) {
+	body := `    isolation_test.go:84: rds-e2e-iso-b: management bridge 10.15.8.11:5432 refused as expected`
+	if hint := extractFileHint(body); hint != "isolation_test.go:84" {
+		t.Errorf("extractFileHint = %q, want isolation_test.go:84", hint)
+	}
+}
+
 func TestExtractFileHint_PicksLastMatch(t *testing.T) {
 	body := "ec2helpers.go:50: setup\n    vpc_test.go:227: Eventually: condition not met"
 	got := extractFileHint(body)

.github/e2e-cells.json

@@ -130,7 +130,7 @@
     "runner": "ci-single",
     "extra": "rds",
     "suite": "rds",
-    "timeout_minutes": 100
+    "timeout_minutes": 120
   },
   {
     "cell": 29,

.github/scripts/suite-sets.sh

@@ -77,8 +77,9 @@ e2e_suite_timeout() {
     # what the queue costs, not what the tests do: single tests logged 11m, 22m
     # and 26m waiting for budget. 17 tests at that concurrency is over an hour
     # before anything goes wrong, and the first pass to run deep into the suite
-    # still had three left at 60m.
-    rds) echo "90m" ;;
+    # still had three left at 60m. The loaded graceful-reboot case adds an
+    # instance that holds its slot through a pgbench load as well as a create.
+    rds) echo "105m" ;;
     # diskperf runs two 16 GiB fio profiles per repetition, each on a volume it
     # creates first. One pass is ~20 minutes on the bare-metal cell and a
     # baseline capture at SPINIFEX_DISKPERF_REPS=3 is three times that.

.github/workflows/apt-packages.yml

@@ -33,8 +33,6 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # The suites the nightly E2E matrix actually installs on. This matches
-        # what detect_os accepts: Debian 13+ and Ubuntu 24.04+.
         image:
           - ubuntu:24.04
           - ubuntu:26.04

.github/workflows/e2e-baremetal.yml

@@ -55,15 +55,12 @@ permissions:
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
-  # Same central ES/APM sink as the other four e2e workflows. Bare-metal had
-  # no default here before — this cell's telemetry never reached the sink at
-  # all (not just missing ci.cell), see bm-bootstrap.sh's telemetry.env step.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
 
 jobs:
   e2e_baremetal:
     name: Bare-Metal Single-Node E2E
-    runs-on: [ self-hosted, ci-baremetal ]
+    runs-on: [self-hosted, ci-baremetal]
     timeout-minutes: 240
     concurrency:
       group: e2e-baremetal-cluster
@@ -72,9 +69,6 @@ jobs:
       BM_ISO_CACHE: /var/cache/spinifex-iso-builder
       BM_CELL: ${{ inputs.cell || '20' }}
       ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ inputs.cell || '20' }}-${{ github.run_id }}
-      # Caller-supplied on workflow_call. A dispatch run has no such input, and
-      # leaving it empty cost this cell its ci.run_id: its telemetry reached the
-      # sink but could not be found by run, only by guessing a time window.
       SPINIFEX_OTEL_ATTRS: ${{ inputs.otel-attrs || format('ci.run_id={0},ci.workflow={1},ci.cell={2},ci.matrix=baremetal/pxe/real-hw', github.run_id, github.workflow, inputs.cell || '20') }}
     steps:
       - name: Clean Workspace
@@ -190,8 +184,6 @@ jobs:
         run: |
           mkdir -p "${ARTIFACT_DIR}"
           set -o pipefail
-          # Per-suite budgets are declared once in suite-sets.sh. The script
-          # applies one timeout to every suite in the list, so the widest wins.
           . "${GITHUB_WORKSPACE}/.github/scripts/suite-sets.sh"
           BM_TEST_TIMEOUT=""
           for suite in ${BM_SUITES}; do

.github/workflows/e2e-ddil.yml

@@ -1,7 +1,5 @@
 name: E2E DDIL
 
-# This workflow is manually triggered while the DDIL scenarios are being
-# validated.
 on:
   workflow_dispatch:
     inputs:
@@ -10,37 +8,31 @@ on:
         required: false
         default: ""
 
-# Shared with e2e-nightly.yml, e2e-suites.yml and e2e-ddil.yml: every one of
-# them provisions on the same runner pools and the same hypervisors, so two in
+# Shared with e2e-nightly.yml and e2e-suites.yml: every one of them provisions
+# many cells on the same runner pools and the same hypervisors, so two in
 # flight compete for their disks and their libvirt guests. Queued rather than
-# cancelled — a run in progress is someone's result, not a stale one.
+# cancelled — a run in progress is someone's result, not a stale one. e2e.yml
+# is outside the group: one cluster per run is cheap enough to overlap.
 concurrency:
   group: e2e-hypervisors
   cancel-in-progress: false
 
 permissions:
   contents: read
-  # actions: read lets the otel-job-id action look up this job's numeric id via
-  # the Actions API — github.job is only the YAML key, no context exposes it.
   actions: read
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
-  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
-  # empty string disables telemetry entirely.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
   ddil:
     name: DDIL Real Multi-Node
-    runs-on: [ self-hosted, ci-multi ]
+    runs-on: [self-hosted, ci-multi]
     timeout-minutes: 60
     env:
       TOFU_LOG: /tmp/e2e-tofu-ddil-${{ github.run_id }}.log
-      # Job-level env wins over the workflow-level default. ci.cell=ddil marks
-      # this workflow's telemetry so a DDIL run's partition-induced errors are
-      # never mistaken for a healthy multi-node run's in the sink.
       SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=ddil
     steps:
       - name: Clean Workspace

.github/workflows/e2e-nightly.yml

@@ -2,7 +2,13 @@ name: E2E Nightly
 
 on:
   schedule:
-    - cron: "0 14 * * 0-4" # 00:00 Mon-Fri AEST = 14:00 Sun-Thu UTC
+    # 14:00 UTC is midnight AEST (01:00 AEDT — cron has no DST), when the
+    # hypervisors are idle. Every day, not weeknights: a weekend gap is two runs
+    # of evidence lost on a suite whose remaining failures are a rate, not a bug.
+    # :17 rather than :00 because GitHub queues scheduled runs behind every other
+    # top-of-hour cron and drops them under load, which is why 2026-09-14 never
+    # fired and why every run in that window started 3h15m-3h45m late.
+    - cron: "17 14 * * *"
   workflow_dispatch:
     inputs:
       cells:
@@ -11,9 +17,6 @@ on:
 
 permissions:
   contents: read
-  # actions: read lets the "Resolve CI job id" step below look up this job's
-  # numeric id via the Actions API — github.job is only the YAML key, no
-  # context exposes the id itself.
   actions: read
 
 # Every cell builds its cluster on the same five hypervisors, so two runs in
@@ -21,30 +24,21 @@ permissions:
 # second rather than cancelling it: a dispatch is usually a verification run
 # and discarding it loses the result someone is waiting on.
 #
-# The group is shared with e2e.yml, e2e-suites.yml and e2e-ddil.yml, which draw
-# on the same runner pools. A per-workflow group let a push-triggered E2E land
-# on top of a nightly cell and take libvirt down under both.
+# The group is shared with e2e-suites.yml and e2e-ddil.yml, which draw on the
+# same runner pools. e2e.yml is deliberately outside it: a single-cluster push
+# run is cheap enough to overlap, and grouping it stalled every push behind a
+# multi-hour nightly.
 concurrency:
   group: e2e-hypervisors
   cancel-in-progress: false
 
-# Scheduled runs are loaded from the default branch (main), so the schedule
-# trigger must stay in sync on main for cron to fire — and so must the
-# concurrency block above. main currently has none, which is why the nightly
-# cron runs ungrouped and a dispatch on any branch can land on top of it.
 env:
   GIT_BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
   UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-26.04.qcow2
-  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
-  # empty string disables telemetry entirely.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
-  # The matrix is built here rather than filtered inside each cell. A job whose
-  # steps all skip still reports success, so gating steps made a one-cell
-  # dispatch look like a green nineteen-job suite. A cell that was not selected
-  # now produces no job at all.
   select:
     runs-on: ubuntu-26.04
     outputs:
@@ -60,9 +54,6 @@ jobs:
       - id: pick
         env:
           CELLS: ${{ github.event_name == 'workflow_dispatch' && inputs.cells || '' }}
-          # Provision, install and teardown around the suite. Measured at ~6
-          # minutes single-node and ~10 multi-node, so this is the floor the
-          # job budget must carry on top of the suite's own.
           SETUP_ALLOWANCE_MIN: 10
         run: |
           if [ -z "$CELLS" ]; then
@@ -125,15 +116,11 @@ jobs:
       max-parallel: 3
       matrix:
         include: ${{ fromJSON(needs.select.outputs.cells) }}
-    runs-on: [ self-hosted, "${{ matrix.runner }}" ]
+    runs-on: [self-hosted, "${{ matrix.runner }}"]
     timeout-minutes: ${{ matrix.timeout_minutes || 50 }}
     env:
       TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}
-      # Job-level env wins over the workflow-level default above, so telemetry
-      # from this cell carries ci.cell/ci.matrix. Twelve cells run under one
-      # run id and every cell names its hosts node1/node2/node3, so without
-      # this every cell's telemetry collapses into one unattributable bucket.
       SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=${{ matrix.cell }},ci.matrix=${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}
 
     steps:
@@ -690,6 +677,66 @@ jobs:
             > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || true
           echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"
 
+      # Whether br-mgmt is a per-host island or one L2 segment spanning the
+      # cluster decides whether a system instance can reach the gateway address
+      # it was baked with. Nothing in a failing run records which one was built,
+      # and the environment is destroyed before it can be asked.
+      - name: Collect management-fabric topology
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
+          OUT="${ARTIFACT_DIR}/mgmt-fabric.txt"
+          INVENTORY=$(tofu output -json inventory 2>/dev/null) || { echo "no inventory; skipping"; exit 0; }
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path'); SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY")
+
+          declare -a WAN=() MGMT=()
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            WAN[$i]=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
+          done
+
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            idx=$((i + 1))
+            {
+              echo "=== node${idx} (${WAN[$i]}) ==="
+              ssh "${SSH_OPTS[@]}" tf-user@"${WAN[$i]}" \
+                'sudo ovs-vsctl list-ports br-mgmt 2>&1 | sed "s/^/ports: /"; \
+                 ip -4 addr show br-mgmt 2>&1 | sed "s/^/addr:  /"; \
+                 sudo ovs-vsctl get Bridge br-mgmt fail_mode 2>&1 | sed "s/^/fail:  /"'
+              echo
+            } >> "$OUT" 2>&1
+            MGMT[$i]=$(ssh "${SSH_OPTS[@]}" tf-user@"${WAN[$i]}" \
+              "ip -4 -o addr show br-mgmt 2>/dev/null | awk '{print \$4}' | cut -d/ -f1" 2>/dev/null)
+          done
+
+          # An island cluster reaches only its own address and that is correct;
+          # a spanned one must reach every peer. Either result is a finding, and
+          # a mixed result is the failure this matrix exists to catch.
+          {
+            echo "=== br-mgmt reachability (row = prober, column = target) ==="
+            printf '%-10s' "from\\to"
+            for i in $(seq 0 $((NODE_COUNT - 1))); do printf '%-18s' "node$((i+1)):${MGMT[$i]:-none}"; done
+            echo
+            for i in $(seq 0 $((NODE_COUNT - 1))); do
+              printf '%-10s' "node$((i+1))"
+              for j in $(seq 0 $((NODE_COUNT - 1))); do
+                if [ -z "${MGMT[$j]}" ]; then printf '%-18s' "-"; continue; fi
+                if ssh "${SSH_OPTS[@]}" tf-user@"${WAN[$i]}" \
+                     "ping -c1 -W1 ${MGMT[$j]} >/dev/null 2>&1"; then
+                  printf '%-18s' ok
+                else
+                  printf '%-18s' FAIL
+                fi
+              done
+              echo
+            done
+          } >> "$OUT" 2>&1
+
+          cat "$OUT"
+
       - name: Install go-junit-report
         if: always()
         run: |
@@ -775,7 +822,7 @@ jobs:
 
   # Bare-metal single-node suite. Doesn't fit the tofu matrix shape, so it
   # runs as a sibling job via the reusable e2e-baremetal workflow. Gated like a
-  # matrix cell ("cell 20"): every weeknight cron, or dispatch with 20 selected
+  # matrix cell ("cell 20"): every nightly cron, or dispatch with 20 selected
   # (blank cells = all). Tests `dev` on schedule to match GIT_BRANCH above.
   baremetal:
     name: cell-20 (baremetal/pxe/real-hw)
@@ -1478,7 +1525,17 @@ jobs:
   # every cell so this runs last; always() so a red run is the one it describes.
   summary:
     name: run summary
-    needs: [select, matrix, baremetal, diskperf, iso-single, iso-multi, performance, stress]
+    needs:
+      [
+        select,
+        matrix,
+        baremetal,
+        diskperf,
+        iso-single,
+        iso-multi,
+        performance,
+        stress,
+      ]
     if: always()
     runs-on: ubuntu-26.04
     steps:
@@ -1488,7 +1545,7 @@ jobs:
           sparse-checkout-cone-mode: false
 
       - name: Collect junit
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@v8
         with:
           pattern: junit-cell-*-${{ github.run_id }}
           path: junit

.github/workflows/e2e-suites.yml

@@ -43,26 +43,22 @@ on:
         required: false
         default: ""
 
-# Shared with e2e-nightly.yml, e2e-suites.yml and e2e-ddil.yml: every one of
-# them provisions on the same runner pools and the same hypervisors, so two in
+# Shared with e2e-nightly.yml and e2e-ddil.yml: every one of them provisions
+# many cells on the same runner pools and the same hypervisors, so two in
 # flight compete for their disks and their libvirt guests. Queued rather than
-# cancelled — a run in progress is someone's result, not a stale one.
+# cancelled — a run in progress is someone's result, not a stale one. e2e.yml
+# is outside the group: one cluster per run is cheap enough to overlap.
 concurrency:
   group: e2e-hypervisors
   cancel-in-progress: false
 
 permissions:
   contents: read
-  # actions: read lets the otel-job-id action look up a provision job's numeric
-  # id via the Actions API — github.job is only the YAML key, no context
-  # exposes it.
   actions: read
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
   TEST_RUN: ${{ inputs.test_run || '' }}
-  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
-  # empty string disables telemetry entirely.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
@@ -81,12 +77,6 @@ jobs:
       matrix: ${{ steps.suites.outputs.matrix }}
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
-      # This job (not single_suite) is where "Bootstrap via Binary Install"
-      # bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env, before the
-      # suite matrix even exists — single_suite's legs share this one
-      # already-booted VM, so daemon telemetry can't be scoped any finer
-      # than "single env" per suite leg. ci.cell=single distinguishes it
-      # from multi_provision's VMs within the same run id.
       SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=single
     steps:
       - name: Reap Orphan Procs

.github/workflows/e2e.yml

@@ -24,27 +24,15 @@ on:
         required: false
         default: ""
 
-# Shared with e2e-nightly.yml, e2e-suites.yml and e2e-ddil.yml: every one of
-# them provisions on the same runner pools and the same hypervisors, so two in
-# flight compete for their disks and their libvirt guests. Queued rather than
-# cancelled — a run in progress is someone's result, not a stale one.
-concurrency:
-  group: e2e-hypervisors
-  cancel-in-progress: false
-
 permissions:
   contents: read
   statuses: write
-  # actions: read lets the otel-job-id action look up a job's numeric id via
-  # the Actions API — github.job is only the YAML key, no context exposes it.
   actions: read
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
   SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam e2e-partialblock' }}
   TEST_RUN: ${{ inputs.test_run || '' }}
-  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
-  # empty string disables telemetry entirely.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 

.github/workflows/gpu-e2e.yml

@@ -56,15 +56,7 @@ concurrency:
   cancel-in-progress: false
 
 env:
-  # OTLP export to the central Elastic sink (wattle is the sink itself — see
-  # Part A.2). Repo variable overrides; carried for correlation even though
-  # the GPU suite's own daemon-side spans are emitted by wattle's already-
-  # configured obs-agent, not by this job.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
-  # ci.cell=gpu keeps this workflow's telemetry separable from the tofu-env
-  # workflows'. Nothing in this job bakes these attrs into a daemon (see the
-  # note above), so the label only takes effect once wattle's obs-agent is
-  # given them; there is deliberately no ci.job_id lookup for the same reason.
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=gpu
   WATTLE_SSH_HOST: ${{ vars.WATTLE_SSH_HOST || '192.168.1.13' }}
   WATTLE_SSH_USER: ${{ vars.WATTLE_SSH_USER || 'tf-user' }}
@@ -78,8 +70,6 @@ jobs:
     env:
       ARTIFACT_DIR: /tmp/spinifex-gpu-e2e-artifacts-${{ github.run_id }}
     steps:
-      # Manual-work mutex gate — see setup note 4 above. Checked BEFORE any
-      # SSH to wattle so a paused run touches the box zero times.
       - name: Manual-work mutex gate
         id: gate
         env:
@@ -122,8 +112,6 @@ jobs:
           echo "${{ secrets.WATTLE_SSH_KEY }}" > "${RUNNER_TEMP}/wattle-ci"
           echo "ssh_opts=-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${RUNNER_TEMP}/wattle-ci" >> "$GITHUB_ENV"
 
-      # Healthcheck: confirm wattle's persistent AWS-compat control plane is
-      # actually up before spending time building/shipping the test binary.
       - name: Healthcheck — describe-regions
         if: steps.gate.outputs.paused != 'true'
         run: |
@@ -131,18 +119,6 @@ jobs:
           ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" \
             'AWS_PROFILE=spinifex aws ec2 describe-regions --output text'
 
-      # Ensure the base EC2 GPU AMI is present — the raw-EC2 GPU passthrough
-      # test is the always-on baseline, so CI self-provisions it rather than
-      # depending on a manual import. `--name` selects the catalog entry (pulls
-      # the prebuilt image from iso.mulgadc.com); `--ami-name` is REQUIRED to
-      # name the AMI ubuntu-26.04-nvidia-gpu-x86_64 — without it the import
-      # falls back to the generic ami-<distro>-<version>-<arch>, which the
-      # tests/e2e/gpu filter would never match. Idempotent — skipped once
-      # wattle's persistent loop-fs already has the named AMI (the 4.4G source
-      # qcow2 is cached in the images dir, so a re-import wouldn't re-download).
-      # `spx admin images import` writes local spinifex storage, so it needs
-      # the `spinifex` group active (sg); aws-CLI calls hit the gateway over
-      # HTTPS and don't.
       - name: Ensure base GPU AMI present (import-if-absent)
         if: steps.gate.outputs.paused != 'true'
         run: |
@@ -171,11 +147,6 @@ jobs:
           '
           ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
 
-      # Preflight — the base GPU AMI is REQUIRED: the setup step above just
-      # ensured it, so a miss here means the import silently failed. Hard-fail
-      # rather than run a false-green all-skipped suite. ECS/EKS node AMIs are
-      # opt-in tiers (heavier — node AMIs + cluster spin-up); their tests
-      # self-skip when absent, so only warn.
       - name: Preflight — verify GPU AMIs
         if: steps.gate.outputs.paused != 'true'
         run: |
@@ -205,10 +176,6 @@ jobs:
           '
           ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
 
-      # Same build command CI's other e2e workflows use to cross-compile a
-      # suite into a shippable binary (tests/e2e/Makefile e2e-build target,
-      # SUITES_BUILD scoped to just gpu). Building on the runner keeps go
-      # toolchain / module cache off wattle entirely.
       - name: Build GPU e2e test binary on runner
         if: steps.gate.outputs.paused != 'true'
         working-directory: tests/e2e
@@ -231,12 +198,6 @@ jobs:
           ssh ${{ env.ssh_opts }} "$VM" "mkdir -p \$HOME/spinifex-gpu-e2e-bin"
           scp ${{ env.ssh_opts }} tests/e2e/_bin/gpu.test "$VM":spinifex-gpu-e2e-bin/gpu.test
 
-      # Same invocation shape e2e.yml uses to run a pre-built suite binary
-      # (SPINIFEX_E2E=1 SPINIFEX_MODE=single, ./<suite>.test -test.v
-      # -test.timeout=NNm), plus SPINIFEX_WAN_IP: wattle's ~24 bridges make
-      # net.InterfaceAddrs() enumeration pick a co-tenant bridge instead of
-      # the advertised gateway, so the override is required here. The harness
-      # should prefer spinifex.toml's advertised bridge.
       - name: Run GPU e2e suite on wattle
         id: e2e
         if: steps.gate.outputs.paused != 'true'
@@ -287,8 +248,6 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "GPU E2E results"
 
-      # Tolerated on failure: the sink being unreachable must not turn a
-      # green test run red.
       - name: Ship test results to Elasticsearch
         if: always() && steps.gate.outputs.paused != 'true'
         continue-on-error: true
@@ -313,14 +272,6 @@ jobs:
           if-no-files-found: warn
           retention-days: 14
 
-      # Cleanup is NEVER a host teardown: it never runs setup-ovn.sh,
-      # dev-reset, or cluster-reset. It only reconciles EC2-API-level state —
-      # terminate any GPU-typed instance still pending/running (the Go
-      # suite's own t.Cleanup already does this on normal pass/fail exit;
-      # this is the backstop for a hard `-test.timeout` kill or a workflow
-      # cancellation, which skip deferred Go cleanups). Termination triggers
-      # gpuManager.Release() on wattle, returning the A1000 to the pool for
-      # the next queued run or manual test.
       - name: Cleanup — terminate leftover GPU instances, release GPU
         if: always() && steps.gate.outputs.paused != 'true'
         run: |

.github/workflows/image-scripts.yml

@@ -1,9 +1,5 @@
 name: Image Scripts
 
-# Separate from spinifex.yml on purpose: scripts/images/ and images/ are shell
-# and mkosi assets baked into system images, not Go, so they get their own
-# path-filtered gate instead of running on every Go contributor's push (see
-# ui.yml for the same pattern applied to the frontend).
 on:
   push:
     branches:
@@ -38,15 +34,6 @@ jobs:
       - name: Run image script tests + shellcheck
         run: make test-images
 
-  # The mkosi EKS image is what customers run, and until now nothing built it:
-  # the composition existed in mkosi-build.sh but had no caller, so a change
-  # that broke it was only found by hand. Cell-25 still boots the Alpine image,
-  # so this is the only gate the Ubuntu one has.
-  #
-  # Self-hosted because the build needs Docker and several GB of disk, matching
-  # publish-system-ami.yml. Its own concurrency group, not e2e-hypervisors: the
-  # build provisions no libvirt guest, and sharing that group would queue image
-  # builds behind whole nightlies.
   eks_mkosi_image:
     name: Build mkosi EKS image
     runs-on: [self-hosted, ci-single]
@@ -58,9 +45,6 @@ jobs:
       - name: Checkout Code
         uses: actions/checkout@v7
 
-      # The profile binaries (eks-token-webhook, eks-gateway-publish, ...) are
-      # built on the host, not in the builder container, so Go is needed here
-      # even though the toolchain image carries mkosi itself.
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
@@ -69,7 +53,5 @@ jobs:
           # disk, and cache-save would re-upload hundreds of MB for nothing.
           cache: false
 
-      # stage_profile_binaries() sets GOWORK=off, so the sub-repo resolves its
-      # own go.mod and the sibling monorepo checkouts are not needed here.
       - name: Build the image
         run: make build-eks-node-mkosi-image

.github/workflows/perf-3node.yml

@@ -0,0 +1,415 @@
+name: Predastore 3-Node Performance
+
+on:
+  workflow_dispatch:
+    inputs:
+      perf_preset:
+        description: "Warp workload preset: smoke (30s) or compare (2m)"
+        required: false
+        default: compare
+        type: choice
+        options:
+          - smoke
+          - compare
+      cluster_ref:
+        description: "Git ref to build the cluster from, e.g. v1.18.0. Empty uses this branch."
+        required: false
+        default: ""
+        type: string
+
+# Shared with e2e.yml and e2e-nightly.yml: this job provisions on the same
+# hypervisors they do, so a run in flight there and here would compete for
+# disks and libvirt guests. Queued, not cancelled, same as those workflows.
+concurrency:
+  group: e2e-hypervisors
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  GIT_BRANCH: ${{ github.ref_name }}
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+
+jobs:
+  perf_3node:
+    name: Predastore Throughput (3-node)
+    # ci-multi is a systemd runner on the LAN, not the ephemeral pool. A pool
+    # VM sits behind the geneve overlay at MTU 1408, which would put the
+    # runner's own network path inside the thing being measured.
+    runs-on: [self-hosted, ci-multi]
+    timeout-minutes: 90
+    env:
+      TOFU_LOG: /tmp/e2e-tofu-perf3node-${{ github.run_id }}.log
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=perf-3node
+    steps:
+      - name: Reap Orphan Procs
+        run: |
+          set +e
+          UID_SELF=$(id -u)
+          WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in
+                "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
+              esac
+            done
+          }
+          reap TERM
+          sleep 3
+          reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          true
+
+      - name: Clean Workspace
+        run: |
+          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
+            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      # ci-multi and ci-single resolve to the same physical host, so the
+      # nvme path cell-23 uses for its throughput run is already there.
+      # Pointing TMPDIR at it keeps warp's payloads off a small /tmp tmpfs.
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+          echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-3node-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/tofu-cluster
+          path: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - uses: actions/setup-go@v7
+        with:
+          go-version-file: spinifex/go.mod
+          cache: false
+
+      - name: Re-enable Go toolchain auto-download
+        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
+
+      # Same-named branch when one exists, dev otherwise. Matches how the
+      # e2e path picks siblings, so this measures the same predastore
+      # commit a cross-repo change would be built and tested against.
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ env.GIT_BRANCH }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - uses: actions/checkout@v7
+        id: predastore
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      # The cluster's ref is a knob and the harness's is not. Measuring an old
+      # release with a new harness is the only way its number can be set beside
+      # a current one; rebuilding both would change the ruler and the thing.
+      - name: Build Install Artifacts
+        working-directory: mulga/scripts/tofu-cluster
+        env:
+          CLUSTER_REF: ${{ inputs.cluster_ref }}
+        run: |
+          ./build-install-artifacts.sh "${CLUSTER_REF:-$GIT_BRANCH}" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-tests.tar" \
+            --skip-e2e-build
+
+      # The libvirt provider dials the hypervisors by bare hostname. A pool VM
+      # sits inside a VPC and cannot resolve them; a LAN runner already can, so
+      # this is self-gating on the lookup rather than on the runner's labels.
+      - name: Map hypervisor names
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          if getent hosts banksia >/dev/null; then
+            echo "hypervisor names already resolve"
+          else
+            sudo tee -a /etc/hosts < network/hosts >/dev/null
+            echo "added $(grep -c '^[0-9]' network/hosts) entries to /etc/hosts"
+          fi
+          # Pinned rather than StrictHostKeyChecking=no: these hops reach the
+          # hypervisors themselves, so an unverified key is the one place in
+          # this job where accepting anything would matter.
+          if ssh-keygen -F banksia >/dev/null 2>&1; then
+            echo "hypervisor host keys already known"
+          else
+            install -d -m 700 ~/.ssh
+            cat network/known_hosts >> ~/.ssh/known_hosts
+            chmod 600 ~/.ssh/known_hosts
+            echo "pinned $(grep -c '^[a-z]' network/known_hosts) host keys"
+          fi
+
+      - name: Provision VMs
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          export TF_VAR_git_branch="${GIT_BRANCH}"
+          quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
+          quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
+          quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
+          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          ok "3-node cluster provisioned"
+
+      - name: Bootstrap via Binary Install
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
+          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
+
+      - name: Resolve cluster SSH details
+        id: cluster
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          SSH_USER=$(echo "$INVENTORY" | jq -r '.ssh_user')
+          REGION=$(echo "$INVENTORY" | jq -r '.region')
+          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          NODES_CSV=""
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
+            if [ -z "$NODES_CSV" ]; then NODES_CSV="$ip"; else NODES_CSV="$NODES_CSV,$ip"; fi
+          done
+          echo "node1_ip=${NODE1_IP}" >> "$GITHUB_OUTPUT"
+          echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
+          echo "ssh_user=${SSH_USER}" >> "$GITHUB_OUTPUT"
+          echo "region=${REGION}" >> "$GITHUB_OUTPUT"
+          echo "nodes_csv=${NODES_CSV}" >> "$GITHUB_OUTPUT"
+
+      # PERF_EXTERNAL_HOSTS wants host:port, and the gate listens on 8443 on
+      # every node. run-info.txt from this run is then comparable with the
+      # nightly's loopback one, since the harness itself never changes.
+      - name: Resolve PERF_EXTERNAL_HOSTS
+        id: perf_hosts
+        run: |
+          set -euo pipefail
+          IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
+          HOSTS=""
+          for ip in "${NODE_LIST[@]}"; do
+            [ -z "$ip" ] && continue
+            if [ -z "$HOSTS" ]; then HOSTS="$ip:8443"; else HOSTS="$HOSTS,$ip:8443"; fi
+          done
+          echo "hosts=${HOSTS}" >> "$GITHUB_OUTPUT"
+
+      # A slow bootstrap should fail here, legibly, rather than inside warp
+      # with a wall of connection-refused errors. Bounded at 5 minutes per
+      # gate so a genuinely dead node still fails the job in finite time.
+      - name: Wait for gates to answer on :8443
+        run: |
+          set -euo pipefail
+          IFS=',' read -ra GATES <<< "${{ steps.perf_hosts.outputs.hosts }}"
+          for gate in "${GATES[@]}"; do
+            echo "waiting for https://$gate"
+            up=0
+            for i in $(seq 1 60); do
+              if curl -k -sS --max-time 5 -o /dev/null "https://$gate/"; then
+                up=1
+                break
+              fi
+              sleep 5
+            done
+            [ "$up" -eq 1 ] || { echo "::error::gate $gate never answered HTTPS on :8443"; exit 1; }
+            echo "gate $gate is up"
+          done
+
+      # Warp reads whole objects, so no workload here touches a ranged read and
+      # the throughput figures stay healthy while ranged GETs are broken. This
+      # probe is the only thing in CI that looks at them.
+      - name: Ranged GET probe
+        env:
+          GATE: ${{ steps.cluster.outputs.node1_ip }}
+          SSH_KEY: ${{ steps.cluster.outputs.ssh_key }}
+          SSH_USER: ${{ steps.cluster.outputs.ssh_user }}
+          NODE1_IP: ${{ steps.cluster.outputs.node1_ip }}
+          CLUSTER_REGION: ${{ steps.cluster.outputs.region }}
+        run: |
+          set -euo pipefail
+          CREDS=$(ssh -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+            "${SSH_USER}@${NODE1_IP}" 'cat ~/.aws/credentials')
+          AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
+          SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
+          echo "::add-mask::$SK"
+          B="ranged-probe-${GITHUB_RUN_ID}"
+          AWSX=(--region "$CLUSTER_REGION" --endpoint-url "https://${GATE}:8443" --no-verify-ssl)
+          export AWS_ACCESS_KEY_ID="$AK" AWS_SECRET_ACCESS_KEY="$SK"
+          aws s3 mb "s3://$B" "${AWSX[@]}"
+          trap 'aws s3 rb "s3://$B" --force "${AWSX[@]}" >/dev/null 2>&1 || true' EXIT
+          # Two sizes: read amplification makes a boundary-crossing range cost
+          # the whole object, so the fault shows as time tracking object size.
+          for mb in 64 256; do
+            head -c $((mb*1024*1024)) /dev/urandom > "${TMPDIR}/probe-${mb}.bin"
+            aws s3 cp "${TMPDIR}/probe-${mb}.bin" "s3://$B/probe-${mb}.bin" --only-show-errors "${AWSX[@]}"
+            rm -f "${TMPDIR}/probe-${mb}.bin"
+          done
+          t() {
+            curl -sk --aws-sigv4 "aws:amz:${CLUSTER_REGION}:s3" -u "$AK:$SK" \
+              -H "Range: bytes=0-$1" -o /dev/null -w '%{time_total}' \
+              --url "https://${GATE}:8443/${B}/probe-${2}.bin"
+          }
+          # 4194304 is the last byte inside the first 4 MiB block; one more
+          # crosses into the second and leaves the single-shard fast path.
+          IN64=$(t 4194303 64);  OUT64=$(t 4194304 64)
+          IN256=$(t 4194303 256); OUT256=$(t 4194304 256)
+          printf '%-10s in-block=%ss  crossing=%ss\n' "64MiB" "$IN64" "$OUT64"
+          printf '%-10s in-block=%ss  crossing=%ss\n' "256MiB" "$IN256" "$OUT256"
+          # The 4x object-size ratio is the signal, and it is compared against
+          # the crossing reads alone so a slow runner cancels out of both sides.
+          awk -v a="$OUT64" -v b="$OUT256" 'BEGIN{
+            r = (a > 0) ? b/a : 0
+            printf "crossing-range cost ratio 256MiB/64MiB = %.2f (object size ratio is 4.00)\n", r
+            if (r > 2.0) {
+              print "::error::a boundary-crossing ranged GET scales with object size, so it is reading the whole object"
+              exit 1
+            }
+          }'
+          echo "ranged GET does not scale with object size"
+
+      # The benchmark works in $TMPDIR and predastore refuses writes below a 5%
+      # free-space watermark, so this asserts the same headroom cell-23 does.
+      # tmpfs is rejected outright: that would measure RAM, not the gates.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
+          case "$fstype" in
+            tmpfs|ramfs)
+              echo "::error::$TMPDIR is $fstype — a storage benchmark there measures RAM"
+              exit 1 ;;
+          esac
+          find "$TMPDIR" -maxdepth 1 -name 'predastore-e2e-performance.*' \
+               -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          pct_used=$(df --output=pcent "$TMPDIR" | tail -1 | tr -dc '0-9')
+          if [ "$pct_used" -ge 90 ] || [ "$avail_gb" -lt 20 ]; then
+            echo "::error::$TMPDIR ($fstype) is ${pct_used}% full with ${avail_gb} GiB free."
+            echo "::error::The benchmark needs headroom above predastore's 5% watermark."
+            exit 1
+          fi
+
+      - name: Install warp
+        working-directory: predastore
+        run: make warp-install
+
+      # PERF_EXTERNAL_HOSTS skips profile selection, port offsetting, and
+      # cluster start/stop, and measures these three gates directly. Same
+      # workloads and analysis as the nightly loopback run, so the numbers
+      # from the two are directly comparable.
+      #
+      # The cluster's credentials are read and used inside this one step, the
+      # way e2e-ddil.yml does it: a step-level env: block is echoed verbatim
+      # into the run log, and a shell variable is not.
+      - name: Run 3-node throughput benchmark
+        working-directory: predastore
+        env:
+          PERF_EXTERNAL_HOSTS: ${{ steps.perf_hosts.outputs.hosts }}
+          PERF_CONFIGS: spinifex-3node
+          PERF_PRESET: ${{ inputs.perf_preset || 'compare' }}
+          # What is deployed, not what the harness was checked out at. With a
+          # cluster_ref those differ, and run-info.txt records the cluster.
+          PERF_EXTERNAL_SHA: ${{ inputs.cluster_ref || steps.predastore.outputs.commit }}
+          SSH_KEY: ${{ steps.cluster.outputs.ssh_key }}
+          SSH_USER: ${{ steps.cluster.outputs.ssh_user }}
+          NODE1_IP: ${{ steps.cluster.outputs.node1_ip }}
+          CLUSTER_REGION: ${{ steps.cluster.outputs.region }}
+        run: |
+          set -euo pipefail
+          # spx admin init mints a random pair at bootstrap and writes it to
+          # node1's ~/.aws/credentials. The benchmark's built-in key is the one
+          # the loopback configs trust, which this cluster has never heard of.
+          CREDS=$(ssh -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+            "${SSH_USER}@${NODE1_IP}" 'cat ~/.aws/credentials')
+          PERF_ACCESS_KEY=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
+          PERF_SECRET_KEY=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
+          if [ -z "$PERF_ACCESS_KEY" ] || [ -z "$PERF_SECRET_KEY" ]; then
+            echo "::error::could not read spinifex AWS credentials from ${NODE1_IP}"
+            exit 1
+          fi
+          echo "::add-mask::$PERF_SECRET_KEY"
+          mkdir -p "$PERF_RESULTS_ROOT"
+          PERF_ACCESS_KEY="$PERF_ACCESS_KEY" \
+          PERF_SECRET_KEY="$PERF_SECRET_KEY" \
+          PERF_REGION="$CLUSTER_REGION" \
+            make e2e-performance 2>&1 | tee "$PERF_RESULTS_ROOT/e2e-performance.log"
+
+      # Report-only, matching cell-23: the baseline comparison and any
+      # regression gate are future work once run-to-run spread is known from
+      # real bare-metal runs.
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: predastore-perf-3node-${{ github.run_id }}
+          path: ${{ env.PERF_RESULTS_ROOT }}
+          if-no-files-found: warn
+          retention-days: 90
+
+      - name: Clean up results
+        if: always()
+        run: rm -rf "${PERF_RESULTS_ROOT}"
+
+      - name: Teardown
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set +e
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          UID_SELF=$(id -u)
+          WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in
+                "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;;
+              esac
+            done
+          }
+          reap TERM
+          sleep 2
+          reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
+          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
+          ok "teardown complete"
+
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-perf3node-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14

.github/workflows/publish-system-ami.yml

@@ -60,7 +60,7 @@ concurrency:
 jobs:
   matrix:
     name: Resolve image matrix
-    runs-on: [ self-hosted, ci-single ]
+    runs-on: [self-hosted, ci-single]
     outputs:
       images: ${{ steps.gen.outputs.images }}
     steps:
@@ -84,7 +84,7 @@ jobs:
   publish:
     name: Build + publish ${{ matrix.name }}
     needs: matrix
-    runs-on: [ self-hosted, ci-single ]
+    runs-on: [self-hosted, ci-single]
     strategy:
       fail-fast: false
       # ci-single is one runner and each GPU build needs ~16G of scratch disk;
@@ -131,9 +131,7 @@ jobs:
         with:
           go-version-file: "spinifex/go.mod"
           # Self-hosted runner: the Go module + build cache already persists on
-          # local disk between runs. setup-go's cache-save ("Post Setup Go")
-          # otherwise re-uploads hundreds of MB to the Actions cache backend,
-          # taking longer than the build itself.
+          # local disk between runs.
           cache: false
 
       - name: Setup Go Workspace
@@ -160,7 +158,6 @@ jobs:
       - name: Build + publish ${{ matrix.name }} to R2
         working-directory: spinifex
         env:
-          # Map the repo's R2 secrets onto the names publish-system-image.sh reads.
           AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
           R2_ENDPOINT: ${{ secrets.CF_R2_ENDPOINT_URL }}

Makefile

@@ -181,7 +181,7 @@ test-harness:
 # JetStream, with only the daemon-side NATS subjects stubbed.
 AWS_MODEL_CONFORMANCE_REPORT ?= $(CURDIR)/.cache/aws-model-conformance-report.txt
 AWS_MODEL_CONFORMANCE_MODE ?= fail
-AWS_MODEL_OPERATION_COVERAGE_REPORT ?= $(CURDIR)/docs/aws-model-operation-coverage.md
+AWS_MODEL_OPERATION_COVERAGE_DIR ?= $(CURDIR)/docs/coverage
 # -count=1 is load-bearing: the conformance report is written by the test binary,
 # so a cached pass skips the run, leaves no report, and the cat below fails the
 # target. It also keeps the conformance gate honest — a cached result would mean
@@ -191,14 +191,13 @@ test-integration:
 	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -count=1 -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
 	@cat $(AWS_MODEL_CONFORMANCE_REPORT)
 	@$(MAKE) --no-print-directory generate-aws-model-coverage
-	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_REPORT)"
+	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_DIR)"
 
 generate-aws-model-coverage:
-	@mkdir -p $(dir $(AWS_MODEL_OPERATION_COVERAGE_REPORT))
-	@go run ./cmd/aws-model-coverage > $(AWS_MODEL_OPERATION_COVERAGE_REPORT)
+	@go run ./cmd/aws-model-coverage -out $(AWS_MODEL_OPERATION_COVERAGE_DIR)
 
 aws-model-coverage: generate-aws-model-coverage
-	@cat $(AWS_MODEL_OPERATION_COVERAGE_REPORT)
+	@go run ./cmd/aws-model-coverage
 
 # Segscan storage oracle: needs the mulga umbrella repo's scripts/segscan
 # checked out alongside spinifex (see spinifex/testutil/segscanoracle), which

README.md

@@ -6,6 +6,7 @@
   <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.27+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
   <a href="https://mulgadc.com"><img src="https://img.shields.io/badge/home-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
+  <a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/docs-spinifex-0969da?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTYuNSAySDIwYTEgMSAwIDAgMSAxIDF2MTQuNWExIDEgMCAwIDEtMSAxSDYuNWExLjUgMS41IDAgMCAwIDAgM0gyMWExIDEgMCAwIDEgMCAySDYuNUEzLjUgMy41IDAgMCAxIDMgMjBWNS41QTMuNSAzLjUgMCAwIDEgNi41IDJ6bTEgNGExIDEgMCAwIDAgMCAyaDlhMSAxIDAgMCAwIDAtMmgtOXptMCA0YTEgMSAwIDAgMCAwIDJoOWExIDEgMCAwIDAgMC0yaC05eiIvPjwvc3ZnPg==" alt="docs.mulgadc.com"></a>
 </p>
 
 <p align="center">
@@ -79,7 +80,7 @@ Spinifex speaks the AWS API surface. The AWS SDKs, AWS CLI, and Terraform all wo
 | **ECR** | Container Registry | Available |
 | **ECS** | Container Service | Available |
 | **RDS** | Databases | Available |
-| **Bedrock** | AI Deployment | Q3 2026 |
+| **Bedrock** | AI Deployment | Q4 2026 |
 
 Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
 

cmd/aws-model-coverage/main.go

@@ -1,9 +1,11 @@
 package main
 
 import (
+	"flag"
 	"fmt"
 	"os"
 
+	"github.com/mulgadc/predastore/s3api"
 	"github.com/mulgadc/spinifex/internal/awsmodel"
 	"github.com/mulgadc/spinifex/spinifex/gateway"
 
@@ -11,6 +13,28 @@ import (
 )
 
 func main() {
+	outputDir := flag.String("out", "", "write one publishable page per service into this directory")
+	flag.Parse()
+
+	coverages, err := compareAll()
+	if err != nil {
+		fail(err)
+	}
+	if *outputDir == "" {
+		fmt.Print(awsmodel.RenderCoverageSummary(coverages))
+		return
+	}
+	if err := awsmodel.WritePages(*outputDir, coverages); err != nil {
+		fail(err)
+	}
+}
+
+func fail(err error) {
+	fmt.Fprintf(os.Stderr, "aws-model-coverage: %v\n", err)
+	os.Exit(1)
+}
+
+func compareAll() ([]awsmodel.OperationCoverage, error) {
 	dispatch := gateway.AWSOperationInventory()
 	coverages := make([]awsmodel.OperationCoverage, 0, len(awsmodel.Services()))
 	for _, service := range awsmodel.Services() {
@@ -23,23 +47,19 @@ func main() {
 				Stubbed:     inventory.Stubbed,
 				Unsupported: inventory.Unsupported,
 			}
+		// Predastore serves the S3 REST surface, and names the operations each
+		// of its routes answers, so the comparison is the same one.
 		case service == awsmodel.S3:
-			modelInventory = awsmodel.DispatchInventory{
-				Opaque: true,
-				Note:   "Spinifex delegates the S3 REST surface to Predastore, which has no operation-name dispatch table to compare mechanically.",
-			}
+			modelInventory = awsmodel.DispatchInventory{Registered: s3api.Operations()}
 		default:
-			fmt.Fprintf(os.Stderr, "aws-model-coverage: no gateway inventory for %s\n", service)
-			os.Exit(1)
+			return nil, fmt.Errorf("no gateway inventory for %s", service)
 		}
 
 		coverage, err := awsmodel.CompareOperations(service, modelInventory)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "aws-model-coverage: %v\n", err)
-			os.Exit(1)
+			return nil, err
 		}
 		coverages = append(coverages, coverage)
 	}
-
-	fmt.Print(awsmodel.RenderCoverageMarkdown(coverages))
+	return coverages, nil
 }

cmd/spinifex/cmd/admin.go

@@ -358,6 +358,7 @@ func init() {
 
 	// Flags for admin init
 	adminInitCmd.Flags().Bool("force", false, "Force re-initialization (overwrites existing config)")
+	adminInitCmd.Flags().Bool("discard-jetstream", true, "With --nodes >= 2, remove this node's existing JetStream streams once every node has joined; =false refuses to form over a non-empty store instead. Ignored with --nodes 1")
 	adminInitCmd.Flags().String("region", "ap-southeast-2", "Mulga region to create")
 	adminInitCmd.Flags().String("az", "ap-southeast-2a", "Mulga AZ to create")
 	adminInitCmd.Flags().String("node", "node1", "Node name, increment for additional nodes (default, node1)")
@@ -1313,6 +1314,15 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		os.Exit(0)
 	}
 
+	if spxRoot == "" {
+		spxRoot = DefaultDataDir()
+	}
+	discardJetStream, _ := cmd.Flags().GetBool("discard-jetstream")
+	if err := checkInitJetStreamStore(jetStreamStoreDir(spxRoot), nodes, discardJetStream); err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+		os.Exit(1)
+	}
+
 	// Preserve the previously-captured operator email across --force re-inits
 	// when --email is omitted (e.g. reset-dev-env.sh workflows). Without this
 	// a reset would silently blank the address.
@@ -1845,6 +1855,17 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 
 	fmt.Printf("✅ All %d nodes joined!\n", expectedNodes)
 
+	// The pre-start check refused a store with streams unless discard was asked
+	// for, so anything left here is removed now that the cluster is committed.
+	if discard, _ := cmd.Flags().GetBool("discard-jetstream"); discard {
+		storeDir := jetStreamStoreDir(spxRoot)
+		if err := discardJetStreamStore(storeDir); err != nil {
+			fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Printf("✅ Discarded this node's pre-formation JetStream store: %s\n", storeDir)
+	}
+
 	// Build cluster topology from formation data
 	allNodes := fs.Nodes()
 	clusterRoutes := formation.BuildClusterRoutes(allNodes)
@@ -2002,19 +2023,27 @@ func joinRetryable(err error, statusCode int) bool {
 }
 
 // checkJoinPreconditions rejects a join that would silently destroy this node's
-// existing cluster identity. Joining adopts the primary's CA and master key,
-// overwriting whatever is here: correct on a freshly installed node, and on one
-// that has been in service it orphans every fragment and volume sealed under the
-// old key. runAdminInit guards the same way before re-initializing.
-func checkJoinPreconditions(configDir string, force bool) error {
+// identity or JetStream data; --force accepts that loss. A running NATS service
+// is refused even with --force, because the store cannot be removed under it.
+func checkJoinPreconditions(configDir, storeDir string, force bool) error {
+	if err := checkStoreDiscardable(); err != nil {
+		return err
+	}
 	if force {
 		return nil
 	}
 	tomlPath := filepath.Join(configDir, "spinifex.toml")
-	if !admin.FileExists(tomlPath) {
-		return nil
+	if admin.FileExists(tomlPath) {
+		return fmt.Errorf("this node is already initialized: %s", tomlPath)
 	}
-	return fmt.Errorf("this node is already initialized: %s", tomlPath)
+	streams, err := localStreams(storeDir)
+	if err != nil {
+		return err
+	}
+	if len(streams) > 0 {
+		return fmt.Errorf("this node's JetStream store %s holds %d stream(s)", storeDir, len(streams))
+	}
+	return nil
 }
 
 // joinDiscardsIdentityMsg spells out what a forced join throws away. Kept out of
@@ -2023,6 +2052,8 @@ const joinDiscardsIdentityMsg = `Joining will discard this node's own cluster id
   - CA certificate and key
   - master key, and any data sealed under it
   - viperblock key, and any volumes encrypted under it
+  - JetStream store (<data-dir>/nats/jetstream): every stream and KV bucket
+    this node holds, so it takes the cluster's copy instead of diverging from it
 
 That is safe on a freshly installed node — an ISO install initializes a
 single-node cluster at first boot, and nothing has been sealed under these keys
@@ -2103,7 +2134,11 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	// Checked before any network call so a node that will not join says so
 	// immediately. Unlike init this exits non-zero: a node that did not join
 	// must not look like success to a provisioning script.
-	if err := checkJoinPreconditions(configDir, force); err != nil {
+	if dataDir == "" {
+		dataDir = DefaultDataDir()
+	}
+	storeDir := jetStreamStoreDir(dataDir)
+	if err := checkJoinPreconditions(configDir, storeDir, force); err != nil {
 		fmt.Fprintf(os.Stderr, "⚠️  %v\n\n%s\n", err, joinDiscardsIdentityMsg)
 		os.Exit(1)
 	}
@@ -2113,11 +2148,6 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		clusterBind = bindIP
 	}
 
-	// Set default data directory
-	if dataDir == "" {
-		dataDir = DefaultDataDir()
-	}
-
 	fmt.Println("🚀 Joining Spinifex cluster...")
 	fmt.Printf("Node: %s\n", node)
 	fmt.Printf("Leader: %s\n", leaderHost)
@@ -2298,6 +2328,14 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
+	// Removed only once formation has succeeded, so a failed join leaves the
+	// node's single-node cluster intact.
+	if err := discardJetStreamStore(storeDir); err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Printf("✅ Discarded this node's pre-formation JetStream store: %s\n", storeDir)
+
 	// Set up config directory
 	if configDir == "" {
 		configDir = DefaultConfigDir()

cmd/spinifex/cmd/admin_kv.go

@@ -0,0 +1,451 @@
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"slices"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/minio/highwayhash"
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/spf13/cobra"
+)
+
+var kvCmd = &cobra.Command{
+	Use:   "kv",
+	Short: "Inspect JetStream KV state",
+}
+
+var kvDigestCmd = &cobra.Command{
+	Use:   "digest",
+	Short: "Print a digest of every stream held by this node's JetStream replica",
+	Long: `Read this node's own copy of every JetStream stream and print one line per
+stream: name, first and last sequence, message count, and a SHA-256 over every
+stored message (sequence, subject, timestamp, headers, data).
+
+Run it on every node and compare the output. Replicas of one stream must
+produce identical lines; any difference means the replicas have diverged, and
+--seqs prints a hash per sequence to find where. For a check that tells a
+diverged replica from one that is merely behind, run it with --json --seqs on
+every node and pass the results to "spx admin kv compare".
+
+It never touches the live store. A direct get over NATS may be answered by any
+replica, so instead the streams tree is copied to --work-dir and read by a
+private in-process server with no listener, which is removed afterwards. The
+copy's stream config is set to one replica with no expiry so it can be loaded.
+
+Writes landing during the copy, or applied but not yet flushed to disk, can
+make a replica's tail differ, so gate API writes and re-run before calling a
+tail difference divergence.`,
+	Run: runKVDigest,
+}
+
+func init() {
+	adminCmd.AddCommand(kvCmd)
+	kvCmd.AddCommand(kvDigestCmd)
+	kvDigestCmd.Flags().String("store-dir", "", "JetStream store to read (default <spinifex-dir>/nats/jetstream)")
+	kvDigestCmd.Flags().String("work-dir", os.TempDir(), "Directory to hold the temporary copy")
+	kvDigestCmd.Flags().StringSlice("stream", nil, "Only these streams (repeatable)")
+	kvDigestCmd.Flags().Bool("seqs", false, "Also print a hash per sequence")
+	kvDigestCmd.Flags().Bool("json", false, "Print one JSON document, the input to \"spx admin kv compare\"")
+}
+
+// nodeDigest is one node's digest report, the unit "kv compare" reads.
+type nodeDigest struct {
+	Host     string         `json:"host"`
+	Store    string         `json:"store"`
+	CopiedAt time.Time      `json:"copied_at"`
+	Skipped  []string       `json:"skipped,omitempty"`
+	Streams  []streamDigest `json:"streams"`
+}
+
+// streamDigest is one stream's content as held by one replica.
+type streamDigest struct {
+	streamMeta
+
+	Name     string      `json:"name"`
+	FirstSeq uint64      `json:"first_seq"`
+	LastSeq  uint64      `json:"last_seq"`
+	Msgs     uint64      `json:"msgs"`
+	Digest   string      `json:"sha256"`
+	Seqs     []seqDigest `json:"seqs,omitempty"`
+}
+
+// streamMeta is the part of the live stream config that decides which replica
+// differences are legitimate. It is read before the copy is rewritten.
+type streamMeta struct {
+	Replicas    int           `json:"replicas,omitempty"`
+	Retention   string        `json:"retention,omitempty"`
+	MaxAge      time.Duration `json:"max_age_ns,omitempty"`
+	AllowMsgTTL bool          `json:"allow_msg_ttl,omitempty"`
+	MaxMsgs     int64         `json:"max_msgs,omitempty"`
+	MaxBytes    int64         `json:"max_bytes,omitempty"`
+}
+
+// seqDigest is the hash of one stored sequence, or "deleted" when the replica
+// has no message at that sequence. Subject, time and TTL are empty then.
+type seqDigest struct {
+	Seq     uint64        `json:"seq"`
+	Hash    string        `json:"hash"`
+	Subject string        `json:"subject,omitempty"`
+	Time    time.Time     `json:"time,omitzero"`
+	TTL     time.Duration `json:"ttl_ns,omitempty"`
+}
+
+// deletedHash marks a sequence the replica holds no message for.
+const deletedHash = "deleted"
+
+func runKVDigest(cmd *cobra.Command, _ []string) {
+	storeDir, _ := cmd.Flags().GetString("store-dir")
+	if storeDir == "" {
+		spxRoot, _ := cmd.Flags().GetString("spinifex-dir")
+		if spxRoot == "" {
+			spxRoot = DefaultDataDir()
+		}
+		storeDir = jetStreamStoreDir(spxRoot)
+	}
+	workParent, _ := cmd.Flags().GetString("work-dir")
+	only, _ := cmd.Flags().GetStringSlice("stream")
+	withSeqs, _ := cmd.Flags().GetBool("seqs")
+	asJSON, _ := cmd.Flags().GetBool("json")
+
+	work, err := os.MkdirTemp(workParent, "spx-kv-digest-")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "kv digest: %v\n", err)
+		os.Exit(1)
+	}
+	defer os.RemoveAll(work)
+
+	copiedAt := time.Now().UTC()
+	skipped, metas, err := copyStreamTree(storeDir, filepath.Join(work, "jetstream"))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "kv digest: %v\n", err)
+		os.RemoveAll(work)
+		os.Exit(1)
+	}
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), 10*time.Minute)
+	defer cancel()
+	digests, err := digestStore(ctx, work, only, withSeqs, metas)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "kv digest: %v\n", err)
+		cancel()
+		os.RemoveAll(work)
+		os.Exit(1)
+	}
+
+	host, _ := os.Hostname()
+	if asJSON {
+		enc := json.NewEncoder(cmd.OutOrStdout())
+		if err := enc.Encode(nodeDigest{Host: host, Store: storeDir, CopiedAt: copiedAt, Skipped: skipped, Streams: digests}); err != nil {
+			fmt.Fprintf(os.Stderr, "kv digest: %v\n", err)
+			os.Exit(1)
+		}
+		return
+	}
+	writeDigests(cmd.OutOrStdout(), host, storeDir, copiedAt, skipped, digests)
+}
+
+// writeDigests prints the report. Only the "#" lines carry node-specific
+// values, so `grep -v '^#'` output from two replicas diffs cleanly.
+func writeDigests(w io.Writer, host, storeDir string, copiedAt time.Time, skipped []string, digests []streamDigest) {
+	fmt.Fprintf(w, "# host=%s store=%s copied_at=%s\n", host, storeDir, copiedAt.Format(time.RFC3339))
+	for _, s := range skipped {
+		fmt.Fprintf(w, "# skipped account stream (not read): %s\n", s)
+	}
+	for _, d := range digests {
+		fmt.Fprintf(w, "%s\tfirst=%d\tlast=%d\tmsgs=%d\tsha256=%s\n", d.Name, d.FirstSeq, d.LastSeq, d.Msgs, d.Digest)
+		for _, s := range d.Seqs {
+			fmt.Fprintf(w, "%s\tseq=%d\t%s\n", d.Name, s.Seq, s.Hash)
+		}
+	}
+}
+
+// globalAccountDir is the account Spinifex's token-authenticated clients use.
+// Streams under any other account are listed but not read.
+const globalAccountDir = "$G"
+
+// copyStreamTree copies <storeDir>/$G/streams to <dst>/$G/streams, leaving out
+// consumer state (obs). It returns streams found under other accounts and each
+// copied stream's config as the live server held it, keyed by stream name.
+func copyStreamTree(storeDir, dst string) (skipped []string, metas map[string]streamMeta, err error) {
+	streams, err := localStreams(storeDir)
+	if err != nil {
+		return nil, nil, err
+	}
+	if len(streams) == 0 {
+		return nil, nil, fmt.Errorf("no streams found under %s", storeDir)
+	}
+	for _, s := range streams {
+		if !strings.HasPrefix(s, globalAccountDir+"/") {
+			skipped = append(skipped, s)
+		}
+	}
+
+	src := filepath.Join(storeDir, globalAccountDir, "streams")
+	out := filepath.Join(dst, globalAccountDir, "streams")
+	err = filepath.WalkDir(src, func(path string, d fs.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			return walkErr
+		}
+		rel, err := filepath.Rel(src, path)
+		if err != nil {
+			return err
+		}
+		// <stream>/obs holds consumers, which are not stream content.
+		if parts := strings.Split(rel, string(filepath.Separator)); d.IsDir() && len(parts) == 2 && parts[1] == "obs" {
+			return filepath.SkipDir
+		}
+		target := filepath.Join(out, rel)
+		if d.IsDir() {
+			return os.MkdirAll(target, 0o700)
+		}
+		if !d.Type().IsRegular() {
+			return nil
+		}
+		return copyFile(path, target)
+	})
+	if err != nil {
+		return nil, nil, fmt.Errorf("copy streams from %s: %w", src, err)
+	}
+	entries, err := os.ReadDir(out)
+	if err != nil {
+		return nil, nil, err
+	}
+	metas = make(map[string]streamMeta, len(entries))
+	for _, e := range entries {
+		if e.IsDir() {
+			meta, err := rewriteStreamMetaStandalone(filepath.Join(out, e.Name()))
+			if err != nil {
+				return nil, nil, err
+			}
+			metas[e.Name()] = meta
+		}
+	}
+	return skipped, metas, nil
+}
+
+// rewriteStreamMetaStandalone makes a copied stream loadable by a standalone
+// server, which refuses replicas > 1 and would expire aged messages on
+// recovery. meta.sum is highwayhash64 keyed by sha256 of the directory name.
+// It returns the settings as they were before the rewrite.
+func rewriteStreamMetaStandalone(streamDir string) (streamMeta, error) {
+	metaPath := filepath.Join(streamDir, "meta.inf")
+	raw, err := os.ReadFile(metaPath)
+	if errors.Is(err, os.ErrNotExist) {
+		return streamMeta{}, nil
+	}
+	if err != nil {
+		return streamMeta{}, err
+	}
+	var orig struct {
+		Replicas    int           `json:"num_replicas"`
+		Retention   string        `json:"retention"`
+		MaxAge      time.Duration `json:"max_age"`
+		AllowMsgTTL bool          `json:"allow_msg_ttl"`
+		MaxMsgs     int64         `json:"max_msgs"`
+		MaxBytes    int64         `json:"max_bytes"`
+	}
+	if err := json.Unmarshal(raw, &orig); err != nil {
+		return streamMeta{}, fmt.Errorf("parse %s: %w", metaPath, err)
+	}
+	kept := streamMeta{
+		Replicas: orig.Replicas, Retention: orig.Retention, MaxAge: orig.MaxAge,
+		AllowMsgTTL: orig.AllowMsgTTL, MaxMsgs: orig.MaxMsgs, MaxBytes: orig.MaxBytes,
+	}
+	dec := json.NewDecoder(bytes.NewReader(raw))
+	dec.UseNumber()
+	var meta map[string]any
+	if err := dec.Decode(&meta); err != nil {
+		return streamMeta{}, fmt.Errorf("parse %s: %w", metaPath, err)
+	}
+	meta["num_replicas"] = 1
+	meta["max_age"] = 0
+	delete(meta, "allow_msg_ttl")
+	delete(meta, "subject_delete_marker_ttl")
+	delete(meta, "placement")
+	out, err := json.Marshal(meta)
+	if err != nil {
+		return streamMeta{}, err
+	}
+	key := sha256.Sum256([]byte(filepath.Base(streamDir)))
+	hh, err := highwayhash.New64(key[:])
+	if err != nil {
+		return streamMeta{}, err
+	}
+	hh.Write(out)
+	if err := os.WriteFile(metaPath, out, 0o600); err != nil {
+		return streamMeta{}, err
+	}
+	return kept, os.WriteFile(filepath.Join(streamDir, "meta.sum"), []byte(hex.EncodeToString(hh.Sum(nil))), 0o600)
+}
+
+func copyFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+	out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(out, in); err != nil {
+		out.Close()
+		return err
+	}
+	return out.Close()
+}
+
+// digestStore opens storeParent (holding jetstream/) in a standalone server
+// that accepts in-process connections only, and digests each stream.
+func digestStore(ctx context.Context, storeParent string, only []string, withSeqs bool, metas map[string]streamMeta) ([]streamDigest, error) {
+	ns, err := server.NewServer(&server.Options{
+		ServerName: "spx-kv-digest",
+		JetStream:  true,
+		StoreDir:   storeParent,
+		DontListen: true,
+		NoLog:      true,
+		NoSigs:     true,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("create reader server: %w", err)
+	}
+	ns.Start()
+	defer func() {
+		ns.Shutdown()
+		ns.WaitForShutdown()
+	}()
+	if !ns.ReadyForConnections(30 * time.Second) {
+		return nil, errors.New("reader server did not become ready")
+	}
+
+	nc, err := nats.Connect("", nats.InProcessServer(ns))
+	if err != nil {
+		return nil, fmt.Errorf("connect to reader server: %w", err)
+	}
+	defer nc.Close()
+	js, err := jetstream.New(nc)
+	if err != nil {
+		return nil, err
+	}
+
+	var names []string
+	lister := js.StreamNames(ctx)
+	for name := range lister.Name() {
+		if len(only) == 0 || slices.Contains(only, name) {
+			names = append(names, name)
+		}
+	}
+	if err := lister.Err(); err != nil {
+		return nil, fmt.Errorf("list streams: %w", err)
+	}
+	sort.Strings(names)
+
+	digests := make([]streamDigest, 0, len(names))
+	for _, name := range names {
+		d, err := digestStream(ctx, js, name, withSeqs)
+		if err != nil {
+			return nil, err
+		}
+		d.streamMeta = metas[name]
+		digests = append(digests, d)
+	}
+	return digests, nil
+}
+
+func digestStream(ctx context.Context, js jetstream.JetStream, name string, withSeqs bool) (streamDigest, error) {
+	s, err := js.Stream(ctx, name)
+	if err != nil {
+		return streamDigest{}, fmt.Errorf("open stream %s: %w", name, err)
+	}
+	info, err := s.Info(ctx)
+	if err != nil {
+		return streamDigest{}, fmt.Errorf("stream info %s: %w", name, err)
+	}
+	st := info.State
+	d := streamDigest{Name: name, FirstSeq: st.FirstSeq, LastSeq: st.LastSeq, Msgs: st.Msgs}
+
+	whole := sha256.New()
+	for seq := st.FirstSeq; st.Msgs > 0 && seq <= st.LastSeq; seq++ {
+		msg, err := s.GetMsg(ctx, seq)
+		sd := seqDigest{Seq: seq}
+		switch {
+		case errors.Is(err, jetstream.ErrMsgNotFound):
+			sd.Hash = deletedHash
+		case err != nil:
+			return streamDigest{}, fmt.Errorf("read %s seq %d: %w", name, seq, err)
+		default:
+			sd.Hash = hashStoredMsg(msg)
+			sd.Subject = msg.Subject
+			sd.Time = msg.Time.UTC()
+			sd.TTL = msgTTL(msg.Header.Get(jetstream.MsgTTLHeader))
+		}
+		fmt.Fprintf(whole, "%d:%s\n", seq, sd.Hash)
+		if withSeqs {
+			d.Seqs = append(d.Seqs, sd)
+		}
+	}
+	d.Digest = hex.EncodeToString(whole.Sum(nil))
+	return d, nil
+}
+
+// hashStoredMsg hashes everything a replica stores for one sequence. Header
+// keys are sorted so the hash does not depend on map iteration order.
+func hashStoredMsg(m *jetstream.RawStreamMsg) string {
+	h := sha256.New()
+	var n [8]byte
+	field := func(b []byte) {
+		binary.BigEndian.PutUint64(n[:], uint64(len(b)))
+		h.Write(n[:])
+		h.Write(b)
+	}
+	binary.BigEndian.PutUint64(n[:], m.Sequence)
+	h.Write(n[:])
+	field([]byte(m.Subject))
+	binary.BigEndian.PutUint64(n[:], uint64(m.Time.UnixNano()))
+	h.Write(n[:])
+	keys := make([]string, 0, len(m.Header))
+	for k := range m.Header {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		field([]byte(k))
+		for _, v := range m.Header[k] {
+			field([]byte(v))
+		}
+	}
+	field(m.Data)
+	return hex.EncodeToString(h.Sum(nil))[:16]
+}
+
+// msgTTL parses a Nats-TTL header: a Go duration or whole seconds. "never", an
+// absent header or an unparseable one yield zero, meaning no expiry.
+func msgTTL(v string) time.Duration {
+	if v == "" {
+		return 0
+	}
+	if d, err := time.ParseDuration(v); err == nil && d > 0 {
+		return d
+	}
+	if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
+		return time.Duration(n) * time.Second
+	}
+	return 0
+}

cmd/spinifex/cmd/admin_kv_compare.go

@@ -0,0 +1,360 @@
+package cmd
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"slices"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+)
+
+var kvCompareCmd = &cobra.Command{
+	Use:   "compare <digest.json>...",
+	Short: "Compare the kv digests of every node and report replicas that have diverged",
+	Long: `Read the "spx admin kv digest --json --seqs" output of every node in the
+cluster, one file per node ("-" reads any number of them from stdin), and
+check that every stream's replicas hold the same content.
+
+A replica that is merely behind is not divergent, so only these fail:
+
+  - the same sequence holds different content on two replicas;
+  - a message held by one replica is missing from another, and nothing on
+    that replica explains it: no later write to the same subject, no expiry
+    under max_age or a message TTL, and no stream limit dropping its head;
+  - a stream is held by fewer nodes than its replica count, or by more;
+  - replicas disagree on the replica count;
+  - no sequence range is common to every replica, so nothing can be checked.
+
+Sequences past the slowest replica's last one are not compared. Digests are
+taken from live stores a few seconds apart, so re-run before trusting a
+failure that could be a write landing between two copies.
+
+Exits 0 when every stream is consistent, 1 on divergence, 2 on bad input.`,
+	Args: cobra.MinimumNArgs(1),
+	Run:  runKVCompare,
+}
+
+func init() {
+	kvCmd.AddCommand(kvCompareCmd)
+}
+
+func runKVCompare(cmd *cobra.Command, args []string) {
+	var nodes []nodeDigest
+	for _, arg := range args {
+		var r io.Reader
+		if arg == "-" {
+			r = cmd.InOrStdin()
+		} else {
+			f, err := os.Open(arg)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "kv compare: %v\n", err)
+				os.Exit(2)
+			}
+			defer f.Close()
+			r = f
+		}
+		got, err := loadNodeDigests(r, arg)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "kv compare: %v\n", err)
+			os.Exit(2)
+		}
+		nodes = append(nodes, got...)
+	}
+	res, err := compareNodeDigests(nodes)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "kv compare: %v\n", err)
+		os.Exit(2)
+	}
+	writeCompareResult(cmd.OutOrStdout(), nodes, res)
+	if res.divergent() {
+		os.Exit(1)
+	}
+}
+
+// loadNodeDigests reads every JSON digest document in r.
+func loadNodeDigests(r io.Reader, source string) ([]nodeDigest, error) {
+	dec := json.NewDecoder(r)
+	var out []nodeDigest
+	for {
+		var d nodeDigest
+		err := dec.Decode(&d)
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("%s: document %d: %w", source, len(out)+1, err)
+		}
+		if d.Host == "" {
+			return nil, fmt.Errorf("%s: document %d has no host; is it \"kv digest --json\" output?", source, len(out)+1)
+		}
+		out = append(out, d)
+	}
+	if len(out) == 0 {
+		return nil, fmt.Errorf("%s: no digest documents", source)
+	}
+	return out, nil
+}
+
+// maxFindingsPerStream caps what one stream prints; the rest are counted.
+const maxFindingsPerStream = 5
+
+type streamVerdict struct {
+	Name     string
+	Holders  []string
+	Problems []string
+	Notes    []string
+}
+
+type kvCompareResult struct {
+	Streams []streamVerdict
+}
+
+func (r kvCompareResult) divergent() bool {
+	return slices.ContainsFunc(r.Streams, func(v streamVerdict) bool { return len(v.Problems) > 0 })
+}
+
+// replicaView is one node's copy of one stream, indexed for the per-sequence rules.
+type replicaView struct {
+	host          string
+	copiedAt      time.Time
+	d             *streamDigest
+	bySeq         map[uint64]seqDigest
+	lastBySubject map[string]uint64
+}
+
+func compareNodeDigests(nodes []nodeDigest) (kvCompareResult, error) {
+	if len(nodes) < 2 {
+		return kvCompareResult{}, fmt.Errorf("need the digests of at least two nodes, got %d", len(nodes))
+	}
+	hosts := make([]string, 0, len(nodes))
+	byName := map[string][]*replicaView{}
+	for i := range nodes {
+		n := &nodes[i]
+		if slices.Contains(hosts, n.Host) {
+			return kvCompareResult{}, fmt.Errorf("two digests are from host %s", n.Host)
+		}
+		hosts = append(hosts, n.Host)
+		for j := range n.Streams {
+			d := &n.Streams[j]
+			v := &replicaView{host: n.Host, copiedAt: n.CopiedAt, d: d,
+				bySeq: make(map[uint64]seqDigest, len(d.Seqs)), lastBySubject: map[string]uint64{}}
+			for _, s := range d.Seqs {
+				v.bySeq[s.Seq] = s
+				if s.Hash != deletedHash && s.Seq > v.lastBySubject[s.Subject] {
+					v.lastBySubject[s.Subject] = s.Seq
+				}
+			}
+			byName[d.Name] = append(byName[d.Name], v)
+		}
+	}
+	names := make([]string, 0, len(byName))
+	for name := range byName {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+
+	var res kvCompareResult
+	for _, name := range names {
+		res.Streams = append(res.Streams, compareStream(name, hosts, byName[name]))
+	}
+	return res, nil
+}
+
+func compareStream(name string, hosts []string, views []*replicaView) streamVerdict {
+	v := streamVerdict{Name: name}
+	for _, r := range views {
+		v.Holders = append(v.Holders, r.host)
+	}
+
+	counts := map[int][]string{}
+	replicas := 0
+	for _, r := range views {
+		if n := r.d.Replicas; n > 0 {
+			counts[n] = append(counts[n], r.host)
+			replicas = max(replicas, n)
+		}
+	}
+	if len(counts) > 1 {
+		var parts []string
+		for n, hs := range counts {
+			parts = append(parts, fmt.Sprintf("%d on %s", n, strings.Join(hs, ",")))
+		}
+		sort.Strings(parts)
+		v.Problems = append(v.Problems, "replicas disagree on the replica count: "+strings.Join(parts, "; "))
+	}
+	switch {
+	case replicas > 0 && len(views) < replicas:
+		missing := slices.DeleteFunc(slices.Clone(hosts), func(h string) bool { return slices.Contains(v.Holders, h) })
+		msg := fmt.Sprintf("held by %d of %d replicas; missing from %s", len(views), replicas, strings.Join(missing, ","))
+		if len(hosts) < replicas {
+			msg += fmt.Sprintf(" (only %d digests given)", len(hosts))
+		}
+		v.Problems = append(v.Problems, msg)
+	case replicas > 0 && len(views) > replicas:
+		v.Problems = append(v.Problems, fmt.Sprintf("held by %d nodes but configured for %d replicas; a node outside the stream's group still holds a copy", len(views), replicas))
+	}
+	if len(views) < 2 {
+		v.Notes = append(v.Notes, "single copy, nothing to compare")
+		return v
+	}
+
+	identical := true
+	for _, r := range views[1:] {
+		if r.d.Digest != views[0].d.Digest || r.d.FirstSeq != views[0].d.FirstSeq || r.d.LastSeq != views[0].d.LastSeq {
+			identical = false
+			break
+		}
+	}
+	if identical {
+		v.Notes = append(v.Notes, fmt.Sprintf("identical, seqs %d-%d", views[0].d.FirstSeq, views[0].d.LastSeq))
+		return v
+	}
+	for _, r := range views {
+		if r.d.Msgs > 0 && len(r.d.Seqs) == 0 {
+			v.Problems = append(v.Problems, "contents differ and "+r.host+"'s digest has no per-sequence detail; re-run digest with --seqs")
+			return v
+		}
+	}
+
+	lo, hi := views[0].d.FirstSeq, views[0].d.LastSeq
+	seqSet := map[uint64]struct{}{}
+	for _, r := range views {
+		lo, hi = max(lo, r.d.FirstSeq), min(hi, r.d.LastSeq)
+		for s := range r.bySeq {
+			seqSet[s] = struct{}{}
+		}
+	}
+	if lo > hi {
+		v.Problems = append(v.Problems, "no sequence range is common to every replica, so nothing could be compared")
+		return v
+	}
+	seqs := make([]uint64, 0, len(seqSet))
+	for s := range seqSet {
+		if s <= hi {
+			seqs = append(seqs, s)
+		}
+	}
+	slices.Sort(seqs)
+
+	var problems []string
+	explained := 0
+	for _, seq := range seqs {
+		var present, absent []*replicaView
+		for _, r := range views {
+			if e, ok := r.bySeq[seq]; ok && e.Hash != deletedHash {
+				present = append(present, r)
+			} else {
+				absent = append(absent, r)
+			}
+		}
+		if len(present) == 0 {
+			continue
+		}
+		byHash := map[string][]string{}
+		for _, r := range present {
+			h := r.bySeq[seq].Hash
+			byHash[h] = append(byHash[h], r.host)
+		}
+		if len(byHash) > 1 {
+			var parts []string
+			for h, hs := range byHash {
+				parts = append(parts, strings.Join(hs, ",")+"="+h)
+			}
+			sort.Strings(parts)
+			problems = append(problems, fmt.Sprintf("seq %d %s holds different content: %s", seq, present[0].bySeq[seq].Subject, strings.Join(parts, " ")))
+			continue
+		}
+		msg := present[0].bySeq[seq]
+		for _, r := range absent {
+			if absenceExplained(r, msg) {
+				explained++
+				continue
+			}
+			problems = append(problems, fmt.Sprintf("seq %d %s is on %s but missing from %s, with no later write to that subject, expiry or limit to explain it",
+				seq, msg.Subject, hostsOf(present), r.host))
+		}
+	}
+	if len(problems) > maxFindingsPerStream {
+		problems = append(problems[:maxFindingsPerStream], fmt.Sprintf("... and %d more", len(problems)-maxFindingsPerStream))
+	}
+	v.Problems = append(v.Problems, problems...)
+
+	note := fmt.Sprintf("seqs %d-%d compared", lo, hi)
+	if explained > 0 {
+		note += fmt.Sprintf(", %d superseded or expired messages missing from some replicas", explained)
+	}
+	var ahead []string
+	for _, r := range views {
+		if r.d.LastSeq > hi {
+			ahead = append(ahead, fmt.Sprintf("%s +%d", r.host, r.d.LastSeq-hi))
+		}
+	}
+	if len(ahead) > 0 {
+		note += ", tail not compared: " + strings.Join(ahead, " ")
+	}
+	v.Notes = append(v.Notes, note)
+	return v
+}
+
+// absenceExplained reports whether a replica lacking msg removed it legitimately:
+// the subject was written again later, the message aged out before that replica
+// was copied, or a stream limit dropped it from a head the replica has moved past.
+func absenceExplained(r *replicaView, msg seqDigest) bool {
+	if msg.Subject == "" {
+		return false
+	}
+	if r.lastBySubject[msg.Subject] > msg.Seq {
+		return true
+	}
+	meta := r.d.streamMeta
+	if meta.Retention != "" && meta.Retention != "limits" {
+		return true
+	}
+	if !msg.Time.IsZero() {
+		if meta.MaxAge > 0 && msg.Time.Add(meta.MaxAge).Before(r.copiedAt) {
+			return true
+		}
+		if msg.TTL > 0 && msg.Time.Add(msg.TTL).Before(r.copiedAt) {
+			return true
+		}
+	}
+	return msg.Seq < r.d.FirstSeq && (meta.MaxMsgs > 0 || meta.MaxBytes > 0)
+}
+
+func hostsOf(views []*replicaView) string {
+	hs := make([]string, len(views))
+	for i, r := range views {
+		hs[i] = r.host
+	}
+	return strings.Join(hs, ",")
+}
+
+func writeCompareResult(w io.Writer, nodes []nodeDigest, res kvCompareResult) {
+	fmt.Fprintf(w, "compared %d digests:", len(nodes))
+	for _, n := range nodes {
+		fmt.Fprintf(w, " %s@%s", n.Host, n.CopiedAt.Format(time.TimeOnly))
+	}
+	fmt.Fprintln(w)
+	bad := 0
+	for _, v := range res.Streams {
+		status := "ok"
+		if len(v.Problems) > 0 {
+			status = "DIVERGED"
+			bad++
+		}
+		fmt.Fprintf(w, "%-9s %s  [%s]\n", status, v.Name, strings.Join(v.Holders, ","))
+		for _, p := range v.Problems {
+			fmt.Fprintf(w, "    ! %s\n", p)
+		}
+		for _, n := range v.Notes {
+			fmt.Fprintf(w, "      %s\n", n)
+		}
+	}
+	fmt.Fprintf(w, "%d streams: %d consistent, %d diverged\n", len(res.Streams), len(res.Streams)-bad, bad)
+}

cmd/spinifex/cmd/admin_kv_compare_test.go

@@ -0,0 +1,233 @@
+package cmd_test
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var t0 = time.Date(2026, 9, 15, 10, 0, 0, 0, time.UTC)
+
+type msg struct {
+	seq     uint64
+	subject string
+	hash    string // "deleted" for a gap
+	at      time.Time
+}
+
+type stream struct {
+	name     string
+	replicas int
+	maxAge   time.Duration
+	maxMsgs  int64
+	msgs     []msg
+	noSeqs   bool
+}
+
+// doc renders one node's digest the way "kv digest --json --seqs" prints it.
+func doc(t *testing.T, host string, copiedAt time.Time, streams ...stream) string {
+	t.Helper()
+	var out []map[string]any
+	for _, s := range streams {
+		var seqs []map[string]any
+		var first, last, count uint64
+		digest := host + ":" // only equality matters to compare
+		for _, m := range s.msgs {
+			if first == 0 {
+				first = m.seq
+			}
+			last = m.seq
+			e := map[string]any{"seq": m.seq, "hash": m.hash}
+			if m.hash != "deleted" {
+				count++
+				e["subject"] = m.subject
+				at := m.at
+				if at.IsZero() {
+					at = t0
+				}
+				e["time"] = at
+			}
+			digest += m.hash + ","
+			seqs = append(seqs, e)
+		}
+		sd := map[string]any{"name": s.name, "first_seq": first, "last_seq": last, "msgs": count, "sha256": digest}
+		if s.replicas > 0 {
+			sd["replicas"] = s.replicas
+		}
+		if s.maxAge > 0 {
+			sd["max_age_ns"] = s.maxAge
+		}
+		if s.maxMsgs > 0 {
+			sd["max_msgs"] = s.maxMsgs
+		}
+		if !s.noSeqs {
+			sd["seqs"] = seqs
+		}
+		out = append(out, sd)
+	}
+	raw, err := json.Marshal(map[string]any{"host": host, "store": "/s", "copied_at": copiedAt, "streams": out})
+	require.NoError(t, err)
+	return string(raw)
+}
+
+func kvs(msgs ...msg) []msg { return msgs }
+
+func TestKVCompareIdentical(t *testing.T) {
+	s := stream{name: "KV_a", replicas: 3, msgs: kvs(msg{1, "k", "h1", t0}, msg{2, "k", "h2", t0})}
+	// The digest field embeds the host, so this exercises the per-seq path too.
+	divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, s), doc(t, "n2", t0, s), doc(t, "n3", t0, s))
+	require.NoError(t, err)
+	assert.False(t, divergent, report)
+	assert.Contains(t, report, "1 streams: 1 consistent, 0 diverged")
+}
+
+// The prod failure: each node wrote its own copy before formation, so the same
+// sequence holds different content on different replicas.
+func TestKVCompareSameSeqDifferentContent(t *testing.T) {
+	a := stream{name: "KV_acct", replicas: 2, msgs: kvs(msg{1, "$KV.acct.000000000002", "aaaa", t0})}
+	b := stream{name: "KV_acct", replicas: 2, msgs: kvs(msg{1, "$KV.acct.000000000002", "bbbb", t0})}
+	divergent, report, err := cmd.CompareKVDigests(doc(t, "node1", t0, a), doc(t, "node2", t0, b))
+	require.NoError(t, err)
+	assert.True(t, divergent)
+	assert.Contains(t, report, "DIVERGED  KV_acct")
+	assert.Contains(t, report, "seq 1 $KV.acct.000000000002 holds different content: node1=aaaa node2=bbbb")
+}
+
+func TestKVCompareMissingMessage(t *testing.T) {
+	full := kvs(msg{1, "k", "h1", t0}, msg{2, "j", "h2", t0}, msg{3, "k", "h3", t0})
+
+	t.Run("superseded by a later write to the subject", func(t *testing.T) {
+		lagging := stream{name: "KV_a", replicas: 2, msgs: full}
+		purged := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{1, "", "deleted", time.Time{}}, full[1], full[2])}
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, lagging), doc(t, "n2", t0, purged))
+		require.NoError(t, err)
+		assert.False(t, divergent, report)
+		assert.Contains(t, report, "1 superseded or expired")
+	})
+
+	t.Run("nothing explains it", func(t *testing.T) {
+		have := stream{name: "KV_a", replicas: 2, msgs: full}
+		lost := stream{name: "KV_a", replicas: 2, msgs: kvs(full[0], msg{2, "", "deleted", time.Time{}}, full[2])}
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, have), doc(t, "n2", t0, lost))
+		require.NoError(t, err)
+		assert.True(t, divergent)
+		assert.Contains(t, report, "seq 2 j is on n1 but missing from n2")
+	})
+
+	t.Run("aged out under max_age before the replica was copied", func(t *testing.T) {
+		old := t0.Add(-time.Hour)
+		have := stream{name: "KV_lease", replicas: 2, maxAge: time.Minute, msgs: kvs(msg{1, "l", "h1", old}, msg{2, "m", "h2", t0})}
+		gone := stream{name: "KV_lease", replicas: 2, maxAge: time.Minute, msgs: kvs(msg{2, "m", "h2", t0})}
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, have), doc(t, "n2", t0, gone))
+		require.NoError(t, err)
+		assert.False(t, divergent, report)
+
+		have.maxAge, gone.maxAge = 0, 0
+		divergent, _, err = cmd.CompareKVDigests(doc(t, "n1", t0, have), doc(t, "n2", t0, gone))
+		require.NoError(t, err)
+		assert.True(t, divergent, "without max_age the same gap is unexplained")
+	})
+
+	t.Run("head dropped by a stream limit", func(t *testing.T) {
+		behind := stream{name: "S", replicas: 2, maxMsgs: 2, msgs: kvs(msg{1, "a", "h1", t0}, msg{2, "b", "h2", t0})}
+		ahead := stream{name: "S", replicas: 2, maxMsgs: 2, msgs: kvs(msg{2, "b", "h2", t0}, msg{3, "c", "h3", t0})}
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, behind), doc(t, "n2", t0, ahead))
+		require.NoError(t, err)
+		assert.False(t, divergent, report)
+	})
+}
+
+func TestKVCompareTailLag(t *testing.T) {
+	behind := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{1, "k", "h1", t0}, msg{2, "k", "h2", t0})}
+	ahead := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{1, "k", "h1", t0}, msg{2, "k", "h2", t0}, msg{3, "k", "h3", t0}, msg{4, "k", "h4", t0})}
+	divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, behind), doc(t, "n2", t0, ahead))
+	require.NoError(t, err)
+	assert.False(t, divergent, report)
+	assert.Contains(t, report, "tail not compared: n2 +2")
+}
+
+func TestKVCompareNoCommonRange(t *testing.T) {
+	a := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{1, "k", "h1", t0})}
+	b := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{5, "k", "h5", t0})}
+	divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, a), doc(t, "n2", t0, b))
+	require.NoError(t, err)
+	assert.True(t, divergent)
+	assert.Contains(t, report, "no sequence range is common")
+}
+
+func TestKVCompareReplicaPlacement(t *testing.T) {
+	s := stream{name: "KV_a", replicas: 3, msgs: kvs(msg{1, "k", "h1", t0})}
+	other := stream{name: "KV_b", replicas: 1, msgs: kvs(msg{1, "k", "h1", t0})}
+
+	t.Run("fewer holders than replicas", func(t *testing.T) {
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, s), doc(t, "n2", t0, s), doc(t, "n3", t0, other))
+		require.NoError(t, err)
+		assert.True(t, divergent)
+		assert.Contains(t, report, "held by 2 of 3 replicas; missing from n3")
+	})
+
+	t.Run("more holders than replicas", func(t *testing.T) {
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, other), doc(t, "n2", t0, other))
+		require.NoError(t, err)
+		assert.True(t, divergent)
+		assert.Contains(t, report, "held by 2 nodes but configured for 1 replicas")
+	})
+
+	t.Run("replica counts disagree", func(t *testing.T) {
+		r1 := s
+		r1.replicas = 1
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, s), doc(t, "n2", t0, r1))
+		require.NoError(t, err)
+		assert.True(t, divergent)
+		assert.Contains(t, report, "replicas disagree on the replica count")
+	})
+
+	t.Run("single replica stream on one node", func(t *testing.T) {
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, other), doc(t, "n2", t0))
+		require.NoError(t, err)
+		assert.False(t, divergent, report)
+	})
+}
+
+func TestKVCompareNeedsSeqs(t *testing.T) {
+	a := stream{name: "KV_a", replicas: 2, noSeqs: true, msgs: kvs(msg{1, "k", "h1", t0})}
+	b := stream{name: "KV_a", replicas: 2, noSeqs: true, msgs: kvs(msg{1, "k", "h2", t0})}
+	divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, a), doc(t, "n2", t0, b))
+	require.NoError(t, err)
+	assert.True(t, divergent)
+	assert.Contains(t, report, "re-run digest with --seqs")
+}
+
+func TestKVCompareInput(t *testing.T) {
+	s := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{1, "k", "h1", t0})}
+
+	// Several documents in one stream, as "-" reads them from stdin.
+	divergent, _, err := cmd.CompareKVDigests(doc(t, "n1", t0, s) + "\n" + doc(t, "n2", t0, s))
+	require.NoError(t, err)
+	assert.False(t, divergent)
+
+	_, _, err = cmd.CompareKVDigests(doc(t, "n1", t0, s))
+	assert.ErrorContains(t, err, "at least two nodes")
+
+	_, _, err = cmd.CompareKVDigests(doc(t, "n1", t0, s), doc(t, "n1", t0, s))
+	assert.ErrorContains(t, err, "two digests are from host n1")
+
+	_, _, err = cmd.CompareKVDigests(`KV_a	first=1	last=1`)
+	assert.Error(t, err, "text digest output is rejected")
+
+	_, _, err = cmd.CompareKVDigests(`{"streams":[]}`)
+	assert.ErrorContains(t, err, "no host")
+
+	_, _, err = cmd.CompareKVDigests("  ")
+	assert.ErrorContains(t, err, "no digest documents")
+}
+
+func TestAdminInitDiscardsJetStreamByDefault(t *testing.T) {
+	assert.Equal(t, "true", cmd.AdminInitFlagDefault("discard-jetstream"))
+	assert.True(t, strings.HasPrefix(cmd.AdminInitFlagDefault("ipsec"), "true"))
+}

cmd/spinifex/cmd/admin_kv_test.go

@@ -0,0 +1,319 @@
+//test:in-package: exercises the unexported digest helpers against a real clustered
+// store, which the cobra command only reaches through a live node.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// kvServer runs a standalone JetStream server on storeParent and returns a KV
+// bucket on it plus a stop function.
+func kvServer(t *testing.T, storeParent, bucket string) (jetstream.KeyValue, func()) {
+	t.Helper()
+	ns, err := server.NewServer(&server.Options{
+		ServerName: "kv-test", JetStream: true, StoreDir: storeParent,
+		DontListen: true, NoLog: true, NoSigs: true,
+	})
+	require.NoError(t, err)
+	ns.Start()
+	require.True(t, ns.ReadyForConnections(10*time.Second))
+	nc, err := nats.Connect("", nats.InProcessServer(ns))
+	require.NoError(t, err)
+	js, err := jetstream.New(nc)
+	require.NoError(t, err)
+	kv, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: bucket, History: 5})
+	require.NoError(t, err)
+	return kv, func() {
+		nc.Close()
+		ns.Shutdown()
+		ns.WaitForShutdown()
+	}
+}
+
+func digestOf(t *testing.T, storeDir string, withSeqs bool) []streamDigest {
+	t.Helper()
+	work := t.TempDir()
+	_, metas, err := copyStreamTree(storeDir, filepath.Join(work, "jetstream"))
+	require.NoError(t, err)
+	d, err := digestStore(context.Background(), work, nil, withSeqs, metas)
+	require.NoError(t, err)
+	return d
+}
+
+func TestKVDigest(t *testing.T) {
+	ctx := context.Background()
+	parent := t.TempDir()
+	kv, stop := kvServer(t, parent, "test")
+	for _, kvp := range [][2]string{{"a", "1"}, {"b", "2"}, {"a", "3"}} {
+		_, err := kv.PutString(ctx, kvp[0], kvp[1])
+		require.NoError(t, err)
+	}
+	require.NoError(t, kv.Delete(ctx, "b"))
+	store := filepath.Join(parent, "jetstream")
+
+	// Copied while the server is live, exactly as it runs on a node.
+	live := digestOf(t, store, true)
+	stop()
+
+	require.Len(t, live, 1)
+	assert.Equal(t, "KV_test", live[0].Name)
+	assert.Equal(t, uint64(1), live[0].FirstSeq)
+	assert.Equal(t, uint64(4), live[0].LastSeq)
+	assert.Equal(t, uint64(4), live[0].Msgs)
+	require.Len(t, live[0].Seqs, 4)
+
+	t.Run("stable across reads and a stopped server", func(t *testing.T) {
+		again := digestOf(t, store, false)
+		assert.Equal(t, live[0].Digest, again[0].Digest)
+	})
+
+	t.Run("source store is left untouched", func(t *testing.T) {
+		_, err := os.Stat(filepath.Join(store, "$G", "streams", "KV_test"))
+		assert.NoError(t, err)
+	})
+
+	// Same keys and values written separately differ in timestamps, which is
+	// exactly what separates two nodes' pre-formation writes.
+	t.Run("independently written store differs", func(t *testing.T) {
+		other := t.TempDir()
+		kv2, stop2 := kvServer(t, other, "test")
+		for _, kvp := range [][2]string{{"a", "1"}, {"b", "2"}, {"a", "3"}} {
+			_, err := kv2.PutString(ctx, kvp[0], kvp[1])
+			require.NoError(t, err)
+		}
+		require.NoError(t, kv2.Delete(ctx, "b"))
+		stop2()
+		d := digestOf(t, filepath.Join(other, "jetstream"), true)
+		assert.NotEqual(t, live[0].Digest, d[0].Digest)
+
+		// Two stores adopted as replicas of one stream: same subjects and
+		// sequences, different content, which compare must call divergent.
+		res, err := compareNodeDigests([]nodeDigest{
+			{Host: "node1", CopiedAt: time.Now(), Streams: live},
+			{Host: "node2", CopiedAt: time.Now(), Streams: d},
+		})
+		require.NoError(t, err)
+		require.True(t, res.divergent())
+		require.Len(t, res.Streams, 1)
+		assert.Contains(t, strings.Join(res.Streams[0].Problems, "\n"), "seq 1 $KV.test.a holds different content")
+	})
+
+	t.Run("report lines", func(t *testing.T) {
+		var buf bytes.Buffer
+		writeDigests(&buf, "node1", store, time.Unix(0, 0), []string{"ACC/S"}, live)
+		out := buf.String()
+		assert.Contains(t, out, "# host=node1 ")
+		assert.Contains(t, out, "# skipped account stream (not read): ACC/S")
+		assert.Contains(t, out, "KV_test\tfirst=1\tlast=4\tmsgs=4\tsha256="+live[0].Digest)
+		assert.Equal(t, 4, strings.Count(out, "KV_test\tseq="))
+	})
+}
+
+func TestKVDigestStreamFilterAndDeletedSeq(t *testing.T) {
+	ctx := context.Background()
+	parent := t.TempDir()
+	// The KV bucket is only there to be excluded by the stream filter.
+	_, stop := kvServer(t, parent, "other")
+	stop()
+
+	// KV streams deny message deletes, so an interior gap needs a plain stream.
+	ns, err := server.NewServer(&server.Options{JetStream: true, StoreDir: parent, DontListen: true, NoLog: true, NoSigs: true})
+	require.NoError(t, err)
+	ns.Start()
+	require.True(t, ns.ReadyForConnections(10*time.Second))
+	nc, err := nats.Connect("", nats.InProcessServer(ns))
+	require.NoError(t, err)
+	js, err := jetstream.New(nc)
+	require.NoError(t, err)
+	s, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ONE", Subjects: []string{"one.>"}})
+	require.NoError(t, err)
+	for _, v := range []string{"v1", "v2", "v3"} {
+		_, err = js.Publish(ctx, "one.k", []byte(v))
+		require.NoError(t, err)
+	}
+	require.NoError(t, s.DeleteMsg(ctx, 2))
+	nc.Close()
+	ns.Shutdown()
+	ns.WaitForShutdown()
+
+	work := t.TempDir()
+	_, metas, err := copyStreamTree(filepath.Join(parent, "jetstream"), filepath.Join(work, "jetstream"))
+	require.NoError(t, err)
+	d, err := digestStore(ctx, work, []string{"ONE"}, true, metas)
+	require.NoError(t, err)
+	require.Len(t, d, 1)
+	assert.Equal(t, "ONE", d[0].Name)
+	assert.Equal(t, uint64(2), d[0].Msgs)
+	require.Len(t, d[0].Seqs, 3)
+	assert.Equal(t, seqDigest{Seq: 2, Hash: "deleted"}, d[0].Seqs[1])
+	for _, i := range []int{0, 2} {
+		assert.Equal(t, "one.k", d[0].Seqs[i].Subject)
+		assert.False(t, d[0].Seqs[i].Time.IsZero())
+	}
+	assert.Equal(t, "limits", d[0].Retention)
+	assert.Equal(t, 1, d[0].Replicas)
+}
+
+func freePort(t *testing.T) int {
+	t.Helper()
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	defer l.Close()
+	return l.Addr().(*net.TCPAddr).Port
+}
+
+// A replicated stream's meta.inf carries num_replicas > 1, which a standalone
+// server refuses to load; this is the case every production node presents.
+func TestKVDigestClusteredReplicas(t *testing.T) {
+	ctx := context.Background()
+	const n = 3
+	ports := make([]int, n)
+	routes := make([]string, n)
+	for i := range ports {
+		ports[i] = freePort(t)
+		routes[i] = fmt.Sprintf("nats://127.0.0.1:%d", ports[i])
+	}
+	stores := make([]string, n)
+	servers := make([]*server.Server, n)
+	for i := range servers {
+		stores[i] = t.TempDir()
+		ns, err := server.NewServer(&server.Options{
+			ServerName: fmt.Sprintf("s%d", i+1), JetStream: true, StoreDir: stores[i],
+			Host: "127.0.0.1", Port: -1, NoLog: true, NoSigs: true,
+			Cluster: server.ClusterOpts{Name: "digest", Host: "127.0.0.1", Port: ports[i]},
+			Routes:  server.RoutesFromStr(strings.Join(routes, ",")),
+		})
+		require.NoError(t, err)
+		ns.Start()
+		servers[i] = ns
+	}
+	t.Cleanup(func() {
+		for _, ns := range servers {
+			ns.Shutdown()
+			ns.WaitForShutdown()
+		}
+	})
+
+	nc, err := nats.Connect("", nats.InProcessServer(servers[0]))
+	require.NoError(t, err)
+	defer nc.Close()
+	js, err := jetstream.New(nc)
+	require.NoError(t, err)
+
+	var kv jetstream.KeyValue
+	require.Eventually(t, func() bool {
+		kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "rep", History: 3, Replicas: n, TTL: time.Hour})
+		return err == nil
+	}, 30*time.Second, 250*time.Millisecond, "clustered KV create: %v", err)
+	for i := range 5 {
+		_, err := kv.PutString(ctx, fmt.Sprintf("k%d", i%2), fmt.Sprintf("v%d", i))
+		require.NoError(t, err)
+	}
+	s, err := js.Stream(ctx, "KV_rep")
+	require.NoError(t, err)
+	require.Eventually(t, func() bool {
+		info, err := s.Info(ctx)
+		if err != nil || info.Cluster == nil || len(info.Cluster.Replicas) != n-1 {
+			return false
+		}
+		for _, r := range info.Cluster.Replicas {
+			if !r.Current || r.Lag != 0 {
+				return false
+			}
+		}
+		return true
+	}, 30*time.Second, 250*time.Millisecond)
+
+	// Current means applied, not flushed; shutdown flushes every block to disk.
+	nc.Close()
+	for _, ns := range servers {
+		ns.Shutdown()
+		ns.WaitForShutdown()
+	}
+
+	var first string
+	var nodes []nodeDigest
+	for i, store := range stores {
+		d := digestOf(t, filepath.Join(store, "jetstream"), true)
+		require.Len(t, d, 1, "replica %d", i+1)
+		assert.Equal(t, uint64(5), d[0].Msgs, "replica %d", i+1)
+		assert.Equal(t, n, d[0].Replicas, "live replica count survives the standalone rewrite")
+		assert.Equal(t, time.Hour, d[0].MaxAge)
+		nodes = append(nodes, nodeDigest{Host: fmt.Sprintf("s%d", i+1), CopiedAt: time.Now(), Streams: d})
+		if i == 0 {
+			first = d[0].Digest
+			continue
+		}
+		assert.Equal(t, first, d[0].Digest, "replica %d matches replica 1", i+1)
+	}
+	res, err := compareNodeDigests(nodes)
+	require.NoError(t, err)
+	assert.False(t, res.divergent(), "%+v", res)
+}
+
+// Recovery applies MaxAge, so without the rewrite an expired bucket copy
+// would read as empty and mask whatever the replica holds.
+func TestKVDigestIgnoresExpiry(t *testing.T) {
+	ctx := context.Background()
+	parent := t.TempDir()
+	ns, err := server.NewServer(&server.Options{JetStream: true, StoreDir: parent, DontListen: true, NoLog: true, NoSigs: true})
+	require.NoError(t, err)
+	ns.Start()
+	require.True(t, ns.ReadyForConnections(10*time.Second))
+	nc, err := nats.Connect("", nats.InProcessServer(ns))
+	require.NoError(t, err)
+	js, err := jetstream.New(nc)
+	require.NoError(t, err)
+	kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "ttl", TTL: time.Second})
+	require.NoError(t, err)
+	_, err = kv.PutString(ctx, "k", "v")
+	require.NoError(t, err)
+
+	work := t.TempDir()
+	_, metas, err := copyStreamTree(filepath.Join(parent, "jetstream"), filepath.Join(work, "jetstream"))
+	require.NoError(t, err)
+	nc.Close()
+	ns.Shutdown()
+	ns.WaitForShutdown()
+
+	time.Sleep(1500 * time.Millisecond)
+	d, err := digestStore(ctx, work, nil, false, metas)
+	require.NoError(t, err)
+	require.Len(t, d, 1)
+	assert.Equal(t, uint64(1), d[0].Msgs)
+}
+
+func TestCopyStreamTree(t *testing.T) {
+	store := seedStore(t, "KV_a")
+	require.NoError(t, os.MkdirAll(filepath.Join(store, "$G", "streams", "KV_a", "obs", "c1"), 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(store, "$G", "streams", "KV_a", "obs", "c1", "o.dat"), []byte("x"), 0o600))
+	require.NoError(t, os.MkdirAll(filepath.Join(store, "ACME", "streams", "ORDERS"), 0o755))
+
+	dst := filepath.Join(t.TempDir(), "jetstream")
+	skipped, _, err := copyStreamTree(store, dst)
+	require.NoError(t, err)
+	assert.Equal(t, []string{"ACME/ORDERS"}, skipped)
+
+	_, err = os.Stat(filepath.Join(dst, "$G", "streams", "KV_a", "msgs", "1.blk"))
+	assert.NoError(t, err, "message blocks are copied")
+	_, err = os.Stat(filepath.Join(dst, "$G", "streams", "KV_a", "obs"))
+	assert.True(t, os.IsNotExist(err), "consumer state is left out")
+
+	_, _, err = copyStreamTree(filepath.Join(t.TempDir(), "absent"), dst)
+	assert.Error(t, err)
+}

cmd/spinifex/cmd/admin_test.go

@@ -1289,6 +1289,9 @@ func TestAMIVolumeSizeGiB(t *testing.T) {
 }
 
 func TestCheckJoinPreconditions(t *testing.T) {
+	withFakeProc(t, nil)
+	noStore := filepath.Join(t.TempDir(), "nats", "jetstream")
+
 	writeToml := func(t *testing.T) string {
 		t.Helper()
 		dir := t.TempDir()
@@ -1299,13 +1302,13 @@ func TestCheckJoinPreconditions(t *testing.T) {
 	}
 
 	t.Run("uninitialized node joins", func(t *testing.T) {
-		if err := checkJoinPreconditions(t.TempDir(), false); err != nil {
+		if err := checkJoinPreconditions(t.TempDir(), noStore, false); err != nil {
 			t.Fatalf("want nil, got %v", err)
 		}
 	})
 
 	t.Run("initialized node is refused", func(t *testing.T) {
-		err := checkJoinPreconditions(writeToml(t), false)
+		err := checkJoinPreconditions(writeToml(t), noStore, false)
 		if err == nil {
 			t.Fatal("want refusal, got nil")
 		}
@@ -1317,7 +1320,7 @@ func TestCheckJoinPreconditions(t *testing.T) {
 	// The guidance is the whole point of the guard: an operator must be able to
 	// tell what a forced join would destroy without reading the source.
 	t.Run("guidance names what is discarded", func(t *testing.T) {
-		for _, want := range []string{"CA certificate", "master key", "viperblock key", "--force"} {
+		for _, want := range []string{"CA certificate", "master key", "viperblock key", "JetStream store", "--force"} {
 			if !strings.Contains(joinDiscardsIdentityMsg, want) {
 				t.Errorf("guidance missing %q", want)
 			}
@@ -1325,10 +1328,27 @@ func TestCheckJoinPreconditions(t *testing.T) {
 	})
 
 	t.Run("force overrides", func(t *testing.T) {
-		if err := checkJoinPreconditions(writeToml(t), true); err != nil {
+		if err := checkJoinPreconditions(writeToml(t), noStore, true); err != nil {
 			t.Fatalf("want nil with force, got %v", err)
 		}
 	})
+
+	// A store with streams and no config is still data the join would destroy.
+	t.Run("uninitialized node with streams is refused", func(t *testing.T) {
+		err := checkJoinPreconditions(t.TempDir(), seedStore(t, "KV_a"), false)
+		if err == nil || !strings.Contains(err.Error(), "JetStream store") {
+			t.Fatalf("want JetStream refusal, got %v", err)
+		}
+	})
+
+	// --force accepts the loss of data, never a wipe under a live server.
+	t.Run("running NATS is refused even with force", func(t *testing.T) {
+		withFakeProc(t, map[string]string{"4242": "/usr/local/bin/spx\x00service\x00nats\x00start\x00"})
+		err := checkJoinPreconditions(writeToml(t), seedStore(t, "KV_a"), true)
+		if !errors.Is(err, errNATSRunning) {
+			t.Fatalf("want errNATSRunning, got %v", err)
+		}
+	})
 }
 
 func TestJoinRetryable(t *testing.T) {

cmd/spinifex/cmd/export_test.go

@@ -4,8 +4,10 @@
 package cmd
 
 import (
+	"bytes"
 	"errors"
 	"strconv"
+	"strings"
 	"time"
 
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
@@ -146,3 +148,28 @@ var PrintAccountTable = printAccountTable
 
 // AccountSummaries exposes accountSummaries for testing.
 var AccountSummaries = accountSummaries
+
+// CompareKVDigests runs "kv compare" over digest JSON documents and returns
+// whether any stream diverged, plus the printed report.
+func CompareKVDigests(docs ...string) (bool, string, error) {
+	var nodes []nodeDigest
+	for i, doc := range docs {
+		got, err := loadNodeDigests(strings.NewReader(doc), "doc"+strconv.Itoa(i))
+		if err != nil {
+			return false, "", err
+		}
+		nodes = append(nodes, got...)
+	}
+	res, err := compareNodeDigests(nodes)
+	if err != nil {
+		return false, "", err
+	}
+	var buf bytes.Buffer
+	writeCompareResult(&buf, nodes, res)
+	return res.divergent(), buf.String(), nil
+}
+
+// AdminInitFlagDefault returns the registered default of an admin init flag.
+func AdminInitFlagDefault(name string) string {
+	return adminInitCmd.Flags().Lookup(name).DefValue
+}

cmd/spinifex/cmd/jetstream_store.go

@@ -0,0 +1,139 @@
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// jetStreamStoreDir is where the NATS service keeps its JetStream data under a
+// node's data directory. The nats.conf template sets store_dir to
+// <data_dir>/nats/, and nats-server appends "jetstream" itself.
+func jetStreamStoreDir(dataDir string) string {
+	return filepath.Join(dataDir, "nats", "jetstream")
+}
+
+// localStreams lists the streams a JetStream store holds on disk, as
+// "<account>/<stream>". A missing store holds nothing.
+func localStreams(storeDir string) ([]string, error) {
+	accounts, err := os.ReadDir(storeDir)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("read JetStream store %s: %w", storeDir, err)
+	}
+	var streams []string
+	for _, acc := range accounts {
+		if !acc.IsDir() {
+			continue
+		}
+		entries, err := os.ReadDir(filepath.Join(storeDir, acc.Name(), "streams"))
+		if errors.Is(err, os.ErrNotExist) {
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("read streams for account %s: %w", acc.Name(), err)
+		}
+		for _, e := range entries {
+			if e.IsDir() {
+				streams = append(streams, acc.Name()+"/"+e.Name())
+			}
+		}
+	}
+	sort.Strings(streams)
+	return streams, nil
+}
+
+// procRoot is overridable in tests so the process scan can run against a fake
+// /proc tree.
+var procRoot = "/proc"
+
+// natsRunningLocally reports whether this host runs the Spinifex NATS service,
+// found as a process whose command line is `spx service nats ...`. Removing a
+// store under a live server races JetStream, which rewrites its index files.
+func natsRunningLocally() (bool, error) {
+	entries, err := os.ReadDir(procRoot)
+	if err != nil {
+		return false, fmt.Errorf("scan processes: %w", err)
+	}
+	for _, e := range entries {
+		if !e.IsDir() || strings.TrimLeft(e.Name(), "0123456789") != "" {
+			continue
+		}
+		raw, err := os.ReadFile(filepath.Join(procRoot, e.Name(), "cmdline"))
+		if err != nil || len(raw) == 0 {
+			continue
+		}
+		if isSpxNATSService(bytes.Split(bytes.TrimRight(raw, "\x00"), []byte{0})) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// isSpxNATSService matches `spx service nats` with any leading path on spx.
+func isSpxNATSService(argv [][]byte) bool {
+	return len(argv) >= 3 &&
+		filepath.Base(string(argv[0])) == "spx" &&
+		string(argv[1]) == "service" &&
+		string(argv[2]) == "nats"
+}
+
+// errNATSRunning is returned whenever a store would be discarded with the NATS
+// service still up. No flag overrides it: the wipe would race the live server.
+var errNATSRunning = errors.New("the NATS service is still running on this node; stop it first (sudo systemctl stop spinifex.target, then confirm `pgrep -af 'spx service'` prints nothing)")
+
+// checkStoreDiscardable fails if the NATS service is running locally.
+func checkStoreDiscardable() error {
+	running, err := natsRunningLocally()
+	if err != nil {
+		return err
+	}
+	if running {
+		return errNATSRunning
+	}
+	return nil
+}
+
+// discardJetStreamStore removes a node's JetStream store before it enters a
+// multi-node cluster. nats-server adopts a same-named local stream and raft
+// catch-up never repairs the prefix it already holds.
+func discardJetStreamStore(storeDir string) error {
+	if err := checkStoreDiscardable(); err != nil {
+		return err
+	}
+	if err := os.RemoveAll(storeDir); err != nil {
+		return fmt.Errorf("remove JetStream store %s: %w", storeDir, err)
+	}
+	if _, err := os.Stat(storeDir); !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("JetStream store %s still present after removal", storeDir)
+	}
+	return nil
+}
+
+// checkInitJetStreamStore refuses a multi-node init over a store holding streams
+// when discard was turned off. Keeping it as a seed is not safe: leadership
+// ignores who holds data, and unassigned local streams are deleted as orphans.
+func checkInitJetStreamStore(storeDir string, nodes int, discard bool) error {
+	if nodes < 2 {
+		return nil
+	}
+	streams, err := localStreams(storeDir)
+	if err != nil {
+		return err
+	}
+	if len(streams) == 0 {
+		return nil
+	}
+	if !discard {
+		return fmt.Errorf("this node's JetStream store %s holds %d stream(s) from before formation; "+
+			"a multi-node cluster cannot adopt them consistently. "+
+			"re-run without --discard-jetstream=false to remove them once every node has joined", storeDir, len(streams))
+	}
+	return checkStoreDiscardable()
+}

cmd/spinifex/cmd/jetstream_store_test.go

@@ -0,0 +1,140 @@
+//test:in-package: swaps the unexported procRoot for a fake /proc tree and asserts on
+// the store helpers the init and join commands call.
+
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// withFakeProc points the process scan at a temp tree holding the given
+// pid -> cmdline entries (NUL-separated argv) for the duration of the test.
+func withFakeProc(t *testing.T, procs map[string]string) {
+	t.Helper()
+	root := t.TempDir()
+	for pid, cmdline := range procs {
+		dir := filepath.Join(root, pid)
+		require.NoError(t, os.MkdirAll(dir, 0o755))
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "cmdline"), []byte(cmdline), 0o644))
+	}
+	prev := procRoot
+	procRoot = root
+	t.Cleanup(func() { procRoot = prev })
+}
+
+// seedStore builds a JetStream store with the named streams under account $G,
+// laid out the way nats-server writes it, and returns the store path.
+func seedStore(t *testing.T, streams ...string) string {
+	t.Helper()
+	store := filepath.Join(t.TempDir(), "nats", "jetstream")
+	for _, s := range streams {
+		dir := filepath.Join(store, "$G", "streams", s, "msgs")
+		require.NoError(t, os.MkdirAll(dir, 0o755))
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "1.blk"), []byte("x"), 0o600))
+	}
+	return store
+}
+
+func TestJetStreamStoreDir(t *testing.T) {
+	assert.Equal(t, "/var/lib/spinifex/nats/jetstream", jetStreamStoreDir("/var/lib/spinifex"))
+}
+
+func TestLocalStreams(t *testing.T) {
+	t.Run("missing store holds nothing", func(t *testing.T) {
+		got, err := localStreams(filepath.Join(t.TempDir(), "absent"))
+		require.NoError(t, err)
+		assert.Empty(t, got)
+	})
+
+	t.Run("lists streams across accounts, sorted", func(t *testing.T) {
+		store := seedStore(t, "KV_b", "KV_a")
+		require.NoError(t, os.MkdirAll(filepath.Join(store, "$SYS", "_js_", "_meta_"), 0o755))
+		require.NoError(t, os.WriteFile(filepath.Join(store, "$G", "streams", "stray-file"), nil, 0o600))
+
+		got, err := localStreams(store)
+		require.NoError(t, err)
+		assert.Equal(t, []string{"$G/KV_a", "$G/KV_b"}, got)
+	})
+}
+
+func TestNATSRunningLocally(t *testing.T) {
+	tests := []struct {
+		name  string
+		procs map[string]string
+		want  bool
+	}{
+		{"no processes", nil, false},
+		{"installed service", map[string]string{"10": "/usr/local/bin/spx\x00service\x00nats\x00start\x00"}, true},
+		{"bare spx on PATH", map[string]string{"10": "spx\x00service\x00nats\x00start\x00"}, true},
+		// The join itself is an spx process and must not count.
+		{"spx admin join", map[string]string{"10": "spx\x00admin\x00join\x00--force\x00"}, false},
+		{"other spx service", map[string]string{"10": "spx\x00service\x00predastore\x00start\x00"}, false},
+		{"nats-server binary", map[string]string{"10": "nats-server\x00service\x00nats\x00"}, false},
+		{"kernel thread, empty cmdline", map[string]string{"2": ""}, false},
+		{"non-pid entry ignored", map[string]string{"self": "spx\x00service\x00nats\x00"}, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withFakeProc(t, tt.procs)
+			got, err := natsRunningLocally()
+			require.NoError(t, err)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestDiscardJetStreamStore(t *testing.T) {
+	t.Run("removes the store", func(t *testing.T) {
+		withFakeProc(t, nil)
+		store := seedStore(t, "KV_a")
+		require.NoError(t, discardJetStreamStore(store))
+		_, err := os.Stat(store)
+		assert.ErrorIs(t, err, os.ErrNotExist)
+	})
+
+	t.Run("missing store is not an error", func(t *testing.T) {
+		withFakeProc(t, nil)
+		assert.NoError(t, discardJetStreamStore(filepath.Join(t.TempDir(), "absent")))
+	})
+
+	t.Run("refuses under a running NATS and leaves the store", func(t *testing.T) {
+		withFakeProc(t, map[string]string{"7": "spx\x00service\x00nats\x00start\x00"})
+		store := seedStore(t, "KV_a")
+		assert.ErrorIs(t, discardJetStreamStore(store), errNATSRunning)
+		_, err := os.Stat(store)
+		assert.NoError(t, err)
+	})
+}
+
+func TestCheckInitJetStreamStore(t *testing.T) {
+	withFakeProc(t, nil)
+
+	t.Run("single-node init keeps its store", func(t *testing.T) {
+		assert.NoError(t, checkInitJetStreamStore(seedStore(t, "KV_a"), 1, false))
+	})
+
+	t.Run("multi-node init over an empty store", func(t *testing.T) {
+		assert.NoError(t, checkInitJetStreamStore(filepath.Join(t.TempDir(), "absent"), 3, false))
+	})
+
+	t.Run("multi-node init over streams is refused without the flag", func(t *testing.T) {
+		err := checkInitJetStreamStore(seedStore(t, "KV_a", "KV_b"), 4, false)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "2 stream(s)")
+		assert.Contains(t, err.Error(), "--discard-jetstream")
+	})
+
+	t.Run("multi-node init over streams proceeds with the flag", func(t *testing.T) {
+		assert.NoError(t, checkInitJetStreamStore(seedStore(t, "KV_a"), 4, true))
+	})
+
+	t.Run("flag does not override a running NATS", func(t *testing.T) {
+		withFakeProc(t, map[string]string{"7": "spx\x00service\x00nats\x00start\x00"})
+		assert.ErrorIs(t, checkInitJetStreamStore(seedStore(t, "KV_a"), 4, true), errNATSRunning)
+	})
+}

docs/README.md

@@ -2,7 +2,9 @@
 
 This directory contains the Spinifex documentation. This is available in individual markdown files, and a generated website to aid navigation and browsing documentation at: [https://docs.mulgadc.com](https://docs.mulgadc.com/)
 
-The [AWS model operation coverage](aws-model-operation-coverage.md) report is generated from Spinifex's dispatch tables and the pinned AWS API models. Run `make aws-model-coverage` to regenerate and print it.
+The [AWS API coverage](coverage/README.md) pages are generated from Spinifex's dispatch tables and the pinned AWS API models, one page per service. Run `make aws-model-coverage` to regenerate them and print a summary. Only `pages.json` and the `intro.md` files under `coverage/` are hand-written; every `README.md` there is generated and will be overwritten.
+
+A published page names the operations Spinifex serves, and nothing else. It carries no percentage, no unimplemented operation and no operation the platform will never offer: the same run writes those to [`coverage/internal-report.md`](coverage/internal-report.md), which no slug in the docs site's configuration points at, so it stays internal. A `notApplicable` declaration in `pages.json` therefore publishes nothing; it keeps a permanent absence out of the report's candidate work.
 
 ## Frontmatter
 

docs/admin/setting-up-your-cluster/README.md

@@ -314,15 +314,16 @@ On first load, your browser will show a TLS warning — Spinifex generates a sel
 
 1. Accept the warning to reach the login page (exact wording varies by browser — e.g. Chrome: *Advanced → Proceed to ...*, Firefox: *Advanced → Accept the Risk and Continue*).
 2. On the login page, click **Download Certificate** and save `spinifex-ca.pem` to your machine.
-3. Install the certificate as a **trusted root** on your workstation:
-
-   - **macOS:** open `spinifex-ca.pem` in Keychain Access → *System* keychain → set *Trust* to **Always Trust**.
-   - **Linux:** `sudo cp spinifex-ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt && sudo update-ca-certificates`
-   - **Windows:** double-click the file → *Install Certificate* → *Local Machine* → *Trusted Root Certification Authorities*.
-   - **Browser-only (Firefox):** *Settings → Privacy & Security → Certificates → View Certificates → Authorities → Import* and tick *Trust this CA to identify websites*.
-
+3. Install the certificate as a **trusted root** on your workstation, following the steps for your platform below.
 4. Restart your browser and reload `https://YOUR_NODE_IP:3000`. The padlock should now show a valid certificate.
 
+Step 3, by platform:
+
+- **macOS:** open `spinifex-ca.pem` in Keychain Access → *System* keychain → set *Trust* to **Always Trust**.
+- **Linux:** `sudo cp spinifex-ca.pem /usr/local/share/ca-certificates/spinifex-ca.crt && sudo update-ca-certificates`
+- **Windows:** double-click the file → *Install Certificate* → *Local Machine* → *Trusted Root Certification Authorities*.
+- **Browser-only (Firefox):** *Settings → Privacy & Security → Certificates → View Certificates → Authorities → Import* and tick *Trust this CA to identify websites*.
+
 > **Why this is required:** the UI logs in by reading your AWS credentials through a trusted TLS channel. Browsers refuse to send credentials over an untrusted connection, so the certificate must be installed as trusted — temporary "Proceed anyway" exceptions won't work for login.
 
 ### Log In with AWS Credentials

docs/admin/update/README.md

@@ -37,20 +37,6 @@ For operators who want to review migrations before they are applied, a manual up
 > [!WARNING]
 > **Swapping the `spx` binary alone is not an upgrade.** Systemd unit files (`KillMode`, `TimeoutStopSec`, drain ordering, and similar) are written once at install time and never re-asserted just because a new binary is in place — a node "upgraded" by replacing `/usr/local/bin/spx` directly keeps running whatever units it was first installed with, which can silently disagree with the new binary's behaviour. Re-running the installer always reinstalls units unconditionally, so it is unaffected. `spx admin upgrade` now reconciles units too, so it closes this gap for operators who update the binary by hand. See [Checking for Unit Drift](#checking-for-unit-drift).
 
-> [!WARNING]
-> **Object storage is exempt when upgrading from v1.15.0 or earlier to v1.16.0.** Predastore's configuration schema and on-disk layout changed with the object storage cutover in v1.16.0, and no migration converts an installation from before it. `spx admin upgrade` will not report anything pending for `predastore.toml`, and updating the binary over such an installation leaves Predastore unable to start. Those clusters have to be re-initialised from scratch, which discards their stored objects — export anything you need first. Upgrades between v1.16.0 and later releases are unaffected.
-
-> [!WARNING]
-> **AMI metadata is exempt when upgrading from v1.16.0 or earlier to the release carrying the EBS-provider decoupling.** AMI metadata moved to `ebsmetadata` documents, and the legacy path that read it from `ami-<id>/config.json` was removed rather than migrated. There is no prefix scan, no fallback and no backfill at daemon start, so an AMI imported before the change becomes invisible to the control plane afterwards: `describe-images --image-ids` answers `InvalidAMIID.NotFound` and launches fail with `AMI has no snapshot ID, cannot perform zero-copy clone`. As with object storage above, `spx admin upgrade` reports nothing pending, because the gap is in stored data rather than in a config file.
->
-> **Re-import affected AMIs after upgrading**, using [`spx admin images import`](/docs/spinifex-admin-cli), which writes metadata in the new location. If the source images are no longer available, the installation has to be re-initialised. Verify before you rely on it — an AMI is only healthy if it resolves by ID, not merely if it appears in the list:
->
-> ```bash
-> aws ec2 describe-images --image-ids <ami-id>
-> ```
->
-> This warning covers AMI metadata specifically, which is what has been observed. Check any other imported state you depend on before upgrading a cluster you cannot rebuild.
-
 ## Instructions
 
 ## Step 1. Re-run the Installer

docs/aws-model-operation-coverage.md

@@ -1,577 +0,0 @@
-# AWS model operation coverage
-
-Generated from the cached `aws-sdk-go v1.55.8` `api-2.json` models and Spinifex's authoritative gateway dispatch tables.
-
-Implemented means a modelled operation is registered to a real handler. Stub and deliberately unsupported handlers are reported separately. Shape conformance does not imply behavioural conformance.
-
-| Service | API version | Modelled | Registered | Implemented | Stubbed | Unsupported | Missing | Extra |
-|---|---:|---:|---:|---:|---:|---:|---:|---:|
-| acm | 2015-12-08 | 15 | 9 | 9 | 0 | 0 | 6 | 0 |
-| ec2 | 2016-11-15 | 625 | 121 | 121 | 0 | 0 | 504 | 0 |
-| ecr | 2015-09-21 | 47 | 39 | 19 | 13 | 7 | 11 | 3 |
-| ecs | 2014-11-13 | 56 | 39 | 31 | 5 | 0 | 20 | 3 |
-| elasticloadbalancingv2 | 2015-12-01 | 46 | 37 | 33 | 0 | 0 | 13 | 4 |
-| iam | 2010-05-08 | 159 | 75 | 75 | 0 | 0 | 84 | 0 |
-| rds | 2014-10-31 | 162 | 43 | 26 | 0 | 12 | 124 | 5 |
-| s3 | 2006-03-01 | 99 | — | — | — | — | — | — |
-| sts | 2011-06-15 | 8 | 4 | 4 | 0 | 0 | 4 | 0 |
-
-## acm
-
-Implements **9 of 15** modelled operations (60.0%).
-
-<details><summary>Implemented (9)</summary>
-
-`AddTagsToCertificate`, `DeleteCertificate`, `DescribeCertificate`, `GetCertificate`, `ImportCertificate`, `ListCertificates`, `ListTagsForCertificate`, `RemoveTagsFromCertificate`
-
-`RequestCertificate`
-
-
-</details>
-
-<details><summary>Missing from dispatch (6)</summary>
-
-`ExportCertificate`, `GetAccountConfiguration`, `PutAccountConfiguration`, `RenewCertificate`, `ResendValidationEmail`, `UpdateCertificateOptions`
-
-
-</details>
-
-<details><summary>Registered stubs (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Deliberately unsupported (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Registered outside the pinned model (0)</summary>
-
-None.
-
-</details>
-
-## ec2
-
-Implements **121 of 625** modelled operations (19.4%).
-
-<details><summary>Implemented (121)</summary>
-
-`AllocateAddress`, `AssociateAddress`, `AssociateIamInstanceProfile`, `AssociateRouteTable`, `AttachInternetGateway`, `AttachNetworkInterface`, `AttachVolume`, `AuthorizeSecurityGroupEgress`
-
-`AuthorizeSecurityGroupIngress`, `CancelCapacityReservation`, `CancelSpotInstanceRequests`, `CopyImage`, `CopySnapshot`, `CreateCapacityReservation`, `CreateEgressOnlyInternetGateway`, `CreateImage`
-
-`CreateInternetGateway`, `CreateKeyPair`, `CreateLaunchTemplate`, `CreateLaunchTemplateVersion`, `CreateNatGateway`, `CreateNetworkInterface`, `CreatePlacementGroup`, `CreateRoute`
-
-`CreateRouteTable`, `CreateSecurityGroup`, `CreateSnapshot`, `CreateSubnet`, `CreateTags`, `CreateVolume`, `CreateVpc`, `DeleteEgressOnlyInternetGateway`
-
-`DeleteInternetGateway`, `DeleteKeyPair`, `DeleteLaunchTemplate`, `DeleteLaunchTemplateVersions`, `DeleteNatGateway`, `DeleteNetworkInterface`, `DeletePlacementGroup`, `DeleteRoute`
-
-`DeleteRouteTable`, `DeleteSecurityGroup`, `DeleteSnapshot`, `DeleteSubnet`, `DeleteTags`, `DeleteVolume`, `DeleteVpc`, `DeregisterImage`
-
-`DescribeAccountAttributes`, `DescribeAddresses`, `DescribeAddressesAttribute`, `DescribeAvailabilityZones`, `DescribeCapacityReservations`, `DescribeEgressOnlyInternetGateways`, `DescribeIamInstanceProfileAssociations`, `DescribeImageAttribute`
-
-`DescribeImages`, `DescribeInstanceAttribute`, `DescribeInstanceCreditSpecifications`, `DescribeInstanceStatus`, `DescribeInstanceTypes`, `DescribeInstances`, `DescribeInternetGateways`, `DescribeKeyPairs`
-
-`DescribeLaunchTemplateVersions`, `DescribeLaunchTemplates`, `DescribeNatGateways`, `DescribeNetworkInterfaces`, `DescribePlacementGroups`, `DescribeRegions`, `DescribeRouteTables`, `DescribeSecurityGroupRules`
-
-`DescribeSecurityGroups`, `DescribeSnapshots`, `DescribeSpotInstanceRequests`, `DescribeSubnets`, `DescribeTags`, `DescribeVolumeStatus`, `DescribeVolumes`, `DescribeVolumesModifications`
-
-`DescribeVpcAttribute`, `DescribeVpcs`, `DetachInternetGateway`, `DetachNetworkInterface`, `DetachVolume`, `DisableEbsEncryptionByDefault`, `DisableSerialConsoleAccess`, `DisassociateAddress`
-
-`DisassociateIamInstanceProfile`, `DisassociateRouteTable`, `EnableEbsEncryptionByDefault`, `EnableSerialConsoleAccess`, `GetConsoleOutput`, `GetEbsEncryptionByDefault`, `GetPasswordData`, `GetSerialConsoleAccessStatus`
-
-`ImportKeyPair`, `ModifyImageAttribute`, `ModifyInstanceAttribute`, `ModifyInstanceMetadataOptions`, `ModifyLaunchTemplate`, `ModifyNetworkInterfaceAttribute`, `ModifySubnetAttribute`, `ModifyVolume`
-
-`ModifyVpcAttribute`, `MonitorInstances`, `RebootInstances`, `RegisterImage`, `ReleaseAddress`, `ReplaceIamInstanceProfileAssociation`, `ReplaceRoute`, `ReplaceRouteTableAssociation`
-
-`RequestSpotInstances`, `ResetImageAttribute`, `RevokeSecurityGroupEgress`, `RevokeSecurityGroupIngress`, `RunInstances`, `StartInstances`, `StopInstances`, `TerminateInstances`
-
-`UnmonitorInstances`
-
-
-</details>
-
-<details><summary>Missing from dispatch (504)</summary>
-
-`AcceptAddressTransfer`, `AcceptReservedInstancesExchangeQuote`, `AcceptTransitGatewayMulticastDomainAssociations`, `AcceptTransitGatewayPeeringAttachment`, `AcceptTransitGatewayVpcAttachment`, `AcceptVpcEndpointConnections`, `AcceptVpcPeeringConnection`, `AdvertiseByoipCidr`
-
-`AllocateHosts`, `AllocateIpamPoolCidr`, `ApplySecurityGroupsToClientVpnTargetNetwork`, `AssignIpv6Addresses`, `AssignPrivateIpAddresses`, `AssignPrivateNatGatewayAddress`, `AssociateClientVpnTargetNetwork`, `AssociateDhcpOptions`
-
-`AssociateEnclaveCertificateIamRole`, `AssociateInstanceEventWindow`, `AssociateIpamByoasn`, `AssociateIpamResourceDiscovery`, `AssociateNatGatewayAddress`, `AssociateSubnetCidrBlock`, `AssociateTransitGatewayMulticastDomain`, `AssociateTransitGatewayPolicyTable`
-
-`AssociateTransitGatewayRouteTable`, `AssociateTrunkInterface`, `AssociateVpcCidrBlock`, `AttachClassicLinkVpc`, `AttachVerifiedAccessTrustProvider`, `AttachVpnGateway`, `AuthorizeClientVpnIngress`, `BundleInstance`
-
-`CancelBundleTask`, `CancelCapacityReservationFleets`, `CancelConversionTask`, `CancelExportTask`, `CancelImageLaunchPermission`, `CancelImportTask`, `CancelReservedInstancesListing`, `CancelSpotFleetRequests`
-
-`ConfirmProductInstance`, `CopyFpgaImage`, `CreateCapacityReservationFleet`, `CreateCarrierGateway`, `CreateClientVpnEndpoint`, `CreateClientVpnRoute`, `CreateCoipCidr`, `CreateCoipPool`
-
-`CreateCustomerGateway`, `CreateDefaultSubnet`, `CreateDefaultVpc`, `CreateDhcpOptions`, `CreateFleet`, `CreateFlowLogs`, `CreateFpgaImage`, `CreateInstanceConnectEndpoint`
-
-`CreateInstanceEventWindow`, `CreateInstanceExportTask`, `CreateIpam`, `CreateIpamExternalResourceVerificationToken`, `CreateIpamPool`, `CreateIpamResourceDiscovery`, `CreateIpamScope`, `CreateLocalGatewayRoute`
-
-`CreateLocalGatewayRouteTable`, `CreateLocalGatewayRouteTableVirtualInterfaceGroupAssociation`, `CreateLocalGatewayRouteTableVpcAssociation`, `CreateManagedPrefixList`, `CreateNetworkAcl`, `CreateNetworkAclEntry`, `CreateNetworkInsightsAccessScope`, `CreateNetworkInsightsPath`
-
-`CreateNetworkInterfacePermission`, `CreatePublicIpv4Pool`, `CreateReplaceRootVolumeTask`, `CreateReservedInstancesListing`, `CreateRestoreImageTask`, `CreateSnapshots`, `CreateSpotDatafeedSubscription`, `CreateStoreImageTask`
-
-`CreateSubnetCidrReservation`, `CreateTrafficMirrorFilter`, `CreateTrafficMirrorFilterRule`, `CreateTrafficMirrorSession`, `CreateTrafficMirrorTarget`, `CreateTransitGateway`, `CreateTransitGatewayConnect`, `CreateTransitGatewayConnectPeer`
-
-`CreateTransitGatewayMulticastDomain`, `CreateTransitGatewayPeeringAttachment`, `CreateTransitGatewayPolicyTable`, `CreateTransitGatewayPrefixListReference`, `CreateTransitGatewayRoute`, `CreateTransitGatewayRouteTable`, `CreateTransitGatewayRouteTableAnnouncement`, `CreateTransitGatewayVpcAttachment`
-
-`CreateVerifiedAccessEndpoint`, `CreateVerifiedAccessGroup`, `CreateVerifiedAccessInstance`, `CreateVerifiedAccessTrustProvider`, `CreateVpcEndpoint`, `CreateVpcEndpointConnectionNotification`, `CreateVpcEndpointServiceConfiguration`, `CreateVpcPeeringConnection`
-
-`CreateVpnConnection`, `CreateVpnConnectionRoute`, `CreateVpnGateway`, `DeleteCarrierGateway`, `DeleteClientVpnEndpoint`, `DeleteClientVpnRoute`, `DeleteCoipCidr`, `DeleteCoipPool`
-
-`DeleteCustomerGateway`, `DeleteDhcpOptions`, `DeleteFleets`, `DeleteFlowLogs`, `DeleteFpgaImage`, `DeleteInstanceConnectEndpoint`, `DeleteInstanceEventWindow`, `DeleteIpam`
-
-`DeleteIpamExternalResourceVerificationToken`, `DeleteIpamPool`, `DeleteIpamResourceDiscovery`, `DeleteIpamScope`, `DeleteLocalGatewayRoute`, `DeleteLocalGatewayRouteTable`, `DeleteLocalGatewayRouteTableVirtualInterfaceGroupAssociation`, `DeleteLocalGatewayRouteTableVpcAssociation`
-
-`DeleteManagedPrefixList`, `DeleteNetworkAcl`, `DeleteNetworkAclEntry`, `DeleteNetworkInsightsAccessScope`, `DeleteNetworkInsightsAccessScopeAnalysis`, `DeleteNetworkInsightsAnalysis`, `DeleteNetworkInsightsPath`, `DeleteNetworkInterfacePermission`
-
-`DeletePublicIpv4Pool`, `DeleteQueuedReservedInstances`, `DeleteSpotDatafeedSubscription`, `DeleteSubnetCidrReservation`, `DeleteTrafficMirrorFilter`, `DeleteTrafficMirrorFilterRule`, `DeleteTrafficMirrorSession`, `DeleteTrafficMirrorTarget`
-
-`DeleteTransitGateway`, `DeleteTransitGatewayConnect`, `DeleteTransitGatewayConnectPeer`, `DeleteTransitGatewayMulticastDomain`, `DeleteTransitGatewayPeeringAttachment`, `DeleteTransitGatewayPolicyTable`, `DeleteTransitGatewayPrefixListReference`, `DeleteTransitGatewayRoute`
-
-`DeleteTransitGatewayRouteTable`, `DeleteTransitGatewayRouteTableAnnouncement`, `DeleteTransitGatewayVpcAttachment`, `DeleteVerifiedAccessEndpoint`, `DeleteVerifiedAccessGroup`, `DeleteVerifiedAccessInstance`, `DeleteVerifiedAccessTrustProvider`, `DeleteVpcEndpointConnectionNotifications`
-
-`DeleteVpcEndpointServiceConfigurations`, `DeleteVpcEndpoints`, `DeleteVpcPeeringConnection`, `DeleteVpnConnection`, `DeleteVpnConnectionRoute`, `DeleteVpnGateway`, `DeprovisionByoipCidr`, `DeprovisionIpamByoasn`
-
-`DeprovisionIpamPoolCidr`, `DeprovisionPublicIpv4PoolCidr`, `DeregisterInstanceEventNotificationAttributes`, `DeregisterTransitGatewayMulticastGroupMembers`, `DeregisterTransitGatewayMulticastGroupSources`, `DescribeAddressTransfers`, `DescribeAggregateIdFormat`, `DescribeAwsNetworkPerformanceMetricSubscriptions`
-
-`DescribeBundleTasks`, `DescribeByoipCidrs`, `DescribeCapacityBlockOfferings`, `DescribeCapacityReservationFleets`, `DescribeCarrierGateways`, `DescribeClassicLinkInstances`, `DescribeClientVpnAuthorizationRules`, `DescribeClientVpnConnections`
-
-`DescribeClientVpnEndpoints`, `DescribeClientVpnRoutes`, `DescribeClientVpnTargetNetworks`, `DescribeCoipPools`, `DescribeConversionTasks`, `DescribeCustomerGateways`, `DescribeDhcpOptions`, `DescribeElasticGpus`
-
-`DescribeExportImageTasks`, `DescribeExportTasks`, `DescribeFastLaunchImages`, `DescribeFastSnapshotRestores`, `DescribeFleetHistory`, `DescribeFleetInstances`, `DescribeFleets`, `DescribeFlowLogs`
-
-`DescribeFpgaImageAttribute`, `DescribeFpgaImages`, `DescribeHostReservationOfferings`, `DescribeHostReservations`, `DescribeHosts`, `DescribeIdFormat`, `DescribeIdentityIdFormat`, `DescribeImportImageTasks`
-
-`DescribeImportSnapshotTasks`, `DescribeInstanceConnectEndpoints`, `DescribeInstanceEventNotificationAttributes`, `DescribeInstanceEventWindows`, `DescribeInstanceTopology`, `DescribeInstanceTypeOfferings`, `DescribeIpamByoasn`, `DescribeIpamExternalResourceVerificationTokens`
-
-`DescribeIpamPools`, `DescribeIpamResourceDiscoveries`, `DescribeIpamResourceDiscoveryAssociations`, `DescribeIpamScopes`, `DescribeIpams`, `DescribeIpv6Pools`, `DescribeLocalGatewayRouteTableVirtualInterfaceGroupAssociations`, `DescribeLocalGatewayRouteTableVpcAssociations`
-
-`DescribeLocalGatewayRouteTables`, `DescribeLocalGatewayVirtualInterfaceGroups`, `DescribeLocalGatewayVirtualInterfaces`, `DescribeLocalGateways`, `DescribeLockedSnapshots`, `DescribeMacHosts`, `DescribeManagedPrefixLists`, `DescribeMovingAddresses`
-
-`DescribeNetworkAcls`, `DescribeNetworkInsightsAccessScopeAnalyses`, `DescribeNetworkInsightsAccessScopes`, `DescribeNetworkInsightsAnalyses`, `DescribeNetworkInsightsPaths`, `DescribeNetworkInterfaceAttribute`, `DescribeNetworkInterfacePermissions`, `DescribePrefixLists`
-
-`DescribePrincipalIdFormat`, `DescribePublicIpv4Pools`, `DescribeReplaceRootVolumeTasks`, `DescribeReservedInstances`, `DescribeReservedInstancesListings`, `DescribeReservedInstancesModifications`, `DescribeReservedInstancesOfferings`, `DescribeScheduledInstanceAvailability`
-
-`DescribeScheduledInstances`, `DescribeSecurityGroupReferences`, `DescribeSnapshotAttribute`, `DescribeSnapshotTierStatus`, `DescribeSpotDatafeedSubscription`, `DescribeSpotFleetInstances`, `DescribeSpotFleetRequestHistory`, `DescribeSpotFleetRequests`
-
-`DescribeSpotPriceHistory`, `DescribeStaleSecurityGroups`, `DescribeStoreImageTasks`, `DescribeTrafficMirrorFilterRules`, `DescribeTrafficMirrorFilters`, `DescribeTrafficMirrorSessions`, `DescribeTrafficMirrorTargets`, `DescribeTransitGatewayAttachments`
-
-`DescribeTransitGatewayConnectPeers`, `DescribeTransitGatewayConnects`, `DescribeTransitGatewayMulticastDomains`, `DescribeTransitGatewayPeeringAttachments`, `DescribeTransitGatewayPolicyTables`, `DescribeTransitGatewayRouteTableAnnouncements`, `DescribeTransitGatewayRouteTables`, `DescribeTransitGatewayVpcAttachments`
-
-`DescribeTransitGateways`, `DescribeTrunkInterfaceAssociations`, `DescribeVerifiedAccessEndpoints`, `DescribeVerifiedAccessGroups`, `DescribeVerifiedAccessInstanceLoggingConfigurations`, `DescribeVerifiedAccessInstances`, `DescribeVerifiedAccessTrustProviders`, `DescribeVolumeAttribute`
-
-`DescribeVpcClassicLink`, `DescribeVpcClassicLinkDnsSupport`, `DescribeVpcEndpointConnectionNotifications`, `DescribeVpcEndpointConnections`, `DescribeVpcEndpointServiceConfigurations`, `DescribeVpcEndpointServicePermissions`, `DescribeVpcEndpointServices`, `DescribeVpcEndpoints`
-
-`DescribeVpcPeeringConnections`, `DescribeVpnConnections`, `DescribeVpnGateways`, `DetachClassicLinkVpc`, `DetachVerifiedAccessTrustProvider`, `DetachVpnGateway`, `DisableAddressTransfer`, `DisableAwsNetworkPerformanceMetricSubscription`
-
-`DisableFastLaunch`, `DisableFastSnapshotRestores`, `DisableImage`, `DisableImageBlockPublicAccess`, `DisableImageDeprecation`, `DisableImageDeregistrationProtection`, `DisableIpamOrganizationAdminAccount`, `DisableSnapshotBlockPublicAccess`
-
-`DisableTransitGatewayRouteTablePropagation`, `DisableVgwRoutePropagation`, `DisableVpcClassicLink`, `DisableVpcClassicLinkDnsSupport`, `DisassociateClientVpnTargetNetwork`, `DisassociateEnclaveCertificateIamRole`, `DisassociateInstanceEventWindow`, `DisassociateIpamByoasn`
-
-`DisassociateIpamResourceDiscovery`, `DisassociateNatGatewayAddress`, `DisassociateSubnetCidrBlock`, `DisassociateTransitGatewayMulticastDomain`, `DisassociateTransitGatewayPolicyTable`, `DisassociateTransitGatewayRouteTable`, `DisassociateTrunkInterface`, `DisassociateVpcCidrBlock`
-
-`EnableAddressTransfer`, `EnableAwsNetworkPerformanceMetricSubscription`, `EnableFastLaunch`, `EnableFastSnapshotRestores`, `EnableImage`, `EnableImageBlockPublicAccess`, `EnableImageDeprecation`, `EnableImageDeregistrationProtection`
-
-`EnableIpamOrganizationAdminAccount`, `EnableReachabilityAnalyzerOrganizationSharing`, `EnableSnapshotBlockPublicAccess`, `EnableTransitGatewayRouteTablePropagation`, `EnableVgwRoutePropagation`, `EnableVolumeIO`, `EnableVpcClassicLink`, `EnableVpcClassicLinkDnsSupport`
-
-`ExportClientVpnClientCertificateRevocationList`, `ExportClientVpnClientConfiguration`, `ExportImage`, `ExportTransitGatewayRoutes`, `GetAssociatedEnclaveCertificateIamRoles`, `GetAssociatedIpv6PoolCidrs`, `GetAwsNetworkPerformanceData`, `GetCapacityReservationUsage`
-
-`GetCoipPoolUsage`, `GetConsoleScreenshot`, `GetDefaultCreditSpecification`, `GetEbsDefaultKmsKeyId`, `GetFlowLogsIntegrationTemplate`, `GetGroupsForCapacityReservation`, `GetHostReservationPurchasePreview`, `GetImageBlockPublicAccessState`
-
-`GetInstanceMetadataDefaults`, `GetInstanceTpmEkPub`, `GetInstanceTypesFromInstanceRequirements`, `GetInstanceUefiData`, `GetIpamAddressHistory`, `GetIpamDiscoveredAccounts`, `GetIpamDiscoveredPublicAddresses`, `GetIpamDiscoveredResourceCidrs`
-
-`GetIpamPoolAllocations`, `GetIpamPoolCidrs`, `GetIpamResourceCidrs`, `GetLaunchTemplateData`, `GetManagedPrefixListAssociations`, `GetManagedPrefixListEntries`, `GetNetworkInsightsAccessScopeAnalysisFindings`, `GetNetworkInsightsAccessScopeContent`
-
-`GetReservedInstancesExchangeQuote`, `GetSecurityGroupsForVpc`, `GetSnapshotBlockPublicAccessState`, `GetSpotPlacementScores`, `GetSubnetCidrReservations`, `GetTransitGatewayAttachmentPropagations`, `GetTransitGatewayMulticastDomainAssociations`, `GetTransitGatewayPolicyTableAssociations`
-
-`GetTransitGatewayPolicyTableEntries`, `GetTransitGatewayPrefixListReferences`, `GetTransitGatewayRouteTableAssociations`, `GetTransitGatewayRouteTablePropagations`, `GetVerifiedAccessEndpointPolicy`, `GetVerifiedAccessGroupPolicy`, `GetVpnConnectionDeviceSampleConfiguration`, `GetVpnConnectionDeviceTypes`
-
-`GetVpnTunnelReplacementStatus`, `ImportClientVpnClientCertificateRevocationList`, `ImportImage`, `ImportInstance`, `ImportSnapshot`, `ImportVolume`, `ListImagesInRecycleBin`, `ListSnapshotsInRecycleBin`
-
-`LockSnapshot`, `ModifyAddressAttribute`, `ModifyAvailabilityZoneGroup`, `ModifyCapacityReservation`, `ModifyCapacityReservationFleet`, `ModifyClientVpnEndpoint`, `ModifyDefaultCreditSpecification`, `ModifyEbsDefaultKmsKeyId`
-
-`ModifyFleet`, `ModifyFpgaImageAttribute`, `ModifyHosts`, `ModifyIdFormat`, `ModifyIdentityIdFormat`, `ModifyInstanceCapacityReservationAttributes`, `ModifyInstanceCreditSpecification`, `ModifyInstanceEventStartTime`
-
-`ModifyInstanceEventWindow`, `ModifyInstanceMaintenanceOptions`, `ModifyInstanceMetadataDefaults`, `ModifyInstancePlacement`, `ModifyIpam`, `ModifyIpamPool`, `ModifyIpamResourceCidr`, `ModifyIpamResourceDiscovery`
-
-`ModifyIpamScope`, `ModifyLocalGatewayRoute`, `ModifyManagedPrefixList`, `ModifyPrivateDnsNameOptions`, `ModifyReservedInstances`, `ModifySecurityGroupRules`, `ModifySnapshotAttribute`, `ModifySnapshotTier`
-
-`ModifySpotFleetRequest`, `ModifyTrafficMirrorFilterNetworkServices`, `ModifyTrafficMirrorFilterRule`, `ModifyTrafficMirrorSession`, `ModifyTransitGateway`, `ModifyTransitGatewayPrefixListReference`, `ModifyTransitGatewayVpcAttachment`, `ModifyVerifiedAccessEndpoint`
-
-`ModifyVerifiedAccessEndpointPolicy`, `ModifyVerifiedAccessGroup`, `ModifyVerifiedAccessGroupPolicy`, `ModifyVerifiedAccessInstance`, `ModifyVerifiedAccessInstanceLoggingConfiguration`, `ModifyVerifiedAccessTrustProvider`, `ModifyVolumeAttribute`, `ModifyVpcEndpoint`
-
-`ModifyVpcEndpointConnectionNotification`, `ModifyVpcEndpointServiceConfiguration`, `ModifyVpcEndpointServicePayerResponsibility`, `ModifyVpcEndpointServicePermissions`, `ModifyVpcPeeringConnectionOptions`, `ModifyVpcTenancy`, `ModifyVpnConnection`, `ModifyVpnConnectionOptions`
-
-`ModifyVpnTunnelCertificate`, `ModifyVpnTunnelOptions`, `MoveAddressToVpc`, `MoveByoipCidrToIpam`, `ProvisionByoipCidr`, `ProvisionIpamByoasn`, `ProvisionIpamPoolCidr`, `ProvisionPublicIpv4PoolCidr`
-
-`PurchaseCapacityBlock`, `PurchaseHostReservation`, `PurchaseReservedInstancesOffering`, `PurchaseScheduledInstances`, `RegisterInstanceEventNotificationAttributes`, `RegisterTransitGatewayMulticastGroupMembers`, `RegisterTransitGatewayMulticastGroupSources`, `RejectTransitGatewayMulticastDomainAssociations`
-
-`RejectTransitGatewayPeeringAttachment`, `RejectTransitGatewayVpcAttachment`, `RejectVpcEndpointConnections`, `RejectVpcPeeringConnection`, `ReleaseHosts`, `ReleaseIpamPoolAllocation`, `ReplaceNetworkAclAssociation`, `ReplaceNetworkAclEntry`
-
-`ReplaceTransitGatewayRoute`, `ReplaceVpnTunnel`, `ReportInstanceStatus`, `RequestSpotFleet`, `ResetAddressAttribute`, `ResetEbsDefaultKmsKeyId`, `ResetFpgaImageAttribute`, `ResetInstanceAttribute`
-
-`ResetNetworkInterfaceAttribute`, `ResetSnapshotAttribute`, `RestoreAddressToClassic`, `RestoreImageFromRecycleBin`, `RestoreManagedPrefixListVersion`, `RestoreSnapshotFromRecycleBin`, `RestoreSnapshotTier`, `RevokeClientVpnIngress`
-
-`RunScheduledInstances`, `SearchLocalGatewayRoutes`, `SearchTransitGatewayMulticastGroups`, `SearchTransitGatewayRoutes`, `SendDiagnosticInterrupt`, `StartNetworkInsightsAccessScopeAnalysis`, `StartNetworkInsightsAnalysis`, `StartVpcEndpointServicePrivateDnsVerification`
-
-`TerminateClientVpnConnections`, `UnassignIpv6Addresses`, `UnassignPrivateIpAddresses`, `UnassignPrivateNatGatewayAddress`, `UnlockSnapshot`, `UpdateSecurityGroupRuleDescriptionsEgress`, `UpdateSecurityGroupRuleDescriptionsIngress`, `WithdrawByoipCidr`
-
-
-</details>
-
-<details><summary>Registered stubs (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Deliberately unsupported (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Registered outside the pinned model (0)</summary>
-
-None.
-
-</details>
-
-## ecr
-
-Implements **19 of 47** modelled operations (40.4%).
-
-<details><summary>Implemented (19)</summary>
-
-`BatchDeleteImage`, `BatchGetImage`, `CreateRepository`, `DeleteLifecyclePolicy`, `DeleteRepository`, `DeleteRepositoryPolicy`, `DescribeImages`, `DescribeRepositories`
-
-`GetAuthorizationToken`, `GetLifecyclePolicy`, `GetLifecyclePolicyPreview`, `GetRepositoryPolicy`, `ListImages`, `ListTagsForResource`, `PutImage`, `PutImageTagMutability`
-
-`PutLifecyclePolicy`, `SetRepositoryPolicy`, `StartLifecyclePolicyPreview`
-
-
-</details>
-
-<details><summary>Missing from dispatch (11)</summary>
-
-`CreatePullThroughCacheRule`, `CreateRepositoryCreationTemplate`, `DeletePullThroughCacheRule`, `DeleteRegistryPolicy`, `DeleteRepositoryCreationTemplate`, `DescribeImageReplicationStatus`, `DescribePullThroughCacheRules`, `DescribeRepositoryCreationTemplates`
-
-`UpdatePullThroughCacheRule`, `UpdateRepositoryCreationTemplate`, `ValidatePullThroughCacheRule`
-
-
-</details>
-
-<details><summary>Registered stubs (13)</summary>
-
-`BatchCheckLayerAvailability`, `CompleteLayerUpload`, `DescribeRegistry`, `GetDownloadUrlForLayer`, `GetRegistryPolicy`, `InitiateLayerUpload`, `ListRepositories`, `PutRegistryPolicy`
-
-`PutReplicationConfiguration`, `ReplicateImage`, `TagResource`, `UntagResource`, `UploadLayerPart`
-
-
-</details>
-
-<details><summary>Deliberately unsupported (7)</summary>
-
-`BatchGetRepositoryScanningConfiguration`, `DescribeImageScanFindings`, `GetImageScanningConfiguration`, `GetRegistryScanningConfiguration`, `PutImageScanningConfiguration`, `PutRegistryScanningConfiguration`, `StartImageScan`
-
-
-</details>
-
-<details><summary>Registered outside the pinned model (3)</summary>
-
-`GetImageScanningConfiguration`, `ListRepositories`, `ReplicateImage`
-
-
-</details>
-
-## ecs
-
-Implements **31 of 56** modelled operations (55.4%).
-
-<details><summary>Implemented (31)</summary>
-
-`CreateCapacityProvider`, `CreateCluster`, `CreateService`, `DeleteCapacityProvider`, `DeleteCluster`, `DeleteService`, `DeregisterContainerInstance`, `DeregisterTaskDefinition`
-
-`DescribeCapacityProviders`, `DescribeClusters`, `DescribeContainerInstances`, `DescribeServices`, `DescribeTaskDefinition`, `DescribeTasks`, `ListClusters`, `ListContainerInstances`
-
-`ListServices`, `ListTagsForResource`, `ListTaskDefinitions`, `ListTasks`, `PutClusterCapacityProviders`, `RegisterContainerInstance`, `RegisterTaskDefinition`, `RunTask`
-
-`StartTask`, `StopTask`, `SubmitTaskStateChange`, `TagResource`, `UntagResource`, `UpdateContainerInstancesState`, `UpdateService`
-
-
-</details>
-
-<details><summary>Missing from dispatch (20)</summary>
-
-`CreateTaskSet`, `DeleteAccountSetting`, `DeleteAttributes`, `DeleteTaskDefinitions`, `DeleteTaskSet`, `DescribeTaskSets`, `DiscoverPollEndpoint`, `ExecuteCommand`
-
-`GetTaskProtection`, `ListAttributes`, `PutAccountSettingDefault`, `PutAttributes`, `SubmitAttachmentStateChanges`, `SubmitContainerStateChange`, `UpdateCapacityProvider`, `UpdateClusterSettings`
-
-`UpdateContainerAgent`, `UpdateServicePrimaryTaskSet`, `UpdateTaskProtection`, `UpdateTaskSet`
-
-
-</details>
-
-<details><summary>Registered stubs (5)</summary>
-
-`ListAccountSettings`, `ListServicesByNamespace`, `ListTaskDefinitionFamilies`, `PutAccountSetting`, `UpdateCluster`
-
-
-</details>
-
-<details><summary>Deliberately unsupported (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Registered outside the pinned model (3)</summary>
-
-`PollAssignments`, `ProvisionCapacity`, `ReportTaskGPU`
-
-
-</details>
-
-## elasticloadbalancingv2
-
-Implements **33 of 46** modelled operations (71.7%).
-
-<details><summary>Implemented (33)</summary>
-
-`AddListenerCertificates`, `AddTags`, `CreateListener`, `CreateLoadBalancer`, `CreateRule`, `CreateTargetGroup`, `DeleteListener`, `DeleteLoadBalancer`
-
-`DeleteRule`, `DeleteTargetGroup`, `DeregisterTargets`, `DescribeListenerCertificates`, `DescribeListeners`, `DescribeLoadBalancerAttributes`, `DescribeLoadBalancers`, `DescribeRules`
-
-`DescribeSSLPolicies`, `DescribeTags`, `DescribeTargetGroupAttributes`, `DescribeTargetGroups`, `DescribeTargetHealth`, `ModifyListener`, `ModifyLoadBalancerAttributes`, `ModifyRule`
-
-`ModifyTargetGroup`, `ModifyTargetGroupAttributes`, `RegisterTargets`, `RemoveListenerCertificates`, `RemoveTags`, `SetIpAddressType`, `SetRulePriorities`, `SetSecurityGroups`
-
-`SetSubnets`
-
-
-</details>
-
-<details><summary>Missing from dispatch (13)</summary>
-
-`AddTrustStoreRevocations`, `CreateTrustStore`, `DeleteSharedTrustStoreAssociation`, `DeleteTrustStore`, `DescribeAccountLimits`, `DescribeTrustStoreAssociations`, `DescribeTrustStoreRevocations`, `DescribeTrustStores`
-
-`GetResourcePolicy`, `GetTrustStoreCaCertificatesBundle`, `GetTrustStoreRevocationContent`, `ModifyTrustStore`, `RemoveTrustStoreRevocations`
-
-
-</details>
-
-<details><summary>Registered stubs (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Deliberately unsupported (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Registered outside the pinned model (4)</summary>
-
-`DescribeListenerAttributes`, `GetLBConfig`, `LBAgentHeartbeat`, `ModifyListenerAttributes`
-
-
-</details>
-
-## iam
-
-Implements **75 of 159** modelled operations (47.2%).
-
-<details><summary>Implemented (75)</summary>
-
-`AddRoleToInstanceProfile`, `AddUserToGroup`, `AttachGroupPolicy`, `AttachRolePolicy`, `AttachUserPolicy`, `CreateAccessKey`, `CreateGroup`, `CreateInstanceProfile`
-
-`CreateOpenIDConnectProvider`, `CreatePolicy`, `CreateRole`, `CreateUser`, `DeleteAccessKey`, `DeleteGroup`, `DeleteGroupPolicy`, `DeleteInstanceProfile`
-
-`DeleteOpenIDConnectProvider`, `DeletePolicy`, `DeleteRole`, `DeleteRolePolicy`, `DeleteUser`, `DeleteUserPolicy`, `DetachGroupPolicy`, `DetachRolePolicy`
-
-`DetachUserPolicy`, `GetAccountSummary`, `GetGroup`, `GetGroupPolicy`, `GetInstanceProfile`, `GetOpenIDConnectProvider`, `GetPolicy`, `GetPolicyVersion`
-
-`GetRole`, `GetRolePolicy`, `GetUser`, `GetUserPolicy`, `ListAccessKeys`, `ListAttachedGroupPolicies`, `ListAttachedRolePolicies`, `ListAttachedUserPolicies`
-
-`ListGroupPolicies`, `ListGroups`, `ListGroupsForUser`, `ListInstanceProfileTags`, `ListInstanceProfiles`, `ListInstanceProfilesForRole`, `ListOpenIDConnectProviderTags`, `ListOpenIDConnectProviders`
-
-`ListPolicies`, `ListPolicyTags`, `ListPolicyVersions`, `ListRolePolicies`, `ListRoleTags`, `ListRoles`, `ListUserPolicies`, `ListUserTags`
-
-`ListUsers`, `PutGroupPolicy`, `PutRolePolicy`, `PutUserPolicy`, `RemoveRoleFromInstanceProfile`, `RemoveUserFromGroup`, `TagInstanceProfile`, `TagOpenIDConnectProvider`
-
-`TagPolicy`, `TagRole`, `TagUser`, `UntagInstanceProfile`, `UntagOpenIDConnectProvider`, `UntagPolicy`, `UntagRole`, `UntagUser`
-
-`UpdateAccessKey`, `UpdateAssumeRolePolicy`, `UpdateRole`
-
-
-</details>
-
-<details><summary>Missing from dispatch (84)</summary>
-
-`AddClientIDToOpenIDConnectProvider`, `ChangePassword`, `CreateAccountAlias`, `CreateLoginProfile`, `CreatePolicyVersion`, `CreateSAMLProvider`, `CreateServiceLinkedRole`, `CreateServiceSpecificCredential`
-
-`CreateVirtualMFADevice`, `DeactivateMFADevice`, `DeleteAccountAlias`, `DeleteAccountPasswordPolicy`, `DeleteLoginProfile`, `DeletePolicyVersion`, `DeleteRolePermissionsBoundary`, `DeleteSAMLProvider`
-
-`DeleteSSHPublicKey`, `DeleteServerCertificate`, `DeleteServiceLinkedRole`, `DeleteServiceSpecificCredential`, `DeleteSigningCertificate`, `DeleteUserPermissionsBoundary`, `DeleteVirtualMFADevice`, `EnableMFADevice`
-
-`GenerateCredentialReport`, `GenerateOrganizationsAccessReport`, `GenerateServiceLastAccessedDetails`, `GetAccessKeyLastUsed`, `GetAccountAuthorizationDetails`, `GetAccountPasswordPolicy`, `GetContextKeysForCustomPolicy`, `GetContextKeysForPrincipalPolicy`
-
-`GetCredentialReport`, `GetLoginProfile`, `GetMFADevice`, `GetOrganizationsAccessReport`, `GetSAMLProvider`, `GetSSHPublicKey`, `GetServerCertificate`, `GetServiceLastAccessedDetails`
-
-`GetServiceLastAccessedDetailsWithEntities`, `GetServiceLinkedRoleDeletionStatus`, `ListAccountAliases`, `ListEntitiesForPolicy`, `ListMFADeviceTags`, `ListMFADevices`, `ListPoliciesGrantingServiceAccess`, `ListSAMLProviderTags`
-
-`ListSAMLProviders`, `ListSSHPublicKeys`, `ListServerCertificateTags`, `ListServerCertificates`, `ListServiceSpecificCredentials`, `ListSigningCertificates`, `ListVirtualMFADevices`, `PutRolePermissionsBoundary`
-
-`PutUserPermissionsBoundary`, `RemoveClientIDFromOpenIDConnectProvider`, `ResetServiceSpecificCredential`, `ResyncMFADevice`, `SetDefaultPolicyVersion`, `SetSecurityTokenServicePreferences`, `SimulateCustomPolicy`, `SimulatePrincipalPolicy`
-
-`TagMFADevice`, `TagSAMLProvider`, `TagServerCertificate`, `UntagMFADevice`, `UntagSAMLProvider`, `UntagServerCertificate`, `UpdateAccountPasswordPolicy`, `UpdateGroup`
-
-`UpdateLoginProfile`, `UpdateOpenIDConnectProviderThumbprint`, `UpdateRoleDescription`, `UpdateSAMLProvider`, `UpdateSSHPublicKey`, `UpdateServerCertificate`, `UpdateServiceSpecificCredential`, `UpdateSigningCertificate`
-
-`UpdateUser`, `UploadSSHPublicKey`, `UploadServerCertificate`, `UploadSigningCertificate`
-
-
-</details>
-
-<details><summary>Registered stubs (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Deliberately unsupported (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Registered outside the pinned model (0)</summary>
-
-None.
-
-</details>
-
-## rds
-
-Implements **26 of 162** modelled operations (16.0%).
-
-<details><summary>Implemented (26)</summary>
-
-`AddTagsToResource`, `CreateDBInstance`, `CreateDBParameterGroup`, `CreateDBSnapshot`, `CreateDBSubnetGroup`, `DeleteDBInstance`, `DeleteDBParameterGroup`, `DeleteDBSnapshot`
-
-`DeleteDBSubnetGroup`, `DescribeDBEngineVersions`, `DescribeDBInstanceAutomatedBackups`, `DescribeDBInstances`, `DescribeDBParameterGroups`, `DescribeDBParameters`, `DescribeDBSnapshots`, `DescribeDBSubnetGroups`
-
-`DescribeEvents`, `DescribeOrderableDBInstanceOptions`, `ListTagsForResource`, `ModifyDBInstance`, `ModifyDBParameterGroup`, `RebootDBInstance`, `RemoveTagsFromResource`, `RestoreDBInstanceFromDBSnapshot`
-
-`StartDBInstance`, `StopDBInstance`
-
-
-</details>
-
-<details><summary>Missing from dispatch (124)</summary>
-
-`AddRoleToDBCluster`, `AddRoleToDBInstance`, `AddSourceIdentifierToSubscription`, `ApplyPendingMaintenanceAction`, `AuthorizeDBSecurityGroupIngress`, `BacktrackDBCluster`, `CancelExportTask`, `CopyDBClusterParameterGroup`
-
-`CopyDBClusterSnapshot`, `CopyDBParameterGroup`, `CopyDBSnapshot`, `CopyOptionGroup`, `CreateBlueGreenDeployment`, `CreateCustomDBEngineVersion`, `CreateDBClusterEndpoint`, `CreateDBClusterParameterGroup`
-
-`CreateDBClusterSnapshot`, `CreateDBProxy`, `CreateDBProxyEndpoint`, `CreateDBSecurityGroup`, `CreateDBShardGroup`, `CreateEventSubscription`, `CreateGlobalCluster`, `CreateIntegration`
-
-`CreateTenantDatabase`, `DeleteBlueGreenDeployment`, `DeleteCustomDBEngineVersion`, `DeleteDBClusterAutomatedBackup`, `DeleteDBClusterEndpoint`, `DeleteDBClusterParameterGroup`, `DeleteDBClusterSnapshot`, `DeleteDBInstanceAutomatedBackup`
-
-`DeleteDBProxy`, `DeleteDBProxyEndpoint`, `DeleteDBSecurityGroup`, `DeleteDBShardGroup`, `DeleteEventSubscription`, `DeleteGlobalCluster`, `DeleteIntegration`, `DeleteTenantDatabase`
-
-`DeregisterDBProxyTargets`, `DescribeAccountAttributes`, `DescribeBlueGreenDeployments`, `DescribeCertificates`, `DescribeDBClusterAutomatedBackups`, `DescribeDBClusterBacktracks`, `DescribeDBClusterEndpoints`, `DescribeDBClusterParameterGroups`
-
-`DescribeDBClusterParameters`, `DescribeDBClusterSnapshotAttributes`, `DescribeDBClusterSnapshots`, `DescribeDBLogFiles`, `DescribeDBProxies`, `DescribeDBProxyEndpoints`, `DescribeDBProxyTargetGroups`, `DescribeDBProxyTargets`
-
-`DescribeDBRecommendations`, `DescribeDBSecurityGroups`, `DescribeDBShardGroups`, `DescribeDBSnapshotAttributes`, `DescribeDBSnapshotTenantDatabases`, `DescribeEngineDefaultClusterParameters`, `DescribeEngineDefaultParameters`, `DescribeEventCategories`
-
-`DescribeEventSubscriptions`, `DescribeExportTasks`, `DescribeGlobalClusters`, `DescribeIntegrations`, `DescribeOptionGroupOptions`, `DescribePendingMaintenanceActions`, `DescribeReservedDBInstances`, `DescribeReservedDBInstancesOfferings`
-
-`DescribeSourceRegions`, `DescribeTenantDatabases`, `DescribeValidDBInstanceModifications`, `DisableHttpEndpoint`, `DownloadDBLogFilePortion`, `EnableHttpEndpoint`, `FailoverGlobalCluster`, `ModifyActivityStream`
-
-`ModifyCertificates`, `ModifyCurrentDBClusterCapacity`, `ModifyCustomDBEngineVersion`, `ModifyDBClusterEndpoint`, `ModifyDBClusterParameterGroup`, `ModifyDBClusterSnapshotAttribute`, `ModifyDBProxy`, `ModifyDBProxyEndpoint`
-
-`ModifyDBProxyTargetGroup`, `ModifyDBRecommendation`, `ModifyDBShardGroup`, `ModifyDBSnapshot`, `ModifyDBSnapshotAttribute`, `ModifyDBSubnetGroup`, `ModifyEventSubscription`, `ModifyGlobalCluster`
-
-`ModifyIntegration`, `ModifyTenantDatabase`, `PromoteReadReplicaDBCluster`, `PurchaseReservedDBInstancesOffering`, `RebootDBCluster`, `RebootDBShardGroup`, `RegisterDBProxyTargets`, `RemoveFromGlobalCluster`
-
-`RemoveRoleFromDBCluster`, `RemoveRoleFromDBInstance`, `RemoveSourceIdentifierFromSubscription`, `ResetDBClusterParameterGroup`, `ResetDBParameterGroup`, `RestoreDBClusterFromS3`, `RestoreDBClusterFromSnapshot`, `RestoreDBClusterToPointInTime`
-
-`RestoreDBInstanceFromS3`, `RevokeDBSecurityGroupIngress`, `StartActivityStream`, `StartDBCluster`, `StartDBInstanceAutomatedBackupsReplication`, `StartExportTask`, `StopActivityStream`, `StopDBCluster`
-
-`StopDBInstanceAutomatedBackupsReplication`, `SwitchoverBlueGreenDeployment`, `SwitchoverGlobalCluster`, `SwitchoverReadReplica`
-
-
-</details>
-
-<details><summary>Registered stubs (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Deliberately unsupported (12)</summary>
-
-`CreateDBCluster`, `CreateDBInstanceReadReplica`, `CreateOptionGroup`, `DeleteDBCluster`, `DeleteOptionGroup`, `DescribeDBClusters`, `DescribeOptionGroups`, `FailoverDBCluster`
-
-`ModifyDBCluster`, `ModifyOptionGroup`, `PromoteReadReplica`, `RestoreDBInstanceToPointInTime`
-
-
-</details>
-
-<details><summary>Registered outside the pinned model (5)</summary>
-
-`AcknowledgeDBBootstrap`, `GetDBBootstrapConfig`, `PollDBCommands`, `RegisterDBInstance`, `SubmitDBStateChange`
-
-
-</details>
-
-## s3
-
-Operation coverage is not enumerable: Spinifex delegates the S3 REST surface to Predastore, which has no operation-name dispatch table to compare mechanically.
-
-## sts
-
-Implements **4 of 8** modelled operations (50.0%).
-
-<details><summary>Implemented (4)</summary>
-
-`AssumeRole`, `AssumeRoleWithWebIdentity`, `GetCallerIdentity`, `GetSessionToken`
-
-
-</details>
-
-<details><summary>Missing from dispatch (4)</summary>
-
-`AssumeRoleWithSAML`, `DecodeAuthorizationMessage`, `GetAccessKeyInfo`, `GetFederationToken`
-
-
-</details>
-
-<details><summary>Registered stubs (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Deliberately unsupported (0)</summary>
-
-None.
-
-</details>
-
-<details><summary>Registered outside the pinned model (0)</summary>
-
-None.
-
-</details>

docs/compute/vpc-networking/README.md

@@ -842,18 +842,8 @@ one entry covers every security group in that VPC.
 own `spinifex.toml`, and there is no live distribution of it. Two properties of
 the current implementation make the operator responsible for consistency:
 
-- **One node writes the ACLs.** SG reconcile runs on a single CAS-elected vpcd
-  leader, which programs the shared OVN northbound DB. Whichever node holds the
-  lease is the one whose `blocked_ports_wan` / `egress_block_exempt_vpcs` is in
-  force. Leadership moves on restart or crash, so if the node configs disagree,
-  the effective policy **changes when the leader changes** and the drift pass
-  flaps the ACLs between the two states. Edit the value **identically on every
-  node**.
-- **It is read at vpcd startup, not hot-reloaded.** The policy is built once when
-  vpcd starts; editing the TOML does nothing until vpcd restarts. Deploy the
-  config change to all nodes and restart vpcd cluster-wide (take the target down
-  and confirm no `spx` process survives — a selective single-service restart is
-  not reliable on the shared binary).
+- **One node writes the ACLs.** SG reconcile runs on a single CAS-elected vpcd leader, which programs the shared OVN northbound DB. Whichever node holds the lease is the one whose `blocked_ports_wan` / `egress_block_exempt_vpcs` is in force. Leadership moves on restart or crash, so if the node configs disagree, the effective policy **changes when the leader changes** and the drift pass flaps the ACLs between the two states. Edit the value **identically on every node**.
+- **It is read at vpcd startup, not hot-reloaded.** The policy is built once when vpcd starts; editing the TOML does nothing until vpcd restarts. Deploy the config change to all nodes and restart vpcd cluster-wide (take the target down and confirm no `spx` process survives — a selective single-service restart is not reliable on the shared binary).
 
 This is the current implementation and is deliberately minimal. A future revision
 will move the exemption into shared cluster state (so a single edit propagates

docs/containers/eks/README.md

@@ -82,12 +82,11 @@ Before creating a cluster you need the supporting infrastructure in place. The T
 - **Spinifex running**, with the AWS CLI configured for the `spinifex` profile (see [Installing Spinifex](/docs/install)) and `kubectl` installed.
 - **The `eks-node` image** registered on the cluster. The console blocks cluster creation until it is present.
 - **A VPC with at least one subnet.** The control plane launches in your first subnet; workers run in the subnets you pass to the node group.
-- **Egress for the workers.** Workers must reach the internet to pull container images:
-  - **Public workers** — an **Internet Gateway** with a default route (`0.0.0.0/0 → IGW`) and public IPs.
-  - **Private workers** — a **NAT Gateway** in a public subnet with a default route (`0.0.0.0/0 → NAT`) from the private subnets.
-- **Two IAM roles:**
-  - A **cluster role** trusted by `eks.amazonaws.com` with `AmazonEKSClusterPolicy` attached.
-  - A **node role** trusted by `ec2.amazonaws.com` with `AmazonEKSWorkerNodePolicy`, `AmazonEKS_CNI_Policy`, and `AmazonEC2ContainerRegistryReadOnly` attached (the last lets workers pull images from [ECR](/docs/ecr)).
+- **Egress for the workers.** Workers must reach the internet to pull container images.
+- **Public workers** need an **Internet Gateway** with a default route (`0.0.0.0/0 → IGW`) and public IPs.
+- **Private workers** need a **NAT Gateway** in a public subnet with a default route (`0.0.0.0/0 → NAT`) from the private subnets.
+- **A cluster role** trusted by `eks.amazonaws.com` with `AmazonEKSClusterPolicy` attached.
+- **A node role** trusted by `ec2.amazonaws.com` with `AmazonEKSWorkerNodePolicy`, `AmazonEKS_CNI_Policy`, and `AmazonEC2ContainerRegistryReadOnly` attached (the last lets workers pull images from [ECR](/docs/ecr)).
 
 ## Instructions
 
@@ -273,4 +272,4 @@ The workbooks form a ladder. **eks-quickstart** is a NodePort demo; **eks-https-
 
 A complete end-to-end example of Spinifex EKS in production: a GPU-accelerated AI inference platform running on a Supermicro X14 2U with two NVIDIA RTX Pro 6000 Blackwell GPUs. The guide provisions an EKS cluster with two GPU worker nodes via VFIO PCIe passthrough, deploys an OpenAI-compatible LLM API (llama.cpp / Llama 3.2 3B) and a real-time YOLO object-detection stream on separate GPU nodes, and wires them together behind an ALB using IAM, ECR, ACM, and EBS — all with standard AWS tooling and a single `AWS_PROFILE` swap.
 
-→ [EKS AI Platform Reference Architecture](/docs/RTX-Pro-6000-EKS)
+→ [EKS AI Platform Reference Architecture](/hardware/supermicro/rtx-pro-6000)

docs/coverage/README.md

@@ -0,0 +1,34 @@
+---
+title: "AWS API Coverage"
+seoTitle: "Which AWS API Operations Spinifex Implements — Spinifex Docs"
+description: "The AWS API operations Spinifex serves, generated from its gateway dispatch tables and the pinned AWS SDK service models on each build, service by service."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - api
+  - operations
+---
+
+# AWS API Coverage
+
+## Overview
+
+The platform serves **381 operations** across the AWS APIs below. Every page names the operations its service implements from the pinned model, generated from the dispatch tables on each build rather than written by hand.
+
+| Service | Operations |
+|---|---:|
+| [ACM](/coverage/acm) | 9 |
+| [EC2](/coverage/ec2) | 125 |
+| [ECR](/coverage/ecr) | 22 |
+| [ECS](/coverage/ecs) | 31 |
+| [EKS](/coverage/eks) | 34 |
+| [ELBv2](/coverage/elbv2) | 34 |
+| [IAM](/coverage/iam) | 76 |
+| [RDS](/coverage/rds) | 26 |
+| [S3](/coverage/s3) | 19 |
+| [STS](/coverage/sts) | 5 |
+| **Total** | **381** |

docs/coverage/acm/README.md

@@ -0,0 +1,43 @@
+---
+title: "ACM API Coverage"
+seoTitle: "AWS Certificate Manager API Coverage — Spinifex Docs"
+description: "The AWS Certificate Manager API operations Spinifex implements, generated from the gateway dispatch tables and the pinned AWS service model on each build."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - acm
+  - certificates
+  - tls
+---
+
+# ACM API Coverage
+
+## Overview
+
+Spinifex implements **9 operations** in the ACM `2015-12-08` API model.
+
+### Import and issuance
+
+Spinifex both stores externally-issued certificates and issues its own for load balancer listener references. Certificates are account-scoped, and a delete is refused while any listener still references the ARN — there is no force flag, matching AWS.
+
+`RequestCertificate` mints an ARN immediately and returns `PENDING_VALIDATION`; it never issues inline except against a tenant private CA, which has no domain to validate. The validation mode is derived from deployment state rather than configured: the DNS provider API where a credential exists, a manual TXT record where the platform hosts the zone, and a private CA otherwise — the only option for a deployment with no publicly delegated domain.
+
+Terraform's canonical certificate, DNS record and validation flow works unmodified in every mode. Where Spinifex owns the record write it emits no `ResourceRecord`, so iterating the validation options yields zero records and the validation resource still blocks correctly by polling until the certificate is issued.
+
+### Operations
+
+| Operation |
+|---|
+| `AddTagsToCertificate` |
+| `DeleteCertificate` |
+| `DescribeCertificate` |
+| `GetCertificate` |
+| `ImportCertificate` |
+| `ListCertificates` |
+| `ListTagsForCertificate` |
+| `RemoveTagsFromCertificate` |
+| `RequestCertificate` |

docs/coverage/acm/intro.md

@@ -0,0 +1,7 @@
+### Import and issuance
+
+Spinifex both stores externally-issued certificates and issues its own for load balancer listener references. Certificates are account-scoped, and a delete is refused while any listener still references the ARN — there is no force flag, matching AWS.
+
+`RequestCertificate` mints an ARN immediately and returns `PENDING_VALIDATION`; it never issues inline except against a tenant private CA, which has no domain to validate. The validation mode is derived from deployment state rather than configured: the DNS provider API where a credential exists, a manual TXT record where the platform hosts the zone, and a private CA otherwise — the only option for a deployment with no publicly delegated domain.
+
+Terraform's canonical certificate, DNS record and validation flow works unmodified in every mode. Where Spinifex owns the record write it emits no `ResourceRecord`, so iterating the validation options yields zero records and the validation resource still blocks correctly by polling until the certificate is issued.

docs/coverage/ec2/README.md

@@ -0,0 +1,155 @@
+---
+title: "EC2 API Coverage"
+seoTitle: "Amazon EC2 API Coverage on Spinifex — Spinifex Docs"
+description: "The Amazon EC2 API operations Spinifex implements, covering instances, EBS volumes, VPC networking, tags, security groups and the rest of the compute surface."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - ec2
+  - compute
+  - vpc
+---
+
+# EC2 API Coverage
+
+## Overview
+
+Spinifex implements **125 operations** in the EC2 `2016-11-15` API model.
+
+### Spot Instances Are a Mock
+
+Spot Instance Requests are a mock over the on-demand `RunInstances` path. A request synchronously launches real VMs on the operator's own compute and is then reported `active` and `fulfilled`. There is no spot market: no bidding, no price rejection, no interruption and no reclamation, and instances are never taken back.
+
+### Operations
+
+| Operation |
+|---|
+| `AllocateAddress` |
+| `AssociateAddress` |
+| `AssociateIamInstanceProfile` |
+| `AssociateRouteTable` |
+| `AttachInternetGateway` |
+| `AttachNetworkInterface` |
+| `AttachVolume` |
+| `AuthorizeSecurityGroupEgress` |
+| `AuthorizeSecurityGroupIngress` |
+| `CancelCapacityReservation` |
+| `CancelSpotInstanceRequests` |
+| `CopyImage` |
+| `CopySnapshot` |
+| `CreateCapacityReservation` |
+| `CreateEgressOnlyInternetGateway` |
+| `CreateImage` |
+| `CreateInternetGateway` |
+| `CreateKeyPair` |
+| `CreateLaunchTemplate` |
+| `CreateLaunchTemplateVersion` |
+| `CreateNatGateway` |
+| `CreateNetworkInterface` |
+| `CreatePlacementGroup` |
+| `CreateRoute` |
+| `CreateRouteTable` |
+| `CreateSecurityGroup` |
+| `CreateSnapshot` |
+| `CreateSubnet` |
+| `CreateTags` |
+| `CreateVolume` |
+| `CreateVpc` |
+| `DeleteEgressOnlyInternetGateway` |
+| `DeleteInternetGateway` |
+| `DeleteKeyPair` |
+| `DeleteLaunchTemplate` |
+| `DeleteLaunchTemplateVersions` |
+| `DeleteNatGateway` |
+| `DeleteNetworkInterface` |
+| `DeletePlacementGroup` |
+| `DeleteRoute` |
+| `DeleteRouteTable` |
+| `DeleteSecurityGroup` |
+| `DeleteSnapshot` |
+| `DeleteSubnet` |
+| `DeleteTags` |
+| `DeleteVolume` |
+| `DeleteVpc` |
+| `DeregisterImage` |
+| `DescribeAccountAttributes` |
+| `DescribeAddresses` |
+| `DescribeAddressesAttribute` |
+| `DescribeAvailabilityZones` |
+| `DescribeCapacityReservations` |
+| `DescribeEgressOnlyInternetGateways` |
+| `DescribeIamInstanceProfileAssociations` |
+| `DescribeImageAttribute` |
+| `DescribeImages` |
+| `DescribeInstanceAttribute` |
+| `DescribeInstanceCreditSpecifications` |
+| `DescribeInstanceStatus` |
+| `DescribeInstanceTypeOfferings` |
+| `DescribeInstanceTypes` |
+| `DescribeInstances` |
+| `DescribeInternetGateways` |
+| `DescribeKeyPairs` |
+| `DescribeLaunchTemplateVersions` |
+| `DescribeLaunchTemplates` |
+| `DescribeNatGateways` |
+| `DescribeNetworkInterfaces` |
+| `DescribePlacementGroups` |
+| `DescribeRegions` |
+| `DescribeRouteTables` |
+| `DescribeSecurityGroupRules` |
+| `DescribeSecurityGroups` |
+| `DescribeSnapshots` |
+| `DescribeSpotInstanceRequests` |
+| `DescribeSubnets` |
+| `DescribeTags` |
+| `DescribeVolumeStatus` |
+| `DescribeVolumes` |
+| `DescribeVolumesModifications` |
+| `DescribeVpcAttribute` |
+| `DescribeVpcs` |
+| `DetachInternetGateway` |
+| `DetachNetworkInterface` |
+| `DetachVolume` |
+| `DisableEbsEncryptionByDefault` |
+| `DisableSerialConsoleAccess` |
+| `DisassociateAddress` |
+| `DisassociateIamInstanceProfile` |
+| `DisassociateRouteTable` |
+| `EnableEbsEncryptionByDefault` |
+| `EnableSerialConsoleAccess` |
+| `GetConsoleOutput` |
+| `GetEbsEncryptionByDefault` |
+| `GetPasswordData` |
+| `GetSecurityGroupsForVpc` |
+| `GetSerialConsoleAccessStatus` |
+| `ImportKeyPair` |
+| `ModifyImageAttribute` |
+| `ModifyInstanceAttribute` |
+| `ModifyInstanceMetadataOptions` |
+| `ModifyLaunchTemplate` |
+| `ModifyNetworkInterfaceAttribute` |
+| `ModifySubnetAttribute` |
+| `ModifyVolume` |
+| `ModifyVpcAttribute` |
+| `MonitorInstances` |
+| `RebootInstances` |
+| `RegisterImage` |
+| `ReleaseAddress` |
+| `ReplaceIamInstanceProfileAssociation` |
+| `ReplaceRoute` |
+| `ReplaceRouteTableAssociation` |
+| `RequestSpotInstances` |
+| `ResetImageAttribute` |
+| `RevokeSecurityGroupEgress` |
+| `RevokeSecurityGroupIngress` |
+| `RunInstances` |
+| `StartInstances` |
+| `StopInstances` |
+| `TerminateInstances` |
+| `UnmonitorInstances` |
+| `UpdateSecurityGroupRuleDescriptionsEgress` |
+| `UpdateSecurityGroupRuleDescriptionsIngress` |

docs/coverage/ec2/intro.md

@@ -0,0 +1,3 @@
+### Spot Instances Are a Mock
+
+Spot Instance Requests are a mock over the on-demand `RunInstances` path. A request synchronously launches real VMs on the operator's own compute and is then reported `active` and `fulfilled`. There is no spot market: no bidding, no price rejection, no interruption and no reclamation, and instances are never taken back.

docs/coverage/ecr/README.md

@@ -0,0 +1,56 @@
+---
+title: "ECR API Coverage"
+seoTitle: "Elastic Container Registry API Coverage — Spinifex Docs"
+description: "The Amazon ECR API operations Spinifex implements, alongside the OCI distribution endpoint that carries the image layers for every repository it serves."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - ecr
+  - containers
+  - registry
+---
+
+# ECR API Coverage
+
+## Overview
+
+Spinifex implements **22 operations** in the ECR `2015-09-21` API model.
+
+### Two endpoints, one registry
+
+Repository metadata is served over the AWS API on the gateway endpoint. Image data moves over the OCI Distribution `/v2/` endpoint on that same host, authenticated by the bearer token `GetAuthorizationToken` mints for `docker login`.
+
+The split explains the stubs below. The layer-transfer operations — `BatchCheckLayerAvailability`, `InitiateLayerUpload`, `UploadLayerPart`, `CompleteLayerUpload` and `GetDownloadUrlForLayer` — are registered stubs because the `/v2/` endpoint carries that traffic instead. A client using `docker` or any OCI-compatible tool never calls them.
+
+Registry replication is a stub for the same kind of reason: a deployment is a single registry, with no cross-region peer to replicate to.
+
+### Operations
+
+| Operation |
+|---|
+| `BatchDeleteImage` |
+| `BatchGetImage` |
+| `CreateRepository` |
+| `DeleteLifecyclePolicy` |
+| `DeleteRepository` |
+| `DeleteRepositoryPolicy` |
+| `DescribeImages` |
+| `DescribeRepositories` |
+| `GetAuthorizationToken` |
+| `GetLifecyclePolicy` |
+| `GetLifecyclePolicyPreview` |
+| `GetRepositoryPolicy` |
+| `ListImages` |
+| `ListTagsForResource` |
+| `PutImage` |
+| `PutImageScanningConfiguration` |
+| `PutImageTagMutability` |
+| `PutLifecyclePolicy` |
+| `SetRepositoryPolicy` |
+| `StartLifecyclePolicyPreview` |
+| `TagResource` |
+| `UntagResource` |

docs/coverage/ecr/intro.md

@@ -0,0 +1,7 @@
+### Two endpoints, one registry
+
+Repository metadata is served over the AWS API on the gateway endpoint. Image data moves over the OCI Distribution `/v2/` endpoint on that same host, authenticated by the bearer token `GetAuthorizationToken` mints for `docker login`.
+
+The split explains the stubs below. The layer-transfer operations — `BatchCheckLayerAvailability`, `InitiateLayerUpload`, `UploadLayerPart`, `CompleteLayerUpload` and `GetDownloadUrlForLayer` — are registered stubs because the `/v2/` endpoint carries that traffic instead. A client using `docker` or any OCI-compatible tool never calls them.
+
+Registry replication is a stub for the same kind of reason: a deployment is a single registry, with no cross-region peer to replicate to.

docs/coverage/ecs/README.md

@@ -0,0 +1,61 @@
+---
+title: "ECS API Coverage"
+seoTitle: "Elastic Container Service API Coverage — Spinifex Docs"
+description: "The Amazon ECS API operations Spinifex implements, covering clusters, services, tasks, container instances and the task definitions they are launched from."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - ecs
+  - containers
+  - orchestration
+---
+
+# ECS API Coverage
+
+## Overview
+
+Spinifex implements **31 operations** in the ECS `2014-11-13` API model.
+
+### No Fargate
+
+Clusters, services and tasks run on EC2 container instances. There is no Fargate launch type, and no operation below provides one.
+
+### Operations
+
+| Operation |
+|---|
+| `CreateCapacityProvider` |
+| `CreateCluster` |
+| `CreateService` |
+| `DeleteCapacityProvider` |
+| `DeleteCluster` |
+| `DeleteService` |
+| `DeregisterContainerInstance` |
+| `DeregisterTaskDefinition` |
+| `DescribeCapacityProviders` |
+| `DescribeClusters` |
+| `DescribeContainerInstances` |
+| `DescribeServices` |
+| `DescribeTaskDefinition` |
+| `DescribeTasks` |
+| `ListClusters` |
+| `ListContainerInstances` |
+| `ListServices` |
+| `ListTagsForResource` |
+| `ListTaskDefinitions` |
+| `ListTasks` |
+| `PutClusterCapacityProviders` |
+| `RegisterContainerInstance` |
+| `RegisterTaskDefinition` |
+| `RunTask` |
+| `StartTask` |
+| `StopTask` |
+| `SubmitTaskStateChange` |
+| `TagResource` |
+| `UntagResource` |
+| `UpdateContainerInstancesState` |
+| `UpdateService` |

docs/coverage/ecs/intro.md

@@ -0,0 +1,3 @@
+### No Fargate
+
+Clusters, services and tasks run on EC2 container instances. There is no Fargate launch type, and no operation below provides one.

docs/coverage/eks/README.md

@@ -0,0 +1,60 @@
+---
+title: "EKS API Coverage"
+seoTitle: "Elastic Kubernetes Service API Coverage — Spinifex Docs"
+description: "The Amazon EKS API operations Spinifex implements, covering clusters, nodegroups, add-ons, access entries and the identity providers they authenticate."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - eks
+  - kubernetes
+  - containers
+---
+
+# EKS API Coverage
+
+## Overview
+
+Spinifex implements **34 operations** in the EKS `2017-11-01` API model.
+
+### Operations
+
+| Operation |
+|---|
+| `AssociateAccessPolicy` |
+| `AssociateIdentityProviderConfig` |
+| `CreateAccessEntry` |
+| `CreateAddon` |
+| `CreateCluster` |
+| `CreateNodegroup` |
+| `DeleteAccessEntry` |
+| `DeleteAddon` |
+| `DeleteCluster` |
+| `DeleteNodegroup` |
+| `DescribeAccessEntry` |
+| `DescribeAddon` |
+| `DescribeAddonVersions` |
+| `DescribeCluster` |
+| `DescribeIdentityProviderConfig` |
+| `DescribeNodegroup` |
+| `DisassociateAccessPolicy` |
+| `DisassociateIdentityProviderConfig` |
+| `ListAccessEntries` |
+| `ListAccessPolicies` |
+| `ListAddons` |
+| `ListAssociatedAccessPolicies` |
+| `ListClusters` |
+| `ListIdentityProviderConfigs` |
+| `ListNodegroups` |
+| `ListTagsForResource` |
+| `TagResource` |
+| `UntagResource` |
+| `UpdateAccessEntry` |
+| `UpdateAddon` |
+| `UpdateClusterConfig` |
+| `UpdateClusterVersion` |
+| `UpdateNodegroupConfig` |
+| `UpdateNodegroupVersion` |

docs/coverage/elbv2/README.md

@@ -0,0 +1,64 @@
+---
+title: "ELBv2 API Coverage"
+seoTitle: "Elastic Load Balancing v2 API Coverage — Spinifex Docs"
+description: "The Elastic Load Balancing v2 API operations Spinifex implements, for both the Application and Network Load Balancers it serves, listeners and target groups."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - elbv2
+  - load balancing
+  - networking
+---
+
+# ELBv2 API Coverage
+
+## Overview
+
+Spinifex implements **34 operations** in the ELBv2 `2015-12-01` API model.
+
+### Two data planes
+
+The data plane is a system-managed load balancer VM, launched automatically during `CreateLoadBalancer`. Application Load Balancers run HAProxy for the L7 surface — rules, fixed responses and redirects over HTTP and HTTPS. Network Load Balancers run nginx `stream` for the L4 surface — TCP, UDP, TLS and TCP_UDP — because HAProxy cannot load-balance UDP. The agent selects the engine from the configuration the control plane delivers, so the choice follows the load balancer type and is not separately configurable.
+
+### Operations
+
+| Operation |
+|---|
+| `AddListenerCertificates` |
+| `AddTags` |
+| `CreateListener` |
+| `CreateLoadBalancer` |
+| `CreateRule` |
+| `CreateTargetGroup` |
+| `DeleteListener` |
+| `DeleteLoadBalancer` |
+| `DeleteRule` |
+| `DeleteTargetGroup` |
+| `DeregisterTargets` |
+| `DescribeAccountLimits` |
+| `DescribeListenerCertificates` |
+| `DescribeListeners` |
+| `DescribeLoadBalancerAttributes` |
+| `DescribeLoadBalancers` |
+| `DescribeRules` |
+| `DescribeSSLPolicies` |
+| `DescribeTags` |
+| `DescribeTargetGroupAttributes` |
+| `DescribeTargetGroups` |
+| `DescribeTargetHealth` |
+| `ModifyListener` |
+| `ModifyLoadBalancerAttributes` |
+| `ModifyRule` |
+| `ModifyTargetGroup` |
+| `ModifyTargetGroupAttributes` |
+| `RegisterTargets` |
+| `RemoveListenerCertificates` |
+| `RemoveTags` |
+| `SetIpAddressType` |
+| `SetRulePriorities` |
+| `SetSecurityGroups` |
+| `SetSubnets` |

docs/coverage/elbv2/intro.md

@@ -0,0 +1,3 @@
+### Two data planes
+
+The data plane is a system-managed load balancer VM, launched automatically during `CreateLoadBalancer`. Application Load Balancers run HAProxy for the L7 surface — rules, fixed responses and redirects over HTTP and HTTPS. Network Load Balancers run nginx `stream` for the L4 surface — TCP, UDP, TLS and TCP_UDP — because HAProxy cannot load-balance UDP. The agent selects the engine from the configuration the control plane delivers, so the choice follows the load balancer type and is not separately configurable.

docs/coverage/iam/README.md

@@ -0,0 +1,106 @@
+---
+title: "IAM API Coverage"
+seoTitle: "AWS IAM API Operation Coverage on Spinifex — Spinifex Docs"
+description: "The AWS IAM API operations Spinifex implements, covering users, roles, policies, groups, instance profiles and the access keys that authenticate them."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - iam
+  - identity
+  - policies
+---
+
+# IAM API Coverage
+
+## Overview
+
+Spinifex implements **76 operations** in the IAM `2010-05-08` API model.
+
+### Scope
+
+All IAM operations are account-scoped. The root user of an account bypasses policy evaluation entirely, as it does on AWS.
+
+### Operations
+
+| Operation |
+|---|
+| `AddRoleToInstanceProfile` |
+| `AddUserToGroup` |
+| `AttachGroupPolicy` |
+| `AttachRolePolicy` |
+| `AttachUserPolicy` |
+| `CreateAccessKey` |
+| `CreateGroup` |
+| `CreateInstanceProfile` |
+| `CreateOpenIDConnectProvider` |
+| `CreatePolicy` |
+| `CreateRole` |
+| `CreateUser` |
+| `DeleteAccessKey` |
+| `DeleteGroup` |
+| `DeleteGroupPolicy` |
+| `DeleteInstanceProfile` |
+| `DeleteOpenIDConnectProvider` |
+| `DeletePolicy` |
+| `DeleteRole` |
+| `DeleteRolePolicy` |
+| `DeleteUser` |
+| `DeleteUserPolicy` |
+| `DetachGroupPolicy` |
+| `DetachRolePolicy` |
+| `DetachUserPolicy` |
+| `GetAccountSummary` |
+| `GetGroup` |
+| `GetGroupPolicy` |
+| `GetInstanceProfile` |
+| `GetOpenIDConnectProvider` |
+| `GetPolicy` |
+| `GetPolicyVersion` |
+| `GetRole` |
+| `GetRolePolicy` |
+| `GetUser` |
+| `GetUserPolicy` |
+| `ListAccessKeys` |
+| `ListAttachedGroupPolicies` |
+| `ListAttachedRolePolicies` |
+| `ListAttachedUserPolicies` |
+| `ListEntitiesForPolicy` |
+| `ListGroupPolicies` |
+| `ListGroups` |
+| `ListGroupsForUser` |
+| `ListInstanceProfileTags` |
+| `ListInstanceProfiles` |
+| `ListInstanceProfilesForRole` |
+| `ListOpenIDConnectProviderTags` |
+| `ListOpenIDConnectProviders` |
+| `ListPolicies` |
+| `ListPolicyTags` |
+| `ListPolicyVersions` |
+| `ListRolePolicies` |
+| `ListRoleTags` |
+| `ListRoles` |
+| `ListUserPolicies` |
+| `ListUserTags` |
+| `ListUsers` |
+| `PutGroupPolicy` |
+| `PutRolePolicy` |
+| `PutUserPolicy` |
+| `RemoveRoleFromInstanceProfile` |
+| `RemoveUserFromGroup` |
+| `TagInstanceProfile` |
+| `TagOpenIDConnectProvider` |
+| `TagPolicy` |
+| `TagRole` |
+| `TagUser` |
+| `UntagInstanceProfile` |
+| `UntagOpenIDConnectProvider` |
+| `UntagPolicy` |
+| `UntagRole` |
+| `UntagUser` |
+| `UpdateAccessKey` |
+| `UpdateAssumeRolePolicy` |
+| `UpdateRole` |

docs/coverage/iam/intro.md

@@ -0,0 +1,3 @@
+### Scope
+
+All IAM operations are account-scoped. The root user of an account bypasses policy evaluation entirely, as it does on AWS.

docs/coverage/pages.json

@@ -0,0 +1,296 @@
+{
+  "category": "Coverage",
+  "basePath": "/coverage",
+  "index": {
+    "name": "All services",
+    "title": "AWS API Coverage",
+    "seoTitle": "Which AWS API Operations Spinifex Implements — Spinifex Docs",
+    "description": "The AWS API operations Spinifex serves, generated from its gateway dispatch tables and the pinned AWS SDK service models on each build, service by service.",
+    "tags": [
+      "aws",
+      "compatibility",
+      "coverage",
+      "api",
+      "operations"
+    ]
+  },
+  "services": {
+    "acm": {
+      "slug": "acm",
+      "name": "ACM",
+      "title": "ACM API Coverage",
+      "seoTitle": "AWS Certificate Manager API Coverage — Spinifex Docs",
+      "description": "The AWS Certificate Manager API operations Spinifex implements, generated from the gateway dispatch tables and the pinned AWS service model on each build.",
+      "tags": [
+        "aws",
+        "compatibility",
+        "coverage",
+        "acm",
+        "certificates",
+        "tls"
+      ]
+    },
+    "ec2": {
+      "slug": "ec2",
+      "name": "EC2",
+      "title": "EC2 API Coverage",
+      "seoTitle": "Amazon EC2 API Coverage on Spinifex — Spinifex Docs",
+      "description": "The Amazon EC2 API operations Spinifex implements, covering instances, EBS volumes, VPC networking, tags, security groups and the rest of the compute surface.",
+      "tags": [
+        "aws",
+        "compatibility",
+        "coverage",
+        "ec2",
+        "compute",
+        "vpc"
+      ],
+      "notApplicable": {
+        "AttachClassicLinkVpc": "ec2-classic",
+        "CancelReservedInstancesListing": "ri-marketplace",
+        "ConfirmProductInstance": "marketplace",
+        "CopyFpgaImage": "fpga",
+        "CreateCarrierGateway": "aws-edge",
+        "CreateFpgaImage": "fpga",
+        "CreateReservedInstancesListing": "ri-marketplace",
+        "DeleteCarrierGateway": "aws-edge",
+        "DeleteFpgaImage": "fpga",
+        "DescribeAggregateIdFormat": "id-format",
+        "DescribeAwsNetworkPerformanceMetricSubscriptions": "aws-backbone",
+        "DescribeCarrierGateways": "aws-edge",
+        "DescribeClassicLinkInstances": "ec2-classic",
+        "DescribeElasticGpus": "retired-accelerators",
+        "DescribeFpgaImageAttribute": "fpga",
+        "DescribeFpgaImages": "fpga",
+        "DescribeIdFormat": "id-format",
+        "DescribeIdentityIdFormat": "id-format",
+        "DescribeMovingAddresses": "ec2-classic",
+        "DescribePrincipalIdFormat": "id-format",
+        "DescribeReservedInstancesListings": "ri-marketplace",
+        "DescribeSpotPriceHistory": "spot-price",
+        "DescribeVpcClassicLink": "ec2-classic",
+        "DescribeVpcClassicLinkDnsSupport": "ec2-classic",
+        "DetachClassicLinkVpc": "ec2-classic",
+        "DisableAwsNetworkPerformanceMetricSubscription": "aws-backbone",
+        "DisableVpcClassicLink": "ec2-classic",
+        "DisableVpcClassicLinkDnsSupport": "ec2-classic",
+        "EnableAwsNetworkPerformanceMetricSubscription": "aws-backbone",
+        "EnableVpcClassicLink": "ec2-classic",
+        "EnableVpcClassicLinkDnsSupport": "ec2-classic",
+        "GetAwsNetworkPerformanceData": "aws-backbone",
+        "ModifyFpgaImageAttribute": "fpga",
+        "ModifyIdFormat": "id-format",
+        "ModifyIdentityIdFormat": "id-format",
+        "MoveAddressToVpc": "ec2-classic",
+        "ReportInstanceStatus": "aws-support",
+        "ResetFpgaImageAttribute": "fpga",
+        "RestoreAddressToClassic": "ec2-classic"
+      },
+      "notes": {
+        "aws-backbone": "These report performance across AWS's own global network between its regions and zones.",
+        "aws-edge": "Carrier gateways route to a telecommunications carrier's 5G network in an AWS Wavelength Zone.",
+        "aws-support": "Reporting an instance as impaired files a report with AWS support about AWS hardware.",
+        "ec2-classic": "EC2-Classic is the pre-VPC network AWS retired in 2022. Every instance here is in a VPC.",
+        "fpga": "FPGA images are built and ingested through AWS's own AFI pipeline for its F1 instances.",
+        "id-format": "These switched resource IDs between the short and long forms during AWS's 2016 migration, which has long since finished.",
+        "marketplace": "Product codes belong to AWS Marketplace, which has no equivalent here.",
+        "retired-accelerators": "Elastic Graphics was retired by AWS, and attached nothing but an AWS-hosted accelerator while it existed.",
+        "ri-marketplace": "The Reserved Instance Marketplace resells a reservation to another AWS customer, so AWS is the counterparty.",
+        "spot-price": "On owned hardware there is no spot-to-on-demand price differential, so any figure would be invented."
+      }
+    },
+    "ecr": {
+      "slug": "ecr",
+      "name": "ECR",
+      "title": "ECR API Coverage",
+      "seoTitle": "Elastic Container Registry API Coverage — Spinifex Docs",
+      "description": "The Amazon ECR API operations Spinifex implements, alongside the OCI distribution endpoint that carries the image layers for every repository it serves.",
+      "tags": [
+        "aws",
+        "compatibility",
+        "coverage",
+        "ecr",
+        "containers",
+        "registry"
+      ]
+    },
+    "ecs": {
+      "slug": "ecs",
+      "name": "ECS",
+      "title": "ECS API Coverage",
+      "seoTitle": "Elastic Container Service API Coverage — Spinifex Docs",
+      "description": "The Amazon ECS API operations Spinifex implements, covering clusters, services, tasks, container instances and the task definitions they are launched from.",
+      "tags": [
+        "aws",
+        "compatibility",
+        "coverage",
+        "ecs",
+        "containers",
+        "orchestration"
+      ],
+      "internal": [
+        "PollAssignments",
+        "ProvisionCapacity",
+        "ReportTaskGPU"
+      ]
+    },
+    "eks": {
+      "slug": "eks",
+      "name": "EKS",
+      "title": "EKS API Coverage",
+      "seoTitle": "Elastic Kubernetes Service API Coverage — Spinifex Docs",
+      "description": "The Amazon EKS API operations Spinifex implements, covering clusters, nodegroups, add-ons, access entries and the identity providers they authenticate.",
+      "tags": [
+        "aws",
+        "compatibility",
+        "coverage",
+        "eks",
+        "kubernetes",
+        "containers"
+      ],
+      "internal": [
+        "GetRecoveryDirective",
+        "ListInternalAddons",
+        "PublishInternal",
+        "WebhookTokenReview"
+      ]
+    },
+    "elasticloadbalancingv2": {
+      "slug": "elbv2",
+      "name": "ELBv2",
+      "title": "ELBv2 API Coverage",
+      "seoTitle": "Elastic Load Balancing v2 API Coverage — Spinifex Docs",
+      "description": "The Elastic Load Balancing v2 API operations Spinifex implements, for both the Application and Network Load Balancers it serves, listeners and target groups.",
+      "tags": [
+        "aws",
+        "compatibility",
+        "coverage",
+        "elbv2",
+        "load balancing",
+        "networking"
+      ],
+      "internal": [
+        "GetLBConfig",
+        "LBAgentHeartbeat"
+      ]
+    },
+    "iam": {
+      "slug": "iam",
+      "name": "IAM",
+      "title": "IAM API Coverage",
+      "seoTitle": "AWS IAM API Operation Coverage on Spinifex — Spinifex Docs",
+      "description": "The AWS IAM API operations Spinifex implements, covering users, roles, policies, groups, instance profiles and the access keys that authenticate them.",
+      "tags": [
+        "aws",
+        "compatibility",
+        "coverage",
+        "iam",
+        "identity",
+        "policies"
+      ],
+      "notApplicable": {
+        "CreateServiceLinkedRole": "service-linked-roles",
+        "CreateServiceSpecificCredential": "codecommit",
+        "DeleteSSHPublicKey": "codecommit",
+        "DeleteServiceLinkedRole": "service-linked-roles",
+        "DeleteServiceSpecificCredential": "codecommit",
+        "DeleteSigningCertificate": "signing-certificates",
+        "GenerateOrganizationsAccessReport": "organizations",
+        "GetOrganizationsAccessReport": "organizations",
+        "GetSSHPublicKey": "codecommit",
+        "GetServiceLinkedRoleDeletionStatus": "service-linked-roles",
+        "ListSSHPublicKeys": "codecommit",
+        "ListServiceSpecificCredentials": "codecommit",
+        "ListSigningCertificates": "signing-certificates",
+        "ResetServiceSpecificCredential": "codecommit",
+        "UpdateSSHPublicKey": "codecommit",
+        "UpdateServiceSpecificCredential": "codecommit",
+        "UpdateSigningCertificate": "signing-certificates",
+        "UploadSSHPublicKey": "codecommit",
+        "UploadSigningCertificate": "signing-certificates"
+      },
+      "notes": {
+        "codecommit": "CodeCommit credentials and SSH keys; the service does not exist here.",
+        "organizations": "AWS Organizations is not offered, so there is no organization to report on.",
+        "service-linked-roles": "Service-linked roles exist for AWS service principals, which this platform has none of.",
+        "signing-certificates": "X.509 signing certificates are an EC2-Classic SOAP credential, retired by AWS."
+      }
+    },
+    "rds": {
+      "slug": "rds",
+      "name": "RDS",
+      "title": "RDS API Coverage",
+      "seoTitle": "Relational Database Service API Coverage — Spinifex Docs",
+      "description": "The Amazon RDS API operations Spinifex implements, for the managed PostgreSQL and MariaDB engines it offers, covering instances, snapshots and parameters.",
+      "tags": [
+        "aws",
+        "compatibility",
+        "coverage",
+        "rds",
+        "databases",
+        "postgresql",
+        "mariadb"
+      ],
+      "internal": [
+        "AcknowledgeDBBootstrap",
+        "GetDBBootstrapConfig",
+        "PollDBCommands",
+        "RegisterDBInstance",
+        "SubmitDBStateChange"
+      ],
+      "notApplicable": {
+        "CreateOptionGroup": "option-groups",
+        "DeleteOptionGroup": "option-groups",
+        "DescribeOptionGroups": "option-groups",
+        "ModifyOptionGroup": "option-groups"
+      },
+      "notes": {
+        "option-groups": "Option groups configure engine add-ons for engines this platform does not offer, such as Oracle and SQL Server."
+      }
+    },
+    "s3": {
+      "slug": "s3",
+      "name": "S3",
+      "title": "S3 API Coverage",
+      "seoTitle": "Amazon S3 API Coverage on Spinifex — Spinifex Docs",
+      "description": "The Amazon S3 API operations Predastore serves on the platform's S3 endpoint, covering buckets, objects, multipart uploads and the policies that guard them.",
+      "implementedBy": "Predastore",
+      "tags": [
+        "aws",
+        "compatibility",
+        "coverage",
+        "s3",
+        "storage",
+        "objects"
+      ],
+      "notApplicable": {
+        "CreateSession": "express-one-zone",
+        "GetBucketAccelerateConfiguration": "accelerate",
+        "GetBucketRequestPayment": "request-payment",
+        "PutBucketAccelerateConfiguration": "accelerate",
+        "PutBucketRequestPayment": "request-payment",
+        "WriteGetObjectResponse": "object-lambda"
+      },
+      "notes": {
+        "accelerate": "Transfer acceleration routes uploads over AWS edge locations, which an on-premise deployment has none of.",
+        "express-one-zone": "Sessions authenticate S3 Express One Zone directory buckets, a storage class this platform does not offer.",
+        "object-lambda": "S3 Object Lambda rewrites a response from a Lambda function, and Lambda is not offered.",
+        "request-payment": "Requester Pays shifts transfer charges to the caller, which needs AWS billing behind it."
+      }
+    },
+    "sts": {
+      "slug": "sts",
+      "name": "STS",
+      "title": "STS API Coverage",
+      "seoTitle": "AWS Security Token Service API Coverage — Spinifex Docs",
+      "description": "The AWS STS API operations Spinifex implements, covering role assumption, session tokens and web identity federation (IRSA), with those it does not offer.",
+      "tags": [
+        "aws",
+        "compatibility",
+        "coverage",
+        "sts",
+        "identity",
+        "credentials"
+      ]
+    }
+  }
+}

docs/coverage/rds/README.md

@@ -0,0 +1,100 @@
+---
+title: "RDS API Coverage"
+seoTitle: "Relational Database Service API Coverage — Spinifex Docs"
+description: "The Amazon RDS API operations Spinifex implements, for the managed PostgreSQL and MariaDB engines it offers, covering instances, snapshots and parameters."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - rds
+  - databases
+  - postgresql
+  - mariadb
+---
+
+# RDS API Coverage
+
+## Overview
+
+Spinifex implements **26 operations** in the RDS `2014-10-31` API model.
+
+### Engines
+
+Spinifex offers PostgreSQL and MariaDB. Each DB instance is one dedicated system-owned VM running the engine directly, launched from a platform AMI and hidden from the customer's EC2 API. `Engine` is fixed at create: there is no in-place engine change, no cross-engine snapshot restore and no migration between the two.
+
+`mysql` is not an accepted engine and is not an alias for `mariadb`. MariaDB is offered under its own AWS engine name, exactly as AWS RDS offers it, so a client — including Terraform's `aws_db_instance` — must set `engine = "mariadb"`.
+
+Engine versions are pinned per engine. An `EngineVersion` naming anything but the pin is rejected, including a narrower minor version, because the AMI makes no promise about which minor it carries.
+
+### The endpoint is private
+
+The engine is reached over a customer-account ENI injected into a subnet of the DB subnet group, so the endpoint is reachable from inside the VPC only. The DB VM has other NICs that no customer security group governs, and the engine binds none of them — the port is not open there at all, rather than open and gated.
+
+### TLS is required by default
+
+Both engines enforce encrypted connections by default: `rds.force_ssl` on PostgreSQL and `require_secure_transport` on MariaDB. For MariaDB this is a deliberate divergence from AWS, which leaves it off. Both are boolean, modifiable and dynamic, so setting either to `0` in a parameter group restores plaintext without a reboot.
+
+### Rejected parameters
+
+A parameter whose omission would create a false safety, security or availability guarantee is rejected with `InvalidParameterValue` rather than silently dropped.
+
+| Parameter | Why it is rejected |
+|-----------|--------------------|
+| `MultiAZ=true` | Single-AZ platform; a standby would not exist |
+| `PubliclyAccessible=true` | The endpoint is a private VPC address |
+| `StorageEncrypted=false` | Unencrypted storage is not offered |
+| `EnableIAMDatabaseAuthentication` | IAM database authentication is not implemented |
+| `Iops`, `StorageThroughput`, `StorageType` ≠ `gp3` | Provisioned performance classes are not implemented |
+| `KmsKeyId`, `TdeCredentialArn` | Storage is encrypted with the cluster key, not a customer-managed one |
+| `AvailabilityZone` | The platform exposes a single zone |
+| `AvailabilityZoneGroup` (orderable options) | It selects a zone or local-zone group, and naming a zone is already refused |
+| `DBSecurityGroups` | EC2-Classic security groups — use `VpcSecurityGroupIds` |
+| `DBClusterIdentifier`, `DBClusterSnapshotIdentifier` | Clustered engines are not offered |
+| `EnableCloudwatchLogsExports` | Log export is not implemented |
+| `EngineVersion` other than the engine's pin, `Engine` on modify | No in-place engine or version change |
+| `Engine=mysql` (and Aurora engines) | Oracle MySQL is not offered; `mariadb` is a distinct engine, not an alias for it |
+| `NewDBInstanceIdentifier` | The identifier is the DNS label and the KV key |
+| `DBPortNumber`, `DBSubnetGroupName` on modify | Both would move the endpoint |
+| `MaxAllocatedStorage` | Storage autoscaling is not implemented |
+| `ManageMasterUserPassword`, `RotateMasterUserPassword` | Secrets Manager integration is not offered |
+| `CACertificateIdentifier` | The serving certificate is minted from the cluster CA |
+| `Domain`, `DomainFqdn` | Active Directory domain join is not offered |
+| `OptionGroupName` | Option groups are not offered |
+| `CustomIamInstanceProfile` | The DB VM's instance profile is platform-owned |
+| `EnableCustomerOwnedIp` | An Outposts feature |
+| `ForceFailover` (reboot) | No standby to fail over to |
+| `DBSnapshotIdentifier` (stop) | Snapshot-on-stop is not implemented |
+
+### Operations
+
+| Operation |
+|---|
+| `AddTagsToResource` |
+| `CreateDBInstance` |
+| `CreateDBParameterGroup` |
+| `CreateDBSnapshot` |
+| `CreateDBSubnetGroup` |
+| `DeleteDBInstance` |
+| `DeleteDBParameterGroup` |
+| `DeleteDBSnapshot` |
+| `DeleteDBSubnetGroup` |
+| `DescribeDBEngineVersions` |
+| `DescribeDBInstanceAutomatedBackups` |
+| `DescribeDBInstances` |
+| `DescribeDBParameterGroups` |
+| `DescribeDBParameters` |
+| `DescribeDBSnapshots` |
+| `DescribeDBSubnetGroups` |
+| `DescribeEvents` |
+| `DescribeOrderableDBInstanceOptions` |
+| `ListTagsForResource` |
+| `ModifyDBInstance` |
+| `ModifyDBParameterGroup` |
+| `RebootDBInstance` |
+| `RemoveTagsFromResource` |
+| `RestoreDBInstanceFromDBSnapshot` |
+| `StartDBInstance` |
+| `StopDBInstance` |

docs/coverage/rds/intro.md

@@ -0,0 +1,46 @@
+### Engines
+
+Spinifex offers PostgreSQL and MariaDB. Each DB instance is one dedicated system-owned VM running the engine directly, launched from a platform AMI and hidden from the customer's EC2 API. `Engine` is fixed at create: there is no in-place engine change, no cross-engine snapshot restore and no migration between the two.
+
+`mysql` is not an accepted engine and is not an alias for `mariadb`. MariaDB is offered under its own AWS engine name, exactly as AWS RDS offers it, so a client — including Terraform's `aws_db_instance` — must set `engine = "mariadb"`.
+
+Engine versions are pinned per engine. An `EngineVersion` naming anything but the pin is rejected, including a narrower minor version, because the AMI makes no promise about which minor it carries.
+
+### The endpoint is private
+
+The engine is reached over a customer-account ENI injected into a subnet of the DB subnet group, so the endpoint is reachable from inside the VPC only. The DB VM has other NICs that no customer security group governs, and the engine binds none of them — the port is not open there at all, rather than open and gated.
+
+### TLS is required by default
+
+Both engines enforce encrypted connections by default: `rds.force_ssl` on PostgreSQL and `require_secure_transport` on MariaDB. For MariaDB this is a deliberate divergence from AWS, which leaves it off. Both are boolean, modifiable and dynamic, so setting either to `0` in a parameter group restores plaintext without a reboot.
+
+### Rejected parameters
+
+A parameter whose omission would create a false safety, security or availability guarantee is rejected with `InvalidParameterValue` rather than silently dropped.
+
+| Parameter | Why it is rejected |
+|-----------|--------------------|
+| `MultiAZ=true` | Single-AZ platform; a standby would not exist |
+| `PubliclyAccessible=true` | The endpoint is a private VPC address |
+| `StorageEncrypted=false` | Unencrypted storage is not offered |
+| `EnableIAMDatabaseAuthentication` | IAM database authentication is not implemented |
+| `Iops`, `StorageThroughput`, `StorageType` ≠ `gp3` | Provisioned performance classes are not implemented |
+| `KmsKeyId`, `TdeCredentialArn` | Storage is encrypted with the cluster key, not a customer-managed one |
+| `AvailabilityZone` | The platform exposes a single zone |
+| `AvailabilityZoneGroup` (orderable options) | It selects a zone or local-zone group, and naming a zone is already refused |
+| `DBSecurityGroups` | EC2-Classic security groups — use `VpcSecurityGroupIds` |
+| `DBClusterIdentifier`, `DBClusterSnapshotIdentifier` | Clustered engines are not offered |
+| `EnableCloudwatchLogsExports` | Log export is not implemented |
+| `EngineVersion` other than the engine's pin, `Engine` on modify | No in-place engine or version change |
+| `Engine=mysql` (and Aurora engines) | Oracle MySQL is not offered; `mariadb` is a distinct engine, not an alias for it |
+| `NewDBInstanceIdentifier` | The identifier is the DNS label and the KV key |
+| `DBPortNumber`, `DBSubnetGroupName` on modify | Both would move the endpoint |
+| `MaxAllocatedStorage` | Storage autoscaling is not implemented |
+| `ManageMasterUserPassword`, `RotateMasterUserPassword` | Secrets Manager integration is not offered |
+| `CACertificateIdentifier` | The serving certificate is minted from the cluster CA |
+| `Domain`, `DomainFqdn` | Active Directory domain join is not offered |
+| `OptionGroupName` | Option groups are not offered |
+| `CustomIamInstanceProfile` | The DB VM's instance profile is platform-owned |
+| `EnableCustomerOwnedIp` | An Outposts feature |
+| `ForceFailover` (reboot) | No standby to fail over to |
+| `DBSnapshotIdentifier` (stop) | Snapshot-on-stop is not implemented |

docs/coverage/s3/README.md

@@ -0,0 +1,55 @@
+---
+title: "S3 API Coverage"
+seoTitle: "Amazon S3 API Coverage on Spinifex — Spinifex Docs"
+description: "The Amazon S3 API operations Predastore serves on the platform's S3 endpoint, covering buckets, objects, multipart uploads and the policies that guard them."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - s3
+  - storage
+  - objects
+---
+
+# S3 API Coverage
+
+## Overview
+
+Predastore implements **19 operations** in the S3 `2006-03-01` API model.
+
+### Predastore serves this endpoint
+
+S3 is the one surface the AWS gateway does not answer itself. Object storage runs on [Predastore](https://github.com/mulgadc/predastore), which serves the S3 REST API directly over its own endpoint.
+
+### Routed is not the same as conforming
+
+This page says an operation is routed to a handler. It does not say the handler's behaviour matches S3 in every case.
+
+That behaviour is measured separately, against the `ceph/s3-tests` suite Ceph RGW, MinIO and Garage are all validated with, and the results are published in Predastore's [S3 compatibility report](https://github.com/mulgadc/predastore/blob/dev/docs/S3-COMPATIBILITY.md).
+
+### Operations
+
+| Operation |
+|---|
+| `AbortMultipartUpload` |
+| `CompleteMultipartUpload` |
+| `CopyObject` |
+| `CreateBucket` |
+| `CreateMultipartUpload` |
+| `DeleteBucket` |
+| `DeleteObject` |
+| `DeleteObjects` |
+| `GetObject` |
+| `HeadBucket` |
+| `HeadObject` |
+| `ListBuckets` |
+| `ListMultipartUploads` |
+| `ListObjects` |
+| `ListObjectsV2` |
+| `ListParts` |
+| `PutObject` |
+| `UploadPart` |
+| `UploadPartCopy` |

docs/coverage/s3/intro.md

@@ -0,0 +1,9 @@
+### Predastore serves this endpoint
+
+S3 is the one surface the AWS gateway does not answer itself. Object storage runs on [Predastore](https://github.com/mulgadc/predastore), which serves the S3 REST API directly over its own endpoint.
+
+### Routed is not the same as conforming
+
+This page says an operation is routed to a handler. It does not say the handler's behaviour matches S3 in every case.
+
+That behaviour is measured separately, against the `ceph/s3-tests` suite Ceph RGW, MinIO and Garage are all validated with, and the results are published in Predastore's [S3 compatibility report](https://github.com/mulgadc/predastore/blob/dev/docs/S3-COMPATIBILITY.md).

docs/coverage/sts/README.md

@@ -0,0 +1,41 @@
+---
+title: "STS API Coverage"
+seoTitle: "AWS Security Token Service API Coverage — Spinifex Docs"
+description: "The AWS STS API operations Spinifex implements, covering role assumption, session tokens and web identity federation (IRSA), with those it does not offer."
+category: "Coverage"
+sections:
+  - overview
+tags:
+  - aws
+  - compatibility
+  - coverage
+  - sts
+  - identity
+  - credentials
+---
+
+# STS API Coverage
+
+## Overview
+
+Spinifex implements **5 operations** in the STS `2011-06-15` API model.
+
+### Trust policies
+
+Trust policies are validated at write time rather than silently narrowed at assume time. `NotPrincipal`, `NotAction`, empty-string `Action` elements and empty `Principal` blocks are all rejected as malformed.
+
+`Condition` blocks are rejected except on `AssumeRoleWithWebIdentity` with `StringEquals`, which is the shape IRSA needs and which Spinifex evaluates at assume time against the token's issuer, subject and audience. Anything wider is refused rather than accepted and ignored, because an accepted-but-unevaluated condition is a silent over-grant.
+
+### Parameters that are refused, not ignored
+
+Several inputs the model describes are deliberately rejected rather than accepted as no-ops: inline session policies and policy ARNs, session tags, and MFA serial numbers and token codes. Each of them would otherwise appear to restrict or strengthen a session that in fact carries the role's full permissions.
+
+### Operations
+
+| Operation |
+|---|
+| `AssumeRole` |
+| `AssumeRoleWithWebIdentity` |
+| `GetAccessKeyInfo` |
+| `GetCallerIdentity` |
+| `GetSessionToken` |

docs/coverage/sts/intro.md

@@ -0,0 +1,9 @@
+### Trust policies
+
+Trust policies are validated at write time rather than silently narrowed at assume time. `NotPrincipal`, `NotAction`, empty-string `Action` elements and empty `Principal` blocks are all rejected as malformed.
+
+`Condition` blocks are rejected except on `AssumeRoleWithWebIdentity` with `StringEquals`, which is the shape IRSA needs and which Spinifex evaluates at assume time against the token's issuer, subject and audience. Anything wider is refused rather than accepted and ignored, because an accepted-but-unevaluated condition is a silent over-grant.
+
+### Parameters that are refused, not ignored
+
+Several inputs the model describes are deliberately rejected rather than accepted as no-ops: inline session policies and policy ARNs, session tags, and MFA serial numbers and token codes. Each of them would otherwise appear to restrict or strengthen a session that in fact carries the role's full permissions.

docs/hardware/cisco/llm-serving/README.md

@@ -19,9 +19,9 @@ resources:
   - title: "GPU Passthrough"
     url: "/docs/gpu-passthrough"
   - title: "Cisco UCS: AWS-compatible cloud at the edge (companion document)"
-    url: "/docs/cisco-ucs-platform-benchmark"
+    url: "/hardware/cisco/platform-benchmark"
   - title: "Vision pipeline (companion document)"
-    url: "/docs/cisco-ucs-vision-pipeline"
+    url: "/hardware/cisco/vision-pipeline"
   - title: "Benchmark and workload materials (CISCO-refarch)"
     url: "https://github.com/tomnewton-mulga/CISCO-refarch"
   - title: "vLLM"
@@ -45,7 +45,7 @@ cluster, comparing Intel AMX-accelerated CPU serving on an `m8i.2xlarge`-class i
 against NVIDIA L4 GPU serving on a `g6.2xlarge`-class instance — a real,
 production-representative LLM-serving stack, with the two accelerators measured independently at matched model, version, and serving configuration.
 
-**Companion architectures:** [Cisco UCS: AWS-compatible cloud at the edge](/docs/cisco-ucs-platform-benchmark) · [Spinifex Vision Pipeline on Cisco UCS](/docs/cisco-ucs-vision-pipeline)
+**Companion architectures:** [Cisco UCS: AWS-compatible cloud at the edge](/hardware/cisco/platform-benchmark) · [Spinifex Vision Pipeline on Cisco UCS](/hardware/cisco/vision-pipeline)
 
 ### Platform
 
@@ -93,10 +93,7 @@ sudo spx admin gpu enable
 
 - One `m8i.2xlarge` (`cpu1`) and one `g6.2xlarge` (`gpu`) instance provisioned and reachable — the `gpu` instance must land on the node with passthrough configured; see [Launching Instances](/docs/launching-instances) for the full provisioning workflow including VPC, key pair, and security group setup
 - ~20 GB free disk per instance for model weights
-- Python 3.14, `pip install vllm` (GPU instance) or the CPU wheel with
-  `--extra-index-url https://download.pytorch.org/whl/cpu` (CPU instance) —
-  vLLM 0.26.0 / PyTorch 2.11.0 on both, `torch+cu130` on the GPU instance,
-  `torch+cpu` on the CPU instance
+- Python 3.14, `pip install vllm` (GPU instance) or the CPU wheel with `--extra-index-url https://download.pytorch.org/whl/cpu` (CPU instance) — vLLM 0.26.0 / PyTorch 2.11.0 on both, `torch+cu130` on the GPU instance, `torch+cpu` on the CPU instance
 
 ## Instructions
 
@@ -129,16 +126,8 @@ VLLM_USE_FLASHINFER_SAMPLER=0 vllm serve ./Qwen2.5-7B-Instruct \
 Two environment-specific workarounds were needed on this cluster, worth recording for
 anyone reproducing this:
 
-- **cpu1**: vLLM's compiled extension (`vllm/_C.abi3.so`) shipped with an executable-stack
-  ELF flag (`GNU_STACK` = RWE) that this guest kernel refuses to `mprotect`
-  (`cannot enable executable stack as shared object requires: Invalid argument`),
-  crashing the server at import time. Fixed with `patchelf --clear-execstack
-  vllm/_C.abi3.so` — a one-time, host-local binary patch, not a vLLM or model issue.
-- **gpu**: this instance has no CUDA toolkit (`nvcc`) installed, only the driver/runtime —
-  fine for running pre-built PyTorch/CUDA kernels, but vLLM's default sampler
-  (FlashInfer) JIT-compiles a kernel on first use and fails without `nvcc`.
-  `VLLM_USE_FLASHINFER_SAMPLER=0` falls back to vLLM's native PyTorch sampler, which
-  needs no compilation step.
+- **cpu1**: vLLM's compiled extension (`vllm/_C.abi3.so`) shipped with an executable-stack ELF flag (`GNU_STACK` = RWE) that this guest kernel refuses to `mprotect` (`cannot enable executable stack as shared object requires: Invalid argument`), crashing the server at import time. Fixed with `patchelf --clear-execstack vllm/_C.abi3.so` — a one-time, host-local binary patch, not a vLLM or model issue.
+- **gpu**: this instance has no CUDA toolkit (`nvcc`) installed, only the driver/runtime — fine for running pre-built PyTorch/CUDA kernels, but vLLM's default sampler (FlashInfer) JIT-compiles a kernel on first use and fails without `nvcc`. `VLLM_USE_FLASHINFER_SAMPLER=0` falls back to vLLM's native PyTorch sampler, which needs no compilation step.
 
 ### 3. Confirm Intel AMX is executing for this workload
 
@@ -181,9 +170,9 @@ across the full matrix. Figures below are the mean of each cell's 3 repeats (med
 metric per run). Raw per-run JSON results are available in the
 [LLM serving repository](https://github.com/mulgadc/cisco-ucs-llm-serving).
 
-<img src="../../../.github/assets/images/cisco-ucs-llm-serving/tpot-vs-concurrency.png" alt="Median time per output token versus concurrency, CPU AMX versus GPU, by input length">
+<img src="../../../../.github/assets/images/cisco-ucs-llm-serving/tpot-vs-concurrency.png" alt="Median time per output token versus concurrency, CPU AMX versus GPU, by input length">
 
-<img src="../../../.github/assets/images/cisco-ucs-llm-serving/throughput-vs-concurrency.png" alt="Total token throughput versus concurrency, CPU AMX versus GPU, by input length">
+<img src="../../../../.github/assets/images/cisco-ucs-llm-serving/throughput-vs-concurrency.png" alt="Total token throughput versus concurrency, CPU AMX versus GPU, by input length">
 
 | Engine | Input tok | Concurrency | TTFT p50 (ms) | TPOT p50 (ms) | E2E p50 (ms) | Output tok/s | Total tok/s |
 |---|---:|---:|---:|---:|---:|---:|---:|

docs/hardware/cisco/platform-benchmark/README.md

@@ -39,7 +39,7 @@ EBS, S3 and EKS-compatible APIs on the cluster, so the operational model is
 familiar: use the AWS CLI, SDKs, Terraform or OpenTofu, Kubernetes manifests and
 CI/CD systems already used for AWS. The change is the endpoint, not the workflow.
 
-**Companion architectures:** [Vision Pipeline on Cisco UCS](/docs/cisco-ucs-vision-pipeline) shows a local AI pipeline that streams inputs from S3; [vLLM Serving on Cisco UCS](/docs/cisco-ucs-llm-serving) shows CPU and GPU model serving on the same cluster.
+**Companion architectures:** [Vision Pipeline on Cisco UCS](/hardware/cisco/vision-pipeline) shows a local AI pipeline that streams inputs from S3; [vLLM Serving on Cisco UCS](/hardware/cisco/llm-serving) shows CPU and GPU model serving on the same cluster.
 
 ### Platform
 
@@ -63,7 +63,7 @@ launched an instance.
 
 ### Hardware Chassis
 
-<img src="../../../.github/assets/images/cisco-ucs-platform-benchmark/reference-architectures-cisco-ucs-platform-hardware1.png" alt="Cisco UCS x Spinifex">
+<img src="../../../../.github/assets/images/cisco-ucs-platform-benchmark/reference-architectures-cisco-ucs-platform-hardware1.png" alt="Cisco UCS x Spinifex">
 
 ## Architecture
 
@@ -219,8 +219,8 @@ kubectl get pods -n default
 ```
 
 Kubernetes pod rescheduling is the platform's primary answer to node failure for
-scheduled workloads. The companion [vision pipeline](/docs/cisco-ucs-vision-pipeline)
-and [vLLM serving](/docs/cisco-ucs-llm-serving) architectures use raw EC2 instances;
+scheduled workloads. The companion [vision pipeline](/hardware/cisco/vision-pipeline)
+and [vLLM serving](/hardware/cisco/llm-serving) architectures use raw EC2 instances;
 EKS is the pattern to reach for when automatic rescheduling matters over manual placement.
 
 ### 5. Storage benchmark
@@ -278,19 +278,11 @@ metrics are in the [benchmark repository](https://github.com/tomnewton-mulga/CIS
 
 ## What this architecture unlocks
 
-- **Cloud-to-edge placement:** run the same EC2, volume, S3 and Kubernetes patterns
-  at the site where latency, privacy, bandwidth or sovereignty requires it.
-- **Familiar operations:** keep AWS CLI profiles, Terraform/OpenTofu modules, SDKs,
-  CI/CD pipelines and deployment tooling rather than introducing a separate
-  edge-only platform.
-- **Local AI with shared data:** place GPU and AMX-capable CPU inference next to the
-  data source while keeping datasets and artifacts available to all instances through S3.
-- **A resilient foundation:** distribute control and storage services across three
-  physical Cisco nodes instead of concentrating the site on one server. Kubernetes
-  rescheduling and Raft-based storage both tolerate a single node loss without
-  operator intervention.
-- **A path back to cloud:** the same APIs at the edge and on AWS make workload
-  movement, burst strategies and consistent application packaging straightforward.
+- **Cloud-to-edge placement:** run the same EC2, volume, S3 and Kubernetes patterns at the site where latency, privacy, bandwidth or sovereignty requires it.
+- **Familiar operations:** keep AWS CLI profiles, Terraform/OpenTofu modules, SDKs, CI/CD pipelines and deployment tooling rather than introducing a separate edge-only platform.
+- **Local AI with shared data:** place GPU and AMX-capable CPU inference next to the data source while keeping datasets and artifacts available to all instances through S3.
+- **A resilient foundation:** distribute control and storage services across three physical Cisco nodes instead of concentrating the site on one server. Kubernetes rescheduling and Raft-based storage both tolerate a single node loss without operator intervention.
+- **A path back to cloud:** the same APIs at the edge and on AWS make workload movement, burst strategies and consistent application packaging straightforward.
 
 The raw measurements are evidence that the platform is functioning as designed; the
 larger outcome is a portable, resilient operating model for workloads that need cloud

docs/hardware/cisco/vision-pipeline/README.md

@@ -25,7 +25,7 @@ resources:
   - title: "GPU Passthrough"
     url: "/docs/gpu-passthrough"
   - title: "Cisco UCS: AWS-compatible cloud at the edge (companion document)"
-    url: "/docs/cisco-ucs-platform-benchmark"
+    url: "/hardware/cisco/platform-benchmark"
   - title: "Ultralytics YOLO"
     url: "https://docs.ultralytics.com"
   - title: "AWS: Accelerate CPU-based AI inference with Intel AMX on EC2"
@@ -48,7 +48,7 @@ cluster running Spinifex: real-time object detection plus a vision-language mode
 producing plain-English scene descriptions, both streaming from the same shared Predastore
 (S3-compatible) bucket as two entirely independent EC2-compatible instances.
 
-**Companion architectures:** [Cisco UCS: AWS-compatible cloud at the edge](/docs/cisco-ucs-platform-benchmark) · [vLLM Serving on Cisco UCS: Intel AMX vs NVIDIA L4](/docs/cisco-ucs-llm-serving)
+**Companion architectures:** [Cisco UCS: AWS-compatible cloud at the edge](/hardware/cisco/platform-benchmark) · [vLLM Serving on Cisco UCS: Intel AMX vs NVIDIA L4](/hardware/cisco/llm-serving)
 
 ### Platform
 
@@ -164,9 +164,9 @@ measures what that kernel difference is worth on YOLO11m:
 | GPU, TensorRT FP16 | 1 | 62.60 | 0.5067 |
 | GPU, TensorRT FP16 | 8 | 118.90 | — |
 
-<img src="../../../.github/assets/images/cisco-ucs-vision-pipeline/precision-throughput.png" alt="YOLO11m throughput by precision and engine — CPU FP32/BF16/INT8 versus GPU FP16, batch 1 and 8">
+<img src="../../../../.github/assets/images/cisco-ucs-vision-pipeline/precision-throughput.png" alt="YOLO11m throughput by precision and engine — CPU FP32/BF16/INT8 versus GPU FP16, batch 1 and 8">
 
-<img src="../../../.github/assets/images/cisco-ucs-vision-pipeline/precision-map.png" alt="COCO mAP50-95 by precision, axis zoomed to 0.49-0.502 to show the near-zero accuracy cost of BF16">
+<img src="../../../../.github/assets/images/cisco-ucs-vision-pipeline/precision-map.png" alt="COCO mAP50-95 by precision, axis zoomed to 0.49-0.502 to show the near-zero accuracy cost of BF16">
 
 **BF16/AMX gives a real 2.4–3.1x throughput gain over true FP32 on this hardware, at
 essentially zero accuracy cost** (Δ mAP50-95 = −0.0001). INT8 costs a small but real
@@ -189,7 +189,7 @@ the instance's own local Viperblock volume:
 | cpu1 (OpenVINO BF16) | 33.02 | 13.44 | −59% |
 | cpu2 (OpenVINO default) | ~34.79 | 13.44 | −61% |
 
-<img src="../../../.github/assets/images/cisco-ucs-vision-pipeline/local-vs-s3.png" alt="Local Viperblock volume versus Predastore-streamed throughput, by worker">
+<img src="../../../../.github/assets/images/cisco-ucs-vision-pipeline/local-vs-s3.png" alt="Local Viperblock volume versus Predastore-streamed throughput, by worker">
 
 A solo cpu1 S3 run (no concurrent workers) scored 12.53 img/s — essentially identical to
 its 3-worker-concurrent number (13.44). **Concurrency from the other two workers cost

docs/hardware/onlogic/hx401/README.md

@@ -33,7 +33,7 @@ The [OnLogic HX401](https://www.onlogic.com/hx401/) is a fanless, passive-cooled
 
 Single-node removes the distributed-storage guarantees of a multi-node cluster — Predastore runs RS(1,0) on one disk, so there is no erasure-coded redundancy. For edge deployments where data is generated locally, processed on-node, and shipped upstream on a schedule, or where the box provides compute capacity next to a sensor or control network rather than acting as a primary data store, this is the expected and appropriate configuration.
 
-<p align="center"><img src="../../../.github/assets/images/onlogic/HX401.png" alt="OnLogic HX401 fanless edge node"></p>
+<p align="center"><img src="../../../../.github/assets/images/onlogic/HX401.png" alt="OnLogic HX401 fanless edge node"></p>
 
 ### Platform
 

docs/hardware/supermicro/arcee-trinity-mi350x/README.md

@@ -147,7 +147,7 @@ ssh -i ~/.ssh/spinifex-key ubuntu@$INSTANCE_IP 'lspci | grep -i amd'
 
 Or install and run `amd-smi` on the instance:
 
-<img src="../../../.github/assets/images/2gpus.png" alt="amd-smi inside the guest VM confirming two MI350Xs are directly attached, each with a unique UUID">
+<img src="../../../../.github/assets/images/2gpus.png" alt="amd-smi inside the guest VM confirming two MI350Xs are directly attached, each with a unique UUID">
 
 Two MI350X entries with distinct UUIDs confirm direct PCIe passthrough.
 
@@ -267,37 +267,37 @@ The lower avg utilisation in the chat session reflects idle gaps between convers
 
 ### Generation throughput
 
-<img src="../../../.github/assets/images/trinity/trinity-throughput.png" alt="Trinity generation throughput — scripted benchmark">
+<img src="../../../../.github/assets/images/trinity/trinity-throughput.png" alt="Trinity generation throughput — scripted benchmark">
 
 Generation and prompt-ingest tok/s over the 5-minute benchmark. The model sustains ~53 tok/s average at continuous load, peaking at 67 tok/s.
 
 ### Per-GPU compute utilisation
 
-<img src="../../../.github/assets/images/trinity/trinity-per-gpu-util.png" alt="Trinity per-GPU utilisation — scripted benchmark">
+<img src="../../../../.github/assets/images/trinity/trinity-per-gpu-util.png" alt="Trinity per-GPU utilisation — scripted benchmark">
 
 GPU 0 and GPU 1 track each other closely throughout the benchmark — TP=2 distributes attention and FFN layers symmetrically across both MI350Xs.
 
 ### Per-GPU VRAM
 
-<img src="../../../.github/assets/images/trinity/trinity-per-gpu-vram.png" alt="Trinity per-GPU VRAM — scripted benchmark">
+<img src="../../../../.github/assets/images/trinity/trinity-per-gpu-vram.png" alt="Trinity per-GPU VRAM — scripted benchmark">
 
 Both GPUs hold 259.6 GB — model weights sharded evenly, plus KV cache. 90.2% of HBM3e is occupied at rest, leaving ~26 GB headroom per GPU for active KV cache during generation.
 
 ### Per-GPU power
 
-<img src="../../../.github/assets/images/trinity/trinity-per-gpu-power.png" alt="Trinity per-GPU power — scripted benchmark">
+<img src="../../../../.github/assets/images/trinity/trinity-per-gpu-power.png" alt="Trinity per-GPU power — scripted benchmark">
 
 Power draw during active generation peaks at ~420 W per GPU, for a combined chassis draw well under the MI350X's 750 W TDP per card. Idle between prompts returns to ~300 W.
 
 ### Latency
 
-<img src="../../../.github/assets/images/trinity/trinity-latency.png" alt="Trinity latency — scripted benchmark">
+<img src="../../../../.github/assets/images/trinity/trinity-latency.png" alt="Trinity latency — scripted benchmark">
 
 Cumulative average TTFT and E2E latency over the benchmark. TTFT settles to ~758 ms avg once the KV cache is warm; E2E tracks request length as expected.
 
 ### KV cache utilisation
 
-<img src="../../../.github/assets/images/trinity/trinity-kv-cache.png" alt="Trinity KV cache utilisation — scripted benchmark">
+<img src="../../../../.github/assets/images/trinity/trinity-kv-cache.png" alt="Trinity KV cache utilisation — scripted benchmark">
 
 KV cache occupancy rises as requests pile up during the sustained load phase. The queue depth stays near zero — the model keeps pace with the 1.5 s prompt cadence.
 

docs/hardware/supermicro/rtx-pro-6000/README.md

@@ -39,7 +39,7 @@ This guide walks through the use of Spinifex to deploy a self-contained AI infer
 
 Specifically, we create an EKS cluster with two worker nodes, each consisting of a g7e.2xlarge EC2 instance with an attached GPU via VFIO passthrough, an ALB to route traffic to each node, ECR for storing and managing our workload images, and all of the associated security and certificate management requirements (IAM, ACM) you would expect from real AWS.
 
-<p align="center"><img src="../../../.github/assets/images/rtx6000/platform.png" alt="AI Platform request flow: HTTPS ingress → ALB routing → EKS ai-platform GPU workers"></p>
+<p align="center"><img src="../../../../.github/assets/images/rtx6000/platform.png" alt="AI Platform request flow: HTTPS ingress → ALB routing → EKS ai-platform GPU workers"></p>
 
 ### Platform
 
@@ -70,7 +70,7 @@ In this case, the two workloads demonstrated could both run comfortably on a sin
 
 ## Architecture
 
-<p align="center"><img src="../../../.github/assets/images/rtx6000/ai-platform-request-flow-2.png" alt="AI Platform request flow: HTTPS ingress → ALB routing → EKS ai-platform GPU workers"></p>
+<p align="center"><img src="../../../../.github/assets/images/rtx6000/ai-platform-request-flow-2.png" alt="AI Platform request flow: HTTPS ingress → ALB routing → EKS ai-platform GPU workers"></p>
 
 ### AWS services exercised
 

docs/hardware/supermicro/smci-h14/README.md

@@ -76,7 +76,7 @@ sudo spx admin gpu enable
 
 `spx admin gpu setup` blacklists the AMD driver on the host and binds each GPU to `vfio-pci`. After reboot, `spx admin gpu enable` verifies the binding and makes the GPU pool available to `RunInstances`.
 
-<img src="../../../.github/assets/images/gpu-enabled.png" alt="Spinifex console confirming PCIe passthrough is active for each GPU">
+<img src="../../../../.github/assets/images/gpu-enabled.png" alt="Spinifex console confirming PCIe passthrough is active for each GPU">
 
 To check available GPU instance types:
 
@@ -177,7 +177,7 @@ ssh -i ~/.ssh/spinifex-key ubuntu@$CHAT_IP \
      amd-smi list'
 ```
 
-<img src="../../../.github/assets/images/2gpus.png" alt="amd-smi inside vm-chat confirming two MI350Xs are directly attached, each with a unique UUID">
+<img src="../../../../.github/assets/images/2gpus.png" alt="amd-smi inside vm-chat confirming two MI350Xs are directly attached, each with a unique UUID">
 
 Two MI350X entries with distinct UUIDs confirm direct PCIe passthrough is working.
 
@@ -216,11 +216,11 @@ The only difference between the two runs was tensor parallelism:
 
 The charts below show the three independent workloads spiking activity on their associated GPUs, measured in GFX utilisation and power draw. YOLO average FPS stays at ~17 despite utilisation in the YOLO VM never exceeding 20%, indicating a bottleneck in networking or encoding rather than inference. Each chart highlights workload independence — pulses of GPU activity share no correlation across VMs.
 
-<img src="../../../.github/assets/images/smci/smci-demo1-gpu-utilisation.png" alt="GPU utilisation across all three VMs — Demo 1 TP=1">
+<img src="../../../../.github/assets/images/smci/smci-demo1-gpu-utilisation.png" alt="GPU utilisation across all three VMs — Demo 1 TP=1">
 
 GPU utilisation per VM across the full Demo 1 capture.
 
-<img src="../../../.github/assets/images/smci/smci-demo2-gpu-power.png" alt="GPU power draw per workload — Demo 2 TP=2">
+<img src="../../../../.github/assets/images/smci/smci-demo2-gpu-power.png" alt="GPU power draw per workload — Demo 2 TP=2">
 
 Power draw per VM during Demo 2.
 
@@ -228,11 +228,11 @@ Power draw per VM during Demo 2.
 
 When TP=2 is enabled, each VM shards its model symmetrically across both GPUs. Inside the chat VM, both GPUs pulse together on every request, each holding exactly **248 GB of VRAM** (weights split + KV cache shards). With TP=1, GPU 1 sat at 0% for the entire run.
 
-<img src="../../../.github/assets/images/smci/smci-demo2-per-gpu-chat.png" alt="Llama 3.3 70B per-GPU activity — TP=2">
+<img src="../../../../.github/assets/images/smci/smci-demo2-per-gpu-chat.png" alt="Llama 3.3 70B per-GPU activity — TP=2">
 
 Per-GPU utilisation inside `vm-chat` with TP=2.
 
-<img src="../../../.github/assets/images/smci/smci-demo1-per-gpu-chat.png" alt="Llama 3.3 70B per-GPU activity — TP=1">
+<img src="../../../../.github/assets/images/smci/smci-demo1-per-gpu-chat.png" alt="Llama 3.3 70B per-GPU activity — TP=1">
 
 Same VM with TP=1.
 
@@ -240,11 +240,11 @@ Same VM with TP=1.
 
 With TP=1, Qwen peaks at ~60 tok/s and Llama caps around 35 tok/s — both compute-bound on a single MI350X. Enabling TP=2 lifts Llama's peak to ~50 tok/s (+47%) as generation is distributed symmetrically across both GPUs. Qwen's throughput is slightly lower under TP=2 due to collective overhead on the shared-memory transport.
 
-<img src="../../../.github/assets/images/smci/smci-demo1-vllm-throughput.png" alt="vLLM generation throughput — TP=1">
+<img src="../../../../.github/assets/images/smci/smci-demo1-vllm-throughput.png" alt="vLLM generation throughput — TP=1">
 
 Generation throughput during Demo 1 (TP=1).
 
-<img src="../../../.github/assets/images/smci/smci-demo2-vllm-throughput.png" alt="vLLM generation throughput — TP=2">
+<img src="../../../../.github/assets/images/smci/smci-demo2-vllm-throughput.png" alt="vLLM generation throughput — TP=2">
 
 Generation throughput during Demo 2 (TP=2).
 

docs/hardware/supermicro/smci-mig/README.md

@@ -30,7 +30,7 @@ Spinifex is an open-source infrastructure platform that brings core AWS services
 
 This guide documents a full bare-metal AI deployment on a single [Supermicro X13](https://www.supermicro.com/en/products/x13) chassis using NVIDIA H200 SXM GPUs. These GPUs include the [NVIDIA Multi-Instance GPU](https://www.nvidia.com/en-au/technologies/multi-instance-gpu/) (MIG) capability, which allows a single GPU to be "sliced" into up to seven independent GPU partitions that are hardware isolated. Each partition is capable of running its own workloads with reserved resources from the "host" GPU.
 
-<p align="center"><img src="../../../.github/assets/images/h200/X13.png" alt="Supermicro X13 8U GPU System"></p>
+<p align="center"><img src="../../../../.github/assets/images/h200/X13.png" alt="Supermicro X13 8U GPU System"></p>
 
 In this setup, each EC2 instance receives an entire H200 via PCIe passthrough and manages its own MIG partitions. This gives each tenant full control over how they slice their GPU — including the ability to run heterogeneous workloads at different partition sizes on the same physical card.
 
@@ -72,7 +72,7 @@ Twelve concurrent inference endpoints in total: 7 fast 3B slots, 2 mid-tier 32B
 ### 1. Verify GPU visibility on host
 Before making any changes, verify that the host can see its GPUs:
 
-<img src="../../../.github/assets/images/h200/nvidia-smi.png" alt="NVIDIA SMI">
+<img src="../../../../.github/assets/images/h200/nvidia-smi.png" alt="NVIDIA SMI">
 
 
 ## Instructions
@@ -183,7 +183,7 @@ aws s3 ls --endpoint-url https://localhost:8443
 ### 5. Enable GPU Passthrough
 Spinifex allows GPUs to be utilised by guest VMS via PCIe-passthrough. This can be enabled via `sudo spx admin gpu setup`. Then, after a reboot, run `sudo spx admin gpu enable`. The Spinifex banner should update to reflect GPU passthrough state after every step:
 
-<p align="center"><img src="../../../.github/assets/images/h200/spinifex-banner1.png" alt="GPU Enabled"></p>
+<p align="center"><img src="../../../../.github/assets/images/h200/spinifex-banner1.png" alt="GPU Enabled"></p>
 
 
 ### 6. Import the GPU AMI
@@ -281,7 +281,7 @@ nvidia-smi | grep "MIG M."
 
 With MIG enabled, we are now able to partition each VM's assigned H200 GPU into several separate GPU instances. This process assigns each GPU slice its own UUID, so each VM goes from seeing one whole GPU to seeing a number of "MIG devices":
 
-<p align="center"><img src="../../../.github/assets/images/h200/mig-activated.png" alt="NVIDIA SMI"></p>
+<p align="center"><img src="../../../../.github/assets/images/h200/mig-activated.png" alt="NVIDIA SMI"></p>
 
 This allows us to run several separate workloads, each assigned to its own GPU instance, on the same physical GPU, and thus utilise more of the overall GPU's resources.
 

docs/identity/iam-groups/README.md

@@ -1,5 +1,5 @@
 ---
-title: "AWS IAM Groups for Shared User Permissions"
+title: "IAM Groups for Shared User Permissions"
 description: "Organise AWS IAM users into groups, manage membership, and attach managed or inline policies once for the whole team instead of granting them user by user."
 category: "Identity"
 tags:

docs/identity/iam-roles-and-instance-profiles/README.md

@@ -1,5 +1,5 @@
 ---
-title: "AWS IAM Roles and Instance Profiles for EC2"
+title: "IAM Roles and Instance Profiles for EC2"
 description: "Create AWS IAM roles with trust policies, wrap them in instance profiles, and launch EC2 instances that receive credentials with no static access keys at all."
 category: "Identity"
 tags:

docs/identity/iam-users-and-policies/README.md

@@ -1,5 +1,5 @@
 ---
-title: "AWS IAM Users, Policies, and Access Keys"
+title: "IAM Users, Policies, and Access Keys"
 description: "Create AWS IAM users, issue and rotate access keys, and write and attach JSON policies that control exactly what each user can do in a Spinifex account."
 category: "Identity"
 tags:

docs/identity/imds/README.md

@@ -1,5 +1,5 @@
 ---
-title: "AWS IMDS: Instance Metadata Service (IMDSv2)"
+title: "IMDS: Instance Metadata Service (IMDSv2)"
 description: "Query AWS instance metadata, read user data, and fetch short-lived IAM role credentials from inside a guest VM using IMDSv2 session tokens on Spinifex."
 category: "Identity"
 tags:

docs/identity/sts/README.md

@@ -1,5 +1,5 @@
 ---
-title: "AWS STS Temporary Credentials and AssumeRole"
+title: "STS Temporary Credentials and AssumeRole"
 description: "Use AWS STS to issue temporary credentials: assume an IAM role, request a session token, or federate Kubernetes workloads with OIDC web identity on Spinifex."
 category: "Identity"
 tags:

docs/install/install-multi-node/README.md

@@ -235,7 +235,9 @@ sudo ovn-sbctl show
 
 Run init and join **concurrently** — init blocks until all nodes have joined.
 
-`--force` is in every command below so the sequence is identical whichever way you installed. It does the work on ISO-installed servers, which arrive as their own single-node cluster: joining replaces that server's CA and master key with server 1's, and `--force` is the confirmation. On `spx admin init` it is idempotent — existing keys, credentials and CA are preserved, and only the config files and server certificate are refreshed. On a freshly installed server there is nothing to lose either way.
+`--force` is in every command below so the sequence is identical whichever way you installed. It does the work on ISO-installed servers, which arrive as their own single-node cluster: joining replaces that server's CA and master key with server 1's and removes its JetStream store, and `--force` is the confirmation. On `spx admin init` it is idempotent — existing keys, credentials and CA are preserved, and only the config files and server certificate are refreshed. On a freshly installed server there is nothing to lose either way.
+
+Every server also discards the JetStream store its single-node cluster wrote. Server 1 removes its own once every server has joined, and each joining server removes its own when it joins. Keeping them is never safe: NATS would adopt each server's copy as a replica of the cluster's stream, and those copies never converge. This is automatic whenever `--nodes` is 2 or more. `--discard-jetstream=false` makes init refuse to form over a non-empty store instead, for a server whose store you want to inspect first. Stop `spinifex.target` on every server first and confirm `pgrep -af 'spx service'` prints nothing: init and join both refuse while NATS is still running.
 
 > [!WARNING]
 > Do not point these commands at a server that has already been in service. Joining discards its master key, orphaning every volume and fragment sealed under it. That is what `--force` overrides, and it is unrecoverable.
@@ -251,6 +253,8 @@ sudo spx admin init --force \
 
 `--nodes 3` is the number of servers init waits for. Set it to your total node count if you are building a larger cluster.
 
+IPsec encrypts the Geneve overlay between servers and is on by default. Joining servers take the setting from server 1, so it is chosen once, on init. On servers that share a trusted private link, `--ipsec=false` leaves the overlay unencrypted in exchange for considerably higher throughput between instances.
+
 The init output displays the join command including the token:
 
 ```
@@ -366,6 +370,32 @@ aws ec2 describe-instance-types
 
 A list of instance types means the gateway, IAM and the cluster behind them are all working.
 
+**4. Every server holds the same cluster state.**
+
+Services report Ready even over a cluster whose JetStream replicas disagree, so check the replicas themselves. On **every server**, take a digest of its local store:
+
+```bash
+sudo spx admin kv digest --json --seqs > kv-digest-$(hostname).json
+```
+
+Copy the files to one server and compare them:
+
+```bash
+spx admin kv compare kv-digest-*.json
+```
+
+```
+compared 3 digests: node1@10:02:11 node2@10:02:14 node3@10:02:18
+ok        KV_spinifex-iam-users  [node1,node2,node3]
+      identical, seqs 1-14
+...
+42 streams: 42 consistent, 0 diverged
+```
+
+The last line must report `0 diverged`, and the command exits non-zero otherwise. The digests are copied from live stores a few seconds apart, so a write landing between two copies can briefly show as a difference: take fresh digests and compare again before acting on one. A difference that persists means the cluster adopted a server's old store as a replica. Do not put that cluster in service — reset the servers and form it again.
+
+A replica that is merely behind is not reported, and neither is a message that one server has already replaced with a newer write to the same key. `spx admin kv compare --help` lists exactly what counts as divergence.
+
 **Congratulations! Your Spinifex cluster is installed.**
 
 Continue to [Setting Up Your Cluster](/docs/setting-up-your-cluster) to import an AMI, create a VPC, and launch your first instance.

docs/terraform-workbooks/rds-quickstart/README.md

@@ -16,8 +16,8 @@ sections:
   - instructions
   - troubleshooting
 resources:
-  - title: "RDS CLI Reference"
-    url: "https://github.com/mulgadc/spinifex/blob/main/docs/COMMANDS.md#rds-postgresql-and-mariadb"
+  - title: "RDS API Coverage"
+    url: "/coverage/rds"
   - title: "Terraform AWS Provider — aws_db_instance"
     url: "https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/db_instance"
   - title: "Spinifex Repository"
@@ -30,7 +30,7 @@ resources:
 
 > The smallest end-to-end RDS example on Spinifex: a VPC with a public client subnet and a private DB subnet, a DB subnet group, a parameter group, a PostgreSQL DB instance, and a client VM with `psql` already pointed at the endpoint.
 
-This workbook is **PostgreSQL throughout** — the engine, the `postgres18` parameter group, port `5432`, the client package and the `~/.pgpass` credential are all PostgreSQL's. Spinifex also serves MariaDB 11.8 under `engine = "mariadb"`; running this workbook against it means changing `engine`, `engine_version`, the parameter-group `family` to `mariadb11.8`, the security-group port to `3306` and the client to `mariadb-client`. See the [RDS command reference](https://github.com/mulgadc/spinifex/blob/main/docs/COMMANDS.md#rds-postgresql-and-mariadb) for both engines' pins, families and limits. Note that `mysql` is not an accepted engine and is not an alias for `mariadb`.
+This workbook is **PostgreSQL throughout** — the engine, the `postgres18` parameter group, port `5432`, the client package and the `~/.pgpass` credential are all PostgreSQL's. Spinifex also serves MariaDB 11.8 under `engine = "mariadb"`; running this workbook against it means changing `engine`, `engine_version`, the parameter-group `family` to `mariadb11.8`, the security-group port to `3306` and the client to `mariadb-client`. See the [RDS API coverage page](/coverage/rds) for both engines' pins, families and limits. Note that `mysql` is not an accepted engine and is not an alias for `mariadb`.
 
 ## Overview
 

go.mod

@@ -4,27 +4,28 @@ go 1.27.0
 
 require (
 	github.com/aws/aws-sdk-go v1.55.8
-	github.com/aws/aws-sdk-go-v2 v1.45.1
+	github.com/aws/aws-sdk-go-v2 v1.47.0
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20
-	github.com/aws/aws-sdk-go-v2/config v1.33.2
-	github.com/aws/aws-sdk-go-v2/credentials v1.20.2
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.19.1
+	github.com/aws/aws-sdk-go-v2/config v1.33.4
+	github.com/aws/aws-sdk-go-v2/credentials v1.20.4
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/charmbracelet/bubbles v1.0.0
 	github.com/charmbracelet/bubbletea v1.3.10
 	github.com/charmbracelet/lipgloss v1.1.0
-	github.com/containerd/containerd/v2 v2.3.4
+	github.com/containerd/containerd/v2 v2.3.5
 	github.com/distribution/reference v0.6.0
 	github.com/go-chi/chi/v5 v5.3.2
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82
-	github.com/jackc/pgx/v5 v5.10.0
+	github.com/jackc/pgx/v5 v5.11.0
 	github.com/klauspost/cpuid/v2 v2.4.0
 	github.com/miekg/dns v1.1.73
-	github.com/mulgadc/bluebottle v1.19.0
-	github.com/mulgadc/northstar v1.19.0
-	github.com/mulgadc/predastore v1.19.0
-	github.com/mulgadc/viperblock v1.19.0
+	github.com/minio/highwayhash v1.0.4
+	github.com/mulgadc/bluebottle v1.20.0
+	github.com/mulgadc/northstar v1.20.0
+	github.com/mulgadc/predastore v1.20.0
+	github.com/mulgadc/viperblock v1.20.0
 	github.com/nats-io/nats-server/v2 v2.14.6
 	github.com/nats-io/nats.go v1.53.1
 	github.com/opencontainers/runtime-spec v1.3.0
@@ -42,11 +43,11 @@ require (
 	go.opentelemetry.io/otel/trace v1.46.0
 	go.uber.org/goleak v1.3.1-0.20260602155401-5f791e3ef83b
 	go.yaml.in/yaml/v3 v3.0.5
-	golang.org/x/crypto v0.56.0
-	golang.org/x/sync v0.22.0
-	golang.org/x/sys v0.47.0
-	golang.org/x/time v0.15.0
-	golang.org/x/tools v0.49.0
+	golang.org/x/crypto v0.57.0
+	golang.org/x/sync v0.23.0
+	golang.org/x/sys v0.48.0
+	golang.org/x/time v0.16.0
+	golang.org/x/tools v0.50.0
 	gopkg.in/ini.v1 v1.67.3
 )
 
@@ -59,18 +60,18 @@ require (
 	github.com/antithesishq/antithesis-sdk-go v0.7.2-default-no-op // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.1 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.1 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.110.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.36.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.41.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.48.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.113.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.50.0 // indirect
 	github.com/aws/smithy-go v1.28.1 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
@@ -95,7 +96,7 @@ require (
 	github.com/containerd/errdefs/pkg v0.3.0 // indirect
 	github.com/containerd/fifo v1.1.0 // indirect
 	github.com/containerd/log v0.1.0 // indirect
-	github.com/containerd/platforms v1.0.0-rc.4 // indirect
+	github.com/containerd/platforms v1.0.0-rc.5 // indirect
 	github.com/containerd/plugin v1.1.0 // indirect
 	github.com/containerd/ttrpc v1.2.8 // indirect
 	github.com/containerd/typeurl/v2 v2.2.3 // indirect
@@ -146,7 +147,6 @@ require (
 	github.com/mdlayher/packet v1.1.2 // indirect
 	github.com/mdlayher/socket v0.5.1 // indirect
 	github.com/minio/crc64nvme v1.1.1 // indirect
-	github.com/minio/highwayhash v1.0.4 // indirect
 	github.com/moby/locker v1.0.1 // indirect
 	github.com/moby/sys/capability v0.4.0 // indirect
 	github.com/moby/sys/mountinfo v0.7.2 // indirect
@@ -196,15 +196,15 @@ require (
 	go.uber.org/nilaway v0.0.0-20260721205819-4a0f7653488a // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
 	golang.org/x/exp/typeparams v0.0.0-20260611194520-c48552f49976 // indirect
-	golang.org/x/mod v0.39.0 // indirect
-	golang.org/x/net v0.58.0 // indirect
-	golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5 // indirect
-	golang.org/x/term v0.45.0 // indirect
-	golang.org/x/text v0.41.0 // indirect
+	golang.org/x/mod v0.41.0 // indirect
+	golang.org/x/net v0.59.0 // indirect
+	golang.org/x/telemetry v0.0.0-20260908163034-4bcc4b2ee518 // indirect
+	golang.org/x/term v0.46.0 // indirect
+	golang.org/x/text v0.42.0 // indirect
 	golang.org/x/vuln v1.6.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect
-	google.golang.org/grpc v1.83.1 // indirect
+	google.golang.org/grpc v1.83.2 // indirect
 	google.golang.org/protobuf v1.36.12 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	sigs.k8s.io/yaml v1.6.0 // indirect

go.sum

@@ -524,72 +524,72 @@ github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZw
 github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
 github.com/aws/aws-sdk-go-v2 v1.25.2/go.mod h1:Evoc5AsmtveRt1komDwIsjHFyrP5tDuF1D1U+6z6pNo=
 github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
-github.com/aws/aws-sdk-go-v2 v1.45.1 h1:iIoG3NaLhV6UZpPXyPXlDj2I9oS8tV/nMcMnITCC6Ks=
-github.com/aws/aws-sdk-go-v2 v1.45.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU=
+github.com/aws/aws-sdk-go-v2 v1.47.0 h1:0jsHallhJCeaU0Ko48c/3FK1ctOQ7NpzggxriJOQ8MQ=
+github.com/aws/aws-sdk-go-v2 v1.47.0/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8=
 github.com/aws/aws-sdk-go-v2/config v1.18.14/go.mod h1:0pI6JQBHKwd0JnwAZS3VCapLKMO++UL2BOkWwyyzTnA=
 github.com/aws/aws-sdk-go-v2/config v1.27.4/go.mod h1:zq2FFXK3A416kiukwpsd+rD4ny6JC7QSkp4QdN1Mp2g=
 github.com/aws/aws-sdk-go-v2/config v1.29.12/go.mod h1:xse1YTjmORlb/6fhkWi8qJh3cvZi4JoVNhc+NbJt4kI=
-github.com/aws/aws-sdk-go-v2/config v1.33.2 h1:Pj4+nF2kc4Z+1BJysVPnX9d5dMN7IYFXR4UJaWK2IpA=
-github.com/aws/aws-sdk-go-v2/config v1.33.2/go.mod h1:Igw+HTwbR2tsTU/ydifAS9EHAFJ2s/FCgkwQWFnAdE4=
+github.com/aws/aws-sdk-go-v2/config v1.33.4 h1:FzvkXKSzwqHni4U7nDigHg4jjtqMpVUuHgmZfSoJVQ0=
+github.com/aws/aws-sdk-go-v2/config v1.33.4/go.mod h1:VZqGZnZsCWVfK/iGPptJIyNIX3XEX6iQU2Rel4sLrr8=
 github.com/aws/aws-sdk-go-v2/credentials v1.13.14/go.mod h1:85ckagDuzdIOnZRwws1eLKnymJs3ZM1QwVC1XcuNGOY=
 github.com/aws/aws-sdk-go-v2/credentials v1.17.4/go.mod h1:+30tpwrkOgvkJL1rUZuRLoxcJwtI/OkeBLYnHxJtVe0=
 github.com/aws/aws-sdk-go-v2/credentials v1.17.65/go.mod h1:4zyjAuGOdikpNYiSGpsGz8hLGmUzlY8pc8r9QQ/RXYQ=
-github.com/aws/aws-sdk-go-v2/credentials v1.20.2 h1:VQjZODPNfdikCX2ZZrltw4zNLkcwjyUFDUl2vT9yTwg=
-github.com/aws/aws-sdk-go-v2/credentials v1.20.2/go.mod h1:OmeHCn28vZylsBvalLDf7t8fuJ2rHYQprJs+7WuxniI=
+github.com/aws/aws-sdk-go-v2/credentials v1.20.4 h1:hTvrJJseKbvw32kmiE0G+u/9ZqpqscjDrTigHIXP2qs=
+github.com/aws/aws-sdk-go-v2/credentials v1.20.4/go.mod h1:gWp9O1ZBWwpcIrgV+mVHk4gZUurAEDkgypu/OXOlIaw=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23/go.mod h1:mOtmAg65GT1HIL/HT/PynwPbS+UG0BgCZ6vhkPqnxWo=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.2/go.mod h1:iRlGzMix0SExQEviAyptRWRGdYNo3+ufW/lCzvKVTUc=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.19.1 h1:YIEBqcqRnpi4Pfv0YHImtgi6czGCwKHANC7SwmUAVD0=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.19.1/go.mod h1:imEf0oufgAo8KAkCHhrOdqGEC0YWx1PPBQH82shSxGw=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0 h1:AM4hHjww+PSFtt6E+UrBrPlZkWsePCLEt9AjkfQX+yM=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0/go.mod h1:3x/yXezeQjpOvBb4jEMxrS8SXvpdvJ5abv6l5c1gWM8=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.2/go.mod h1:wRQv0nN6v9wDXuWThpovGQjqF1HFdcgWjporw14lS8k=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.1 h1:pc138gM1CW+XPc60rEwUlwwuwWFQK16CI1T7v1F9Oec=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.1/go.mod h1:1+koxpPIbfBdfzP6vojm5/zTpTQ/micYwlxIiNB3TxI=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 h1:Hp/VgjP0BysR3OgLlR057Vz2LcbbVnoWeJ+3qWiS/fY=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3/go.mod h1:nwGV5qw7F1IZPgxCvA/ph8N2TAuz+BkRG/bXn808qMA=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.2/go.mod h1:tyF5sKccmDz0Bv4NrstEr+/9YkSPJHrcO7UsUKf7pWM=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.1 h1:K0JsbZQj+1h208Ro1zHeA4l7bMp0NvRffHQ91q8Ol1s=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.1/go.mod h1:W3/vL6EtCIatICGy9ab29QhMuae+cOKPWcMxv02CO+Q=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 h1:MUaM4f+kj1ZIBPZfUS8cxP1GKXXZtHJjAthy93AN7SM=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3/go.mod h1:6YmVmEVRI5ZZzRjCSsb9SryKH0hAlMRdgA7kG9aDvBU=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30/go.mod h1:vsbq62AOBwQ1LJ/GWKFxX8beUEYeRp/Agitrxee2/qM=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.1 h1:yhw5KD1phVyP9vijxOUzDfEtJx+bt+L63k+VfuiYFAA=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.1/go.mod h1:ZW2e0d7DYlRxlS9hEiMXE47gTdX5KRN4byUiNbUpG+Q=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 h1:fuSCw4Z2qfRCztMPO3GXJNSiEp6Wee+WOLwrHHUMy9c=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3/go.mod h1:6SxcHheD1pPR5+kWm1wGvjlL/YqUsh267sAfEmN4K7A=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.1 h1:s67hBfG5t9rn1NCvDuB4E3QIep3UFhHPtaIqFDjV3N8=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.1/go.mod h1:FpvjBMXtSNMLPmDJsWwcY5cRnqJlpS2y1R6n4pvzs4k=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.3 h1:BHKCSX4QXERe8So8rbWqaM7owqOmDJxATXgJwGng22A=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.3/go.mod h1:GqWeeKfYfezihA2KfFL9l7ohEdZWe1tuFWh3GfyNSnE=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23/go.mod h1:9uPh+Hrz2Vn6oMnQYiUi/zbh3ovbnQk19YKINkQny44=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.2/go.mod h1:Ru7vg1iQ7cR4i7SZ/JTLYN9kaXtbL69UdgG0OQWQxW0=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.1 h1:RmmWQPREQdk9U+PfqeHW3MqZaBaNK7TpV9W3RY+b+7g=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.1/go.mod h1:0A3W4F+68ZnNk5XcNL/e9HFMwnP8RlEicFfy6eOEDyw=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.1 h1:ZMbtPZZQRca+3+XYQne9PBvRiYpHZlNJJOZfE9WNfT0=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.1/go.mod h1:YAGWQdCYlVCoqrzvfv3RLxO6zKwti7gsAULOGWPLYv4=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.110.0 h1:He8vaTTqAAJrux/KdpjFXNWueLJZyKqE49QEXoqAu4I=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.110.0/go.mod h1:CUr46sCpGAg/rHaclRyhJX0LJAmH73uWSJPPSaMUrSk=
-github.com/aws/aws-sdk-go-v2/service/signin v1.8.0 h1:bSvKIoLuRGFqGwASgeCQncCJDi9YKKBDEmCEZzOX1uU=
-github.com/aws/aws-sdk-go-v2/service/signin v1.8.0/go.mod h1:9IqUlsJDbUPcg6cgx3WEzXdjrbWzLDQrak0aaSqlTcI=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 h1:bON1rJf67TSTDCKg816AAIE4xSTtoo9tl0XRkO72R+I=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3/go.mod h1:c5BBpjJcQXpfeq9iASyVKA3T6vX6B6LEXY4mL/gklDY=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.3 h1:L8vIOxylma91TcR96NFTEC07G3JDwSl+CvK2b+IODms=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.3/go.mod h1:fmPIZQzTExYuBNWFyi1P7IoDjvskgphXqK1yObMzusM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.113.0 h1:0slwIjBv1sEigSUW4EfTtNw9mbHl0PlOUPX9Y1C/eLE=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.113.0/go.mod h1:/uA+2Qj4jd5qBWagVC1AyzzDFXVK997E7U04w7Kw0wI=
+github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 h1:ZD5qFpWcaOKdTuhBi431pIDkCgrMkMlMT6jlpSPoIRI=
+github.com/aws/aws-sdk-go-v2/service/signin v1.10.0/go.mod h1:8Nuuf+tR346PjJ3MvZPh9pekbLiLQFWJhzMXfwy7alA=
 github.com/aws/aws-sdk-go-v2/service/sso v1.12.3/go.mod h1:jtLIhd+V+lft6ktxpItycqHqiVXrPIRjWIsFIlzMriw=
 github.com/aws/aws-sdk-go-v2/service/sso v1.20.1/go.mod h1:RsYqzYr2F2oPDdpy+PdhephuZxTfjHQe7SOBcZGoAU8=
 github.com/aws/aws-sdk-go-v2/service/sso v1.25.2/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI=
-github.com/aws/aws-sdk-go-v2/service/sso v1.36.0 h1:iivsh357VnfIc18IFWSuoyQEluf8frfWf4cL2Y0JUQw=
-github.com/aws/aws-sdk-go-v2/service/sso v1.36.0/go.mod h1:tWuiVBUtPBr8/rgRiYS8Uf85sHcAN+G7XS3D3CEoUh8=
+github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 h1:JGeeBcMlhg1xtOXYpeCaTQBZObtXMPQCUqBcmr65NRA=
+github.com/aws/aws-sdk-go-v2/service/sso v1.38.0/go.mod h1:XwteswG9EOMRFm73UT0t+MbTwyLxMrEXkU6e+v92Lzo=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.3/go.mod h1:zVwRrfdSmbRZWkUkWjOItY7SOalnFnq/Yg2LVPqDjwc=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.1/go.mod h1:YjAPFn4kGFqKC54VsHs5fn5B6d+PCY2tziEa3U/GB5Y=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.41.0 h1:wVxM3QzSKIK8tSN6OGgezp9OK91lCLH2zhmRInN9rFM=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.41.0/go.mod h1:naFe83jSMuYkH+QjQPX8n1MLhBkeCFM5Lsnh5m5wz3c=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 h1:obhahQXDEdVEv8y5bTKXR30LVaxYe1kyYM0L7l2Iq+k=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0/go.mod h1:6twZZ/aXHNy1vXUO8koUbp++MYzMASkOgEBdkbJYmO0=
 github.com/aws/aws-sdk-go-v2/service/sts v1.18.4/go.mod h1:1mKZHLLpDMHTNSYPJ7qrcnCQdHCWsNQaT0xRvq2u80s=
 github.com/aws/aws-sdk-go-v2/service/sts v1.28.1/go.mod h1:uQ7YYKZt3adCRrdCBREm1CD3efFLOUNH77MrUCvx5oA=
 github.com/aws/aws-sdk-go-v2/service/sts v1.33.17/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4=
-github.com/aws/aws-sdk-go-v2/service/sts v1.48.0 h1:RzZVCzYM19vhJCT5s6vO2wN8ie770Li/TmbAZ9B6N7E=
-github.com/aws/aws-sdk-go-v2/service/sts v1.48.0/go.mod h1:mKo/CzaCz8qytGW70NG4vIIGAx1HXTlb5lHNkC5k3lk=
+github.com/aws/aws-sdk-go-v2/service/sts v1.50.0 h1:khXV3+K5D3f4e8xtplaRdSFn1bEg3gj5EBHQvbCOZbQ=
+github.com/aws/aws-sdk-go-v2/service/sts v1.50.0/go.mod h1:/8JRcdTt//hG0Q4BTmGbuOplT7ABe+5rdtqUHqXvYIM=
 github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
 github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
@@ -701,8 +701,8 @@ github.com/containerd/console v1.0.5 h1:R0ymNeydRqH2DmakFNdmjR2k0t7UPuiOV/N/27/q
 github.com/containerd/console v1.0.5/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk=
 github.com/containerd/containerd/api v1.11.1 h1:h8nfoDW9+fNsC/9TwiAHj8B1GzXKtR4eFtkhi/X5RLU=
 github.com/containerd/containerd/api v1.11.1/go.mod h1:CaQFRu+N1MtbgL6JDOJLUB1hCKESU1lD6MuTJhgtdlw=
-github.com/containerd/containerd/v2 v2.3.4 h1:c2PJo/9UGVdiiw8SwrxuLxWGY+9b3jQ6Xp9zntneIvI=
-github.com/containerd/containerd/v2 v2.3.4/go.mod h1:a30D8fWZJ1Uzx/2WpjLbLsxBkq9He41pe8ENW+QZ3LY=
+github.com/containerd/containerd/v2 v2.3.5 h1:9MYlI81gUcOZ0WsCkSMtvOU7rTR3hqAoa2eCzhoLlkA=
+github.com/containerd/containerd/v2 v2.3.5/go.mod h1:RXDyLPaI3zoO7dFdAW9/54W4cix+z3A6larieufC9mg=
 github.com/containerd/continuity v0.5.0 h1:7a85HZpCSs+1Zps0Ee3DPSuAWY+0SJM1JNM51nlEVDg=
 github.com/containerd/continuity v0.5.0/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE=
 github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
@@ -713,8 +713,8 @@ github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY
 github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o=
 github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
-github.com/containerd/platforms v1.0.0-rc.4 h1:M42JrUT4zfZTqtkUwkr0GzmUWbfyO5VO0Q5b3op97T4=
-github.com/containerd/platforms v1.0.0-rc.4/go.mod h1:lKlMXyLybmBedS/JJm11uDofzI8L2v0J2ZbYvNsbq1A=
+github.com/containerd/platforms v1.0.0-rc.5 h1:vXd569rDrz8LeMXzAnBsy6LADV5YtsD8oyaRarxdmSU=
+github.com/containerd/platforms v1.0.0-rc.5/go.mod h1:lKlMXyLybmBedS/JJm11uDofzI8L2v0J2ZbYvNsbq1A=
 github.com/containerd/plugin v1.1.0 h1:O+7lczNJVMy8rz0YNx3xGB8tTf5qY4i5abF041Ew19U=
 github.com/containerd/plugin v1.1.0/go.mod h1:qBTum+A8lJ6lO44A19Eo7y1OlcLj4OWFH1DA/vnHmcc=
 github.com/containerd/ttrpc v1.2.8 h1:xbVu6D4qF2jihdh9rDVOKqUMiFBQk6YctTdo1zk087Y=
@@ -1175,8 +1175,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
-github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
-github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
+github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
+github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
 github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
 github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jezek/xgb v1.0.0/go.mod h1:nrhwO0FX/enq75I7Y7G8iN1ubpSGZEiA3v9e9GyRFlk=
@@ -1327,14 +1327,14 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
 github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
-github.com/mulgadc/bluebottle v1.19.0 h1:KjaH4TbAAZAEfGVd47rgsBW4fLBm0eF3+kzE3mF95lo=
-github.com/mulgadc/bluebottle v1.19.0/go.mod h1:sg+gqQmHRdrS5ObBGfIRhl9sBvGbqMhLZElnC4otG/c=
-github.com/mulgadc/northstar v1.19.0 h1:iC3RaAPPBcsNzIKQKGMBKOMQgpUh2ZCArZK2Yzjf21E=
-github.com/mulgadc/northstar v1.19.0/go.mod h1:m23lGk5aHv/yTbR0XZT4rZobj6EZ4Cu8p1yzNA5npxU=
-github.com/mulgadc/predastore v1.19.0 h1:y47JrsOcBZb8391ndY8muT2uODIHUpeNP7c2Gs1TuqI=
-github.com/mulgadc/predastore v1.19.0/go.mod h1:mGMHNTPIsKFMC0CegmXYc5tHqQgVvDDCof3K3hL48pI=
-github.com/mulgadc/viperblock v1.19.0 h1:8gHHDy8pKtv2iqp6UmEpsHitDK/gxNLVHZ1GPqR6Bic=
-github.com/mulgadc/viperblock v1.19.0/go.mod h1:bjsD4znZgK3t5RkSlPrHE81FrDijy8vBcW2ESTnkNvo=
+github.com/mulgadc/bluebottle v1.20.0 h1:Udi+aOA14CgDLfhanw4Rsa5HqkVt+/9e6mxoQWTRCNs=
+github.com/mulgadc/bluebottle v1.20.0/go.mod h1:4rjLL3B0xZBPmxacaXPkw6sPOZZgyuCDBFmPSOC81yE=
+github.com/mulgadc/northstar v1.20.0 h1:nkWTuJv1aagrJUQiZHruo7xzu9RBtLHEzrQGABElwls=
+github.com/mulgadc/northstar v1.20.0/go.mod h1:3GB49VUE9rA2idp88se5jAMv3ql1Oq9kIcT29WAZwDU=
+github.com/mulgadc/predastore v1.20.0 h1:BBk02ZWWbCGqy0PkYEXh5Yk5lbOUuDEcMS++voxSc6E=
+github.com/mulgadc/predastore v1.20.0/go.mod h1:YDPhz9EZLw3nrlSkpDSQkXArJIk9TB/kpcp1wfsfFnU=
+github.com/mulgadc/viperblock v1.20.0 h1:g17NkHQcXGSXBGp20UcAPrklzTGXdyERXdaldo1aKM4=
+github.com/mulgadc/viperblock v1.20.0/go.mod h1:q+Zs0C1pKfJVj8oALXZrZl33Sd+vPX0yc8ZroFtJBxM=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -1905,8 +1905,8 @@ golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVo
 golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
 golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
 golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
-golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y=
-golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I=
+golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
+golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -2020,8 +2020,8 @@ golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
 golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
 golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
 golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU=
-golang.org/x/mod v0.39.0 h1:UF5zwQdCRRUpHfyPwr7d4UrGiVeldIsogtzWVnczL74=
-golang.org/x/mod v0.39.0/go.mod h1:bvIbwjQ0HUFFf5AKukeeYQG4ZBUG9yxQbR9aEweIwYY=
+golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
+golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -2134,8 +2134,8 @@ golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
 golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
 golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
 golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
-golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
-golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
+golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
+golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -2228,8 +2228,8 @@ golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
-golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
-golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
+golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -2372,8 +2372,8 @@ golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
-golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
+golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0=
 golang.org/x/telemetry v0.0.0-20250710130107-8d8967aff50b/go.mod h1:4ZwOYna0/zsOKwuR5X/m0QFOJpSZvAxFfkQT+Erd9D4=
@@ -2386,8 +2386,8 @@ golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2/go.mod h1:b7fPSJ0pKZ3c
 golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4/go.mod h1:g5NllXBEermZrmR51cJDQxmJUHUOfRAaNyWBM+R+548=
 golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c/go.mod h1:TpUTTEp9frx7rTdLpC9gFG9kdI7zVLFTFFlqaH2Cncw=
 golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa/go.mod h1:kHjTxDEnAu6/Nl9lDkzjWpR+bmKfxeiRuSDlsMb70gE=
-golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5 h1:ZUSxONxc981v7AW7QUg+I9WwZzSTTJ019ENBYr5pV/Q=
-golang.org/x/telemetry v0.0.0-20260811182544-a038080d80e5/go.mod h1:LVehoXe41cL5SCVQilsV7Gg6BNG+Js6P9PhSbYTIUkQ=
+golang.org/x/telemetry v0.0.0-20260908163034-4bcc4b2ee518 h1:F5BWKvW126NXR74uxkxuc1jQHhm/rwm/J3rSiFyuRs4=
+golang.org/x/telemetry v0.0.0-20260908163034-4bcc4b2ee518/go.mod h1:i+ivNqjDnTF3WTElsdk5g9V5DTSBYgdNo7xTU9SDwYA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -2435,8 +2435,8 @@ golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
 golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
 golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
 golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
-golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
-golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
+golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
+golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2481,8 +2481,8 @@ golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
 golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
 golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
 golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
-golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
-golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
+golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
+golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -2498,8 +2498,9 @@ golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
 golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
-golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
 golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
+golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
+golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -2597,8 +2598,8 @@ golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg
 golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
 golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
 golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=
-golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
-golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
+golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU=
+golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0=
 golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM=
 golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY=
 golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM=
@@ -3126,8 +3127,8 @@ google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhH
 google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
 google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4=
 google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I=
-google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y=
-google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
+google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=
+google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y=
 google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b/go.mod h1:IBqQ7wSUJ2Ep09a8rMWFsg4fmI2r38zwsq8a0GgxXpM=

internal/awsmodel/coverage.go

@@ -123,65 +123,113 @@ func toSet(values []string) map[string]bool {
 	return set
 }
 
-// RenderCoverageMarkdown renders a reproducible divergence report suitable for
-// publishing directly or checking as a generated artefact.
-func RenderCoverageMarkdown(coverages []OperationCoverage) string {
-	ordered := append([]OperationCoverage(nil), coverages...)
-	sort.Slice(ordered, func(i, j int) bool { return ordered[i].Service < ordered[j].Service })
-
+// RenderCoverageSummary renders the cross-service counts as plain text for a
+// terminal. The published pages carry the per-operation detail.
+func RenderCoverageSummary(coverages []OperationCoverage) string {
 	var report strings.Builder
-	fmt.Fprintf(&report, "# AWS model operation coverage\n\n")
-	fmt.Fprintf(&report, "Generated from the cached `aws-sdk-go %s` `api-2.json` models and Spinifex's authoritative gateway dispatch tables.\n\n", SourceSDKVersion)
-	report.WriteString("Implemented means a modelled operation is registered to a real handler. Stub and deliberately unsupported handlers are reported separately. Shape conformance does not imply behavioural conformance.\n\n")
-	report.WriteString("| Service | API version | Modelled | Registered | Implemented | Stubbed | Unsupported | Missing | Extra |\n")
-	report.WriteString("|---|---:|---:|---:|---:|---:|---:|---:|---:|\n")
-	for _, coverage := range ordered {
+	fmt.Fprintf(&report, "AWS model operation coverage (aws-sdk-go %s)\n\n", SourceSDKVersion)
+	for _, coverage := range sortedCoverages(coverages) {
 		if coverage.Opaque {
-			fmt.Fprintf(&report, "| %s | %s | %d | — | — | — | — | — | — |\n",
-				coverage.Service, coverage.APIVersion, len(coverage.Modelled))
+			fmt.Fprintf(&report, "%-24s %-12s not enumerable\n", coverage.Service, coverage.APIVersion)
 			continue
 		}
-		fmt.Fprintf(&report, "| %s | %s | %d | %d | %d | %d | %d | %d | %d |\n",
-			coverage.Service, coverage.APIVersion, len(coverage.Modelled), len(coverage.Registered),
-			len(coverage.Implemented), len(coverage.Stubbed), len(coverage.Unsupported), len(coverage.Missing), len(coverage.Extra))
-	}
-
-	for _, coverage := range ordered {
-		fmt.Fprintf(&report, "\n## %s\n\n", coverage.Service)
-		if coverage.Opaque {
-			fmt.Fprintf(&report, "Operation coverage is not enumerable: %s\n", coverage.Note)
-			continue
-		}
-		percentage := float64(0)
-		if len(coverage.Modelled) != 0 {
-			percentage = 100 * float64(len(coverage.Implemented)) / float64(len(coverage.Modelled))
-		}
-		fmt.Fprintf(&report, "Implements **%d of %d** modelled operations (%.1f%%).\n", len(coverage.Implemented), len(coverage.Modelled), percentage)
-		writeOperationList(&report, "Implemented", coverage.Implemented)
-		writeOperationList(&report, "Missing from dispatch", coverage.Missing)
-		writeOperationList(&report, "Registered stubs", coverage.Stubbed)
-		writeOperationList(&report, "Deliberately unsupported", coverage.Unsupported)
-		writeOperationList(&report, "Registered outside the pinned model", coverage.Extra)
-		if coverage.Note != "" {
-			fmt.Fprintf(&report, "\nNote: %s\n", coverage.Note)
-		}
+		fmt.Fprintf(&report, "%-24s %-12s %3d of %3d implemented (%5.1f%%), %3d stubbed, %3d unsupported, %3d outside the model\n",
+			coverage.Service, coverage.APIVersion, len(coverage.Implemented), len(coverage.Modelled),
+			coverage.ImplementedPercent(), len(coverage.Stubbed), len(coverage.Unsupported), len(coverage.Extra))
 	}
 	return report.String()
 }
 
-func writeOperationList(report *strings.Builder, title string, operations []string) {
-	fmt.Fprintf(report, "\n<details><summary>%s (%d)</summary>\n\n", title, len(operations))
-	if len(operations) == 0 {
-		report.WriteString("None.\n")
-	} else {
-		for start := 0; start < len(operations); start += 8 {
-			end := min(start+8, len(operations))
-			quoted := make([]string, 0, end-start)
-			for _, operation := range operations[start:end] {
-				quoted = append(quoted, "`"+operation+"`")
-			}
-			report.WriteString(strings.Join(quoted, ", ") + "\n\n")
+// RenderInternalReport renders the full comparison, including the operations
+// the published pages leave out. It is not a site page: the docs build renders
+// only the slugs it is configured with, and this file is not one of them.
+//
+// A declared not-applicable operation is listed apart from the gaps, with the
+// reason, so a list of candidate work never carries one nobody intends to
+// build.
+func RenderInternalReport(coverages []OperationCoverage, pages PageSet) string {
+	var report strings.Builder
+	report.WriteString("# AWS Model Operation Coverage (internal)\n\n")
+	report.WriteString("Generated by `make aws-model-coverage`. This file is not published: it carries the counts and the unimplemented operations that the pages under this directory deliberately leave out.\n\n")
+	report.WriteString("**Not implemented** is the candidate work. **Not applicable** is declared in `pages.json` and is not work: those operations describe something this platform will not have.\n\n")
+	fmt.Fprintf(&report, "Pinned model source: `aws-sdk-go %s`.\n\n", SourceSDKVersion)
+
+	for _, coverage := range sortedCoverages(coverages) {
+		page := pages.Services[coverage.Service]
+		fmt.Fprintf(&report, "## %s\n\n", coverage.Service)
+		if coverage.Opaque {
+			fmt.Fprintf(&report, "`%s` — not enumerable. %s\n\n", coverage.APIVersion, coverage.Note)
+			continue
 		}
+
+		missing, notApplicable := splitNotApplicable(coverage.Missing, page)
+		refused, declaredRefusals := splitNotApplicable(coverage.Unsupported, page)
+		notApplicable = append(notApplicable, declaredRefusals...)
+		sort.Strings(notApplicable)
+
+		fmt.Fprintf(&report, "`%s` — %d of %d modelled operations implemented (%.1f%%), %d stubbed, %d not applicable, %d registered outside the pinned model.\n\n",
+			coverage.APIVersion, len(coverage.Implemented), len(coverage.Modelled), coverage.ImplementedPercent(),
+			len(coverage.Stubbed), len(notApplicable), len(coverage.Extra))
+		writeOperationList(&report, "Not implemented", missing)
+		writeOperationList(&report, "Stubbed", coverage.Stubbed)
+		writeOperationList(&report, "Registered and refused", refused)
+		writeOperationList(&report, "Registered outside the pinned model", coverage.Extra)
+		writeNotApplicableList(&report, notApplicable, page)
 	}
-	report.WriteString("\n</details>\n")
+	return report.String()
+}
+
+// splitNotApplicable divides operations into those still open and those the
+// page declares the platform will never serve.
+func splitNotApplicable(operations []string, page PageMetadata) (open, notApplicable []string) {
+	for _, operation := range operations {
+		if _, declared := page.NotApplicable[operation]; declared {
+			notApplicable = append(notApplicable, operation)
+			continue
+		}
+		open = append(open, operation)
+	}
+	return open, notApplicable
+}
+
+// writeNotApplicableList records why each declared operation is not work,
+// grouped by the reason it was declared under.
+func writeNotApplicableList(report *strings.Builder, operations []string, page PageMetadata) {
+	if len(operations) == 0 {
+		return
+	}
+	byNote := map[string][]string{}
+	for _, operation := range operations {
+		key := page.NotApplicable[operation]
+		byNote[key] = append(byNote[key], operation)
+	}
+
+	report.WriteString("### Not applicable — do not build\n\n")
+	for _, key := range noteOrder(page.Notes) {
+		if len(byNote[key]) == 0 {
+			continue
+		}
+		fmt.Fprintf(report, "%s: %s\n\n", key, page.Notes[key])
+		for _, operation := range byNote[key] {
+			fmt.Fprintf(report, "- `%s`\n", operation)
+		}
+		report.WriteString("\n")
+	}
+}
+
+func writeOperationList(report *strings.Builder, heading string, operations []string) {
+	if len(operations) == 0 {
+		return
+	}
+	fmt.Fprintf(report, "### %s\n\n", heading)
+	for _, operation := range operations {
+		fmt.Fprintf(report, "- `%s`\n", operation)
+	}
+	report.WriteString("\n")
+}
+
+func sortedCoverages(coverages []OperationCoverage) []OperationCoverage {
+	ordered := append([]OperationCoverage(nil), coverages...)
+	sort.Slice(ordered, func(i, j int) bool { return ordered[i].Service < ordered[j].Service })
+	return ordered
 }

internal/awsmodel/coverage_test.go

@@ -32,20 +32,21 @@ func TestCompareOperationsRejectsInvalidInventory(t *testing.T) {
 	}
 }
 
-func TestRenderCoverageMarkdown(t *testing.T) {
+func TestRenderCoverageSummary(t *testing.T) {
 	coverage, err := CompareOperations(STS, DispatchInventory{Registered: []string{"AssumeRole"}})
 	if err != nil {
 		t.Fatal(err)
 	}
-	report := RenderCoverageMarkdown([]OperationCoverage{coverage})
+	report := RenderCoverageSummary([]OperationCoverage{coverage})
 	for _, want := range []string{
-		"# AWS model operation coverage",
-		"| sts | 2011-06-15 | 8 | 1 | 1 | 0 | 0 | 7 | 0 |",
-		"Implements **1 of 8** modelled operations (12.5%).",
-		"<summary>Missing from dispatch (7)</summary>",
+		"aws-sdk-go " + SourceSDKVersion,
+		"1 of   8 implemented ( 12.5%)",
 	} {
 		if !strings.Contains(report, want) {
-			t.Errorf("report does not contain %q:\n%s", want, report)
+			t.Errorf("summary does not contain %q:\n%s", want, report)
 		}
 	}
+	if strings.Contains(report, "<details>") {
+		t.Errorf("summary contains raw HTML the docs site would escape:\n%s", report)
+	}
 }

internal/awsmodel/embed.go

@@ -14,6 +14,7 @@ var modelFiles = map[Service]string{
 	EC2:                    "models/apis/ec2/2016-11-15/api-2.json",
 	ECR:                    "models/apis/ecr/2015-09-21/api-2.json",
 	ECS:                    "models/apis/ecs/2014-11-13/api-2.json",
+	EKS:                    "models/apis/eks/2017-11-01/api-2.json",
 	ElasticLoadBalancingV2: "models/apis/elasticloadbalancingv2/2015-12-01/api-2.json",
 	IAM:                    "models/apis/iam/2010-05-08/api-2.json",
 	RDS:                    "models/apis/rds/2014-10-31/api-2.json",

internal/awsmodel/model.go

@@ -28,6 +28,7 @@ const (
 	EC2                    Service = "ec2"
 	ECR                    Service = "ecr"
 	ECS                    Service = "ecs"
+	EKS                    Service = "eks"
 	ElasticLoadBalancingV2 Service = "elasticloadbalancingv2"
 	IAM                    Service = "iam"
 	RDS                    Service = "rds"

internal/awsmodel/model_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestServices(t *testing.T) {
-	want := []Service{ACM, EC2, ECR, ECS, ElasticLoadBalancingV2, IAM, RDS, S3, STS}
+	want := []Service{ACM, EC2, ECR, ECS, EKS, ElasticLoadBalancingV2, IAM, RDS, S3, STS}
 	if got := Services(); !reflect.DeepEqual(got, want) {
 		t.Fatalf("Services() = %v, want %v", got, want)
 	}
@@ -24,6 +24,7 @@ func TestLoadAllModels(t *testing.T) {
 		EC2:                    {"2016-11-15", 625, 3198},
 		ECR:                    {"2015-09-21", 47, 330},
 		ECS:                    {"2014-11-13", 56, 400},
+		EKS:                    {"2017-11-01", 56, 298},
 		ElasticLoadBalancingV2: {"2015-12-01", 46, 357},
 		IAM:                    {"2010-05-08", 159, 497},
 		RDS:                    {"2014-10-31", 162, 745},

internal/awsmodel/page.go

@@ -0,0 +1,410 @@
+package awsmodel
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+)
+
+// Operation states. Only Implemented reaches a published page; the rest sort
+// the internal report. The symbol carries the status at a glance and the label
+// disambiguates it; the docs site escapes inline SVG, so these are runes.
+const (
+	StatusImplemented    = "✅ Implemented"
+	StatusStub           = "🟡 Stub"
+	StatusNotApplicable  = "⛔ Not applicable"
+	StatusNotImplemented = "❌ Not implemented"
+	StatusOutsideModel   = "🔒 Outside the pinned model"
+)
+
+// Frontmatter length rules enforced by the docs site. See docs/README.md.
+const (
+	seoTitleSuffix = " — Spinifex Docs"
+	seoTitleMin    = 50
+	seoTitleMax    = 60
+	descriptionMin = 150
+	descriptionMax = 160
+)
+
+var slugPattern = regexp.MustCompile(`^[a-z0-9]+(-[a-z0-9]+)*$`)
+
+// OperationStatus is one row of a published coverage table. NoteKey names the
+// shared explanation behind the row's status, where one is declared.
+type OperationStatus struct {
+	Operation string
+	Status    string
+	NoteKey   string
+}
+
+// ImplementedPercent is the share of modelled operations bound to a real
+// handler, or zero for a model with no operations.
+func (c OperationCoverage) ImplementedPercent() float64 {
+	if len(c.Modelled) == 0 {
+		return 0
+	}
+	return 100 * float64(len(c.Implemented)) / float64(len(c.Modelled))
+}
+
+// OperationStatuses returns every modelled operation with its dispatch state,
+// followed by the registered operations the pinned model does not describe,
+// less the internal routes the page declares.
+//
+// The page's NotApplicable declares operations the platform will never serve,
+// mapped to the note key explaining why. Nothing published carries that claim:
+// it separates a permanent absence from candidate work in the internal report.
+// A handler that refuses today may be implemented tomorrow, so a refusal on its
+// own is a gap like any other rather than a "never".
+func (c OperationCoverage) OperationStatuses(page PageMetadata) ([]OperationStatus, error) {
+	registered := toSet(c.Registered)
+	stubbed := toSet(c.Stubbed)
+	unsupported := toSet(c.Unsupported)
+	modelled := toSet(c.Modelled)
+	extra := toSet(c.Extra)
+	notApplicable := page.NotApplicable
+
+	for operation, noteKey := range notApplicable {
+		switch {
+		case noteKey == "":
+			return nil, fmt.Errorf("awsmodel: %s operation %q is declared not applicable with no note", c.Service, operation)
+		case !modelled[operation]:
+			return nil, fmt.Errorf("awsmodel: %s operation %q is declared not applicable but is not in the pinned model", c.Service, operation)
+		case registered[operation] && !unsupported[operation]:
+			return nil, fmt.Errorf("awsmodel: %s operation %q is declared not applicable but is registered to a handler", c.Service, operation)
+		}
+	}
+
+	internal := map[string]bool{}
+	for _, operation := range page.Internal {
+		if !extra[operation] {
+			return nil, fmt.Errorf("awsmodel: %s route %q is declared internal but is not a registered route outside the pinned model", c.Service, operation)
+		}
+		internal[operation] = true
+	}
+
+	statuses := make([]OperationStatus, 0, len(c.Modelled)+len(c.Extra))
+	for _, operation := range c.Modelled {
+		status := StatusNotImplemented
+		switch {
+		case stubbed[operation]:
+			status = StatusStub
+		// A registered handler that refuses serves nothing, so it reads as the
+		// gap it is unless the page declares why it is permanent.
+		case unsupported[operation]:
+			status = StatusNotImplemented
+		case registered[operation]:
+			status = StatusImplemented
+		}
+		if noteKey, ok := notApplicable[operation]; ok {
+			statuses = append(statuses, OperationStatus{Operation: operation, Status: StatusNotApplicable, NoteKey: noteKey})
+			continue
+		}
+		statuses = append(statuses, OperationStatus{Operation: operation, Status: status})
+	}
+	// A route outside the pinned model is still only as real as its handler, so
+	// its dispatch state wins: being absent from the model says nothing about
+	// whether the platform answers it.
+	for _, operation := range c.Extra {
+		if internal[operation] {
+			continue
+		}
+		status := StatusOutsideModel
+		switch {
+		case stubbed[operation]:
+			status = StatusStub
+		case unsupported[operation]:
+			status = StatusNotImplemented
+		}
+		statuses = append(statuses, OperationStatus{Operation: operation, Status: status})
+	}
+	return statuses, nil
+}
+
+// PageMetadata is the checked-in frontmatter for one published coverage page.
+// It is hand-maintained rather than derived, because a description carrying a
+// live count would drift outside the length the docs site enforces.
+type PageMetadata struct {
+	Slug        string   `json:"slug"`
+	Name        string   `json:"name"`
+	Title       string   `json:"title"`
+	SEOTitle    string   `json:"seoTitle"`
+	Description string   `json:"description"`
+	Tags        []string `json:"tags"`
+
+	// ImplementedBy names the component that serves the surface, where it is
+	// not Spinifex itself. Predastore serves S3, and the page says so.
+	ImplementedBy string `json:"implementedBy,omitempty"`
+
+	// Notes are the shared explanations a page footnotes, keyed by a short
+	// name. One note covers every operation that shares a reason.
+	Notes map[string]string `json:"notes,omitempty"`
+
+	// NotApplicable maps an operation the platform will never serve to the key
+	// of the note explaining why. Checked against the model, the notes and the
+	// dispatch tables at render, so a typo or a later implementation fails the
+	// build rather than publishing a claim that has quietly become false.
+	NotApplicable map[string]string `json:"notApplicable,omitempty"`
+
+	// Internal names registered routes that carry platform plumbing rather than
+	// a tenant-callable AWS action, which the page leaves out. A route outside
+	// the pinned model that is not declared here is published, so a real AWS
+	// operation newer than the pin surfaces instead of disappearing quietly.
+	Internal []string `json:"internal,omitempty"`
+}
+
+// PageSet is the metadata for the index page and every service page. BasePath
+// is the site section the pages are published under; the index answers it, so
+// the index alone carries no slug.
+type PageSet struct {
+	Category string                   `json:"category"`
+	BasePath string                   `json:"basePath"`
+	Index    PageMetadata             `json:"index"`
+	Services map[Service]PageMetadata `json:"services"`
+}
+
+// ParsePageSet decodes docs/compatibility/pages.json and checks it describes
+// exactly the services the loader knows about, with publishable frontmatter.
+func ParsePageSet(contents []byte) (PageSet, error) {
+	decoder := json.NewDecoder(bytes.NewReader(contents))
+	decoder.DisallowUnknownFields()
+	var pages PageSet
+	if err := decoder.Decode(&pages); err != nil {
+		return PageSet{}, fmt.Errorf("awsmodel: parse coverage pages: %w", err)
+	}
+	if pages.Category == "" {
+		return PageSet{}, fmt.Errorf("awsmodel: coverage pages: category is empty")
+	}
+	if !strings.HasPrefix(pages.BasePath, "/") {
+		return PageSet{}, fmt.Errorf("awsmodel: coverage pages: basePath %q is not a site path", pages.BasePath)
+	}
+
+	slugs := map[string]bool{}
+	if err := validateIndexPage(pages.Index); err != nil {
+		return PageSet{}, err
+	}
+	// A known service may be left out. S3 is: Predastore serves that surface,
+	// so the gateway dispatch tables hold no honest number to publish for it.
+	for _, service := range Services() {
+		page, ok := pages.Services[service]
+		if !ok {
+			continue
+		}
+		if err := validatePage(string(service), page, slugs); err != nil {
+			return PageSet{}, err
+		}
+	}
+	for service := range pages.Services {
+		if _, ok := modelFiles[service]; !ok {
+			return PageSet{}, fmt.Errorf("awsmodel: coverage pages: unknown service %q", service)
+		}
+	}
+	return pages, nil
+}
+
+// The index answers the section's base path, so it has no slug of its own.
+func validateIndexPage(page PageMetadata) error {
+	if page.Slug != "" {
+		return fmt.Errorf("awsmodel: coverage page index: slug %q is set, but the index answers the base path", page.Slug)
+	}
+	return validateFrontmatter("index", page)
+}
+
+func validatePage(owner string, page PageMetadata, slugs map[string]bool) error {
+	fail := func(format string, args ...any) error {
+		return fmt.Errorf("awsmodel: coverage page %s: %s", owner, fmt.Sprintf(format, args...))
+	}
+	if !slugPattern.MatchString(page.Slug) {
+		return fail("slug %q is not lowercase kebab-case", page.Slug)
+	}
+	if slugs[page.Slug] {
+		return fail("slug %q is used by another page", page.Slug)
+	}
+	slugs[page.Slug] = true
+	return validateFrontmatter(owner, page)
+}
+
+func validateFrontmatter(owner string, page PageMetadata) error {
+	fail := func(format string, args ...any) error {
+		return fmt.Errorf("awsmodel: coverage page %s: %s", owner, fmt.Sprintf(format, args...))
+	}
+	if page.Name == "" || page.Title == "" {
+		return fail("name and title are required")
+	}
+	if len(page.Tags) == 0 {
+		return fail("at least one tag is required")
+	}
+	if !strings.HasSuffix(page.SEOTitle, seoTitleSuffix) {
+		return fail("seoTitle %q does not end with %q", page.SEOTitle, seoTitleSuffix)
+	}
+	if length := utf8.RuneCountInString(page.SEOTitle); length < seoTitleMin || length > seoTitleMax {
+		return fail("seoTitle is %d characters, want %d-%d", length, seoTitleMin, seoTitleMax)
+	}
+	if length := utf8.RuneCountInString(page.Description); length < descriptionMin || length > descriptionMax {
+		return fail("description is %d characters, want %d-%d", length, descriptionMin, descriptionMax)
+	}
+	return nil
+}
+
+// noteOrder returns the note keys in the order they are published. Sorted by
+// key rather than by first use, so adding an operation cannot renumber a note
+// that is already published.
+func noteOrder(notes map[string]string) []string {
+	keys := make([]string, 0, len(notes))
+	for key := range notes {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+// checkNotes rejects a note nothing references and a reference to a note that
+// does not exist. The notes reach the internal report rather than a page, and
+// the check keeps them honest wherever they are read.
+func checkNotes(page PageMetadata) error {
+	referenced := map[string]bool{}
+	for operation, key := range page.NotApplicable {
+		if _, ok := page.Notes[key]; !ok {
+			return fmt.Errorf("awsmodel: coverage page %s: operation %q references undefined note %q", page.Slug, operation, key)
+		}
+		referenced[key] = true
+	}
+
+	for _, key := range noteOrder(page.Notes) {
+		if !referenced[key] {
+			return fmt.Errorf("awsmodel: coverage page %s: note %q is never referenced", page.Slug, key)
+		}
+		if page.Notes[key] == "" {
+			return fmt.Errorf("awsmodel: coverage page %s: note %q is empty", page.Slug, key)
+		}
+	}
+	return nil
+}
+
+// RenderServicePage renders one publishable coverage page. intro is optional
+// hand-written prose emitted above the table; it may not open a new page
+// section, so its headings must be level three or deeper.
+func RenderServicePage(coverage OperationCoverage, pages PageSet, intro string) (string, error) {
+	if err := checkIntro(string(coverage.Service), intro); err != nil {
+		return "", err
+	}
+	page, ok := pages.Services[coverage.Service]
+	if !ok {
+		return "", fmt.Errorf("awsmodel: coverage pages: no entry for service %q", coverage.Service)
+	}
+	// An opaque service has no dispatch table to count, so there is no honest
+	// number to put on a page. It is left out until a real one exists.
+	if coverage.Opaque {
+		return "", fmt.Errorf("awsmodel: %s coverage is not enumerable, so it has no publishable page", coverage.Service)
+	}
+
+	// A page exists to say what the platform serves. A service with nothing
+	// behind it has no such statement to make.
+	if len(coverage.Implemented) == 0 {
+		return "", fmt.Errorf("awsmodel: %s has no implemented operations, so it has no publishable page", coverage.Service)
+	}
+
+	statuses, err := coverage.OperationStatuses(page)
+	if err != nil {
+		return "", err
+	}
+	if err := checkNotes(page); err != nil {
+		return "", err
+	}
+
+	var body strings.Builder
+	writeFrontmatter(&body, page, pages.Category)
+	fmt.Fprintf(&body, "# %s\n\n## Overview\n\n", page.Title)
+
+	noun := "operations"
+	if len(coverage.Implemented) == 1 {
+		noun = "operation"
+	}
+	implementer := page.ImplementedBy
+	if implementer == "" {
+		implementer = "Spinifex"
+	}
+	fmt.Fprintf(&body, "%s implements **%d %s** in the %s `%s` API model.\n\n",
+		implementer, len(coverage.Implemented), noun, page.Name, coverage.APIVersion)
+
+	if intro != "" {
+		body.WriteString(strings.TrimSpace(intro) + "\n\n")
+	}
+
+	writeOperations(&body, statuses)
+	return body.String(), nil
+}
+
+// writeOperations publishes one row per operation the platform implements from
+// the pinned model, and nothing else. There is no status column because every
+// row carries the same status: a gap, a stub, a route the model does not
+// describe and an operation the platform will never offer are all the internal
+// report's to carry.
+func writeOperations(body *strings.Builder, statuses []OperationStatus) {
+	body.WriteString("### Operations\n\n| Operation |\n|---|\n")
+	for _, status := range statuses {
+		if status.Status != StatusImplemented {
+			continue
+		}
+		fmt.Fprintf(body, "| `%s` |\n", status.Operation)
+	}
+}
+
+// RenderIndexPage renders the cross-service summary that links to each page.
+func RenderIndexPage(coverages []OperationCoverage, pages PageSet, intro string) (string, error) {
+	if err := checkIntro("index", intro); err != nil {
+		return "", err
+	}
+
+	var body strings.Builder
+	writeFrontmatter(&body, pages.Index, pages.Category)
+	fmt.Fprintf(&body, "# %s\n\n## Overview\n\n", pages.Index.Title)
+	total := 0
+	for _, coverage := range coverages {
+		if _, ok := pages.Services[coverage.Service]; ok {
+			total += len(coverage.Implemented)
+		}
+	}
+
+	fmt.Fprintf(&body, "The platform serves **%d operations** across the AWS APIs below. Every page names the operations its service implements from the pinned model, generated from the dispatch tables on each build rather than written by hand.\n\n", total)
+	body.WriteString("| Service | Operations |\n|---|---:|\n")
+	for _, coverage := range sortedCoverages(coverages) {
+		page, ok := pages.Services[coverage.Service]
+		if !ok {
+			continue
+		}
+		fmt.Fprintf(&body, "| [%s](%s/%s) | %d |\n",
+			page.Name, pages.BasePath, page.Slug, len(coverage.Implemented))
+	}
+	fmt.Fprintf(&body, "| **Total** | **%d** |\n", total)
+
+	if intro != "" {
+		body.WriteString("\n" + strings.TrimSpace(intro) + "\n")
+	}
+	return body.String(), nil
+}
+
+func writeFrontmatter(body *strings.Builder, page PageMetadata, category string) {
+	body.WriteString("---\n")
+	fmt.Fprintf(body, "title: %s\n", strconv.Quote(page.Title))
+	fmt.Fprintf(body, "seoTitle: %s\n", strconv.Quote(page.SEOTitle))
+	fmt.Fprintf(body, "description: %s\n", strconv.Quote(page.Description))
+	fmt.Fprintf(body, "category: %s\n", strconv.Quote(category))
+	body.WriteString("sections:\n  - overview\ntags:\n")
+	for _, tag := range page.Tags {
+		fmt.Fprintf(body, "  - %s\n", tag)
+	}
+	body.WriteString("---\n\n")
+}
+
+func checkIntro(owner, intro string) error {
+	for line := range strings.SplitSeq(intro, "\n") {
+		if strings.HasPrefix(line, "# ") || strings.HasPrefix(line, "## ") {
+			return fmt.Errorf("awsmodel: %s intro heading %q opens a new page section; use level three or deeper", owner, strings.TrimSpace(line))
+		}
+	}
+	return nil
+}

internal/awsmodel/page_test.go

@@ -0,0 +1,442 @@
+package awsmodel_test
+
+import (
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+
+	. "github.com/mulgadc/spinifex/internal/awsmodel"
+)
+
+func testPageSet(t *testing.T) PageSet {
+	t.Helper()
+	contents, err := os.ReadFile(filepath.Join("..", "..", "docs", "coverage", "pages.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	pages, err := ParsePageSet(contents)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return pages
+}
+
+func TestParsePageSetAcceptsTheCheckedInFile(t *testing.T) {
+	pages := testPageSet(t)
+
+	if len(pages.Services) == 0 {
+		t.Fatal("no service pages are configured")
+	}
+	// Predastore serves S3, so the page says so rather than crediting Spinifex.
+	if implementer := pages.Services[S3].ImplementedBy; implementer != "Predastore" {
+		t.Errorf("S3 is implemented by %q, want Predastore", implementer)
+	}
+}
+
+func TestParsePageSetRejectsUnpublishableMetadata(t *testing.T) {
+	// A well-formed index, so each case below fails only for its own reason.
+	index := `"index":{"name":"All","title":"All","seoTitle":"` + strings.Repeat("x", 39) +
+		` — Spinifex Docs","description":"` + strings.Repeat("x", 155) + `","tags":["aws"]}`
+	head := `{"category":"Coverage","basePath":"/coverage",`
+
+	for name, contents := range map[string]string{
+		"unknown service":  head + index + `,"services":{"notaservice":{}}}`,
+		"missing category": `{"basePath":"/coverage",` + index + `,"services":{}}`,
+		"missing basePath": `{"category":"Coverage",` + index + `,"services":{}}`,
+		"unknown field":    head + `"colour":"blue",` + index + `,"services":{}}`,
+		"index with a slug": head + `"index":{"slug":"aws","name":"All","title":"All","seoTitle":"` +
+			strings.Repeat("x", 39) + ` — Spinifex Docs","description":"` + strings.Repeat("x", 155) + `","tags":["aws"]}}`,
+		"short seoTitle": head + `"index":{"name":"All","title":"All",` +
+			`"seoTitle":"Too short — Spinifex Docs","description":"` + strings.Repeat("x", 155) + `","tags":["aws"]}}`,
+		"missing suffix": head + `"index":{"name":"All","title":"All",` +
+			`"seoTitle":"` + strings.Repeat("x", 55) + `","description":"` + strings.Repeat("x", 155) + `","tags":["aws"]}}`,
+		"short description": head + `"index":{"name":"All","title":"All",` +
+			`"seoTitle":"` + strings.Repeat("x", 39) + ` — Spinifex Docs","description":"too short","tags":["aws"]}}`,
+		"bad service slug": head + index + `,"services":{"sts":{"slug":"STS Coverage","name":"STS","title":"STS",` +
+			`"seoTitle":"` + strings.Repeat("x", 39) + ` — Spinifex Docs","description":"` + strings.Repeat("x", 155) + `","tags":["aws"]}}}`,
+	} {
+		t.Run(name, func(t *testing.T) {
+			if _, err := ParsePageSet([]byte(contents)); err == nil {
+				t.Fatal("ParsePageSet accepted unpublishable metadata")
+			}
+		})
+	}
+}
+
+func TestRenderServicePagePublishesOnlyImplementedOperations(t *testing.T) {
+	pages := testPageSet(t)
+	coverage, err := CompareOperations(STS, DispatchInventory{
+		Registered:  []string{"AssumeRole", "GetCallerIdentity", "GetSessionToken", "PublishInternal"},
+		Stubbed:     []string{"GetSessionToken"},
+		Unsupported: []string{"GetCallerIdentity"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	page, err := RenderServicePage(coverage, pages, "### Scope\n\nHand-written prose.\n")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{
+		`title: "STS API Coverage"`,
+		`category: "Coverage"`,
+		"sections:\n  - overview\n",
+		"# STS API Coverage\n\n## Overview\n",
+		"Spinifex implements **1 operation** in the STS `2011-06-15` API model.",
+		"### Scope\n\nHand-written prose.",
+		"### Operations\n\n| Operation |\n|---|\n| `AssumeRole` |\n",
+	} {
+		if !strings.Contains(page, want) {
+			t.Errorf("page does not contain %q:\n%s", want, page)
+		}
+	}
+	// Everything other than an implemented operation from the pinned model is
+	// the internal report's: a stub answers with an error, a refusal serves
+	// nothing, and a route the model does not describe has no page to sit on.
+	for _, absent := range []string{
+		"GetSessionToken", "GetCallerIdentity", "PublishInternal",
+		StatusStub, StatusOutsideModel, StatusNotImplemented, "Status",
+	} {
+		if strings.Contains(page, absent) {
+			t.Errorf("page publishes %q:\n%s", absent, page)
+		}
+	}
+	if strings.Contains(page, "<details") || strings.Contains(page, "<summary") {
+		t.Errorf("page contains raw HTML the docs site would escape:\n%s", page)
+	}
+}
+
+// A published page states what the platform serves. An operation Spinifex has
+// not implemented is not a row on it, and no count invites reading the page as
+// a score against AWS.
+func TestRenderServicePagePublishesNeitherGapsNorScores(t *testing.T) {
+	pages := testPageSet(t)
+	coverage, err := CompareOperations(STS, DispatchInventory{Registered: []string{"AssumeRole"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	page, err := RenderServicePage(coverage, pages, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(page, StatusNotImplemented) {
+		t.Errorf("page publishes an unimplemented operation:\n%s", page)
+	}
+	if strings.Contains(page, "%") || strings.Contains(page, " of the ") {
+		t.Errorf("page publishes a coverage score:\n%s", page)
+	}
+	for _, operation := range coverage.Missing {
+		if strings.Contains(page, "| `"+operation+"` |") {
+			t.Errorf("page has a row for unimplemented operation %q", operation)
+		}
+	}
+	for _, operation := range coverage.Implemented {
+		if !strings.Contains(page, "| `"+operation+"` |") {
+			t.Errorf("page has no row for implemented operation %q", operation)
+		}
+	}
+}
+
+// A service page says what Spinifex serves, so a service that serves nothing
+// has no page rather than an empty one.
+func TestRenderServicePageRefusesAServiceWithNoImplementation(t *testing.T) {
+	pages := testPageSet(t)
+	coverage, err := CompareOperations(STS, DispatchInventory{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := RenderServicePage(coverage, pages, ""); err == nil {
+		t.Fatal("RenderServicePage published a page for a service with no implemented operations")
+	}
+}
+
+func TestRenderServicePageWithoutIntro(t *testing.T) {
+	pages := testPageSet(t)
+	coverage, err := CompareOperations(STS, DispatchInventory{Registered: []string{"AssumeRole"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	page, err := RenderServicePage(coverage, pages, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(page, "\n\n\n") {
+		t.Errorf("page has a blank run where the intro would be:\n%s", page)
+	}
+}
+
+func TestRenderServicePageRejectsIntroOpeningASection(t *testing.T) {
+	pages := testPageSet(t)
+	coverage, err := CompareOperations(STS, DispatchInventory{Registered: []string{"AssumeRole"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := RenderServicePage(coverage, pages, "## Troubleshooting\n"); err == nil {
+		t.Fatal("RenderServicePage accepted an intro that opens a new page section")
+	}
+}
+
+// A "never" claim is only worth publishing if it cannot quietly become false.
+func TestOperationStatusesRejectsStaleNotApplicableClaims(t *testing.T) {
+	coverage, err := CompareOperations(STS, DispatchInventory{Registered: []string{"AssumeRole"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for name, declared := range map[string]map[string]string{
+		"not in the model": {"NotAnOperation": "a reason"},
+		"no reason":        {"GetFederationToken": ""},
+		"already implemented": {
+			"AssumeRole": "claims the platform will never serve an operation it already serves",
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			if _, err := coverage.OperationStatuses(PageMetadata{NotApplicable: declared}); err == nil {
+				t.Fatal("OperationStatuses accepted a claim that cannot hold")
+			}
+		})
+	}
+}
+
+func TestOperationStatusesCarriesTheNoteKey(t *testing.T) {
+	coverage, err := CompareOperations(STS, DispatchInventory{Registered: []string{"AssumeRole"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	statuses, err := coverage.OperationStatuses(PageMetadata{
+		NotApplicable: map[string]string{"GetFederationToken": "no-broker"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, status := range statuses {
+		if status.Operation != "GetFederationToken" {
+			continue
+		}
+		if status.Status != StatusNotApplicable || status.NoteKey != "no-broker" {
+			t.Fatalf("status = %q, note key = %q", status.Status, status.NoteKey)
+		}
+		return
+	}
+	t.Fatal("declared operation is missing from the table")
+}
+
+// Platform plumbing is left out of a page about the AWS API, but only where it
+// is declared: an undeclared route stays visible, so a real AWS operation newer
+// than the pinned model cannot disappear by being registered.
+func TestOperationStatusesHidesDeclaredInternalRoutes(t *testing.T) {
+	coverage, err := CompareOperations(STS, DispatchInventory{
+		Registered: []string{"AssumeRole", "PublishInternal", "AheadOfThePin"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	statuses, err := coverage.OperationStatuses(PageMetadata{Internal: []string{"PublishInternal"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, status := range statuses {
+		if status.Operation == "PublishInternal" {
+			t.Error("a declared internal route is published")
+		}
+	}
+	if !slices.ContainsFunc(statuses, func(s OperationStatus) bool {
+		return s.Operation == "AheadOfThePin" && s.Status == StatusOutsideModel
+	}) {
+		t.Error("an undeclared route outside the pinned model is not published")
+	}
+}
+
+func TestOperationStatusesRejectsAnInternalRouteThatIsModelled(t *testing.T) {
+	coverage, err := CompareOperations(STS, DispatchInventory{Registered: []string{"AssumeRole"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := coverage.OperationStatuses(PageMetadata{Internal: []string{"AssumeRole"}}); err == nil {
+		t.Fatal("OperationStatuses hid a modelled AWS operation as internal plumbing")
+	}
+}
+
+// A page says what the platform serves. An operation it will never serve is not
+// that, so neither the row nor the reason behind it reaches the page: both are
+// the internal report's to carry.
+func TestRenderServicePagePublishesNoDeclaredAbsence(t *testing.T) {
+	pages := testPageSet(t)
+	iam := pages.Services[IAM]
+	if len(iam.NotApplicable) == 0 {
+		t.Skip("no operations are declared not applicable for IAM")
+	}
+	coverage, err := CompareOperations(IAM, DispatchInventory{Registered: []string{"CreateUser"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	page, err := RenderServicePage(coverage, pages, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(page, StatusNotApplicable) || strings.Contains(page, "### Notes") {
+		t.Errorf("page publishes a declared absence:\n%s", page)
+	}
+	for operation := range iam.NotApplicable {
+		if strings.Contains(page, "`"+operation+"`") {
+			t.Errorf("declared operation %q is published", operation)
+		}
+	}
+	for _, text := range iam.Notes {
+		if strings.Contains(page, text) {
+			t.Errorf("note %q is published", text)
+		}
+	}
+}
+
+// "Not applicable" says the platform will never serve an operation, which a
+// handler refusing it today does not establish. Only the page's declaration
+// publishes that claim; an undeclared refusal is a gap like any other.
+func TestRenderServicePageDoesNotPublishAnUndeclaredRefusalAsPermanent(t *testing.T) {
+	pages := testPageSet(t)
+	coverage, err := CompareOperations(STS, DispatchInventory{
+		Registered:  []string{"AssumeRole", "GetFederationToken"},
+		Unsupported: []string{"GetFederationToken"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	page, err := RenderServicePage(coverage, pages, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(page, "| `GetFederationToken` |") {
+		t.Errorf("an undeclared refusal is published as a permanent claim:\n%s", page)
+	}
+}
+
+func TestRenderServicePageRejectsBrokenNoteReferences(t *testing.T) {
+	pages := testPageSet(t)
+	coverage, err := CompareOperations(STS, DispatchInventory{Registered: []string{"AssumeRole"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	sts := pages.Services[STS]
+
+	for name, mutate := range map[string]func(*PageMetadata){
+		"undefined note": func(p *PageMetadata) {
+			p.NotApplicable = map[string]string{"GetFederationToken": "nonexistent"}
+		},
+		"unreferenced note": func(p *PageMetadata) {
+			p.Notes = map[string]string{"orphan": "Nothing points at this."}
+		},
+		"empty note": func(p *PageMetadata) {
+			p.Notes = map[string]string{"blank": ""}
+			p.NotApplicable = map[string]string{"GetFederationToken": "blank"}
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			broken := sts
+			mutate(&broken)
+			pages.Services[STS] = broken
+			defer func() { pages.Services[STS] = sts }()
+
+			if _, err := RenderServicePage(coverage, pages, ""); err == nil {
+				t.Fatal("RenderServicePage accepted a broken note reference")
+			}
+		})
+	}
+}
+
+// An opaque service has no dispatch table to count, so publishing a page for it
+// would report 0% where the truth is that the surface is served elsewhere.
+func TestRenderServicePageRefusesOpaqueCoverage(t *testing.T) {
+	pages := testPageSet(t)
+	coverage, err := CompareOperations(STS, DispatchInventory{
+		Opaque: true,
+		Note:   "Served elsewhere.",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := RenderServicePage(coverage, pages, ""); err == nil {
+		t.Fatal("RenderServicePage published a page for a service with no enumerable coverage")
+	}
+}
+
+// The total counts the pages it links and nothing else, so a service with no
+// page of its own is not folded into the headline figure.
+func TestRenderIndexPageTotalsOnlyPublishedServices(t *testing.T) {
+	pages := testPageSet(t)
+	sts, err := CompareOperations(STS, DispatchInventory{Registered: []string{"AssumeRole", "GetSessionToken"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	iam, err := CompareOperations(IAM, DispatchInventory{Registered: []string{"CreateUser"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	// A service the page set does not describe, so the index neither links nor
+	// counts it.
+	s3, err := CompareOperations(S3, DispatchInventory{Registered: []string{"ListBuckets"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	delete(pages.Services, S3)
+
+	index, err := RenderIndexPage([]OperationCoverage{sts, iam, s3}, pages, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{
+		"The platform serves **3 operations** across the AWS APIs below.",
+		"| **Total** | **3** |",
+	} {
+		if !strings.Contains(index, want) {
+			t.Errorf("index does not contain %q:\n%s", want, index)
+		}
+	}
+}
+
+func TestRenderIndexPageLinksEveryPublishedService(t *testing.T) {
+	pages := testPageSet(t)
+	coverages := make([]OperationCoverage, 0, len(Services()))
+	for _, service := range Services() {
+		coverage, err := CompareOperations(service, DispatchInventory{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		coverages = append(coverages, coverage)
+	}
+
+	index, err := RenderIndexPage(coverages, pages, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(index, "%") {
+		t.Errorf("index publishes a coverage score:\n%s", index)
+	}
+	if !strings.Contains(index, "| **Total** |") {
+		t.Errorf("index does not total the operations it lists:\n%s", index)
+	}
+	for _, service := range Services() {
+		page, published := pages.Services[service]
+		if !published {
+			if strings.Contains(index, pages.BasePath+"/"+string(service)) {
+				t.Errorf("index links unpublished service %q", service)
+			}
+			continue
+		}
+		if !strings.Contains(index, "["+page.Name+"]("+pages.BasePath+"/"+page.Slug+")") {
+			t.Errorf("index does not link %q at %s/%s", page.Name, pages.BasePath, page.Slug)
+		}
+	}
+}

internal/awsmodel/write.go

@@ -0,0 +1,91 @@
+package awsmodel
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+const (
+	pagesFile    = "pages.json"
+	introFile    = "intro.md"
+	pageFile     = "README.md"
+	internalFile = "internal-report.md"
+)
+
+// WritePages renders the publishable coverage pages into outputDir, which must
+// already hold pages.json. A service without page metadata is skipped, and any
+// README.md already there is overwritten: only pages.json and the intro.md
+// files are hand-maintained.
+func WritePages(outputDir string, coverages []OperationCoverage) error {
+	contents, err := os.ReadFile(filepath.Join(outputDir, pagesFile))
+	if err != nil {
+		return fmt.Errorf("awsmodel: read coverage page metadata: %w", err)
+	}
+	pages, err := ParsePageSet(contents)
+	if err != nil {
+		return err
+	}
+
+	for _, coverage := range coverages {
+		page, ok := pages.Services[coverage.Service]
+		if !ok {
+			continue
+		}
+		intro, err := readIntro(outputDir, page.Slug)
+		if err != nil {
+			return err
+		}
+		body, err := RenderServicePage(coverage, pages, intro)
+		if err != nil {
+			return err
+		}
+		if err := writePage(outputDir, page.Slug, body); err != nil {
+			return err
+		}
+	}
+
+	intro, err := readIntro(outputDir, pages.Index.Slug)
+	if err != nil {
+		return err
+	}
+	body, err := RenderIndexPage(coverages, pages, intro)
+	if err != nil {
+		return err
+	}
+	if err := writePage(outputDir, pages.Index.Slug, body); err != nil {
+		return err
+	}
+
+	report := RenderInternalReport(coverages, pages)
+	if err := os.WriteFile(filepath.Join(outputDir, internalFile), []byte(report), 0o600); err != nil {
+		return fmt.Errorf("awsmodel: write internal coverage report: %w", err)
+	}
+	return nil
+}
+
+// readIntro returns the hand-written prose for a page, or "" where the page has
+// none. An intro is optional by design: most services have nothing to add.
+func readIntro(outputDir, slug string) (string, error) {
+	contents, err := os.ReadFile(filepath.Join(outputDir, slug, introFile))
+	if errors.Is(err, fs.ErrNotExist) {
+		return "", nil
+	}
+	if err != nil {
+		return "", fmt.Errorf("awsmodel: read %s intro: %w", slug, err)
+	}
+	return string(contents), nil
+}
+
+func writePage(outputDir, slug, body string) error {
+	dir := filepath.Join(outputDir, slug)
+	if err := os.MkdirAll(dir, 0o750); err != nil {
+		return fmt.Errorf("awsmodel: create %s page directory: %w", slug, err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, pageFile), []byte(body), 0o600); err != nil {
+		return fmt.Errorf("awsmodel: write %s page: %w", slug, err)
+	}
+	return nil
+}

internal/awsmodel/write_test.go

@@ -0,0 +1,166 @@
+package awsmodel_test
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	. "github.com/mulgadc/spinifex/internal/awsmodel"
+)
+
+// stagePages copies the checked-in metadata into a temp directory so a write
+// test cannot touch docs/coverage.
+func stagePages(t *testing.T) string {
+	t.Helper()
+	dir := t.TempDir()
+	contents, err := os.ReadFile(filepath.Join("..", "..", "docs", "coverage", "pages.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "pages.json"), contents, 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return dir
+}
+
+func stsCoverage(t *testing.T) OperationCoverage {
+	t.Helper()
+	coverage, err := CompareOperations(STS, DispatchInventory{Registered: []string{"AssumeRole"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return coverage
+}
+
+func TestWritePagesWritesAServicePageAndTheIndex(t *testing.T) {
+	dir := stagePages(t)
+
+	if err := WritePages(dir, []OperationCoverage{stsCoverage(t)}); err != nil {
+		t.Fatal(err)
+	}
+
+	service, err := os.ReadFile(filepath.Join(dir, "sts", "README.md"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(service), "| `AssumeRole` |") {
+		t.Errorf("service page is missing its operation table:\n%s", service)
+	}
+	index, err := os.ReadFile(filepath.Join(dir, "README.md"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(index), "[STS](/coverage/sts)") {
+		t.Errorf("index does not link the service page:\n%s", index)
+	}
+}
+
+// The gaps the pages leave out are still written down, in a file the docs site
+// is not configured to render.
+func TestWritePagesWritesTheInternalReport(t *testing.T) {
+	dir := stagePages(t)
+	coverage := stsCoverage(t)
+
+	if err := WritePages(dir, []OperationCoverage{coverage}); err != nil {
+		t.Fatal(err)
+	}
+
+	report, err := os.ReadFile(filepath.Join(dir, "internal-report.md"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, operation := range coverage.Missing {
+		if !strings.Contains(string(report), "- `"+operation+"`") {
+			t.Errorf("internal report omits unimplemented operation %q:\n%s", operation, report)
+		}
+	}
+	if !strings.Contains(string(report), "% ") && !strings.Contains(string(report), "%)") {
+		t.Errorf("internal report carries no coverage figure:\n%s", report)
+	}
+}
+
+func TestWritePagesIncludesAnIntroAndOverwritesTheGeneratedPage(t *testing.T) {
+	dir := stagePages(t)
+	pageDir := filepath.Join(dir, "sts")
+	if err := os.MkdirAll(pageDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(pageDir, "intro.md"), []byte("### Scope\n\nCarried prose.\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(pageDir, "README.md"), []byte("stale hand edit"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := WritePages(dir, []OperationCoverage{stsCoverage(t)}); err != nil {
+		t.Fatal(err)
+	}
+
+	page, err := os.ReadFile(filepath.Join(pageDir, "README.md"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(string(page), "stale hand edit") {
+		t.Error("a hand edit to a generated page survived regeneration")
+	}
+	if !strings.Contains(string(page), "### Scope\n\nCarried prose.") {
+		t.Errorf("page does not carry its intro:\n%s", page)
+	}
+}
+
+// A service the page set does not describe is skipped rather than crashing the
+// run or publishing a page with no metadata on it.
+func TestWritePagesSkipsAServiceWithNoMetadata(t *testing.T) {
+	dir := stageWithout(t, S3)
+	coverage, err := CompareOperations(S3, DispatchInventory{Registered: []string{"ListBuckets"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := WritePages(dir, []OperationCoverage{coverage}); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := os.Stat(filepath.Join(dir, "s3")); !os.IsNotExist(err) {
+		t.Errorf("a page was written for a service with no metadata: %v", err)
+	}
+}
+
+// stageWithout stages the checked-in metadata with one service dropped, for the
+// cases about a service the page set does not describe.
+func stageWithout(t *testing.T, service Service) string {
+	t.Helper()
+	dir := stagePages(t)
+	path := filepath.Join(dir, "pages.json")
+	contents, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var pages map[string]any
+	if err := json.Unmarshal(contents, &pages); err != nil {
+		t.Fatal(err)
+	}
+	services, ok := pages["services"].(map[string]any)
+	if !ok {
+		t.Fatalf("staged metadata has no services: %s", contents)
+	}
+	delete(services, string(service))
+
+	reduced, err := json.Marshal(pages)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(path, reduced, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	return dir
+}
+
+func TestWritePagesReportsMissingMetadata(t *testing.T) {
+	if err := WritePages(t.TempDir(), nil); err == nil {
+		t.Fatal("WritePages accepted a directory with no pages.json")
+	}
+}

scripts/install-node.sh

@@ -32,6 +32,9 @@
 #   --identity FILE          SSH private key
 #   --hosts-file FILE        One host per line; blank lines and # comments ignored
 #   --token-ttl D            Join token validity (default: 30m)
+#   --ipsec on|off           Encrypt Geneve between nodes with OVN IPsec
+#                            (default: on). Joining nodes take it from the first
+#                            host; off is for nodes on a trusted private link.
 #   --wipe                   Reset every host to its pre-install state first.
 #                            Destroys all data on ALL hosts, including the first.
 #                            Confirmed separately from --yes; unattended runs
@@ -77,6 +80,7 @@ AZ=""
 EMAIL=""
 PORT=4432
 TOKEN_TTL="30m"
+IPSEC="on"
 RUN_SMOKE=false
 WIPE=false
 MANAGE_FIREWALL=true
@@ -96,6 +100,12 @@ FIREWALL_TIMEOUT=180
 # How long to wait for every ovn-controller to register its chassis.
 CHASSIS_TIMEOUT=120
 
+# How many times to compare the nodes' JetStream digests, and the pause between.
+# Digests are copied from live stores seconds apart, so one round can catch a
+# write that has reached some replicas and not yet the others.
+KV_ROUNDS=5
+KV_RETRY_SLEEP=15
+
 FIREWALL_HELPER="/usr/local/lib/spinifex/spinifex-firewall-apply"
 FIREWALL_PEERS="/etc/spinifex/firewall/peers.nft"
 
@@ -114,6 +124,7 @@ while [[ $# -gt 0 ]]; do
         --user)                SSH_USER="$2"; shift 2 ;;
         --identity)            IDENTITY="$2"; shift 2 ;;
         --token-ttl)           TOKEN_TTL="$2"; shift 2 ;;
+        --ipsec)               IPSEC="$2"; shift 2 ;;
         --hosts-file)
             while IFS= read -r line; do
                 line="${line%%#*}"
@@ -127,7 +138,7 @@ while [[ $# -gt 0 ]]; do
         --no-firewall)  MANAGE_FIREWALL=false; shift ;;
         --yes|-y)   ASSUME_YES=true; shift ;;
         --dry-run)  DRY_RUN=true; shift ;;
-        -h|--help)  sed -n '2,58p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
+        -h|--help)  sed -n '2,61p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
         -*)         echo "ERROR: unknown option: $1" >&2; exit 2 ;;
         *)          HOSTS+=("$1"); shift ;;
     esac
@@ -161,6 +172,12 @@ for required in "--external-pool:$EXT_POOL" "--external-gateway:$EXT_GATEWAY" "-
          --external-gateway 216.218.163.97 --external-prefix-len 27"
 done
 
+case "$IPSEC" in
+    on)  IPSEC_ENABLED=true ;;
+    off) IPSEC_ENABLED=false ;;
+    *)   fail "--ipsec must be on or off, got: $IPSEC" ;;
+esac
+
 [[ "$EXT_POOL" =~ ^[0-9.]+-[0-9.]+$ ]] || fail "--external-pool must be START-END, got: $EXT_POOL"
 [[ "$EXT_PREFIX" =~ ^[0-9]+$ ]] && [ "$EXT_PREFIX" -ge 1 ] && [ "$EXT_PREFIX" -le 32 ] ||
     fail "--external-prefix-len must be 1-32, got: $EXT_PREFIX"
@@ -368,6 +385,22 @@ for host in "${HOSTS[@]}"; do
     on "$host" "sudo systemctl stop spinifex.target" || fail "$host: could not stop spinifex.target"
 done
 
+# Init and join remove each node's JetStream store and refuse while NATS still
+# runs. A stopped target is not proof of that: its storage units can outlive it.
+# Matched on the service argv so the ssh shell running this cannot match itself.
+for host in "${HOSTS[@]}"; do
+    $DRY_RUN && { echo "  [dry-run] $host: wait for every 'spx service' process to exit"; continue; }
+    on "$host" "for u in spinifex-predastore spinifex-viperblock spinifex-nats; do
+            systemctl is-active --quiet \$u && sudo systemctl stop \$u
+        done
+        for _ in \$(seq 1 45); do
+            pgrep -f '^(/usr/local/bin/)?spx service ' >/dev/null || exit 0
+            sleep 2
+        done
+        pgrep -af '^(/usr/local/bin/)?spx service ' >&2
+        exit 1" || fail "$host: spx services still running 90s after stopping spinifex.target"
+done
+
 # --- Disarm the host firewall ----------------------------------------------
 #
 # The policy lives in the kernel and outlives the services, so stopping the
@@ -379,6 +412,32 @@ done
 declare -a WAS_ARMED
 for i in $(seq 0 $((N - 1))); do WAS_ARMED[i]=false; done
 
+# Anything that aborts between here and the re-arm below leaves every node it
+# disarmed open on a public address, and `fail` is called from a dozen places in
+# between. The 2026-09-11 rebuild ended with all four nodes unarmed for exactly
+# that reason: one node failed to restore its own policy, verification failed,
+# and the three that had been disarmed for formation were never put back.
+DISARMED_HOSTS=()
+restore_firewall_on_abort() {
+    local status=$?
+    if [ "$status" -eq 0 ] || [ ${#DISARMED_HOSTS[@]} -eq 0 ]; then
+        return "$status"
+    fi
+    echo ""
+    log "aborting — re-arming the ${#DISARMED_HOSTS[@]} host(s) disarmed for formation"
+    local host
+    for host in "${DISARMED_HOSTS[@]}"; do
+        if on "$host" "sudo systemctl restart spinifex-daemon"; then
+            log "  $host re-armed"
+        else
+            log "  $host: COULD NOT RE-ARM — it is open on its public addresses."
+            log "    Repair: ssh $SSH_USER@$host 'sudo systemctl restart spinifex-daemon'"
+        fi
+    done
+    return "$status"
+}
+trap restore_firewall_on_abort EXIT
+
 if $MANAGE_FIREWALL; then
     log "disarming the host firewall while the cluster forms"
     for i in $(seq 0 $((N - 1))); do
@@ -389,6 +448,7 @@ if $MANAGE_FIREWALL; then
         on "$host" "if [ -x $FIREWALL_HELPER ]; then sudo $FIREWALL_HELPER disable; fi" ||
             fail "$host: could not disarm the host firewall"
         if ${WAS_ARMED[$i]}; then
+            DISARMED_HOSTS+=("$host")
             log "  $host disarmed"
         else
             log "  $host had no policy loaded"
@@ -499,9 +559,12 @@ done
 # --advertise must be explicit whenever --bind is pinned. resolveAdvertiseIP
 # echoes a non-wildcard --bind straight back, so binding services to the lan
 # plane without this publishes the internal address as the node's public dial
-# target.
+# target. --discard-jetstream is init's default for a formation, spelled out
+# because a node that kept its own store would never converge with the rest.
 init_args=(
     --force
+    --discard-jetstream
+    --ipsec="$IPSEC_ENABLED"
     --node "${NODE_NAMES[0]}"
     --nodes "$N"
     --bind "${LAN_IPS[0]}"
@@ -550,8 +613,8 @@ else
 fi
 
 # --force: each joining node arrived from the ISO already initialized, with its
-# own CA and master key. Joining replaces them with the init node's, and
-# `spx admin join` refuses to do that silently.
+# own CA, master key and JetStream store. Joining replaces the keys with the init
+# node's and removes the store, and `spx admin join` refuses to do that silently.
 #
 # Every join must run concurrently. `spx admin join` registers and then blocks
 # until the formation server has all N nodes, so joining one at a time deadlocks
@@ -688,6 +751,26 @@ done
 
 log "  OVN Southbound: $chassis chassis registered"
 
+# Joiners copy the IPsec setting from the init node's formation response, so a
+# node that disagrees here got its config from somewhere else.
+ipsec_mismatch=""
+for i in $(seq 0 $((N - 1))); do
+    got=$(out "${HOSTS[$i]}" "sudo grep -oP '^ipsec_enabled\s*=\s*\K\w+' /etc/spinifex/spinifex.toml")
+    got="${got//[$'\r\n']/}"
+    [ "$got" = "$IPSEC_ENABLED" ] || ipsec_mismatch+=" ${HOSTS[$i]}=${got:-unset}"
+done
+[ -z "$ipsec_mismatch" ] ||
+    fail "ipsec_enabled should be $IPSEC_ENABLED on every node, found:$ipsec_mismatch"
+
+if ! $IPSEC_ENABLED; then
+    nb_remotes=""
+    for i in $(seq 0 $((DB_NODES - 1))); do nb_remotes+="${nb_remotes:+,}tcp:${LAN_IPS[$i]}:6641"; done
+    nb_ipsec=$(out "${HOSTS[0]}" "sudo ovn-nbctl --db=$nb_remotes --no-leader-only get NB_Global . ipsec")
+    [ "${nb_ipsec//[$'\r\n']/}" = "false" ] ||
+        fail "--ipsec off, but NB_Global.ipsec is '${nb_ipsec//[$'\r\n']/}' — Geneve would demand encryption no node provides"
+fi
+log "  IPsec: $IPSEC on every node"
+
 # --- Re-arm the host firewall ----------------------------------------------
 #
 # The daemon derives the peer sets from cluster membership, so starting the
@@ -722,8 +805,17 @@ if $MANAGE_FIREWALL; then
 
     for i in $(seq 0 $((N - 1))); do
         host="${HOSTS[$i]}"
+        # Said at this volume because it used to read as a routine note and is
+        # not one: this host is joining a cluster on public addresses with
+        # nothing in front of it. A node that arrives unarmed is usually a reset
+        # that failed after deleting the policy, not a deliberate choice.
         if ! ${WAS_ARMED[$i]}; then
-            log "  $host: firewall was not armed before forming, leaving it off"
+            log "  $host: WARNING — no firewall policy before forming, leaving it OFF."
+            log "    This node is open on its public addresses. If that was not"
+            log "    deliberate, arm it with:"
+            log "      ssh $SSH_USER@$host 'sudo env SETUP_STAGES=firewall \\"
+            log "        /usr/local/share/spinifex/setup.sh --firewall on'"
+            log "      ssh $SSH_USER@$host 'sudo systemctl restart spinifex-daemon'"
             continue
         fi
 
@@ -763,12 +855,58 @@ if $MANAGE_FIREWALL; then
 
         log "  $host armed: $(grep -c . <<<"$loaded") peers, $encap_count tunnel endpoints"
     done
+
+    # Every disarmed host has been put back and verified, so a later failure is
+    # not a firewall failure and the abort handler has nothing left to repair.
+    DISARMED_HOSTS=()
 else
     echo ""
     log "host firewall: left alone (--no-firewall). If these nodes arrived armed they"
     log "               are still scoped to themselves and will drop cluster traffic."
 fi
 
+# --- Verify the JetStream replicas agree ------------------------------------
+#
+# Every node discarded its own store on the way in, so each stream's replicas
+# must hold the same content. A formation that adopted a stale store still
+# reports Ready nodes; only reading the replicas themselves shows it.
+
+echo ""
+log "checking that every node holds the same JetStream state"
+kv_rc=1
+kv_report=""
+for round in $(seq 1 "$KV_ROUNDS"); do
+    digests=""
+    for host in "${HOSTS[@]}"; do
+        d=$(out "$host" "sudo spx admin kv digest --json --seqs")
+        [ -n "$d" ] || fail "$host: spx admin kv digest produced no output — run it there to see why"
+        digests+="$d"$'\n'
+    done
+    kv_rc=0
+    kv_report=$(ssh "${SSH_OPTS[@]}" "$SSH_USER@${HOSTS[0]}" "sudo spx admin kv compare -" <<<"$digests" 2>&1) ||
+        kv_rc=$?
+    [ "$kv_rc" -eq 1 ] || break
+    [ "$round" -lt "$KV_ROUNDS" ] || break
+    log "  round $round of $KV_ROUNDS: replicas differ, checking again in ${KV_RETRY_SLEEP}s"
+    sleep "$KV_RETRY_SLEEP"
+done
+
+case "$kv_rc" in
+    0)
+        log "  $(tail -1 <<<"$kv_report")"
+        ;;
+    1)
+        grep -E '^DIVERGED|^    !|streams:' <<<"$kv_report" | sed 's/^/  /' >&2
+        fail "JetStream replicas still disagree after $KV_ROUNDS rounds.
+       The cluster has adopted inconsistent state and must not be put in service.
+       Reset these nodes and form the cluster again."
+        ;;
+    *)
+        sed 's/^/  | /' <<<"$kv_report" >&2
+        fail "could not compare the JetStream digests (spx admin kv compare exited $kv_rc)"
+        ;;
+esac
+
 # The pool is the one thing the operator supplied by hand and the one thing
 # nothing else validates, so print what actually landed. dns_servers in
 # particular is detected rather than passed, and is wrong often enough to be

scripts/node-reset.sh

@@ -103,17 +103,69 @@ fi
 log "stopping services"
 run sudo systemctl stop spinifex.target 2>/dev/null || true
 run sudo systemctl reset-failed 'spinifex-*' 2>/dev/null || true
-run sudo pkill -x qemu-system-x86_64 2>/dev/null || true
-run sudo pkill -x qemu-system-aarch64 2>/dev/null || true
+
+# Stopping the target is not the same as its services being stopped.
+# spinifex-shutdown.service is ordered After= the storage services so its drain
+# runs while they are still up, which queues their stop jobs behind its
+# ExecStop, and `systemctl stop` returns as soon as the target itself is
+# inactive. NATS surviving that is what lets JetStream rewrite its index after
+# the wipe and before the verification, failing the reset on a file the script
+# had already deleted. update-nodes.sh carries the same wait.
+if ! $DRY_RUN; then
+    for unit in spinifex-predastore spinifex-viperblock spinifex-nats; do
+        if systemctl is-active --quiet "$unit"; then
+            sudo systemctl stop "$unit" || true
+        fi
+    done
+    elapsed=0
+    while [ -n "$(pgrep -x spx || true)" ]; do
+        if [ "$elapsed" -ge 120 ]; then
+            echo "ERROR: spx still running after 120s:" >&2
+            pgrep -ax spx >&2 || true
+            echo "  Wiping under a live service races JetStream, which rewrites" >&2
+            echo "  its index after the rm. Stop them manually and re-run." >&2
+            exit 1
+        fi
+        sleep 2
+        elapsed=$((elapsed + 2))
+    done
+fi
+
+# `pgrep -x` and `pkill -x` match against comm, which the kernel truncates to 15
+# characters, so `qemu-system-x86_64` is stored as `qemu-system-x86` and an -x
+# match on the full name can never fire. Match the prefix instead: it is 11
+# characters, well under the limit, and covers both x86_64 and aarch64. This is
+# the same rule qemuProcessPrefix applies in spinifex/vm/orphan_scan.go.
+#
+# Deliberately not -f. That matches the whole command line, where argv[0] may
+# carry a path, so an anchored pattern misses the process it was written for.
+#
+# Scoped by owner, because a host carrying libvirt guests alongside Spinifex
+# ones has both under the same comm and an unscoped kill would take the wrong
+# set down. The daemon runs guests as its own service user on an installed node;
+# a dev install runs them as whoever invoked spx.
+QEMU_PGREP=('^qemu-system')
+for qemu_user in spinifex-daemon spinifex "${SUDO_USER:-$(id -un)}"; do
+    if id -u "$qemu_user" >/dev/null 2>&1; then
+        QEMU_PGREP=(-u "$qemu_user" "${QEMU_PGREP[@]}")
+        break
+    fi
+done
+qemu_pids() { sudo pgrep "${QEMU_PGREP[@]}" 2>/dev/null || true; }
+
+if [ -n "$(qemu_pids)" ]; then
+    log "signalling QEMU guests owned by $qemu_user"
+    run sudo pkill "${QEMU_PGREP[@]}" 2>/dev/null || true
+fi
 
 # Viperblock state must not be torn out from under a live guest, so wait for
 # QEMU to actually exit rather than assuming the signal was enough.
 if ! $DRY_RUN; then
     elapsed=0
-    while pgrep -x 'qemu-system-x86_64|qemu-system-aarch64' >/dev/null 2>&1; do
+    while [ -n "$(qemu_pids)" ]; do
         if [ "$elapsed" -ge 30 ]; then
             echo "ERROR: QEMU still running after 30s:" >&2
-            pgrep -af 'qemu-system-' >&2 || true
+            sudo pgrep -a "${QEMU_PGREP[@]}" >&2 || true
             echo "  Kill them manually and re-run." >&2
             exit 1
         fi
@@ -150,9 +202,28 @@ run sudo rm -f /etc/openvswitch/system-id.conf
 # stays open forever. Read the arming mode now and regenerate the stage after
 # the wipe. peers.nft is the one file here that really is cluster state, and
 # regenerating does not bring it back.
+#
+# Reading it from $ETC_DIR alone is not enough, because a run that fails between
+# the wipe and the restore has already deleted the file. The retry then reads
+# the absence as "this node was never armed", skips the restore, and
+# install-node.sh honours that and forms the cluster with an open node on a
+# public address. Stash the mode outside every wiped tree so the retry can still
+# find it, and keep it there until the policy is back.
+FIREWALL_STASH=/var/lib/spinifex-reset/firewall-mode
+
 FIREWALL_MODE=""
 if [ -r "$ETC_DIR/firewall/mode" ]; then
     FIREWALL_MODE=$(sudo cat "$ETC_DIR/firewall/mode" 2>/dev/null | tr -d '[:space:]')
+elif [ -r "$FIREWALL_STASH" ]; then
+    FIREWALL_MODE=$(sudo cat "$FIREWALL_STASH" 2>/dev/null | tr -d '[:space:]')
+    log "recovered the firewall mode from an interrupted reset: $FIREWALL_MODE"
+fi
+
+# Absent from both means this node genuinely has no policy — a fresh install or
+# a dev tree — and arming one here would be inventing a decision nobody made.
+if [ -n "$FIREWALL_MODE" ] && ! $DRY_RUN; then
+    sudo mkdir -p "$(dirname "$FIREWALL_STASH")"
+    printf '%s\n' "$FIREWALL_MODE" | sudo tee "$FIREWALL_STASH" >/dev/null
 fi
 
 # Named in the transcript because the risk runs the other way too: an allowlist
@@ -217,6 +288,18 @@ if ! $KEEP_DATA && [ -d "$DATA_DIR/predastore/cluster" ]; then
     run sudo rm -rf "$DATA_DIR/predastore/cluster"
 fi
 
+# JetStream needs the same treatment for the same reason, and the consequence of
+# skipping it is worse. Emptying the files leaves a $G/streams/KV_<name> directory
+# per stream the old cluster held; NATS enumerates those on start and reports them
+# as HA assets it must restore, so a four-node meta group sits at size=4 with no
+# leader, elected over hundreds of assets that have no data behind them. Every
+# service then fails to reach JetStream and the daemon aborts its bootstrap.
+# The directory is recreated on start, so its ownership is not installed state.
+if ! $KEEP_DATA && [ -d "$DATA_DIR/nats/jetstream" ]; then
+    log "removing the JetStream store"
+    run sudo rm -rf "$DATA_DIR/nats/jetstream"
+fi
+
 # Directories may survive — a mountpoint cannot be removed, and neither can the
 # path leading down to one. Files may not: a surviving file is state that would
 # carry into the new cluster, which is the exact failure this script exists to
@@ -237,13 +320,31 @@ fi
 
 # Put the firewall policy back. Only this stage runs, so the node stays on the
 # build it was already running — nothing is downloaded or reinstalled.
+#
+# A node that arrived armed and leaves unarmed is a failure, not a warning. It
+# sits on a public address with nothing in front of it, and the only signal used
+# to be a log line that reads as cosmetic. Exit non-zero so the caller stops
+# rather than forming a cluster around it, and leave the stash in place so the
+# retry still knows what mode to restore.
 SETUP_SH=/usr/local/share/spinifex/setup.sh
-if [ -n "$FIREWALL_MODE" ] && [ -x "$SETUP_SH" ]; then
+if [ -n "$FIREWALL_MODE" ]; then
+    if [ ! -x "$SETUP_SH" ]; then
+        echo "ERROR: $SETUP_SH is missing, cannot restore the firewall policy." >&2
+        echo "  This node was armed ($FIREWALL_MODE) and is now open. Reinstall it" >&2
+        echo "  or arm it by hand before forming a cluster." >&2
+        $DRY_RUN || exit 1
+    fi
     log "restoring the host firewall policy (mode: $FIREWALL_MODE)"
-    run sudo env SETUP_STAGES=firewall "$SETUP_SH" --firewall "$FIREWALL_MODE" ||
-        log "  WARNING: could not restore the firewall policy — this node will not re-arm"
-elif [ -n "$FIREWALL_MODE" ]; then
-    log "  WARNING: $SETUP_SH is missing, cannot restore the firewall policy"
+    if run sudo env SETUP_STAGES=firewall "$SETUP_SH" --firewall "$FIREWALL_MODE"; then
+        run sudo rm -f "$FIREWALL_STASH"
+    else
+        echo "ERROR: could not restore the firewall policy." >&2
+        echo "  This node was armed ($FIREWALL_MODE) and is now open on its public" >&2
+        echo "  addresses. Repair with:" >&2
+        echo "    sudo env SETUP_STAGES=firewall $SETUP_SH --firewall $FIREWALL_MODE" >&2
+        echo "    sudo systemctl restart spinifex-daemon" >&2
+        $DRY_RUN || exit 1
+    fi
 fi
 
 if [ ! -d /etc/spinifex ]; then

spinifex/awserrors/awserrors_test.go

@@ -115,6 +115,26 @@ func TestResolveErrorDetail_ErrorfMessageWins(t *testing.T) {
 	}
 }
 
+// TestResolveErrorDetail_ErrorfMessageSurvivesOuterWrapper is the core
+// anti-leak assertion: a call site's own %w wrapper, added purely for its
+// logs, must not replace or blend with the curated message Errorf recorded.
+func TestResolveErrorDetail_ErrorfMessageSurvivesOuterWrapper(t *testing.T) {
+	inner := Errorf(ErrorInsufficientAddressCapacity, "no capacity for m5.large on any eligible node")
+	outer := fmt.Errorf("dial predastore 10.0.1.5:8443: %w", inner)
+
+	code, message, ok := ResolveErrorDetail(outer)
+	if !ok || code != ErrorInsufficientAddressCapacity {
+		t.Fatalf("ResolveErrorDetail() code = (%q, %v), want (%q, true)", code, ok, ErrorInsufficientAddressCapacity)
+	}
+	want := "no capacity for m5.large on any eligible node"
+	if message != want {
+		t.Errorf("ResolveErrorDetail() message = %q, want %q", message, want)
+	}
+	if strings.Contains(message, "dial predastore") {
+		t.Errorf("ResolveErrorDetail() message %q leaked the outer wrapper's context", message)
+	}
+}
+
 // TestResolveErrorDetail_PlainWrapCarriesNoMessage is the compatibility case:
 // a generic %w wrapper not produced by Errorf must not be mistaken for a
 // client-facing message, even though it sits right next to the bare code.

spinifex/daemon/cluster_ready_test.go

@@ -62,7 +62,10 @@ func closedPort(t *testing.T) string {
 // TestWaitForClusterReady_ReadyImmediately asserts the loop returns on the first
 // pass without sleeping when both dependencies are already reachable.
 func TestWaitForClusterReady_ReadyImmediately(t *testing.T) {
-	_, nc := testutil.StartTestNATS(t)
+	// JetStream, not a bare NATS server: viperblock readiness means the KV the
+	// mount claims its volume lease from can answer, and a connection to a
+	// server whose JetStream is still starting reports connected regardless.
+	_, nc, _ := testutil.StartTestJetStream(t)
 
 	sleeps := 0
 	stubClusterReadySleep(t, func(time.Duration) { sleeps++ })
@@ -79,7 +82,7 @@ func TestWaitForClusterReady_ReadyImmediately(t *testing.T) {
 // while predastore is unreachable and returns once it starts serving, rather
 // than latching the first not-ready result.
 func TestWaitForClusterReady_PredastoreBecomesReady(t *testing.T) {
-	_, nc := testutil.StartTestNATS(t)
+	_, nc, _ := testutil.StartTestJetStream(t)
 
 	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		w.WriteHeader(http.StatusForbidden)
@@ -108,6 +111,31 @@ func TestWaitForClusterReady_PredastoreBecomesReady(t *testing.T) {
 	assert.NotContains(t, logs.String(), "Cluster readiness timeout")
 }
 
+// TestCheckViperblockReady_ConnectedButJetStreamNotServing is the regression
+// guard for a guest lost across a cluster update. A TCP connection to NATS says
+// nothing about whether JetStream can serve, so readiness used to pass while the
+// volume-lease KV was still unavailable, recovery ran into a lease timeout, and
+// the instance was parked in error permanently.
+func TestCheckViperblockReady_ConnectedButJetStreamNotServing(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	require.True(t, nc.IsConnected(), "the connection itself must be up, or this proves nothing")
+
+	d := &Daemon{config: &config.Config{}, natsConn: nc}
+
+	assert.False(t, d.checkViperblockReady(),
+		"connected to a server with no JetStream is not ready: the lease KV cannot answer")
+}
+
+// TestCheckViperblockReady_JetStreamServing is the other half: once JetStream
+// answers, readiness must pass rather than holding startup open.
+func TestCheckViperblockReady_JetStreamServing(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+
+	d := &Daemon{config: &config.Config{}, natsConn: nc}
+
+	assert.True(t, d.checkViperblockReady())
+}
+
 // TestWaitForClusterReady_TimesOut asserts the deadline branch: with viperblock
 // permanently unreachable the loop gives up at maxWait, logs the warning and
 // returns so recovery proceeds anyway, instead of blocking startup forever.

spinifex/daemon/daemon.go

@@ -174,6 +174,10 @@ type Daemon struct {
 	cancel                context.CancelFunc
 	shutdownWg            sync.WaitGroup
 
+	// recoveryRetryWg tracks the post-restore retry loop, which writes the
+	// local state file when it finishes. Tests wait on it before cleanup.
+	recoveryRetryWg sync.WaitGroup
+
 	// systemDispatchWg tracks in-flight system.LaunchInstance / TerminateInstance
 	// handlers. Each runs in its own goroutine so a slow VM boot never blocks
 	// the NATS subscription. Used by tests to await dispatch completion.
@@ -970,10 +974,13 @@ func (d *Daemon) subscribeAll() error {
 		{"ec2.DeleteSecurityGroup", handleNATSRequest(d.node, d.vpcService.DeleteSecurityGroup), "spinifex-workers"},
 		{"ec2.DescribeSecurityGroups", handleNATSRequest(d.node, d.vpcService.DescribeSecurityGroups), "spinifex-workers"},
 		{"ec2.DescribeSecurityGroupRules", handleNATSRequest(d.node, d.vpcService.DescribeSecurityGroupRules), "spinifex-workers"},
+		{"ec2.GetSecurityGroupsForVpc", handleNATSRequest(d.node, d.vpcService.GetSecurityGroupsForVpc), "spinifex-workers"},
 		{"ec2.AuthorizeSecurityGroupIngress", handleNATSRequest(d.node, d.vpcService.AuthorizeSecurityGroupIngress), "spinifex-workers"},
 		{"ec2.AuthorizeSecurityGroupEgress", handleNATSRequest(d.node, d.vpcService.AuthorizeSecurityGroupEgress), "spinifex-workers"},
 		{"ec2.RevokeSecurityGroupIngress", handleNATSRequest(d.node, d.vpcService.RevokeSecurityGroupIngress), "spinifex-workers"},
 		{"ec2.RevokeSecurityGroupEgress", handleNATSRequest(d.node, d.vpcService.RevokeSecurityGroupEgress), "spinifex-workers"},
+		{"ec2.UpdateSecurityGroupRuleDescriptionsIngress", handleNATSRequest(d.node, d.vpcService.UpdateSecurityGroupRuleDescriptionsIngress), "spinifex-workers"},
+		{"ec2.UpdateSecurityGroupRuleDescriptionsEgress", handleNATSRequest(d.node, d.vpcService.UpdateSecurityGroupRuleDescriptionsEgress), "spinifex-workers"},
 		{"ec2.ModifyInstanceAttribute", handleNATSRequest(d.node, d.instanceService.ModifyInstanceAttribute), "spinifex-workers"},
 		{"ec2.ModifyInstanceMetadataOptions", handleNATSRequest(d.node, d.instanceService.ModifyInstanceMetadataOptions), "spinifex-workers"},
 		{"ec2.MonitorInstances", handleNATSRequest(d.node, d.monitorInstances), "spinifex-workers"},
@@ -1335,10 +1342,7 @@ func (d *Daemon) startLocal() error {
 	}
 
 	// Detect management bridge for system instance control plane NICs.
-	mgmtBridge := "br-mgmt"
-	if d.config.Daemon.MgmtBridge != "" {
-		mgmtBridge = d.config.Daemon.MgmtBridge
-	}
+	mgmtBridge := d.mgmtBridgeName()
 	bridgeIP, bridgeErr := host.GetBridgeIPv4(mgmtBridge)
 	if bridgeErr != nil {
 		slog.Warn("Management bridge not detected, system instances will not get mgmt NIC", "bridge", mgmtBridge, "err", bridgeErr)
@@ -1967,6 +1971,7 @@ func (d *Daemon) startCluster() error {
 				opts = append(opts, handlers_ec2_vpc.BootstrapIDs{
 					VpcId:    d.clusterConfig.Bootstrap.VpcId,
 					SubnetId: d.clusterConfig.Bootstrap.SubnetId,
+					IgwId:    d.clusterConfig.Bootstrap.IgwId,
 				})
 			}
 			if _, err := d.vpcService.EnsureDefaultVPC(accountID, opts...); err != nil {
@@ -2352,12 +2357,33 @@ func (d *Daemon) waitForClusterReady() {
 	slog.Warn("Cluster readiness timeout, proceeding with recovery anyway", "max_wait_ms", otelsetup.Millis(maxWait))
 }
 
-// checkViperblockReady reports whether viperblock is reachable via NATS.
+// checkViperblockReady reports whether viperblock can actually serve a mount,
+// which needs more than a live NATS socket: mounting claims a JetStream KV
+// lease before it reads any state, so JetStream must be serving too.
+//
+// Testing only IsConnected was why waitForClusterReady did not prevent the
+// race it exists to prevent. The client reconnects as soon as nats-server
+// accepts TCP, which is well before JetStream has quorum, so readiness passed
+// roughly 30 seconds before the first lease claim could have succeeded — and
+// every guest recovering in that window was failed and latched.
+//
+// AccountInfo is the probe because it is the cheapest round trip that proves
+// JetStream is answering, and it changes nothing.
 func (d *Daemon) checkViperblockReady() bool {
-	if d.natsConn == nil {
+	if d.natsConn == nil || !d.natsConn.IsConnected() {
 		return false
 	}
-	return d.natsConn.IsConnected()
+
+	js, err := jetstream.New(d.natsConn)
+	if err != nil {
+		return false
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	_, err = js.AccountInfo(ctx)
+	return err == nil
 }
 
 // predastoreReadinessClient is hoisted to package scope so the 2s readiness
@@ -2416,11 +2442,29 @@ func (d *Daemon) LoadState() error {
 
 // restoreInstances delegates to vm.Manager.Restore and syncs the local state
 // file so it matches in-memory state.
+//
+// It then starts the recovery retry loop in the background. Restore is a single
+// pass: any instance whose relaunch lost a race with a still-starting dependency
+// is parked in StateError and, before this, stayed there until an operator
+// noticed. The loop reconsiders exactly those, once the backing store is
+// confirmed ready, and returns as soon as there is nothing left to retry.
+//
+// Backgrounded because it must not hold up the rest of daemon start: the guests
+// it is recovering are already down, and blocking here would keep the API from
+// coming up while it waits.
 func (d *Daemon) restoreInstances() error {
 	d.vmMgr.Restore()
 	if err := d.WriteState(); err != nil {
 		slog.Error("Failed to persist local state after restore", "error", err)
 	}
+
+	d.recoveryRetryWg.Go(func() {
+		d.vmMgr.RunRecoveryRetryLoop(d.shuttingDown.Load)
+		if err := d.WriteState(); err != nil {
+			slog.Error("Failed to persist local state after recovery retry", "error", err)
+		}
+	})
+
 	return nil
 }
 

spinifex/daemon/daemon_handlers.go

@@ -73,7 +73,8 @@ func respondWithError(nodeID string, msg *nats.Msg, errCode string) {
 // an opaque ServerInternal, leaving the reason visible only in the daemon log.
 // Mirrors utils.ServeNATSRequestCtx, which has always preserved the message.
 func respondWithServiceError(nodeID string, msg *nats.Msg, err error) {
-	payload := utils.GenerateErrorPayloadWithMessage(awserrors.ValidErrorCodeFromError(err), err.Error())
+	_, message, _ := awserrors.ResolveErrorDetail(err)
+	payload := utils.GenerateErrorPayloadWithMessage(awserrors.ValidErrorCodeFromError(err), message)
 	respondNATSMsg(nodeID, msg, payload)
 }
 

spinifex/daemon/daemon_handlers_error_test.go

@@ -44,31 +44,61 @@ func requestErrorEnvelope(t *testing.T, subject string, fn func(context.Context,
 	return errResp
 }
 
-// An unregistered error sanitizes to ServerInternal, but its reason must still
-// reach the caller. This is the ACM force-renew refusal: without the message the
-// CLI prints a bare "ServerInternal" and the operator has to read the daemon log
-// to learn that only PRIVATE_CA certificates can be force-renewed.
-func TestHandleNATSRequest_UncodedErrorCarriesMessage(t *testing.T) {
+// A plain errors.New carries no client-facing message: only awserrors.Errorf
+// records one. Falling back to err.Error() here would put whatever context a
+// caller wrapped in for its own logs onto the wire, which is the disclosure
+// risk this contract exists to prevent.
+func TestHandleNATSRequest_UncodedErrorOmitsMessage(t *testing.T) {
 	const reason = "acm renewal: arn:aws:acm:ap-southeast-2:000000000001:certificate/abc is a AMAZON_ISSUED certificate; only PRIVATE_CA certificates can be force-renewed"
 
 	errResp := requestErrorEnvelope(t, "test.err.msg.uncoded", func(context.Context, *testInput, string) (*testOutput, error) {
 		return nil, errors.New(reason)
 	})
 
+	assert.Equal(t, awserrors.ErrorServerInternal, errResp["Code"])
+	assert.Nil(t, errResp["Message"])
+}
+
+// The ACM force-renew refusal this test mirrors now builds its error with
+// awserrors.Errorf specifically so the operator sees the actionable reason
+// instead of a bare "ServerInternal", without relying on the rejected
+// err.Error() fallback.
+func TestHandleNATSRequest_ErrorfMessageOnUnregisteredCodeCarriesMessage(t *testing.T) {
+	const reason = "acm renewal: arn:aws:acm:ap-southeast-2:000000000001:certificate/abc is a AMAZON_ISSUED certificate; only PRIVATE_CA certificates can be force-renewed"
+
+	errResp := requestErrorEnvelope(t, "test.err.msg.uncoded.errorf", func(context.Context, *testInput, string) (*testOutput, error) {
+		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "%s", reason)
+	})
+
 	assert.Equal(t, awserrors.ErrorServerInternal, errResp["Code"])
 	assert.Equal(t, reason, errResp["Message"],
 		"the actionable reason must survive the transport, not only the daemon log")
 }
 
-// A wrapped coded error keeps the real code (so the gateway maps the right HTTP
-// status) and carries the wrapping context alongside it.
-func TestHandleNATSRequest_WrappedCodedErrorCarriesMessage(t *testing.T) {
+// A bare coded error wrapped only for the handler's own log line keeps the
+// real code (so the gateway maps the right HTTP status) but must not leak the
+// wrapping context onto the wire — that context is internal detail, not a
+// message the producing call site attached via awserrors.Errorf.
+func TestHandleNATSRequest_WrappedCodedErrorOmitsWrapperContext(t *testing.T) {
 	errResp := requestErrorEnvelope(t, "test.err.msg.wrapped", func(context.Context, *testInput, string) (*testOutput, error) {
 		return nil, fmt.Errorf("vol-123 is attached to i-456: %w", errors.New(awserrors.ErrorVolumeInUse))
 	})
 
 	assert.Equal(t, awserrors.ErrorVolumeInUse, errResp["Code"])
-	assert.Contains(t, errResp["Message"], "vol-123 is attached to i-456")
+	assert.Nil(t, errResp["Message"])
+}
+
+// A wrapped Errorf-built error keeps its curated message across the wire, but
+// drops an outer wrapper added purely for the handler's own log line.
+func TestHandleNATSRequest_WrappedErrorfCarriesMessageNotWrapperContext(t *testing.T) {
+	const curated = "vol-123 is attached to i-456"
+	errResp := requestErrorEnvelope(t, "test.err.msg.wrapped.errorf", func(context.Context, *testInput, string) (*testOutput, error) {
+		cause := awserrors.Errorf(awserrors.ErrorVolumeInUse, "%s", curated)
+		return nil, fmt.Errorf("detach: %w", cause)
+	})
+
+	assert.Equal(t, awserrors.ErrorVolumeInUse, errResp["Code"])
+	assert.Equal(t, curated, errResp["Message"])
 }
 
 // A bare code carries no message. Handlers returning errors.New(awserrors.X) are

spinifex/daemon/daemon_handlers_instance.go

@@ -164,6 +164,13 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) string {
 		return outcomeError
 	}
 
+	// A multi-node spread carries the reservation ID the gateway minted once
+	// for the whole call. reservation is the same pointer every instance
+	// holds, so this override reaches all of them.
+	if gatewayReservationID := utils.ReservationIDFromMsg(msg); gatewayReservationID != "" {
+		reservation.SetReservationId(gatewayReservationID)
+	}
+
 	// PlacementGroupNode is daemon-local identity, set after prepare.
 	for _, instance := range instances {
 		if instance.PlacementGroupName != "" {

spinifex/daemon/daemon_handlers_test.go

@@ -798,13 +798,14 @@ func TestHandleEC2Events_RebootRunningInstance(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, reply)
 
-	// vm.Reboot returns a generic "QMP system_reset" error when the
-	// QMPClient encoder/decoder is nil, which handleRebootInstance maps
-	// to ServerInternal via its default branch.
+	// The daemon admits the reboot and runs it in the background, so the reply
+	// carries no error even though this guest's QMP client cannot be reset. A
+	// reply that waited for the guest would arrive after the caller gave up.
 	var errResp map[string]any
 	err = json.Unmarshal(reply.Data, &errResp)
 	require.NoError(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, errResp["Code"])
+	assert.NotContains(t, errResp, "Code", "an admitted reboot must not answer with an error")
+	t.Cleanup(daemon.vmMgr.WaitForBackgroundWork)
 
 	// Instance should remain in running state (reboot doesn't change state)
 	var status vm.InstanceState

spinifex/daemon/daemon_handlers_vpc.go

@@ -122,32 +122,26 @@ func (d *Daemon) ensureDefaultVPCInfrastructureFor(ctx context.Context, accountI
 		return
 	}
 
-	// Find the default VPC for this account
-	descOut, err := d.vpcService.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{}, accountID)
+	// The claim names the default VPC and the gateway ID every node agreed on.
+	info, err := d.vpcService.DefaultVPC(ctx, accountID)
 	if err != nil {
-		slog.WarnContext(ctx, "DescribeVpcs failed for default VPC infrastructure, retrying", "accountID", accountID, "err", err)
+		slog.WarnContext(ctx, "Default VPC lookup failed for default VPC infrastructure, retrying", "accountID", accountID, "err", err)
 		time.Sleep(500 * time.Millisecond)
-		descOut, err = d.vpcService.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{}, accountID)
+		info, err = d.vpcService.DefaultVPC(ctx, accountID)
 		if err != nil {
-			slog.ErrorContext(ctx, "DescribeVpcs failed for default VPC infrastructure after retry",
+			slog.ErrorContext(ctx, "Default VPC lookup failed for default VPC infrastructure after retry",
 				"accountID", accountID, "err", err)
 			return
 		}
 	}
-	var defaultVpcId string
-	for _, vpc := range descOut.Vpcs {
-		if vpc.IsDefault != nil && *vpc.IsDefault {
-			defaultVpcId = *vpc.VpcId
-			break
-		}
-	}
-	if defaultVpcId == "" {
+	if info == nil || info.VpcId == "" {
 		return
 	}
+	defaultVpcId := info.VpcId
 
 	// Check if IGW already attached. Uses the intent view: an attach requested
-	// by an earlier pass but not yet confirmed still counts, or this creates a
-	// second gateway for the same VPC every time it runs inside that window.
+	// by an earlier pass but not yet confirmed still counts, and a gateway the
+	// user attached in place of the default one is left alone.
 	existingIGW, err := d.igwService.AttachmentIntent(ctx, accountID, defaultVpcId)
 	if err != nil {
 		slog.WarnContext(ctx, "IGW lookup failed for default VPC infrastructure, retrying",
@@ -161,27 +155,13 @@ func (d *Daemon) ensureDefaultVPCInfrastructureFor(ctx context.Context, accountI
 		}
 	}
 	if existingIGW == nil {
-		// Create and attach an IGW — use bootstrap ID if available
-		var createOut *ec2.CreateInternetGatewayOutput
-		var err error
-		if accountID == admin.DefaultAccountID() && d.clusterConfig != nil && d.clusterConfig.Bootstrap.IgwId != "" {
-			createOut, err = d.igwService.CreateInternetGatewayWithID(&ec2.CreateInternetGatewayInput{}, accountID, d.clusterConfig.Bootstrap.IgwId)
-		} else {
-			createOut, err = d.igwService.CreateInternetGateway(ctx, &ec2.CreateInternetGatewayInput{}, accountID)
-		}
+		created, err := d.igwService.CreateAttachedInternetGateway(ctx, accountID, info.InternetGatewayId, defaultVpcId)
 		if err != nil {
-			slog.ErrorContext(ctx, "Failed to create default IGW", "accountID", accountID, "err", err)
+			slog.ErrorContext(ctx, "Failed to attach default IGW", "igwId", info.InternetGatewayId, "vpcId", defaultVpcId, "err", err)
 			return
 		}
-		igwId := *createOut.InternetGateway.InternetGatewayId
-		_, err = d.igwService.AttachInternetGateway(ctx, &ec2.AttachInternetGatewayInput{
-			InternetGatewayId: &igwId,
-			VpcId:             &defaultVpcId,
-		}, accountID)
-		if err != nil {
-			slog.ErrorContext(ctx, "Failed to attach default IGW", "igwId", igwId, "vpcId", defaultVpcId, "err", err)
-		} else {
-			slog.InfoContext(ctx, "Attached default IGW to default VPC", "igwId", igwId, "vpcId", defaultVpcId, "accountID", accountID)
+		if created {
+			slog.InfoContext(ctx, "Attached default IGW to default VPC", "igwId", info.InternetGatewayId, "vpcId", defaultVpcId, "accountID", accountID)
 		}
 	}
 

spinifex/daemon/daemon_handlers_vpc_test.go

@@ -2,6 +2,7 @@ package daemon
 
 import (
 	"encoding/json"
+	"sync"
 	"testing"
 	"time"
 
@@ -24,10 +25,30 @@ import (
 func createDefaultVPCTestDaemon(t *testing.T) (*Daemon, *nats.Conn) {
 	t.Helper()
 
-	daemon := createTestDaemon(t, sharedNATSURL)
-
 	_, nc, _ := testutil.StartTestJetStream(t)
 	testutil.StubVpcdSGResponder(t, nc)
+	return wireDefaultVPCServices(t, createTestDaemon(t, sharedNATSURL), nc), nc
+}
+
+// createDefaultVPCTestDaemons wires n daemons onto one JetStream, each through
+// its own connection, standing in for the nodes of one cluster.
+func createDefaultVPCTestDaemons(t *testing.T, n int) []*Daemon {
+	t.Helper()
+
+	ns, nc, _ := testutil.StartTestJetStream(t)
+	testutil.StubVpcdSGResponder(t, nc)
+	daemons := make([]*Daemon, n)
+	for i := range n {
+		conn, err := nats.Connect(ns.ClientURL())
+		require.NoError(t, err)
+		t.Cleanup(conn.Close)
+		daemons[i] = wireDefaultVPCServices(t, createTestDaemon(t, sharedNATSURL), conn)
+	}
+	return daemons
+}
+
+func wireDefaultVPCServices(t *testing.T, daemon *Daemon, nc *nats.Conn) *Daemon {
+	t.Helper()
 
 	vpcSvc, err := handlers_ec2_vpc.NewVPCServiceImplWithNATS(t.Context(), daemon.config, nc)
 	require.NoError(t, err)
@@ -41,7 +62,7 @@ func createDefaultVPCTestDaemon(t *testing.T) (*Daemon, *nats.Conn) {
 	require.NoError(t, err)
 	daemon.routeTableService = rtbSvc
 
-	return daemon, nc
+	return daemon
 }
 
 // defaultVPCID returns the account's default VPC ID, or "" when it has none.
@@ -87,6 +108,50 @@ func TestHandleAccountCreated_BuildsDefaultVPCInfrastructure(t *testing.T) {
 	assert.Equal(t, 1, defaultRoutes, "the default VPC must get its 0.0.0.0/0 route")
 }
 
+func TestHandleAccountCreated_RacingNodesAttachOneGateway(t *testing.T) {
+	daemons := createDefaultVPCTestDaemons(t, 4)
+
+	const accountID = "000000000066"
+	evt, err := json.Marshal(map[string]string{"account_id": accountID})
+	require.NoError(t, err)
+
+	// The IAM event and the gateway's request can land on different nodes, and
+	// every node must converge on one gateway and one default route.
+	start := make(chan struct{})
+	var wg sync.WaitGroup
+	for _, d := range daemons {
+		wg.Go(func() {
+			<-start
+			assert.Equal(t, outcomeSuccess, d.handleAccountCreated(&nats.Msg{Subject: "account.created", Data: evt}))
+		})
+	}
+	close(start)
+	wg.Wait()
+
+	d := daemons[0]
+	vpcID := defaultVPCID(t, d, accountID)
+	require.NotEmpty(t, vpcID)
+
+	igws, err := d.igwService.DescribeInternetGateways(t.Context(), &ec2.DescribeInternetGatewaysInput{}, accountID)
+	require.NoError(t, err)
+	require.Len(t, igws.InternetGateways, 1, "racing nodes attached more than one gateway")
+	igw, err := d.igwService.AttachmentIntent(t.Context(), accountID, vpcID)
+	require.NoError(t, err)
+	require.NotNil(t, igw)
+
+	rtbs, err := d.routeTableService.DescribeRouteTables(t.Context(), &ec2.DescribeRouteTablesInput{}, accountID)
+	require.NoError(t, err)
+	require.Len(t, rtbs.RouteTables, 1)
+	var defaultRoutes int
+	for _, r := range rtbs.RouteTables[0].Routes {
+		if aws.StringValue(r.DestinationCidrBlock) == "0.0.0.0/0" {
+			defaultRoutes++
+			assert.Equal(t, aws.StringValue(igw.InternetGatewayId), aws.StringValue(r.GatewayId))
+		}
+	}
+	assert.Equal(t, 1, defaultRoutes)
+}
+
 func TestHandleAccountCreated_RejectsUnusableEvents(t *testing.T) {
 	tests := []struct {
 		name string

spinifex/daemon/daemon_mgmt_ip.go

@@ -1,12 +1,15 @@
 package daemon
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"log/slog"
 	"net"
 	"sync"
+	"time"
 
+	"github.com/mulgadc/spinifex/spinifex/network/host"
 	"github.com/mulgadc/spinifex/spinifex/vm"
 	"github.com/nats-io/nats.go/jetstream"
 )
@@ -326,3 +329,59 @@ func (a *MgmtIPAllocator) AllocatedCount() int {
 	defer a.mu.Unlock()
 	return len(a.allocated)
 }
+
+// defaultMgmtBridge is the management bridge assumed when config names none.
+const defaultMgmtBridge = "br-mgmt"
+
+// mgmtNeighTimeout bounds one `ip neigh` invocation. The kernel answers at once;
+// the budget covers a wedged exec, not slow work.
+const mgmtNeighTimeout = 5 * time.Second
+
+// Kernel neighbour-table hooks, indirected so the unit suite can observe them
+// without running `ip neigh` as root. Same idiom as health.go's gateDialFn.
+var (
+	flushMgmtNeigh = host.FlushNeigh
+	primeMgmtNeigh = host.ReplaceNeigh
+)
+
+// mgmtBridgeName returns the configured management bridge, or the default.
+func (d *Daemon) mgmtBridgeName() string {
+	if d.config != nil && d.config.Daemon.MgmtBridge != "" {
+		return d.config.Daemon.MgmtBridge
+	}
+	return defaultMgmtBridge
+}
+
+// invalidateMgmtNeigh drops the host ARP entry for a released management IP.
+// br-mgmt is one flat L2 segment and the address is re-handed within seconds
+// with a fresh MAC, so a surviving entry blackholes the next holder.
+func (d *Daemon) invalidateMgmtNeigh(instanceID, mgmtIP string) {
+	// mgmtBridgeIP is empty when startLocal found no bridge, so there is no
+	// neighbour table to hold a stale entry and `ip neigh` would only ENODEV.
+	if mgmtIP == "" || d.mgmtBridgeIP == "" {
+		return
+	}
+	bridge := d.mgmtBridgeName()
+	ctx, cancel := context.WithTimeout(context.Background(), mgmtNeighTimeout)
+	defer cancel()
+	if err := flushMgmtNeigh(ctx, nil, bridge, mgmtIP); err != nil {
+		slog.Warn("Failed to flush mgmt ARP entry; a recycled address may blackhole until the entry expires",
+			"instanceId", instanceID, "mgmtIP", mgmtIP, "bridge", bridge, "err", err)
+	}
+}
+
+// primeMgmtNeighEntry programs the ARP entry for a freshly assigned management
+// IP. Preferred over relying on the release-side flush alone: the MAC is known
+// here, so the first dial does not race an ARP round trip.
+func (d *Daemon) primeMgmtNeighEntry(instanceID, mgmtIP, mgmtMAC string) {
+	if mgmtIP == "" || mgmtMAC == "" {
+		return
+	}
+	bridge := d.mgmtBridgeName()
+	ctx, cancel := context.WithTimeout(context.Background(), mgmtNeighTimeout)
+	defer cancel()
+	if err := primeMgmtNeigh(ctx, nil, bridge, mgmtIP, mgmtMAC); err != nil {
+		slog.Warn("Failed to prime mgmt ARP entry; the first dial falls back to ARP resolution",
+			"instanceId", instanceID, "mgmtIP", mgmtIP, "mgmtMAC", mgmtMAC, "bridge", bridge, "err", err)
+	}
+}

spinifex/daemon/daemon_mgmt_ip_test.go

@@ -7,6 +7,8 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/network/host"
 	"github.com/mulgadc/spinifex/spinifex/vm"
 	"github.com/nats-io/nats.go"
 )
@@ -617,3 +619,140 @@ func TestMgmtIPAllocator_Allocate_RefusedWhenKVNeverBound(t *testing.T) {
 		t.Fatal("expected allocation to be refused with no KV bound")
 	}
 }
+
+// recordingPlumber satisfies vm.NetworkPlumber for the cleanup-path tests. Only
+// CleanupTap is exercised; the rest are here to satisfy the interface.
+type recordingPlumber struct {
+	cleanedTaps []string
+}
+
+var _ vm.NetworkPlumber = (*recordingPlumber)(nil)
+
+func (p *recordingPlumber) SetupTap(vm.TapSpec) error { return nil }
+func (p *recordingPlumber) CleanupTap(name string) error {
+	p.cleanedTaps = append(p.cleanedTaps, name)
+	return nil
+}
+func (p *recordingPlumber) EnsureIMDSDatapathBridge() error         { return nil }
+func (p *recordingPlumber) AttachIMDSDatapath(_, _, _ string) error { return nil }
+func (p *recordingPlumber) DetachIMDSDatapath(string) error         { return nil }
+func (p *recordingPlumber) EnsureVPCHostPort(_, _, _ string) error  { return nil }
+func (p *recordingPlumber) RemoveVPCHostPort(string) error          { return nil }
+
+// neighCall is one recorded flush or prime.
+type neighCall struct {
+	dev string
+	ip  string
+	mac string
+}
+
+// captureNeighHooks swaps the kernel neighbour hooks for recorders and restores
+// them when the test ends.
+func captureNeighHooks(t *testing.T) (flushed, primed *[]neighCall) {
+	t.Helper()
+	var f, p []neighCall
+	origFlush, origPrime := flushMgmtNeigh, primeMgmtNeigh
+	flushMgmtNeigh = func(_ context.Context, _ host.Runner, dev, ip string) error {
+		f = append(f, neighCall{dev: dev, ip: ip})
+		return nil
+	}
+	primeMgmtNeigh = func(_ context.Context, _ host.Runner, dev, ip, mac string) error {
+		p = append(p, neighCall{dev: dev, ip: ip, mac: mac})
+		return nil
+	}
+	t.Cleanup(func() { flushMgmtNeigh, primeMgmtNeigh = origFlush, origPrime })
+	return &f, &p
+}
+
+// A released address is re-handed within seconds with a different MAC, so the
+// stale entry has to go or the next holder blackholes until the kernel expires
+// it. This is the defect behind the nightly rds isolation failure.
+func TestCleanupMgmtNetwork_FlushesNeighForReleasedAddress(t *testing.T) {
+	flushed, _ := captureNeighHooks(t)
+
+	alloc, err := NewMgmtIPAllocator("10.15.8.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	plumber := &recordingPlumber{}
+	d := &Daemon{networkPlumber: plumber, mgmtIPAllocator: alloc, mgmtBridgeIP: "10.15.8.1"}
+
+	newInstanceCleanerAdapter(d).CleanupMgmtNetwork(&vm.VM{ID: "i-abc", MgmtIP: "10.15.8.10"})
+
+	if len(*flushed) != 1 {
+		t.Fatalf("flush calls = %d, want 1", len(*flushed))
+	}
+	if got := (*flushed)[0]; got.ip != "10.15.8.10" || got.dev != "br-mgmt" {
+		t.Errorf("flushed %+v, want ip 10.15.8.10 on br-mgmt", got)
+	}
+	if len(plumber.cleanedTaps) != 1 {
+		t.Errorf("tap cleanup calls = %d, want 1", len(plumber.cleanedTaps))
+	}
+}
+
+// An instance that never got a mgmt NIC has no address to invalidate, and
+// flushing "" would be an error the operator cannot act on.
+func TestCleanupMgmtNetwork_NoMgmtIPSkipsFlush(t *testing.T) {
+	flushed, _ := captureNeighHooks(t)
+
+	d := &Daemon{networkPlumber: &recordingPlumber{}}
+	newInstanceCleanerAdapter(d).CleanupMgmtNetwork(&vm.VM{ID: "i-abc"})
+
+	if len(*flushed) != 0 {
+		t.Errorf("flush calls = %d, want 0", len(*flushed))
+	}
+}
+
+func TestInvalidateMgmtNeigh_UsesConfiguredBridge(t *testing.T) {
+	flushed, _ := captureNeighHooks(t)
+
+	d := &Daemon{config: &config.Config{}, mgmtBridgeIP: "10.15.8.1"}
+	d.config.Daemon.MgmtBridge = "br-ctrl"
+	d.invalidateMgmtNeigh("i-abc", "10.15.8.10")
+
+	if len(*flushed) != 1 || (*flushed)[0].dev != "br-ctrl" {
+		t.Fatalf("flushed %+v, want one call on br-ctrl", *flushed)
+	}
+}
+
+// startLocal tolerates an absent bridge, and every persisted instance still
+// carries a mgmt IP. Flushing against a device that is not there would warn
+// about a blackhole that cannot happen, on every terminate.
+func TestInvalidateMgmtNeigh_NoBridgeIsNoOp(t *testing.T) {
+	flushed, _ := captureNeighHooks(t)
+
+	d := &Daemon{}
+	d.invalidateMgmtNeigh("i-abc", "10.15.8.10")
+
+	if len(*flushed) != 0 {
+		t.Errorf("flush calls = %d, want 0", len(*flushed))
+	}
+}
+
+// The MAC is known at attach time, so the entry is programmed directly rather
+// than left for the first dial to resolve — the same choice the EIP path makes.
+func TestPrimeMgmtNeighEntry_ProgramsAddressAndMAC(t *testing.T) {
+	_, primed := captureNeighHooks(t)
+
+	d := &Daemon{}
+	d.primeMgmtNeighEntry("i-abc", "10.15.8.10", "02:d9:f3:a8:fb:be")
+
+	if len(*primed) != 1 {
+		t.Fatalf("prime calls = %d, want 1", len(*primed))
+	}
+	want := neighCall{dev: "br-mgmt", ip: "10.15.8.10", mac: "02:d9:f3:a8:fb:be"}
+	if (*primed)[0] != want {
+		t.Errorf("primed %+v, want %+v", (*primed)[0], want)
+	}
+}
+
+func TestPrimeMgmtNeighEntry_NoMACIsNoOp(t *testing.T) {
+	_, primed := captureNeighHooks(t)
+
+	d := &Daemon{}
+	d.primeMgmtNeighEntry("i-abc", "10.15.8.10", "")
+
+	if len(*primed) != 0 {
+		t.Errorf("prime calls = %d, want 0", len(*primed))
+	}
+}