release: update go.mod for v1.20.0

RegisterTargets read the target group, appended in memory and wrote the whole record back with no revision guard, so two concurrent registrations lost one. Terraform parallelises by default, so any target group with more than one member hit it, and the apply exited 0 with half the fleet not behind the load balancer.

Every writer of the record now goes through the store's existing CAS rather than a wholesale Put, because a CAS loop on one writer is defeated by a non-CAS writer on the same key. Contention stays per target group; no global lock is introduced.

RequestCertificate returned InternalErrorException for a missing tenant CA, which is a client-actionable precondition rather than a server fault. Worse, that code is classified retryable, so the provider backed off and retried instead of failing, and a single certificate sat at Still creating for over nine minutes. It now returns ACM's not-found code, which is terminal, and carries a message naming the missing CA and how to create one.

ListCertificates omitted CertificateSummaryList entirely on an empty account, since the SDK marshaller drops a nil slice unconditionally.

Closes: mulga-c6khe
Closes: mulga-wbjal
Closes: mulga-l6yft
Closes: mulga-61h3h

.github/ISSUE_TEMPLATE/1-bug.yml

@@ -0,0 +1,59 @@
+name: Bug report
+description: Something is not working as it should.
+labels: ["bug"]
+body:
+  - type: dropdown
+    id: component
+    attributes:
+      label: Component
+      options:
+        - EC2 / instances
+        - EBS / volumes
+        - S3 / object store
+        - VPC / networking
+        - IAM / authentication
+        - EKS
+        - API gateway
+        - Web UI
+        - CLI / daemon
+        - Observability / telemetry
+        - Build / CI / packaging
+        - Documentation
+        - Other / not sure
+    validations:
+      required: true
+
+  - type: input
+    id: version
+    attributes:
+      label: Spinifex version
+      description: Output of `spinifex --version`, or the commit SHA you are on.
+      placeholder: v1.0.0, or commit abc1234
+    validations:
+      required: true
+
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened
+      description: What you saw, what you expected instead, and how to reproduce it.
+    validations:
+      required: true
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Logs and output
+      description: Relevant log lines, error messages or command output.
+      render: shell
+
+  - type: textarea
+    id: platform
+    attributes:
+      label: Platform
+      description: Where you are running it, and how you deployed it.
+      placeholder: |
+        OS: Debian 13
+        Kernel: 6.12
+        Arch: amd64
+        Deployment: single node, systemd

.github/ISSUE_TEMPLATE/2-feature.yml

@@ -0,0 +1,11 @@
+name: Feature request
+description: Suggest something new.
+labels: ["enhancement"]
+body:
+  - type: textarea
+    id: proposal
+    attributes:
+      label: What would you like to see?
+      description: What you are trying to do, and what stops you today.
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/3-docs.yml

@@ -0,0 +1,20 @@
+name: Documentation
+description: Something in the docs is wrong, missing or unclear.
+labels: ["documentation"]
+body:
+  - type: input
+    id: location
+    attributes:
+      label: Where
+      description: Link to the page, or the path to the file in this repository.
+      placeholder: https://docs.mulgadc.com/..., or README.md
+    validations:
+      required: true
+
+  - type: textarea
+    id: problem
+    attributes:
+      label: What is wrong
+      description: What is inaccurate, missing or confusing, and what it should say instead.
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Documentation
+    url: https://docs.mulgadc.com
+    about: Guides, configuration reference and API compatibility notes.

.github/actions/e2e-analyze/analyze.go

@@ -91,9 +91,12 @@ type SuiteReport struct {
 	Label     string // short name, e.g. "single"
 	Total     int
 	FailCount int
-	Root      *Failure    // nil if no failures
-	Cascades  []Failure   // everything else that failed in this suite
-	Buckets   [][]Failure // failures grouped by signature (root's bucket first)
+	// Unresolved counts testcases go-junit-report could not find an outcome
+	// for. They are not failures and the suite can be green with hundreds.
+	Unresolved int
+	Root       *Failure    // nil if no failures
+	Cascades   []Failure   // everything else that failed in this suite
+	Buckets    [][]Failure // failures grouped by signature (root's bucket first)
 	// EndedAt is go-junit-report's <testsuite timestamp>, which records when
 	// the XML was produced — i.e. when the suite finished, not when it began.
 	EndedAt time.Time
@@ -131,21 +134,22 @@ var cascadeMarkers = []string{
 	"Expected value not to be nil",
 }
 
-// teardownPatterns match the content a test logs from t.Cleanup or from the
-// harness's on-failure diagnostics — i.e. after it has already failed. They are
-// the LAST file:line lines in the failure body, so extractErrorLine's
-// keep-the-last rule picks them over the assertion unless they are skipped.
-// Anchored deliberately: a fragment loose enough to match an assertion would
-// blank out the one line the report exists to show.
-var teardownPatterns = []*regexp.Regexp{
+// nonFailurePatterns match content a test emits AFTER the assertion that failed
+// — t.Cleanup, harness diagnostics, or a t.Logf on a passing branch — which
+// extractErrorLine's keep-the-last rule would otherwise pick over the assertion.
+// Go emits t.Logf and t.Errorf identically, so anchor every entry: a fragment
+// loose enough to match an assertion blanks out the line the report exists to
+// show. `\bas expected\b` unanchored also matches "did not fail as expected".
+var nonFailurePatterns = []*regexp.Regexp{
 	regexp.MustCompile(`^DB instance \S+ is gone$`),
 	regexp.MustCompile(`^db diagnostics \S+:`),
+	regexp.MustCompile(`^\S+: management bridge \S+:\d+ refused as expected$`),
 }
 
-// isTeardownLine reports whether a candidate error line was emitted after the
-// failure it would otherwise be mistaken for.
-func isTeardownLine(content string) bool {
-	for _, re := range teardownPatterns {
+// isNonFailureLine reports whether a candidate error line was emitted after the
+// failure it would otherwise be mistaken for, or reports an expected outcome.
+func isNonFailureLine(content string) bool {
+	for _, re := range nonFailurePatterns {
 		if re.MatchString(content) {
 			return true
 		}
@@ -242,12 +246,12 @@ func extractErrorLine(body string) string {
 			// progress lines from the same file:line pattern before the
 			// failing assertion finally fires. Cleanup runs after it, though,
 			// so its lines are last of all and have to be skipped.
-			if content := strings.TrimSpace(m[1]); !isTeardownLine(content) {
+			if content := strings.TrimSpace(m[1]); !isNonFailureLine(content) {
 				goFileLine = content
 			}
 			continue
 		}
-		if fileHintRe.MatchString(l) && !isTeardownLine(l) {
+		if fileHintRe.MatchString(l) && !isNonFailureLine(l) {
 			lastTestLine = l
 		}
 	}
@@ -293,15 +297,23 @@ func extractFileHint(body string) string {
 	}
 	// Scanned line by line rather than across the body so a cleanup line's own
 	// file:line does not become the site the report points a reader at.
-	hint := ""
+	hint, filtered := "", ""
 	for _, raw := range strings.Split(body, "\n") {
 		l := strings.TrimSpace(raw)
-		if m := goFileLineRe.FindStringSubmatch(l); m != nil && isTeardownLine(strings.TrimSpace(m[1])) {
+		all := fileHintRe.FindAllString(l, -1)
+		if len(all) == 0 {
 			continue
 		}
-		if all := fileHintRe.FindAllString(l, -1); len(all) > 0 {
-			hint = all[len(all)-1]
+		if m := goFileLineRe.FindStringSubmatch(l); m != nil && isNonFailureLine(strings.TrimSpace(m[1])) {
+			filtered = all[len(all)-1]
+			continue
 		}
+		hint = all[len(all)-1]
+	}
+	// A body whose only file:line was allowlisted still gets a pointer: coarse
+	// beats none, and extractErrorLine has the same last-resort shape.
+	if hint == "" {
+		return filtered
 	}
 	return hint
 }
@@ -366,7 +378,7 @@ func computeLeafSet(cases []junitTC) map[string]bool {
 func computeRolledUpSet(cases []junitTC) map[string]bool {
 	rolledUp := make(map[string]bool, len(cases))
 	for _, tc := range cases {
-		if tc.Failure == nil && tc.Error == nil {
+		if tc.Failure == nil && !isRealError(tc.Error) {
 			continue
 		}
 		// Mark every ancestor: the failure belongs to the deepest case that
@@ -412,6 +424,18 @@ func parseStartTime(s string) time.Time {
 	return t
 }
 
+// noResultMessage is go-junit-report's marker for a testcase it saw start and
+// never saw finish. On cell-20 it accounted for 93 of the 120 testcases and 24
+// of the 25 reported failures, none of which had failed.
+const noResultMessage = "No test result found"
+
+// isRealError separates a genuine <error> from the parser's no-result marker.
+// A parent must not be treated as rolled up by a child whose outcome is merely
+// unknown, or the parent's own failure is dropped along with it.
+func isRealError(e *junitFailure) bool {
+	return e != nil && strings.TrimSpace(e.Message) != noResultMessage
+}
+
 // suiteLabel turns "junit-single.xml" into "single".
 func suiteLabel(path string) string {
 	base := filepath.Base(path)
@@ -457,6 +481,17 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 			case tc.Failure != nil:
 				body = tc.Failure.Body
 			case tc.Error != nil:
+				// go-junit-report emits this when a `=== RUN` line has no
+				// matching result line. It means the outcome is unknown, not
+				// that the test errored, and it attaches the test's own log
+				// output — which then reads as an assertion.
+				if !isRealError(tc.Error) {
+					rep.Unresolved++
+					if leaf {
+						cumul += tc.Time
+					}
+					continue
+				}
 				body = tc.Error.Body
 			default:
 				if leaf {
@@ -553,6 +588,18 @@ func ParseFile(path string, data []byte) (SuiteReport, error) {
 }
 
 // Render writes the markdown report.
+// writeUnresolved names the testcases whose outcome the parser could not read.
+// Worth surfacing because a suite that leaves most of its subtests unresolved
+// is reporting on far less than its test count suggests.
+func writeUnresolved(b *strings.Builder, s SuiteReport) {
+	if s.Unresolved == 0 {
+		return
+	}
+	fmt.Fprintf(b, "ℹ️ %d of %d testcases have no result line in the log, so `go-junit-report` "+
+		"could not read their outcome. Not failures — but this suite reports on %d tests, not %d.\n\n",
+		s.Unresolved, s.Total, s.Total-s.Unresolved, s.Total)
+}
+
 func Render(r Report) string {
 	var b strings.Builder
 	fmt.Fprintf(&b, "## %s\n\n", r.Title)
@@ -579,9 +626,11 @@ func Render(r Report) string {
 	for _, s := range r.Suites {
 		if s.FailCount == 0 {
 			fmt.Fprintf(&b, "### Suite `%s`: ✅ pass (%d tests)\n\n", s.Label, s.Total)
+			writeUnresolved(&b, s)
 			continue
 		}
 		fmt.Fprintf(&b, "### Suite `%s`: %d failed, 1 root cause likely\n\n", s.Label, s.FailCount)
+		writeUnresolved(&b, s)
 
 		if s.Root != nil {
 			// XML order is completion order, so "earliest" only means earliest

.github/actions/e2e-analyze/analyze_test.go

@@ -141,6 +141,46 @@ func TestExtractErrorLine_SkipsCleanupLoggedAfterTheFailure(t *testing.T) {
 	}
 }
 
+// A subtest looping over two subjects calls t.Errorf for the first and t.Logf
+// for the second, so the success line is last in the body. Nightly run
+// 34383800404 reported that log line as the failure.
+func TestExtractErrorLine_SkipsSuccessLoggedAfterTheFailure(t *testing.T) {
+	body := `    isolation_test.go:80: rds-e2e-iso-a-1788981456: management bridge 10.15.8.10:5432 could not be tested: want connection refused, got dial tcp 10.15.8.10:5432: i/o timeout
+    isolation_test.go:84: rds-e2e-iso-b-1788981456: management bridge 10.15.8.11:5432 refused as expected`
+	got := extractErrorLine(body)
+	if !strings.Contains(got, "could not be tested") {
+		t.Errorf("extractErrorLine = %q, want the assertion at isolation_test.go:80", got)
+	}
+	if hint := extractFileHint(body); hint != "isolation_test.go:80" {
+		t.Errorf("extractFileHint = %q, want isolation_test.go:80", hint)
+	}
+}
+
+// "as expected" reads the same in an affirmation and in its negation, so an
+// unanchored pattern suppresses the assertion it was meant to outrank and
+// promotes a harness progress line in its place.
+func TestExtractErrorLine_KeepsNegatedExpectationAssertions(t *testing.T) {
+	body := `    isolation_test.go:60: waiting for the db instance to become available
+    isolation_test.go:80: rds-e2e-iso-a: management bridge 10.15.8.10:5432 was not refused as expected: got a connection`
+	got := extractErrorLine(body)
+	if !strings.Contains(got, "was not refused as expected") {
+		t.Errorf("extractErrorLine = %q, want the assertion at isolation_test.go:80", got)
+	}
+	if hint := extractFileHint(body); hint != "isolation_test.go:80" {
+		t.Errorf("extractFileHint = %q, want isolation_test.go:80", hint)
+	}
+}
+
+// Every candidate allowlisted leaves extractFileHint with nothing to return,
+// and unlike extractErrorLine it has no last-resort branch. A coarse pointer
+// beats sending the reader to no file at all.
+func TestExtractFileHint_FallsBackWhenEveryCandidateFiltered(t *testing.T) {
+	body := `    isolation_test.go:84: rds-e2e-iso-b: management bridge 10.15.8.11:5432 refused as expected`
+	if hint := extractFileHint(body); hint != "isolation_test.go:84" {
+		t.Errorf("extractFileHint = %q, want isolation_test.go:84", hint)
+	}
+}
+
 func TestExtractFileHint_PicksLastMatch(t *testing.T) {
 	body := "ec2helpers.go:50: setup\n    vpc_test.go:227: Eventually: condition not met"
 	got := extractFileHint(body)
@@ -174,6 +214,55 @@ func TestParseFile_SkipsEmptyParentFailure(t *testing.T) {
 	}
 }
 
+// go-junit-report emits <error message="No test result found"> for a `=== RUN`
+// with no matching result line, and attaches the test's own log output. On a
+// baremetal cell that was 93 of 120 testcases and 24 of 25 reported failures,
+// none of which had failed: "ALB internal: 20/20 successful" read as an error.
+func TestParseFile_NoResultFoundIsNotAFailure(t *testing.T) {
+	xml := []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="3" failures="1" errors="2">
+  <testcase name="TestLB/Internal_ALB" time="0.0"><error message="No test result found"><![CDATA[
+    lb_test.go:88: ALB internal: 20/20 successful, 2 unique
+]]></error></testcase>
+  <testcase name="TestLB/Internal_NLB" time="0.0"><error message="No test result found"></error></testcase>
+  <testcase name="TestX/RealFailure" time="0.5"><failure message="Failed"><![CDATA[
+    foo_test.go:10:
+        Messages: real assertion message
+]]></failure></testcase>
+</testsuite></testsuites>`)
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	if sr.FailCount != 1 {
+		t.Fatalf("FailCount = %d, want 1 (no-result entries are not failures)", sr.FailCount)
+	}
+	if sr.Unresolved != 2 {
+		t.Fatalf("Unresolved = %d, want 2", sr.Unresolved)
+	}
+	if sr.Root == nil || sr.Root.Name != "TestX/RealFailure" {
+		t.Fatalf("expected the real failure as root, got %+v", sr.Root)
+	}
+}
+
+// An <error> that is not the parser's no-result marker is a genuine error and
+// must still be reported.
+func TestParseFile_KeepsARealError(t *testing.T) {
+	xml := []byte(`<?xml version="1.0"?>
+<testsuites><testsuite name="" tests="1" failures="0" errors="1">
+  <testcase name="TestX" time="0.5"><error message="panic"><![CDATA[
+    foo_test.go:10: panic: runtime error: index out of range
+]]></error></testcase>
+</testsuite></testsuites>`)
+	sr, err := ParseFile("junit-x.xml", xml)
+	if err != nil {
+		t.Fatalf("ParseFile: %v", err)
+	}
+	if sr.FailCount != 1 || sr.Unresolved != 0 {
+		t.Fatalf("FailCount = %d, Unresolved = %d, want 1 and 0", sr.FailCount, sr.Unresolved)
+	}
+}
+
 func TestParseFile_SkipsParentRolledUpBySubtests(t *testing.T) {
 	// A parent that logs during cleanup gets a non-empty body, and a t.Log like
 	// "DB instance … is gone" — which passing tests emit too — reads exactly

.github/actions/junit-to-es/action.yml

@@ -0,0 +1,43 @@
+name: JUnit to Elasticsearch
+description: >-
+  Bulk-index one doc per JUnit test case into the mulga Elasticsearch sink,
+  so per-test pass rates outlive the 90-day artifact retention. Call it once
+  per job right after the JUnit XML is produced (with `if: always()`), next to
+  the junit-report action that renders the same files into the job summary.
+
+inputs:
+  junit-glob:
+    description: Shell glob matching the JUnit XML files to ship.
+    required: true
+  cell:
+    description: >-
+      Cell label stored as labels.ci_cell — the same value the job puts in
+      ci.cell for its OTel attributes (single, multi, gpu, ddil, 20, ...).
+      Without it, results from different topologies collapse into one bucket.
+    required: true
+  es-sink:
+    description: host:port of the Elasticsearch sink.
+    required: false
+    default: "192.168.1.13:9200"
+  index:
+    description: Target index.
+    required: false
+    default: mulga-ci-test-results
+
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      env:
+        JUNIT_GLOB: ${{ inputs.junit-glob }}
+        JUNIT_CELL: ${{ inputs.cell }}
+        MULGA_ES_SINK: ${{ inputs.es-sink }}
+        ES_INDEX: ${{ inputs.index }}
+        RUN_ID: ${{ github.run_id }}
+        RUN_ATTEMPT: ${{ github.run_attempt }}
+        WORKFLOW: ${{ github.workflow }}
+        BRANCH: ${{ github.head_ref || github.ref_name }}
+        SHA: ${{ github.sha }}
+        ARTIFACT_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+      run: |
+        python3 "${{ github.action_path }}/ship.py"

.github/actions/junit-to-es/ship.py

@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+"""Ship per-test JUnit results to the Elasticsearch sink.
+
+One doc per leaf test case, bulk-indexed into mulga-ci-test-results. The
+mulga-ci-runs summary says which suite failed; this says which test, with the
+failure text, so a per-test pass-rate trend outlives artifact retention.
+
+Inputs (env):
+    JUNIT_GLOB      Glob matching JUnit XML files. No matches is not an error:
+                    a leg that never ran produces none.
+    JUNIT_CELL      Cell label for these results (single, multi, gpu, 20, ...).
+    MULGA_ES_SINK   host:port of the sink. Anonymous requests are superuser
+                    there, so no credentials are involved.
+    ES_INDEX        Target index; defaults to mulga-ci-test-results.
+    RUN_ID, RUN_ATTEMPT, WORKFLOW, BRANCH, SHA, ARTIFACT_URL
+"""
+
+from __future__ import annotations
+
+import glob
+import json
+import os
+import re
+import sys
+import urllib.error
+import urllib.request
+import xml.etree.ElementTree as ET
+from datetime import datetime, timezone
+
+# Go test output carries colour codes into the JUnit CDATA (harness.Phase
+# emits them), which are noise in a stored failure message.
+_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
+
+# Enough failure text to recognise a failure; the artifact log stays the full
+# record.
+_MAX_MESSAGE = 4000
+
+# go-junit-report emits this when a `=== RUN` line has no matching result line,
+# which means the log it was fed was truncated or filtered. It is a parse gap,
+# not a test outcome, so it is stored under its own status and left out of every
+# pass rate rather than counted as a failure.
+_NO_RESULT = "No test result found"
+
+
+def _strip_ansi(s: str) -> str:
+    return _ANSI_RE.sub("", s)
+
+
+def _truncate(s: str) -> str:
+    s = s.strip()
+    return s if len(s) <= _MAX_MESSAGE else s[: _MAX_MESSAGE - 1] + "…"
+
+
+def _suite_from_path(path: str) -> str:
+    base = os.path.basename(path)
+    if base.endswith(".xml"):
+        base = base[:-4]
+    if base.startswith("junit-"):
+        base = base[len("junit-") :]
+    return base
+
+
+def _cases(path: str) -> list[dict]:
+    """Parse one JUnit file into leaf test cases."""
+    try:
+        root = ET.parse(path).getroot()
+    except ET.ParseError as exc:
+        print(f"::warning::junit-to-es: {path} is not parseable ({exc}), skipped")
+        return []
+
+    # JUnit XML has two shapes: <testsuite> as root, or <testsuites> wrapping
+    # several. .iter() handles both.
+    out: list[dict] = []
+    for s in root.iter("testsuite"):
+        for tc in s.findall("testcase"):
+            status = "pass"
+            fail_type = ""
+            message = ""
+            for tag in ("failure", "error"):
+                node = tc.find(tag)
+                if node is not None:
+                    status = "fail" if tag == "failure" else "error"
+                    fail_type = tag
+                    message = _strip_ansi(
+                        ((node.get("message") or "") + "\n" + (node.text or "")).strip()
+                    )
+                    if (node.get("message") or "").strip() == _NO_RESULT:
+                        status = "unknown"
+                    break
+            if status == "pass" and tc.find("skipped") is not None:
+                status = "skip"
+            out.append(
+                {
+                    "name": tc.get("name", ""),
+                    "classname": tc.get("classname", ""),
+                    "time": float(tc.get("time", 0) or 0),
+                    "status": status,
+                    "fail_type": fail_type,
+                    "message": message,
+                }
+            )
+
+    # Drop container cases: Go reports a parent t.Run as a rollup of its
+    # children, so keeping both double-counts one outcome. Same rule the job
+    # summary renderer applies.
+    names = {c["name"] for c in out}
+    containers = {n.rsplit("/", 1)[0] for n in names if "/" in n}
+    return [c for c in out if c["name"] not in containers]
+
+
+def _doc(cell: str, suite: str, case: dict, ts: str, ci: dict) -> dict:
+    full = case["name"]
+    top, _, sub = full.partition("/")
+    doc = {
+        "@timestamp": ts,
+        "suite": suite,
+        "status": case["status"],
+        "duration_seconds": case["time"],
+        "artifact_url": ci["artifact_url"],
+        "labels": {"ci_run_id": ci["run_id"], "ci_cell": cell},
+        "test": {
+            "name": top,
+            "subtest": sub,
+            "full_name": full,
+            "classname": case["classname"],
+        },
+        "ci": {
+            "workflow": ci["workflow"],
+            "branch": ci["branch"],
+            "sha": ci["sha"],
+            "run_attempt": ci["run_attempt"],
+        },
+    }
+    if case["fail_type"]:
+        doc["failure"] = {
+            "type": case["fail_type"],
+            "message": _truncate(case["message"]),
+        }
+    return doc
+
+
+def _doc_id(ci: dict, cell: str, suite: str, full_name: str) -> str:
+    # Deterministic so a replayed post overwrites instead of double-counting.
+    # run_attempt is part of it because a re-run is a distinct result.
+    return f"{ci['run_id']}-{ci['run_attempt']}-{cell}-{suite}-{full_name}"
+
+
+def _bulk(sink: str, index: str, body: str) -> int:
+    req = urllib.request.Request(
+        f"http://{sink}/{index}/_bulk",
+        data=body.encode("utf-8"),
+        headers={"Content-Type": "application/x-ndjson"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=60) as resp:
+            payload = json.loads(resp.read().decode("utf-8"))
+    except (urllib.error.URLError, OSError) as exc:
+        print(f"::error::junit-to-es: bulk POST to {sink}/{index} failed: {exc}")
+        return 1
+
+    if payload.get("errors"):
+        for item in payload.get("items", []):
+            err = next(iter(item.values())).get("error")
+            if err:
+                print(f"::error::junit-to-es: bulk rejected a doc: {json.dumps(err)}")
+                return 1
+    return 0
+
+
+def main() -> int:
+    pattern = os.environ.get("JUNIT_GLOB", "").strip()
+    if not pattern:
+        print("::error::junit-to-es: JUNIT_GLOB is empty")
+        return 1
+    cell = os.environ.get("JUNIT_CELL", "").strip()
+    if not cell:
+        print("::error::junit-to-es: JUNIT_CELL is empty")
+        return 1
+    sink = os.environ.get("MULGA_ES_SINK", "").strip()
+    if not sink:
+        print("::error::junit-to-es: MULGA_ES_SINK is empty")
+        return 1
+    index = os.environ.get("ES_INDEX", "").strip() or "mulga-ci-test-results"
+
+    ci = {
+        "run_id": os.environ.get("RUN_ID", ""),
+        "run_attempt": os.environ.get("RUN_ATTEMPT", ""),
+        "workflow": os.environ.get("WORKFLOW", ""),
+        "branch": os.environ.get("BRANCH", ""),
+        "sha": os.environ.get("SHA", ""),
+        "artifact_url": os.environ.get("ARTIFACT_URL", ""),
+    }
+    ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    lines: list[str] = []
+    counts: dict[str, int] = {"pass": 0, "fail": 0, "error": 0, "skip": 0, "unknown": 0}
+    for path in sorted(glob.glob(pattern)):
+        suite = _suite_from_path(path)
+        for case in _cases(path):
+            doc = _doc(cell, suite, case, ts, ci)
+            counts[case["status"]] = counts.get(case["status"], 0) + 1
+            meta = {"index": {"_id": _doc_id(ci, cell, suite, doc["test"]["full_name"])}}
+            lines.append(json.dumps(meta))
+            lines.append(json.dumps(doc))
+
+    if not lines:
+        # Not an error on its own: the leg may have been skipped, or died
+        # before any test ran. The run summary doc records that separately.
+        print(f"junit-to-es: no JUnit files matched {pattern}, nothing to post")
+        return 0
+
+    rc = _bulk(sink, index, "\n".join(lines) + "\n")
+    if rc == 0:
+        print(
+            f"junit-to-es: posted {sum(counts.values())} test docs "
+            f"(cell {cell}) to {sink}/{index} — {counts['pass']} pass, "
+            f"{counts['fail']} fail, {counts['error']} error, "
+            f"{counts['skip']} skip, {counts['unknown']} unknown"
+        )
+    return rc
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/e2e-cells.json

@@ -0,0 +1,147 @@
+[
+  {
+    "cell": 1,
+    "install": "binary",
+    "topo": "single",
+    "net": "dhcp",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "",
+    "suite": ""
+  },
+  {
+    "cell": 2,
+    "install": "binary",
+    "topo": "single",
+    "net": "static",
+    "host": "ubuntu-26.04",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "",
+    "suite": ""
+  },
+  {
+    "cell": 3,
+    "install": "binary",
+    "topo": "single",
+    "net": "dhcp",
+    "host": "ubuntu-26.04",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "",
+    "suite": ""
+  },
+  {
+    "cell": 8,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "dhcp",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": ""
+  },
+  {
+    "cell": 9,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "static",
+    "host": "ubuntu-26.04",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": ""
+  },
+  {
+    "cell": 10,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "dhcp",
+    "host": "ubuntu-26.04",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": ""
+  },
+  {
+    "cell": 17,
+    "install": "binary",
+    "topo": "single",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-pseudo",
+    "extra": "tofu-examples",
+    "suite": ""
+  },
+  {
+    "cell": 18,
+    "install": "binary",
+    "topo": "single",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "reboot-single",
+    "suite": ""
+  },
+  {
+    "cell": 19,
+    "install": "binary",
+    "topo": "single",
+    "net": "nat",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "nat-single",
+    "suite": ""
+  },
+  {
+    "cell": 25,
+    "install": "binary",
+    "topo": "single",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-single-nvme",
+    "extra": "eks",
+    "suite": "eks"
+  },
+  {
+    "cell": 26,
+    "install": "binary",
+    "topo": "single",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "ecs",
+    "suite": "ecs"
+  },
+  {
+    "cell": 27,
+    "install": "binary",
+    "topo": "single",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-single",
+    "extra": "rds",
+    "suite": "rds",
+    "timeout_minutes": 120
+  },
+  {
+    "cell": 29,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "static",
+    "host": "debian-13",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": "storagefault",
+    "timeout_minutes": 100
+  }
+]

.github/scripts/ci-fmt.sh

@@ -61,3 +61,37 @@ quiet_run() {
     return "$rc"
   fi
 }
+
+# Connection-layer failures from the libvirt tofu provider. The socket drops
+# when a hypervisor is carrying several environments at once, which is the
+# steady state here. A resource that failed to build says something else.
+CI_LIBVIRT_TRANSIENT='Unable to Connect to Libvirt|error connecting to libvirt|procedure interrupted|Cannot recv data|Cannot write data|End of file while reading data'
+
+# retry_run LABEL LOG_FILE -- CMD ARGS …
+# quiet_run with a bounded retry, taken only when that attempt's own output
+# names a libvirt connection failure. Every other failure is reported on the
+# first attempt, so a real bug never becomes a slow flake.
+retry_run() {
+  local label="$1" log="$2"; shift 2
+  local _attempts="${CI_RETRY_ATTEMPTS:-3}" _delay="${CI_RETRY_DELAY:-30}" _i _rc=1 _mark
+  for ((_i = 1; _i <= _attempts; _i++)); do
+    _mark=$(wc -c < "$log" 2>/dev/null || echo 0)
+    # rc must be read inside the else: an `if` whose condition fails and has no
+    # else exits 0, so reading $? after `fi` reports success for a failed run.
+    if "$@" >> "$log" 2>&1; then
+      if [[ $_i -gt 1 ]]; then ok "$label (libvirt recovered on attempt $_i)"; else ok "$label"; fi
+      return 0
+    else
+      _rc=$?
+    fi
+    tail -c "+$((_mark + 1))" "$log" | grep -Eq "$CI_LIBVIRT_TRANSIENT" || break
+    [[ $_i -eq $_attempts ]] && break
+    warn "$label: libvirt connection dropped (attempt $_i/$_attempts), retrying in ${_delay}s"
+    sleep "$_delay"
+  done
+  bad "$label (rc=$_rc) — last 80 lines of $(basename "$log"):"
+  group_open "tail $(basename "$log")"
+  tail -n 80 "$log"
+  group_close
+  return "$_rc"
+}

.github/scripts/e2e-run-summary.py

@@ -0,0 +1,220 @@
+#!/usr/bin/env python3
+"""Render one run-level table for the e2e nightly into the job summary.
+
+Reading a nightly meant opening nineteen jobs, and the Actions UI does not
+distinguish a cell that failed from one that was cancelled out from under it —
+a distinction every investigation of this suite has needed.
+
+Inputs (env):
+    GITHUB_TOKEN         Token with actions:read.
+    GITHUB_REPOSITORY    owner/repo.
+    GITHUB_RUN_ID        Run to summarise.
+    JUNIT_ROOT           Directory holding downloaded junit-cell-* artifacts.
+    GITHUB_STEP_SUMMARY  Appended to.
+
+Test counts come from the junit artifacts; result, duration and runner come
+from the jobs API, so a cell that died before writing junit still gets a row.
+"""
+
+from __future__ import annotations
+
+import glob
+import json
+import os
+import sys
+import urllib.error
+import urllib.request
+import xml.etree.ElementTree as ET
+from datetime import datetime
+
+API = "https://api.github.com"
+
+# A cell whose job produced no verdict is not a pass and not a failure. Keeping
+# them apart is the point of the table.
+VERDICT_ICON = {
+    "success": "✅",
+    "failure": "❌",
+    "cancelled": "🚫",
+    "skipped": "⏭️",
+    "timed_out": "⏱️",
+}
+
+
+def fetch_jobs(repo: str, run_id: str, token: str) -> list[dict]:
+    jobs: list[dict] = []
+    page = 1
+    while True:
+        url = f"{API}/repos/{repo}/actions/runs/{run_id}/jobs?per_page=100&page={page}"
+        req = urllib.request.Request(
+            url,
+            headers={
+                "Authorization": f"Bearer {token}",
+                "Accept": "application/vnd.github+json",
+                "X-GitHub-Api-Version": "2022-11-28",
+            },
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=30) as resp:
+                payload = json.load(resp)
+        except (urllib.error.URLError, TimeoutError) as exc:
+            print(f"::warning::jobs API page {page} failed: {exc}", file=sys.stderr)
+            break
+        batch = payload.get("jobs", [])
+        jobs.extend(batch)
+        if len(batch) < 100:
+            break
+        page += 1
+    return jobs
+
+
+def duration_s(job: dict) -> int | None:
+    started, completed = job.get("started_at"), job.get("completed_at")
+    if not started or not completed:
+        return None
+    fmt = "%Y-%m-%dT%H:%M:%SZ"
+    try:
+        return int((datetime.strptime(completed, fmt) - datetime.strptime(started, fmt)).total_seconds())
+    except ValueError:
+        return None
+
+
+def human(seconds: int | None) -> str:
+    if seconds is None:
+        return "—"
+    if seconds < 0:
+        return "0s"
+    return f"{seconds // 60}m{seconds % 60:02d}s" if seconds >= 60 else f"{seconds}s"
+
+
+def junit_counts(root: str) -> dict[str, dict]:
+    """Map cell number to its aggregate junit counts and failing test names."""
+    per_cell: dict[str, dict] = {}
+    for path in glob.glob(os.path.join(root, "**", "junit-*.xml"), recursive=True):
+        cell = cell_from_artifact_path(path)
+        if cell is None:
+            continue
+        acc = per_cell.setdefault(cell, {"tests": 0, "failures": 0, "skipped": 0, "failing": []})
+        try:
+            root_el = ET.parse(path).getroot()
+        except ET.ParseError as exc:
+            print(f"::warning::unparseable junit {path}: {exc}", file=sys.stderr)
+            continue
+        suites = root_el.iter("testsuite") if root_el.tag != "testsuite" else [root_el]
+        for suite in suites:
+            acc["tests"] += int(suite.get("tests") or 0)
+            acc["failures"] += int(suite.get("failures") or 0) + int(suite.get("errors") or 0)
+            acc["skipped"] += int(suite.get("skipped") or 0)
+        for case in root_el.iter("testcase"):
+            if case.find("failure") is not None or case.find("error") is not None:
+                name = case.get("name") or "?"
+                if name not in acc["failing"]:
+                    acc["failing"].append(name)
+    return per_cell
+
+
+def cell_from_artifact_path(path: str) -> str | None:
+    """Recover the cell number from a junit-cell-<n>-<run_id> artifact directory."""
+    for part in path.split(os.sep):
+        if part.startswith("junit-cell-"):
+            rest = part[len("junit-cell-"):]
+            cell = rest.split("-", 1)[0]
+            return cell if cell.isdigit() else None
+    return None
+
+
+def cell_number(job_name: str) -> str | None:
+    """cell-19 (single/static/debian-13) -> 19."""
+    if not job_name.startswith("cell-"):
+        return None
+    cell = job_name[len("cell-"):].split(" ", 1)[0]
+    return cell if cell.isdigit() else None
+
+
+def main() -> int:
+    repo = os.environ.get("GITHUB_REPOSITORY", "")
+    run_id = os.environ.get("GITHUB_RUN_ID", "")
+    token = os.environ.get("GITHUB_TOKEN", "")
+    summary_path = os.environ.get("GITHUB_STEP_SUMMARY", "")
+    junit_root = os.environ.get("JUNIT_ROOT", "")
+
+    if not (repo and run_id and token and summary_path):
+        print("::warning::e2e-run-summary: missing required environment", file=sys.stderr)
+        return 0
+
+    jobs = fetch_jobs(repo, run_id, token)
+    counts = junit_counts(junit_root) if junit_root and os.path.isdir(junit_root) else {}
+
+    rows = []
+    for job in jobs:
+        cell = cell_number(job.get("name", ""))
+        if cell is None:
+            continue
+        rows.append((int(cell), job))
+    rows.sort(key=lambda r: r[0])
+
+    lines = [f"## e2e nightly — run summary ({len(rows)} cells)", ""]
+    if not rows:
+        lines.append("No cell jobs found for this run.")
+        write(summary_path, lines)
+        return 0
+
+    lines += [
+        "| Cell | Topology | Result | Duration | Runner | Tests | Failed | Skipped |",
+        "| --- | --- | --- | --- | --- | --- | --- | --- |",
+    ]
+
+    tally: dict[str, int] = {}
+    total_tests = total_failed = total_seconds = 0
+    failing_detail: list[tuple[int, list[str]]] = []
+
+    for cell, job in rows:
+        name = job.get("name", "")
+        # A cell dispatched to a reusable workflow reports as
+        # "cell-20 (baremetal/pxe/real-hw) / Bare-Metal Single-Node E2E".
+        topo = name.split("(", 1)[1].split(")", 1)[0] if "(" in name else "—"
+        verdict = job.get("conclusion") or job.get("status") or "unknown"
+        tally[verdict] = tally.get(verdict, 0) + 1
+        secs = duration_s(job)
+        total_seconds += secs or 0
+        c = counts.get(str(cell), {})
+        total_tests += c.get("tests", 0)
+        total_failed += c.get("failures", 0)
+        if c.get("failing"):
+            failing_detail.append((cell, c["failing"]))
+        lines.append(
+            f"| {cell} | {topo} | {VERDICT_ICON.get(verdict, '❔')} {verdict} | {human(secs)} "
+            f"| {job.get('runner_name') or '—'} | {c.get('tests', '—')} "
+            f"| {c.get('failures', '—')} | {c.get('skipped', '—')} |"
+        )
+
+    verdicts = ", ".join(f"{n} {v}" for v, n in sorted(tally.items(), key=lambda kv: -kv[1]))
+    lines += [
+        "",
+        f"**Cells:** {verdicts}",
+        f"**Tests:** {total_tests} run, {total_failed} failed",
+        f"**Cell wall clock:** {human(total_seconds)} across all cells",
+    ]
+
+    slowest = sorted(rows, key=lambda r: duration_s(r[1]) or 0, reverse=True)[:3]
+    if slowest:
+        lines.append(
+            "**Slowest:** " + ", ".join(f"cell {c} ({human(duration_s(j))})" for c, j in slowest)
+        )
+
+    if failing_detail:
+        lines += ["", "<details><summary>Failing tests</summary>", ""]
+        for cell, names in failing_detail:
+            lines.append(f"- **cell {cell}** — " + ", ".join(f"`{n}`" for n in names))
+        lines += ["", "</details>"]
+
+    write(summary_path, lines)
+    return 0
+
+
+def write(path: str, lines: list[str]) -> None:
+    with open(path, "a", encoding="utf-8") as fh:
+        fh.write("\n".join(lines) + "\n")
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scripts/ensure-libguestfs.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Prepare a runner to build a guest image with build-system-image.sh.
+#
+# Installs libguestfs + qemu-utils if they are missing and makes a host kernel
+# readable, so the libguestfs appliance can boot. Nothing here touches a host
+# block device: virt-customize works inside an isolated userspace appliance, so
+# this is safe on a shared runner.
+#
+# Callers must still export LIBGUESTFS_BACKEND=direct themselves — it has to be
+# set in the environment the build runs in, which is not this subshell.
+set -euo pipefail
+
+# Three nightly service cells share one ci-single runner and start together, so
+# concurrent apt-get runs are the normal case rather than the exception. Two of
+# them lost the lock outright before this guard existed. Skipping the install
+# when the tools are already present avoids most of the contention; the lock
+# timeout covers the rest, including apt runs started by anything else on the
+# box.
+if command -v virt-customize >/dev/null && command -v qemu-img >/dev/null; then
+    echo "libguestfs-tools and qemu-utils already present"
+else
+    APT_OPTS=(-o DPkg::Lock::Timeout=600)
+    sudo apt-get "${APT_OPTS[@]}" update -qq
+    sudo apt-get "${APT_OPTS[@]}" install -y --no-install-recommends \
+        libguestfs-tools qemu-utils
+fi
+
+# Debian/Ubuntu ship /boot/vmlinuz-* mode 0600 and the appliance needs to read a
+# host kernel. Tolerated on failure: a runner whose kernel is already readable
+# has no vmlinuz to chmod under some images.
+sudo chmod 0644 /boot/vmlinuz-* || true

.github/scripts/suite-sets.sh

@@ -24,23 +24,88 @@
 # behaviour, so a second node adds no assertion. Its DB VMs are capped at four
 # concurrent by the suite's own semaphore, which is what keeps it inside the
 # job's 50-minute budget alongside the suites above.
-E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds"
+# quota belongs to both topologies because the gate it asserts is gateway code
+# that runs identically on either, and because a quota left switched on would
+# cap every suite sharing the environment — so it must be proved to restore
+# itself wherever it runs, not only on one node.
+# storagefault runs last in both lists and belongs to both topologies, for
+# different reasons. On one node it is RS(1,0), so freezing the only predastore
+# is a total outage and the guest-survival assertion is unambiguous. On three it
+# is RS(2,1) with degraded writes on, so two must be frozen to break the shard
+# floor — and only there is a cross-node restart possible at all, which is the
+# case that takes a volume over from a node holding writes the backend never
+# received. It is last because it SIGSTOPs a cluster-wide
+# service: anything sharing the environment would see the same outage and report
+# a failure that says nothing about itself.
+E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota storagefault"
 
 # Suites runnable against a multi-node environment.
-E2E_SUITES_MULTI="multinode lb cert"
+E2E_SUITES_MULTI="multinode lb cert quota storagefault"
 
 # grep -xE alternation form, for narrowing a requested set to the eligible one.
 E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"
 E2E_SUITES_MULTI_RE="$(printf '%s' "${E2E_SUITES_MULTI}" | tr ' ' '|')"
 
-# e2e-nightly runs a narrower set than the two sets above, deliberately: its
-# ~19 cells each have a 35-minute budget and exist to prove that every
-# install / network / host-OS permutation boots and serves. Suite breadth is
-# the PR gate's job, and it already runs on every push. These are subsets, and
-# e2e_suites_assert_subset below enforces that.
-E2E_SUITES_NIGHTLY_SINGLE="single cert"
+# What every e2e-nightly permutation cell must prove, deliberately narrower
+# than the two sets above: those cells have a 35-minute budget and exist to show
+# that every install / network / host-OS combination boots and serves. The
+# service suites are not deferred to the PR gate — eks, ecs and rds each run
+# nightly in their own dedicated cell, because each needs a guest AMI built
+# first and a service failure should not mask whether a host OS boots. These
+# are subsets, and e2e_suites_assert_subset below enforces that.
+#
+# storagefault has a dedicated cell for two reasons. It SIGSTOPs a cluster-wide
+# service, so in a permutation cell every suite sharing that environment would
+# report the outage as its own failure. And a full pass is ~63 minutes, nearly
+# all of it deliberate waiting, which is too long to add to the PR gate — where
+# it has never run, since the gate's default suite list does not name it.
+#
+# Its cell is three-node: the RS(2,1) shard floor and the cross-node takeover
+# after a failed seal do not exist on one node, and they are the assertions
+# worth a nightly.
+E2E_SUITES_NIGHTLY_SINGLE="single cert iam"
 E2E_SUITES_NIGHTLY_MULTI="multinode cert lb"
 
+# How long a suite gets, for the same reason the lists above live here: the
+# timeout was written out separately in each workflow and drifted, so rds ran
+# 50m on the PR gate and 30m nightly and panicked there. A suite needing more
+# than the default declares it once, here.
+#   e2e_suite_timeout <suite>
+e2e_suite_timeout() {
+  case "$1" in
+    # rds serialises its DB VMs behind a 3 GiB semaphore, so its wall clock is
+    # what the queue costs, not what the tests do: single tests logged 11m, 22m
+    # and 26m waiting for budget. 17 tests at that concurrency is over an hour
+    # before anything goes wrong, and the first pass to run deep into the suite
+    # still had three left at 60m. The loaded graceful-reboot case adds an
+    # instance that holds its slot through a pgbench load as well as a create.
+    rds) echo "105m" ;;
+    # diskperf runs two 16 GiB fio profiles per repetition, each on a volume it
+    # creates first. One pass is ~20 minutes on the bare-metal cell and a
+    # baseline capture at SPINIFEX_DISKPERF_REPS=3 is three times that.
+    diskperf) echo "180m" ;;
+    # storagefault is mostly deliberate waiting: one test alone holds predastore
+    # SIGSTOPped for seven minutes to prove a guest survives it. A full pass on
+    # dev-prod is ~63 minutes, so the 30m default truncated it mid-suite.
+    storagefault) echo "90m" ;;
+    *) echo "30m" ;;
+  esac
+}
+
+# One matrix leg per suite, carrying the job budget that suite needs. The outer
+# timeout must always be later than the `go test` one inside it: go test reports
+# where it stopped, a cancelled job reports only "The operation was canceled".
+#   e2e_suite_matrix <suite-list> <floor-minutes> [allowance-minutes]
+e2e_suite_matrix() {
+  local list="$1" floor="$2" allowance="${3:-15}" suite budget
+  for suite in ${list}; do
+    budget="$(e2e_suite_timeout "${suite}")"
+    budget=$((${budget%m} + allowance))
+    if [ "${budget}" -lt "${floor}" ]; then budget="${floor}"; fi
+    printf '%s\t%s\n' "${suite}" "${budget}"
+  done | jq -cRs 'split("\n")|map(select(length>0)|split("\t")|{suite:.[0],timeout:(.[1]|tonumber)})'
+}
+
 # Fail loudly when a narrowed list names a suite its topology cannot run. This
 # is the guard that stops the lists drifting again: a suite added to a nightly
 # subset without also being declared eligible above stops the job here rather

.github/workflows/apt-packages.yml

@@ -33,8 +33,6 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # The suites the nightly E2E matrix actually installs on. Ubuntu 22.04 is
-        # accepted by detect_os but has never been exercised and does not resolve.
         image:
           - ubuntu:24.04
           - ubuntu:26.04

.github/workflows/e2e-baremetal.yml

@@ -1,7 +1,26 @@
 name: E2E Bare-Metal
 
 on:
+  # Mirrors the suite selection of workflow_call so a baseline capture can be
+  # driven by hand. A dispatch run sees only the inputs declared here, so
+  # omitting these would silently fall back to the correctness set.
   workflow_dispatch:
+    inputs:
+      suites:
+        description: "Space-separated e2e suites to run on the node. Default is the correctness set; a performance run names its own so its numbers come off an otherwise idle cell"
+        required: false
+        type: string
+        default: "single cert lb"
+      suite-env:
+        description: "Extra environment for the suite invocation, e.g. SPINIFEX_DISKPERF_REPS=3"
+        required: false
+        type: string
+        default: ""
+      cell:
+        description: "Nightly cell number this run reports as. Names the artifact bundle and the ES dimension, so two callers of this workflow stay distinguishable in the sink"
+        required: false
+        type: string
+        default: "20"
   workflow_call:
     inputs:
       ref:
@@ -14,6 +33,21 @@ on:
         required: false
         type: string
         default: ""
+      suites:
+        description: "Space-separated e2e suites to run on the node. Default is the correctness set; a performance caller names its own so its numbers come off an otherwise idle cell"
+        required: false
+        type: string
+        default: "single cert lb"
+      suite-env:
+        description: "Extra environment for the suite invocation, e.g. SPINIFEX_DISKPERF_REPS=3"
+        required: false
+        type: string
+        default: ""
+      cell:
+        description: "Nightly cell number this run reports as. Names the artifact bundle and the ES dimension, so two callers of this workflow stay distinguishable in the sink"
+        required: false
+        type: string
+        default: "20"
 
 permissions:
   contents: read
@@ -21,25 +55,21 @@ permissions:
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
-  # Same central ES/APM sink as the other four e2e workflows. Bare-metal had
-  # no default here before — this cell's telemetry never reached the sink at
-  # all (not just missing ci.cell), see bm-bootstrap.sh's telemetry.env step.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
 
 jobs:
   e2e_baremetal:
     name: Bare-Metal Single-Node E2E
-    runs-on: [ self-hosted, ci-baremetal ]
-    timeout-minutes: 120
+    runs-on: [self-hosted, ci-baremetal]
+    timeout-minutes: 240
     concurrency:
       group: e2e-baremetal-cluster
       cancel-in-progress: false
     env:
       BM_ISO_CACHE: /var/cache/spinifex-iso-builder
-      ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-20-${{ github.run_id }}
-      # Caller-supplied (workflow_call input); empty for a manual
-      # workflow_dispatch run, which bm-bootstrap.sh treats as "no attrs".
-      SPINIFEX_OTEL_ATTRS: ${{ inputs.otel-attrs }}
+      BM_CELL: ${{ inputs.cell || '20' }}
+      ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ inputs.cell || '20' }}-${{ github.run_id }}
+      SPINIFEX_OTEL_ATTRS: ${{ inputs.otel-attrs || format('ci.run_id={0},ci.workflow={1},ci.cell={2},ci.matrix=baremetal/pxe/real-hw', github.run_id, github.workflow, inputs.cell || '20') }}
     steps:
       - name: Clean Workspace
         run: |
@@ -74,6 +104,10 @@ jobs:
           ssh-keygen -y -f "${RUNNER_TEMP}/bm-ci" > "${RUNNER_TEMP}/bm-ci.pub"
 
       - name: Build Artifacts
+        env:
+          # The node runs pre-built binaries, so a suite absent here cannot be
+          # selected later. Same list as the run step for that reason.
+          BM_SUITES: ${{ inputs.suites || 'single cert lb' }}
         run: |
           mulga/scripts/baremetal/bm-build-artifacts.sh \
             "$GITHUB_WORKSPACE" \
@@ -119,6 +153,21 @@ jobs:
       - name: Install go-junit-report on runner
         run: go install github.com/jstemmer/go-junit-report/v2@latest
 
+      # The eks suite's CreateCluster boots a control-plane VM from the eks-node
+      # AMI. Built here because libguestfs needs source and network, which the
+      # runner has and the PXE-booted node does not; bm-bootstrap.sh imports it.
+      - name: Build eks-node AMI (libguestfs, runner)
+        if: contains(inputs.suites, 'eks')
+        run: |
+          set -euo pipefail
+          # direct backend = no libvirt. Set here rather than in the script:
+          # it has to be in the environment the build itself runs in.
+          export LIBGUESTFS_BACKEND=direct
+          # spinifex is the workspace root here, unlike the nightly where it is
+          # checked out into a subdirectory.
+          "${GITHUB_WORKSPACE}/.github/scripts/ensure-libguestfs.sh"
+          make build-eks-node-image
+
       - name: Run Bare-Metal E2E
         env:
           BM_CI_PASSWORD: ${{ secrets.BM_CI_PASSWORD }}
@@ -126,9 +175,25 @@ jobs:
           BM_EPDU_ENV: ${{ runner.temp }}/epdu.env
           BM_UBUNTU_IMAGE: ${{ env.BM_ISO_CACHE }}/ubuntu-26.04-cloudimg-amd64.img
           SPX_EXTERNAL_POOL: ${{ env.SPINIFEX_CI_POOL }}
+          BM_SUITES: ${{ inputs.suites || 'single cert lb' }}
+          BM_SUITE_ENV: ${{ inputs.suite-env }}
+          # Empty unless the eks build above ran, and bm-bootstrap.sh skips the
+          # import when unset, so the other suites are unaffected.
+          BM_EKS_IMAGE: ${{ contains(inputs.suites, 'eks') && '/tmp/eks-node-image-build/eks-node-alpine.raw' || '' }}
+          BM_EKS_MANIFEST: ${{ contains(inputs.suites, 'eks') && format('{0}/scripts/images/eks-node/manifest.conf', github.workspace) || '' }}
         run: |
           mkdir -p "${ARTIFACT_DIR}"
           set -o pipefail
+          . "${GITHUB_WORKSPACE}/.github/scripts/suite-sets.sh"
+          BM_TEST_TIMEOUT=""
+          for suite in ${BM_SUITES}; do
+            tmo="$(e2e_suite_timeout "${suite}")"
+            if [ -z "${BM_TEST_TIMEOUT}" ] || [ "${tmo%m}" -gt "${BM_TEST_TIMEOUT%m}" ]; then
+              BM_TEST_TIMEOUT="${tmo}"
+            fi
+          done
+          export BM_TEST_TIMEOUT
+          echo "suites=${BM_SUITES} test-timeout=${BM_TEST_TIMEOUT}"
           sudo -E mulga/scripts/baremetal/run-baremetal-e2e.sh \
             --vmlinuz   "${BM_ISO_CACHE}/vmlinuz" \
             --initrd    "${BM_ISO_CACHE}/initrd.img" \
@@ -139,22 +204,40 @@ jobs:
       - name: Generate JUnit report
         if: always()
         run: |
-          LOG="${ARTIFACT_DIR}/test-baremetal.log"
+          # suite-output.log, not test-baremetal.log. The harness streams the
+          # full suite output to the first and a grep-filtered summary to its
+          # stdout, and that filter anchors --- PASS at column 0, so every
+          # indented subtest result is dropped. Parsed from the filtered log,
+          # go-junit-report saw a === RUN with no result and emitted
+          # <error message="No test result found"> — 17 of 39 testcases on a
+          # green cell-28, 93 of 120 on cell-20.
+          LOG="${ARTIFACT_DIR}/suite-output.log"
+          [ -s "$LOG" ] || LOG="${ARTIFACT_DIR}/test-baremetal.log"
           [ -f "$LOG" ] || exit 0
           go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-baremetal.xml" || true
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: ${{ env.BM_CELL }}
+
       - name: Analyze e2e failures
         if: always()
         uses: ./.github/actions/e2e-analyze
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           log-dir: ${{ env.ARTIFACT_DIR }}
-          title: "E2E failure analysis (cell 20)"
+          title: "E2E failure analysis (cell ${{ env.BM_CELL }})"
 
       - uses: actions/upload-artifact@v7
         if: always()
         with:
-          name: e2e-artifacts-cell-20-${{ github.run_id }}
+          name: e2e-artifacts-cell-${{ env.BM_CELL }}-${{ github.run_id }}
           path: ${{ env.ARTIFACT_DIR }}
           if-no-files-found: warn
           retention-days: 14

.github/workflows/e2e-ddil.yml

@@ -1,7 +1,5 @@
 name: E2E DDIL
 
-# This workflow is manually triggered while the DDIL scenarios are being
-# validated.
 on:
   workflow_dispatch:
     inputs:
@@ -10,29 +8,31 @@ on:
         required: false
         default: ""
 
+# Shared with e2e-nightly.yml and e2e-suites.yml: every one of them provisions
+# many cells on the same runner pools and the same hypervisors, so two in
+# flight compete for their disks and their libvirt guests. Queued rather than
+# cancelled — a run in progress is someone's result, not a stale one. e2e.yml
+# is outside the group: one cluster per run is cheap enough to overlap.
+concurrency:
+  group: e2e-hypervisors
+  cancel-in-progress: false
+
 permissions:
   contents: read
-  # actions: read lets the otel-job-id action look up this job's numeric id via
-  # the Actions API — github.job is only the YAML key, no context exposes it.
   actions: read
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
-  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
-  # empty string disables telemetry entirely.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
   ddil:
     name: DDIL Real Multi-Node
-    runs-on: [ self-hosted, ci-multi ]
+    runs-on: [self-hosted, ci-multi]
     timeout-minutes: 60
     env:
       TOFU_LOG: /tmp/e2e-tofu-ddil-${{ github.run_id }}.log
-      # Job-level env wins over the workflow-level default. ci.cell=ddil marks
-      # this workflow's telemetry so a DDIL run's partition-induced errors are
-      # never mistaken for a healthy multi-node run's in the sink.
       SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=ddil
     steps:
       - name: Clean Workspace
@@ -103,7 +103,7 @@ jobs:
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "VMs provisioned"
 
       - name: Bootstrap via Binary Install
@@ -180,6 +180,16 @@ jobs:
           junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
           title: "DDIL scenario results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ runner.temp }}/ddil-results/junit-*.xml
+          cell: ddil
+
       - name: Collect Cluster Logs
         if: failure()
         working-directory: mulga/scripts/tofu-cluster
@@ -218,7 +228,7 @@ jobs:
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu destroy"          "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu destroy"          "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
           ok "teardown complete"
 

.github/workflows/e2e-nightly.yml

@@ -2,86 +2,135 @@ name: E2E Nightly
 
 on:
   schedule:
-    - cron: "0 14 * * 0-4" # 00:00 Mon-Fri AEST = 14:00 Sun-Thu UTC
+    # 14:00 UTC is midnight AEST (01:00 AEDT — cron has no DST), when the
+    # hypervisors are idle. Every day, not weeknights: a weekend gap is two runs
+    # of evidence lost on a suite whose remaining failures are a rate, not a bug.
+    # :17 rather than :00 because GitHub queues scheduled runs behind every other
+    # top-of-hour cron and drops them under load, which is why 2026-09-14 never
+    # fired and why every run in that window started 3h15m-3h45m late.
+    - cron: "17 14 * * *"
   workflow_dispatch:
     inputs:
       cells:
-        description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-19 tofu matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection"
+        description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection"
         required: false
 
 permissions:
   contents: read
-  # actions: read lets the "Resolve CI job id" step below look up this job's
-  # numeric id via the Actions API — github.job is only the YAML key, no
-  # context exposes the id itself.
   actions: read
 
-# Scheduled runs are loaded from the default branch (main), so the schedule
-# trigger must stay in sync on main for cron to fire.
+# Every cell builds its cluster on the same five hypervisors, so two runs in
+# flight at once compete for their disks and their libvirt guests. Queue the
+# second rather than cancelling it: a dispatch is usually a verification run
+# and discarding it loses the result someone is waiting on.
+#
+# The group is shared with e2e-suites.yml and e2e-ddil.yml, which draw on the
+# same runner pools. e2e.yml is deliberately outside it: a single-cluster push
+# run is cheap enough to overlap, and grouping it stalled every push behind a
+# multi-hour nightly.
+concurrency:
+  group: e2e-hypervisors
+  cancel-in-progress: false
+
 env:
   GIT_BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
   UBUNTU_GOLD_IMAGE: /mnt/disk1/gold-images/spinifex-image-builder-ubuntu-26.04.qcow2
-  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
-  # empty string disables telemetry entirely.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
+  select:
+    runs-on: ubuntu-26.04
+    outputs:
+      cells: ${{ steps.pick.outputs.cells }}
+      any: ${{ steps.pick.outputs.any }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          sparse-checkout: |
+            .github/e2e-cells.json
+            .github/scripts/suite-sets.sh
+          sparse-checkout-cone-mode: false
+      - id: pick
+        env:
+          CELLS: ${{ github.event_name == 'workflow_dispatch' && inputs.cells || '' }}
+          SETUP_ALLOWANCE_MIN: 10
+        run: |
+          if [ -z "$CELLS" ]; then
+            sel=$(jq -c . .github/e2e-cells.json)
+          else
+            sel=$(jq -c --arg c "$CELLS" \
+              '[.[] | . as $e | select(($c | split(",")) | index($e.cell|tostring))]' \
+              .github/e2e-cells.json)
+          fi
+          echo "cells=$sel" >> "$GITHUB_OUTPUT"
+          [ "$(jq 'length' <<<"$sel")" -gt 0 ] && any=true || any=false
+          echo "any=$any" >> "$GITHUB_OUTPUT"
+          echo "selected: $(jq -c '[.[].cell]' <<<"$sel")"
+
+          # The inner timeout must always fire before the outer one. go test
+          # reports where it stopped and the suite's own failure; a job timeout
+          # reports only "The operation was canceled". cell-27 ran a 50m rds
+          # suite inside a 50m job and could never finish either way.
+          . .github/scripts/suite-sets.sh
+          # Cells with no suite are filtered in jq, not with an empty-field test:
+          # tab is IFS whitespace, so read collapses a run of them and an empty
+          # middle column silently shifts every later one left.
+          bad=0
+          while IFS=$'\t' read -r cell suite budget; do
+            want="$(e2e_suite_timeout "$suite")"; want="${want%m}"
+            if [ "$budget" -lt $((want + SETUP_ALLOWANCE_MIN)) ]; then
+              echo "::error::cell-$cell gives suite '$suite' ${want}m inside a ${budget}m job. Raise timeout_minutes in .github/e2e-cells.json to at least $((want + SETUP_ALLOWANCE_MIN)), or lower the suite budget in suite-sets.sh." >&2
+              bad=1
+            fi
+          done < <(jq -r '.[]|select((.suite//"")!="")|[.cell,.suite,(.timeout_minutes//50)]|@tsv' <<<"$sel")
+
+          # A permutation cell names no suite and runs the nightly subset for its
+          # topology, so its bound is the longest of those. max() and not sum():
+          # whichever suite hangs must panic on its own timeout first, and a sum
+          # would demand a budget the set has never come close to using.
+          while IFS=$'\t' read -r cell topo budget; do
+            case "$topo" in
+              *multi) set -- ${E2E_SUITES_NIGHTLY_MULTI} ;;
+              *)      set -- ${E2E_SUITES_NIGHTLY_SINGLE} ;;
+            esac
+            want=0
+            for suite in "$@"; do
+              t="$(e2e_suite_timeout "$suite")"; t="${t%m}"
+              [ "$t" -gt "$want" ] && want="$t"
+            done
+            if [ "$budget" -lt $((want + SETUP_ALLOWANCE_MIN)) ]; then
+              echo "::error::cell-$cell runs the $topo nightly subset, whose longest suite asks ${want}m, inside a ${budget}m job. Raise timeout_minutes in .github/e2e-cells.json to at least $((want + SETUP_ALLOWANCE_MIN))." >&2
+              bad=1
+            fi
+          done < <(jq -r '.[]|select((.suite//"")=="")|[.cell,.topo,(.timeout_minutes//50)]|@tsv' <<<"$sel")
+          [ "$bad" -eq 0 ]
+
   matrix:
     name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{
       matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
+    needs: select
+    if: needs.select.outputs.any == 'true'
     strategy:
       fail-fast: false
       max-parallel: 3
       matrix:
-        include:
-          - { cell: 1,  install: binary, topo: single,     net: dhcp,   host: debian-13,    arch: x86_64, runner: ci-single, extra: "" }
-          - { cell: 2,  install: binary, topo: single,     net: static, host: ubuntu-26.04, arch: x86_64, runner: ci-single, extra: "" }
-          - { cell: 3,  install: binary, topo: single,     net: dhcp,   host: ubuntu-26.04, arch: x86_64, runner: ci-single, extra: "" }
-          - { cell: 8,  install: binary, topo: real-multi, net: dhcp,   host: debian-13,    arch: x86_64, runner: ci-multi,  extra: "" }
-          - { cell: 9,  install: binary, topo: real-multi, net: static, host: ubuntu-26.04, arch: x86_64, runner: ci-multi,  extra: "" }
-          - { cell: 10, install: binary, topo: real-multi, net: dhcp,   host: ubuntu-26.04, arch: x86_64, runner: ci-multi,  extra: "" }
-          - { cell: 17, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-pseudo, extra: "tofu-examples" }
-          - { cell: 18, install: binary, topo: single,     net: static, host: debian-13,    arch: x86_64, runner: ci-single, extra: "reboot-single" }
-          - { cell: 19, install: binary, topo: single,     net: nat,    host: debian-13,    arch: x86_64, runner: ci-single, extra: "nat-single" }
-
-    runs-on: [ self-hosted, "${{ matrix.runner }}" ]
-    timeout-minutes: ${{ matrix.extra == 'tofu-examples' && 45 || 35 }}
+        include: ${{ fromJSON(needs.select.outputs.cells) }}
+    runs-on: [self-hosted, "${{ matrix.runner }}"]
+    timeout-minutes: ${{ matrix.timeout_minutes || 50 }}
     env:
       TOFU_LOG: /tmp/e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-nightly-cell-${{ matrix.cell }}-${{ github.run_id }}
-      # Job-level env wins over the workflow-level default above, so telemetry
-      # from this cell carries ci.cell/ci.matrix. Nine cells run under one
-      # run id and every cell names its hosts node1/node2/node3, so without
-      # this every cell's telemetry collapses into one unattributable bucket.
       SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=${{ matrix.cell }},ci.matrix=${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}
 
     steps:
-      # blank or scheduled run = full matrix.
-      # Input must be comma-separated with no whitespace (e.g. '1,2,17', not '1, 2').
-      - name: Gate on Dispatch Input
-        id: gate
-        env:
-          CELL: ${{ matrix.cell }}
-          CELLS: ${{ github.event_name == 'workflow_dispatch' && inputs.cells || '' }}
-        run: |
-          if [ -z "$CELLS" ] || [[ ",${CELLS}," == *",${CELL},"* ]]; then
-            echo "selected=true" >> "$GITHUB_OUTPUT"
-            echo "cell ${CELL} selected"
-          else
-            echo "selected=false" >> "$GITHUB_OUTPUT"
-            echo "cell ${CELL} not in '${CELLS}' — subsequent steps will skip"
-          fi
-
       - name: Clean Workspace
-        if: steps.gate.outputs.selected == 'true'
         run: |
           sudo umount -R -l /var/cache/spinifex-iso-builder/rootfs 2>/dev/null || true
           if [ ! -c /dev/null ]; then sudo rm -f /dev/null; sudo mknod -m 666 /dev/null c 1 3; fi
           sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
       - uses: actions/create-github-app-token@v3
-        if: steps.gate.outputs.selected == 'true'
         id: app-token
         with:
           client-id: ${{ secrets.APP_ID }}
@@ -89,7 +138,6 @@ jobs:
           repositories: mulga
 
       - uses: actions/checkout@v7
-        if: steps.gate.outputs.selected == 'true'
         with:
           repository: mulgadc/mulga
           token: ${{ steps.app-token.outputs.token }}
@@ -97,31 +145,47 @@ jobs:
           path: mulga
 
       - uses: actions/checkout@v7
-        if: steps.gate.outputs.selected == 'true'
         with:
           path: spinifex
 
-      # Nine cells share one run id and every cell names its hosts
+      # Twelve cells share one run id and every cell names its hosts
       # node1/node2/node3, so ci.cell above separates them and ci.job_id here
       # links each cell's telemetry to the job log that produced it. Runs
       # before Bootstrap bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env.
       - name: Resolve CI job id
-        if: steps.gate.outputs.selected == 'true'
         uses: ./spinifex/.github/actions/otel-job-id
         with:
           job-name: "cell-${{ matrix.cell }} (${{ matrix.topo }}/${{ matrix.net }}/${{ matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
 
       - name: Build Install Artifacts
-        if: steps.gate.outputs.selected == 'true'
         working-directory: mulga/scripts/tofu-cluster
+        env:
+          MATRIX_SUITE: ${{ matrix.suite }}
         run: |
+          # The script's default set is the nightly baseline. A service cell
+          # runs one suite outside it, and asks for that binary by name.
           ./build-install-artifacts.sh "${GIT_BRANCH}" \
             "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
             "${RUNNER_TEMP}/setup.sh" \
-            "${RUNNER_TEMP}/spinifex-tests.tar"
+            "${RUNNER_TEMP}/spinifex-tests.tar" \
+            --extra-suites "${MATRIX_SUITE}"
+
+      # Every runner agent runs on one host, while their nodes sit on several, so
+      # this samples the runner's host and not the node's hypervisor. Kept for
+      # runner-side contention; the node's own capacity is sampled after.
+      - name: Sample runner-host load (before)
+        if: always()
+        run: |
+          mkdir -p "${ARTIFACT_DIR}"
+          {
+            echo "=== before: $(date -u +%Y-%m-%dT%H:%M:%SZ) runner=${RUNNER_NAME} host=$(hostname) ==="
+            echo "loadavg: $(cat /proc/loadavg)"
+            grep -E '^(MemTotal|MemAvailable|SwapFree):' /proc/meminfo || true
+            echo "cpu(steal is field 8): $(grep '^cpu ' /proc/stat || true)"
+            echo "running qemu: $(pgrep -c qemu-system-x86 || echo 0)"
+          } >> "${ARTIFACT_DIR}/hypervisor-load.txt"
 
       - name: Provision VMs
-        if: steps.gate.outputs.selected == 'true'
         working-directory: mulga/scripts/tofu-cluster
         env:
           MATRIX_HOST: ${{ matrix.host }}
@@ -136,14 +200,15 @@ jobs:
             ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
           esac
 
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           tofu init -input=false >> "$TOFU_LOG" 2>&1
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
           tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
-          tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
+          # shellcheck disable=SC2086
+          retry_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false
 
       - name: Bootstrap via Binary Install
-        if: steps.gate.outputs.selected == 'true'
         working-directory: mulga/scripts/tofu-cluster
         env:
           MATRIX_NET: ${{ matrix.net }}
@@ -166,7 +231,7 @@ jobs:
             --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
 
       - name: Run Single Node E2E
-        if: steps.gate.outputs.selected == 'true' && matrix.topo == 'single' && matrix.extra == ''
+        if: matrix.topo == 'single' && matrix.extra == '' && matrix.suite == ''
         working-directory: mulga/scripts/tofu-cluster
         run: |
           set -euo pipefail
@@ -194,7 +259,7 @@ jobs:
              done; exit \$FAIL"
 
       - name: Run NAT Uplink E2E
-        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'nat-single'
+        if: matrix.extra == 'nat-single'
         working-directory: mulga/scripts/tofu-cluster
         # Skips cleanly (exit 0) when the tested branch predates the natuplink
         # suite so scheduled runs against dev stay green until the merge lands.
@@ -216,7 +281,7 @@ jobs:
              if [ \$rc -ne 0 ]; then echo 'natuplink: FAILED' >> '${ARTIFACT_DIR}/suite-status.txt'; fi; exit \$rc"
 
       - name: Run Multi Node E2E
-        if: steps.gate.outputs.selected == 'true' && matrix.topo == 'real-multi'
+        if: matrix.topo == 'real-multi' && matrix.suite == ''
         working-directory: mulga/scripts/tofu-cluster
         run: |
           set -euo pipefail
@@ -251,8 +316,221 @@ jobs:
                if [ \$rc -ne 0 ]; then echo \"\$SUITE: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; FAIL=1; fi; \
              done; exit \$FAIL"
 
+      # The dedicated suite cells only. Resolved once here rather than in each
+      # step below, which would repeat the same three lines five times.
+      - name: Resolve VM SSH details
+        id: vm
+        if: matrix.suite != '' && matrix.topo == 'single'
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          INVENTORY=$(tofu output -json inventory)
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          echo "wan_ip=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')" >> "$GITHUB_OUTPUT"
+          echo "ssh_key=${SSH_KEY/#\~/$HOME}" >> "$GITHUB_OUTPUT"
+
+      # The eks suite's CreateCluster boots a control-plane VM from the eks-node
+      # AMI, so that AMI must be in the VM's image store before the suite runs.
+      # build-system-image.sh customizes with libguestfs (an isolated userspace
+      # appliance touching no host block device), so it is safe on the shared
+      # runner. The build needs source + network (runner only) and the import
+      # needs predastore/NATS (VM only), hence the split.
+      - name: Build eks-node AMI (libguestfs, runner)
+        if: matrix.suite == 'eks'
+        working-directory: spinifex
+        run: |
+          set -euo pipefail
+          # direct backend = no libvirt. Set here rather than in the script:
+          # it has to be in the environment the build itself runs in.
+          export LIBGUESTFS_BACKEND=direct
+          "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ensure-libguestfs.sh"
+          make build-eks-node-image
+
+      - name: Import eks-node AMI on VM
+        if: matrix.suite == 'eks'
+        run: |
+          set -euo pipefail
+          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
+            spinifex/scripts/images/eks-node/manifest.conf)"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          scp "${SSH_OPTS[@]}" \
+            /tmp/eks-node-image-build/eks-node-alpine.raw \
+            "$VM":/tmp/eks-node-alpine.raw
+          ssh "${SSH_OPTS[@]}" "$VM" \
+            "spx admin images import --file /tmp/eks-node-alpine.raw \
+               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
+               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1
+
+      # The eks data-plane subtest (TestEKS/KubectlGetNodes) drives the full
+      # `aws eks get-token` -> kubectl -> apiserver -> token-webhook -> gateway
+      # broker -> STS verify path. The e2e binaries run ON the VM, so kubectl
+      # must be on the VM's PATH or harness.NewKubectl skips the subtest and the
+      # cell reports green having never exercised the data plane. Pinned to the
+      # k3s server version (setup.sh K3S_VERSION) and checksum-verified.
+      - name: Install kubectl on VM
+        if: matrix.suite == 'eks'
+        run: |
+          set -euo pipefail
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          ssh "${SSH_OPTS[@]}" tf-user@${{ steps.vm.outputs.wan_ip }} 'bash -s' <<'EOF'
+          set -euo pipefail
+          KVER=v1.32.5
+          case "$(uname -m)" in
+            x86_64)  ARCH=amd64 ;;
+            aarch64) ARCH=arm64 ;;
+            *) echo "unsupported arch $(uname -m)" >&2; exit 1 ;;
+          esac
+          cd /tmp
+          curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl"
+          curl -fsSLO "https://dl.k8s.io/release/${KVER}/bin/linux/${ARCH}/kubectl.sha256"
+          echo "$(cat kubectl.sha256)  kubectl" | sha256sum -c -
+          sudo install -m 0755 kubectl /usr/local/bin/kubectl
+          kubectl version --client
+          EOF
+
+      # RegisterContainerInstance boots the container instance from the ecs-node
+      # AMI, and resolveECSNodeAMI matches on the managed-by tag rather than the
+      # name, so the tag is not optional. The image directory is named for
+      # IMAGE_NAME in the manifest (ecs-agent), not for the make target.
+      - name: Build ecs-node AMI (libguestfs, runner)
+        if: matrix.suite == 'ecs'
+        working-directory: spinifex
+        run: |
+          set -euo pipefail
+          # direct backend = no libvirt. Set here rather than in the script:
+          # it has to be in the environment the build itself runs in.
+          export LIBGUESTFS_BACKEND=direct
+          "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ensure-libguestfs.sh"
+          make build-ecs-node-image
+
+      - name: Import ecs-node AMI on VM
+        if: matrix.suite == 'ecs'
+        run: |
+          set -euo pipefail
+          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
+            spinifex/scripts/images/ecs-agent/manifest.conf)"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          scp "${SSH_OPTS[@]}" \
+            /tmp/ecs-agent-image-build/ecs-agent-alpine.raw \
+            "$VM":/tmp/ecs-agent-alpine.raw
+          ssh "${SSH_OPTS[@]}" "$VM" \
+            "spx admin images import --file /tmp/ecs-agent-alpine.raw \
+               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
+               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1
+
+      # CreateDBInstance boots the DB VM from an RDS AMI, so both engines' images
+      # must be in the store before the suite runs. The suite's psql and mariadb
+      # clients need no step here: they run inside a client VM that installs them
+      # from its own user-data, because a private-only endpoint is unreachable
+      # from the runner and from the cluster host alike.
+      - name: Build RDS AMIs (libguestfs, runner)
+        if: matrix.suite == 'rds'
+        working-directory: spinifex
+        run: |
+          set -euo pipefail
+          # direct backend = no libvirt. Set here rather than in the script:
+          # it has to be in the environment the build itself runs in.
+          export LIBGUESTFS_BACKEND=direct
+          "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ensure-libguestfs.sh"
+          for IMAGE in rds-postgres rds-mariadb; do
+            make "build-${IMAGE}-image"
+          done
+
+      - name: Import RDS AMIs on VM
+        if: matrix.suite == 'rds'
+        run: |
+          set -euo pipefail
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          for IMAGE in rds-postgres rds-mariadb; do
+            eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
+              "spinifex/scripts/images/${IMAGE}/manifest.conf")"
+            # SYSTEM_TAG carries four tags, not one: managed-by marks it a system
+            # image, engine/engine-version are what an EngineVersion request
+            # resolves against, and the contract tag gates the data-volume format
+            # grant. --tag is repeatable, so expand rather than pass the whole
+            # string as a single value.
+            TAG_ARGS=""
+            for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
+            scp "${SSH_OPTS[@]}" \
+              "/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" \
+              "$VM":"/tmp/${IMAGE}-alpine.raw"
+            ssh "${SSH_OPTS[@]}" "$VM" \
+              "spx admin images import --file /tmp/${IMAGE}-alpine.raw \
+                 --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
+                 --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
+          done
+
+      - name: Run Service Suite E2E
+        if: matrix.suite != '' && matrix.topo == 'single'
+        env:
+          SUITE: ${{ matrix.suite }}
+        run: |
+          set -euo pipefail
+          # Same guard as the permutation steps, against the full single-node
+          # set: a cell naming a suite this topology cannot fully exercise fails
+          # here rather than skipping half its assertions.
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          e2e_suites_assert_subset "e2e-nightly cell-${{ matrix.cell }}" "${SUITE}" "${E2E_SUITES_SINGLE}"
+          TMO="$(e2e_suite_timeout "${SUITE}")"
+
+          ssh -tt -o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}" \
+            tf-user@${{ steps.vm.outputs.wan_ip }} \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
+             echo \"::group::suite ${SUITE}\"; \
+             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./${SUITE}.test -test.v -test.timeout=${TMO} \
+               2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log'; \
+             rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
+             if [ \$rc -ne 0 ]; then echo \"${SUITE}: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; fi; \
+             exit \$rc"
+
+      # The multi-node twin of the step above. Separate rather than merged
+      # because the suite needs every node's address to reach the whole cluster,
+      # which the single-node path has no way to supply.
+      - name: Run Multi Suite E2E
+        if: matrix.suite != '' && matrix.topo == 'real-multi'
+        working-directory: mulga/scripts/tofu-cluster
+        env:
+          SUITE: ${{ matrix.suite }}
+        run: |
+          set -euo pipefail
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          e2e_suites_assert_subset "e2e-nightly cell-${{ matrix.cell }}" "${SUITE}" "${E2E_SUITES_MULTI}"
+          TMO="$(e2e_suite_timeout "${SUITE}")"
+
+          INVENTORY=$(tofu output -json inventory)
+          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+
+          NODES_CSV=""
+          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
+            NODES_CSV="${NODES_CSV:+$NODES_CSV,}$ip"
+          done
+
+          ssh -tt -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
+            "set -u; mkdir -p '${ARTIFACT_DIR}'; cd \$HOME/e2e-tests/_bin; \
+             echo \"::group::suite ${SUITE}\"; \
+             date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
+             GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
+               SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
+               SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
+               ARTIFACT_DIR='${ARTIFACT_DIR}' \
+               ./${SUITE}.test -test.v -test.timeout=${TMO} \
+               2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log'; \
+             rc=\${PIPESTATUS[0]}; echo \"::endgroup::\"; \
+             if [ \$rc -ne 0 ]; then echo \"${SUITE}: FAILED\" >> '${ARTIFACT_DIR}/suite-status.txt'; fi; \
+             exit \$rc"
+
       - name: Import RDS PostgreSQL AMI
-        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples'
+        if: matrix.extra == 'tofu-examples'
         working-directory: mulga/scripts/tofu-cluster
         run: |
           set -euo pipefail
@@ -264,7 +542,7 @@ jobs:
             "spx admin images import --name spinifex-rds-postgres"
 
       - name: Run Tofu Workbooks
-        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'tofu-examples'
+        if: matrix.extra == 'tofu-examples'
         working-directory: mulga/scripts/tofu-cluster
         # The driver emits go test -v markers per workbook; teeing to
         # test-tofu.log feeds the same go-junit-report -> e2e-analyze -> upload
@@ -285,7 +563,7 @@ jobs:
           exit $rc
 
       - name: Run Reboot Resilience E2E
-        if: steps.gate.outputs.selected == 'true' && matrix.extra == 'reboot-single'
+        if: matrix.extra == 'reboot-single'
         working-directory: mulga/scripts/tofu-cluster
         # reboot.test runs from the runner (NOT via ssh-and-run) so it survives the
         # systemctl-reboot-induced ssh disconnect mid-run.
@@ -320,8 +598,46 @@ jobs:
           if [ $rc -ne 0 ]; then echo "reboot: FAILED" >> "${ARTIFACT_DIR}/suite-status.txt"; fi
           exit $rc
 
+      - name: Sample runner-host load (after)
+        if: always()
+        run: |
+          mkdir -p "${ARTIFACT_DIR}"
+          {
+            echo "=== after: $(date -u +%Y-%m-%dT%H:%M:%SZ) runner=${RUNNER_NAME} host=$(hostname) ==="
+            echo "loadavg: $(cat /proc/loadavg)"
+            grep -E '^(MemTotal|MemAvailable|SwapFree):' /proc/meminfo || true
+            echo "cpu(steal is field 8): $(grep '^cpu ' /proc/stat || true)"
+            echo "running qemu: $(pgrep -c qemu-system-x86 || echo 0)"
+          } >> "${ARTIFACT_DIR}/hypervisor-load.txt"
+
+      # The node, not the runner's host, is what the guests are scheduled on, so
+      # its load is the figure that explains a starved guest. Never fails the
+      # cell: a node already gone is the case this is trying to describe.
+      - name: Sample node load (after)
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
+          INVENTORY=$(tofu output -json inventory 2>/dev/null)
+          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip' 2>/dev/null)
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path' 2>/dev/null)
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          [ -n "$NODE1_IP" ] && [ "$NODE1_IP" != "null" ] || exit 0
+          [ -n "$SSH_KEY" ] && [ "$SSH_KEY" != "null" ] || exit 0
+          {
+            echo "=== node after: $(date -u +%Y-%m-%dT%H:%M:%SZ) ip=${NODE1_IP} ==="
+            ssh -o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=10 \
+              -i "$SSH_KEY" "tf-user@${NODE1_IP}" \
+              'echo "host: $(hostname)"; echo "loadavg: $(cat /proc/loadavg)";
+               grep -E "^(MemTotal|MemAvailable|SwapFree):" /proc/meminfo;
+               echo "cpu(steal is field 8): $(grep "^cpu " /proc/stat)";
+               echo "running qemu: $(pgrep -c qemu-system-x86 || echo 0)"' 2>&1
+          } >> "${ARTIFACT_DIR}/node-load.txt"
+          exit 0
+
       - name: Collect e2e artifacts
-        if: always() && steps.gate.outputs.selected == 'true'
+        if: always()
         working-directory: mulga/scripts/tofu-cluster
         run: |
           set +e
@@ -342,20 +658,93 @@ jobs:
               | tar -C "${ARTIFACT_DIR}/vm-console-node${idx}" -xf - 2>/dev/null
             ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
               > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
+            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \
+              > "${ARTIFACT_DIR}/openvswitch-ipsec-node${idx}.log" 2>/dev/null
+            # The unit filter above drops kernel and OOM messages, which are
+            # exactly what explains one node behaving unlike its siblings.
+            # Gzipped on the far side; the whole journal is far larger.
+            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo journalctl --no-pager --output=short-iso --since "90 min ago" | gzip -6' \
+              > "${ARTIFACT_DIR}/full-journal-node${idx}.log.gz" 2>/dev/null
+            # ovn-controller and ovs-vswitchd log to files, never to journald:
+            # without these a cross-chassis black-hole cannot be attributed to
+            # IPsec versus SB port binding.
+            mkdir -p "${ARTIFACT_DIR}/ovs-logs-node${idx}"
+            ssh "${SSH_OPTS[@]}" tf-user@${IP} 'sudo tar -C /var/log -cf - ovn openvswitch 2>/dev/null || true' \
+              | tar -C "${ARTIFACT_DIR}/ovs-logs-node${idx}" -xf - 2>/dev/null
           done
 
           cat "${ARTIFACT_DIR}"/spinifex-journal-node*.log \
             > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || true
           echo "collected: $(ls "${ARTIFACT_DIR}" 2>/dev/null | wc -l) entries"
 
+      # Whether br-mgmt is a per-host island or one L2 segment spanning the
+      # cluster decides whether a system instance can reach the gateway address
+      # it was baked with. Nothing in a failing run records which one was built,
+      # and the environment is destroyed before it can be asked.
+      - name: Collect management-fabric topology
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set +e
+          mkdir -p "${ARTIFACT_DIR}"
+          OUT="${ARTIFACT_DIR}/mgmt-fabric.txt"
+          INVENTORY=$(tofu output -json inventory 2>/dev/null) || { echo "no inventory; skipping"; exit 0; }
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path'); SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "$SSH_KEY")
+
+          declare -a WAN=() MGMT=()
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            WAN[$i]=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
+          done
+
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            idx=$((i + 1))
+            {
+              echo "=== node${idx} (${WAN[$i]}) ==="
+              ssh "${SSH_OPTS[@]}" tf-user@"${WAN[$i]}" \
+                'sudo ovs-vsctl list-ports br-mgmt 2>&1 | sed "s/^/ports: /"; \
+                 ip -4 addr show br-mgmt 2>&1 | sed "s/^/addr:  /"; \
+                 sudo ovs-vsctl get Bridge br-mgmt fail_mode 2>&1 | sed "s/^/fail:  /"'
+              echo
+            } >> "$OUT" 2>&1
+            MGMT[$i]=$(ssh "${SSH_OPTS[@]}" tf-user@"${WAN[$i]}" \
+              "ip -4 -o addr show br-mgmt 2>/dev/null | awk '{print \$4}' | cut -d/ -f1" 2>/dev/null)
+          done
+
+          # An island cluster reaches only its own address and that is correct;
+          # a spanned one must reach every peer. Either result is a finding, and
+          # a mixed result is the failure this matrix exists to catch.
+          {
+            echo "=== br-mgmt reachability (row = prober, column = target) ==="
+            printf '%-10s' "from\\to"
+            for i in $(seq 0 $((NODE_COUNT - 1))); do printf '%-18s' "node$((i+1)):${MGMT[$i]:-none}"; done
+            echo
+            for i in $(seq 0 $((NODE_COUNT - 1))); do
+              printf '%-10s' "node$((i+1))"
+              for j in $(seq 0 $((NODE_COUNT - 1))); do
+                if [ -z "${MGMT[$j]}" ]; then printf '%-18s' "-"; continue; fi
+                if ssh "${SSH_OPTS[@]}" tf-user@"${WAN[$i]}" \
+                     "ping -c1 -W1 ${MGMT[$j]} >/dev/null 2>&1"; then
+                  printf '%-18s' ok
+                else
+                  printf '%-18s' FAIL
+                fi
+              done
+              echo
+            done
+          } >> "$OUT" 2>&1
+
+          cat "$OUT"
+
       - name: Install go-junit-report
-        if: always() && steps.gate.outputs.selected == 'true'
+        if: always()
         run: |
           go install github.com/jstemmer/go-junit-report/v2@latest
           echo "$HOME/go/bin" >> "$GITHUB_PATH"
 
       - name: Convert test logs to JUnit XML
-        if: always() && steps.gate.outputs.selected == 'true'
+        if: always()
         run: |
           shopt -s nullglob
           for LOG in "${ARTIFACT_DIR}"/test-*.log; do
@@ -363,8 +752,18 @@ jobs:
             go-junit-report -in "$LOG" -out "${ARTIFACT_DIR}/junit-${SUITE}.xml" || true
           done
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: ${{ matrix.cell }}
+
       - name: Analyze e2e failures
-        if: always() && steps.gate.outputs.selected == 'true'
+        if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
@@ -372,18 +771,29 @@ jobs:
           title: "E2E failure analysis (cell ${{ matrix.cell }})"
 
       - uses: actions/upload-artifact@v7
-        if: always() && steps.gate.outputs.selected == 'true'
+        if: always()
         with:
           name: e2e-artifacts-cell-${{ matrix.cell }}-${{ github.run_id }}
           path: ${{ env.ARTIFACT_DIR }}
           if-no-files-found: warn
           retention-days: 14
 
+      # The XML on its own, so the roll-up can read every cell's counts without
+      # pulling nineteen journals it has no use for.
+      - name: Upload junit only
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: junit-cell-${{ matrix.cell }}-${{ github.run_id }}
+          path: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          if-no-files-found: ignore
+          retention-days: 14
+
       # Gated like every other step, not just always(): a cell the dispatch
       # deselected never checked out or ran tofu init, so destroying would fail
       # on missing plugins and report the cell red for correctly doing nothing.
       - name: Teardown
-        if: always() && steps.gate.outputs.selected == 'true'
+        if: always()
         working-directory: mulga/scripts/tofu-cluster
         env:
           MATRIX_HOST: ${{ matrix.host }}
@@ -394,13 +804,15 @@ jobs:
           case "$MATRIX_HOST" in
             ubuntu-*) TOFU_VAR_ARGS="-var=host_image_override=${UBUNTU_GOLD_IMAGE}" ;;
           esac
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
           timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
-          tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1
+          # shellcheck disable=SC2086
+          retry_run "tofu destroy" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
 
       - name: Upload tofu log
-        if: always() && steps.gate.outputs.selected == 'true'
+        if: always()
         uses: actions/upload-artifact@v7
         with:
           name: tofu-log-cell-${{ matrix.cell }}-${{ github.run_id }}
@@ -410,7 +822,7 @@ jobs:
 
   # Bare-metal single-node suite. Doesn't fit the tofu matrix shape, so it
   # runs as a sibling job via the reusable e2e-baremetal workflow. Gated like a
-  # matrix cell ("cell 20"): every weeknight cron, or dispatch with 20 selected
+  # matrix cell ("cell 20"): every nightly cron, or dispatch with 20 selected
   # (blank cells = all). Tests `dev` on schedule to match GIT_BRANCH above.
   baremetal:
     name: cell-20 (baremetal/pxe/real-hw)
@@ -430,6 +842,34 @@ jobs:
       otel-attrs: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=20,ci.matrix=baremetal/pxe/real-hw
     secrets: inherit
 
+  # The guest disk-performance gate. Same bare-metal node as cell 20 and
+  # therefore queued behind it by the shared concurrency group, which is the
+  # point: perf numbers off a cell still running other suites are noise, and
+  # the availability assertions measure host responsiveness under a single
+  # tenant's load, which a co-tenant correctness suite would confound.
+  #
+  # It reimages the cluster first, like every other caller of that workflow, so
+  # it is a fresh node rather than whatever cell-20 left behind.
+  diskperf:
+    name: cell-28 (baremetal/diskperf)
+    needs: baremetal
+    if: >-
+      !cancelled() && (
+        github.event_name == 'schedule' ||
+        inputs.cells == '' ||
+        contains(format(',{0},', inputs.cells), ',28,')
+      )
+    permissions:
+      contents: read
+      statuses: write
+    uses: ./.github/workflows/e2e-baremetal.yml
+    with:
+      ref: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+      cell: "28"
+      suites: diskperf
+      otel-attrs: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=28,ci.matrix=baremetal/pxe/diskperf
+    secrets: inherit
+
   # ISO cells. Unlike 1-19 these need no tofu: the harnesses run QEMU directly
   # on the runner. The nodes are therefore L1, which keeps the instances they
   # launch at L2 and their benchmark numbers comparable between the two cells.
@@ -496,12 +936,13 @@ jobs:
 
       - uses: actions/checkout@v7
         with:
+          ref: ${{ env.GIT_BRANCH }}
           path: mulga/spinifex
 
       - name: Resolve sibling refs
         id: siblings
         env:
-          BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+          BRANCH: ${{ env.GIT_BRANCH }}
         run: bash .github/resolve-sibling-refs.sh
         working-directory: mulga/spinifex
 
@@ -615,6 +1056,22 @@ jobs:
           EOF
           ./test/test-workload.sh --env /tmp/cell21.env --benchmark
 
+      # The node's own journal, which nothing else preserves: the serial log
+      # stops at login and the harness owns the only node there is. Pulled
+      # before teardown and never fatal, so a passing night keeps it too.
+      - name: Collect node journal
+        if: always() && steps.smoke.outputs.node_ip != ''
+        env:
+          NODE_IP: ${{ steps.smoke.outputs.node_ip }}
+        run: |
+          mkdir -p /tmp/cell21-artifacts-$GITHUB_RUN_ID
+          timeout 180 ssh -o BatchMode=yes -o StrictHostKeyChecking=no \
+            -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 \
+            -i "$HOME/.ssh/spinifex-key" "spinifex@${NODE_IP}" \
+            'sudo journalctl -b --no-pager --output=short-iso | gzip -6' \
+            > "/tmp/cell21-artifacts-$GITHUB_RUN_ID/node-journal.log.gz" ||
+            echo "::warning::could not pull the node journal from ${NODE_IP}"
+
       - name: Collect logs
         if: always()
         run: |
@@ -678,7 +1135,7 @@ jobs:
     # them together would need 32 GiB on one box.
     needs: [iso-single]
     if: >-
-      always() && (
+      !cancelled() && (
       github.event_name == 'schedule' ||
       inputs.cells == '' ||
       contains(format(',{0},', inputs.cells), ',22,') )
@@ -727,12 +1184,13 @@ jobs:
 
       - uses: actions/checkout@v7
         with:
+          ref: ${{ env.GIT_BRANCH }}
           path: mulga/spinifex
 
       - name: Resolve sibling refs
         id: siblings
         env:
-          BRANCH: ${{ github.event_name == 'schedule' && 'dev' || github.ref_name }}
+          BRANCH: ${{ env.GIT_BRANCH }}
         run: bash .github/resolve-sibling-refs.sh
         working-directory: mulga/spinifex
 
@@ -832,13 +1290,14 @@ jobs:
   # bottlebrush, so "a different label" would not have separated them. needs: on
   # a matrix job waits for every leg, so this starts on a quiet machine.
   #
-  # always() because a failed cell must not silently skip the benchmark and
-  # leave a green-looking night with no number in it.
+  # !cancelled() rather than always(): a failed cell must not silently skip the
+  # benchmark and leave a green-looking night with no number in it, but always()
+  # also overrides cancellation and keeps the run starting jobs after a cancel.
   performance:
     name: cell-23 (predastore/throughput)
     needs: [matrix, baremetal, iso-single, iso-multi]
     if: >-
-      always() && (
+      !cancelled() && (
       github.event_name == 'schedule' ||
       inputs.cells == '' ||
       contains(format(',{0},', inputs.cells), ',23,') )
@@ -950,12 +1409,15 @@ jobs:
     name: cell-24 (predastore/fault-injection)
     needs: [matrix, baremetal, iso-single, iso-multi, performance]
     if: >-
-      always() && (
+      !cancelled() && (
       github.event_name == 'schedule' ||
       inputs.cells == '' ||
       contains(format(',{0},', inputs.cells), ',24,') )
     runs-on: [self-hosted, ci-single]
-    timeout-minutes: 30
+    # The thirteen scenarios run serially and are ~35 minutes of work on their
+    # own, measured from the per-scenario predastore matrix, before checkout
+    # and build. 30 could not have passed whatever the disk did.
+    timeout-minutes: 75
     env:
       # Freezes a follower. STRESS_HOST=leader is a known failure: a gate keeps
       # the frozen node first in its meta read order and pays the full client
@@ -968,12 +1430,20 @@ jobs:
       - name: Clean Workspace
         run: sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
 
+      # STRESS_WORK_ROOT is set as well as TMPDIR because the harness ignores
+      # TMPDIR for the scenarios that need the most disk: large-object and
+      # multipart-upload default to $HOME/.cache/predastore-e2e, on the theory
+      # that TMPDIR is commonly tmpfs. Here it is a 938G nvme, so pointing them
+      # at it is both correct and the only path with room.
       - name: Configure work directories
         run: |
           E2E_ROOT=/mnt/nvme/e2e
-          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
-            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          for d in work stress-work; do
+            mkdir -p "$E2E_ROOT/$d" 2>/dev/null ||
+              { sudo mkdir -p "$E2E_ROOT/$d" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/$d"; }
+          done
           echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+          echo "STRESS_WORK_ROOT=$E2E_ROOT/stress-work" >> "$GITHUB_ENV"
           echo "STRESS_RESULTS_ROOT=$E2E_ROOT/predastore-stress-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
 
       - name: Checkout Spinifex
@@ -1001,15 +1471,25 @@ jobs:
       - name: Preflight
         run: |
           set -euo pipefail
-          mkdir -p "$TMPDIR"
-          find "$TMPDIR" -maxdepth 1 -name 'predastore-e2e-stress.*' \
-               -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
-          df -h "$TMPDIR"
-          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
-          if [ "$avail_gb" -lt 10 ]; then
-            echo "::error::$TMPDIR has ${avail_gb} GiB free; the run needs headroom above predastore's 5% watermark."
-            exit 1
-          fi
+          mkdir -p "$TMPDIR" "$STRESS_WORK_ROOT"
+          for d in "$TMPDIR" "$STRESS_WORK_ROOT" "$HOME/.cache/predastore-e2e"; do
+            [ -d "$d" ] || continue
+            find "$d" -maxdepth 1 -name 'predastore-e2e-stress.*' \
+                 -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
+          done
+          # Both paths, because they can be different filesystems and the one
+          # holding the large objects is not the one TMPDIR names. large-object
+          # needs 2GiB and 4GiB sharded with parity, multipart 1GiBx8.
+          rc=0
+          for d in "$TMPDIR" "$STRESS_WORK_ROOT"; do
+            df -h "$d"
+            avail_gb=$(df -BG --output=avail "$d" | tail -1 | tr -dc '0-9')
+            if [ "$avail_gb" -lt 40 ]; then
+              echo "::error::$d has ${avail_gb} GiB free; the run needs headroom above predastore's 5% watermark and room for the 1GiBx8 multipart case."
+              rc=1
+            fi
+          done
+          exit "$rc"
 
       # Leftovers from a run killed before its cleanup trap: a surviving node
       # holds its ports and fails this run somewhere unrelated.
@@ -1039,3 +1519,40 @@ jobs:
         run: |
           pkill -KILL -f '/bin/s3d -config' || true
           rm -rf "$STRESS_RESULTS_ROOT"
+
+  # Reading a nightly meant opening every cell job, and the Actions UI does not
+  # separate a cell that failed from one cancelled out from under it. needs:
+  # every cell so this runs last; always() so a red run is the one it describes.
+  summary:
+    name: run summary
+    needs:
+      [
+        select,
+        matrix,
+        baremetal,
+        diskperf,
+        iso-single,
+        iso-multi,
+        performance,
+        stress,
+      ]
+    if: always()
+    runs-on: ubuntu-26.04
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          sparse-checkout: .github/scripts/e2e-run-summary.py
+          sparse-checkout-cone-mode: false
+
+      - name: Collect junit
+        uses: actions/download-artifact@v8
+        with:
+          pattern: junit-cell-*-${{ github.run_id }}
+          path: junit
+        continue-on-error: true
+
+      - name: Write run summary
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          JUNIT_ROOT: junit
+        run: python3 .github/scripts/e2e-run-summary.py

.github/workflows/e2e-suites.yml

@@ -43,18 +43,22 @@ on:
         required: false
         default: ""
 
+# Shared with e2e-nightly.yml and e2e-ddil.yml: every one of them provisions
+# many cells on the same runner pools and the same hypervisors, so two in
+# flight compete for their disks and their libvirt guests. Queued rather than
+# cancelled — a run in progress is someone's result, not a stale one. e2e.yml
+# is outside the group: one cluster per run is cheap enough to overlap.
+concurrency:
+  group: e2e-hypervisors
+  cancel-in-progress: false
+
 permissions:
   contents: read
-  # actions: read lets the otel-job-id action look up a provision job's numeric
-  # id via the Actions API — github.job is only the YAML key, no context
-  # exposes it.
   actions: read
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
   TEST_RUN: ${{ inputs.test_run || '' }}
-  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
-  # empty string disables telemetry entirely.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
@@ -73,12 +77,6 @@ jobs:
       matrix: ${{ steps.suites.outputs.matrix }}
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
-      # This job (not single_suite) is where "Bootstrap via Binary Install"
-      # bakes SPINIFEX_OTEL_ATTRS into the VM's telemetry.env, before the
-      # suite matrix even exists — single_suite's legs share this one
-      # already-booted VM, so daemon telemetry can't be scoped any finer
-      # than "single env" per suite leg. ci.cell=single distinguishes it
-      # from multi_provision's VMs within the same run id.
       SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=single
     steps:
       - name: Reap Orphan Procs
@@ -143,10 +141,9 @@ jobs:
           LIST="${REQUESTED:-${E2E_SUITES_SINGLE}}"
           e2e_suites_assert_subset "single_suites" "${LIST}" "${E2E_SUITES_SINGLE}"
           echo "list=${LIST}" >> "$GITHUB_OUTPUT"
-          # Unquoted on purpose: the space-separated list must word-split into
-          # one JSON element per suite.
-          # shellcheck disable=SC2086
-          JSON=$(printf '%s\n' ${LIST} | jq -R . | jq -cs .)
+          # 45 is the floor this job has always had, so resolving each leg's
+          # budget can only raise it — storagefault and rds ask for more.
+          JSON=$(e2e_suite_matrix "${LIST}" 45)
           echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
 
       - name: Reclaim docker build cache
@@ -182,7 +179,7 @@ jobs:
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "VM provisioned"
 
       - name: Bootstrap via Binary Install
@@ -311,14 +308,16 @@ jobs:
           retention-days: 14
 
   single_suite:
-    name: Single env · suite
+    name: Single env · ${{ matrix.suite }}
     needs: single_provision
     runs-on: [self-hosted, ci-single]
-    timeout-minutes: 45
+    # Per leg, from suite-sets.sh: one number for every suite would have to be
+    # storagefault's, and every other leg would then hang for 105 minutes.
+    timeout-minutes: ${{ matrix.timeout }}
     strategy:
       fail-fast: false
       matrix:
-        suite: ${{ fromJSON(needs.single_provision.outputs.matrix) }}
+        include: ${{ fromJSON(needs.single_provision.outputs.matrix) }}
     env:
       WAN_IP: ${{ needs.single_provision.outputs.wan_ip }}
       SSH_KEY: ${{ needs.single_provision.outputs.ssh_key }}
@@ -343,6 +342,8 @@ jobs:
         run: |
           set +e
           SUITE="${{ matrix.suite }}"
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          TMO="$(e2e_suite_timeout "${SUITE}")"
           VM_HOST="tf-user@${WAN_IP}"
           TEST_RUN_FLAG=""
           [ -n "${TEST_RUN}" ] && TEST_RUN_FLAG="-test.run=${TEST_RUN}"
@@ -360,7 +361,7 @@ jobs:
              GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                SPINIFEX_E2E_RUN_ID='${{ github.run_id }}' \
                ARTIFACT_DIR='${ARTIFACT_DIR}' \
-               ./${SUITE}.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
+               ./${SUITE}.test -test.v -test.timeout=${TMO} ${TEST_RUN_FLAG} \
                2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log' \
                     | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
              exit \${PIPESTATUS[0]}"
@@ -394,6 +395,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "E2E ${{ matrix.suite }} results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: single
+
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -506,7 +517,7 @@ jobs:
           find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 
@@ -617,7 +628,7 @@ jobs:
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "3-node cluster provisioned"
 
       - name: Bootstrap via Binary Install
@@ -659,10 +670,9 @@ jobs:
           LIST="${REQUESTED:-${E2E_SUITES_MULTI}}"
           e2e_suites_assert_subset "multi_suites" "${LIST}" "${E2E_SUITES_MULTI}"
           echo "list=${LIST}" >> "$GITHUB_OUTPUT"
-          # Unquoted on purpose: the space-separated list must word-split into
-          # one JSON element per suite.
-          # shellcheck disable=SC2086
-          JSON=$(printf '%s\n' ${LIST} | jq -R . | jq -cs .)
+          # 60 is the floor this job has always had; multi-node legs carry a
+          # heavier setup than single-node ones, which is why it is higher.
+          JSON=$(e2e_suite_matrix "${LIST}" 60)
           echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
 
       - name: Build multinode binaries on runner
@@ -712,14 +722,15 @@ jobs:
           retention-days: 14
 
   multi_suite:
-    name: Multi env · suite
+    name: Multi env · ${{ matrix.suite }}
     needs: multi_provision
     runs-on: [self-hosted, ci-multi]
-    timeout-minutes: 60
+    # See the note on single_suite: the budget is the suite's, not the job's.
+    timeout-minutes: ${{ matrix.timeout }}
     strategy:
       fail-fast: false
       matrix:
-        suite: ${{ fromJSON(needs.multi_provision.outputs.matrix) }}
+        include: ${{ fromJSON(needs.multi_provision.outputs.matrix) }}
     env:
       NODE1_IP: ${{ needs.multi_provision.outputs.node1_ip }}
       SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }}
@@ -745,6 +756,8 @@ jobs:
         run: |
           set +e
           SUITE="${{ matrix.suite }}"
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          TMO="$(e2e_suite_timeout "${SUITE}")"
           VM_HOST="tf-user@${NODE1_IP}"
           TEST_RUN_FLAG=""
           [ -n "${TEST_RUN}" ] && TEST_RUN_FLAG="-test.run=${TEST_RUN}"
@@ -763,7 +776,7 @@ jobs:
                SPINIFEX_NODES='${NODES_CSV}' SPINIFEX_NODE_IPS='${NODES_CSV}' \
                SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
                ARTIFACT_DIR='${ARTIFACT_DIR}' \
-               ./${SUITE}.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
+               ./${SUITE}.test -test.v -test.timeout=${TMO} ${TEST_RUN_FLAG} \
                2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log' \
                     | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
              exit \${PIPESTATUS[0]}"
@@ -796,6 +809,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "Multi-Node E2E ${{ matrix.suite }} results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: multi
+
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -879,7 +902,7 @@ jobs:
           find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 

.github/workflows/e2e.yml

@@ -7,6 +7,7 @@ on:
       - dev
     paths:
       - "**/*.go"
+      - "!**/*_test.go"
       - go.mod
       - go.sum
       - Makefile
@@ -26,20 +27,67 @@ on:
 permissions:
   contents: read
   statuses: write
-  # actions: read lets the otel-job-id action look up a job's numeric id via
-  # the Actions API — github.job is only the YAML key, no context exposes it.
   actions: read
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
   SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb e2e-iam e2e-partialblock' }}
   TEST_RUN: ${{ inputs.test_run || '' }}
-  # OTLP export to the central Elastic sink (wattle). Repo variable overrides;
-  # empty string disables telemetry entirely.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
+  # The inner timeout must fire before the outer one. `go test` reports where it
+  # stopped and the suite's own failure; a job cancelled on timeout reports only
+  # "The operation was canceled" and takes the artifacts with it.
+  #
+  # A job budget cannot be a literal here because the suite set is a dispatch
+  # input: storagefault asks for 90m and diskperf for 180m, both eligible on one
+  # node, so any fixed number is either wrong for them or wasteful for everyone
+  # else. Resolving it from suite-sets.sh keeps the one source of truth and
+  # leaves the default push set at exactly the 60m it has always had.
+  plan:
+    name: Resolve Job Budgets
+    runs-on: ubuntu-26.04
+    timeout-minutes: 5
+    outputs:
+      single_timeout: ${{ steps.budgets.outputs.single_timeout }}
+      multi_timeout: ${{ steps.budgets.outputs.multi_timeout }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          path: spinifex
+          sparse-checkout: .github/scripts
+
+      - name: Resolve Budgets
+        id: budgets
+        env:
+          # Provision, install and teardown around the suites, plus the headroom
+          # a multi-suite job needs for whatever ran before the one that hangs.
+          SETUP_ALLOWANCE_MIN: 15
+          # What the job has always had. Kept as a floor so resolving the budget
+          # can only ever raise it, never quietly shorten an existing run.
+          FLOOR_MIN: 60
+        run: |
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          # Only max() is a sound bound: the suites share one job, so a sum would
+          # demand hours for a set that has never taken them. Max is what makes
+          # the hanging suite panic on its own timeout first, which is the point.
+          budget() {
+            local want="${FLOOR_MIN}" s t
+            for s in $(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE "$1"); do
+              t="$(e2e_suite_timeout "${s}")"
+              t=$(( ${t%m} + SETUP_ALLOWANCE_MIN ))
+              [ "${t}" -gt "${want}" ] && want="${t}"
+            done
+            echo "${want}"
+          }
+          single=$(budget "${E2E_SUITES_SINGLE_RE}")
+          multi=$(budget "${E2E_SUITES_MULTI_RE}")
+          echo "single_timeout=${single}" >> "$GITHUB_OUTPUT"
+          echo "multi_timeout=${multi}" >> "$GITHUB_OUTPUT"
+          echo "suites='${SUITES}' -> single ${single}m, multi ${multi}m"
+
   e2e_single:
     name: Single-Node E2E
     # The dispatch check gates on exactly the suites this job's Run step
@@ -51,8 +99,9 @@ jobs:
     # tear down. Push triggers keep running via the null-check on
     # inputs.suites.
     if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-iam') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-ecs') || contains(github.event.inputs.suites, 'e2e-storagegrowth') || contains(github.event.inputs.suites, 'e2e-partialblock') || contains(github.event.inputs.suites, 'e2e-rds')
+    needs: plan
     runs-on: [self-hosted, ci-single]
-    timeout-minutes: 60
+    timeout-minutes: ${{ fromJSON(needs.plan.outputs.single_timeout) }}
     env:
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
@@ -152,7 +201,7 @@ jobs:
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "VM provisioned"
 
       - name: Bootstrap via Binary Install
@@ -357,6 +406,10 @@ jobs:
           # suite-sets.sh also carries the per-suite rationale.
           . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
           SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE "${E2E_SUITES_SINGLE_RE}" | tr '\n' ' ')
+          # Resolve each suite's timeout here, where suite-sets.sh is readable, and
+          # carry the pairs into the remote loop below — the VM has no checkout.
+          SUITE_TMOS=""
+          for s in ${SUITES_RUN}; do SUITE_TMOS="${SUITE_TMOS}${s}:$(e2e_suite_timeout "${s}") "; done
           if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
             echo "no in-VM suites requested (SUITES=${SUITES}); skipping"
             echo "rc=0" >> "$GITHUB_OUTPUT"
@@ -388,7 +441,7 @@ jobs:
                echo \"::group::suite \$SUITE\"; \
                printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
-               case \$SUITE in rds) TMO=50m ;; *) TMO=30m ;; esac; \
+               TMO=30m; for kv in ${SUITE_TMOS}; do case \$kv in \$SUITE:*) TMO=\${kv#*:} ;; esac; done; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
                  ./\$SUITE.test -test.v -test.timeout=\$TMO ${TEST_RUN_FLAG} \
@@ -464,6 +517,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "E2E results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: single
+
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -510,7 +573,7 @@ jobs:
           find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 
@@ -539,8 +602,9 @@ jobs:
     # provisions the 3-node env when requested on its own, and the env stays
     # unprovisioned when none of them are asked for.
     if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-multinode') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-cert')
+    needs: plan
     runs-on: [self-hosted, ci-multi]
-    timeout-minutes: 60
+    timeout-minutes: ${{ fromJSON(needs.plan.outputs.multi_timeout) }}
     env:
       TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-multi-artifacts-${{ github.run_id }}
@@ -636,7 +700,7 @@ jobs:
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
           quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ok "3-node cluster provisioned"
 
       - name: Bootstrap via Binary Install
@@ -762,6 +826,8 @@ jobs:
           # Allow-list, not a deny-list — see the single-node job for why.
           . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
           SUITES_RUN=$(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE "${E2E_SUITES_MULTI_RE}" | tr '\n' ' ')
+          SUITE_TMOS=""
+          for s in ${SUITES_RUN}; do SUITE_TMOS="${SUITE_TMOS}${s}:$(e2e_suite_timeout "${s}") "; done
           if [ -z "$(echo "${SUITES_RUN}" | tr -d ' ')" ]; then
             echo "no multinode-eligible suites requested (SUITES=${SUITES}); skipping"
             echo "rc=0" >> "$GITHUB_OUTPUT"
@@ -793,13 +859,14 @@ jobs:
                echo \"::group::suite \$SUITE\"; \
                printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
+               TMO=30m; for kv in ${SUITE_TMOS}; do case \$kv in \$SUITE:*) TMO=\${kv#*:} ;; esac; done; \
                GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=multinode \
                  SPINIFEX_NODES='${NODES_CSV}' \
                  SPINIFEX_NODE_IPS='${NODES_CSV}' \
                  SPINIFEX_SSH_USER=tf-user \
                  SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
-                 ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \
+                 ./\$SUITE.test -test.v -test.timeout=\$TMO ${TEST_RUN_FLAG} \
                  2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \
                       | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \
                rc=\${PIPESTATUS[0]}; \
@@ -903,6 +970,16 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "Multi-Node E2E results"
 
+      # Tolerated on failure: the sink being unreachable must not turn a
+      # green test run red.
+      - name: Ship test results to Elasticsearch
+        if: always()
+        continue-on-error: true
+        uses: ./spinifex/.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: multi
+
       - name: Analyze e2e failures
         if: always()
         uses: ./spinifex/.github/actions/e2e-analyze
@@ -952,7 +1029,7 @@ jobs:
           find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
-          quiet_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
           ok "teardown complete"
 

.github/workflows/gpu-e2e.yml

@@ -56,15 +56,7 @@ concurrency:
   cancel-in-progress: false
 
 env:
-  # OTLP export to the central Elastic sink (wattle is the sink itself — see
-  # Part A.2). Repo variable overrides; carried for correlation even though
-  # the GPU suite's own daemon-side spans are emitted by wattle's already-
-  # configured obs-agent, not by this job.
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
-  # ci.cell=gpu keeps this workflow's telemetry separable from the tofu-env
-  # workflows'. Nothing in this job bakes these attrs into a daemon (see the
-  # note above), so the label only takes effect once wattle's obs-agent is
-  # given them; there is deliberately no ci.job_id lookup for the same reason.
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=gpu
   WATTLE_SSH_HOST: ${{ vars.WATTLE_SSH_HOST || '192.168.1.13' }}
   WATTLE_SSH_USER: ${{ vars.WATTLE_SSH_USER || 'tf-user' }}
@@ -78,8 +70,6 @@ jobs:
     env:
       ARTIFACT_DIR: /tmp/spinifex-gpu-e2e-artifacts-${{ github.run_id }}
     steps:
-      # Manual-work mutex gate — see setup note 4 above. Checked BEFORE any
-      # SSH to wattle so a paused run touches the box zero times.
       - name: Manual-work mutex gate
         id: gate
         env:
@@ -122,8 +112,6 @@ jobs:
           echo "${{ secrets.WATTLE_SSH_KEY }}" > "${RUNNER_TEMP}/wattle-ci"
           echo "ssh_opts=-o StrictHostKeyChecking=no -o ConnectTimeout=10 -i ${RUNNER_TEMP}/wattle-ci" >> "$GITHUB_ENV"
 
-      # Healthcheck: confirm wattle's persistent AWS-compat control plane is
-      # actually up before spending time building/shipping the test binary.
       - name: Healthcheck — describe-regions
         if: steps.gate.outputs.paused != 'true'
         run: |
@@ -131,18 +119,6 @@ jobs:
           ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" \
             'AWS_PROFILE=spinifex aws ec2 describe-regions --output text'
 
-      # Ensure the base EC2 GPU AMI is present — the raw-EC2 GPU passthrough
-      # test is the always-on baseline, so CI self-provisions it rather than
-      # depending on a manual import. `--name` selects the catalog entry (pulls
-      # the prebuilt image from iso.mulgadc.com); `--ami-name` is REQUIRED to
-      # name the AMI ubuntu-26.04-nvidia-gpu-x86_64 — without it the import
-      # falls back to the generic ami-<distro>-<version>-<arch>, which the
-      # tests/e2e/gpu filter would never match. Idempotent — skipped once
-      # wattle's persistent loop-fs already has the named AMI (the 4.4G source
-      # qcow2 is cached in the images dir, so a re-import wouldn't re-download).
-      # `spx admin images import` writes local spinifex storage, so it needs
-      # the `spinifex` group active (sg); aws-CLI calls hit the gateway over
-      # HTTPS and don't.
       - name: Ensure base GPU AMI present (import-if-absent)
         if: steps.gate.outputs.paused != 'true'
         run: |
@@ -171,11 +147,6 @@ jobs:
           '
           ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
 
-      # Preflight — the base GPU AMI is REQUIRED: the setup step above just
-      # ensured it, so a miss here means the import silently failed. Hard-fail
-      # rather than run a false-green all-skipped suite. ECS/EKS node AMIs are
-      # opt-in tiers (heavier — node AMIs + cluster spin-up); their tests
-      # self-skip when absent, so only warn.
       - name: Preflight — verify GPU AMIs
         if: steps.gate.outputs.paused != 'true'
         run: |
@@ -205,10 +176,6 @@ jobs:
           '
           ssh ${{ env.ssh_opts }} "${WATTLE_SSH_USER}@${WATTLE_SSH_HOST}" "$REMOTE"
 
-      # Same build command CI's other e2e workflows use to cross-compile a
-      # suite into a shippable binary (tests/e2e/Makefile e2e-build target,
-      # SUITES_BUILD scoped to just gpu). Building on the runner keeps go
-      # toolchain / module cache off wattle entirely.
       - name: Build GPU e2e test binary on runner
         if: steps.gate.outputs.paused != 'true'
         working-directory: tests/e2e
@@ -231,12 +198,6 @@ jobs:
           ssh ${{ env.ssh_opts }} "$VM" "mkdir -p \$HOME/spinifex-gpu-e2e-bin"
           scp ${{ env.ssh_opts }} tests/e2e/_bin/gpu.test "$VM":spinifex-gpu-e2e-bin/gpu.test
 
-      # Same invocation shape e2e.yml uses to run a pre-built suite binary
-      # (SPINIFEX_E2E=1 SPINIFEX_MODE=single, ./<suite>.test -test.v
-      # -test.timeout=NNm), plus SPINIFEX_WAN_IP: wattle's ~24 bridges make
-      # net.InterfaceAddrs() enumeration pick a co-tenant bridge instead of
-      # the advertised gateway, so the override is required here. The harness
-      # should prefer spinifex.toml's advertised bridge.
       - name: Run GPU e2e suite on wattle
         id: e2e
         if: steps.gate.outputs.paused != 'true'
@@ -287,6 +248,14 @@ jobs:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
           title: "GPU E2E results"
 
+      - name: Ship test results to Elasticsearch
+        if: always() && steps.gate.outputs.paused != 'true'
+        continue-on-error: true
+        uses: ./.github/actions/junit-to-es
+        with:
+          junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
+          cell: gpu
+
       - name: Analyze e2e failures
         if: always() && steps.gate.outputs.paused != 'true'
         uses: ./.github/actions/e2e-analyze
@@ -303,14 +272,6 @@ jobs:
           if-no-files-found: warn
           retention-days: 14
 
-      # Cleanup is NEVER a host teardown: it never runs setup-ovn.sh,
-      # dev-reset, or cluster-reset. It only reconciles EC2-API-level state —
-      # terminate any GPU-typed instance still pending/running (the Go
-      # suite's own t.Cleanup already does this on normal pass/fail exit;
-      # this is the backstop for a hard `-test.timeout` kill or a workflow
-      # cancellation, which skip deferred Go cleanups). Termination triggers
-      # gpuManager.Release() on wattle, returning the A1000 to the pool for
-      # the next queued run or manual test.
       - name: Cleanup — terminate leftover GPU instances, release GPU
         if: always() && steps.gate.outputs.paused != 'true'
         run: |

.github/workflows/image-scripts.yml

@@ -1,9 +1,5 @@
 name: Image Scripts
 
-# Separate from spinifex.yml on purpose: scripts/images/ and images/ are shell
-# and mkosi assets baked into system images, not Go, so they get their own
-# path-filtered gate instead of running on every Go contributor's push (see
-# ui.yml for the same pattern applied to the frontend).
 on:
   push:
     branches:
@@ -37,3 +33,25 @@ jobs:
 
       - name: Run image script tests + shellcheck
         run: make test-images
+
+  eks_mkosi_image:
+    name: Build mkosi EKS image
+    runs-on: [self-hosted, ci-single]
+    concurrency:
+      group: eks-mkosi-image-${{ github.ref }}
+      cancel-in-progress: true
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7
+
+      - name: Set up Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: "go.mod"
+          # Self-hosted: the module + build cache already persists on local
+          # disk, and cache-save would re-upload hundreds of MB for nothing.
+          cache: false
+
+      - name: Build the image
+        run: make build-eks-node-mkosi-image

.github/workflows/perf-3node.yml

@@ -0,0 +1,415 @@
+name: Predastore 3-Node Performance
+
+on:
+  workflow_dispatch:
+    inputs:
+      perf_preset:
+        description: "Warp workload preset: smoke (30s) or compare (2m)"
+        required: false
+        default: compare
+        type: choice
+        options:
+          - smoke
+          - compare
+      cluster_ref:
+        description: "Git ref to build the cluster from, e.g. v1.18.0. Empty uses this branch."
+        required: false
+        default: ""
+        type: string
+
+# Shared with e2e.yml and e2e-nightly.yml: this job provisions on the same
+# hypervisors they do, so a run in flight there and here would compete for
+# disks and libvirt guests. Queued, not cancelled, same as those workflows.
+concurrency:
+  group: e2e-hypervisors
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+env:
+  GIT_BRANCH: ${{ github.ref_name }}
+  SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
+
+jobs:
+  perf_3node:
+    name: Predastore Throughput (3-node)
+    # ci-multi is a systemd runner on the LAN, not the ephemeral pool. A pool
+    # VM sits behind the geneve overlay at MTU 1408, which would put the
+    # runner's own network path inside the thing being measured.
+    runs-on: [self-hosted, ci-multi]
+    timeout-minutes: 90
+    env:
+      TOFU_LOG: /tmp/e2e-tofu-perf3node-${{ github.run_id }}.log
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=perf-3node
+    steps:
+      - name: Reap Orphan Procs
+        run: |
+          set +e
+          UID_SELF=$(id -u)
+          WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in
+                "$WS"|"$WS"/*) echo "reap $sig pid=$pid cwd=$cwd"; kill "-$sig" "$pid" 2>/dev/null ;;
+              esac
+            done
+          }
+          reap TERM
+          sleep 3
+          reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          true
+
+      - name: Clean Workspace
+        run: |
+          awk -v w="$GITHUB_WORKSPACE" '$2 ~ "^"w {print $2}' /proc/mounts \
+            | sort -r | xargs -r -I{} sudo umount -l -R {} || true
+          sudo rm -rf --one-file-system "$GITHUB_WORKSPACE"/*
+
+      # ci-multi and ci-single resolve to the same physical host, so the
+      # nvme path cell-23 uses for its throughput run is already there.
+      # Pointing TMPDIR at it keeps warp's payloads off a small /tmp tmpfs.
+      - name: Configure work directories
+        run: |
+          E2E_ROOT=/mnt/nvme/e2e
+          mkdir -p "$E2E_ROOT/work" 2>/dev/null ||
+            { sudo mkdir -p "$E2E_ROOT/work" && sudo chown "$(id -u):$(id -g)" "$E2E_ROOT" "$E2E_ROOT/work"; }
+          echo "TMPDIR=$E2E_ROOT/work" >> "$GITHUB_ENV"
+          echo "PERF_RESULTS_ROOT=$E2E_ROOT/predastore-perf-3node-$GITHUB_RUN_ID" >> "$GITHUB_ENV"
+
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+          repositories: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          repository: mulgadc/mulga
+          token: ${{ steps.app-token.outputs.token }}
+          sparse-checkout: scripts/tofu-cluster
+          path: mulga
+
+      - uses: actions/checkout@v7
+        with:
+          path: spinifex
+
+      - uses: actions/setup-go@v7
+        with:
+          go-version-file: spinifex/go.mod
+          cache: false
+
+      - name: Re-enable Go toolchain auto-download
+        run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
+
+      # Same-named branch when one exists, dev otherwise. Matches how the
+      # e2e path picks siblings, so this measures the same predastore
+      # commit a cross-repo change would be built and tested against.
+      - name: Resolve sibling refs
+        id: siblings
+        env:
+          BRANCH: ${{ env.GIT_BRANCH }}
+        run: bash .github/resolve-sibling-refs.sh
+        working-directory: spinifex
+
+      - uses: actions/checkout@v7
+        id: predastore
+        with:
+          repository: mulgadc/predastore
+          ref: ${{ steps.siblings.outputs.predastore }}
+          path: predastore
+
+      # The cluster's ref is a knob and the harness's is not. Measuring an old
+      # release with a new harness is the only way its number can be set beside
+      # a current one; rebuilding both would change the ruler and the thing.
+      - name: Build Install Artifacts
+        working-directory: mulga/scripts/tofu-cluster
+        env:
+          CLUSTER_REF: ${{ inputs.cluster_ref }}
+        run: |
+          ./build-install-artifacts.sh "${CLUSTER_REF:-$GIT_BRANCH}" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-tests.tar" \
+            --skip-e2e-build
+
+      # The libvirt provider dials the hypervisors by bare hostname. A pool VM
+      # sits inside a VPC and cannot resolve them; a LAN runner already can, so
+      # this is self-gating on the lookup rather than on the runner's labels.
+      - name: Map hypervisor names
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          if getent hosts banksia >/dev/null; then
+            echo "hypervisor names already resolve"
+          else
+            sudo tee -a /etc/hosts < network/hosts >/dev/null
+            echo "added $(grep -c '^[0-9]' network/hosts) entries to /etc/hosts"
+          fi
+          # Pinned rather than StrictHostKeyChecking=no: these hops reach the
+          # hypervisors themselves, so an unverified key is the one place in
+          # this job where accepting anything would matter.
+          if ssh-keygen -F banksia >/dev/null 2>&1; then
+            echo "hypervisor host keys already known"
+          else
+            install -d -m 700 ~/.ssh
+            cat network/known_hosts >> ~/.ssh/known_hosts
+            chmod 600 ~/.ssh/known_hosts
+            echo "pinned $(grep -c '^[a-z]' network/known_hosts) host keys"
+          fi
+
+      - name: Provision VMs
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set -euo pipefail
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          export TF_VAR_git_branch="${GIT_BRANCH}"
+          quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
+          quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "tofu destroy (pre-clean)" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false || true
+          quiet_run "scrub-env (pre-clean)"    "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}" || true
+          retry_run "tofu apply"               "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          ok "3-node cluster provisioned"
+
+      - name: Bootstrap via Binary Install
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
+          ./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
+            "${RUNNER_TEMP}/setup.sh" \
+            "${RUNNER_TEMP}/spinifex-distro.tar.gz" \
+            --test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
+
+      - name: Resolve cluster SSH details
+        id: cluster
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          INVENTORY=$(tofu output -json inventory)
+          NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
+          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
+          SSH_KEY="${SSH_KEY/#\~/$HOME}"
+          SSH_USER=$(echo "$INVENTORY" | jq -r '.ssh_user')
+          REGION=$(echo "$INVENTORY" | jq -r '.region')
+          NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
+          NODES_CSV=""
+          for i in $(seq 0 $((NODE_COUNT - 1))); do
+            ip=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
+            if [ -z "$NODES_CSV" ]; then NODES_CSV="$ip"; else NODES_CSV="$NODES_CSV,$ip"; fi
+          done
+          echo "node1_ip=${NODE1_IP}" >> "$GITHUB_OUTPUT"
+          echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
+          echo "ssh_user=${SSH_USER}" >> "$GITHUB_OUTPUT"
+          echo "region=${REGION}" >> "$GITHUB_OUTPUT"
+          echo "nodes_csv=${NODES_CSV}" >> "$GITHUB_OUTPUT"
+
+      # PERF_EXTERNAL_HOSTS wants host:port, and the gate listens on 8443 on
+      # every node. run-info.txt from this run is then comparable with the
+      # nightly's loopback one, since the harness itself never changes.
+      - name: Resolve PERF_EXTERNAL_HOSTS
+        id: perf_hosts
+        run: |
+          set -euo pipefail
+          IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
+          HOSTS=""
+          for ip in "${NODE_LIST[@]}"; do
+            [ -z "$ip" ] && continue
+            if [ -z "$HOSTS" ]; then HOSTS="$ip:8443"; else HOSTS="$HOSTS,$ip:8443"; fi
+          done
+          echo "hosts=${HOSTS}" >> "$GITHUB_OUTPUT"
+
+      # A slow bootstrap should fail here, legibly, rather than inside warp
+      # with a wall of connection-refused errors. Bounded at 5 minutes per
+      # gate so a genuinely dead node still fails the job in finite time.
+      - name: Wait for gates to answer on :8443
+        run: |
+          set -euo pipefail
+          IFS=',' read -ra GATES <<< "${{ steps.perf_hosts.outputs.hosts }}"
+          for gate in "${GATES[@]}"; do
+            echo "waiting for https://$gate"
+            up=0
+            for i in $(seq 1 60); do
+              if curl -k -sS --max-time 5 -o /dev/null "https://$gate/"; then
+                up=1
+                break
+              fi
+              sleep 5
+            done
+            [ "$up" -eq 1 ] || { echo "::error::gate $gate never answered HTTPS on :8443"; exit 1; }
+            echo "gate $gate is up"
+          done
+
+      # Warp reads whole objects, so no workload here touches a ranged read and
+      # the throughput figures stay healthy while ranged GETs are broken. This
+      # probe is the only thing in CI that looks at them.
+      - name: Ranged GET probe
+        env:
+          GATE: ${{ steps.cluster.outputs.node1_ip }}
+          SSH_KEY: ${{ steps.cluster.outputs.ssh_key }}
+          SSH_USER: ${{ steps.cluster.outputs.ssh_user }}
+          NODE1_IP: ${{ steps.cluster.outputs.node1_ip }}
+          CLUSTER_REGION: ${{ steps.cluster.outputs.region }}
+        run: |
+          set -euo pipefail
+          CREDS=$(ssh -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+            "${SSH_USER}@${NODE1_IP}" 'cat ~/.aws/credentials')
+          AK=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
+          SK=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
+          echo "::add-mask::$SK"
+          B="ranged-probe-${GITHUB_RUN_ID}"
+          AWSX=(--region "$CLUSTER_REGION" --endpoint-url "https://${GATE}:8443" --no-verify-ssl)
+          export AWS_ACCESS_KEY_ID="$AK" AWS_SECRET_ACCESS_KEY="$SK"
+          aws s3 mb "s3://$B" "${AWSX[@]}"
+          trap 'aws s3 rb "s3://$B" --force "${AWSX[@]}" >/dev/null 2>&1 || true' EXIT
+          # Two sizes: read amplification makes a boundary-crossing range cost
+          # the whole object, so the fault shows as time tracking object size.
+          for mb in 64 256; do
+            head -c $((mb*1024*1024)) /dev/urandom > "${TMPDIR}/probe-${mb}.bin"
+            aws s3 cp "${TMPDIR}/probe-${mb}.bin" "s3://$B/probe-${mb}.bin" --only-show-errors "${AWSX[@]}"
+            rm -f "${TMPDIR}/probe-${mb}.bin"
+          done
+          t() {
+            curl -sk --aws-sigv4 "aws:amz:${CLUSTER_REGION}:s3" -u "$AK:$SK" \
+              -H "Range: bytes=0-$1" -o /dev/null -w '%{time_total}' \
+              --url "https://${GATE}:8443/${B}/probe-${2}.bin"
+          }
+          # 4194304 is the last byte inside the first 4 MiB block; one more
+          # crosses into the second and leaves the single-shard fast path.
+          IN64=$(t 4194303 64);  OUT64=$(t 4194304 64)
+          IN256=$(t 4194303 256); OUT256=$(t 4194304 256)
+          printf '%-10s in-block=%ss  crossing=%ss\n' "64MiB" "$IN64" "$OUT64"
+          printf '%-10s in-block=%ss  crossing=%ss\n' "256MiB" "$IN256" "$OUT256"
+          # The 4x object-size ratio is the signal, and it is compared against
+          # the crossing reads alone so a slow runner cancels out of both sides.
+          awk -v a="$OUT64" -v b="$OUT256" 'BEGIN{
+            r = (a > 0) ? b/a : 0
+            printf "crossing-range cost ratio 256MiB/64MiB = %.2f (object size ratio is 4.00)\n", r
+            if (r > 2.0) {
+              print "::error::a boundary-crossing ranged GET scales with object size, so it is reading the whole object"
+              exit 1
+            }
+          }'
+          echo "ranged GET does not scale with object size"
+
+      # The benchmark works in $TMPDIR and predastore refuses writes below a 5%
+      # free-space watermark, so this asserts the same headroom cell-23 does.
+      # tmpfs is rejected outright: that would measure RAM, not the gates.
+      - name: Preflight
+        run: |
+          set -euo pipefail
+          mkdir -p "$TMPDIR"
+          fstype=$(df --output=fstype "$TMPDIR" | tail -1 | tr -d ' ')
+          case "$fstype" in
+            tmpfs|ramfs)
+              echo "::error::$TMPDIR is $fstype — a storage benchmark there measures RAM"
+              exit 1 ;;
+          esac
+          find "$TMPDIR" -maxdepth 1 -name 'predastore-e2e-performance.*' \
+               -type d -mtime +0 -exec rm -rf {} + 2>/dev/null || true
+          df -h "$TMPDIR"
+          avail_gb=$(df -BG --output=avail "$TMPDIR" | tail -1 | tr -dc '0-9')
+          pct_used=$(df --output=pcent "$TMPDIR" | tail -1 | tr -dc '0-9')
+          if [ "$pct_used" -ge 90 ] || [ "$avail_gb" -lt 20 ]; then
+            echo "::error::$TMPDIR ($fstype) is ${pct_used}% full with ${avail_gb} GiB free."
+            echo "::error::The benchmark needs headroom above predastore's 5% watermark."
+            exit 1
+          fi
+
+      - name: Install warp
+        working-directory: predastore
+        run: make warp-install
+
+      # PERF_EXTERNAL_HOSTS skips profile selection, port offsetting, and
+      # cluster start/stop, and measures these three gates directly. Same
+      # workloads and analysis as the nightly loopback run, so the numbers
+      # from the two are directly comparable.
+      #
+      # The cluster's credentials are read and used inside this one step, the
+      # way e2e-ddil.yml does it: a step-level env: block is echoed verbatim
+      # into the run log, and a shell variable is not.
+      - name: Run 3-node throughput benchmark
+        working-directory: predastore
+        env:
+          PERF_EXTERNAL_HOSTS: ${{ steps.perf_hosts.outputs.hosts }}
+          PERF_CONFIGS: spinifex-3node
+          PERF_PRESET: ${{ inputs.perf_preset || 'compare' }}
+          # What is deployed, not what the harness was checked out at. With a
+          # cluster_ref those differ, and run-info.txt records the cluster.
+          PERF_EXTERNAL_SHA: ${{ inputs.cluster_ref || steps.predastore.outputs.commit }}
+          SSH_KEY: ${{ steps.cluster.outputs.ssh_key }}
+          SSH_USER: ${{ steps.cluster.outputs.ssh_user }}
+          NODE1_IP: ${{ steps.cluster.outputs.node1_ip }}
+          CLUSTER_REGION: ${{ steps.cluster.outputs.region }}
+        run: |
+          set -euo pipefail
+          # spx admin init mints a random pair at bootstrap and writes it to
+          # node1's ~/.aws/credentials. The benchmark's built-in key is the one
+          # the loopback configs trust, which this cluster has never heard of.
+          CREDS=$(ssh -i "${SSH_KEY}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+            "${SSH_USER}@${NODE1_IP}" 'cat ~/.aws/credentials')
+          PERF_ACCESS_KEY=$(echo "$CREDS" | awk -F '= *' '/^aws_access_key_id/ {print $2; exit}')
+          PERF_SECRET_KEY=$(echo "$CREDS" | awk -F '= *' '/^aws_secret_access_key/ {print $2; exit}')
+          if [ -z "$PERF_ACCESS_KEY" ] || [ -z "$PERF_SECRET_KEY" ]; then
+            echo "::error::could not read spinifex AWS credentials from ${NODE1_IP}"
+            exit 1
+          fi
+          echo "::add-mask::$PERF_SECRET_KEY"
+          mkdir -p "$PERF_RESULTS_ROOT"
+          PERF_ACCESS_KEY="$PERF_ACCESS_KEY" \
+          PERF_SECRET_KEY="$PERF_SECRET_KEY" \
+          PERF_REGION="$CLUSTER_REGION" \
+            make e2e-performance 2>&1 | tee "$PERF_RESULTS_ROOT/e2e-performance.log"
+
+      # Report-only, matching cell-23: the baseline comparison and any
+      # regression gate are future work once run-to-run spread is known from
+      # real bare-metal runs.
+      - uses: actions/upload-artifact@v7
+        if: always()
+        with:
+          name: predastore-perf-3node-${{ github.run_id }}
+          path: ${{ env.PERF_RESULTS_ROOT }}
+          if-no-files-found: warn
+          retention-days: 90
+
+      - name: Clean up results
+        if: always()
+        run: rm -rf "${PERF_RESULTS_ROOT}"
+
+      - name: Teardown
+        if: always()
+        working-directory: mulga/scripts/tofu-cluster
+        run: |
+          set +e
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          UID_SELF=$(id -u)
+          WS="${GITHUB_WORKSPACE}"
+          reap() {
+            local sig="$1"
+            for pid in $(pgrep -u "$UID_SELF" -x tofu) $(pgrep -u "$UID_SELF" -x terraform); do
+              cwd=$(readlink "/proc/$pid/cwd" 2>/dev/null) || continue
+              case "$cwd" in
+                "$WS"|"$WS"/*) kill "-$sig" "$pid" 2>/dev/null ;;
+              esac
+            done
+          }
+          reap TERM
+          sleep 2
+          reap KILL
+          find "$WS" -name '.terraform.tfstate.lock.info' -delete 2>/dev/null
+          quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
+          quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
+          retry_run "tofu destroy"          "$TOFU_LOG" timeout 600 tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
+          quiet_run "scrub-env"             "$TOFU_LOG" ./scrub-env.sh "${SPINIFEX_CI_ENV}"
+          ok "teardown complete"
+
+      - name: Upload tofu log
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: tofu-log-perf3node-${{ github.run_id }}
+          path: ${{ env.TOFU_LOG }}
+          if-no-files-found: ignore
+          retention-days: 14

.github/workflows/publish-system-ami.yml

@@ -60,7 +60,7 @@ concurrency:
 jobs:
   matrix:
     name: Resolve image matrix
-    runs-on: [ self-hosted, ci-single ]
+    runs-on: [self-hosted, ci-single]
     outputs:
       images: ${{ steps.gen.outputs.images }}
     steps:
@@ -84,7 +84,7 @@ jobs:
   publish:
     name: Build + publish ${{ matrix.name }}
     needs: matrix
-    runs-on: [ self-hosted, ci-single ]
+    runs-on: [self-hosted, ci-single]
     strategy:
       fail-fast: false
       # ci-single is one runner and each GPU build needs ~16G of scratch disk;
@@ -131,9 +131,7 @@ jobs:
         with:
           go-version-file: "spinifex/go.mod"
           # Self-hosted runner: the Go module + build cache already persists on
-          # local disk between runs. setup-go's cache-save ("Post Setup Go")
-          # otherwise re-uploads hundreds of MB to the Actions cache backend,
-          # taking longer than the build itself.
+          # local disk between runs.
           cache: false
 
       - name: Setup Go Workspace
@@ -160,7 +158,6 @@ jobs:
       - name: Build + publish ${{ matrix.name }} to R2
         working-directory: spinifex
         env:
-          # Map the repo's R2 secrets onto the names publish-system-image.sh reads.
           AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
           R2_ENDPOINT: ${{ secrets.CF_R2_ENDPOINT_URL }}

.github/workflows/release.yml

@@ -46,7 +46,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v7
         with:
-          go-version: "1.26.6"
+          go-version: "1.27.0"
 
       - name: Build microVM artifacts (lb-agent + kernel + initramfs)
         run: cd spinifex && make build-microvm-image
@@ -148,6 +148,13 @@ jobs:
   verify:
     needs: release
     runs-on: [ self-hosted, ci-single ]
+    # Shared with the e2e workflows: this job provisions on the same runner
+    # pools and the same hypervisors, so two in flight compete for their disks
+    # and their libvirt guests. Job-level, so the build jobs are not queued
+    # behind a nightly that has nothing to do with them.
+    concurrency:
+      group: e2e-hypervisors
+      cancel-in-progress: false
     timeout-minutes: 25
     env:
       TOFU_LOG: /tmp/release-verify-${{ github.run_id }}.log
@@ -174,6 +181,13 @@ jobs:
           sparse-checkout: scripts/tofu-cluster
           path: mulga
 
+      # Only the shared shell helpers: this job builds nothing from source, but
+      # it drives the same libvirt provider the e2e cells retry around.
+      - uses: actions/checkout@v7
+        with:
+          sparse-checkout: .github/scripts
+          path: spinifex
+
       - name: Download Release Assets
         uses: actions/download-artifact@v8
         with:
@@ -183,11 +197,12 @@ jobs:
       - name: Provision VM
         working-directory: mulga/scripts/tofu-cluster
         run: |
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           tofu init -input=false >> "$TOFU_LOG" 2>&1
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
           tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
-          tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
+          retry_run "tofu apply" "$TOFU_LOG" tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
 
       - name: Bootstrap via Binary Install
         working-directory: mulga/scripts/tofu-cluster
@@ -200,8 +215,9 @@ jobs:
         if: always()
         working-directory: mulga/scripts/tofu-cluster
         run: |
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
-          tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
+          retry_run "tofu destroy" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
 
   baremetal:

.github/workflows/spinifex.yml

@@ -89,8 +89,11 @@ jobs:
       # Build artifacts follow source, so the key rolls per run and falls back to
       # this job's newest entry. Keyed per job because -race, -covermode=atomic
       # and plain builds produce disjoint entries and would evict each other.
-      - name: Cache Go build
-        uses: actions/cache@v6
+      # Restore runs everywhere; the matching save is gated to dev and main, so
+      # a PR reads the base branch's entry without spending a minute uploading
+      # one only that PR would ever read.
+      - name: Restore Go build cache
+        uses: actions/cache/restore@v6
         with:
           path: ~/.cache/go-build
           key: gobuild-${{ github.job }}-${{ github.run_id }}
@@ -131,91 +134,26 @@ jobs:
       - name: Check Diff Coverage
         run: cd spinifex && scripts/diff-coverage.sh coverage.out --base HEAD~1
 
-  # In-process integration tier (gateway.SetupRoutes over embedded NATS
-  # JetStream). Its own job so it runs in parallel with the unit tiers rather
-  # than adding to their wall time, and it still blocks a PR — unlike e2e.yml's
-  # "Integration (front gate)" job, which is push-triggered against main/dev on
-  # a self-hosted runner and so never gates a PR before merge.
-  integration_tests:
-    name: Integration Tests
-    runs-on: ubuntu-26.04
+      # Compiled output stays valid whatever the tests said, and skipping the
+      # save on a red run would leave the next one rebuilding it, so this is
+      # gated on the branch rather than on success.
+      # Go bumps an entry's mtime when a run uses it, so this run's working set
+      # survives and objects superseded by later commits do not. Runs after the
+      # work, so it costs this job nothing and only shrinks what the next reads.
+      - name: Trim Go build cache
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        run: |
+          C="$(go env GOCACHE)"
+          before=$(du -sm "$C" | cut -f1)
+          find "$C" -type f -mmin +240 -delete
+          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
 
-    steps:
-      - name: Checkout Spinifex
-        uses: actions/checkout@v7
-        with:
-          path: spinifex
-          fetch-depth: 2
-
-      - name: Resolve sibling refs
-        id: siblings
-        env:
-          BRANCH: ${{ github.head_ref || github.ref_name }}
-        run: bash .github/resolve-sibling-refs.sh
-        working-directory: spinifex
-
-      - name: Checkout Viperblock
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/viperblock
-          ref: ${{ steps.siblings.outputs.viperblock }}
-          path: viperblock
-
-      - name: Checkout Predastore
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/predastore
-          ref: ${{ steps.siblings.outputs.predastore }}
-          path: predastore
-
-      - name: Checkout Northstar
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/northstar
-          ref: ${{ steps.siblings.outputs.northstar }}
-          path: northstar
-
-      - name: Checkout Bluebottle
-        uses: actions/checkout@v7
-        with:
-          repository: mulgadc/bluebottle
-          ref: ${{ steps.siblings.outputs.bluebottle }}
-          path: bluebottle
-
-      # setup-go's own cache only writes on a key miss, so it froze a snapshot at
-      # the last go.sum change and never refreshed it. Both caches are managed
-      # below instead.
-      - name: Setup Go
-        uses: actions/setup-go@v7
-        with:
-          go-version-file: "spinifex/go.mod"
-          cache: false
-
-      # Module contents follow the five go.sum files, so a content key is exact.
-      # setup-go hashed spinifex/go.sum alone, which left the sibling modules'
-      # dependencies out of the key and re-downloading on every run.
-      - name: Cache Go modules
-        uses: actions/cache@v6
-        with:
-          path: ~/go/pkg/mod
-          key: gomod-${{ hashFiles('spinifex/go.sum', 'viperblock/go.sum', 'predastore/go.sum', 'northstar/go.sum', 'bluebottle/go.sum') }}
-          restore-keys: gomod-
-
-      # Build artifacts follow source, so the key rolls per run and falls back to
-      # this job's newest entry. Keyed per job because -race, -covermode=atomic
-      # and plain builds produce disjoint entries and would evict each other.
-      - name: Cache Go build
-        uses: actions/cache@v6
+      - name: Save Go build cache
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        uses: actions/cache/save@v6
         with:
           path: ~/.cache/go-build
           key: gobuild-${{ github.job }}-${{ github.run_id }}
-          restore-keys: gobuild-${{ github.job }}-
-
-      - name: Setup Go Workspace
-        run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
-
-      - name: Run Integration Tests
-        run: make -C spinifex test-integration
 
   race_detection:
     name: Race Detection
@@ -284,8 +222,11 @@ jobs:
       # Build artifacts follow source, so the key rolls per run and falls back to
       # this job's newest entry. Keyed per job because -race, -covermode=atomic
       # and plain builds produce disjoint entries and would evict each other.
-      - name: Cache Go build
-        uses: actions/cache@v6
+      # Restore runs everywhere; the matching save is gated to dev and main, so
+      # a PR reads the base branch's entry without spending a minute uploading
+      # one only that PR would ever read.
+      - name: Restore Go build cache
+        uses: actions/cache/restore@v6
         with:
           path: ~/.cache/go-build
           key: gobuild-${{ github.job }}-${{ github.run_id }}
@@ -297,8 +238,26 @@ jobs:
       - name: Run Tests with Race Detector
         run: make -C spinifex test-race GOTESTFLAGS=-count=1
 
+      # Go bumps an entry's mtime when a run uses it, so this run's working set
+      # survives and objects superseded by later commits do not. Runs after the
+      # work, so it costs this job nothing and only shrinks what the next reads.
+      - name: Trim Go build cache
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        run: |
+          C="$(go env GOCACHE)"
+          before=$(du -sm "$C" | cut -f1)
+          find "$C" -type f -mmin +240 -delete
+          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
+
+      - name: Save Go build cache
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        uses: actions/cache/save@v6
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}
+
   lint_and_security:
-    name: Lint and Security
+    name: Lint + Security + Integration
     runs-on: ubuntu-26.04
 
     steps:
@@ -306,6 +265,7 @@ jobs:
         uses: actions/checkout@v7
         with:
           path: spinifex
+          fetch-depth: 2
 
       - name: Resolve sibling refs
         id: siblings
@@ -364,8 +324,11 @@ jobs:
       # Build artifacts follow source, so the key rolls per run and falls back to
       # this job's newest entry. Keyed per job because -race, -covermode=atomic
       # and plain builds produce disjoint entries and would evict each other.
-      - name: Cache Go build
-        uses: actions/cache@v6
+      # Restore runs everywhere; the matching save is gated to dev and main, so
+      # a PR reads the base branch's entry without spending a minute uploading
+      # one only that PR would ever read.
+      - name: Restore Go build cache
+        uses: actions/cache/restore@v6
         with:
           path: ~/.cache/go-build
           key: gobuild-${{ github.job }}-${{ github.run_id }}
@@ -377,8 +340,30 @@ jobs:
       - name: Lint
         uses: golangci/golangci-lint-action@v9
         with:
-          version: v2.12.2
+          version: v2.13.0
           working-directory: spinifex
 
       - name: Govulncheck
         run: make -C spinifex govulncheck
+
+      - name: Run Integration Tests
+        if: ${{ !cancelled() }}
+        run: make -C spinifex test-integration
+
+      # Go bumps an entry's mtime when a run uses it, so this run's working set
+      # survives and objects superseded by later commits do not. Runs after the
+      # work, so it costs this job nothing and only shrinks what the next reads.
+      - name: Trim Go build cache
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        run: |
+          C="$(go env GOCACHE)"
+          before=$(du -sm "$C" | cut -f1)
+          find "$C" -type f -mmin +240 -delete
+          echo "trimmed ${before}MB -> $(du -sm "$C" | cut -f1)MB"
+
+      - name: Save Go build cache
+        if: ${{ !cancelled() && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/main') }}
+        uses: actions/cache/save@v6
+        with:
+          path: ~/.cache/go-build
+          key: gobuild-${{ github.job }}-${{ github.run_id }}

.gitignore

@@ -34,9 +34,11 @@ Thumbs.db
 # Environment
 .env
 .venv/
+.cache/
+
+# VScode
+.vscode/
 
 # Test reports
 *-report.txt
-
-# Per-worktree golangci-lint cache
-/.cache/
+/rds-agent

.golangci.yml

@@ -26,7 +26,7 @@ linters:
     - errorlint # catches ==/!= on errors, type assertions on errors, and %v instead of %w — all break on wrapped errors
     - exptostd # detects golang.org/x/exp/ functions replaceable by stdlib
     - fatcontext # detects nested contexts in loops
-    - forbidigo # blocks the legacy JetStream/KV symbols
+    - forbidigo # blocks forbidden identifiers
     - gocheckcompilerdirectives # validates go compiler directive comments (//go:)
     - godot # checks that declaration comments end in a period
     - goprintffuncname # checks that printf-like functions are named with f at the end
@@ -92,6 +92,8 @@ linters:
     forbidigo:
       analyze-types: true
       forbid:
+        - pattern: '^panic$'
+          msg: panic terminates the process; return an error or explicitly justify the panic.
         - pattern: '\.JetStream$'
           pkg: '^github\.com/nats-io/nats\.go$'
           msg: legacy nc.JetStream() is superseded; construct with jetstream.New(nc).
@@ -105,6 +107,7 @@ linters:
         - G116 # bidi chars — bidichk is more thorough
         - G117 # use of net/http serve functions (we handle TLS separately)
         - G703 # path traversal taint analysis — false positives on config file loading
+        - G704 # SSRF taint analysis — false positives on resolver-derived endpoint URLs
         - G705 # XSS taint analysis — false positives on internal error responses
 
     govet:
@@ -116,6 +119,7 @@ linters:
 
     modernize:
       disable:
+        - embedlit # its fix promotes embedded fields into the outer literal, which does not compile
         - newexpr # don't replace aws.String() with new() — AWS SDK idiom
         - stringsbuilder # don't replace small string concatenations with strings.Builder
 
@@ -166,6 +170,7 @@ linters:
           - bodyclose
           - errcheck
           - errchkjson
+          - forbidigo
           - gosec
 
       # spinifex-specific: gosec exclusions

Makefile

@@ -1,11 +1,6 @@
 GO_PROJECT_NAME := spx
 SHELL := /bin/bash
 
-# golangci-lint's default cache (~/.cache/golangci-lint) is keyed by module
-# path, not worktree path, so parallel worktrees of this module collide and
-# surface each other's issues. Scope it per worktree.
-export GOLANGCI_LINT_CACHE := $(CURDIR)/.cache/golangci-lint
-
 # GOFIPS140 is a global build-config change: it alters every package's action ID,
 # stdlib included, so a target that omits it shares no build-cache entry with one
 # that sets it. Exported once here rather than per-recipe so the two can't drift.
@@ -84,7 +79,7 @@ build-ecs-agent: ## Build the ecs-agent (ships in the ECS-AMI guest; not in host
 
 build-loadgen: ## Build spx-loadgen (drives the stress harness's measured load; not shipped to nodes)
 	@echo -e "\n....Building spx-loadgen"
-	GOFIPS140=v1.0.0 go build -ldflags "-s -w" -o ./bin/spx-loadgen ./cmd/spx-loadgen
+	go build -ldflags "-s -w" -o ./bin/spx-loadgen ./cmd/spx-loadgen
 
 build-system-image: ## Build a system image from manifest (use IMAGE=lb or IMAGE=eks-node)
 ifndef IMAGE
@@ -108,6 +103,12 @@ import-eks-node-image: ## Build + register the eks-node AMI (requires a running
 publish-eks-node-image: ## Build + publish the eks-node AMI to Cloudflare R2 (needs R2_ENDPOINT + AWS_* env)
 	./scripts/publish-system-image.sh scripts/images/eks-node/manifest.conf --build
 
+# The Ubuntu/systemd EKS image, as opposed to build-eks-node-image's Alpine one.
+# Composition lives in mkosi-build.sh's image_profiles table, so the image name
+# is the whole interface here.
+build-eks-node-mkosi-image: ## Build the Ubuntu mkosi EKS node image (server+agent; role at first boot)
+	./scripts/mkosi-build.sh --image spinifex-eks-node
+
 build-ecs-node-image: ## Build the spinifex-ecs-node AMI (Alpine + containerd + ecs-agent; IMPORT=1 to register)
 	$(MAKE) build-system-image IMAGE=ecs-agent
 
@@ -168,15 +169,19 @@ test-build-scripts:
 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the
 # default `go test ./spinifex/...`. Runs with mocked AWS clients — no
 # infrastructure required, safe to run in CI without a cluster.
+#
+# diskperf/ is here for the same reason: its baseline comparison and probe
+# accounting decide whether the disk-performance gate passes, and that
+# arithmetic has to be covered off-cluster or the gate is trusted untested.
 test-harness:
 	@echo -e "\n....Running e2e harness unit tests...."
-	$(_Q)LOG_IGNORE=1 go test -tags=e2e -timeout 60s ./tests/e2e/harness/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test -tags=e2e -timeout 60s ./tests/e2e/harness/... ./tests/e2e/diskperf/... $(_RACEQ)
 
 # In-process integration tier: the real gateway router against embedded NATS
 # JetStream, with only the daemon-side NATS subjects stubbed.
 AWS_MODEL_CONFORMANCE_REPORT ?= $(CURDIR)/.cache/aws-model-conformance-report.txt
 AWS_MODEL_CONFORMANCE_MODE ?= fail
-AWS_MODEL_OPERATION_COVERAGE_REPORT ?= $(CURDIR)/docs/aws-model-operation-coverage.md
+AWS_MODEL_OPERATION_COVERAGE_DIR ?= $(CURDIR)/docs/coverage
 # -count=1 is load-bearing: the conformance report is written by the test binary,
 # so a cached pass skips the run, leaves no report, and the cat below fails the
 # target. It also keeps the conformance gate honest — a cached result would mean
@@ -186,14 +191,13 @@ test-integration:
 	$(_Q)LOG_IGNORE=1 AWS_MODEL_CONFORMANCE_MODE=$(AWS_MODEL_CONFORMANCE_MODE) AWS_MODEL_CONFORMANCE_REPORT=$(AWS_MODEL_CONFORMANCE_REPORT) go test -count=1 -tags=integration -timeout 60s ./tests/integration/... $(_RACEQ)
 	@cat $(AWS_MODEL_CONFORMANCE_REPORT)
 	@$(MAKE) --no-print-directory generate-aws-model-coverage
-	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_REPORT)"
+	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_DIR)"
 
 generate-aws-model-coverage:
-	@mkdir -p $(dir $(AWS_MODEL_OPERATION_COVERAGE_REPORT))
-	@go run ./cmd/aws-model-coverage > $(AWS_MODEL_OPERATION_COVERAGE_REPORT)
+	@go run ./cmd/aws-model-coverage -out $(AWS_MODEL_OPERATION_COVERAGE_DIR)
 
 aws-model-coverage: generate-aws-model-coverage
-	@cat $(AWS_MODEL_OPERATION_COVERAGE_REPORT)
+	@go run ./cmd/aws-model-coverage
 
 # Segscan storage oracle: needs the mulga umbrella repo's scripts/segscan
 # checked out alongside spinifex (see spinifex/testutil/segscanoracle), which
@@ -344,9 +348,9 @@ install-system:
 		ovn-central ovn-host openvswitch-switch dhcpcd-base
 
 install-go:
-	@echo -e "\n....Installing Go 1.26.6 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.27.0 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.26.6.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.27.0.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi

README.md

@@ -3,9 +3,10 @@
 </p>
 
 <p align="center">
-  <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.26+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
+  <a href="https://go.dev"><img src="https://img.shields.io/badge/Go-1.27+-00ADD8?style=flat-square&logo=go&logoColor=white" alt="Go"></a>
   <a href="LICENSE"><img src="https://img.shields.io/badge/License-AGPL--3.0-3fb950?style=flat-square" alt="License"></a>
   <a href="https://mulgadc.com"><img src="https://img.shields.io/badge/home-mulga-orange?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTE2LjcxOCA4Ljg5MWMtMS4yODUgMS4zNy0zLjE3OCAyLjMxNy00LjY5NSAzLjQ0LS44NTQuNjMtMy4wODkgMi4yNy0xLjYxIDMuMzQ0IDEuNzg4IDEuMjk4IDYuMjQzLjE2NCA3Ljc4NS0xLjI2Ljg0LS43NzUuODE0LTEuODIyLS4zMjgtMi4yNTktMS4yMzUtLjQ3Mi0yLjY2LS4xMTEtMy45MTMuMDg3LS4wNDIuMDA3LS4xMzEuMDU1LS4xMjMtLjAyLjUyMy0uMzM2Ljk5My0uNzUgMS41MDMtMS4xMDIuNDkyLS4zNCAxLjA5Ny0uODI3IDEuNy0uODM4IDEuODcxLS4wMzQgMy43OTkuODkgNC4yODcgMi44MTUuODExIDMuMjAzLTMuMDA2IDUuNzE1LTUuNzg0IDUuOTQybDEuNjE0LS43NDdjLjYwNS0uMzYgMS4yMTctLjczNCAxLjc1Mi0xLjE5Ni4xMzMtLjExNS4yMy0uMjYzLjM0Mi0uMzczLjAyNy0uMDI2LjMwMi0uMTQ0LjE0NC0uMTUtMS40NDEuOTQ1LTMuMTI3IDEuNTcyLTQuODEzIDEuOTMyLS41OC4xMjMtMS4xOTUuMjQyLTEuNzg1LjI1Ny4wMTUuMDguMDg3LjA2NC4xNC4wOC40NzYuMTU0IDEuMDIuMjQ1IDEuNTE2LjMyLjA2OC4wNTItLjAwNi4wNzQtLjA2LjA4MS0uNDQ4LjA1OC0uOTIzLjE0My0xLjM3LjE2Ni0xLjI3LjA2NC0yLjU1LS4wNjgtMy43NzctLjM4bC0uNjktMS45NTdjLS4wNi4wMDYtLjA2My4wNi0uMDc5LjEwMy0uMDYuMTctLjMwNSAxLjQxNy0uMzg3IDEuNDM1LS42Ni0uMi0xLjI1MS0uNjQ3LTEuNzM2LTEuMTMybDEuMTUtMi4wNi0xLjgzNSAxLjAxMWMtLjI1OC0uNTgzLS4zMDUtMS4yNDMtLjIxOS0xLjg3bDIuMzc1LTEuMThjLS43MzktLjA0OS0xLjQ4Ni4wNjgtMi4yMi4xNDUuMTI1LS4zNi4yNzctLjcxOC40NjItMS4wNTEuMDY3LS4xMjIuNDI2LS43MjEuNTItLjcyNC44Ni4xNjQgMS43ODYuMjQxIDIuNjQ4LjA3NGwtMS44Ny0uNzcxLS4wNjktLjA5M2MuMzgyLS4zNi43Ny0uNzE3IDEuMTk5LTEuMDI0LjgwNC4yMzQgMS42My40MjIgMi40NzMuNDMzTDkuMzUgOS4zNDJhNyA3IDAgMCAxIC41NjctLjQwMWMuMTYtLjEwMy42MDktLjQwMi43NzItLjM4LjY1LjIyMSAxLjMyNC4zNzggMi4wMS40MzMtLjMxMy0uMzI1LS45MDktLjU2NC0xLjIxLS44NjgtLjAzMy0uMDM0LS4wNTgtLjAzNi0uMDQzLS4wOTguNzg3LS40NTIgMS42NjItLjkwMSAyLjM0LTEuNTE3LjY5LS42MjkgMS40MjEtMS42NTYuOTI0LTIuNjA1LjU3My4xODIgMS4wNjcuOTA2IDEuMDU0IDEuNTEyLS4wMzQgMS42NzYtMS44MjIgMy4xNy0yLjk0MyA0LjIyMiAxLjczMi0uNzI4IDMuNzE0LTIuMjMgMy43MS00LjMwNS0uMDAzLTEuNjg4LTEuNTQtMi4zNjUtMi45OTMtMi41MThhMy4yIDMuMiAwIDAgMS0uMzg1IDEuMDg3Yy0uNDI4LjcxOS0xLjMwMiAxLjE2OC0xLjc1OCAxLjkxNS0uMzExLjUxLS4zNyAxLjE5NS0xLjAzMSAxLjM5LS4yNy4wOC0uNjE3LjA5My0uODk3LjA5NGwuNjE1LS4zMjVjLjY4OS0uNTA2LjY3Ny0xLjQyIDEuMDk5LTIuMS0uMDUtLjA3LS41NDUtLjItLjY2LS4yMjktLjUxLS4xMjUtMS4zNzUtLjI4NS0xLjg4Ni0uMjUtLjE1Ny4wMS0uODY5LjEyMS0uOTMuMjQyLS4wODguMTc3LjM0MS45My40NjggMS4xMDEuMDI1LjAzNS4wNzcuMDI2LjA4LjAzLjAyNC4wMzEuMDA3LjEwNS0uMDYuMDhhNSA1IDAgMCAxLS40MDQtLjI4M2MtLjUwMy0uMzg4LTEuMDc4LS45NzQtMS40OTktMS40NDctLjA2OC0uMTY2LS4xNi0uMzUuMDAyLS40ODcuODg4LS41NSAxLjc2OC0xLjEyNiAyLjY3LTEuNjUxIDIuNjcxLTEuNTU4IDUuNzExLTMuMzE4IDguMjY3LS40NjUgMi4xMTkgMi4zNjYgMS41MTQgNS4yMTItLjUxMiA3LjM3MnptLTguMjI0LTUuMzhjLjY5OS0uMTIyIDIuMDE4LjU3MiAyLjIzNS0uNDU2LjAyMS0uMS0uMDM2LS4xNTcuMDItLjI1NS4wNDMtLjA3My4yODYtLjI1LjM3LS4zMTguMjctLjIxNy41NzctLjM4Ny44NDItLjYxMi0xLjAyOS4xNjYtMi4wNjUuNzU2LTIuOTY0IDEuMjc3LS4xNDkuMDg2LS4zMS4xNzYtLjQ1MS4yNzMtLjAzNy4wMjUtLjA3NS0uMDA2LS4wNTMuMDltLTQuODMgMTAuODQ0Yy0xLjcxNSAxLjg4MS0xLjMyNSA0LjU3LjQ5NCA2LjIxNyAxLjgxMSAxLjY0MSA0LjY2IDIuMjIzIDcuMDQ4IDIuMTA4IDIuMTUyLS4xMDMgNC4zMzctLjgxMiA2LjQ2LS4wOTEuODI1LjI4IDEuNTQ2LjgwNSAyLjE2IDEuNDExLS4wODMtLjQtLjMyNC0uODE5LS41NTgtMS4xNTktMi4xMy0zLjA5NS02LjI3LTEuOTM1LTkuNDI2LTIuNTUzLTEuNzExLS4zMzUtMy40OTEtMS4xMTgtNC41MzMtMi41NjctLjkwMS0xLjI1My0xLjA0Mi0yLjczLS41OTUtNC4xOTQtLjA5MS0uMDktLjk1Mi43MjEtMS4wNS44MjhtNi4zNjYtOC42NjdjLS4zODIuMzM5LS43ODcuNjYtMS4yMTIuOTQ4bC0xLjIwNy41OWMxLjAyMi4wMzkgMi4wOC0uNTQ2IDIuNDItMS41MzciLz48L3N2Zz4=" alt="mulgadc.com"></a>
+  <a href="https://docs.mulgadc.com"><img src="https://img.shields.io/badge/docs-spinifex-0969da?style=flat-square&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9IiNmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZD0iTTYuNSAySDIwYTEgMSAwIDAgMSAxIDF2MTQuNWExIDEgMCAwIDEtMSAxSDYuNWExLjUgMS41IDAgMCAwIDAgM0gyMWExIDEgMCAwIDEgMCAySDYuNUEzLjUgMy41IDAgMCAxIDMgMjBWNS41QTMuNSAzLjUgMCAwIDEgNi41IDJ6bTEgNGExIDEgMCAwIDAgMCAyaDlhMSAxIDAgMCAwIDAtMmgtOXptMCA0YTEgMSAwIDAgMCAwIDJoOWExIDEgMCAwIDAgMC0yaC05eiIvPjwvc3ZnPg==" alt="docs.mulgadc.com"></a>
 </p>
 
 <p align="center">
@@ -13,9 +14,10 @@
   <a href="#the-platform">Platform</a> ·
   <a href="#three-ways-to-deploy">Deploy</a> ·
   <a href="#aws-compatibility">AWS Compatibility</a> ·
+  <a href="#live-demo">Live Demo</a> ·
   <a href="#core-components">Components</a> ·
   <a href="#architecture-at-a-glance">Architecture</a> ·
-  <a href="#installation">Installation</a> ·
+  <a href="#self-hosted-installation">Installation</a> ·
   <a href="https://docs.mulgadc.com">Docs</a>
 </p>
 
@@ -29,11 +31,9 @@ Spinifex, built by [Mulga](https://mulgadc.com), recreates the AWS services your
 
 Spinifex is an open-source infrastructure platform that recreates the AWS service surface (EC2, EBS, S3, VPC, IAM) on bare-metal, edge, and on-premises environments. Run cloud-native software without a hyperscaler.
 
-Most AWS alternatives wrap an API and call it a cloud. We rebuilt the engineering underneath: distributed object storage with erasure coding, block storage built to survive failure, and compute on bare metal. That depth is the reason your software runs unchanged, instead of rewritten.
+Built for teams that need:
 
-It's built for teams that need:
-
-- The AWS tooling they already use (AWS CLI, SDKs, Terraform), with nothing to rewrite
+- The AWS tooling you already use (AWS CLI, SDKs, Terraform), with nothing to rewrite
 - Full control of the stack: your hardware, your network, your data, your keys
 - A cloud that keeps running offline, through disconnection, with no external control plane
 - An open core (AGPL-3.0), auditable and yours to keep, with no lock-in
@@ -50,15 +50,15 @@ From commodity hardware up to unmodified AWS tooling, every layer is replaceable
 
 ## Three Ways To Deploy
 
-One AWS-compatible surface, three deployment shapes. Pick the one that matches your reality; the platform on top is the same.
+The same AWS-compatible platform runs in three shapes. Pick the one that matches your hardware.
 
 ### Neocloud
 
-Lift and shift onto a partner Neocloud. Move workloads off the hyperscalers and onto our Neocloud partner ecosystem without rewriting them. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
+Lift and shift onto a partner Neocloud without rewriting workloads. GPU capacity (H100 / H200 / B200), cheaper, available now. Change an endpoint, keep the software.
 
 ### On-premise
 
-Bring your own hardware and host Spinifex in your own data centre. A real multi-node HA cluster integrated with your storage and networking, with full control of stack, data, and jurisdiction. A predictable bill, no egress surprises, an auditable open-source core.
+Host Spinifex on your own hardware in your own data centre. You get a multi-node HA cluster integrated with your storage and networking, so the stack, the data, and the jurisdiction stay yours.
 
 ### Edge
 
@@ -66,7 +66,7 @@ Cloud where the cloud can't reach. Air-gapped sites, vehicles, vessels, factorie
 
 ## AWS Compatibility
 
-Speak the AWS API surface, natively. The AWS SDKs, AWS CLI, and Terraform: everything you deploy on AWS deploys on Spinifex unchanged. At the edge, on-premise, or on a partner Neocloud.
+Spinifex speaks the AWS API surface. The AWS SDKs, AWS CLI, and Terraform all work unchanged, whether you deploy at the edge, on-premise, or on a partner Neocloud.
 
 | Service | What it is | Status |
 | --- | --- | --- |
@@ -80,19 +80,37 @@ Speak the AWS API surface, natively. The AWS SDKs, AWS CLI, and Terraform: every
 | **ECR** | Container Registry | Available |
 | **ECS** | Container Service | Available |
 | **RDS** | Databases | Available |
-| **Bedrock** | AI Deployment | Q3 2026 |
+| **Bedrock** | AI Deployment | Q4 2026 |
 
 Roadmap items ship under the same AWS API surface. Code written for AWS today keeps working the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
 
+## Live Demo
+
+Create a live sandbox in under a minute. The fastest way to experience the stack. Try Spinifex against one of your own workloads on infrastructure we run for you.
+
+* Live in under a minute. No card, no install.
+* Yours for 72 hours to test and experiment.
+* Real AWS-compatible endpoint, access keys and console.
+* EC2, EBS, S3, VPC, IAM, EKS and RDS, ready to call.
+* Point your existing AWS CLI or Terraform straight at it.
+
+Sign up at [https://mulgadc.com/signup](https://mulgadc.com/signup)
+
 ## Core Components
 
-### Spinifex (Compute Service – EC2 Alternative)
+### Spinifex (Compute and Platform Services – EC2, EKS, ECS, RDS)
 
-Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs similar to EC2. It manages lifecycle operations like start, stop, and terminate, using QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs using the AWS CLI, SDKs, or Terraform—without needing Kubernetes or heavyweight orchestrators. Keep in mind, you can also setup a Kubernetes environment using Spinifex with underlying instances.
+Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing an EC2 compatible API. It manages lifecycle operations like start, stop, and terminate through QEMU's QMP interface. Designed to be straightforward and scriptable, Spinifex lets you launch VMs with the AWS CLI, SDKs, or Terraform, without Kubernetes or a heavyweight orchestrator.
 
-- EC2-like VM management on bare metal
-- Launches with cloud-init metadata support
-- Works with standard AWS tooling
+The managed services build on that same instance layer. An EKS node group, an ECS container instance, and an RDS database are Spinifex VMs on your own hardware, each driven through its own AWS API.
+
+- **EC2** – VM lifecycle on bare metal, cloud-init and IMDS metadata, key pairs, tags, spot instances, capacity reservations, and GPU passthrough
+- **EKS** – Managed Kubernetes on k3s, with HA control planes, managed node groups, cluster add-ons, an OIDC provider, and IAM access entries
+- **ECS** – Task and service scheduling across container instances, with awsvpc ENIs, capacity providers, rolling deployments, and GPU-aware placement
+- **ECR** – Private container registry backed by Predastore, with content-addressed blobs deduplicated per account and lifecycle policies
+- **RDS** – Managed PostgreSQL and MariaDB, with parameter groups, automated backups, and DNS endpoints served by Northstar
+- **ALB / NLB** – Layer 7 load balancing on HAProxy and layer 4 on nginx `stream`, with health checks and ACM-issued listener certificates
+- **VPC and IAM** – OVN-backed networking and security groups, with roles, policies, and SigV4 auth applied across every service
 
 ### Viperblock (Block Storage – EBS Alternative)
 
@@ -101,12 +119,12 @@ Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs
 - Fast, durable virtual disks
 - Replication for resilience
 - Exposed over NBD or embedded in VMs
-- Supports high performance WAL logs using local NVMe drives to reduce IO traffic to S3.
-- In memory read/write block cache for blazing performance.
+- High performance WAL logs on local NVMe drives, reducing IO traffic to S3
+- In-memory read/write block cache
 
 ### Predastore (Object Storage – S3-Compatible)
 
-[Predastore](https://github.com/mulgadc/predastore) is a fully S3-compatible object storage system. It supports the AWS S3 API, including Signature V4 authentication, multipart uploads, and Terraform provisioning. Data is chunked and distributed across nodes using Reed-Solomon erasure coding, making it fault-tolerant and ideal for large-scale or low-bandwidth scenarios.
+[Predastore](https://github.com/mulgadc/predastore) is a fully S3-compatible object storage system. It supports the AWS S3 API, including Signature V4 authentication, multipart uploads, and Terraform provisioning. Predastore chunks data and distributes it across nodes using Reed-Solomon erasure coding, so it survives node loss and suits large-scale or low-bandwidth sites.
 
 - S3-compatible API and auth
 - Multipart uploads, streaming reads/writes
@@ -128,16 +146,7 @@ Spinifex is a minimal VM orchestration layer built on top of QEMU, exposing APIs
 
 Every AWS API call is authenticated at the gateway, published to a NATS subject, and answered by whichever daemon claims it. Daemons are stateless: scale horizontally by starting more. No etcd, no Kubernetes, no external control plane. Just systemd units, a NATS cluster, and your hardware. Deep dive: **[`docs/DESIGN.md`](docs/DESIGN.md)**.
 
-## Key Features
-
-- **AWS-compatible APIs.** Use the AWS CLI, SDKs, and Terraform you already know. Repoint your endpoint and ship; nothing to rewrite.
-- **Zero cloud dependency.** Runs entirely on your hardware. No phone-home, no control plane, no external authority. Works fully offline.
-- **Bare-metal compute.** QEMU-based with direct hardware access. No hypervisor tax, no abstraction overhead.
-- **Built-in storage.** Block and object storage included: NVMe caching, Reed-Solomon erasure coding, and replication out of the box.
-- **Edge-first architecture.** Designed for disconnected, contested, and resource-constrained environments from day one.
-- **Open core, no lock-in.** AGPL-3.0 with a commercial option. Inspect it, modify it, deploy it. The platform is yours to keep.
-
-## Installation
+## Self-hosted Installation
 
 Installation requires an Ubuntu 26.04 or Debian 13 system. See the detailed documentation at [docs.mulgadc.com](https://docs.mulgadc.com) for installing Spinifex.
 
@@ -173,31 +182,21 @@ aws ec2 describe-instance-types
 
 For a complete development environment see the [Source Install](https://docs.mulgadc.com/docs/install-source) documentation.
 
-### Component Repositories
-
-Spinifex coordinates these independent components:
-
-- **[Predastore](https://github.com/mulgadc/predastore)** - S3-compatible object storage
-- **[Viperblock](https://github.com/mulgadc/viperblock)** - EBS-compatible block storage
-- **[Northstar](https://github.com/mulgadc/northstar)** - Authoritative DNS server
-
-Each component can be developed independently. See component-specific documentation for focused development guides.
-
 ## Spinifex UI
 
-Spinifex ships with a built-in web console — an optional alternative to the AWS CLI, SDKs, and Terraform. If you're familiar with the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
+Spinifex ships with a built-in web console, an optional alternative to the AWS CLI, SDKs, and Terraform. If you know the AWS Management Console, the Spinifex UI fills the same role: a browser-based view of your instances, volumes, buckets, VPCs, and IAM resources, without leaving your own network.
 
 <p align="center">
-  <img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console — dashboard view" width="900">
+  <img src=".github/assets/spinifex-ui.jpg" alt="Spinifex web console dashboard" width="900">
 </p>
 
-The console is served by each node on port `3000` over TLS, and becomes available as soon as `spinifex.target` is up:
+Each node serves the console on port `3000` over TLS, and it comes up with `spinifex.target`:
 
-- **Same API, different surface.** Every action in the UI is the same AWS SigV4 call the CLI makes — so RBAC, audit trails, and IAM policies apply uniformly.
-- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed — no separate user database.
-- **Self-hosted, works offline.** The UI is embedded in the Spinifex binary and served from the node itself. No external CDN, no analytics calls, no cloud dependency.
+- **Same API, different surface.** The UI issues the same AWS SigV4 calls the CLI does, so RBAC, audit trails, and IAM policies apply the same way.
+- **Single sign-on against your AWS credentials.** Log in with the access keys from `~/.aws/credentials` on the node where Spinifex is installed. There is no separate user database.
+- **Self-hosted, works offline.** The Spinifex binary embeds the UI and each node serves it directly, with no external CDN or analytics calls.
 
-For the full walkthrough — first-time TLS certificate trust, login, and feature tour — see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
+For the full walkthrough, covering first-time TLS certificate trust, login, and a feature tour, see [**Launching the Web UI**](https://docs.mulgadc.com/docs/setting-up-your-cluster#7-launching-the-web-ui) in the cluster setup guide.
 
 ## Development Philosophy
 
@@ -205,16 +204,9 @@ For the full walkthrough — first-time TLS certificate trust, login, and featur
 
 Spinifex is developed by experienced infrastructure engineers with deep AWS expertise, including former AWS team members who understand the intricacies of building production-grade cloud services. Our team brings decades of combined experience from AWS, enterprise infrastructure, and edge computing environments.
 
-**Real-World Experience:**
-
-- Production AWS service development and operations
-- Large-scale infrastructure deployment and management
-- Edge computing and resource-constrained environments
-- Enterprise security and compliance requirements
-
 ### AI-Assisted Development
 
-While Spinifex is architected and implemented by experienced engineers, we leverage **Claude Code** (Anthropic's AI coding assistant) to accelerate certain development tasks. This approach combines human expertise with AI efficiency:
+Spinifex is architected and implemented by experienced engineers, with **Claude Code** accelerating certain development tasks. The split keeps the quality and reliability standards that production infrastructure demands:
 
 **How We Use Claude Code:**
 
@@ -230,12 +222,10 @@ While Spinifex is architected and implemented by experienced engineers, we lever
 - **Performance Optimization**: Real-world performance tuning and benchmarking
 - **Production Operations**: Deployment strategies and operational procedures
 
-This hybrid approach ensures Spinifex benefits from both proven engineering expertise and modern development acceleration, while maintaining the quality and reliability standards required for production infrastructure.
-
 ## Trademarks
 
 "AWS", "Amazon Web Services", and all related service names (EC2, EBS, S3, VPC, IAM, and others) are trademarks of Amazon.com, Inc. or its affiliates. Mulga Defense Corporation and Spinifex are independent and not affiliated with, endorsed by, or sponsored by Amazon.com, Inc. or Amazon Web Services, Inc. References to AWS services describe interoperability and compatibility only.
 
 ## License
 
-Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE). You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.
+Spinifex is open source under the [GNU Affero General Public License v3.0](LICENSE), with a commercial license available. You're free to use, modify, and deploy it anywhere you need reliable infrastructure without depending on centralized cloud platforms.

SECURITY.md

@@ -0,0 +1,24 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+Please do not open a public issue for a security vulnerability.
+
+Report it privately, either way works:
+
+- [Open a private security advisory](https://github.com/mulgadc/spinifex/security/advisories/new) on this repository.
+- Email <security@mulgadc.com>.
+
+Please include enough for us to reproduce it: the version or commit, the configuration involved, and the steps that trigger it. If you have a proof of concept, include that too.
+
+## What Happens Next
+
+We aim to acknowledge a report within three business days. We will confirm whether we can reproduce it, agree a fix and a disclosure timeline with you, and credit you in the advisory unless you would rather we did not.
+
+## Scope
+
+Spinifex is one part of the [Mulga](https://mulgadc.com) stack. If a vulnerability spans more than one component, report it once against whichever repository you found it in and we will handle the rest.
+
+## Supported Versions
+
+Fixes land on the latest release and on `main`. Older releases are not patched.

build/Dockerfile.distro

@@ -1,6 +1,6 @@
 ARG VERSION=dev
 
-FROM golang:1.26-bookworm AS builder
+FROM golang:1.27-bookworm AS builder
 
 ARG VERSION
 

build/systemd/spinifex-predastore.service

@@ -1,4 +1,4 @@
-# spinifex-unit-version: 2
+# spinifex-unit-version: 3
 [Unit]
 Description=Spinifex Predastore (S3-compatible Object Storage)
 PartOf=spinifex.target
@@ -7,9 +7,9 @@ Wants=spinifex-nats.service
 
 [Service]
 EnvironmentFile=-/etc/spinifex/telemetry.env
-# Guest block I/O arrives with no caller trace; sample those roots at 5% so
-# they don't drown request traces. Request-linked spans are always kept.
-Environment=MULGA_ROOT_TRACE_RATIO=0.05
+# No MULGA_ROOT_TRACE_RATIO here: the sink's tail sampler sheds guest block I/O
+# instead, and it can see whether a trace errored. A head ratio decides before
+# the error exists, so it discarded failures at the same rate as successes.
 Type=simple
 Slice=spinifex-system.slice
 User=spinifex-storage

build/systemd/spinifex-ui.service

@@ -1,4 +1,4 @@
-# spinifex-unit-version: 1
+# spinifex-unit-version: 2
 [Unit]
 Description=Spinifex Web UI
 PartOf=spinifex.target
@@ -24,6 +24,7 @@ MemoryMax=20%
 Environment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem
 Environment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key
 Environment=SPINIFEX_UI_BASE_DIR=/var/lib/spinifex/spinifex-ui/
+Environment=SPINIFEX_UI_OCHRE_ENABLED=false
 
 # Filesystem access
 NoNewPrivileges=yes

cmd/aws-model-coverage/main.go

@@ -1,14 +1,40 @@
 package main
 
 import (
+	"flag"
 	"fmt"
 	"os"
 
+	"github.com/mulgadc/predastore/s3api"
 	"github.com/mulgadc/spinifex/internal/awsmodel"
 	"github.com/mulgadc/spinifex/spinifex/gateway"
+
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {
+	outputDir := flag.String("out", "", "write one publishable page per service into this directory")
+	flag.Parse()
+
+	coverages, err := compareAll()
+	if err != nil {
+		fail(err)
+	}
+	if *outputDir == "" {
+		fmt.Print(awsmodel.RenderCoverageSummary(coverages))
+		return
+	}
+	if err := awsmodel.WritePages(*outputDir, coverages); err != nil {
+		fail(err)
+	}
+}
+
+func fail(err error) {
+	fmt.Fprintf(os.Stderr, "aws-model-coverage: %v\n", err)
+	os.Exit(1)
+}
+
+func compareAll() ([]awsmodel.OperationCoverage, error) {
 	dispatch := gateway.AWSOperationInventory()
 	coverages := make([]awsmodel.OperationCoverage, 0, len(awsmodel.Services()))
 	for _, service := range awsmodel.Services() {
@@ -21,23 +47,19 @@ func main() {
 				Stubbed:     inventory.Stubbed,
 				Unsupported: inventory.Unsupported,
 			}
+		// Predastore serves the S3 REST surface, and names the operations each
+		// of its routes answers, so the comparison is the same one.
 		case service == awsmodel.S3:
-			modelInventory = awsmodel.DispatchInventory{
-				Opaque: true,
-				Note:   "Spinifex delegates the S3 REST surface to Predastore, which has no operation-name dispatch table to compare mechanically.",
-			}
+			modelInventory = awsmodel.DispatchInventory{Registered: s3api.Operations()}
 		default:
-			fmt.Fprintf(os.Stderr, "aws-model-coverage: no gateway inventory for %s\n", service)
-			os.Exit(1)
+			return nil, fmt.Errorf("no gateway inventory for %s", service)
 		}
 
 		coverage, err := awsmodel.CompareOperations(service, modelInventory)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "aws-model-coverage: %v\n", err)
-			os.Exit(1)
+			return nil, err
 		}
 		coverages = append(coverages, coverage)
 	}
-
-	fmt.Print(awsmodel.RenderCoverageMarkdown(coverages))
+	return coverages, nil
 }

cmd/dhcptest/main.go

@@ -24,6 +24,8 @@ import (
 	"github.com/insomniacslk/dhcp/dhcpv4"
 	"github.com/insomniacslk/dhcp/dhcpv4/nclient4"
 	"github.com/mulgadc/spinifex/spinifex/network/external/dhcp"
+
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {

cmd/ecr-credential-provider/main.go

@@ -24,7 +24,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/guestenv"
 	"github.com/mulgadc/spinifex/internal/imdscreds"
 
-	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 const (

cmd/ecs-agent/credentials/imds.go

@@ -9,9 +9,9 @@ import (
 )
 
 // IMDSProvider fetches instance-role credentials from IMDS via the AWS SDK's
-// EC2 IMDS client and ec2rolecreds provider, cached with aws.CredentialsCache
-// (which refetches once the real expiry passes; ec2rolecreds also caps
-// Expires to at most an hour out). Safe for concurrent use.
+// EC2 IMDS client and ec2rolecreds provider, cached with aws.CredentialsCache,
+// which refetches once the real expiry passes. A failed refresh is an error
+// here, not an extended expiry. Safe for concurrent use.
 type IMDSProvider struct {
 	cache *aws.CredentialsCache
 }

cmd/ecs-agent/credentials/imds_test.go

@@ -88,6 +88,63 @@ func TestRetrieve_RefetchesWhenExpired(t *testing.T) {
 	}
 }
 
+// imdsStubFailAfterFirst serves creds on the first credential request and 500s
+// on every one after it, so a test can drive a refresh that fails against a
+// credential the cache has already stored.
+func imdsStubFailAfterFirst(t *testing.T, creds map[string]string) *httptest.Server {
+	t.Helper()
+	var hits atomic.Int32
+	mux := http.NewServeMux()
+	mux.HandleFunc("/latest/api/token", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
+		_, _ = w.Write([]byte("v2-token"))
+	})
+	mux.HandleFunc("/latest/meta-data/iam/security-credentials/", func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/latest/meta-data/iam/security-credentials/" {
+			_, _ = w.Write([]byte("node-role"))
+			return
+		}
+		if hits.Add(1) > 1 {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		_ = json.NewEncoder(w).Encode(creds)
+	})
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+// A refresh that fails must surface as an error. Left to itself the SDK's
+// ec2rolecreds implements HandleFailToRefresh, which hands back the same dead
+// key with Expires pushed 5-15 minutes out, so the agent would sign requests
+// with a credential the control plane has already rejected — indefinitely,
+// since every subsequent refresh fails the same way.
+func TestRetrieve_FailedRefreshDoesNotExtendExpiry(t *testing.T) {
+	srv := imdsStubFailAfterFirst(t, map[string]string{
+		"Code":            "Success",
+		"AccessKeyId":     "AKIA",
+		"SecretAccessKey": "secret",
+		"Token":           "session",
+		// Already expired, so the next Retrieve must go back to IMDS.
+		"Expiration": time.Now().Add(-time.Minute).UTC().Format(time.RFC3339),
+	})
+
+	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")
+	first, err := p.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("first Retrieve: %v", err)
+	}
+
+	got, err := p.Retrieve(context.Background())
+	if err == nil {
+		t.Fatalf("refresh failure was reported as success: %+v", got)
+	}
+	if got.Expiration.After(first.Expiration) {
+		t.Errorf("expiry extended on refresh failure: %v -> %v", first.Expiration, got.Expiration)
+	}
+}
+
 func TestRetrieve_CancelledContext(t *testing.T) {
 	srv := imdsStub(t, nil, map[string]string{"Code": "Success", "AccessKeyId": "A", "SecretAccessKey": "B"})
 	p := NewIMDSProvider(srv.Client(), srv.URL+"/latest")

cmd/ecs-agent/main.go

@@ -17,7 +17,7 @@ import (
 	"os/signal"
 	"syscall"
 
-	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {

cmd/ecs-agent/runtime/runtime_test.go

@@ -1,71 +1,9 @@
 package runtime
 
 import (
-	"context"
-	"errors"
 	"testing"
 )
 
-// staticResolver is a test Resolver returning fixed creds or an error.
-type staticResolver struct {
-	user, pass, endpoint string
-	err                  error
-}
-
-func (s staticResolver) Authorize(_ context.Context, _ string) (string, string, string, error) {
-	return s.user, s.pass, s.endpoint, s.err
-}
-
-func TestFakePuller_RecordsAndDefaultsRef(t *testing.T) {
-	f := &FakePuller{}
-	img, err := f.Pull(context.Background(), PullSpec{Ref: "repo/app:1"}, staticResolver{user: "AWS"})
-	if err != nil {
-		t.Fatalf("Pull: %v", err)
-	}
-	if img.Ref != "repo/app:1" {
-		t.Errorf("Ref defaulted wrong: %q", img.Ref)
-	}
-	if len(f.Pulls) != 1 || f.Pulls[0].Ref != "repo/app:1" {
-		t.Errorf("Pulls = %+v", f.Pulls)
-	}
-	if len(f.Authzd) != 1 || f.Authzd[0] != "AWS" {
-		t.Errorf("Authzd = %+v", f.Authzd)
-	}
-}
-
-func TestFakePuller_ResolverError(t *testing.T) {
-	f := &FakePuller{}
-	_, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, staticResolver{err: errors.New("boom")})
-	if err == nil {
-		t.Fatal("expected resolver error to propagate")
-	}
-}
-
-func TestFakePuller_ProgrammedError(t *testing.T) {
-	f := &FakePuller{Err: errors.New("pull failed")}
-	if _, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil); err == nil {
-		t.Fatal("expected programmed error")
-	}
-}
-
-func TestFakePuller_OnPullOverrides(t *testing.T) {
-	want := Image{Ref: "x", Digest: "sha256:abc", Size: 42}
-	f := &FakePuller{Err: errors.New("ignored"), OnPull: func(PullSpec) (Image, error) {
-		return want, nil
-	}}
-	got, err := f.Pull(context.Background(), PullSpec{Ref: "r"}, nil)
-	if err != nil || got != want {
-		t.Fatalf("got %+v, %v; want %+v", got, err, want)
-	}
-}
-
-func TestFakePuller_Close(t *testing.T) {
-	f := &FakePuller{}
-	if err := f.Close(); err != nil || !f.Closed {
-		t.Fatalf("Close: err=%v closed=%v", err, f.Closed)
-	}
-}
-
 func TestNormalizeRef(t *testing.T) {
 	cases := map[string]string{
 		"nginx:alpine":      "docker.io/library/nginx:alpine",

cmd/eks-gateway-fetch/main.go

@@ -40,7 +40,7 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/eksgw"
 
-	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 // stagedAddon mirrors the gateway's StagedAddonManifest wire shape; duplicated

cmd/eks-gateway-publish/main.go

@@ -31,7 +31,7 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/eksgw"
 
-	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 const (

cmd/eks-konnectivity-cert/main.go

@@ -38,6 +38,8 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
+
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {

cmd/eks-token-webhook/cert.go

@@ -15,7 +15,7 @@ import (
 	"os"
 	"time"
 
-	"github.com/mulgadc/spinifex/internal/tlsconfig"
+	"github.com/mulgadc/bluebottle/pkg/tlsconfig"
 )
 
 // serverTLSConfig is the webhook's loopback serving TLS config.

cmd/eks-token-webhook/main.go

@@ -27,6 +27,8 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/eksgw"
 	handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
+
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 // config is the webhook's runtime configuration, sourced from the first-boot

cmd/eks-webhook-cert/main.go

@@ -32,6 +32,8 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
+
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {

cmd/installer/install/steps_test.go

@@ -887,13 +887,6 @@ func TestInstallBootToolZFSStopsWhenADiskFails(t *testing.T) {
 	f.mustNotRun(t, "SPINIFEX_BOOT_TOOL_NVRAM=1")
 }
 
-func TestRefreshBootToolNoteOnlyAppliesToZFS(t *testing.T) {
-	// Log-only, but it is the operator's only record of the disk-replacement
-	// command, so both branches are walked.
-	refreshBootToolNote(DiskConfig{FS: FSZFSRAID1, Disks: disks(2, 40)})
-	refreshBootToolNote(DiskConfig{FS: FSExt4, Disks: disks(1, 40)})
-}
-
 func TestMaskSystemdUnitIsIdempotent(t *testing.T) {
 	root := t.TempDir()
 	if err := os.MkdirAll(filepath.Join(root, "etc/systemd/system"), 0o755); err != nil {
@@ -913,14 +906,6 @@ func TestMaskSystemdUnitIsIdempotent(t *testing.T) {
 	}
 }
 
-func TestCopyAssetsAreBestEffort(t *testing.T) {
-	root := t.TempDir()
-	// Neither asset exists outside the live ISO; a missing one must not fail
-	// the install, only the branding.
-	copyGrubFont(root)
-	copySplashImage(root)
-}
-
 func TestWriteFstabOnZFSHoldsNothingButSwap(t *testing.T) {
 	root := withMountRoot(t)
 	targetSkeleton(t, root)

cmd/installer/install/zfs.go

@@ -182,8 +182,7 @@ func ImportablePools() ([]string, error) {
 		// zpool exits non-zero when there is nothing to import, which is the
 		// normal case. Anything else — no binary, no /dev/zfs — must surface, or
 		// a machine with an existing pool looks empty to the caller.
-		var exitErr *exec.ExitError
-		if errors.As(err, &exitErr) {
+		if _, ok := errors.AsType[*exec.ExitError](err); ok {
 			return nil, nil
 		}
 		return nil, fmt.Errorf("zpool import: %w", err)

cmd/installer/main.go

@@ -5,10 +5,10 @@ import (
 	"os"
 	"strings"
 
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 	"github.com/mulgadc/spinifex/cmd/installer/autoinstall"
 	"github.com/mulgadc/spinifex/cmd/installer/install"
 	"github.com/mulgadc/spinifex/cmd/installer/ui"
-	_ "github.com/mulgadc/spinifex/internal/fipsboot"
 )
 
 func main() {

cmd/installer/ui/wizard_test.go

@@ -0,0 +1,896 @@
+//test:in-package — the wizard model, its screen constants and the key handlers
+//under test are all unexported, and the tests read model fields directly to
+//assert which screen a key sequence landed on.
+
+package ui
+
+import (
+	"slices"
+	"strings"
+	"testing"
+
+	tea "github.com/charmbracelet/bubbletea"
+	"github.com/mulgadc/spinifex/cmd/installer/install"
+)
+
+// namedKeys maps the key names the handlers switch on to their tea.KeyType.
+// Anything not listed is sent as runes.
+var namedKeys = map[string]tea.KeyType{
+	"enter":     tea.KeyEnter,
+	"esc":       tea.KeyEsc,
+	"tab":       tea.KeyTab,
+	"shift+tab": tea.KeyShiftTab,
+	"up":        tea.KeyUp,
+	"down":      tea.KeyDown,
+	"left":      tea.KeyLeft,
+	"right":     tea.KeyRight,
+	" ":         tea.KeySpace,
+	"backspace": tea.KeyBackspace,
+	"ctrl+c":    tea.KeyCtrlC,
+}
+
+// keyMsg builds the tea.KeyMsg for a key name and checks it round-trips, so a
+// mis-built message fails here rather than silently exercising the wrong branch.
+func keyMsg(t *testing.T, key string) tea.KeyMsg {
+	t.Helper()
+	msg := tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(key)}
+	if kt, ok := namedKeys[key]; ok {
+		msg = tea.KeyMsg{Type: kt}
+	}
+	if got := msg.String(); got != key {
+		t.Fatalf("built key %q, which the handlers see as %q", key, got)
+	}
+	return msg
+}
+
+// press feeds keys through Update, which is the only entry point the bubbletea
+// runtime uses, and returns the resulting model.
+func press(t *testing.T, m model, keys ...string) model {
+	t.Helper()
+	for _, k := range keys {
+		next, _ := m.Update(keyMsg(t, k))
+		typed, ok := next.(model)
+		if !ok {
+			t.Fatalf("Update returned %T, not a model", next)
+		}
+		m = typed
+	}
+	return m
+}
+
+// typeText sends a string one rune at a time, the way a terminal does.
+func typeText(t *testing.T, m model, s string) model {
+	t.Helper()
+	for _, r := range s {
+		m = press(t, m, string(r))
+	}
+	return m
+}
+
+func wizardModel(t *testing.T, diskCount, nicCount int) model {
+	t.Helper()
+	disks := make([]install.Disk, 0, diskCount)
+	for i := range diskCount {
+		disks = append(disks, install.Disk{
+			Path:             "/dev/sd" + string(rune('a'+i)),
+			Bytes:            200 << 30,
+			LogicalBlockSize: 512,
+		})
+	}
+	m := newModel(disks, nics(nicCount))
+	m.width, m.height = 100, 40
+	return m
+}
+
+// The wizard has to be completable from the welcome screen using only the keys
+// the help lines advertise, and produce the config the operator entered.
+func TestWizardWalkthroughProducesTheEnteredConfig(t *testing.T) {
+	m := wizardModel(t, 1, 2)
+
+	m = press(t, m, "enter") // welcome → disk
+	if m.screen != screenDisk {
+		t.Fatalf("screen = %v, want the disk screen", m.screen)
+	}
+
+	m = press(t, m, "enter") // preselected disk is valid → erase confirmation
+	if m.screen != screenDiskConfirm {
+		t.Fatalf("screen = %v, want the erase confirmation", m.screen)
+	}
+
+	m = typeText(t, m, "yes")
+	m = press(t, m, "enter") // → network roles
+	if m.screen != screenNetworkRoles {
+		t.Fatalf("screen = %v, want the network roles screen", m.screen)
+	}
+
+	m = press(t, m, "down", "down", "down", "enter") // Continue → identity
+	if m.screen != screenIdentity {
+		t.Fatalf("screen = %v, want the identity screen", m.screen)
+	}
+
+	m = typeText(t, m, "node1")
+	m = press(t, m, "enter") // → credentials
+	if m.screen != screenPassword {
+		t.Fatalf("screen = %v, want the credentials screen", m.screen)
+	}
+
+	m = typeText(t, m, "admin@example.com")
+	m = press(t, m, "enter") // email accepted → password
+	m = typeText(t, m, "hunter2hunter2")
+	m = press(t, m, "enter") // → confirm password
+	m = typeText(t, m, "hunter2hunter2")
+	m = press(t, m, "enter") // → final confirmation
+	if m.screen != screenConfirm {
+		t.Fatalf("screen = %v, want the final confirmation (err: %q)", m.screen, m.validationErr)
+	}
+
+	m = press(t, m, "y")
+	if m.screen != screenDone {
+		t.Fatalf("screen = %v, want done", m.screen)
+	}
+	if m.result == nil {
+		t.Fatal("result is nil after confirming")
+	}
+
+	cfg := m.result
+	if cfg.Hostname != "node1" {
+		t.Errorf("Hostname = %q, want node1", cfg.Hostname)
+	}
+	if cfg.Email != "admin@example.com" {
+		t.Errorf("Email = %q, want admin@example.com", cfg.Email)
+	}
+	if cfg.RootPassword != "hunter2hunter2" {
+		t.Errorf("RootPassword = %q, want what was typed", cfg.RootPassword)
+	}
+	if !slices.Equal(cfg.Storage.Paths(), []string{"/dev/sda"}) {
+		t.Errorf("Storage.Paths() = %v, want the preselected disk", cfg.Storage.Paths())
+	}
+	if cfg.WAN.Interface != "eno1" {
+		t.Errorf("WAN interface = %q, want the first NIC", cfg.WAN.Interface)
+	}
+	if err := cfg.Validate(); err != nil {
+		t.Errorf("the wizard produced a config it would refuse to install: %v", err)
+	}
+}
+
+// Esc has to walk back the way it came, or an operator who overshoots is stuck
+// with a wizard they can only cancel out of.
+func TestEscapeWalksBackThroughTheScreens(t *testing.T) {
+	m := wizardModel(t, 1, 2)
+	m = press(t, m, "enter") // disk
+	m = press(t, m, "enter") // erase confirmation
+
+	for _, step := range []struct {
+		key  string
+		want screen
+	}{
+		{"esc", screenDisk},
+		{"esc", screenWelcome},
+	} {
+		m = press(t, m, step.key)
+		if m.screen != step.want {
+			t.Fatalf("after esc: screen = %v, want %v", m.screen, step.want)
+		}
+	}
+}
+
+// Ctrl+C is handled before any screen sees the key, so it works everywhere.
+func TestCtrlCCancelsWithAnError(t *testing.T) {
+	m := wizardModel(t, 1, 1)
+	next, cmd := m.Update(keyMsg(t, "ctrl+c"))
+	final, ok := next.(model)
+	if !ok {
+		t.Fatalf("Update returned %T", next)
+	}
+	if final.err == nil || !strings.Contains(final.err.Error(), "cancelled") {
+		t.Fatalf("err = %v, want a cancellation", final.err)
+	}
+	if cmd == nil {
+		t.Error("ctrl+c must return tea.Quit")
+	}
+}
+
+// A window resize has to reach the model, since every view lays out against it.
+func TestWindowResizeIsStored(t *testing.T) {
+	m := model{}
+	next, _ := m.Update(tea.WindowSizeMsg{Width: 120, Height: 50})
+	resized, ok := next.(model)
+	if !ok {
+		t.Fatalf("Update returned %T", next)
+	}
+	if resized.width != 120 || resized.height != 50 {
+		t.Errorf("size = %dx%d, want 120x50", resized.width, resized.height)
+	}
+}
+
+// The erase prompt is the last point of no return, so anything other than an
+// explicit "yes" has to keep the operator on it.
+func TestEraseConfirmationRequiresYes(t *testing.T) {
+	m := wizardModel(t, 1, 1)
+	m = press(t, m, "enter", "enter") // → erase confirmation
+
+	m = typeText(t, m, "no")
+	m = press(t, m, "enter")
+	if m.screen != screenDiskConfirm {
+		t.Fatalf("screen = %v, want to stay on the confirmation", m.screen)
+	}
+	if !strings.Contains(m.validationErr, "yes") {
+		t.Errorf("validationErr = %q, want it to say what to type", m.validationErr)
+	}
+	if !strings.Contains(m.viewDiskConfirm(100), m.validationErr) {
+		t.Error("the confirmation screen must show the error it set")
+	}
+}
+
+// The filesystem selector and the disk list share one screen, so the cursor has
+// to move between them and stop at both ends.
+func TestDiskScreenCursorStaysInRange(t *testing.T) {
+	m := wizardModel(t, 2, 1)
+	m.screen = screenDisk
+
+	m = press(t, m, "up", "up")
+	if m.storageCursor != fsRow {
+		t.Errorf("storageCursor = %d, want it clamped to the filesystem row", m.storageCursor)
+	}
+
+	m = press(t, m, "down", "down", "down", "down")
+	if m.storageCursor != len(m.disks) {
+		t.Errorf("storageCursor = %d, want it clamped to the last disk", m.storageCursor)
+	}
+}
+
+// Left/right only cycle the filesystem while the cursor is on that row, and
+// they wrap, so one key reaches every topology.
+func TestFilesystemCyclesOnlyOnItsOwnRow(t *testing.T) {
+	m := wizardModel(t, 2, 1)
+	m.screen = screenDisk
+
+	m = press(t, m, "left")
+	if got := m.fsType(); got != install.AllFSTypes[len(install.AllFSTypes)-1] {
+		t.Errorf("fsType = %v, want the cycle to wrap backwards to the last", got)
+	}
+	m = press(t, m, "right")
+	if got := m.fsType(); got != install.AllFSTypes[0] {
+		t.Errorf("fsType = %v, want the cycle to wrap forwards to the first", got)
+	}
+
+	m = press(t, m, "down") // onto the first disk
+	before := m.fsType()
+	m = press(t, m, "left", "right")
+	if m.fsType() != before {
+		t.Errorf("fsType changed to %v while the cursor was on a disk", m.fsType())
+	}
+}
+
+// Space toggles the disk under the cursor, and r cycles its role — the two keys
+// the storage screen exists for.
+func TestDiskScreenSelectionKeys(t *testing.T) {
+	m := wizardModel(t, 2, 1)
+	m.screen = screenDisk
+
+	// The second disk is unselected; the first arrives preselected as os.
+	m = press(t, m, "down", "down", " ")
+	if !slices.Contains(m.selected, 1) {
+		t.Fatalf("selected = %v, want the second disk added", m.selected)
+	}
+
+	roleBefore := m.roleOf[slices.Index(m.selected, 1)]
+	m = press(t, m, "r")
+	if got := m.roleOf[slices.Index(m.selected, 1)]; got == roleBefore {
+		t.Errorf("role stayed %q after r", got)
+	}
+
+	m = press(t, m, " ")
+	if slices.Contains(m.selected, 1) {
+		t.Fatalf("selected = %v, want the second disk removed again", m.selected)
+	}
+	if len(m.roleOf) != len(m.selected) {
+		t.Errorf("roleOf (%d) and selected (%d) drifted", len(m.roleOf), len(m.selected))
+	}
+}
+
+// A selection that cannot be installed has to be reported on the screen the
+// operator can still fix it on, not part-way through the install.
+func TestDiskScreenRefusesAnInvalidSelection(t *testing.T) {
+	m := wizardModel(t, 2, 1)
+	m.screen = screenDisk
+	m.fsCursor = slices.Index(install.AllFSTypes, install.FSZFSRAID1) // needs 2 disks
+
+	m = press(t, m, "enter")
+	if m.screen != screenDisk {
+		t.Fatalf("screen = %v, want to stay on the disk screen", m.screen)
+	}
+	if m.validationErr == "" {
+		t.Fatal("an invalid selection must set a validation error")
+	}
+	if !strings.Contains(m.viewDisk(100), m.validationErr) {
+		t.Error("the disk screen must render the error it set")
+	}
+
+	// Selecting the second disk satisfies the mirror and clears the way.
+	m = press(t, m, "down", "down", " ", "enter")
+	if m.screen != screenDiskConfirm {
+		t.Fatalf("screen = %v, want to advance once the mirror has two disks", m.screen)
+	}
+}
+
+// The ZFS options screen is only reachable on a pool, since its settings have
+// no meaning on ext4.
+func TestZFSOptionsScreenIsZFSOnly(t *testing.T) {
+	m := wizardModel(t, 2, 1)
+	m.screen = screenDisk
+
+	m = press(t, m, "a")
+	if m.screen != screenDisk {
+		t.Fatalf("screen = %v, want a to do nothing on ext4", m.screen)
+	}
+
+	m.fsCursor = slices.Index(install.AllFSTypes, install.FSZFSRAID1)
+	m = press(t, m, "a")
+	if m.screen != screenZFSOptions {
+		t.Fatalf("screen = %v, want the ZFS options screen", m.screen)
+	}
+}
+
+// Every option defaults to auto and cycles through its own choice list, with
+// the cursor clamped to the rows that exist.
+func TestZFSOptionsCycleTheirValues(t *testing.T) {
+	m := wizardModel(t, 2, 1)
+	m.screen = screenZFSOptions
+
+	if got := zfsOptions[0].value(&m.zfs); got != "auto" {
+		t.Errorf("ashift starts at %q, want auto", got)
+	}
+	m = press(t, m, "right")
+	if got := zfsOptions[0].value(&m.zfs); got == "auto" {
+		t.Error("right must move ashift off auto")
+	}
+	m = press(t, m, "left")
+	if got := zfsOptions[0].value(&m.zfs); got != "auto" {
+		t.Errorf("ashift = %q after cycling back, want auto", got)
+	}
+
+	m = press(t, m, "up", "up")
+	if m.zfsCursor != 0 {
+		t.Errorf("zfsCursor = %d, want it clamped to the first row", m.zfsCursor)
+	}
+	for range len(zfsOptions) + 2 {
+		m = press(t, m, "down")
+	}
+	if m.zfsCursor != len(zfsOptions)-1 {
+		t.Errorf("zfsCursor = %d, want it clamped to the last row", m.zfsCursor)
+	}
+
+	// The compression choices are strings rather than ints, so they exercise
+	// the other half of cycle.
+	m.zfsCursor = 1
+	m = press(t, m, "right")
+	if m.zfs.Compress == "" {
+		t.Error("right must set a compression algorithm")
+	}
+
+	m = press(t, m, "enter")
+	if m.screen != screenDisk {
+		t.Fatalf("screen = %v, want enter to return to the disk screen", m.screen)
+	}
+}
+
+// The screen has to show what each option is currently set to, since that is
+// the only place the pool's geometry is stated before it is created.
+func TestZFSOptionsViewShowsValuesAndTheFocusedNote(t *testing.T) {
+	m := wizardModel(t, 2, 1)
+	m.screen = screenZFSOptions
+	m.zfs.Compress = "zstd"
+
+	got := m.viewZFSOptions(100)
+	for _, want := range []string{"ashift", "auto", "compression", "zstd", zfsOptions[0].note} {
+		if !strings.Contains(got, want) {
+			t.Errorf("ZFS options screen is missing %q:\n%s", want, got)
+		}
+	}
+	if strings.Contains(got, zfsOptions[1].note) {
+		t.Error("only the focused row's note should be shown")
+	}
+}
+
+// Credentials are three fields on one screen, so focus has to cycle both ways
+// and land somewhere sensible after a validation failure.
+func TestCredentialsFocusCyclesBothWays(t *testing.T) {
+	m := wizardModel(t, 1, 1)
+	m.screen = screenPassword
+	m = m.setCredsFocus(0)
+
+	m = press(t, m, "tab")
+	if m.credsFocus != 1 || !m.passwordInput.Focused() {
+		t.Fatalf("credsFocus = %d, want the password field focused", m.credsFocus)
+	}
+	m = press(t, m, "tab", "tab")
+	if m.credsFocus != 0 || !m.emailInput.Focused() {
+		t.Fatalf("credsFocus = %d, want the cycle to wrap to email", m.credsFocus)
+	}
+	m = press(t, m, "shift+tab")
+	if m.credsFocus != 2 || !m.passwordConfirmInput.Focused() {
+		t.Fatalf("credsFocus = %d, want shift+tab to wrap backwards to confirm", m.credsFocus)
+	}
+}
+
+// Each credential rule has to stop the wizard on the screen and put the cursor
+// on the field at fault, not just refuse to advance.
+func TestCredentialValidation(t *testing.T) {
+	tests := []struct {
+		name       string
+		email      string
+		password   string
+		confirm    string
+		wantErr    string
+		wantFocus  int
+		wantScreen screen
+	}{
+		{
+			name: "rejects a malformed email", email: "not-an-email",
+			password: "hunter2hunter2", confirm: "hunter2hunter2",
+			wantErr: "email", wantFocus: 0, wantScreen: screenPassword,
+		},
+		{
+			name: "rejects an empty password", email: "admin@example.com",
+			wantErr: "Password is required", wantFocus: 1, wantScreen: screenPassword,
+		},
+		{
+			name: "rejects a mismatch", email: "admin@example.com",
+			password: "hunter2hunter2", confirm: "hunter2hunter3",
+			wantErr: "do not match", wantFocus: 2, wantScreen: screenPassword,
+		},
+		{
+			name: "accepts a matching pair", email: "admin@example.com",
+			password: "hunter2hunter2", confirm: "hunter2hunter2",
+			wantScreen: screenConfirm,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			m := wizardModel(t, 1, 1)
+			m.screen = screenPassword
+			m.emailInput.SetValue(tt.email)
+			m.passwordInput.SetValue(tt.password)
+			m.passwordConfirmInput.SetValue(tt.confirm)
+			m = m.setCredsFocus(2) // enter on confirm runs every rule
+
+			m = press(t, m, "enter")
+
+			if m.screen != tt.wantScreen {
+				t.Fatalf("screen = %v, want %v (err %q)", m.screen, tt.wantScreen, m.validationErr)
+			}
+			if tt.wantErr == "" {
+				if m.validationErr != "" {
+					t.Fatalf("validationErr = %q, want none", m.validationErr)
+				}
+				return
+			}
+			if !strings.Contains(strings.ToLower(m.validationErr), strings.ToLower(tt.wantErr)) {
+				t.Errorf("validationErr = %q, want it to mention %q", m.validationErr, tt.wantErr)
+			}
+			if m.credsFocus != tt.wantFocus {
+				t.Errorf("credsFocus = %d, want the cursor on field %d", m.credsFocus, tt.wantFocus)
+			}
+			if !strings.Contains(m.viewPassword(100), m.validationErr) {
+				t.Error("the credentials screen must render the error it set")
+			}
+		})
+	}
+}
+
+// The hostname ends up in the config and in DNS, so an empty one has to be
+// caught on the screen rather than installed.
+func TestHostnameIsRequired(t *testing.T) {
+	m := wizardModel(t, 1, 1)
+	m.screen = screenIdentity
+	m.hostnameInput.Focus()
+
+	m = press(t, m, "enter")
+	if m.screen != screenIdentity {
+		t.Fatalf("screen = %v, want to stay on identity", m.screen)
+	}
+	if !strings.Contains(m.validationErr, "Hostname") {
+		t.Errorf("validationErr = %q, want it to name the field", m.validationErr)
+	}
+	if !m.hostnameInput.Focused() {
+		t.Error("the hostname field must keep focus after a failed submit")
+	}
+	if !strings.Contains(m.viewIdentity(100), m.validationErr) {
+		t.Error("the identity screen must render the error it set")
+	}
+}
+
+// Tab order follows the visible fields, which change with the addressing mode,
+// so it has to be derived from them rather than fixed.
+func TestRoleEditorTabsThroughVisibleFieldsOnly(t *testing.T) {
+	m := wizardModel(t, 1, 2)
+	m.screen = screenNetworkRole
+	m.roleCursor = 0 // wan, DHCP by default
+
+	m = press(t, m, "tab")
+	if m.roles[0].focus != roleFieldMethod {
+		t.Fatalf("focus = %v, want the IP method field", m.roles[0].focus)
+	}
+	m = press(t, m, "tab")
+	if m.roles[0].focus != roleFieldNIC {
+		t.Fatalf("focus = %v, want the DHCP form to wrap after two fields", m.roles[0].focus)
+	}
+
+	// Switching to static reveals the address fields, and tab must reach them.
+	m = press(t, m, "tab", "right")
+	if m.roles[0].dhcp {
+		t.Fatal("right on the method field must select static addressing")
+	}
+	m = press(t, m, "tab")
+	if m.roles[0].focus != roleFieldIP || !m.roles[0].ip.Focused() {
+		t.Fatalf("focus = %v, want the IP field focused", m.roles[0].focus)
+	}
+	m = press(t, m, "shift+tab")
+	if m.roles[0].focus != roleFieldMethod {
+		t.Fatalf("focus = %v, want shift+tab to go back", m.roles[0].focus)
+	}
+}
+
+// A focused text field has to receive typed runes, since that is the only way
+// an address is entered.
+func TestRoleEditorTypesIntoTheFocusedField(t *testing.T) {
+	m := wizardModel(t, 1, 2)
+	m.screen = screenNetworkRole
+	m.roles[0].dhcp = false
+	m.roles[0].focus = roleFieldIP
+	m.roles[0].focusCurrent()
+
+	m = typeText(t, m, "192.168.1.10")
+	if got := m.roles[0].ip.Value(); got != "192.168.1.10" {
+		t.Fatalf("ip = %q, want what was typed", got)
+	}
+	if got := m.roles[0].mask.Value(); got != "" {
+		t.Errorf("mask = %q, want the keys to reach only the focused field", got)
+	}
+}
+
+// The interface selector wraps and offers the fold option on lan and vpc only,
+// so a single-NIC node stays configurable with one key.
+func TestNICSelectorWrapsAndFoldsWhereAllowed(t *testing.T) {
+	m := wizardModel(t, 1, 2)
+	m.screen = screenNetworkRole
+
+	// wan cannot fold, so it wraps across the real NICs only.
+	m.roleCursor = 0
+	m = press(t, m, "left")
+	if m.roles[0].nic != len(m.nics)-1 {
+		t.Fatalf("wan nic = %d, want it to wrap to the last NIC", m.roles[0].nic)
+	}
+	m = press(t, m, "right")
+	if m.roles[0].nic != 0 {
+		t.Fatalf("wan nic = %d, want it to wrap back to the first", m.roles[0].nic)
+	}
+
+	// lan may fold, so the cycle includes the unbound position.
+	m.roleCursor = 1
+	m.roles[1].nic = 0
+	m = press(t, m, "left")
+	if m.roles[1].nic != foldedNIC {
+		t.Fatalf("lan nic = %d, want the fold position", m.roles[1].nic)
+	}
+	if !m.roles[1].folded() {
+		t.Error("the lan role should read as folded")
+	}
+}
+
+// Per-role validation runs on Enter and keeps the operator on the editor, since
+// an invalid address here fails the install after the disk is partitioned.
+func TestRoleEditorValidatesBeforeLeaving(t *testing.T) {
+	m := wizardModel(t, 1, 2)
+	m.screen = screenNetworkRole
+	m.roleCursor = 0
+	m.roles[0].dhcp = false
+	m.roles[0].ip.SetValue("not-an-ip")
+
+	m = press(t, m, "enter")
+	if m.screen != screenNetworkRole {
+		t.Fatalf("screen = %v, want to stay on the role editor", m.screen)
+	}
+	if !strings.Contains(m.validationErr, "IP address") {
+		t.Errorf("validationErr = %q, want it to name the field", m.validationErr)
+	}
+	if !strings.Contains(m.viewNetworkRole(100), m.validationErr) {
+		t.Error("the role editor must render the error it set")
+	}
+
+	// A prefix length is rejected here because netgen parses the mask with
+	// net.ParseIP and would fail the install itself.
+	m.roles[0].ip.SetValue("192.168.1.10")
+	m.roles[0].mask.SetValue("24")
+	m = press(t, m, "enter")
+	if !strings.Contains(m.validationErr, "dotted-decimal") {
+		t.Errorf("validationErr = %q, want the mask rule", m.validationErr)
+	}
+
+	m.roles[0].mask.SetValue("255.255.255.0")
+	m.roles[0].gateway.SetValue("192.168.1.1")
+	m = press(t, m, "enter")
+	if m.screen != screenNetworkRoles {
+		t.Fatalf("screen = %v, want a valid role to return to the overview (err %q)", m.screen, m.validationErr)
+	}
+}
+
+// Esc abandons the editor without validating, so a half-typed address is not a
+// trap the operator has to finish before backing out.
+func TestRoleEditorEscapeSkipsValidation(t *testing.T) {
+	m := wizardModel(t, 1, 2)
+	m.screen = screenNetworkRole
+	m.roles[0].dhcp = false
+	m.roles[0].ip.SetValue("192.")
+
+	m = press(t, m, "esc")
+	if m.screen != screenNetworkRoles {
+		t.Fatalf("screen = %v, want the overview", m.screen)
+	}
+	if m.validationErr != "" {
+		t.Errorf("validationErr = %q, want esc to clear it", m.validationErr)
+	}
+}
+
+// The editor shows only the fields that apply, so the DHCP form stays two lines
+// and Advanced is what reveals VLAN and MTU.
+func TestRoleEditorViewShowsTheApplicableFields(t *testing.T) {
+	m := wizardModel(t, 1, 3)
+	m.screen = screenNetworkRole
+	m.roleCursor = 0
+
+	dhcpView := m.viewNetworkRole(100)
+	if !strings.Contains(dhcpView, "DHCP") || !strings.Contains(dhcpView, "WAN") {
+		t.Errorf("DHCP view is missing the plane or the method:\n%s", dhcpView)
+	}
+	for _, hidden := range []string{"IP address", "Subnet mask", "VLAN id"} {
+		if strings.Contains(dhcpView, hidden) {
+			t.Errorf("DHCP view should not show %q:\n%s", hidden, dhcpView)
+		}
+	}
+
+	m.roles[0].dhcp = false
+	m.advanced = true
+	staticView := m.viewNetworkRole(100)
+	for _, want := range []string{"IP address", "Subnet mask", "Gateway", "DNS", "VLAN id", "MTU"} {
+		if !strings.Contains(staticView, want) {
+			t.Errorf("static advanced view is missing %q:\n%s", want, staticView)
+		}
+	}
+
+	// Gateway and DNS belong to the plane holding the default route, so a
+	// static lan must not offer them.
+	m.roleCursor = 1
+	m.roles[1].nic = 1
+	m.roles[1].dhcp = false
+	lanView := m.viewNetworkRole(100)
+	for _, hidden := range []string{"Gateway", "DNS"} {
+		if strings.Contains(lanView, hidden) {
+			t.Errorf("static lan view should not show %q:\n%s", hidden, lanView)
+		}
+	}
+}
+
+// A folded role names the plane it landed on, on both screens, so the operator
+// can see where the traffic actually goes before committing.
+func TestFoldedRolesAreNamedNotBlank(t *testing.T) {
+	m := wizardModel(t, 1, 1) // one NIC folds lan and vpc onto wan
+
+	overview := m.viewNetworkRoles(100)
+	if !strings.Contains(overview, "folds onto") {
+		t.Errorf("overview does not state where the folded roles land:\n%s", overview)
+	}
+
+	m.screen = screenNetworkRole
+	m.roleCursor = 2
+	if got := m.viewNetworkRole(100); !strings.Contains(got, "fold onto") {
+		t.Errorf("role editor does not offer the fold plainly:\n%s", got)
+	}
+
+	m.screen = screenConfirm
+	if got := m.viewConfirm(100); !strings.Contains(got, "folded onto") {
+		t.Errorf("confirmation does not state the folded planes:\n%s", got)
+	}
+}
+
+// Continue is refused while the config would fail, and the message is the one
+// install.Config.Validate produced.
+func TestRolesOverviewRefusesAnInvalidConfig(t *testing.T) {
+	m := wizardModel(t, 1, 2)
+	m.screen = screenNetworkRoles
+	m.roles[1].nic = 0 // lan on the same NIC as wan, no VLAN to separate them
+	m.roleCursor = continueRow
+
+	m = press(t, m, "enter")
+	if m.screen != screenNetworkRoles {
+		t.Fatalf("screen = %v, want to stay on the overview", m.screen)
+	}
+	if !strings.Contains(m.validationErr, "VLAN") {
+		t.Errorf("validationErr = %q, want the shared-NIC rule", m.validationErr)
+	}
+	// The message is wrapped to the box, so match its opening rather than
+	// the whole line.
+	if !strings.Contains(m.viewNetworkRoles(100), "both bind eno1") {
+		t.Error("the overview must render the error it set")
+	}
+
+	// a toggles Advanced for the whole node rather than per role.
+	m = press(t, m, "a")
+	if !m.advanced {
+		t.Error("a must turn Advanced on")
+	}
+	if !strings.Contains(m.viewNetworkRoles(100), "Advanced: on") {
+		t.Error("the overview must show Advanced is on")
+	}
+}
+
+// The final screen is the last chance to catch a wrong disk or address, so the
+// summary has to carry what was entered.
+func TestConfirmScreenSummarisesTheEnteredConfig(t *testing.T) {
+	m := wizardModel(t, 1, 3)
+	m.screen = screenConfirm
+	m.hostnameInput.SetValue("node7")
+	m.roles[1].dhcp = false
+	m.roles[1].ip.SetValue("10.0.0.3")
+	m.roles[1].mask.SetValue("255.255.255.0")
+
+	got := m.viewConfirm(100)
+	for _, want := range []string{"node7", "/dev/sda", "ext4", "eno1", "10.0.0.3/255.255.255.0", "br-lan", "will erase"} {
+		if !strings.Contains(got, want) {
+			t.Errorf("confirmation is missing %q:\n%s", want, got)
+		}
+	}
+}
+
+// n on the confirmation cancels rather than installing.
+func TestConfirmScreenCancelsOnN(t *testing.T) {
+	m := wizardModel(t, 1, 1)
+	m.screen = screenConfirm
+
+	m = press(t, m, "n")
+	if m.err == nil {
+		t.Fatal("n must set the cancellation error")
+	}
+	if m.result != nil {
+		t.Error("a cancelled install must produce no config")
+	}
+}
+
+// View dispatches on the screen, so each one has to render its own content
+// rather than falling through to an empty frame.
+func TestViewRendersEveryScreen(t *testing.T) {
+	tests := []struct {
+		screen screen
+		want   string
+	}{
+		{screenWelcome, "Enter to begin"},
+		{screenDisk, "Select Installation Disk"},
+		{screenZFSOptions, "ZFS Options"},
+		{screenDiskConfirm, "Confirm Disk Erasure"},
+		{screenNetworkRoles, "Network Planes"},
+		{screenNetworkRole, "Configure WAN plane"},
+		{screenIdentity, "Node Identity"},
+		{screenPassword, "Administrator Credentials"},
+		{screenConfirm, "Confirm Installation"},
+		{screenDone, "Installation complete"},
+	}
+
+	for _, tt := range tests {
+		m := wizardModel(t, 1, 2)
+		m.screen = tt.screen
+		if got := m.View(); !strings.Contains(got, tt.want) {
+			t.Errorf("screen %v does not render %q:\n%s", tt.screen, tt.want, got)
+		}
+	}
+}
+
+// The disk screen has to state what the selection actually builds, since the
+// cost of the redundancy choice is otherwise invisible until after the install.
+func TestDiskViewStatesTheResultingGeometry(t *testing.T) {
+	m := wizardModel(t, 3, 1)
+	m.screen = screenDisk
+
+	// ext4 with a role per drive: the mountpoints are the summary that matters.
+	m = press(t, m, "down", "down", " ", "down", " ")
+	if got := m.viewDisk(100); !strings.Contains(got, "/var/lib/spinifex") {
+		t.Errorf("ext4 view does not state the role layout:\n%s", got)
+	}
+	// Matched spares are suggested, never applied.
+	if got := m.viewDisk(100); !strings.Contains(got, "survive a disk failure") {
+		t.Errorf("view does not suggest the pool the matched disks allow:\n%s", got)
+	}
+
+	// A mirror with too few disks quotes the requirement, not a capacity.
+	m = wizardModel(t, 3, 1)
+	m.screen = screenDisk
+	m.fsCursor = slices.Index(install.AllFSTypes, install.FSZFSRAID1)
+	if got := m.viewDisk(100); !strings.Contains(got, "1 selected") {
+		t.Errorf("view does not state what the mirror is missing:\n%s", got)
+	}
+
+	// Once buildable it quotes usable capacity and the failures tolerated.
+	m = press(t, m, "down", "down", " ")
+	got := m.viewDisk(100)
+	if !strings.Contains(got, "usable") || !strings.Contains(got, "survives 1 disk failure") {
+		t.Errorf("view does not quote the pool geometry:\n%s", got)
+	}
+	if !strings.Contains(got, "A for ZFS options") {
+		t.Errorf("the ZFS help line should replace the role key:\n%s", got)
+	}
+}
+
+// The overview names the card behind each port, which is what distinguishes the
+// 25gbe uplink from the onboard 1gbe.
+func TestRolesOverviewNamesTheHardware(t *testing.T) {
+	m := wizardModel(t, 1, 2)
+	m.nics[0].AltName = "enp1s0f0"
+	m.advanced = true
+
+	got := m.viewNetworkRoles(100)
+	for _, want := range []string{"eno1", "25Gbps", "Mellanox", "enp1s0f0", "VLAN", "MTU"} {
+		if !strings.Contains(got, want) {
+			t.Errorf("overview is missing %q:\n%s", want, got)
+		}
+	}
+}
+
+func TestHumanBytes(t *testing.T) {
+	tests := []struct {
+		bytes int64
+		want  string
+	}{
+		{512, "512B"},
+		{2 << 20, "2.0M"},
+		{3 << 30, "3.0G"},
+		{2 << 40, "2.0T"},
+	}
+	for _, tt := range tests {
+		if got := humanBytes(tt.bytes); got != tt.want {
+			t.Errorf("humanBytes(%d) = %q, want %q", tt.bytes, got, tt.want)
+		}
+	}
+}
+
+// The overview writes an address the way an operator does, and falls back to
+// the raw mask rather than dropping it when it is not dotted-decimal.
+func TestFormatCIDR(t *testing.T) {
+	tests := []struct {
+		name       string
+		addr, mask string
+		want       string
+	}{
+		{"dotted mask becomes a prefix", "10.0.0.3", "255.255.255.0", "10.0.0.3/24"},
+		{"short mask keeps its prefix", "10.0.0.3", "255.255.0.0", "10.0.0.3/16"},
+		{"no mask yet", "10.0.0.3", "", "10.0.0.3"},
+		{"no address is blank", "", "255.255.255.0", ""},
+		{"unparseable mask is shown as typed", "10.0.0.3", "24", "10.0.0.3/24"},
+		{"whitespace is trimmed", " 10.0.0.3 ", " 255.255.255.0 ", "10.0.0.3/24"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := formatCIDR(tt.addr, tt.mask); got != tt.want {
+				t.Errorf("formatCIDR(%q, %q) = %q, want %q", tt.addr, tt.mask, got, tt.want)
+			}
+		})
+	}
+}
+
+// Messages that are not keys reach the focused input, which is what keeps the
+// cursor blinking on the two single-field screens.
+func TestNonKeyMessagesReachTheActiveInput(t *testing.T) {
+	m := wizardModel(t, 1, 1)
+	m.screen = screenIdentity
+	m.hostnameInput.Focus()
+
+	next, _ := m.Update(struct{ tea.Msg }{})
+	if _, ok := next.(model); !ok {
+		t.Fatalf("Update returned %T", next)
+	}
+
+	m.screen = screenWelcome
+	if _, cmd := m.Update(struct{ tea.Msg }{}); cmd != nil {
+		t.Error("a screen with no input must issue no command")
+	}
+}

cmd/lb-agent/main.go

@@ -21,7 +21,7 @@ import (
 	"github.com/mulgadc/bluebottle/pkg/otelsetup"
 	"github.com/mulgadc/spinifex/spinifex/lbagent"
 
-	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {

cmd/rds-agent/agent.go

@@ -15,6 +15,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/gwsign"
 	"github.com/mulgadc/spinifex/internal/rdsgw"
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 )
 
 // Overridable via -ldflags "-X main.version=...".
@@ -406,10 +407,10 @@ func retryObserved(ctx context.Context, what string, fn func(context.Context) er
 		}
 		if attempt >= retryErrorAttempt {
 			slog.ErrorContext(ctx, "rds-agent: "+what+" still failing, retrying",
-				"attempt", attempt, "retryIn", delay, "err", err)
+				"attempt", attempt, "retry_in_ms", otelsetup.Millis(delay), "err", err)
 		} else {
 			slog.WarnContext(ctx, "rds-agent: "+what+" failed, retrying",
-				"attempt", attempt, "retryIn", delay, "err", err)
+				"attempt", attempt, "retry_in_ms", otelsetup.Millis(delay), "err", err)
 		}
 
 		select {

cmd/rds-agent/agent_test.go

@@ -207,6 +207,8 @@ func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput
 		MasterUsername:       "master",
 		MasterUserPassword:   password,
 		Port:                 5432,
+		ListenAddress:        testListenAddress,
+		ClientCIDR:           testClientCIDR,
 		DataVolumeID:         "vol-data-01",
 		DataVolumeSerial:     "voldata01",
 		VMGeneration:         1,
@@ -217,6 +219,13 @@ func bootstrapOutput(password *string) *handlers_rds.GetDBBootstrapConfigOutput
 
 const testPayloadID = "bp-0123456789abcdef"
 
+// The customer ENI address the engine binds, and the VPC range its client
+// authentication admits. writeHandoff refuses a config without either.
+const (
+	testListenAddress = "10.0.1.20"
+	testClientCIDR    = "10.0.0.0/16"
+)
+
 // Writes the completion receipt rds-init leaves behind, so the acknowledgement
 // step finds what a finished initialization would have produced.
 func writeReceipt(t *testing.T, a *Agent, payloadID, dbInstanceIdentifier string) {

cmd/rds-agent/bootstrap.go

@@ -96,6 +96,15 @@ func writeHandoff(dir string, cfg *handlers_rds.GetDBBootstrapConfigOutput) erro
 	if cfg.MasterUsername == "" {
 		return fmt.Errorf("bootstrap config carries no master username")
 	}
+	// Half a handoff would leave rds-init deciding what to bind and who to admit,
+	// and the only answers it could reach on its own are the wildcards this
+	// contract exists to remove.
+	if cfg.ListenAddress == "" {
+		return fmt.Errorf("bootstrap config carries no listen address for the engine to bind")
+	}
+	if cfg.ClientCIDR == "" {
+		return fmt.Errorf("bootstrap config carries no client CIDR to scope client authentication to")
+	}
 	if err := os.MkdirAll(dir, handoffDirMode); err != nil {
 		return fmt.Errorf("create handoff dir %s: %w", dir, err)
 	}
@@ -134,6 +143,8 @@ func renderBootstrapEnv(cfg *handlers_rds.GetDBBootstrapConfigOutput) string {
 	writeEnvLine(&b, "RDS_MODE", cfg.Mode)
 	writeEnvLine(&b, "RDS_DB_INSTANCE_IDENTIFIER", cfg.DBInstanceIdentifier)
 	writeEnvLine(&b, "RDS_MASTER_USERNAME", cfg.MasterUsername)
+	writeEnvLine(&b, "RDS_LISTEN_ADDRESS", cfg.ListenAddress)
+	writeEnvLine(&b, "RDS_CLIENT_CIDR", cfg.ClientCIDR)
 	writeEnvLine(&b, "RDS_DATA_VOLUME_ID", cfg.DataVolumeID)
 	writeEnvLine(&b, "RDS_DATA_VOLUME_SERIAL", cfg.DataVolumeSerial)
 	writeEnvLine(&b, "RDS_VM_GENERATION", strconv.FormatInt(cfg.VMGeneration, 10))

cmd/rds-agent/bootstrap_test.go

@@ -18,6 +18,8 @@ func TestWriteHandoff_WritesEveryFileRootOnly(t *testing.T) {
 		Mode:               handlers_rds.BootstrapModeInitialize,
 		Engine:             "postgres",
 		MasterUsername:     "master",
+		ListenAddress:      "10.0.1.20",
+		ClientCIDR:         "10.0.0.0/16",
 		MasterUserPassword: &password,
 		DBName:             "appdb",
 		Port:               5433,
@@ -85,6 +87,8 @@ func TestWriteHandoff_AttachOmitsPassword(t *testing.T) {
 		Mode:           handlers_rds.BootstrapModeAttach,
 		Engine:         "postgres",
 		MasterUsername: "master",
+		ListenAddress:  "10.0.1.20",
+		ClientCIDR:     "10.0.0.0/16",
 		Port:           5432,
 	}); err != nil {
 		t.Fatalf("writeHandoff: %v", err)
@@ -116,6 +120,8 @@ func TestWriteHandoff_QuotesShellMetacharacters(t *testing.T) {
 		Mode:               handlers_rds.BootstrapModeInitialize,
 		Engine:             "postgres",
 		MasterUsername:     "master",
+		ListenAddress:      "10.0.1.20",
+		ClientCIDR:         "10.0.0.0/16",
 		MasterUserPassword: &password,
 		Port:               5432,
 	}); err != nil {
@@ -137,6 +143,8 @@ func TestWriteHandoff_SkipsHalfATLSPair(t *testing.T) {
 		Mode:               handlers_rds.BootstrapModeAttach,
 		Engine:             "postgres",
 		MasterUsername:     "master",
+		ListenAddress:      "10.0.1.20",
+		ClientCIDR:         "10.0.0.0/16",
 		Port:               5432,
 		ServingCertificate: "CERT",
 	}); err != nil {
@@ -177,6 +185,8 @@ func TestWriteHandoff_TightensExistingDirectory(t *testing.T) {
 		Mode:           handlers_rds.BootstrapModeAttach,
 		Engine:         "postgres",
 		MasterUsername: "master",
+		ListenAddress:  "10.0.1.20",
+		ClientCIDR:     "10.0.0.0/16",
 		Port:           5432,
 	}); err != nil {
 		t.Fatalf("writeHandoff: %v", err)

cmd/rds-agent/command.go

@@ -50,7 +50,13 @@ func newCommandRegistry(engine engineOps, storage storageOps) commandRegistry {
 			if err != nil {
 				return "", err
 			}
-			return "", engine.Quiesce(ctx, params[handlers_rds.CommandParamQuiesceLabel], hold)
+			if err := engine.Quiesce(ctx, params[handlers_rds.CommandParamQuiesceLabel], hold); err != nil {
+				return "", err
+			}
+			// After the hold, so what is flushed is the engine's checkpointed
+			// state. Its own files are not the risk — those it repairs from the
+			// WAL — but the parameter file beside them it never opens.
+			return "", storage.SyncDataMount(ctx)
 		},
 		handlers_rds.CommandUnquiesce: func(ctx context.Context, cmd handlers_rds.Command) (string, error) {
 			return "", engine.Unquiesce(ctx)

cmd/rds-agent/config.go

@@ -13,7 +13,6 @@ const (
 	defaultEnvFile    = "/etc/spinifex-rds/agent.env"
 	defaultGatewayCA  = "/etc/spinifex-rds/gateway-ca.pem"
 	defaultHandoffDir = "/run/spinifex-rds"
-	defaultEngineHost = "127.0.0.1"
 	// Where setup.sh stamps the engine the image bakes. The agent builds its
 	// engine implementation from this file rather than from anything delivered.
 	defaultEngineFile         = "/etc/spinifex-rds/engine"
@@ -38,7 +37,6 @@ type config struct {
 	// version alone rather than clearing it.
 	EngineVersion string
 	HandoffDir    string
-	EngineHost    string
 	EnginePort    int
 	PollWait      time.Duration
 
@@ -89,7 +87,6 @@ func loadConfig(envFile string) config {
 		EngineFile:           get("RDS_ENGINE_FILE"),
 		EngineVersion:        get("RDS_ENGINE_VERSION"),
 		HandoffDir:           get("RDS_HANDOFF_DIR"),
-		EngineHost:           get("RDS_ENGINE_HOST"),
 		EngineBinDir:         get("RDS_ENGINE_BIN"),
 		EngineDataDir:        get("RDS_ENGINE_DATA"),
 		SocketDir:            get("RDS_SOCKET_DIR"),
@@ -108,9 +105,6 @@ func loadConfig(envFile string) config {
 	if cfg.HandoffDir == "" {
 		cfg.HandoffDir = defaultHandoffDir
 	}
-	if cfg.EngineHost == "" {
-		cfg.EngineHost = defaultEngineHost
-	}
 	if cfg.EngineFile == "" {
 		cfg.EngineFile = defaultEngineFile
 	}

cmd/rds-agent/config_test.go

@@ -26,8 +26,8 @@ func TestLoadConfig_Defaults(t *testing.T) {
 	if cfg.GatewayCA != defaultGatewayCA || cfg.HandoffDir != defaultHandoffDir {
 		t.Errorf("CA/handoff = %q/%q, want the built-in defaults", cfg.GatewayCA, cfg.HandoffDir)
 	}
-	if cfg.EngineHost != defaultEngineHost || cfg.PollWait != defaultPollWait {
-		t.Errorf("engine host/pollWait = %q/%v, want the built-in defaults", cfg.EngineHost, cfg.PollWait)
+	if cfg.PollWait != defaultPollWait {
+		t.Errorf("pollWait = %v, want the built-in default", cfg.PollWait)
 	}
 	if cfg.EngineProbeTimeout != defaultEngineProbeTimeout {
 		t.Errorf("engine probe timeout = %v, want %v", cfg.EngineProbeTimeout, defaultEngineProbeTimeout)

cmd/rds-agent/engine.go

@@ -8,6 +8,7 @@ import (
 	"os/exec"
 	"os/user"
 	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"
 	"syscall"
@@ -107,10 +108,12 @@ var engineLayouts = map[string]engineLayout{
 		service:   "postgresql",
 		dataMount: "/var/lib/postgresql",
 		port:      5432,
+		// logfile in the image's /etc/conf.d/postgresql. The postmaster states
+		// which setting it refused there and nowhere else — the console shows only
+		// that pg_ctl could not start it, on a guest with no SSH.
+		errorLog:  "/var/log/postgresql/postmaster.log",
 		newProbe:  newPostgresProbe,
-		newEngine: func(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
-			return newPostgresEngine(cfg, run, startSess, probe), nil
-		},
+		newEngine: newPostgresEngineFromCatalog,
 	},
 	engineMariaDB: {
 		binDir:    "/usr/bin",
@@ -133,6 +136,61 @@ var engineLayouts = map[string]engineLayout{
 	},
 }
 
+// How much of the engine's error log a probe reason carries. The server names
+// what it refused on in its last few lines, and the reason reaches a customer
+// through StatusInfos, so this quotes the tail rather than the whole file.
+const (
+	engineErrorLogTailBytes = 4096
+	engineErrorLogTailLines = 4
+)
+
+// Appends the last few lines of the engine's error log to reason. An unset path,
+// an unreadable file or an empty one leaves reason alone: the probe's own
+// statement is still true, and a guessed cause would be worse than none.
+func withEngineLogTail(reason, path string) string {
+	tail := engineLogTail(path)
+	if tail == "" {
+		return reason
+	}
+	return reason + "; the engine's log ends: " + tail
+}
+
+func engineLogTail(path string) string {
+	if path == "" {
+		return ""
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return ""
+	}
+	defer func() { _ = f.Close() }()
+
+	info, err := f.Stat()
+	if err != nil {
+		return ""
+	}
+	offset := max(info.Size()-engineErrorLogTailBytes, 0)
+	buf := make([]byte, info.Size()-offset)
+	if _, err := f.ReadAt(buf, offset); err != nil {
+		return ""
+	}
+
+	lines := strings.Split(string(buf), "\n")
+	// A read that started mid-file almost certainly started mid-line, and half a
+	// line is more misleading than one fewer.
+	if offset > 0 && len(lines) > 0 {
+		lines = lines[1:]
+	}
+	kept := make([]string, 0, engineErrorLogTailLines)
+	for i := len(lines) - 1; i >= 0 && len(kept) < engineErrorLogTailLines; i-- {
+		if line := strings.TrimSpace(lines[i]); line != "" {
+			kept = append(kept, line)
+		}
+	}
+	slices.Reverse(kept)
+	return strings.Join(kept, " | ")
+}
+
 // Builds the health probe the image's engine supplies.
 func newProbe(cfg config, run probeRunner) (*engineProbe, error) {
 	layout, err := layoutFor(cfg)

cmd/rds-agent/engine_log_test.go

@@ -148,7 +148,7 @@ func TestEngineLogTail(t *testing.T) {
 		},
 		{
 			name:        "drops a line the byte cap cut in half",
-			body:        strings.Repeat("x", mariadbErrorLogTailBytes) + "\nlast\n",
+			body:        strings.Repeat("x", engineErrorLogTailBytes) + "\nlast\n",
 			want:        "last",
 			wantMissing: "x",
 		},
@@ -209,3 +209,54 @@ func TestWithEngineLogTailLeavesReasonAloneWithoutALog(t *testing.T) {
 		t.Errorf("withEngineLogTail() = %q, want the reason unchanged", got)
 	}
 }
+
+const postgresConfD = "../../scripts/images/rds-postgres/postgresql.confd"
+
+// "engine did not respond" is where a postmaster that refused a setting lands,
+// and it is the same message as an engine that never got as far as trying. Only
+// the server's own log separates them, and the guest has no other way out.
+func TestPostgresProbe_QuotesTheEngineLogWhenTheEngineIsAbsent(t *testing.T) {
+	log := writeEngineLog(t, strings.Join([]string{
+		`2026-09-06 10:20:33 UTC [712] LOG:  starting PostgreSQL 18.0`,
+		`2026-09-06 10:20:33 UTC [712] FATAL:  could not map anonymous shared memory`,
+		"",
+	}, "\n"))
+
+	state, message := postgresProbeState("/run/postgresql", log,
+		func(context.Context, string, ...string) (int, string, error) { return 2, "", nil })(t.Context(), 5432)
+
+	if state != engineAbsent {
+		t.Errorf("state = %v, want absent", state)
+	}
+	if !strings.Contains(message, "could not map anonymous shared memory") {
+		t.Errorf("message = %q, want the postmaster's own refusal quoted", message)
+	}
+}
+
+func TestPostgresProbe_SurvivesAnAbsentEngineLog(t *testing.T) {
+	state, message := postgresProbeState("/run/postgresql", filepath.Join(t.TempDir(), "nope.log"),
+		func(context.Context, string, ...string) (int, string, error) { return 2, "", nil })(t.Context(), 5432)
+
+	if state != engineAbsent {
+		t.Errorf("state = %v, want absent", state)
+	}
+	if message != "engine did not respond on /run/postgresql:5432" {
+		t.Errorf("message = %q, want the bare reason when there is no log to quote", message)
+	}
+}
+
+// The layout names a path the image has to be directing the postmaster to, or
+// the tail above quotes nothing on the one failure it exists for.
+func TestPostgresErrorLogPathMatchesTheImage(t *testing.T) {
+	raw, err := os.ReadFile(postgresConfD)
+	if err != nil {
+		t.Fatalf("read postgresql.confd: %v", err)
+	}
+	m := regexp.MustCompile(`(?m)^logfile="(.*)"$`).FindStringSubmatch(string(raw))
+	if m == nil {
+		t.Fatal("postgresql.confd has no logfile assignment")
+	}
+	if got := engineLayouts[enginePostgres].errorLog; got != m[1] {
+		t.Errorf("layout errorLog = %q, but the image sets logfile = %q", got, m[1])
+	}
+}

cmd/rds-agent/engine_mariadb.go

@@ -100,11 +100,6 @@ const (
 	// rds-init keeps for the platform and never hands to the customer.
 	mariadbSuperuser                  = "root"
 	mariadbProbeConnectTimeoutSeconds = 3
-	// How much of the engine's error log a probe reason carries. The server names
-	// what it refused on in its last few lines, and the reason reaches a customer
-	// through StatusInfos, so this quotes the tail rather than the whole file.
-	mariadbErrorLogTailBytes = 4096
-	mariadbErrorLogTailLines = 4
 	// How much of the probe client's stderr a reason carries, on the same grounds
 	// as the log tail above.
 	mariadbProbeStderrMaxBytes = 512
@@ -280,53 +275,6 @@ func collapseProbeStderr(stderr string) string {
 	return line
 }
 
-// Appends the last few lines of the engine's error log to reason. An unset path,
-// an unreadable file or an empty one leaves reason alone: the probe's own
-// statement is still true, and a guessed cause would be worse than none.
-func withEngineLogTail(reason, path string) string {
-	tail := engineLogTail(path)
-	if tail == "" {
-		return reason
-	}
-	return reason + "; the engine's log ends: " + tail
-}
-
-func engineLogTail(path string) string {
-	if path == "" {
-		return ""
-	}
-	f, err := os.Open(path)
-	if err != nil {
-		return ""
-	}
-	defer func() { _ = f.Close() }()
-
-	info, err := f.Stat()
-	if err != nil {
-		return ""
-	}
-	offset := max(info.Size()-mariadbErrorLogTailBytes, 0)
-	buf := make([]byte, info.Size()-offset)
-	if _, err := f.ReadAt(buf, offset); err != nil {
-		return ""
-	}
-
-	lines := strings.Split(string(buf), "\n")
-	// A read that started mid-file almost certainly started mid-line, and half a
-	// line is more misleading than one fewer.
-	if offset > 0 && len(lines) > 0 {
-		lines = lines[1:]
-	}
-	kept := make([]string, 0, mariadbErrorLogTailLines)
-	for i := len(lines) - 1; i >= 0 && len(kept) < mariadbErrorLogTailLines; i-- {
-		if line := strings.TrimSpace(lines[i]); line != "" {
-			kept = append(kept, line)
-		}
-	}
-	slices.Reverse(kept)
-	return strings.Join(kept, " | ")
-}
-
 func readPidFile(path string) (int, error) {
 	raw, err := os.ReadFile(path)
 	if err != nil {

cmd/rds-agent/engine_mariadb_test.go

@@ -129,10 +129,11 @@ func newScriptedTestMariaDBEngine(t *testing.T, run commandRunner) *mariadbEngin
 // that is registered: what a spec says about a parameter is what the guest must
 // see, and an unlisted name must read as static rather than as dynamic.
 func TestControlPlaneRulesFrom_TakesTheClassificationFromTheCatalog(t *testing.T) {
-	rules := controlPlaneRulesFrom(postgresEngineMeta)
+	meta := testPostgresEngineMeta(t)
+	rules := controlPlaneRulesFrom(meta)
 
 	for name, want := range map[string]bool{"shared_buffers": true, "work_mem": false} {
-		spec, ok := postgresEngineMeta.LookupParameter(name)
+		spec, ok := meta.LookupParameter(name)
 		if !ok {
 			t.Fatalf("the catalog no longer carries %s", name)
 		}

cmd/rds-agent/engine_postgres.go

@@ -41,11 +41,16 @@ var _ engine = (*postgresEngine)(nil)
 // is not there. Resolved on PATH, where the client package puts it.
 const postgresProbeBinary = "pg_isready"
 
+// The socket directory rather than loopback: the engine binds the customer ENI
+// and nothing else, so there is no TCP path for the probe to take. It also keeps
+// liveness independent of the network path, of the pg_hba scope and of the TLS
+// enforcement rule — libpq reads a host beginning with / as a socket directory,
+// and the generated `local ... peer` line covers it.
 func newPostgresProbe(cfg config, run probeRunner) *engineProbe {
-	return newEngineProbe(cfg.EnginePort, postgresProbeState(cfg.EngineHost, run))
+	return newEngineProbe(cfg.EnginePort, postgresProbeState(cfg.SocketDir, cfg.EngineErrorLog, run))
 }
 
-func postgresProbeState(host string, run probeRunner) probeStateFn {
+func postgresProbeState(host, errorLog string, run probeRunner) probeStateFn {
 	return func(ctx context.Context, port int64) (engineState, string) {
 		portArg := strconv.FormatInt(port, 10)
 		code, _, err := run(ctx, postgresProbeBinary, "-h", host, "-p", portArg, "-q")
@@ -58,26 +63,18 @@ func postgresProbeState(host string, run probeRunner) probeStateFn {
 		case code == 0:
 			return engineServing, ""
 		case code == 1:
-			return engineRecovering, "engine is rejecting connections (startup or recovery)"
+			return engineRecovering, withEngineLogTail(
+				"engine is rejecting connections (startup or recovery)", errorLog)
 		default:
-			return engineAbsent, fmt.Sprintf("engine did not respond on %s:%s", host, portArg)
+			// A postmaster that refused a setting never opens the socket, so this is
+			// the state a bad parameter lands in. The reason for it is only in the
+			// server's own log, and it reaches the customer through StatusInfos.
+			return engineAbsent, withEngineLogTail(
+				fmt.Sprintf("engine did not respond on %s:%s", host, portArg), errorLog)
 		}
 	}
 }
 
-// Resolved once, under the name the control plane knows this implementation by.
-// A lookup failure is a mismatch between the agent and the control plane it was
-// built against, so it fails at startup rather than at the first rotation.
-var postgresEngineMeta = mustLookupEngine(enginePostgres)
-
-func mustLookupEngine(name string) handlers_rds.Engine {
-	engine, err := handlers_rds.LookupEngine(name)
-	if err != nil {
-		panic("rds-agent: " + err.Error())
-	}
-	return engine
-}
-
 // The include the resolved set is rendered to, and the copy of the last one the
 // engine accepted. Both live beside the data rather than in /etc, matching
 // rds-init: a class change boots a fresh root volume and would revert them.
@@ -109,9 +106,18 @@ hostnossl all all ::/0 reject
 	postgresForceSSLRuleCount = 2
 )
 
-func newPostgresEngine(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) *postgresEngine {
+// The layout's factory resolves the control plane metadata during startup.
+func newPostgresEngineFromCatalog(cfg config, run commandRunner, startSess sessionRunner, probe *engineProbe) (engine, error) {
+	meta, err := handlers_rds.LookupEngine(enginePostgres)
+	if err != nil {
+		return nil, fmt.Errorf("this image bakes %s, which this build's control plane does not offer: %w", enginePostgres, err)
+	}
+	return newPostgresEngine(cfg, meta, run, startSess, probe), nil
+}
+
+func newPostgresEngine(cfg config, meta handlers_rds.Engine, run commandRunner, startSess sessionRunner, probe *engineProbe) *postgresEngine {
 	return &postgresEngine{
-		meta:      postgresEngineMeta,
+		meta:      meta,
 		run:       run,
 		startSess: startSess,
 		psql:      filepath.Join(cfg.EngineBinDir, "psql"),

cmd/rds-agent/engine_test.go

@@ -140,10 +140,19 @@ func (r *recordingRunner) run(_ context.Context, c command) (string, error) {
 	return r.out, r.err
 }
 
+func testPostgresEngineMeta(t *testing.T) handlers_rds.Engine {
+	t.Helper()
+	meta, err := handlers_rds.LookupEngine(enginePostgres)
+	if err != nil {
+		t.Fatalf("LookupEngine(%s): %v", enginePostgres, err)
+	}
+	return meta
+}
+
 func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
 	t.Helper()
 	cfg := testEngineConfig(t)
-	return newPostgresEngine(cfg, withPostgresReadBacks(cfg.EngineDataDir, run), nil, newPostgresProbe(cfg, staticProbe(0)))
+	return newPostgresEngine(cfg, testPostgresEngineMeta(t), withPostgresReadBacks(cfg.EngineDataDir, run), nil, newPostgresProbe(cfg, staticProbe(0)))
 }
 
 // The same engine with the read-backs left to the runner, for the cases that are
@@ -152,7 +161,7 @@ func newTestEngine(t *testing.T, run commandRunner) *postgresEngine {
 func newScriptedTestEngine(t *testing.T, run commandRunner) *postgresEngine {
 	t.Helper()
 	cfg := testEngineConfig(t)
-	return newPostgresEngine(cfg, run, nil, newPostgresProbe(cfg, staticProbe(0)))
+	return newPostgresEngine(cfg, testPostgresEngineMeta(t), run, nil, newPostgresProbe(cfg, staticProbe(0)))
 }
 
 func testEngineConfig(t *testing.T) config {
@@ -479,10 +488,11 @@ func (r *tlsReadBackRunner) run(ctx context.Context, c command) (string, error)
 	return r.recordingRunner.run(ctx, c)
 }
 
-func forceSSLParameter(value string) []handlers_rds.Parameter {
+func forceSSLParameter(t *testing.T, value string) []handlers_rds.Parameter {
+	t.Helper()
 	return []handlers_rds.Parameter{
 		{Name: "work_mem", Value: "4096"},
-		{Name: postgresEngineMeta.TLSEnforcementParameter(), Value: value},
+		{Name: testPostgresEngineMeta(t).TLSEnforcementParameter(), Value: value},
 	}
 }
 
@@ -492,7 +502,7 @@ func TestPostgresEngine_ApplyParametersWritesTheEnforcementRule(t *testing.T) {
 	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
 	engine := newScriptedTestEngine(t, runner.run)
 
-	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("1")); err != nil {
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1")); err != nil {
 		t.Fatalf("ApplyParameters: %v", err)
 	}
 	rule, err := os.ReadFile(engine.forceSSLRulePath())
@@ -516,7 +526,7 @@ func TestPostgresEngine_ApplyParametersRemovesAStaleEnforcementRule(t *testing.T
 		t.Fatalf("write the rule the restored volume carried: %v", err)
 	}
 
-	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("0")); err != nil {
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "0")); err != nil {
 		t.Fatalf("ApplyParameters: %v", err)
 	}
 	if _, err := os.Stat(engine.forceSSLRulePath()); !os.IsNotExist(err) {
@@ -547,7 +557,7 @@ func TestPostgresEngine_ApplyParametersRefusesAnUnparsableEnforcementValue(t *te
 	runner := &tlsReadBackRunner{ssl: "on"}
 	engine := newScriptedTestEngine(t, runner.run)
 
-	_, err := engine.ApplyParameters(t.Context(), forceSSLParameter("yes"))
+	_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "yes"))
 	if err == nil || !strings.Contains(err.Error(), "neither 1 nor 0") {
 		t.Fatalf("ApplyParameters error = %v, want a refusal naming the unreadable value", err)
 	}
@@ -562,7 +572,7 @@ func TestPostgresEngine_ApplyParametersRefusesToEnforceWithoutTLS(t *testing.T)
 	runner := &tlsReadBackRunner{ssl: "off"}
 	engine := newScriptedTestEngine(t, runner.run)
 
-	_, err := engine.ApplyParameters(t.Context(), forceSSLParameter("1"))
+	_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1"))
 	if err == nil || !strings.Contains(err.Error(), "serving without TLS") {
 		t.Fatalf("ApplyParameters error = %v, want a refusal naming the engine's own TLS state", err)
 	}
@@ -592,7 +602,7 @@ func TestPostgresEngine_ApplyParametersFailsAnUnverifiableEnforcement(t *testing
 			runner := &tlsReadBackRunner{ssl: "on", ruleRows: tc.ruleRows, brokenRows: tc.brokenRows}
 			engine := newScriptedTestEngine(t, runner.run)
 
-			_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(tc.value))
+			_, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, tc.value))
 			if err == nil || !strings.Contains(err.Error(), tc.want) {
 				t.Fatalf("ApplyParameters error = %v, want %q", err, tc.want)
 			}
@@ -606,13 +616,13 @@ func TestPostgresEngine_ApplyParametersFailsAnUnverifiableEnforcement(t *testing
 func TestPostgresEngine_ApplyParametersWithdrawsBothFilesTogether(t *testing.T) {
 	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
 	engine := newScriptedTestEngine(t, runner.run)
-	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("1")); err != nil {
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1")); err != nil {
 		t.Fatalf("first ApplyParameters: %v", err)
 	}
 
 	// Turning enforcement off, verified against a pg_hba that still carries the
 	// rule — so the apply fails after both files have already been replaced.
-	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("0")); err == nil {
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "0")); err == nil {
 		t.Fatal("ApplyParameters succeeded against an unverifiable enforcement")
 	}
 	installed, err := os.ReadFile(engine.params.installedPath())
@@ -639,14 +649,14 @@ func TestPostgresEngine_RestoreLastKnownGoodReDerivesEnforcement(t *testing.T) {
 	runner := &tlsReadBackRunner{ssl: "on", ruleRows: postgresForceSSLRuleCount}
 	engine := newScriptedTestEngine(t, runner.run)
 
-	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("1")); err != nil {
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "1")); err != nil {
 		t.Fatalf("ApplyParameters(enforcing): %v", err)
 	}
 	if err := engine.RecordServingParameters(t.Context()); err != nil {
 		t.Fatalf("RecordServingParameters: %v", err)
 	}
 	runner.ruleRows = 0
-	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter("0")); err != nil {
+	if _, err := engine.ApplyParameters(t.Context(), forceSSLParameter(t, "0")); err != nil {
 		t.Fatalf("ApplyParameters(not enforcing): %v", err)
 	}
 	if _, err := os.Stat(engine.forceSSLRulePath()); !os.IsNotExist(err) {

cmd/rds-agent/health.go

@@ -28,8 +28,7 @@ func execProbeRunner(ctx context.Context, name string, args ...string) (int, str
 	if ctxErr := ctx.Err(); ctxErr != nil {
 		return -1, out, ctxErr
 	}
-	var exitErr *exec.ExitError
-	if errors.As(err, &exitErr) {
+	if exitErr, ok := errors.AsType[*exec.ExitError](err); ok {
 		return exitErr.ExitCode(), out, nil
 	}
 	return -1, out, err

cmd/rds-agent/health_test.go

@@ -110,6 +110,37 @@ func TestExecProbeRunner_CapturesTheClientsStderr(t *testing.T) {
 	}
 }
 
-func testProbeConfig() config {
-	return config{EngineHost: defaultEngineHost, EnginePort: engineLayouts[enginePostgres].port}
+// The engine binds the customer ENI and nothing else, so a loopback probe would
+// report a serving engine as absent. libpq reads a host beginning with / as a
+// socket directory, which is also what keeps liveness off the network path.
+func TestPostgresProbe_ProbesTheUnixSocket(t *testing.T) {
+	cfg := testProbeConfig()
+	var probed []string
+	probe := newPostgresProbe(cfg, func(_ context.Context, name string, args ...string) (int, string, error) {
+		probed = append([]string{name}, args...)
+		return 0, "", nil
+	})
+
+	if state, _ := probe.state(context.Background()); state != engineServing {
+		t.Fatalf("probe state = %v, want engineServing", state)
+	}
+	host := ""
+	for i, arg := range probed {
+		if arg == "-h" && i+1 < len(probed) {
+			host = probed[i+1]
+		}
+	}
+	if host != cfg.SocketDir {
+		t.Errorf("probe host = %q, want the socket directory %q", host, cfg.SocketDir)
+	}
+	if !strings.HasPrefix(host, "/") {
+		t.Errorf("probe host = %q, which libpq reads as a TCP host rather than a socket directory", host)
+	}
+}
+
+func testProbeConfig() config {
+	return config{
+		SocketDir:  engineLayouts[enginePostgres].socketDir,
+		EnginePort: engineLayouts[enginePostgres].port,
+	}
 }

cmd/rds-agent/heartbeat.go

@@ -90,10 +90,26 @@ func (h *heartbeater) beat(ctx context.Context) {
 	h.setInterval(time.Duration(out.HeartbeatIntervalSeconds) * time.Second)
 }
 
+// How often a not-yet-serving engine is reported. The control plane owns the
+// steady-state cadence, but while the engine has never answered it is actively
+// waiting on this instance — a restart is not finished until a healthy beat
+// lands — and a full interval of silence there is pure added downtime.
+const startupHeartbeatInterval = 5 * time.Second
+
+func (h *heartbeater) nextInterval() time.Duration {
+	if h.probe != nil && !h.probe.seenHealthy && startupHeartbeatInterval < h.interval {
+		return startupHeartbeatInterval
+	}
+	return h.interval
+}
+
 // A timer rather than a ticker, so a cadence change takes effect on the next
-// beat, not at a restart.
+// beat, not at a restart. The first beat is not waited for: a just-registered
+// agent has told the control plane nothing about the engine yet, and holding
+// that back for an interval makes every restart look longer than it was.
 func (h *heartbeater) Run(ctx context.Context) {
-	timer := time.NewTimer(h.interval)
+	h.beat(ctx)
+	timer := time.NewTimer(h.nextInterval())
 	defer timer.Stop()
 	for {
 		select {
@@ -101,7 +117,7 @@ func (h *heartbeater) Run(ctx context.Context) {
 			return
 		case <-timer.C:
 			h.beat(ctx)
-			timer.Reset(h.interval)
+			timer.Reset(h.nextInterval())
 		}
 	}
 }

cmd/rds-agent/heartbeat_test.go

@@ -88,3 +88,34 @@ func (r *countingServingRecorder) RecordServingParameters(context.Context) error
 	r.calls++
 	return nil
 }
+
+// A restart is not finished until a healthy beat lands, so the two cadence
+// rules that shorten it are worth pinning: report at once rather than after a
+// full interval, and keep reporting quickly while the engine has never served.
+func TestHeartbeater_ReportsPromptlyUntilTheEngineHasServed(t *testing.T) {
+	code := 2
+	probe := newPostgresProbe(testProbeConfig(), func(context.Context, string, ...string) (int, string, error) {
+		return code, "", nil
+	})
+	cp := newFakeControlPlane()
+	h := newHeartbeater(cp, probe, nil, 0)
+
+	if got := h.nextInterval(); got != startupHeartbeatInterval {
+		t.Errorf("interval before the engine has served = %s, want %s", got, startupHeartbeatInterval)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go h.Run(ctx)
+	waitFor(t, func() bool { return len(cp.snapshotStates()) > 0 }, "the first heartbeat")
+	cancel()
+
+	if states := cp.snapshotStates(); states[0].health != handlers_rds.EngineHealthStarting {
+		t.Errorf("first beat health = %q, want %q", states[0].health, handlers_rds.EngineHealthStarting)
+	}
+
+	code = 0
+	h.beat(context.Background())
+	if got := h.nextInterval(); got != h.interval {
+		t.Errorf("interval after the engine served = %s, want the control plane's %s", got, h.interval)
+	}
+}

cmd/rds-agent/main.go

@@ -16,21 +16,32 @@ import (
 	"os/signal"
 	"syscall"
 
-	_ "github.com/mulgadc/spinifex/internal/fipsboot"
+	_ "github.com/mulgadc/bluebottle/pkg/fipsboot"
 )
 
 func main() {
 	cfg := loadConfig(defaultEnvFile)
 
-	agent, err := New(cfg)
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	// Retried rather than fatal: construction resolves the IMDS credential chain,
+	// so a metadata service that is not answering yet would otherwise end the
+	// process with the engine still down and nothing left to bring it back.
+	var agent *Agent
+	err := retry(ctx, "startup", func(context.Context) error {
+		var err error
+		agent, err = New(cfg)
+		return err
+	})
 	if err != nil {
+		if errors.Is(err, context.Canceled) {
+			return
+		}
 		slog.Error("rds-agent: startup failed", "err", err)
 		os.Exit(1)
 	}
 
-	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
-	defer stop()
-
 	// A shutdown signal cancels whatever boot retry loop was running; that is a
 	// clean stop, not a failure.
 	if err := agent.Run(ctx); err != nil && !errors.Is(err, context.Canceled) {

cmd/rds-agent/paramrecovery.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"log/slog"
 	"time"
 
@@ -66,7 +67,7 @@ func (g *paramGuard) Run(ctx context.Context) {
 	}
 
 	message := handlers_rds.ParameterRollbackMessage
-	slog.Error("rds-agent: "+message, "after", g.after)
+	slog.Error("rds-agent: "+message, "after_ms", otelsetup.Millis(g.after))
 	if g.cp != nil {
 		if _, err := g.cp.SubmitState(ctx, g.id, handlers_rds.EngineHealthUnhealthy, message); err != nil {
 			slog.Error("rds-agent: reporting the parameter rollback failed", "err", err)

cmd/rds-agent/quiesce.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"io"
 	"log/slog"
 	"os/exec"
@@ -72,7 +73,7 @@ func (q *quiesceState) beginHoldLocked(label string, session engineSession, hold
 		session: session,
 		expiry:  time.AfterFunc(hold, func() { q.expire(label) }),
 	}
-	slog.Info("rds-agent: engine quiesced for backup", "label", label, "hold", hold)
+	slog.Info("rds-agent: engine quiesced for backup", "label", label, "hold_ms", otelsetup.Millis(hold))
 }
 
 // Takes the hold off the engine, if one is still there, and disarms its

cmd/rds-agent/quiesce_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"slices"
 	"strings"
 	"sync"
@@ -132,6 +133,50 @@ func TestCommandRegistry_QuiesceCarriesTheLabelAndDeadline(t *testing.T) {
 	}
 }
 
+// The engine's own hold settles only the files it owns. Everything else on that
+// volume — the parameter file the platform writes beside the datadir — is dirty
+// page cache until this, and a snapshot of it restores an engine that will not
+// start.
+func TestCommandRegistry_QuiesceFlushesTheDataMount(t *testing.T) {
+	storage := &fakeStorage{}
+	reply := newCommander(nil, newCommandRegistry(&fakeEngine{}, storage), 0).execute(context.Background(),
+		quiesceCommand("cmd-5a", "360"))
+
+	if reply.Status != handlers_rds.CommandStatusSucceeded {
+		t.Fatalf("reply = %+v, want succeeded", reply)
+	}
+	if !storage.synced {
+		t.Error("the data mount was not flushed, so the snapshot may not hold what the guest wrote")
+	}
+}
+
+// A snapshot taken after a flush that did not happen is the failure this whole
+// path exists to prevent, so it is reported rather than swallowed. The hold is
+// left on the engine: it expires on its own deadline.
+func TestCommandRegistry_QuiesceFailsWhenTheDataMountCannotBeFlushed(t *testing.T) {
+	storage := &fakeStorage{syncErr: errors.New("input/output error")}
+	reply := newCommander(nil, newCommandRegistry(&fakeEngine{}, storage), 0).execute(context.Background(),
+		quiesceCommand("cmd-5b", "360"))
+
+	if reply.Status != handlers_rds.CommandStatusFailed {
+		t.Fatalf("reply = %+v, want failed", reply)
+	}
+	if !strings.Contains(reply.Message, "input/output error") {
+		t.Errorf("reply message = %q, want the flush error", reply.Message)
+	}
+}
+
+func quiesceCommand(id, deadline string) handlers_rds.Command {
+	return handlers_rds.Command{
+		CommandID: id,
+		Type:      handlers_rds.CommandQuiesce,
+		Parameters: []handlers_rds.Parameter{
+			{Name: handlers_rds.CommandParamQuiesceLabel, Value: "orders-db-pre-upgrade"},
+			{Name: handlers_rds.CommandParamQuiesceDeadlineSeconds, Value: deadline},
+		},
+	}
+}
+
 // A hold the guest chose the length of would not be the bound the control plane
 // is relying on, so a missing or unreadable deadline is refused.
 func TestCommandRegistry_QuiesceRejectsAnUnusableDeadline(t *testing.T) {

cmd/rds-agent/register.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"log/slog"
 	"time"
 )
@@ -20,7 +21,7 @@ func (a *Agent) register(ctx context.Context) error {
 		a.hb.setInterval(time.Duration(out.HeartbeatIntervalSeconds) * time.Second)
 
 		slog.Info("rds-agent: registered",
-			"dbInstanceIdentifier", a.id.DBInstanceIdentifier, "heartbeat", a.hb.interval)
+			"dbInstanceIdentifier", a.id.DBInstanceIdentifier, "heartbeat_ms", otelsetup.Millis(a.hb.interval))
 		return nil
 	})
 }

cmd/rds-agent/storage.go

@@ -8,6 +8,8 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+
+	"golang.org/x/sys/unix"
 )
 
 // The in-guest half of a storage grow: the control plane has already grown the
@@ -17,6 +19,9 @@ type storageOps interface {
 	// Extends the filesystem at the data mount onto its whole device, and
 	// reports what it did for the command reply.
 	GrowFilesystem(ctx context.Context) (string, error)
+	// Flushes every dirty page on the data mount to its device. A backup hold
+	// settles the engine's own files and nothing else on that filesystem.
+	SyncDataMount(ctx context.Context) error
 }
 
 // Tool names rather than paths: the image installs them from
@@ -50,6 +55,23 @@ func newGuestStorage(cfg config, run commandRunner) *guestStorage {
 	}
 }
 
+// syncfs rather than fsync: the whole filesystem, not one file. Config the
+// engine never opens lives on this volume too, and a snapshot that catches it
+// half-written leaves a restored instance the engine cannot start from.
+func (g *guestStorage) SyncDataMount(ctx context.Context) error {
+	f, err := os.Open(g.dataMount)
+	if err != nil {
+		return fmt.Errorf("open the data mount %s to flush it: %w", g.dataMount, err)
+	}
+	defer func() { _ = f.Close() }()
+
+	if err := unix.Syncfs(int(f.Fd())); err != nil {
+		return fmt.Errorf("flush the data mount %s: %w", g.dataMount, err)
+	}
+	slog.InfoContext(ctx, "rds-agent: data mount flushed for backup", "dataMount", g.dataMount)
+	return nil
+}
+
 // The device and filesystem behind the data mount, resolved from the kernel's
 // own mount table rather than from anything the control plane asserted.
 type dataMount struct {

cmd/rds-agent/storage_test.go

@@ -15,8 +15,10 @@ import (
 // is that the command reaches it rather than what it does to a filesystem.
 type fakeStorage struct {
 	grown   bool
+	synced  bool
 	message string
 	err     error
+	syncErr error
 }
 
 var _ storageOps = (*fakeStorage)(nil)
@@ -26,6 +28,11 @@ func (f *fakeStorage) GrowFilesystem(context.Context) (string, error) {
 	return f.message, f.err
 }
 
+func (f *fakeStorage) SyncDataMount(context.Context) error {
+	f.synced = true
+	return f.syncErr
+}
+
 // runLog records what a grow shelled out to, in order, so the test asserts on
 // the tools invoked rather than on a filesystem it would have to create.
 type runLog struct {
@@ -227,3 +234,25 @@ func TestCommandRegistry_GrowFilesystemReachesTheGuestStorage(t *testing.T) {
 		t.Errorf("reply = %+v, want succeeded carrying what the grow reported", reply)
 	}
 }
+
+func TestSyncDataMount_FlushesTheMountItWasGiven(t *testing.T) {
+	g := &guestStorage{dataMount: t.TempDir()}
+
+	if err := g.SyncDataMount(context.Background()); err != nil {
+		t.Fatalf("SyncDataMount: %v", err)
+	}
+}
+
+// The control plane snapshots on the strength of this reply, so a mount that is
+// not there has to be an error rather than a flush of nothing.
+func TestSyncDataMount_FailsWhenTheMountIsNotThere(t *testing.T) {
+	g := &guestStorage{dataMount: filepath.Join(t.TempDir(), "absent")}
+
+	err := g.SyncDataMount(context.Background())
+	if err == nil {
+		t.Fatal("SyncDataMount succeeded on a mount that does not exist")
+	}
+	if !strings.Contains(err.Error(), "absent") {
+		t.Errorf("error = %v, want the mount it could not open", err)
+	}
+}

cmd/spinifex/cmd/admin.go

@@ -25,20 +25,22 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"uuid"
 
-	"github.com/google/uuid"
+	"github.com/mulgadc/bluebottle/pkg/masterkey"
+	"github.com/mulgadc/bluebottle/pkg/safecast"
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/config"
 	"github.com/mulgadc/spinifex/spinifex/ebsmetadata"
 	"github.com/mulgadc/spinifex/spinifex/ebsprovider"
 	"github.com/mulgadc/spinifex/spinifex/formation"
 	"github.com/mulgadc/spinifex/spinifex/gpu"
-	handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot"
 	handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
 	"github.com/mulgadc/spinifex/spinifex/hostdns"
 	"github.com/mulgadc/spinifex/spinifex/network/host"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/services/viperblockd/vbwire"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -217,6 +219,18 @@ decision made per volume.`,
 	Run: runVolumesOrphansCmd,
 }
 
+var volumesUnsealedCmd = &cobra.Command{
+	Use:   "unsealed",
+	Short: "Report volumes whose writes have not been confirmed to the backend",
+	Long: `List volumes a node has open, or had open, without a confirmed seal to the
+backend. The node named holds writes that predastore may not have.
+
+Starting such a volume elsewhere is allowed and is preferred over leaving the
+instance unable to run, but it opens from the last checkpoint that did reach
+the backend. This is the list of volumes where that would cost something.`,
+	Run: runVolumesUnsealedCmd,
+}
+
 var accountCmd = &cobra.Command{
 	Use:   "account",
 	Short: "Manage Spinifex accounts",
@@ -317,6 +331,7 @@ func init() {
 
 	adminCmd.AddCommand(volumesCmd)
 	volumesCmd.AddCommand(volumesOrphansCmd)
+	volumesCmd.AddCommand(volumesUnsealedCmd)
 
 	adminCmd.AddCommand(accountCmd)
 	accountCmd.AddCommand(accountCreateCmd)
@@ -343,6 +358,7 @@ func init() {
 
 	// Flags for admin init
 	adminInitCmd.Flags().Bool("force", false, "Force re-initialization (overwrites existing config)")
+	adminInitCmd.Flags().Bool("discard-jetstream", true, "With --nodes >= 2, remove this node's existing JetStream streams once every node has joined; =false refuses to form over a non-empty store instead. Ignored with --nodes 1")
 	adminInitCmd.Flags().String("region", "ap-southeast-2", "Mulga region to create")
 	adminInitCmd.Flags().String("az", "ap-southeast-2a", "Mulga AZ to create")
 	adminInitCmd.Flags().String("node", "node1", "Node name, increment for additional nodes (default, node1)")
@@ -415,15 +431,11 @@ func init() {
 	imagesRemoveCmd.Flags().String("image-id", "", "AMI ID to remove (required)")
 	imagesRemoveCmd.Flags().Bool("force", false, "Bypass dependency, ownership and config-corrupt checks (salvage mode)")
 	imagesRemoveCmd.Flags().Bool("yes", false, "Skip interactive confirmation prompt")
-	if err := imagesRemoveCmd.MarkFlagRequired("image-id"); err != nil {
-		panic(err)
-	}
+	_ = imagesRemoveCmd.MarkFlagRequired("image-id")
 
 	imagesPromoteCmd.Flags().String("image-id", "", "AMI ID to promote to system image (required)")
 	imagesPromoteCmd.Flags().Bool("yes", false, "Skip interactive confirmation prompt")
-	if err := imagesPromoteCmd.MarkFlagRequired("image-id"); err != nil {
-		panic(err)
-	}
+	_ = imagesPromoteCmd.MarkFlagRequired("image-id")
 }
 
 const bytesPerGiB = 1024 * 1024 * 1024
@@ -449,7 +461,7 @@ func amiVolumeSizeGiB(sizeBytes int64) uint64 {
 	if sizeBytes <= 0 {
 		return 0
 	}
-	return utils.SafeInt64ToUint64((sizeBytes + bytesPerGiB - 1) / bytesPerGiB)
+	return safecast.Int64ToUint64((sizeBytes + bytesPerGiB - 1) / bytesPerGiB)
 }
 
 func runimagesImportCmd(cmd *cobra.Command, args []string) {
@@ -697,7 +709,7 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 	err = admin.ImportImage(context.Background(), provider, admin.ImportOpts{
 		VolumeID:         volumeId,
 		NodeID:           appConfig.Node,
-		SizeBytes:        utils.SafeUint64ToInt64(ami.VolumeSizeGiB * bytesPerGiB),
+		SizeBytes:        safecast.Uint64ToInt64(ami.VolumeSizeGiB * bytesPerGiB),
 		AvailabilityZone: node.AZ,
 		SourcePath:       extractedImagePath,
 		Snapshot:         true,
@@ -735,10 +747,10 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) {
 // DescribeSnapshots call cannot resolve it. The volume ID matches ami.ImageID:
 // admin.ImportImage creates the volume under the AMI's own ID.
 func registerImportedAMISnapshot(store objectstore.ObjectStore, bucket string, ami ebsmetadata.AMI, az string, encrypted bool) error {
-	cfg := &handlers_ec2_snapshot.SnapshotConfig{
+	cfg := ebsmetadata.Snapshot{
 		SnapshotID:       ami.SnapshotID,
 		VolumeID:         ami.ImageID,
-		VolumeSize:       utils.SafeUint64ToInt64(ami.VolumeSizeGiB),
+		VolumeSize:       safecast.Uint64ToInt64(ami.VolumeSizeGiB),
 		State:            "completed",
 		Progress:         "100%",
 		StartTime:        time.Now(),
@@ -747,7 +759,7 @@ func registerImportedAMISnapshot(store objectstore.ObjectStore, bucket string, a
 		OwnerID:          utils.GlobalAccountID,
 		AvailabilityZone: az,
 	}
-	if err := handlers_ec2_snapshot.WriteSnapshotConfig(store, bucket, ami.SnapshotID, cfg); err != nil {
+	if err := ebsmetadata.NewStore(store, bucket).PutSnapshot(context.Background(), cfg); err != nil {
 		return fmt.Errorf("register snapshot metadata: %w", err)
 	}
 	return nil
@@ -846,9 +858,9 @@ func runimagesRemoveCmd(cmd *cobra.Command, args []string) {
 		}
 	}
 	fmt.Printf("  Backing storage: %s/      (%d objects, %s)\n",
-		preview.ImageID, preview.AMIObjectCount, utils.HumanBytes(utils.SafeInt64ToUint64(preview.AMIBytesTotal)))
+		preview.ImageID, preview.AMIObjectCount, utils.HumanBytes(safecast.Int64ToUint64(preview.AMIBytesTotal)))
 	fmt.Printf("                   %s/ (%d objects, %s)\n",
-		admin.SnapPrefix(preview.ImageID), preview.SnapObjectCount, utils.HumanBytes(utils.SafeInt64ToUint64(preview.SnapBytesTotal)))
+		admin.SnapPrefix(preview.ImageID), preview.SnapObjectCount, utils.HumanBytes(safecast.Int64ToUint64(preview.SnapBytesTotal)))
 	fmt.Println()
 
 	// Account-owned guard before salvage / dependents — the AWS-flow hint is
@@ -904,7 +916,7 @@ func runimagesRemoveCmd(cmd *cobra.Command, args []string) {
 	// BytesDeleted is logical: predastore reclaims the underlying disk space
 	// asynchronously via background compaction, not at delete time.
 	fmt.Printf("✅ Removed AMI %s (%d objects, %s marked for deletion; disk space is reclaimed by background compaction).\n",
-		imageID, res.ObjectsDeleted, utils.HumanBytes(utils.SafeInt64ToUint64(res.BytesDeleted)))
+		imageID, res.ObjectsDeleted, utils.HumanBytes(safecast.Int64ToUint64(res.BytesDeleted)))
 }
 
 func printDependents(w io.Writer, d admin.Dependents) {
@@ -1302,6 +1314,15 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		os.Exit(0)
 	}
 
+	if spxRoot == "" {
+		spxRoot = DefaultDataDir()
+	}
+	discardJetStream, _ := cmd.Flags().GetBool("discard-jetstream")
+	if err := checkInitJetStreamStore(jetStreamStoreDir(spxRoot), nodes, discardJetStream); err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+		os.Exit(1)
+	}
+
 	// Preserve the previously-captured operator email across --force re-inits
 	// when --email is omitted (e.g. reset-dev-env.sh workflows). Without this
 	// a reset would silently blank the address.
@@ -1603,6 +1624,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		ClusterName:   clusterName,
 
 		PredastoreHostID:          predastoreHostID,
+		PredastoreAdminPort:       admin.PredastoreAdminPort,
 		CompactionIntervalSeconds: compactionInterval,
 		Services:                  services,
 
@@ -1833,6 +1855,17 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 
 	fmt.Printf("✅ All %d nodes joined!\n", expectedNodes)
 
+	// The pre-start check refused a store with streams unless discard was asked
+	// for, so anything left here is removed now that the cluster is committed.
+	if discard, _ := cmd.Flags().GetBool("discard-jetstream"); discard {
+		storeDir := jetStreamStoreDir(spxRoot)
+		if err := discardJetStreamStore(storeDir); err != nil {
+			fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Printf("✅ Discarded this node's pre-formation JetStream store: %s\n", storeDir)
+	}
+
 	// Build cluster topology from formation data
 	allNodes := fs.Nodes()
 	clusterRoutes := formation.BuildClusterRoutes(allNodes)
@@ -1904,6 +1937,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		ClusterName:   clusterName,
 
 		PredastoreHostID:          predastoreHostID,
+		PredastoreAdminPort:       admin.PredastoreAdminPort,
 		CompactionIntervalSeconds: compactionInterval,
 		Services:                  services,
 		RemoteNodes:               buildRemoteNodes(allNodes, node, northstarConfigPath),
@@ -1989,19 +2023,27 @@ func joinRetryable(err error, statusCode int) bool {
 }
 
 // checkJoinPreconditions rejects a join that would silently destroy this node's
-// existing cluster identity. Joining adopts the primary's CA and master key,
-// overwriting whatever is here: correct on a freshly installed node, and on one
-// that has been in service it orphans every fragment and volume sealed under the
-// old key. runAdminInit guards the same way before re-initializing.
-func checkJoinPreconditions(configDir string, force bool) error {
+// identity or JetStream data; --force accepts that loss. A running NATS service
+// is refused even with --force, because the store cannot be removed under it.
+func checkJoinPreconditions(configDir, storeDir string, force bool) error {
+	if err := checkStoreDiscardable(); err != nil {
+		return err
+	}
 	if force {
 		return nil
 	}
 	tomlPath := filepath.Join(configDir, "spinifex.toml")
-	if !admin.FileExists(tomlPath) {
-		return nil
+	if admin.FileExists(tomlPath) {
+		return fmt.Errorf("this node is already initialized: %s", tomlPath)
 	}
-	return fmt.Errorf("this node is already initialized: %s", tomlPath)
+	streams, err := localStreams(storeDir)
+	if err != nil {
+		return err
+	}
+	if len(streams) > 0 {
+		return fmt.Errorf("this node's JetStream store %s holds %d stream(s)", storeDir, len(streams))
+	}
+	return nil
 }
 
 // joinDiscardsIdentityMsg spells out what a forced join throws away. Kept out of
@@ -2010,6 +2052,8 @@ const joinDiscardsIdentityMsg = `Joining will discard this node's own cluster id
   - CA certificate and key
   - master key, and any data sealed under it
   - viperblock key, and any volumes encrypted under it
+  - JetStream store (<data-dir>/nats/jetstream): every stream and KV bucket
+    this node holds, so it takes the cluster's copy instead of diverging from it
 
 That is safe on a freshly installed node — an ISO install initializes a
 single-node cluster at first boot, and nothing has been sealed under these keys
@@ -2090,7 +2134,11 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	// Checked before any network call so a node that will not join says so
 	// immediately. Unlike init this exits non-zero: a node that did not join
 	// must not look like success to a provisioning script.
-	if err := checkJoinPreconditions(configDir, force); err != nil {
+	if dataDir == "" {
+		dataDir = DefaultDataDir()
+	}
+	storeDir := jetStreamStoreDir(dataDir)
+	if err := checkJoinPreconditions(configDir, storeDir, force); err != nil {
 		fmt.Fprintf(os.Stderr, "⚠️  %v\n\n%s\n", err, joinDiscardsIdentityMsg)
 		os.Exit(1)
 	}
@@ -2100,11 +2148,6 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		clusterBind = bindIP
 	}
 
-	// Set default data directory
-	if dataDir == "" {
-		dataDir = DefaultDataDir()
-	}
-
 	fmt.Println("🚀 Joining Spinifex cluster...")
 	fmt.Printf("Node: %s\n", node)
 	fmt.Printf("Leader: %s\n", leaderHost)
@@ -2285,6 +2328,14 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
+	// Removed only once formation has succeeded, so a failed join leaves the
+	// node's single-node cluster intact.
+	if err := discardJetStreamStore(storeDir); err != nil {
+		fmt.Fprintf(os.Stderr, "❌ Error: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Printf("✅ Discarded this node's pre-formation JetStream store: %s\n", storeDir)
+
 	// Set up config directory
 	if configDir == "" {
 		configDir = DefaultConfigDir()
@@ -2463,6 +2514,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		ClusterName:   creds.ClusterName,
 
 		PredastoreHostID:          predastoreHostID,
+		PredastoreAdminPort:       admin.PredastoreAdminPort,
 		CompactionIntervalSeconds: compactionInterval,
 		Services:                  services,
 		RemoteNodes:               buildRemoteNodes(statusResp.Nodes, node, northstarConfigPath),
@@ -2582,7 +2634,7 @@ func initIAMServiceFromConfig() (*handlers_iam.IAMServiceImpl, *config.ClusterCo
 	}
 
 	masterKeyPath := filepath.Join(cfg.NodeBaseDir(), "config", "master.key")
-	masterKey, err := handlers_iam.LoadMasterKey(masterKeyPath)
+	masterKey, err := masterkey.ReadShared(masterKeyPath)
 	if err != nil {
 		nc.Close()
 		return nil, nil, nil, nil, fmt.Errorf("load master key: %w", err)
@@ -2649,7 +2701,7 @@ func runAccountCreate(cmd *cobra.Command, args []string) {
 		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
 		os.Exit(1)
 	}
-	reservationOwner := "spx-cli-" + uuid.NewString()
+	reservationOwner := "spx-cli-" + uuid.NewV4().String()
 	switch err := names.Reserve(ctx, name, reservationOwner); {
 	case errors.Is(err, handlers_iam.ErrNameTaken):
 		fmt.Fprintf(os.Stderr, "Error: account %q already exists\n", name)
@@ -2844,9 +2896,44 @@ func northstarFromFormation(creds *formation.SharedCredentials, dirs configDirs)
 	}, filepath.Join(dirs.Northstar, "northstar.toml")
 }
 
+// resolverAllowFrom returns the addresses permitted to recurse against this
+// node's northstar: every cluster node, on whichever address its peers dial.
+//
+// Guests are covered without appearing here. vpcd's per-tap DNS shim relays
+// guest queries to a node's :5300 from a node address, so trusting the nodes
+// trusts every guest behind them, and no guest CIDR is needed.
+func resolverAllowFrom(settings admin.ConfigSettings) []string {
+	seen := make(map[string]bool)
+	var addrs []string
+	add := func(ip string) {
+		ip = strings.TrimSpace(ip)
+		if ip == "" || ip == "0.0.0.0" || seen[ip] {
+			return
+		}
+		seen[ip] = true
+		addrs = append(addrs, ip)
+	}
+
+	// Both local addresses: a node resolving through its own advertised address
+	// is seen arriving from that address over lo, not from 127.0.0.1.
+	add(settings.AdvertiseIP)
+	add(settings.BindIP)
+	add(settings.ClusterBindIP)
+	for _, peer := range settings.RemoteNodes {
+		host := peer.Host
+		if h, _, err := net.SplitHostPort(host); err == nil {
+			host = h
+		}
+		add(host)
+	}
+	return addrs
+}
+
 // generateAndWriteConfigs renders the standard config files (spinifex.toml,
 // awsgw.toml, nats.conf, and optionally predastore.toml) from templates.
 func generateAndWriteConfigs(dirs configDirs, spinifexTomlPath string, settings admin.ConfigSettings, skipPredastore bool) error {
+	settings.ResolverAllowFrom = resolverAllowFrom(settings)
+
 	// A one-sided pair must disable Northstar wholesale. Rendering only the
 	// public stanza or only the secret file advertises a resolver that cannot run.
 	northstarEnabled := settings.NorthstarAccessKey != "" && settings.NorthstarSecretKey != ""
@@ -2988,7 +3075,7 @@ type writeBootstrapResult struct {
 func ensureMasterKey(configDir string) (key []byte, existed bool, err error) {
 	keyPath := filepath.Join(configDir, "master.key")
 	if admin.FileExists(keyPath) {
-		key, err = handlers_iam.LoadMasterKey(keyPath)
+		key, err = masterkey.ReadShared(keyPath)
 		if err != nil {
 			return nil, false, fmt.Errorf("load master key: %w", err)
 		}
@@ -3081,7 +3168,7 @@ func ensureViperblockEncryptionKey(configDir string) ([]byte, string, error) {
 	}
 	keyPath := filepath.Join(viperblockDir, "encryption.key")
 	if admin.FileExists(keyPath) {
-		key, err := handlers_iam.LoadMasterKey(keyPath)
+		key, err := masterkey.ReadShared(keyPath)
 		if err != nil {
 			return nil, "", fmt.Errorf("load viperblock encryption key: %w", err)
 		}
@@ -3662,3 +3749,37 @@ func printChecksumError(w io.Writer, imageFile, imageName string, image utils.Im
 	fmt.Fprintln(w, "The cached file was left in place. To re-download and retry:")
 	fmt.Fprintf(w, "  spx admin images import --name %s --force\n", imageName)
 }
+
+func runVolumesUnsealedCmd(_ *cobra.Command, _ []string) {
+	_, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Could not connect to the cluster: %v\n", err)
+		os.Exit(1)
+	}
+	defer nc.Close()
+
+	unsealed, err := vbwire.ListUnsealedVolumes(context.Background(), nc)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Could not read the unsealed volumes:", err)
+		os.Exit(1)
+	}
+	if len(unsealed) == 0 {
+		fmt.Println("No unsealed volumes: every volume's writes are confirmed to the backend.")
+		return
+	}
+
+	sort.Slice(unsealed, func(i, j int) bool { return unsealed[i].VolumeID < unsealed[j].VolumeID })
+	fmt.Printf("%d volume(s) with writes the backend may not have:\n\n", len(unsealed))
+	for _, volume := range unsealed {
+		fmt.Printf("  %s\n", volume.VolumeID)
+		fmt.Printf("    held by: %s\n", volume.Owner)
+		if !volume.Since.IsZero() {
+			fmt.Printf("    since:   %s\n", volume.Since.Format(time.RFC3339))
+		}
+		fmt.Printf("    reason:  %s\n", volume.Reason)
+	}
+	fmt.Println()
+	fmt.Println("A volume open right now is listed here and that is normal — the entry")
+	fmt.Println("clears on its next successful seal. One naming a node that is gone is")
+	fmt.Println("the case worth acting on: starting it elsewhere loses writes made since.")
+}

cmd/spinifex/cmd/admin_account_quota.go

@@ -0,0 +1,185 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strconv"
+
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	handlers_quota "github.com/mulgadc/spinifex/spinifex/handlers/quota"
+	"github.com/spf13/cobra"
+)
+
+// quotaDimensions maps each CLI flag to the override field it sets. One table
+// drives the flag registration and the parse, so a new dimension cannot be
+// added to one and forgotten in the other.
+var quotaDimensions = []struct {
+	flag  string
+	usage string
+	field func(*handlers_quota.Overrides) **int
+}{
+	{"vcpus", "Total vCPUs across running and stopped instances", func(o *handlers_quota.Overrides) **int { return &o.VCPUs }},
+	{"vpcs", "VPCs", func(o *handlers_quota.Overrides) **int { return &o.VPCs }},
+	{"subnets", "Subnets, counted per account rather than per VPC", func(o *handlers_quota.Overrides) **int { return &o.Subnets }},
+	{"eips", "Elastic IP addresses", func(o *handlers_quota.Overrides) **int { return &o.EIPs }},
+	{"volumes", "EBS volumes", func(o *handlers_quota.Overrides) **int { return &o.Volumes }},
+	{"volumes-gib", "Total EBS capacity in GiB", func(o *handlers_quota.Overrides) **int { return &o.VolumesGiB }},
+	{"rds-instances", "RDS database instances", func(o *handlers_quota.Overrides) **int { return &o.RDSInstances }},
+	{"load-balancers", "Load balancers, ALB and NLB together", func(o *handlers_quota.Overrides) **int { return &o.LoadBalancers }},
+}
+
+var accountQuotaCmd = &cobra.Command{
+	Use:   "quota",
+	Short: "Inspect and adjust an account's service quotas",
+	Long: `Inspect and adjust the service quotas applied to one account.
+
+Every account inherits the [quota] limits from awsgw.toml. An override raises or
+lowers individual dimensions for one account without affecting any other, so a
+customer can be given more vCPUs while every other tenant stays on the baseline.
+
+Only the dimensions passed as flags are changed; the rest keep inheriting. Use
+-1 for no limit on a dimension, and --clear to drop every override.`,
+}
+
+var accountQuotaGetCmd = &cobra.Command{
+	Use:   "get <account-id>",
+	Short: "Show the limits in force for an account",
+	Long: `Show the limits in force for an account and where each came from:
+"config" for an inherited baseline, "override" for one set on this account.`,
+	Args: cobra.ExactArgs(1),
+	RunE: runAccountQuotaGet,
+}
+
+var accountQuotaSetCmd = &cobra.Command{
+	Use:   "set <account-id>",
+	Short: "Override one or more limits for an account",
+	Long: `Override one or more limits for an account. Dimensions not named keep
+inheriting the configured baseline.
+
+  spx admin account quota set 000000000005 --vcpus 32
+  spx admin account quota set 000000000005 --clear`,
+	Args: cobra.ExactArgs(1),
+	RunE: runAccountQuotaSet,
+}
+
+// Both subcommands are served only by the /admin surface, so unlike the account
+// commands there is no NATS path and no --remote to opt into.
+func init() {
+	accountCmd.AddCommand(accountQuotaCmd)
+	accountQuotaCmd.AddCommand(accountQuotaGetCmd)
+	accountQuotaCmd.AddCommand(accountQuotaSetCmd)
+
+	for _, c := range []*cobra.Command{accountQuotaGetCmd, accountQuotaSetCmd} {
+		c.Flags().String("endpoint", "", "Gateway endpoint (default: this node's AWS gateway)")
+		c.Flags().String("region", "", "SigV4 region (default: this node's region)")
+		c.Flags().String("ca-bundle", "", "CA certificate (default: this node's CA)")
+	}
+
+	for _, d := range quotaDimensions {
+		accountQuotaSetCmd.Flags().Int(d.flag, 0, d.usage+" (-1 for no limit)")
+	}
+	accountQuotaSetCmd.Flags().Bool("clear", false, "Remove every override, returning the account to the configured limits")
+}
+
+func runAccountQuotaGet(cmd *cobra.Command, args []string) error {
+	target, err := resolveAdminTarget(cmd)
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), adminRequestTimeout)
+	defer cancel()
+
+	var out gateway.AccountQuotaResponse
+	req := gateway.AccountQuotaRequest{AccountID: args[0]}
+	if err := callAdmin(ctx, target, "GetAccountQuota", req, &out); err != nil {
+		return err
+	}
+	printAccountQuota(&out)
+	return nil
+}
+
+func runAccountQuotaSet(cmd *cobra.Command, args []string) error {
+	target, err := resolveAdminTarget(cmd)
+	if err != nil {
+		return err
+	}
+
+	over, changed, err := quotaOverridesFromFlags(cmd)
+	if err != nil {
+		return err
+	}
+	clearAll, err := cmd.Flags().GetBool("clear")
+	if err != nil {
+		return err
+	}
+	// Refusing both is not pedantry: --clear silently winning would discard
+	// limits the caller believed it was setting in the same breath.
+	if clearAll && changed {
+		return fmt.Errorf("--clear cannot be combined with a dimension flag")
+	}
+	if !clearAll && !changed {
+		return fmt.Errorf("no dimension given; pass at least one limit flag, or --clear to remove every override")
+	}
+	if clearAll {
+		over = handlers_quota.Overrides{}
+	}
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), adminRequestTimeout)
+	defer cancel()
+
+	var out gateway.AccountQuotaResponse
+	req := gateway.AccountQuotaRequest{AccountID: args[0], Overrides: over}
+	if err := callAdmin(ctx, target, "PutAccountQuota", req, &out); err != nil {
+		return err
+	}
+	printAccountQuota(&out)
+	return nil
+}
+
+// quotaOverridesFromFlags builds the override set from the flags the caller
+// actually passed. An unset flag is left nil so the dimension keeps inheriting,
+// which is why Changed is consulted rather than the flag's value.
+func quotaOverridesFromFlags(cmd *cobra.Command) (handlers_quota.Overrides, bool, error) {
+	var over handlers_quota.Overrides
+	changed := false
+	for _, d := range quotaDimensions {
+		if !cmd.Flags().Changed(d.flag) {
+			continue
+		}
+		value, err := cmd.Flags().GetInt(d.flag)
+		if err != nil {
+			return over, false, err
+		}
+		if value < handlers_quota.Unlimited {
+			return over, false, fmt.Errorf("--%s must be 0 or greater, or %d for no limit", d.flag, handlers_quota.Unlimited)
+		}
+		*d.field(&over) = &value
+		changed = true
+	}
+	return over, changed, nil
+}
+
+func printAccountQuota(resp *gateway.AccountQuotaResponse) {
+	fmt.Printf("Account %s\n\n", resp.AccountID)
+	fmt.Printf("%-16s %10s  %s\n", "DIMENSION", "LIMIT", "SOURCE")
+	fmt.Printf("%-16s %10s  %s\n", "---------", "-----", "------")
+
+	names := make([]string, 0, len(resp.Limits))
+	for name := range resp.Limits {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+	for _, name := range names {
+		limit := strconv.Itoa(resp.Limits[name])
+		if resp.Limits[name] == handlers_quota.Unlimited {
+			limit = "unlimited"
+		}
+		fmt.Printf("%-16s %10s  %s\n", name, limit, resp.Source[name])
+	}
+
+	if resp.Overrides.UpdatedAt != "" {
+		fmt.Printf("\nOverride set by %s at %s\n", resp.Overrides.UpdatedBy, resp.Overrides.UpdatedAt)
+	}
+}

cmd/spinifex/cmd/admin_account_quota_test.go

@@ -0,0 +1,140 @@
+//test:in-package — drives the unexported flag table, parser and printer the quota subcommands are built from.
+package cmd
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/gateway"
+	handlers_quota "github.com/mulgadc/spinifex/spinifex/handlers/quota"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/require"
+)
+
+// newQuotaSetCmd builds a command carrying the same flags as the real one, so
+// the parse is tested without mutating the package-level command between tests.
+func newQuotaSetCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "set"}
+	for _, d := range quotaDimensions {
+		cmd.Flags().Int(d.flag, 0, d.usage)
+	}
+	cmd.Flags().Bool("clear", false, "")
+	return cmd
+}
+
+// Only the flags actually passed become overrides. Every other dimension must
+// stay nil so it keeps inheriting, which is the whole point of a sparse record.
+func TestQuotaOverridesFromFlagsIsSparse(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--vcpus", "32", "--rds-instances", "8"}))
+
+	over, changed, err := quotaOverridesFromFlags(cmd)
+	require.NoError(t, err)
+	require.True(t, changed)
+
+	require.NotNil(t, over.VCPUs)
+	require.Equal(t, 32, *over.VCPUs)
+	require.NotNil(t, over.RDSInstances)
+	require.Equal(t, 8, *over.RDSInstances)
+
+	require.Nil(t, over.VPCs)
+	require.Nil(t, over.EIPs)
+	require.Nil(t, over.Volumes)
+	require.Nil(t, over.VolumesGiB)
+	require.Nil(t, over.Subnets)
+	require.Nil(t, over.LoadBalancers)
+}
+
+// A flag left unset must not be read as an explicit 0, which would deny every
+// request on that dimension instead of inheriting it.
+func TestQuotaOverridesFromFlagsUnsetIsNotZero(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--vpcs", "8"}))
+
+	over, changed, err := quotaOverridesFromFlags(cmd)
+	require.NoError(t, err)
+	require.True(t, changed)
+	require.Nil(t, over.VCPUs, "an unset flag must not become an explicit zero")
+}
+
+// Passing 0 deliberately is a real limit and must be preserved.
+func TestQuotaOverridesFromFlagsExplicitZero(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--eips", "0"}))
+
+	over, changed, err := quotaOverridesFromFlags(cmd)
+	require.NoError(t, err)
+	require.True(t, changed)
+	require.NotNil(t, over.EIPs)
+	require.Equal(t, 0, *over.EIPs)
+}
+
+func TestQuotaOverridesFromFlagsUnlimited(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--vcpus", "-1"}))
+
+	over, _, err := quotaOverridesFromFlags(cmd)
+	require.NoError(t, err)
+	require.NotNil(t, over.VCPUs)
+	require.Equal(t, handlers_quota.Unlimited, *over.VCPUs)
+}
+
+// Anything below the Unlimited sentinel is a typo, not a smaller limit.
+func TestQuotaOverridesFromFlagsRejectsBelowUnlimited(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags([]string{"--vcpus", "-2"}))
+
+	_, _, err := quotaOverridesFromFlags(cmd)
+	require.Error(t, err)
+}
+
+// No flags at all means nothing changed, which the caller turns into an error
+// rather than a write that silently does nothing.
+func TestQuotaOverridesFromFlagsNoneChanged(t *testing.T) {
+	cmd := newQuotaSetCmd()
+	require.NoError(t, cmd.ParseFlags(nil))
+
+	over, changed, err := quotaOverridesFromFlags(cmd)
+	require.NoError(t, err)
+	require.False(t, changed)
+	require.True(t, over.Empty())
+}
+
+// The table must name every dimension, label its source, and render the
+// Unlimited sentinel as a word rather than as -1, which reads like a bug.
+func TestPrintAccountQuota(t *testing.T) {
+	resp := &gateway.AccountQuotaResponse{
+		AccountID: "000000000042",
+		Limits:    map[string]int{"vcpus": 32, "vpcs": handlers_quota.Unlimited, "eips": 4},
+		Source:    map[string]string{"vcpus": "override", "vpcs": "override", "eips": "config"},
+		Overrides: handlers_quota.Overrides{UpdatedBy: "operator", UpdatedAt: "2026-08-20T01:00:00Z"},
+	}
+
+	out := captureStdout(t, func() { printAccountQuota(resp) })
+
+	require.Contains(t, out, "000000000042")
+	require.Contains(t, out, "vcpus")
+	require.Contains(t, out, "32")
+	require.Contains(t, out, "override")
+	require.Contains(t, out, "config")
+	require.Contains(t, out, "unlimited")
+	require.NotContains(t, out, "-1")
+	require.Contains(t, out, "operator")
+}
+
+// Every dimension the override record carries must be reachable from the CLI,
+// or a limit exists that an operator cannot set.
+func TestQuotaDimensionsCoverEveryOverrideField(t *testing.T) {
+	var over handlers_quota.Overrides
+	for _, d := range quotaDimensions {
+		v := 1
+		*d.field(&over) = &v
+	}
+	require.False(t, over.Empty())
+
+	seen := map[string]bool{}
+	for _, d := range quotaDimensions {
+		require.False(t, seen[d.flag], "duplicate flag %q", d.flag)
+		seen[d.flag] = true
+	}
+	require.Len(t, quotaDimensions, 8)
+}

cmd/spinifex/cmd/admin_acm.go

@@ -60,16 +60,12 @@ func init() {
 	certCreateTenantCACmd.Flags().StringSlice("domain", nil, "Domain the tenant CA is permitted to sign for (repeatable)")
 	certCreateTenantCACmd.Flags().Bool("regenerate", false, "Replace an existing tenant CA with a fresh root (invalidates every device's trust store — requires confirmation)")
 	certCreateTenantCACmd.Flags().Bool("yes", false, "Skip the --regenerate confirmation prompt")
-	if err := certCreateTenantCACmd.MarkFlagRequired("domain"); err != nil {
-		panic(err)
-	}
+	_ = certCreateTenantCACmd.MarkFlagRequired("domain")
 
 	certCmd.AddCommand(certForceRenewCmd)
 	certForceRenewCmd.Flags().String("arn", "", "Certificate ARN to force-renew (required)")
 	certForceRenewCmd.Flags().String("account", "", "Account ID that owns the certificate; defaults to the bootstrap account")
-	if err := certForceRenewCmd.MarkFlagRequired("arn"); err != nil {
-		panic(err)
-	}
+	_ = certForceRenewCmd.MarkFlagRequired("arn")
 }
 
 func runCertForceRenew(cmd *cobra.Command, _ []string) {

cmd/spinifex/cmd/admin_kv.go

@@ -0,0 +1,451 @@
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"slices"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/minio/highwayhash"
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/spf13/cobra"
+)
+
+var kvCmd = &cobra.Command{
+	Use:   "kv",
+	Short: "Inspect JetStream KV state",
+}
+
+var kvDigestCmd = &cobra.Command{
+	Use:   "digest",
+	Short: "Print a digest of every stream held by this node's JetStream replica",
+	Long: `Read this node's own copy of every JetStream stream and print one line per
+stream: name, first and last sequence, message count, and a SHA-256 over every
+stored message (sequence, subject, timestamp, headers, data).
+
+Run it on every node and compare the output. Replicas of one stream must
+produce identical lines; any difference means the replicas have diverged, and
+--seqs prints a hash per sequence to find where. For a check that tells a
+diverged replica from one that is merely behind, run it with --json --seqs on
+every node and pass the results to "spx admin kv compare".
+
+It never touches the live store. A direct get over NATS may be answered by any
+replica, so instead the streams tree is copied to --work-dir and read by a
+private in-process server with no listener, which is removed afterwards. The
+copy's stream config is set to one replica with no expiry so it can be loaded.
+
+Writes landing during the copy, or applied but not yet flushed to disk, can
+make a replica's tail differ, so gate API writes and re-run before calling a
+tail difference divergence.`,
+	Run: runKVDigest,
+}
+
+func init() {
+	adminCmd.AddCommand(kvCmd)
+	kvCmd.AddCommand(kvDigestCmd)
+	kvDigestCmd.Flags().String("store-dir", "", "JetStream store to read (default <spinifex-dir>/nats/jetstream)")
+	kvDigestCmd.Flags().String("work-dir", os.TempDir(), "Directory to hold the temporary copy")
+	kvDigestCmd.Flags().StringSlice("stream", nil, "Only these streams (repeatable)")
+	kvDigestCmd.Flags().Bool("seqs", false, "Also print a hash per sequence")
+	kvDigestCmd.Flags().Bool("json", false, "Print one JSON document, the input to \"spx admin kv compare\"")
+}
+
+// nodeDigest is one node's digest report, the unit "kv compare" reads.
+type nodeDigest struct {
+	Host     string         `json:"host"`
+	Store    string         `json:"store"`
+	CopiedAt time.Time      `json:"copied_at"`
+	Skipped  []string       `json:"skipped,omitempty"`
+	Streams  []streamDigest `json:"streams"`
+}
+
+// streamDigest is one stream's content as held by one replica.
+type streamDigest struct {
+	streamMeta
+
+	Name     string      `json:"name"`
+	FirstSeq uint64      `json:"first_seq"`
+	LastSeq  uint64      `json:"last_seq"`
+	Msgs     uint64      `json:"msgs"`
+	Digest   string      `json:"sha256"`
+	Seqs     []seqDigest `json:"seqs,omitempty"`
+}
+
+// streamMeta is the part of the live stream config that decides which replica
+// differences are legitimate. It is read before the copy is rewritten.
+type streamMeta struct {
+	Replicas    int           `json:"replicas,omitempty"`
+	Retention   string        `json:"retention,omitempty"`
+	MaxAge      time.Duration `json:"max_age_ns,omitempty"`
+	AllowMsgTTL bool          `json:"allow_msg_ttl,omitempty"`
+	MaxMsgs     int64         `json:"max_msgs,omitempty"`
+	MaxBytes    int64         `json:"max_bytes,omitempty"`
+}
+
+// seqDigest is the hash of one stored sequence, or "deleted" when the replica
+// has no message at that sequence. Subject, time and TTL are empty then.
+type seqDigest struct {
+	Seq     uint64        `json:"seq"`
+	Hash    string        `json:"hash"`
+	Subject string        `json:"subject,omitempty"`
+	Time    time.Time     `json:"time,omitzero"`
+	TTL     time.Duration `json:"ttl_ns,omitempty"`
+}
+
+// deletedHash marks a sequence the replica holds no message for.
+const deletedHash = "deleted"
+
+func runKVDigest(cmd *cobra.Command, _ []string) {
+	storeDir, _ := cmd.Flags().GetString("store-dir")
+	if storeDir == "" {
+		spxRoot, _ := cmd.Flags().GetString("spinifex-dir")
+		if spxRoot == "" {
+			spxRoot = DefaultDataDir()
+		}
+		storeDir = jetStreamStoreDir(spxRoot)
+	}
+	workParent, _ := cmd.Flags().GetString("work-dir")
+	only, _ := cmd.Flags().GetStringSlice("stream")
+	withSeqs, _ := cmd.Flags().GetBool("seqs")
+	asJSON, _ := cmd.Flags().GetBool("json")
+
+	work, err := os.MkdirTemp(workParent, "spx-kv-digest-")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "kv digest: %v\n", err)
+		os.Exit(1)
+	}
+	defer os.RemoveAll(work)
+
+	copiedAt := time.Now().UTC()
+	skipped, metas, err := copyStreamTree(storeDir, filepath.Join(work, "jetstream"))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "kv digest: %v\n", err)
+		os.RemoveAll(work)
+		os.Exit(1)
+	}
+
+	ctx, cancel := context.WithTimeout(cmd.Context(), 10*time.Minute)
+	defer cancel()
+	digests, err := digestStore(ctx, work, only, withSeqs, metas)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "kv digest: %v\n", err)
+		cancel()
+		os.RemoveAll(work)
+		os.Exit(1)
+	}
+
+	host, _ := os.Hostname()
+	if asJSON {
+		enc := json.NewEncoder(cmd.OutOrStdout())
+		if err := enc.Encode(nodeDigest{Host: host, Store: storeDir, CopiedAt: copiedAt, Skipped: skipped, Streams: digests}); err != nil {
+			fmt.Fprintf(os.Stderr, "kv digest: %v\n", err)
+			os.Exit(1)
+		}
+		return
+	}
+	writeDigests(cmd.OutOrStdout(), host, storeDir, copiedAt, skipped, digests)
+}
+
+// writeDigests prints the report. Only the "#" lines carry node-specific
+// values, so `grep -v '^#'` output from two replicas diffs cleanly.
+func writeDigests(w io.Writer, host, storeDir string, copiedAt time.Time, skipped []string, digests []streamDigest) {
+	fmt.Fprintf(w, "# host=%s store=%s copied_at=%s\n", host, storeDir, copiedAt.Format(time.RFC3339))
+	for _, s := range skipped {
+		fmt.Fprintf(w, "# skipped account stream (not read): %s\n", s)
+	}
+	for _, d := range digests {
+		fmt.Fprintf(w, "%s\tfirst=%d\tlast=%d\tmsgs=%d\tsha256=%s\n", d.Name, d.FirstSeq, d.LastSeq, d.Msgs, d.Digest)
+		for _, s := range d.Seqs {
+			fmt.Fprintf(w, "%s\tseq=%d\t%s\n", d.Name, s.Seq, s.Hash)
+		}
+	}
+}
+
+// globalAccountDir is the account Spinifex's token-authenticated clients use.
+// Streams under any other account are listed but not read.
+const globalAccountDir = "$G"
+
+// copyStreamTree copies <storeDir>/$G/streams to <dst>/$G/streams, leaving out
+// consumer state (obs). It returns streams found under other accounts and each
+// copied stream's config as the live server held it, keyed by stream name.
+func copyStreamTree(storeDir, dst string) (skipped []string, metas map[string]streamMeta, err error) {
+	streams, err := localStreams(storeDir)
+	if err != nil {
+		return nil, nil, err
+	}
+	if len(streams) == 0 {
+		return nil, nil, fmt.Errorf("no streams found under %s", storeDir)
+	}
+	for _, s := range streams {
+		if !strings.HasPrefix(s, globalAccountDir+"/") {
+			skipped = append(skipped, s)
+		}
+	}
+
+	src := filepath.Join(storeDir, globalAccountDir, "streams")
+	out := filepath.Join(dst, globalAccountDir, "streams")
+	err = filepath.WalkDir(src, func(path string, d fs.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			return walkErr
+		}
+		rel, err := filepath.Rel(src, path)
+		if err != nil {
+			return err
+		}
+		// <stream>/obs holds consumers, which are not stream content.
+		if parts := strings.Split(rel, string(filepath.Separator)); d.IsDir() && len(parts) == 2 && parts[1] == "obs" {
+			return filepath.SkipDir
+		}
+		target := filepath.Join(out, rel)
+		if d.IsDir() {
+			return os.MkdirAll(target, 0o700)
+		}
+		if !d.Type().IsRegular() {
+			return nil
+		}
+		return copyFile(path, target)
+	})
+	if err != nil {
+		return nil, nil, fmt.Errorf("copy streams from %s: %w", src, err)
+	}
+	entries, err := os.ReadDir(out)
+	if err != nil {
+		return nil, nil, err
+	}
+	metas = make(map[string]streamMeta, len(entries))
+	for _, e := range entries {
+		if e.IsDir() {
+			meta, err := rewriteStreamMetaStandalone(filepath.Join(out, e.Name()))
+			if err != nil {
+				return nil, nil, err
+			}
+			metas[e.Name()] = meta
+		}
+	}
+	return skipped, metas, nil
+}
+
+// rewriteStreamMetaStandalone makes a copied stream loadable by a standalone
+// server, which refuses replicas > 1 and would expire aged messages on
+// recovery. meta.sum is highwayhash64 keyed by sha256 of the directory name.
+// It returns the settings as they were before the rewrite.
+func rewriteStreamMetaStandalone(streamDir string) (streamMeta, error) {
+	metaPath := filepath.Join(streamDir, "meta.inf")
+	raw, err := os.ReadFile(metaPath)
+	if errors.Is(err, os.ErrNotExist) {
+		return streamMeta{}, nil
+	}
+	if err != nil {
+		return streamMeta{}, err
+	}
+	var orig struct {
+		Replicas    int           `json:"num_replicas"`
+		Retention   string        `json:"retention"`
+		MaxAge      time.Duration `json:"max_age"`
+		AllowMsgTTL bool          `json:"allow_msg_ttl"`
+		MaxMsgs     int64         `json:"max_msgs"`
+		MaxBytes    int64         `json:"max_bytes"`
+	}
+	if err := json.Unmarshal(raw, &orig); err != nil {
+		return streamMeta{}, fmt.Errorf("parse %s: %w", metaPath, err)
+	}
+	kept := streamMeta{
+		Replicas: orig.Replicas, Retention: orig.Retention, MaxAge: orig.MaxAge,
+		AllowMsgTTL: orig.AllowMsgTTL, MaxMsgs: orig.MaxMsgs, MaxBytes: orig.MaxBytes,
+	}
+	dec := json.NewDecoder(bytes.NewReader(raw))
+	dec.UseNumber()
+	var meta map[string]any
+	if err := dec.Decode(&meta); err != nil {
+		return streamMeta{}, fmt.Errorf("parse %s: %w", metaPath, err)
+	}
+	meta["num_replicas"] = 1
+	meta["max_age"] = 0
+	delete(meta, "allow_msg_ttl")
+	delete(meta, "subject_delete_marker_ttl")
+	delete(meta, "placement")
+	out, err := json.Marshal(meta)
+	if err != nil {
+		return streamMeta{}, err
+	}
+	key := sha256.Sum256([]byte(filepath.Base(streamDir)))
+	hh, err := highwayhash.New64(key[:])
+	if err != nil {
+		return streamMeta{}, err
+	}
+	hh.Write(out)
+	if err := os.WriteFile(metaPath, out, 0o600); err != nil {
+		return streamMeta{}, err
+	}
+	return kept, os.WriteFile(filepath.Join(streamDir, "meta.sum"), []byte(hex.EncodeToString(hh.Sum(nil))), 0o600)
+}
+
+func copyFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+	out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(out, in); err != nil {
+		out.Close()
+		return err
+	}
+	return out.Close()
+}
+
+// digestStore opens storeParent (holding jetstream/) in a standalone server
+// that accepts in-process connections only, and digests each stream.
+func digestStore(ctx context.Context, storeParent string, only []string, withSeqs bool, metas map[string]streamMeta) ([]streamDigest, error) {
+	ns, err := server.NewServer(&server.Options{
+		ServerName: "spx-kv-digest",
+		JetStream:  true,
+		StoreDir:   storeParent,
+		DontListen: true,
+		NoLog:      true,
+		NoSigs:     true,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("create reader server: %w", err)
+	}
+	ns.Start()
+	defer func() {
+		ns.Shutdown()
+		ns.WaitForShutdown()
+	}()
+	if !ns.ReadyForConnections(30 * time.Second) {
+		return nil, errors.New("reader server did not become ready")
+	}
+
+	nc, err := nats.Connect("", nats.InProcessServer(ns))
+	if err != nil {
+		return nil, fmt.Errorf("connect to reader server: %w", err)
+	}
+	defer nc.Close()
+	js, err := jetstream.New(nc)
+	if err != nil {
+		return nil, err
+	}
+
+	var names []string
+	lister := js.StreamNames(ctx)
+	for name := range lister.Name() {
+		if len(only) == 0 || slices.Contains(only, name) {
+			names = append(names, name)
+		}
+	}
+	if err := lister.Err(); err != nil {
+		return nil, fmt.Errorf("list streams: %w", err)
+	}
+	sort.Strings(names)
+
+	digests := make([]streamDigest, 0, len(names))
+	for _, name := range names {
+		d, err := digestStream(ctx, js, name, withSeqs)
+		if err != nil {
+			return nil, err
+		}
+		d.streamMeta = metas[name]
+		digests = append(digests, d)
+	}
+	return digests, nil
+}
+
+func digestStream(ctx context.Context, js jetstream.JetStream, name string, withSeqs bool) (streamDigest, error) {
+	s, err := js.Stream(ctx, name)
+	if err != nil {
+		return streamDigest{}, fmt.Errorf("open stream %s: %w", name, err)
+	}
+	info, err := s.Info(ctx)
+	if err != nil {
+		return streamDigest{}, fmt.Errorf("stream info %s: %w", name, err)
+	}
+	st := info.State
+	d := streamDigest{Name: name, FirstSeq: st.FirstSeq, LastSeq: st.LastSeq, Msgs: st.Msgs}
+
+	whole := sha256.New()
+	for seq := st.FirstSeq; st.Msgs > 0 && seq <= st.LastSeq; seq++ {
+		msg, err := s.GetMsg(ctx, seq)
+		sd := seqDigest{Seq: seq}
+		switch {
+		case errors.Is(err, jetstream.ErrMsgNotFound):
+			sd.Hash = deletedHash
+		case err != nil:
+			return streamDigest{}, fmt.Errorf("read %s seq %d: %w", name, seq, err)
+		default:
+			sd.Hash = hashStoredMsg(msg)
+			sd.Subject = msg.Subject
+			sd.Time = msg.Time.UTC()
+			sd.TTL = msgTTL(msg.Header.Get(jetstream.MsgTTLHeader))
+		}
+		fmt.Fprintf(whole, "%d:%s\n", seq, sd.Hash)
+		if withSeqs {
+			d.Seqs = append(d.Seqs, sd)
+		}
+	}
+	d.Digest = hex.EncodeToString(whole.Sum(nil))
+	return d, nil
+}
+
+// hashStoredMsg hashes everything a replica stores for one sequence. Header
+// keys are sorted so the hash does not depend on map iteration order.
+func hashStoredMsg(m *jetstream.RawStreamMsg) string {
+	h := sha256.New()
+	var n [8]byte
+	field := func(b []byte) {
+		binary.BigEndian.PutUint64(n[:], uint64(len(b)))
+		h.Write(n[:])
+		h.Write(b)
+	}
+	binary.BigEndian.PutUint64(n[:], m.Sequence)
+	h.Write(n[:])
+	field([]byte(m.Subject))
+	binary.BigEndian.PutUint64(n[:], uint64(m.Time.UnixNano()))
+	h.Write(n[:])
+	keys := make([]string, 0, len(m.Header))
+	for k := range m.Header {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		field([]byte(k))
+		for _, v := range m.Header[k] {
+			field([]byte(v))
+		}
+	}
+	field(m.Data)
+	return hex.EncodeToString(h.Sum(nil))[:16]
+}
+
+// msgTTL parses a Nats-TTL header: a Go duration or whole seconds. "never", an
+// absent header or an unparseable one yield zero, meaning no expiry.
+func msgTTL(v string) time.Duration {
+	if v == "" {
+		return 0
+	}
+	if d, err := time.ParseDuration(v); err == nil && d > 0 {
+		return d
+	}
+	if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
+		return time.Duration(n) * time.Second
+	}
+	return 0
+}

cmd/spinifex/cmd/admin_kv_compare.go

@@ -0,0 +1,360 @@
+package cmd
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"slices"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+)
+
+var kvCompareCmd = &cobra.Command{
+	Use:   "compare <digest.json>...",
+	Short: "Compare the kv digests of every node and report replicas that have diverged",
+	Long: `Read the "spx admin kv digest --json --seqs" output of every node in the
+cluster, one file per node ("-" reads any number of them from stdin), and
+check that every stream's replicas hold the same content.
+
+A replica that is merely behind is not divergent, so only these fail:
+
+  - the same sequence holds different content on two replicas;
+  - a message held by one replica is missing from another, and nothing on
+    that replica explains it: no later write to the same subject, no expiry
+    under max_age or a message TTL, and no stream limit dropping its head;
+  - a stream is held by fewer nodes than its replica count, or by more;
+  - replicas disagree on the replica count;
+  - no sequence range is common to every replica, so nothing can be checked.
+
+Sequences past the slowest replica's last one are not compared. Digests are
+taken from live stores a few seconds apart, so re-run before trusting a
+failure that could be a write landing between two copies.
+
+Exits 0 when every stream is consistent, 1 on divergence, 2 on bad input.`,
+	Args: cobra.MinimumNArgs(1),
+	Run:  runKVCompare,
+}
+
+func init() {
+	kvCmd.AddCommand(kvCompareCmd)
+}
+
+func runKVCompare(cmd *cobra.Command, args []string) {
+	var nodes []nodeDigest
+	for _, arg := range args {
+		var r io.Reader
+		if arg == "-" {
+			r = cmd.InOrStdin()
+		} else {
+			f, err := os.Open(arg)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "kv compare: %v\n", err)
+				os.Exit(2)
+			}
+			defer f.Close()
+			r = f
+		}
+		got, err := loadNodeDigests(r, arg)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "kv compare: %v\n", err)
+			os.Exit(2)
+		}
+		nodes = append(nodes, got...)
+	}
+	res, err := compareNodeDigests(nodes)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "kv compare: %v\n", err)
+		os.Exit(2)
+	}
+	writeCompareResult(cmd.OutOrStdout(), nodes, res)
+	if res.divergent() {
+		os.Exit(1)
+	}
+}
+
+// loadNodeDigests reads every JSON digest document in r.
+func loadNodeDigests(r io.Reader, source string) ([]nodeDigest, error) {
+	dec := json.NewDecoder(r)
+	var out []nodeDigest
+	for {
+		var d nodeDigest
+		err := dec.Decode(&d)
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("%s: document %d: %w", source, len(out)+1, err)
+		}
+		if d.Host == "" {
+			return nil, fmt.Errorf("%s: document %d has no host; is it \"kv digest --json\" output?", source, len(out)+1)
+		}
+		out = append(out, d)
+	}
+	if len(out) == 0 {
+		return nil, fmt.Errorf("%s: no digest documents", source)
+	}
+	return out, nil
+}
+
+// maxFindingsPerStream caps what one stream prints; the rest are counted.
+const maxFindingsPerStream = 5
+
+type streamVerdict struct {
+	Name     string
+	Holders  []string
+	Problems []string
+	Notes    []string
+}
+
+type kvCompareResult struct {
+	Streams []streamVerdict
+}
+
+func (r kvCompareResult) divergent() bool {
+	return slices.ContainsFunc(r.Streams, func(v streamVerdict) bool { return len(v.Problems) > 0 })
+}
+
+// replicaView is one node's copy of one stream, indexed for the per-sequence rules.
+type replicaView struct {
+	host          string
+	copiedAt      time.Time
+	d             *streamDigest
+	bySeq         map[uint64]seqDigest
+	lastBySubject map[string]uint64
+}
+
+func compareNodeDigests(nodes []nodeDigest) (kvCompareResult, error) {
+	if len(nodes) < 2 {
+		return kvCompareResult{}, fmt.Errorf("need the digests of at least two nodes, got %d", len(nodes))
+	}
+	hosts := make([]string, 0, len(nodes))
+	byName := map[string][]*replicaView{}
+	for i := range nodes {
+		n := &nodes[i]
+		if slices.Contains(hosts, n.Host) {
+			return kvCompareResult{}, fmt.Errorf("two digests are from host %s", n.Host)
+		}
+		hosts = append(hosts, n.Host)
+		for j := range n.Streams {
+			d := &n.Streams[j]
+			v := &replicaView{host: n.Host, copiedAt: n.CopiedAt, d: d,
+				bySeq: make(map[uint64]seqDigest, len(d.Seqs)), lastBySubject: map[string]uint64{}}
+			for _, s := range d.Seqs {
+				v.bySeq[s.Seq] = s
+				if s.Hash != deletedHash && s.Seq > v.lastBySubject[s.Subject] {
+					v.lastBySubject[s.Subject] = s.Seq
+				}
+			}
+			byName[d.Name] = append(byName[d.Name], v)
+		}
+	}
+	names := make([]string, 0, len(byName))
+	for name := range byName {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+
+	var res kvCompareResult
+	for _, name := range names {
+		res.Streams = append(res.Streams, compareStream(name, hosts, byName[name]))
+	}
+	return res, nil
+}
+
+func compareStream(name string, hosts []string, views []*replicaView) streamVerdict {
+	v := streamVerdict{Name: name}
+	for _, r := range views {
+		v.Holders = append(v.Holders, r.host)
+	}
+
+	counts := map[int][]string{}
+	replicas := 0
+	for _, r := range views {
+		if n := r.d.Replicas; n > 0 {
+			counts[n] = append(counts[n], r.host)
+			replicas = max(replicas, n)
+		}
+	}
+	if len(counts) > 1 {
+		var parts []string
+		for n, hs := range counts {
+			parts = append(parts, fmt.Sprintf("%d on %s", n, strings.Join(hs, ",")))
+		}
+		sort.Strings(parts)
+		v.Problems = append(v.Problems, "replicas disagree on the replica count: "+strings.Join(parts, "; "))
+	}
+	switch {
+	case replicas > 0 && len(views) < replicas:
+		missing := slices.DeleteFunc(slices.Clone(hosts), func(h string) bool { return slices.Contains(v.Holders, h) })
+		msg := fmt.Sprintf("held by %d of %d replicas; missing from %s", len(views), replicas, strings.Join(missing, ","))
+		if len(hosts) < replicas {
+			msg += fmt.Sprintf(" (only %d digests given)", len(hosts))
+		}
+		v.Problems = append(v.Problems, msg)
+	case replicas > 0 && len(views) > replicas:
+		v.Problems = append(v.Problems, fmt.Sprintf("held by %d nodes but configured for %d replicas; a node outside the stream's group still holds a copy", len(views), replicas))
+	}
+	if len(views) < 2 {
+		v.Notes = append(v.Notes, "single copy, nothing to compare")
+		return v
+	}
+
+	identical := true
+	for _, r := range views[1:] {
+		if r.d.Digest != views[0].d.Digest || r.d.FirstSeq != views[0].d.FirstSeq || r.d.LastSeq != views[0].d.LastSeq {
+			identical = false
+			break
+		}
+	}
+	if identical {
+		v.Notes = append(v.Notes, fmt.Sprintf("identical, seqs %d-%d", views[0].d.FirstSeq, views[0].d.LastSeq))
+		return v
+	}
+	for _, r := range views {
+		if r.d.Msgs > 0 && len(r.d.Seqs) == 0 {
+			v.Problems = append(v.Problems, "contents differ and "+r.host+"'s digest has no per-sequence detail; re-run digest with --seqs")
+			return v
+		}
+	}
+
+	lo, hi := views[0].d.FirstSeq, views[0].d.LastSeq
+	seqSet := map[uint64]struct{}{}
+	for _, r := range views {
+		lo, hi = max(lo, r.d.FirstSeq), min(hi, r.d.LastSeq)
+		for s := range r.bySeq {
+			seqSet[s] = struct{}{}
+		}
+	}
+	if lo > hi {
+		v.Problems = append(v.Problems, "no sequence range is common to every replica, so nothing could be compared")
+		return v
+	}
+	seqs := make([]uint64, 0, len(seqSet))
+	for s := range seqSet {
+		if s <= hi {
+			seqs = append(seqs, s)
+		}
+	}
+	slices.Sort(seqs)
+
+	var problems []string
+	explained := 0
+	for _, seq := range seqs {
+		var present, absent []*replicaView
+		for _, r := range views {
+			if e, ok := r.bySeq[seq]; ok && e.Hash != deletedHash {
+				present = append(present, r)
+			} else {
+				absent = append(absent, r)
+			}
+		}
+		if len(present) == 0 {
+			continue
+		}
+		byHash := map[string][]string{}
+		for _, r := range present {
+			h := r.bySeq[seq].Hash
+			byHash[h] = append(byHash[h], r.host)
+		}
+		if len(byHash) > 1 {
+			var parts []string
+			for h, hs := range byHash {
+				parts = append(parts, strings.Join(hs, ",")+"="+h)
+			}
+			sort.Strings(parts)
+			problems = append(problems, fmt.Sprintf("seq %d %s holds different content: %s", seq, present[0].bySeq[seq].Subject, strings.Join(parts, " ")))
+			continue
+		}
+		msg := present[0].bySeq[seq]
+		for _, r := range absent {
+			if absenceExplained(r, msg) {
+				explained++
+				continue
+			}
+			problems = append(problems, fmt.Sprintf("seq %d %s is on %s but missing from %s, with no later write to that subject, expiry or limit to explain it",
+				seq, msg.Subject, hostsOf(present), r.host))
+		}
+	}
+	if len(problems) > maxFindingsPerStream {
+		problems = append(problems[:maxFindingsPerStream], fmt.Sprintf("... and %d more", len(problems)-maxFindingsPerStream))
+	}
+	v.Problems = append(v.Problems, problems...)
+
+	note := fmt.Sprintf("seqs %d-%d compared", lo, hi)
+	if explained > 0 {
+		note += fmt.Sprintf(", %d superseded or expired messages missing from some replicas", explained)
+	}
+	var ahead []string
+	for _, r := range views {
+		if r.d.LastSeq > hi {
+			ahead = append(ahead, fmt.Sprintf("%s +%d", r.host, r.d.LastSeq-hi))
+		}
+	}
+	if len(ahead) > 0 {
+		note += ", tail not compared: " + strings.Join(ahead, " ")
+	}
+	v.Notes = append(v.Notes, note)
+	return v
+}
+
+// absenceExplained reports whether a replica lacking msg removed it legitimately:
+// the subject was written again later, the message aged out before that replica
+// was copied, or a stream limit dropped it from a head the replica has moved past.
+func absenceExplained(r *replicaView, msg seqDigest) bool {
+	if msg.Subject == "" {
+		return false
+	}
+	if r.lastBySubject[msg.Subject] > msg.Seq {
+		return true
+	}
+	meta := r.d.streamMeta
+	if meta.Retention != "" && meta.Retention != "limits" {
+		return true
+	}
+	if !msg.Time.IsZero() {
+		if meta.MaxAge > 0 && msg.Time.Add(meta.MaxAge).Before(r.copiedAt) {
+			return true
+		}
+		if msg.TTL > 0 && msg.Time.Add(msg.TTL).Before(r.copiedAt) {
+			return true
+		}
+	}
+	return msg.Seq < r.d.FirstSeq && (meta.MaxMsgs > 0 || meta.MaxBytes > 0)
+}
+
+func hostsOf(views []*replicaView) string {
+	hs := make([]string, len(views))
+	for i, r := range views {
+		hs[i] = r.host
+	}
+	return strings.Join(hs, ",")
+}
+
+func writeCompareResult(w io.Writer, nodes []nodeDigest, res kvCompareResult) {
+	fmt.Fprintf(w, "compared %d digests:", len(nodes))
+	for _, n := range nodes {
+		fmt.Fprintf(w, " %s@%s", n.Host, n.CopiedAt.Format(time.TimeOnly))
+	}
+	fmt.Fprintln(w)
+	bad := 0
+	for _, v := range res.Streams {
+		status := "ok"
+		if len(v.Problems) > 0 {
+			status = "DIVERGED"
+			bad++
+		}
+		fmt.Fprintf(w, "%-9s %s  [%s]\n", status, v.Name, strings.Join(v.Holders, ","))
+		for _, p := range v.Problems {
+			fmt.Fprintf(w, "    ! %s\n", p)
+		}
+		for _, n := range v.Notes {
+			fmt.Fprintf(w, "      %s\n", n)
+		}
+	}
+	fmt.Fprintf(w, "%d streams: %d consistent, %d diverged\n", len(res.Streams), len(res.Streams)-bad, bad)
+}

cmd/spinifex/cmd/admin_kv_compare_test.go

@@ -0,0 +1,233 @@
+package cmd_test
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/cmd/spinifex/cmd"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var t0 = time.Date(2026, 9, 15, 10, 0, 0, 0, time.UTC)
+
+type msg struct {
+	seq     uint64
+	subject string
+	hash    string // "deleted" for a gap
+	at      time.Time
+}
+
+type stream struct {
+	name     string
+	replicas int
+	maxAge   time.Duration
+	maxMsgs  int64
+	msgs     []msg
+	noSeqs   bool
+}
+
+// doc renders one node's digest the way "kv digest --json --seqs" prints it.
+func doc(t *testing.T, host string, copiedAt time.Time, streams ...stream) string {
+	t.Helper()
+	var out []map[string]any
+	for _, s := range streams {
+		var seqs []map[string]any
+		var first, last, count uint64
+		digest := host + ":" // only equality matters to compare
+		for _, m := range s.msgs {
+			if first == 0 {
+				first = m.seq
+			}
+			last = m.seq
+			e := map[string]any{"seq": m.seq, "hash": m.hash}
+			if m.hash != "deleted" {
+				count++
+				e["subject"] = m.subject
+				at := m.at
+				if at.IsZero() {
+					at = t0
+				}
+				e["time"] = at
+			}
+			digest += m.hash + ","
+			seqs = append(seqs, e)
+		}
+		sd := map[string]any{"name": s.name, "first_seq": first, "last_seq": last, "msgs": count, "sha256": digest}
+		if s.replicas > 0 {
+			sd["replicas"] = s.replicas
+		}
+		if s.maxAge > 0 {
+			sd["max_age_ns"] = s.maxAge
+		}
+		if s.maxMsgs > 0 {
+			sd["max_msgs"] = s.maxMsgs
+		}
+		if !s.noSeqs {
+			sd["seqs"] = seqs
+		}
+		out = append(out, sd)
+	}
+	raw, err := json.Marshal(map[string]any{"host": host, "store": "/s", "copied_at": copiedAt, "streams": out})
+	require.NoError(t, err)
+	return string(raw)
+}
+
+func kvs(msgs ...msg) []msg { return msgs }
+
+func TestKVCompareIdentical(t *testing.T) {
+	s := stream{name: "KV_a", replicas: 3, msgs: kvs(msg{1, "k", "h1", t0}, msg{2, "k", "h2", t0})}
+	// The digest field embeds the host, so this exercises the per-seq path too.
+	divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, s), doc(t, "n2", t0, s), doc(t, "n3", t0, s))
+	require.NoError(t, err)
+	assert.False(t, divergent, report)
+	assert.Contains(t, report, "1 streams: 1 consistent, 0 diverged")
+}
+
+// The prod failure: each node wrote its own copy before formation, so the same
+// sequence holds different content on different replicas.
+func TestKVCompareSameSeqDifferentContent(t *testing.T) {
+	a := stream{name: "KV_acct", replicas: 2, msgs: kvs(msg{1, "$KV.acct.000000000002", "aaaa", t0})}
+	b := stream{name: "KV_acct", replicas: 2, msgs: kvs(msg{1, "$KV.acct.000000000002", "bbbb", t0})}
+	divergent, report, err := cmd.CompareKVDigests(doc(t, "node1", t0, a), doc(t, "node2", t0, b))
+	require.NoError(t, err)
+	assert.True(t, divergent)
+	assert.Contains(t, report, "DIVERGED  KV_acct")
+	assert.Contains(t, report, "seq 1 $KV.acct.000000000002 holds different content: node1=aaaa node2=bbbb")
+}
+
+func TestKVCompareMissingMessage(t *testing.T) {
+	full := kvs(msg{1, "k", "h1", t0}, msg{2, "j", "h2", t0}, msg{3, "k", "h3", t0})
+
+	t.Run("superseded by a later write to the subject", func(t *testing.T) {
+		lagging := stream{name: "KV_a", replicas: 2, msgs: full}
+		purged := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{1, "", "deleted", time.Time{}}, full[1], full[2])}
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, lagging), doc(t, "n2", t0, purged))
+		require.NoError(t, err)
+		assert.False(t, divergent, report)
+		assert.Contains(t, report, "1 superseded or expired")
+	})
+
+	t.Run("nothing explains it", func(t *testing.T) {
+		have := stream{name: "KV_a", replicas: 2, msgs: full}
+		lost := stream{name: "KV_a", replicas: 2, msgs: kvs(full[0], msg{2, "", "deleted", time.Time{}}, full[2])}
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, have), doc(t, "n2", t0, lost))
+		require.NoError(t, err)
+		assert.True(t, divergent)
+		assert.Contains(t, report, "seq 2 j is on n1 but missing from n2")
+	})
+
+	t.Run("aged out under max_age before the replica was copied", func(t *testing.T) {
+		old := t0.Add(-time.Hour)
+		have := stream{name: "KV_lease", replicas: 2, maxAge: time.Minute, msgs: kvs(msg{1, "l", "h1", old}, msg{2, "m", "h2", t0})}
+		gone := stream{name: "KV_lease", replicas: 2, maxAge: time.Minute, msgs: kvs(msg{2, "m", "h2", t0})}
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, have), doc(t, "n2", t0, gone))
+		require.NoError(t, err)
+		assert.False(t, divergent, report)
+
+		have.maxAge, gone.maxAge = 0, 0
+		divergent, _, err = cmd.CompareKVDigests(doc(t, "n1", t0, have), doc(t, "n2", t0, gone))
+		require.NoError(t, err)
+		assert.True(t, divergent, "without max_age the same gap is unexplained")
+	})
+
+	t.Run("head dropped by a stream limit", func(t *testing.T) {
+		behind := stream{name: "S", replicas: 2, maxMsgs: 2, msgs: kvs(msg{1, "a", "h1", t0}, msg{2, "b", "h2", t0})}
+		ahead := stream{name: "S", replicas: 2, maxMsgs: 2, msgs: kvs(msg{2, "b", "h2", t0}, msg{3, "c", "h3", t0})}
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, behind), doc(t, "n2", t0, ahead))
+		require.NoError(t, err)
+		assert.False(t, divergent, report)
+	})
+}
+
+func TestKVCompareTailLag(t *testing.T) {
+	behind := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{1, "k", "h1", t0}, msg{2, "k", "h2", t0})}
+	ahead := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{1, "k", "h1", t0}, msg{2, "k", "h2", t0}, msg{3, "k", "h3", t0}, msg{4, "k", "h4", t0})}
+	divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, behind), doc(t, "n2", t0, ahead))
+	require.NoError(t, err)
+	assert.False(t, divergent, report)
+	assert.Contains(t, report, "tail not compared: n2 +2")
+}
+
+func TestKVCompareNoCommonRange(t *testing.T) {
+	a := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{1, "k", "h1", t0})}
+	b := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{5, "k", "h5", t0})}
+	divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, a), doc(t, "n2", t0, b))
+	require.NoError(t, err)
+	assert.True(t, divergent)
+	assert.Contains(t, report, "no sequence range is common")
+}
+
+func TestKVCompareReplicaPlacement(t *testing.T) {
+	s := stream{name: "KV_a", replicas: 3, msgs: kvs(msg{1, "k", "h1", t0})}
+	other := stream{name: "KV_b", replicas: 1, msgs: kvs(msg{1, "k", "h1", t0})}
+
+	t.Run("fewer holders than replicas", func(t *testing.T) {
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, s), doc(t, "n2", t0, s), doc(t, "n3", t0, other))
+		require.NoError(t, err)
+		assert.True(t, divergent)
+		assert.Contains(t, report, "held by 2 of 3 replicas; missing from n3")
+	})
+
+	t.Run("more holders than replicas", func(t *testing.T) {
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, other), doc(t, "n2", t0, other))
+		require.NoError(t, err)
+		assert.True(t, divergent)
+		assert.Contains(t, report, "held by 2 nodes but configured for 1 replicas")
+	})
+
+	t.Run("replica counts disagree", func(t *testing.T) {
+		r1 := s
+		r1.replicas = 1
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, s), doc(t, "n2", t0, r1))
+		require.NoError(t, err)
+		assert.True(t, divergent)
+		assert.Contains(t, report, "replicas disagree on the replica count")
+	})
+
+	t.Run("single replica stream on one node", func(t *testing.T) {
+		divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, other), doc(t, "n2", t0))
+		require.NoError(t, err)
+		assert.False(t, divergent, report)
+	})
+}
+
+func TestKVCompareNeedsSeqs(t *testing.T) {
+	a := stream{name: "KV_a", replicas: 2, noSeqs: true, msgs: kvs(msg{1, "k", "h1", t0})}
+	b := stream{name: "KV_a", replicas: 2, noSeqs: true, msgs: kvs(msg{1, "k", "h2", t0})}
+	divergent, report, err := cmd.CompareKVDigests(doc(t, "n1", t0, a), doc(t, "n2", t0, b))
+	require.NoError(t, err)
+	assert.True(t, divergent)
+	assert.Contains(t, report, "re-run digest with --seqs")
+}
+
+func TestKVCompareInput(t *testing.T) {
+	s := stream{name: "KV_a", replicas: 2, msgs: kvs(msg{1, "k", "h1", t0})}
+
+	// Several documents in one stream, as "-" reads them from stdin.
+	divergent, _, err := cmd.CompareKVDigests(doc(t, "n1", t0, s) + "\n" + doc(t, "n2", t0, s))
+	require.NoError(t, err)
+	assert.False(t, divergent)
+
+	_, _, err = cmd.CompareKVDigests(doc(t, "n1", t0, s))
+	assert.ErrorContains(t, err, "at least two nodes")
+
+	_, _, err = cmd.CompareKVDigests(doc(t, "n1", t0, s), doc(t, "n1", t0, s))
+	assert.ErrorContains(t, err, "two digests are from host n1")
+
+	_, _, err = cmd.CompareKVDigests(`KV_a	first=1	last=1`)
+	assert.Error(t, err, "text digest output is rejected")
+
+	_, _, err = cmd.CompareKVDigests(`{"streams":[]}`)
+	assert.ErrorContains(t, err, "no host")
+
+	_, _, err = cmd.CompareKVDigests("  ")
+	assert.ErrorContains(t, err, "no digest documents")
+}
+
+func TestAdminInitDiscardsJetStreamByDefault(t *testing.T) {
+	assert.Equal(t, "true", cmd.AdminInitFlagDefault("discard-jetstream"))
+	assert.True(t, strings.HasPrefix(cmd.AdminInitFlagDefault("ipsec"), "true"))
+}

cmd/spinifex/cmd/admin_kv_test.go

@@ -0,0 +1,319 @@
+//test:in-package: exercises the unexported digest helpers against a real clustered
+// store, which the cobra command only reaches through a live node.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// kvServer runs a standalone JetStream server on storeParent and returns a KV
+// bucket on it plus a stop function.
+func kvServer(t *testing.T, storeParent, bucket string) (jetstream.KeyValue, func()) {
+	t.Helper()
+	ns, err := server.NewServer(&server.Options{
+		ServerName: "kv-test", JetStream: true, StoreDir: storeParent,
+		DontListen: true, NoLog: true, NoSigs: true,
+	})
+	require.NoError(t, err)
+	ns.Start()
+	require.True(t, ns.ReadyForConnections(10*time.Second))
+	nc, err := nats.Connect("", nats.InProcessServer(ns))
+	require.NoError(t, err)
+	js, err := jetstream.New(nc)
+	require.NoError(t, err)
+	kv, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: bucket, History: 5})
+	require.NoError(t, err)
+	return kv, func() {
+		nc.Close()
+		ns.Shutdown()
+		ns.WaitForShutdown()
+	}
+}
+
+func digestOf(t *testing.T, storeDir string, withSeqs bool) []streamDigest {
+	t.Helper()
+	work := t.TempDir()
+	_, metas, err := copyStreamTree(storeDir, filepath.Join(work, "jetstream"))
+	require.NoError(t, err)
+	d, err := digestStore(context.Background(), work, nil, withSeqs, metas)
+	require.NoError(t, err)
+	return d
+}
+
+func TestKVDigest(t *testing.T) {
+	ctx := context.Background()
+	parent := t.TempDir()
+	kv, stop := kvServer(t, parent, "test")
+	for _, kvp := range [][2]string{{"a", "1"}, {"b", "2"}, {"a", "3"}} {
+		_, err := kv.PutString(ctx, kvp[0], kvp[1])
+		require.NoError(t, err)
+	}
+	require.NoError(t, kv.Delete(ctx, "b"))
+	store := filepath.Join(parent, "jetstream")
+
+	// Copied while the server is live, exactly as it runs on a node.
+	live := digestOf(t, store, true)
+	stop()
+
+	require.Len(t, live, 1)
+	assert.Equal(t, "KV_test", live[0].Name)
+	assert.Equal(t, uint64(1), live[0].FirstSeq)
+	assert.Equal(t, uint64(4), live[0].LastSeq)
+	assert.Equal(t, uint64(4), live[0].Msgs)
+	require.Len(t, live[0].Seqs, 4)
+
+	t.Run("stable across reads and a stopped server", func(t *testing.T) {
+		again := digestOf(t, store, false)
+		assert.Equal(t, live[0].Digest, again[0].Digest)
+	})
+
+	t.Run("source store is left untouched", func(t *testing.T) {
+		_, err := os.Stat(filepath.Join(store, "$G", "streams", "KV_test"))
+		assert.NoError(t, err)
+	})
+
+	// Same keys and values written separately differ in timestamps, which is
+	// exactly what separates two nodes' pre-formation writes.
+	t.Run("independently written store differs", func(t *testing.T) {
+		other := t.TempDir()
+		kv2, stop2 := kvServer(t, other, "test")
+		for _, kvp := range [][2]string{{"a", "1"}, {"b", "2"}, {"a", "3"}} {
+			_, err := kv2.PutString(ctx, kvp[0], kvp[1])
+			require.NoError(t, err)
+		}
+		require.NoError(t, kv2.Delete(ctx, "b"))
+		stop2()
+		d := digestOf(t, filepath.Join(other, "jetstream"), true)
+		assert.NotEqual(t, live[0].Digest, d[0].Digest)
+
+		// Two stores adopted as replicas of one stream: same subjects and
+		// sequences, different content, which compare must call divergent.
+		res, err := compareNodeDigests([]nodeDigest{
+			{Host: "node1", CopiedAt: time.Now(), Streams: live},
+			{Host: "node2", CopiedAt: time.Now(), Streams: d},
+		})
+		require.NoError(t, err)
+		require.True(t, res.divergent())
+		require.Len(t, res.Streams, 1)
+		assert.Contains(t, strings.Join(res.Streams[0].Problems, "\n"), "seq 1 $KV.test.a holds different content")
+	})
+
+	t.Run("report lines", func(t *testing.T) {
+		var buf bytes.Buffer
+		writeDigests(&buf, "node1", store, time.Unix(0, 0), []string{"ACC/S"}, live)
+		out := buf.String()
+		assert.Contains(t, out, "# host=node1 ")
+		assert.Contains(t, out, "# skipped account stream (not read): ACC/S")
+		assert.Contains(t, out, "KV_test\tfirst=1\tlast=4\tmsgs=4\tsha256="+live[0].Digest)
+		assert.Equal(t, 4, strings.Count(out, "KV_test\tseq="))
+	})
+}
+
+func TestKVDigestStreamFilterAndDeletedSeq(t *testing.T) {
+	ctx := context.Background()
+	parent := t.TempDir()
+	// The KV bucket is only there to be excluded by the stream filter.
+	_, stop := kvServer(t, parent, "other")
+	stop()
+
+	// KV streams deny message deletes, so an interior gap needs a plain stream.
+	ns, err := server.NewServer(&server.Options{JetStream: true, StoreDir: parent, DontListen: true, NoLog: true, NoSigs: true})
+	require.NoError(t, err)
+	ns.Start()
+	require.True(t, ns.ReadyForConnections(10*time.Second))
+	nc, err := nats.Connect("", nats.InProcessServer(ns))
+	require.NoError(t, err)
+	js, err := jetstream.New(nc)
+	require.NoError(t, err)
+	s, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "ONE", Subjects: []string{"one.>"}})
+	require.NoError(t, err)
+	for _, v := range []string{"v1", "v2", "v3"} {
+		_, err = js.Publish(ctx, "one.k", []byte(v))
+		require.NoError(t, err)
+	}
+	require.NoError(t, s.DeleteMsg(ctx, 2))
+	nc.Close()
+	ns.Shutdown()
+	ns.WaitForShutdown()
+
+	work := t.TempDir()
+	_, metas, err := copyStreamTree(filepath.Join(parent, "jetstream"), filepath.Join(work, "jetstream"))
+	require.NoError(t, err)
+	d, err := digestStore(ctx, work, []string{"ONE"}, true, metas)
+	require.NoError(t, err)
+	require.Len(t, d, 1)
+	assert.Equal(t, "ONE", d[0].Name)
+	assert.Equal(t, uint64(2), d[0].Msgs)
+	require.Len(t, d[0].Seqs, 3)
+	assert.Equal(t, seqDigest{Seq: 2, Hash: "deleted"}, d[0].Seqs[1])
+	for _, i := range []int{0, 2} {
+		assert.Equal(t, "one.k", d[0].Seqs[i].Subject)
+		assert.False(t, d[0].Seqs[i].Time.IsZero())
+	}
+	assert.Equal(t, "limits", d[0].Retention)
+	assert.Equal(t, 1, d[0].Replicas)
+}
+
+func freePort(t *testing.T) int {
+	t.Helper()
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	defer l.Close()
+	return l.Addr().(*net.TCPAddr).Port
+}
+
+// A replicated stream's meta.inf carries num_replicas > 1, which a standalone
+// server refuses to load; this is the case every production node presents.
+func TestKVDigestClusteredReplicas(t *testing.T) {
+	ctx := context.Background()
+	const n = 3
+	ports := make([]int, n)
+	routes := make([]string, n)
+	for i := range ports {
+		ports[i] = freePort(t)
+		routes[i] = fmt.Sprintf("nats://127.0.0.1:%d", ports[i])
+	}
+	stores := make([]string, n)
+	servers := make([]*server.Server, n)
+	for i := range servers {
+		stores[i] = t.TempDir()
+		ns, err := server.NewServer(&server.Options{
+			ServerName: fmt.Sprintf("s%d", i+1), JetStream: true, StoreDir: stores[i],
+			Host: "127.0.0.1", Port: -1, NoLog: true, NoSigs: true,
+			Cluster: server.ClusterOpts{Name: "digest", Host: "127.0.0.1", Port: ports[i]},
+			Routes:  server.RoutesFromStr(strings.Join(routes, ",")),
+		})
+		require.NoError(t, err)
+		ns.Start()
+		servers[i] = ns
+	}
+	t.Cleanup(func() {
+		for _, ns := range servers {
+			ns.Shutdown()
+			ns.WaitForShutdown()
+		}
+	})
+
+	nc, err := nats.Connect("", nats.InProcessServer(servers[0]))
+	require.NoError(t, err)
+	defer nc.Close()
+	js, err := jetstream.New(nc)
+	require.NoError(t, err)
+
+	var kv jetstream.KeyValue
+	require.Eventually(t, func() bool {
+		kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "rep", History: 3, Replicas: n, TTL: time.Hour})
+		return err == nil
+	}, 30*time.Second, 250*time.Millisecond, "clustered KV create: %v", err)
+	for i := range 5 {
+		_, err := kv.PutString(ctx, fmt.Sprintf("k%d", i%2), fmt.Sprintf("v%d", i))
+		require.NoError(t, err)
+	}
+	s, err := js.Stream(ctx, "KV_rep")
+	require.NoError(t, err)
+	require.Eventually(t, func() bool {
+		info, err := s.Info(ctx)
+		if err != nil || info.Cluster == nil || len(info.Cluster.Replicas) != n-1 {
+			return false
+		}
+		for _, r := range info.Cluster.Replicas {
+			if !r.Current || r.Lag != 0 {
+				return false
+			}
+		}
+		return true
+	}, 30*time.Second, 250*time.Millisecond)
+
+	// Current means applied, not flushed; shutdown flushes every block to disk.
+	nc.Close()
+	for _, ns := range servers {
+		ns.Shutdown()
+		ns.WaitForShutdown()
+	}
+
+	var first string
+	var nodes []nodeDigest
+	for i, store := range stores {
+		d := digestOf(t, filepath.Join(store, "jetstream"), true)
+		require.Len(t, d, 1, "replica %d", i+1)
+		assert.Equal(t, uint64(5), d[0].Msgs, "replica %d", i+1)
+		assert.Equal(t, n, d[0].Replicas, "live replica count survives the standalone rewrite")
+		assert.Equal(t, time.Hour, d[0].MaxAge)
+		nodes = append(nodes, nodeDigest{Host: fmt.Sprintf("s%d", i+1), CopiedAt: time.Now(), Streams: d})
+		if i == 0 {
+			first = d[0].Digest
+			continue
+		}
+		assert.Equal(t, first, d[0].Digest, "replica %d matches replica 1", i+1)
+	}
+	res, err := compareNodeDigests(nodes)
+	require.NoError(t, err)
+	assert.False(t, res.divergent(), "%+v", res)
+}
+
+// Recovery applies MaxAge, so without the rewrite an expired bucket copy
+// would read as empty and mask whatever the replica holds.
+func TestKVDigestIgnoresExpiry(t *testing.T) {
+	ctx := context.Background()
+	parent := t.TempDir()
+	ns, err := server.NewServer(&server.Options{JetStream: true, StoreDir: parent, DontListen: true, NoLog: true, NoSigs: true})
+	require.NoError(t, err)
+	ns.Start()
+	require.True(t, ns.ReadyForConnections(10*time.Second))
+	nc, err := nats.Connect("", nats.InProcessServer(ns))
+	require.NoError(t, err)
+	js, err := jetstream.New(nc)
+	require.NoError(t, err)
+	kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "ttl", TTL: time.Second})
+	require.NoError(t, err)
+	_, err = kv.PutString(ctx, "k", "v")
+	require.NoError(t, err)
+
+	work := t.TempDir()
+	_, metas, err := copyStreamTree(filepath.Join(parent, "jetstream"), filepath.Join(work, "jetstream"))
+	require.NoError(t, err)
+	nc.Close()
+	ns.Shutdown()
+	ns.WaitForShutdown()
+
+	time.Sleep(1500 * time.Millisecond)
+	d, err := digestStore(ctx, work, nil, false, metas)
+	require.NoError(t, err)
+	require.Len(t, d, 1)
+	assert.Equal(t, uint64(1), d[0].Msgs)
+}
+
+func TestCopyStreamTree(t *testing.T) {
+	store := seedStore(t, "KV_a")
+	require.NoError(t, os.MkdirAll(filepath.Join(store, "$G", "streams", "KV_a", "obs", "c1"), 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(store, "$G", "streams", "KV_a", "obs", "c1", "o.dat"), []byte("x"), 0o600))
+	require.NoError(t, os.MkdirAll(filepath.Join(store, "ACME", "streams", "ORDERS"), 0o755))
+
+	dst := filepath.Join(t.TempDir(), "jetstream")
+	skipped, _, err := copyStreamTree(store, dst)
+	require.NoError(t, err)
+	assert.Equal(t, []string{"ACME/ORDERS"}, skipped)
+
+	_, err = os.Stat(filepath.Join(dst, "$G", "streams", "KV_a", "msgs", "1.blk"))
+	assert.NoError(t, err, "message blocks are copied")
+	_, err = os.Stat(filepath.Join(dst, "$G", "streams", "KV_a", "obs"))
+	assert.True(t, os.IsNotExist(err), "consumer state is left out")
+
+	_, _, err = copyStreamTree(filepath.Join(t.TempDir(), "absent"), dst)
+	assert.Error(t, err)
+}

cmd/spinifex/cmd/admin_ochre.go

@@ -16,13 +16,14 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/bluebottle/pkg/masterkey"
+	"github.com/mulgadc/bluebottle/pkg/safecast"
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/ebsmetadata"
 	"github.com/mulgadc/spinifex/spinifex/ebsprovider"
 	gateway_bedrock "github.com/mulgadc/spinifex/spinifex/gateway/bedrock"
 	"github.com/mulgadc/spinifex/spinifex/gateway/bedrock/hfhub"
-	handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot"
-	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/nats-io/nats.go"
@@ -134,12 +135,11 @@ func init() {
 	_ = ochreWeightsRemoveCmd.MarkFlagRequired("model-id")
 
 	ochreWeightsPullCmd.Flags().String("model-id", "", "Self-host catalog model ID this pull is for (required)")
-	ochreWeightsPullCmd.Flags().String("hf-repo", "", "Hugging Face repo, e.g. meta-llama/Llama-3.2-1B-Instruct (required)")
-	ochreWeightsPullCmd.Flags().String("revision", "main", "Hugging Face branch, tag or commit SHA to resolve and pin")
+	ochreWeightsPullCmd.Flags().String("hf-repo", "", "Hugging Face repo, e.g. meta-llama/Llama-3.2-1B-Instruct (default: the catalog entry's repo)")
+	ochreWeightsPullCmd.Flags().String("revision", "main", "Hugging Face branch, tag or commit SHA to resolve and pin (default: the catalog entry's pinned revision)")
 	ochreWeightsPullCmd.Flags().String("s3-uri", "", "predastore destination prefix, e.g. s3://bucket/prefix/ (default: s3://ochre-weights/<repo>/<sha>/)")
 	ochreWeightsPullCmd.Flags().String("hf-token", "", "Hugging Face token for a gated repo (falls back to HF_TOKEN, then a stored platform credential)")
 	_ = ochreWeightsPullCmd.MarkFlagRequired("model-id")
-	_ = ochreWeightsPullCmd.MarkFlagRequired("hf-repo")
 
 	ochreCredentialsSetCmd.Flags().String("vendor", "", "Vendor to store a credential for, e.g. huggingface (required)")
 	ochreCredentialsSetCmd.Flags().String("account", "", "Account ID to store the credential under (default: platform account)")
@@ -180,20 +180,25 @@ type ochrePullManifest struct {
 	PulledAt    time.Time `json:"pulled_at"`
 }
 
-// allowedPullFileNames are the fixed-name Hugging Face artefacts pull fetches
-// alongside safetensors shards -- config and tokenizer files needed to serve,
-// never weights themselves. Matched by basename so a nested path (e.g.
-// Llama's original/tokenizer.model) still qualifies.
-var allowedPullFileNames = map[string]bool{
+// rootOnlyPullFileNames are the fixed-name config artefacts pull fetches only
+// at the repo root. A same-named file in a subdir is a component config, not
+// the model's -- e.g. sentence-transformers' 1_Pooling/config.json -- and must
+// never flatten onto the root config.json, so nested matches are dropped.
+var rootOnlyPullFileNames = map[string]bool{
 	"config.json":                  true,
 	"tokenizer_config.json":        true,
-	"tokenizer.json":               true,
-	"tokenizer.model":              true,
 	"generation_config.json":       true,
 	"special_tokens_map.json":      true,
 	"model.safetensors.index.json": true,
 }
 
+// nestablePullFileNames are the tokenizer artefacts matched by basename at any
+// depth: some repos keep them in a subdir (e.g. Llama's original/tokenizer.model).
+var nestablePullFileNames = map[string]bool{
+	"tokenizer.json":  true,
+	"tokenizer.model": true,
+}
+
 // selectPullFiles filters a Hugging Face tree listing down to the
 // safetensors-only set pull downloads (D4): every *.safetensors file plus
 // the fixed config/tokenizer set. Anything else -- notably .bin/.pt pickle
@@ -205,7 +210,14 @@ func selectPullFiles(entries []hfhub.TreeEntry) []hfhub.TreeEntry {
 		if e.Type != "file" {
 			continue
 		}
-		if strings.HasSuffix(e.Path, ".safetensors") || allowedPullFileNames[path.Base(e.Path)] {
+		base := path.Base(e.Path)
+		nested := strings.Contains(e.Path, "/")
+		switch {
+		case strings.HasSuffix(e.Path, ".safetensors"):
+			selected = append(selected, e)
+		case nestablePullFileNames[base]:
+			selected = append(selected, e)
+		case rootOnlyPullFileNames[base] && !nested:
 			selected = append(selected, e)
 		}
 	}
@@ -324,8 +336,13 @@ func pullFilesToPrefix(ctx context.Context, hf *hfhub.Client, store objectstore.
 	defer os.RemoveAll(tmpDir)
 
 	var uploaded []string
+	seen := make(map[string]string, len(files))
 	for _, f := range files {
 		key := prefix + path.Base(f.Path)
+		if prev, dup := seen[key]; dup {
+			return uploaded, fmt.Errorf("refusing to overwrite %s: both %s and %s flatten to it", key, prev, f.Path)
+		}
+		seen[key] = f.Path
 		if err := pullOneFile(ctx, hf, store, hfRepo, sha, f.Path, bucket, key, tmpDir); err != nil {
 			return uploaded, fmt.Errorf("pull %s: %w", f.Path, err)
 		}
@@ -334,18 +351,40 @@ func pullFilesToPrefix(ctx context.Context, hf *hfhub.Client, store objectstore.
 	return uploaded, nil
 }
 
+// weightsStagingRefusal builds the model-ID refusal error weights staging and
+// pulling share. Unknown IDs and provider-served IDs get distinct wording so
+// a caller can tell which one it hit; taking found/selfHost as plain bools
+// keeps this testable without a real catalog entry for the provider case.
+func weightsStagingRefusal(modelID, operation string, found, _ bool) error {
+	if !found {
+		return fmt.Errorf("unknown model ID %q: not present in the Ochre catalog", modelID)
+	}
+	return fmt.Errorf("%q is a provider-served model, not self-host; weights %s does not apply", modelID, operation)
+}
+
 // runPullWeights holds 'ochre weights pull' decision and side-effect logic:
 // catalog validation, ref resolution to an immutable commit SHA (D3), tree
 // listing and safetensors-only filtering (D4), streaming each selected file
 // into predastore (D6), and finally the ochre-pull.json manifest. It never
 // touches the weights KV store; 'stage' consumes the printed s3:// URI
 // separately. A failure removes every object already written (D5).
-func runPullWeights(ctx context.Context, hf *hfhub.Client, store objectstore.ObjectStore, modelID, hfRepo, revision, s3URIFlag string) (string, error) {
-	if _, found, selfHost := gateway_bedrock.LookupServingSpec(modelID); !found || !selfHost {
-		if !found {
-			return "", fmt.Errorf("unknown model ID %q: not present in the Ochre catalog", modelID)
-		}
-		return "", fmt.Errorf("%q is a provider-served model, not self-host; weights pull does not apply", modelID)
+func runPullWeights(ctx context.Context, hf *hfhub.Client, store objectstore.ObjectStore, modelID, hfRepo, revision string, revisionExplicit bool, s3URIFlag string) (string, error) {
+	spec, found, selfHost := gateway_bedrock.LookupServingSpec(modelID)
+	if !found || !selfHost {
+		return "", weightsStagingRefusal(modelID, "pull", found, selfHost)
+	}
+
+	// The catalog carries the canonical repo and a pinned revision so a bare
+	// pull cannot silently grab a regressed upstream default; an explicit flag
+	// still overrides either.
+	if hfRepo == "" {
+		hfRepo = spec.HFRepo
+	}
+	if hfRepo == "" {
+		return "", fmt.Errorf("no --hf-repo given and %q has no catalog HF repo", modelID)
+	}
+	if !revisionExplicit && spec.HFRevision != "" {
+		revision = spec.HFRevision
 	}
 
 	fmt.Printf("Resolving %s@%s ...\n", hfRepo, revision)
@@ -594,12 +633,12 @@ func buildWeightsImage(srcDir, imagePath string, contentBytes int64) error {
 // metadata.json sitting alongside them, so without this the endpoint launcher
 // cannot see the snapshot at all and fails with InvalidSnapshot.NotFound.
 func registerWeightsSnapshot(store objectstore.ObjectStore, bucket, snapshotID, volumeID string, volumeSize uint64, az string, encrypted bool) error {
-	cfg := &handlers_ec2_snapshot.SnapshotConfig{
+	cfg := ebsmetadata.Snapshot{
 		SnapshotID: snapshotID,
 		VolumeID:   volumeID,
 		// GiB, not bytes: CreateVolume compares this against a requested Size
 		// already in GiB, and would reject every clone if handed raw bytes.
-		VolumeSize:       utils.SafeUint64ToInt64(volumeSize / bytesPerGiB),
+		VolumeSize:       safecast.Uint64ToInt64(volumeSize / bytesPerGiB),
 		State:            "completed",
 		Progress:         "100%",
 		StartTime:        time.Now(),
@@ -608,7 +647,7 @@ func registerWeightsSnapshot(store objectstore.ObjectStore, bucket, snapshotID,
 		OwnerID:          utils.GlobalAccountID,
 		AvailabilityZone: az,
 	}
-	if err := handlers_ec2_snapshot.WriteSnapshotConfig(store, bucket, snapshotID, cfg); err != nil {
+	if err := ebsmetadata.NewStore(store, bucket).PutSnapshot(context.Background(), cfg); err != nil {
 		return fmt.Errorf("register snapshot metadata: %w", err)
 	}
 	return nil
@@ -636,10 +675,7 @@ type weightsSnapshotChecker func(ctx context.Context, snapshotID string) (bool,
 // fake materializer.
 func runStageWeights(ctx context.Context, store objectstore.ObjectStore, weightsStore *gateway_bedrock.WeightsStore, tmpDirFlag, modelID, s3URI string, materialize weightsMaterializer, checkSnapshotLive weightsSnapshotChecker) (string, error) {
 	if _, found, selfHost := gateway_bedrock.LookupServingSpec(modelID); !found || !selfHost {
-		if !found {
-			return "", fmt.Errorf("unknown model ID %q: not present in the Ochre catalog", modelID)
-		}
-		return "", fmt.Errorf("%q is a provider-served model, not self-host; weights staging does not apply", modelID)
+		return "", weightsStagingRefusal(modelID, "staging", found, selfHost)
 	}
 
 	bucket, prefix, err := parseWeightsS3URI(s3URI)
@@ -749,7 +785,7 @@ func materializeWeightsVolume(ctx context.Context, provider ebsprovider.EBSProvi
 	if err := admin.ImportImage(ctx, provider, admin.ImportOpts{
 		VolumeID:         volumeId,
 		NodeID:           nodeID,
-		SizeBytes:        utils.SafeUint64ToInt64(volumeBytes),
+		SizeBytes:        safecast.Uint64ToInt64(volumeBytes),
 		AvailabilityZone: node.AZ,
 		SourcePath:       imagePath,
 		Snapshot:         true,
@@ -769,13 +805,14 @@ func materializeWeightsVolume(ctx context.Context, provider ebsprovider.EBSProvi
 }
 
 // newWeightsSnapshotChecker is the real weightsSnapshotChecker: it reads the
-// EC2 control plane's metadata.json for snapshotID from the node's
-// predastore bucket -- the same record registerWeightsSnapshot writes at
-// stage time -- and treats a missing object as gone. Any other read failure
-// is a real error, not a signal to guess the snapshot's existence.
+// EC2 control plane's snapshot document for snapshotID under the global
+// account -- the same record registerWeightsSnapshot writes at stage time --
+// and treats a missing object as gone. Any other read failure is a real
+// error, not a signal to guess the snapshot's existence.
 func newWeightsSnapshotChecker(store objectstore.ObjectStore, bucket string) weightsSnapshotChecker {
+	metadata := ebsmetadata.NewStore(store, bucket)
 	return func(ctx context.Context, snapshotID string) (bool, error) {
-		if _, err := handlers_ec2_snapshot.ReadSnapshotConfig(ctx, store, bucket, snapshotID); err != nil {
+		if _, err := metadata.GetSnapshot(ctx, utils.GlobalAccountID, snapshotID); err != nil {
 			if objectstore.IsNoSuchKeyError(err) {
 				return false, nil
 			}
@@ -969,7 +1006,7 @@ func resolveHFToken(ctx context.Context, cmd *cobra.Command, cfg *config.Cluster
 	}
 
 	masterKeyPath := filepath.Join(cfg.NodeBaseDir(), "config", "master.key")
-	masterKey, err := handlers_iam.LoadMasterKey(masterKeyPath)
+	masterKey, err := masterkey.ReadShared(masterKeyPath)
 	if err != nil {
 		return ""
 	}
@@ -989,6 +1026,7 @@ func runOchreWeightsPull(cmd *cobra.Command, _ []string) {
 	modelID, _ := cmd.Flags().GetString("model-id")
 	hfRepo, _ := cmd.Flags().GetString("hf-repo")
 	revision, _ := cmd.Flags().GetString("revision")
+	revisionExplicit := cmd.Flags().Changed("revision")
 	s3URI, _ := cmd.Flags().GetString("s3-uri")
 
 	appConfig, nc, err := loadConfigAndConnectFn()
@@ -1006,13 +1044,13 @@ func runOchreWeightsPull(cmd *cobra.Command, _ []string) {
 	hf := hfhub.NewClient(token)
 	store := objectstore.NewS3ObjectStoreFromConfig(node.Predastore.Host, node.Predastore.Region, node.Predastore.AccessKey, node.Predastore.SecretKey)
 
-	finalURI, err := runPullWeights(ctx, hf, store, modelID, hfRepo, revision, s3URI)
+	finalURI, err := runPullWeights(ctx, hf, store, modelID, hfRepo, revision, revisionExplicit, s3URI)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 		ochreExit(1)
 		return
 	}
-	fmt.Printf("✅ Pulled %s@%s into %s\n", hfRepo, revision, finalURI)
+	fmt.Printf("✅ Pulled into %s\n", finalURI)
 	fmt.Println(finalURI)
 }
 
@@ -1040,7 +1078,7 @@ func ochreCredentialsStore() (*gateway_bedrock.CredentialStore, func(), error) {
 		return nil, nil, fmt.Errorf("connect to cluster: %w", err)
 	}
 	masterKeyPath := filepath.Join(cfg.NodeBaseDir(), "config", "master.key")
-	masterKey, err := handlers_iam.LoadMasterKey(masterKeyPath)
+	masterKey, err := masterkey.ReadShared(masterKeyPath)
 	if err != nil {
 		nc.Close()
 		return nil, nil, fmt.Errorf("load master key: %w", err)
@@ -1138,15 +1176,11 @@ func init() {
 		c.Flags().String("account-id", "", "12-digit account ID to change (required)")
 		c.Flags().String("model-id", "", "Model ID to change (e.g. meta.llama3-2-1b-instruct-v1:0)")
 		c.Flags().Bool("all-models", false, "Apply to every model in the platform catalog")
-		if err := c.MarkFlagRequired("account-id"); err != nil {
-			panic(err)
-		}
+		_ = c.MarkFlagRequired("account-id")
 	}
 
 	adminOchreAccessListCmd.Flags().String("account-id", "", "12-digit account ID to inspect (required)")
-	if err := adminOchreAccessListCmd.MarkFlagRequired("account-id"); err != nil {
-		panic(err)
-	}
+	_ = adminOchreAccessListCmd.MarkFlagRequired("account-id")
 }
 
 // ochreAccessStore connects to the cluster and returns the grant store along

cmd/spinifex/cmd/admin_ochre_appliance.go

@@ -29,10 +29,13 @@ var ochreApplianceTeardownCmd = &cobra.Command{
 instance, this node's VPC host port, and its KV record.
 
 This is a destructive, cluster-wide operator action -- every account's vector
-index data stored in the appliance is lost with it, and it is not
-automatically rebuilt. A future daemon startup re-provisions a fresh
-appliance (a new instance, a new password) only if Ochre remains enabled;
-existing index registry records are not restored.
+index data stored in the appliance is lost with it. By default the index
+registry and KB/DataSource metadata survive, exactly as they survive a
+restart: a future daemon startup re-provisions a fresh appliance (a new
+instance, a new password) and reconciles each surviving index by re-creating
+it and re-ingesting from its recorded source, with no operator action and no
+change to its id. --purge-metadata additionally wipes that metadata for an
+intentional full destroy, after which nothing is auto-restored.
 
 Requires --confirm.`,
 	Run: runOchreApplianceTeardown,
@@ -44,40 +47,47 @@ func init() {
 
 	ochreApplianceTeardownCmd.Flags().Bool("confirm", false,
 		"Required: confirms you intend to destroy the shared platform appliance and every account's vector index data with it")
+	ochreApplianceTeardownCmd.Flags().Bool("purge-metadata", false,
+		"Also wipe the index registry and KB/DataSource records, disabling auto-restore on the next re-provision")
 }
 
 // applianceTeardownFn indirects the NATS call so runApplianceTeardownGuarded
 // is testable without a live daemon, mirroring vectorServiceFn/
 // endpointServiceFn.
-var applianceTeardownFn = func(ctx context.Context) error {
+var applianceTeardownFn = func(ctx context.Context, purgeMetadata bool) error {
 	_, nc, err := loadConfigAndConnectFn()
 	if err != nil {
 		return err
 	}
 	defer nc.Close()
 	_, err = utils.NATSRequest[handlers_ochrevector.TeardownApplianceResponse](ctx, nc,
-		handlers_ochrevector.SubjectTeardownAppliance, &handlers_ochrevector.TeardownApplianceRequest{}, applianceTeardownTimeout, utils.GlobalAccountID)
+		handlers_ochrevector.SubjectTeardownAppliance, &handlers_ochrevector.TeardownApplianceRequest{PurgeMetadata: purgeMetadata},
+		applianceTeardownTimeout, utils.GlobalAccountID)
 	return err
 }
 
 // runApplianceTeardownGuarded is the testable core of 'ochre appliance
 // teardown': it refuses without --confirm before making any NATS call, so a
 // missing flag never even opens a connection to the cluster.
-func runApplianceTeardownGuarded(ctx context.Context, confirmed bool, teardown func(context.Context) error) (string, error) {
+func runApplianceTeardownGuarded(ctx context.Context, confirmed, purgeMetadata bool, teardown func(context.Context, bool) error) (string, error) {
 	if !confirmed {
 		return "", errors.New("refusing to tear down the shared platform appliance without --confirm: " +
 			"this destroys every account's vector index data and is not automatically rebuilt")
 	}
-	if err := teardown(ctx); err != nil {
+	if err := teardown(ctx, purgeMetadata); err != nil {
 		return "", err
 	}
+	if purgeMetadata {
+		return "✅ Platform appliance torn down, index/KB metadata purged.", nil
+	}
 	return "✅ Platform appliance torn down.", nil
 }
 
 func runOchreApplianceTeardown(cmd *cobra.Command, _ []string) {
 	confirmed, _ := cmd.Flags().GetBool("confirm")
+	purgeMetadata, _ := cmd.Flags().GetBool("purge-metadata")
 
-	msg, err := runApplianceTeardownGuarded(context.Background(), confirmed, applianceTeardownFn)
+	msg, err := runApplianceTeardownGuarded(context.Background(), confirmed, purgeMetadata, applianceTeardownFn)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 		ochreExit(1)

cmd/spinifex/cmd/admin_ochre_appliance_test.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"testing"
 
+	"github.com/mulgadc/spinifex/spinifex/testutil"
 	"github.com/stretchr/testify/require"
 )
 
@@ -17,12 +18,12 @@ import (
 // reach a NATS call, let alone destroy the appliance.
 func TestRunApplianceTeardownGuarded_RefusesWithoutConfirm(t *testing.T) {
 	called := false
-	teardown := func(context.Context) error {
+	teardown := func(context.Context, bool) error {
 		called = true
 		return nil
 	}
 
-	_, err := runApplianceTeardownGuarded(context.Background(), false, teardown)
+	_, err := runApplianceTeardownGuarded(context.Background(), false, false, teardown)
 	require.Error(t, err)
 	require.ErrorContains(t, err, "--confirm")
 	require.False(t, called, "teardown must not run without --confirm")
@@ -32,12 +33,12 @@ func TestRunApplianceTeardownGuarded_RefusesWithoutConfirm(t *testing.T) {
 // drives the teardown call through and reports success.
 func TestRunApplianceTeardownGuarded_ConfirmedInvokesTeardown(t *testing.T) {
 	called := false
-	teardown := func(context.Context) error {
+	teardown := func(context.Context, bool) error {
 		called = true
 		return nil
 	}
 
-	msg, err := runApplianceTeardownGuarded(context.Background(), true, teardown)
+	msg, err := runApplianceTeardownGuarded(context.Background(), true, false, teardown)
 	require.NoError(t, err)
 	require.True(t, called)
 	require.Contains(t, msg, "torn down")
@@ -47,19 +48,49 @@ func TestRunApplianceTeardownGuarded_ConfirmedInvokesTeardown(t *testing.T) {
 // failure (e.g. the daemon reporting no appliance up) surfaces to the caller
 // rather than being swallowed behind a generic success message.
 func TestRunApplianceTeardownGuarded_TeardownErrorSurfaces(t *testing.T) {
-	teardown := func(context.Context) error {
+	teardown := func(context.Context, bool) error {
 		return errors.New("ochrevector: platform appliance is not enabled or not up on this node")
 	}
 
-	_, err := runApplianceTeardownGuarded(context.Background(), true, teardown)
+	_, err := runApplianceTeardownGuarded(context.Background(), true, false, teardown)
 	require.Error(t, err)
 	require.ErrorContains(t, err, "not enabled or not up")
 }
 
+// TestRunApplianceTeardownGuarded_PurgeMetadataThreadsThrough proves the
+// --purge-metadata flag reaches the teardown call and is reflected in the
+// success message, so an operator can tell which mode ran.
+func TestRunApplianceTeardownGuarded_PurgeMetadataThreadsThrough(t *testing.T) {
+	var gotPurge bool
+	teardown := func(_ context.Context, purgeMetadata bool) error {
+		gotPurge = purgeMetadata
+		return nil
+	}
+
+	msg, err := runApplianceTeardownGuarded(context.Background(), true, true, teardown)
+	require.NoError(t, err)
+	require.True(t, gotPurge, "purgeMetadata must reach the teardown call")
+	require.Contains(t, msg, "metadata purged")
+}
+
+// TestApplianceTeardownFn_ThreadsPurgeMetadataOverNATS drives the real
+// (unstubbed) applianceTeardownFn through runApplianceTeardownGuarded, so the
+// actual NATSRequest call -- including the PurgeMetadata field on the wire
+// request -- executes rather than a test double standing in for it. No
+// daemon is subscribed to the teardown subject in this test, so the call
+// fails fast; the point is that it's reached and built with PurgeMetadata set.
+func TestApplianceTeardownFn_ThreadsPurgeMetadataOverNATS(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	stubConnect(t, fakeClusterConfig(), nc, nil)
+
+	_, err := runApplianceTeardownGuarded(context.Background(), true, true, applianceTeardownFn)
+	require.Error(t, err, "no daemon is subscribed to the teardown subject in this test")
+}
+
 // withApplianceTeardown swaps applianceTeardownFn for one that never dials
 // NATS, so the Run wrapper's real flag/exit control flow is exercised
 // without a live daemon.
-func withApplianceTeardown(t *testing.T, fn func(context.Context) error) {
+func withApplianceTeardown(t *testing.T, fn func(context.Context, bool) error) {
 	t.Helper()
 	orig := applianceTeardownFn
 	t.Cleanup(func() { applianceTeardownFn = orig })
@@ -67,7 +98,7 @@ func withApplianceTeardown(t *testing.T, fn func(context.Context) error) {
 }
 
 func TestRunOchreApplianceTeardown_MissingConfirmExits1(t *testing.T) {
-	withApplianceTeardown(t, func(context.Context) error {
+	withApplianceTeardown(t, func(context.Context, bool) error {
 		t.Fatal("teardown must not be called without --confirm")
 		return nil
 	})
@@ -78,7 +109,7 @@ func TestRunOchreApplianceTeardown_MissingConfirmExits1(t *testing.T) {
 }
 
 func TestRunOchreApplianceTeardown_ConfirmedPrintsSuccess(t *testing.T) {
-	withApplianceTeardown(t, func(context.Context) error { return nil })
+	withApplianceTeardown(t, func(context.Context, bool) error { return nil })
 
 	cmd := *ochreApplianceTeardownCmd
 	require.NoError(t, cmd.Flags().Set("confirm", "true"))
@@ -92,7 +123,7 @@ func TestRunOchreApplianceTeardown_ConfirmedPrintsSuccess(t *testing.T) {
 }
 
 func TestRunOchreApplianceTeardown_TeardownErrorExits1(t *testing.T) {
-	withApplianceTeardown(t, func(context.Context) error {
+	withApplianceTeardown(t, func(context.Context, bool) error {
 		return errors.New("ochrevector: platform appliance is not enabled or not up on this node")
 	})
 
@@ -102,3 +133,21 @@ func TestRunOchreApplianceTeardown_TeardownErrorExits1(t *testing.T) {
 	code := withOchreExitCapture(t, func() { runOchreApplianceTeardown(&cmd, nil) })
 	require.Equal(t, 1, code)
 }
+
+// TestRunOchreApplianceTeardown_PurgeMetadataFlagReachesTeardownFn proves
+// the --purge-metadata flag is read and passed through the Run wrapper.
+func TestRunOchreApplianceTeardown_PurgeMetadataFlagReachesTeardownFn(t *testing.T) {
+	var gotPurge bool
+	withApplianceTeardown(t, func(_ context.Context, purgeMetadata bool) error {
+		gotPurge = purgeMetadata
+		return nil
+	})
+
+	cmd := *ochreApplianceTeardownCmd
+	require.NoError(t, cmd.Flags().Set("confirm", "true"))
+	require.NoError(t, cmd.Flags().Set("purge-metadata", "true"))
+
+	code := withOchreExitCapture(t, func() { runOchreApplianceTeardown(&cmd, nil) })
+	require.Equal(t, -1, code)
+	require.True(t, gotPurge, "--purge-metadata must reach applianceTeardownFn")
+}

cmd/spinifex/cmd/admin_ochre_endpoint.go

@@ -79,6 +79,7 @@ func init() {
 	_ = ochreEndpointDescribeCmd.MarkFlagRequired("model-id")
 
 	ochreEndpointDeleteCmd.Flags().String("model-id", "", "Model ID whose endpoint to tear down (required)")
+	ochreEndpointDeleteCmd.Flags().String("account", "", "Account ID scoping the target (defaults to the shared platform account; set to tear down a pinned provisioned-throughput endpoint)")
 	_ = ochreEndpointDeleteCmd.MarkFlagRequired("model-id")
 }
 
@@ -331,8 +332,35 @@ func runOchreEndpointList(_ *cobra.Command, _ []string) {
 	fmt.Println(msg)
 }
 
+// deleteEndpointOutput is the testable core of 'ochre endpoint delete': it
+// tears the endpoint down and returns an honest message — a no-op that found
+// no record must not claim a teardown, and it points the operator at --account
+// so a pinned, account-scoped record they can see in 'list' is reachable.
+func deleteEndpointOutput(ctx context.Context, svc handlers_bedrock.EndpointService, modelID, accountID string) (string, error) {
+	out, err := svc.Delete(ctx, &handlers_bedrock.DeleteEndpointInput{ModelID: modelID, AccountID: accountID}, utils.GlobalAccountID)
+	if err != nil {
+		return "", err
+	}
+	if !out.Removed {
+		return fmt.Sprintf("No serving endpoint for %s under account %s; nothing to tear down. "+
+			"If 'endpoint list' shows one under a different account, pass --account.\n",
+			modelID, resolveAccountLabel(accountID)), nil
+	}
+	return fmt.Sprintf("Endpoint for %s torn down.\n", modelID), nil
+}
+
+// resolveAccountLabel renders the account an empty flag resolves to, so the
+// no-op message names the shared platform account rather than an empty string.
+func resolveAccountLabel(accountID string) string {
+	if accountID == "" {
+		return utils.GlobalAccountID + " (shared platform)"
+	}
+	return accountID
+}
+
 func runOchreEndpointDelete(cmd *cobra.Command, _ []string) {
 	modelID, _ := cmd.Flags().GetString("model-id")
+	accountID, _ := cmd.Flags().GetString("account")
 
 	svc, closeFn, err := endpointServiceFn()
 	if err != nil {
@@ -342,10 +370,11 @@ func runOchreEndpointDelete(cmd *cobra.Command, _ []string) {
 	}
 	defer closeFn()
 
-	if _, err := svc.Delete(context.Background(), &handlers_bedrock.DeleteEndpointInput{ModelID: modelID}, utils.GlobalAccountID); err != nil {
+	msg, err := deleteEndpointOutput(context.Background(), svc, modelID, accountID)
+	if err != nil {
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 		ochreExit(1)
 		return
 	}
-	fmt.Printf("Endpoint for %s torn down.\n", modelID)
+	fmt.Print(msg)
 }

cmd/spinifex/cmd/admin_ochre_endpoint_test.go

@@ -14,17 +14,19 @@ import (
 // fakeEndpointService serves scripted Describe responses so the wait loop can
 // be exercised without a daemon, a VM or a real cold start.
 type fakeEndpointService struct {
-	ensure      handlers_bedrock.EndpointRecord
-	ensureErr   error
-	describes   []handlers_bedrock.EndpointRecord
-	describeErr error
-	list        []handlers_bedrock.EndpointRecord
-	listErr     error
-	deleteErr   error
+	ensure        handlers_bedrock.EndpointRecord
+	ensureErr     error
+	describes     []handlers_bedrock.EndpointRecord
+	describeErr   error
+	list          []handlers_bedrock.EndpointRecord
+	listErr       error
+	deleteErr     error
+	deleteRemoved bool
 
 	describeCalls  int
 	deleteCalls    int
 	describeInputs []*handlers_bedrock.DescribeEndpointInput
+	deleteInputs   []*handlers_bedrock.DeleteEndpointInput
 }
 
 func (f *fakeEndpointService) Ensure(_ context.Context, _ *handlers_bedrock.EnsureEndpointInput, _ string) (*handlers_bedrock.EnsureEndpointOutput, error) {
@@ -53,12 +55,13 @@ func (f *fakeEndpointService) List(_ context.Context, _ *handlers_bedrock.ListEn
 	return &handlers_bedrock.ListEndpointsOutput{Endpoints: f.list}, nil
 }
 
-func (f *fakeEndpointService) Delete(_ context.Context, _ *handlers_bedrock.DeleteEndpointInput, _ string) (*handlers_bedrock.DeleteEndpointOutput, error) {
+func (f *fakeEndpointService) Delete(_ context.Context, in *handlers_bedrock.DeleteEndpointInput, _ string) (*handlers_bedrock.DeleteEndpointOutput, error) {
 	f.deleteCalls++
+	f.deleteInputs = append(f.deleteInputs, in)
 	if f.deleteErr != nil {
 		return nil, f.deleteErr
 	}
-	return &handlers_bedrock.DeleteEndpointOutput{}, nil
+	return &handlers_bedrock.DeleteEndpointOutput{Removed: f.deleteRemoved}, nil
 }
 
 var _ handlers_bedrock.EndpointService = (*fakeEndpointService)(nil)
@@ -409,7 +412,7 @@ func TestRunOchreEndpointList_ConnectFailureExits1(t *testing.T) {
 }
 
 func TestRunOchreEndpointDelete_ReportsTeardown(t *testing.T) {
-	svc := &fakeEndpointService{}
+	svc := &fakeEndpointService{deleteRemoved: true}
 	withEndpointService(t, svc, nil)
 
 	cmd := *ochreEndpointDeleteCmd
@@ -424,6 +427,45 @@ func TestRunOchreEndpointDelete_ReportsTeardown(t *testing.T) {
 	require.Contains(t, out, "torn down")
 }
 
+// TestRunOchreEndpointDelete_NoRecordDoesNotClaimTeardown is 6z9vg's honesty
+// guard: a delete that removed nothing must not print "torn down", and must
+// point the operator at --account so a pinned, account-scoped record they can
+// see in 'list' is reachable.
+func TestRunOchreEndpointDelete_NoRecordDoesNotClaimTeardown(t *testing.T) {
+	svc := &fakeEndpointService{deleteRemoved: false}
+	withEndpointService(t, svc, nil)
+
+	cmd := *ochreEndpointDeleteCmd
+	require.NoError(t, cmd.Flags().Set("model-id", testModelID))
+
+	var out string
+	code := withOchreExitCapture(t, func() {
+		out = captureStdout(t, func() { runOchreEndpointDelete(&cmd, nil) })
+	})
+	require.Equal(t, -1, code)
+	require.NotContains(t, out, "torn down")
+	require.Contains(t, out, "--account")
+}
+
+// TestRunOchreEndpointDelete_AccountFlagReachesInput is 6z9vg's core seam: the
+// --account flag must reach DeleteEndpointInput.AccountID, which is how an
+// operator tears down a pinned endpoint the bare GlobalAccountID key misses.
+func TestRunOchreEndpointDelete_AccountFlagReachesInput(t *testing.T) {
+	svc := &fakeEndpointService{deleteRemoved: true}
+	withEndpointService(t, svc, nil)
+
+	cmd := *ochreEndpointDeleteCmd
+	require.NoError(t, cmd.Flags().Set("model-id", testModelID))
+	require.NoError(t, cmd.Flags().Set("account", testAccountID))
+
+	code := withOchreExitCapture(t, func() {
+		_ = captureStdout(t, func() { runOchreEndpointDelete(&cmd, nil) })
+	})
+	require.Equal(t, -1, code)
+	require.Len(t, svc.deleteInputs, 1)
+	require.Equal(t, testAccountID, svc.deleteInputs[0].AccountID)
+}
+
 func TestRunOchreEndpointDelete_ErrorExits1(t *testing.T) {
 	withEndpointService(t, &fakeEndpointService{deleteErr: errors.New("ModelNotReadyException")}, nil)
 

cmd/spinifex/cmd/admin_ochre_pull_test.go

@@ -32,10 +32,11 @@ import (
 // --- selectPullFiles / anySafetensors (D4) ---
 
 // TestSelectPullFiles_SafetensorsAndConfigOnly covers D4's filter: every
-// *.safetensors file, the fixed config/tokenizer set, and the shard index are
-// kept (matched by basename, so a nested path like Llama's
-// original/tokenizer.model still qualifies); a pickle .bin checkpoint and
-// directory entries are dropped.
+// *.safetensors file, the fixed config set (root only), tokenizer files (at
+// any depth, so Llama's original/tokenizer.model qualifies), and the shard
+// index are kept; a pickle .bin, a directory entry, and a nested config.json
+// (a sentence-transformers component config that must never flatten onto the
+// model's own config.json) are dropped.
 func TestSelectPullFiles_SafetensorsAndConfigOnly(t *testing.T) {
 	entries := []hfhub.TreeEntry{
 		{Type: "file", Path: "config.json"},
@@ -47,6 +48,7 @@ func TestSelectPullFiles_SafetensorsAndConfigOnly(t *testing.T) {
 		{Type: "file", Path: "model.safetensors.index.json"},
 		{Type: "directory", Path: "original"},
 		{Type: "file", Path: "original/tokenizer.model"},
+		{Type: "file", Path: "1_Pooling/config.json"},
 	}
 
 	selected := selectPullFiles(entries)
@@ -64,6 +66,7 @@ func TestSelectPullFiles_SafetensorsAndConfigOnly(t *testing.T) {
 		"model.safetensors.index.json",
 		"original/tokenizer.model",
 	}, paths)
+	assert.NotContains(t, paths, "1_Pooling/config.json", "a nested config.json must be dropped, never flattened onto the model config")
 }
 
 // TestAnySafetensors_TrueWhenPresent covers the pass case: at least one
@@ -227,15 +230,17 @@ const (
 // TestRunPullWeights_UnknownModelRefused covers the catalog gate: an
 // unrecognised model ID must refuse before any Hugging Face or S3 work.
 func TestRunPullWeights_UnknownModelRefused(t *testing.T) {
-	_, err := runPullWeights(context.Background(), deadHFClient(), explodingObjectStore{t: t}, "not-a-real-model", testHFRepo, "main", "")
+	_, err := runPullWeights(context.Background(), deadHFClient(), explodingObjectStore{t: t}, "not-a-real-model", testHFRepo, "main", true, "")
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "not present in the Ochre catalog")
 }
 
-// TestRunPullWeights_ProviderModelRefused covers the self-host gate: a
-// provider-served model must refuse before any network work, same as stage.
-func TestRunPullWeights_ProviderModelRefused(t *testing.T) {
-	_, err := runPullWeights(context.Background(), deadHFClient(), explodingObjectStore{t: t}, providerModelID, testHFRepo, "main", "")
+// TestWeightsStagingRefusal_PullOperationWording covers the self-host gate's
+// message for the 'pull' operation. v1 ships no provider-tier catalog entry,
+// so this drives weightsStagingRefusal directly rather than through a real
+// LookupServingSpec lookup.
+func TestWeightsStagingRefusal_PullOperationWording(t *testing.T) {
+	err := weightsStagingRefusal(providerModelID, "pull", true, false)
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "not self-host")
 }
@@ -256,7 +261,7 @@ func TestRunPullWeights_ResolvesUploadsAndWritesManifest(t *testing.T) {
 	hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
 	store := objectstore.NewMemoryObjectStore()
 
-	finalURI, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", "")
+	finalURI, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", true, "")
 	require.NoError(t, err)
 	assert.Equal(t, defaultPullPrefix(testHFRepo, testHFSHA), finalURI)
 
@@ -287,7 +292,7 @@ func TestRunPullWeights_ExplicitS3URIOverridesDefault(t *testing.T) {
 	hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
 	store := objectstore.NewMemoryObjectStore()
 
-	finalURI, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", "s3://custom-bucket/my-prefix")
+	finalURI, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", true, "s3://custom-bucket/my-prefix")
 	require.NoError(t, err)
 	assert.Equal(t, "s3://custom-bucket/my-prefix/", finalURI)
 }
@@ -300,7 +305,7 @@ func TestRunPullWeights_NoSafetensorsAborts(t *testing.T) {
 	hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
 	store := objectstore.NewMemoryObjectStore()
 
-	_, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", "")
+	_, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", true, "")
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "no *.safetensors")
 	assert.Equal(t, 0, store.Count(), "a refused pull must land nothing")
@@ -315,7 +320,7 @@ func TestRunPullWeights_GatedRepoAbortsCleanBeforeAnyUpload(t *testing.T) {
 		hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
 		store := objectstore.NewMemoryObjectStore()
 
-		_, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", "")
+		_, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", true, "")
 		require.Error(t, err)
 		assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorAccessDeniedException), "status %d: got %v", status, err)
 		assert.Equal(t, 0, store.Count(), "a gated pull must land nothing")
@@ -356,11 +361,57 @@ func TestRunPullWeights_MidStreamFailureCleansUpPartialPrefix(t *testing.T) {
 	hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
 	store := &failAfterNPutsStore{MemoryObjectStore: objectstore.NewMemoryObjectStore(), n: 2}
 
-	_, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", "")
+	_, err := runPullWeights(context.Background(), hf, store, selfHostModelID, testHFRepo, "main", true, "")
 	require.Error(t, err)
 	assert.Equal(t, 0, store.Count(), "objects uploaded before the failure must be cleaned up")
 }
 
+// TestRunPullWeights_DefaultsRepoFromCatalog covers the catalog repo default:
+// with no --hf-repo, pull uses the self-host entry's canonical HFRepo rather
+// than refusing, so an operator need not restate a repo the catalog knows.
+func TestRunPullWeights_DefaultsRepoFromCatalog(t *testing.T) {
+	files := []fakeHFFile{{"config.json", "{}"}, {"tokenizer_config.json", "{}"}, {"tokenizer.json", "{}"}, {"model.safetensors", "x"}}
+	srv := newFakeHFServer(t, testHFRepo, "main", testHFSHA, files)
+	hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
+	store := objectstore.NewMemoryObjectStore()
+
+	finalURI, err := runPullWeights(context.Background(), hf, store, selfHostModelID, "", "main", false, "")
+	require.NoError(t, err)
+	assert.Equal(t, defaultPullPrefix(testHFRepo, testHFSHA), finalURI)
+}
+
+// TestRunPullWeights_DefaultsRevisionFromCatalogPin covers the pinned-revision
+// default: a bare pull of an entry carrying HFRevision resolves that pin, not
+// the "main" flag default, so a regressed upstream main cannot be picked up.
+func TestRunPullWeights_DefaultsRevisionFromCatalogPin(t *testing.T) {
+	const nomicModelID = "nomic-embed-text-v1.5"
+	const nomicRepo = "nomic-ai/nomic-embed-text-v1.5"
+	const pinnedRevision = "e5cf08a"
+	files := []fakeHFFile{{"config.json", "{}"}, {"tokenizer_config.json", "{}"}, {"tokenizer.json", "{}"}, {"model.safetensors", "x"}}
+	srv := newFakeHFServer(t, nomicRepo, pinnedRevision, testHFSHA, files)
+	hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
+	store := objectstore.NewMemoryObjectStore()
+
+	finalURI, err := runPullWeights(context.Background(), hf, store, nomicModelID, "", "main", false, "")
+	require.NoError(t, err)
+	assert.Equal(t, defaultPullPrefix(nomicRepo, testHFSHA), finalURI)
+}
+
+// TestPullFilesToPrefix_RefusesDuplicateFlattenedKey covers the flatten guard:
+// two source paths sharing a basename would collide at one destination key, so
+// the pull aborts rather than silently overwriting one with the other.
+func TestPullFilesToPrefix_RefusesDuplicateFlattenedKey(t *testing.T) {
+	files := []fakeHFFile{{"a/tokenizer.model", "first"}, {"b/tokenizer.model", "second"}}
+	srv := newFakeHFServer(t, testHFRepo, "main", testHFSHA, files)
+	hf := &hfhub.Client{BaseURL: srv.URL, HTTP: srv.Client()}
+	store := objectstore.NewMemoryObjectStore()
+
+	entries := []hfhub.TreeEntry{{Type: "file", Path: "a/tokenizer.model"}, {Type: "file", Path: "b/tokenizer.model"}}
+	_, err := pullFilesToPrefix(context.Background(), hf, store, testHFRepo, testHFSHA, "ochre-weights", "prefix/", entries)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "flatten to it")
+}
+
 // --- resolveHFToken (D2) ---
 
 func newOchreWeightsPullTestCmd(t *testing.T, modelID, hfRepo, revision, s3URI, hfToken string) *cobra.Command {

cmd/spinifex/cmd/admin_ochre_test.go

@@ -15,20 +15,22 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/mulgadc/spinifex/spinifex/config"
+	"github.com/mulgadc/spinifex/spinifex/ebsmetadata"
 	gateway_bedrock "github.com/mulgadc/spinifex/spinifex/gateway/bedrock"
-	handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/testutil"
+	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/nats-io/nats.go"
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-// selfHostModelID and providerModelID are real catalog entries: the former
-// is self-host (weights staging applies), the latter is provider-served
-// (weights staging must refuse it), so tests exercise LookupServingSpec's
-// found/selfHost distinction against the real catalog rather than a stub.
+// selfHostModelID is a real catalog entry (weights staging applies).
+// providerModelID is an arbitrary ID not present in the catalog at all: v1
+// ships no provider-tier entry, so the found-but-not-selfHost case is
+// exercised directly against weightsStagingRefusal instead, not through a
+// real LookupServingSpec lookup.
 const (
 	selfHostModelID = "meta.llama3-2-1b-instruct-v1:0"
 	providerModelID = "anthropic.claude-3-5-sonnet-20240620-v1:0"
@@ -404,24 +406,17 @@ func TestRunStageWeights_UnknownModelIDRefused(t *testing.T) {
 	assert.Empty(t, entries, "refusal must not write a KV entry")
 }
 
-// TestRunStageWeights_ProviderModelIDRefused covers refusal on a model ID
-// that exists in the catalog but is provider-served, not self-host. The
-// error must be distinguishable from the unknown-model-ID case, since
-// LookupServingSpec returns found and selfHost separately for exactly this
-// reason.
-func TestRunStageWeights_ProviderModelIDRefused(t *testing.T) {
-	weightsStore := newWeightsStoreForTest(t)
-
-	_, err := runStageWeights(context.Background(), explodingObjectStore{t: t}, weightsStore, t.TempDir(),
-		providerModelID, "s3://models/claude/", explodingMaterializer(t), explodingSnapshotChecker(t))
+// TestWeightsStagingRefusal_ProviderServedDistinguishesFromUnknown covers
+// refusal for a model ID that is found in the catalog but is provider-served,
+// not self-host. The error must be distinguishable from the unknown-model-ID
+// case; v1 ships no provider-tier entry, so this drives weightsStagingRefusal
+// directly with explicit found/selfHost values rather than a real lookup.
+func TestWeightsStagingRefusal_ProviderServedDistinguishesFromUnknown(t *testing.T) {
+	err := weightsStagingRefusal(providerModelID, "staging", true, false)
 
 	require.Error(t, err)
 	assert.ErrorContains(t, err, "provider-served")
 	assert.NotContains(t, err.Error(), "unknown model ID", "must be distinguishable from the unknown-model-ID refusal")
-
-	entries, listErr := weightsStore.ListWeights(context.Background())
-	require.NoError(t, listErr)
-	assert.Empty(t, entries, "refusal must not write a KV entry")
 }
 
 // TestRunStageWeights_MissingRequiredFilesRefusedBeforeMaterializing covers
@@ -1100,7 +1095,7 @@ func TestRegisterWeightsSnapshot_WritesEC2ReadableMetadata(t *testing.T) {
 	require.NoError(t, registerWeightsSnapshot(store, bucket,
 		"snap-vol-abc", "vol-abc", 12*bytesPerGiB, "ap-southeast-2a", true))
 
-	cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(context.Background(), store, bucket, "snap-vol-abc")
+	cfg, err := ebsmetadata.NewStore(store, bucket).GetSnapshot(context.Background(), utils.GlobalAccountID, "snap-vol-abc")
 	require.NoError(t, err)
 	assert.Equal(t, "snap-vol-abc", cfg.SnapshotID)
 	assert.Equal(t, "vol-abc", cfg.VolumeID)
@@ -1117,7 +1112,7 @@ func TestRegisterWeightsSnapshot_UnencryptedVolume(t *testing.T) {
 	require.NoError(t, registerWeightsSnapshot(store, "predastore",
 		"snap-vol-plain", "vol-plain", bytesPerGiB, "ap-southeast-2a", false))
 
-	cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(context.Background(), store, "predastore", "snap-vol-plain")
+	cfg, err := ebsmetadata.NewStore(store, "predastore").GetSnapshot(context.Background(), utils.GlobalAccountID, "snap-vol-plain")
 	require.NoError(t, err)
 	assert.False(t, cfg.Encrypted)
 }

cmd/spinifex/cmd/admin_ochre_vector.go

@@ -96,6 +96,26 @@ truncated in the default table; pass --json for the full response.`,
 	Run: runOchreVectorQuery,
 }
 
+var ochreVectorBackupCmd = &cobra.Command{
+	Use:   "backup",
+	Short: "Back up an account's vector-store schema to predastore",
+	Long: `backup runs pg_dump against the platform appliance, scoped to --account's own
+kb_<account> schema, gzips the output, and uploads it to predastore under an
+account-scoped, timestamped object key. Backup is an explicit operator
+action -- it never runs automatically on appliance teardown.`,
+	Run: runOchreVectorBackup,
+}
+
+var ochreVectorRestoreCmd = &cobra.Command{
+	Use:   "restore",
+	Short: "Restore an account's vector-store schema from a predastore backup",
+	Long: `restore imports --from (an object key a prior backup wrote) back into
+--account's schema, replacing its existing tables cleanly, then repairs the
+account role's grants so query/ingest access resumes exactly as before the
+backup. Safe to run against an existing schema.`,
+	Run: runOchreVectorRestore,
+}
+
 func init() {
 	ochreCmd.AddCommand(ochreVectorCmd)
 	ochreVectorCmd.AddCommand(ochreVectorIndexCmd)
@@ -106,6 +126,8 @@ func init() {
 	ochreVectorCmd.AddCommand(ochreVectorJobCmd)
 	ochreVectorJobCmd.AddCommand(ochreVectorJobDescribeCmd)
 	ochreVectorCmd.AddCommand(ochreVectorQueryCmd)
+	ochreVectorCmd.AddCommand(ochreVectorBackupCmd)
+	ochreVectorCmd.AddCommand(ochreVectorRestoreCmd)
 
 	ochreVectorIndexCreateCmd.Flags().String("name", "", "Human-readable index name (required)")
 	ochreVectorIndexCreateCmd.Flags().Int("dimension", 0, "Embedding vector dimension (required)")
@@ -129,6 +151,14 @@ func init() {
 	ochreVectorQueryCmd.Flags().Bool("json", false, "Print full, untruncated results as JSON")
 	_ = ochreVectorQueryCmd.MarkFlagRequired("index")
 	_ = ochreVectorQueryCmd.MarkFlagRequired("text")
+
+	ochreVectorBackupCmd.Flags().String("account", "", "Account ID to back up (required)")
+	_ = ochreVectorBackupCmd.MarkFlagRequired("account")
+
+	ochreVectorRestoreCmd.Flags().String("account", "", "Account ID to restore into (required)")
+	ochreVectorRestoreCmd.Flags().String("from", "", "Backup object key to restore from, as printed by 'backup' (required)")
+	_ = ochreVectorRestoreCmd.MarkFlagRequired("account")
+	_ = ochreVectorRestoreCmd.MarkFlagRequired("from")
 }
 
 // vectorServiceFn indirects the NATS-backed client so the Run functions'
@@ -142,6 +172,59 @@ var vectorServiceFn = func() (handlers_ochrevector.VectorService, func(), error)
 	return handlers_ochrevector.NewNATSVectorService(nc), nc.Close, nil
 }
 
+// vectorBackupRestoreTimeout bounds the backup/restore NATS round trip: it
+// covers the appliance's own pg_dump/psql run, not just a KV read/write, so
+// it is far more generous than the other vector subjects' timeouts.
+const vectorBackupRestoreTimeout = 10 * time.Minute
+
+// backupAccountFn indirects the NATS call so runBackupAccount is testable
+// without a live daemon, mirroring vectorServiceFn/applianceTeardownFn.
+// Backup/restore are operator-only (unlike VectorService's methods): the
+// target account is named in the request payload, not derived from the
+// caller's own header identity, since the operator CLI always connects as
+// the global account but must be able to target any tenant's schema.
+var backupAccountFn = func(ctx context.Context, accountID string) (*handlers_ochrevector.BackupAccountResponse, error) {
+	_, nc, err := loadConfigAndConnectFn()
+	if err != nil {
+		return nil, err
+	}
+	defer nc.Close()
+	return utils.NATSRequest[handlers_ochrevector.BackupAccountResponse](ctx, nc,
+		handlers_ochrevector.SubjectBackupAccount, &handlers_ochrevector.BackupAccountRequest{AccountID: accountID},
+		vectorBackupRestoreTimeout, utils.GlobalAccountID)
+}
+
+// restoreAccountFn indirects the NATS call so runRestoreAccount is testable
+// without a live daemon, mirroring backupAccountFn.
+var restoreAccountFn = func(ctx context.Context, accountID, objectKey string) error {
+	_, nc, err := loadConfigAndConnectFn()
+	if err != nil {
+		return err
+	}
+	defer nc.Close()
+	_, err = utils.NATSRequest[handlers_ochrevector.RestoreAccountResponse](ctx, nc,
+		handlers_ochrevector.SubjectRestoreAccount, &handlers_ochrevector.RestoreAccountRequest{AccountID: accountID, ObjectKey: objectKey},
+		vectorBackupRestoreTimeout, utils.GlobalAccountID)
+	return err
+}
+
+// runBackupAccount is the testable core of 'ochre vector backup'.
+func runBackupAccount(ctx context.Context, accountID string, backup func(context.Context, string) (*handlers_ochrevector.BackupAccountResponse, error)) (string, error) {
+	out, err := backup(ctx, accountID)
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("✅ Backed up account %s to %s (%d bytes).", accountID, out.ObjectKey, out.SizeBytes), nil
+}
+
+// runRestoreAccount is the testable core of 'ochre vector restore'.
+func runRestoreAccount(ctx context.Context, accountID, objectKey string, restore func(context.Context, string, string) error) (string, error) {
+	if err := restore(ctx, accountID, objectKey); err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("✅ Restored account %s from %s.", accountID, objectKey), nil
+}
+
 // formatIndexRecord renders one index record as aligned key/value lines.
 func formatIndexRecord(rec handlers_ochrevector.Record) string {
 	rows := [][2]string{
@@ -448,3 +531,28 @@ func runOchreVectorQuery(cmd *cobra.Command, _ []string) {
 	}
 	fmt.Println(out)
 }
+
+func runOchreVectorBackup(cmd *cobra.Command, _ []string) {
+	accountID, _ := cmd.Flags().GetString("account")
+
+	msg, err := runBackupAccount(context.Background(), accountID, backupAccountFn)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		ochreExit(1)
+		return
+	}
+	fmt.Println(msg)
+}
+
+func runOchreVectorRestore(cmd *cobra.Command, _ []string) {
+	accountID, _ := cmd.Flags().GetString("account")
+	objectKey, _ := cmd.Flags().GetString("from")
+
+	msg, err := runRestoreAccount(context.Background(), accountID, objectKey, restoreAccountFn)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		ochreExit(1)
+		return
+	}
+	fmt.Println(msg)
+}

cmd/spinifex/cmd/admin_ochre_vector_test.go

@@ -30,6 +30,8 @@ type fakeVectorService struct {
 	queryErr       error
 	listJobs       []handlers_ochrevector.JobRecord
 	listJobsErr    error
+	stopJob        handlers_ochrevector.JobRecord
+	stopJobErr     error
 
 	createIndexInputs []*handlers_ochrevector.CreateIndexRequest
 	deleteIndexInputs []*handlers_ochrevector.DeleteIndexRequest
@@ -94,6 +96,17 @@ func (f *fakeVectorService) ListJobs(_ context.Context, _ *handlers_ochrevector.
 	return &handlers_ochrevector.ListJobsResponse{Jobs: f.listJobs}, nil
 }
 
+func (f *fakeVectorService) StopJob(_ context.Context, req *handlers_ochrevector.StopJobRequest, _ string) (*handlers_ochrevector.StopJobResponse, error) {
+	if f.stopJobErr != nil {
+		return nil, f.stopJobErr
+	}
+	job := f.stopJob
+	if job.ID == "" {
+		job.ID = req.JobID
+	}
+	return &handlers_ochrevector.StopJobResponse{Job: job}, nil
+}
+
 var _ handlers_ochrevector.VectorService = (*fakeVectorService)(nil)
 
 const testIndexID = "idx-0123456789abcdef0"
@@ -382,3 +395,123 @@ func TestRunOchreVectorQuery_ConnectFailureExits1(t *testing.T) {
 	code := withOchreExitCapture(t, func() { runOchreVectorQuery(&cmd, nil) })
 	require.Equal(t, 1, code)
 }
+
+const testBackupAccountID = "123456789012"
+
+func TestRunBackupAccount_PrintsObjectKeyAndSize(t *testing.T) {
+	backup := func(_ context.Context, accountID string) (*handlers_ochrevector.BackupAccountResponse, error) {
+		require.Equal(t, testBackupAccountID, accountID)
+		return &handlers_ochrevector.BackupAccountResponse{ObjectKey: testBackupAccountID + "/backup.sql.gz", SizeBytes: 4096}, nil
+	}
+
+	msg, err := runBackupAccount(context.Background(), testBackupAccountID, backup)
+	require.NoError(t, err)
+	require.Contains(t, msg, testBackupAccountID+"/backup.sql.gz")
+	require.Contains(t, msg, "4096")
+}
+
+func TestRunBackupAccount_ErrorSurfaces(t *testing.T) {
+	backup := func(context.Context, string) (*handlers_ochrevector.BackupAccountResponse, error) {
+		return nil, errors.New("ochrevector: appliance not available")
+	}
+
+	_, err := runBackupAccount(context.Background(), testBackupAccountID, backup)
+	require.ErrorContains(t, err, "not available")
+}
+
+func TestRunRestoreAccount_ReportsSuccess(t *testing.T) {
+	var gotAccount, gotKey string
+	restore := func(_ context.Context, accountID, objectKey string) error {
+		gotAccount, gotKey = accountID, objectKey
+		return nil
+	}
+
+	msg, err := runRestoreAccount(context.Background(), testBackupAccountID, testBackupAccountID+"/backup.sql.gz", restore)
+	require.NoError(t, err)
+	require.Equal(t, testBackupAccountID, gotAccount)
+	require.Equal(t, testBackupAccountID+"/backup.sql.gz", gotKey)
+	require.Contains(t, msg, testBackupAccountID)
+}
+
+func TestRunRestoreAccount_ErrorSurfaces(t *testing.T) {
+	restore := func(context.Context, string, string) error {
+		return errors.New("ochrevector: object key does not belong to account")
+	}
+
+	_, err := runRestoreAccount(context.Background(), testBackupAccountID, "wrong/backup.sql.gz", restore)
+	require.ErrorContains(t, err, "does not belong to account")
+}
+
+// withBackupAccount swaps backupAccountFn for one that never dials NATS, so
+// the Run wrapper's flag/exit control flow is exercised without a live
+// daemon, mirroring withApplianceTeardown.
+func withBackupAccount(t *testing.T, fn func(context.Context, string) (*handlers_ochrevector.BackupAccountResponse, error)) {
+	t.Helper()
+	orig := backupAccountFn
+	t.Cleanup(func() { backupAccountFn = orig })
+	backupAccountFn = fn
+}
+
+// withRestoreAccount mirrors withBackupAccount for restoreAccountFn.
+func withRestoreAccount(t *testing.T, fn func(context.Context, string, string) error) {
+	t.Helper()
+	orig := restoreAccountFn
+	t.Cleanup(func() { restoreAccountFn = orig })
+	restoreAccountFn = fn
+}
+
+func TestRunOchreVectorBackup_PrintsSuccess(t *testing.T) {
+	withBackupAccount(t, func(context.Context, string) (*handlers_ochrevector.BackupAccountResponse, error) {
+		return &handlers_ochrevector.BackupAccountResponse{ObjectKey: testBackupAccountID + "/backup.sql.gz", SizeBytes: 1024}, nil
+	})
+
+	cmd := *ochreVectorBackupCmd
+	require.NoError(t, cmd.Flags().Set("account", testBackupAccountID))
+
+	var out string
+	code := withOchreExitCapture(t, func() {
+		out = captureStdout(t, func() { runOchreVectorBackup(&cmd, nil) })
+	})
+	require.Equal(t, -1, code)
+	require.Contains(t, out, testBackupAccountID+"/backup.sql.gz")
+}
+
+func TestRunOchreVectorBackup_ErrorExits1(t *testing.T) {
+	withBackupAccount(t, func(context.Context, string) (*handlers_ochrevector.BackupAccountResponse, error) {
+		return nil, errors.New("ochrevector: appliance not available")
+	})
+
+	cmd := *ochreVectorBackupCmd
+	require.NoError(t, cmd.Flags().Set("account", testBackupAccountID))
+
+	code := withOchreExitCapture(t, func() { runOchreVectorBackup(&cmd, nil) })
+	require.Equal(t, 1, code)
+}
+
+func TestRunOchreVectorRestore_PrintsSuccess(t *testing.T) {
+	withRestoreAccount(t, func(context.Context, string, string) error { return nil })
+
+	cmd := *ochreVectorRestoreCmd
+	require.NoError(t, cmd.Flags().Set("account", testBackupAccountID))
+	require.NoError(t, cmd.Flags().Set("from", testBackupAccountID+"/backup.sql.gz"))
+
+	var out string
+	code := withOchreExitCapture(t, func() {
+		out = captureStdout(t, func() { runOchreVectorRestore(&cmd, nil) })
+	})
+	require.Equal(t, -1, code)
+	require.Contains(t, out, testBackupAccountID)
+}
+
+func TestRunOchreVectorRestore_ErrorExits1(t *testing.T) {
+	withRestoreAccount(t, func(context.Context, string, string) error {
+		return errors.New("ochrevector: restore requires an object key")
+	})
+
+	cmd := *ochreVectorRestoreCmd
+	require.NoError(t, cmd.Flags().Set("account", testBackupAccountID))
+	require.NoError(t, cmd.Flags().Set("from", testBackupAccountID+"/backup.sql.gz"))
+
+	code := withOchreExitCapture(t, func() { runOchreVectorRestore(&cmd, nil) })
+	require.Equal(t, 1, code)
+}

cmd/spinifex/cmd/admin_test.go

@@ -4,11 +4,14 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"net/netip"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 
+	"github.com/mulgadc/northstar/pkg/backend"
+	nsconfig "github.com/mulgadc/northstar/pkg/config"
 	"github.com/mulgadc/spinifex/spinifex/admin"
 	"github.com/mulgadc/spinifex/spinifex/config"
 	"github.com/mulgadc/spinifex/spinifex/formation"
@@ -1286,6 +1289,9 @@ func TestAMIVolumeSizeGiB(t *testing.T) {
 }
 
 func TestCheckJoinPreconditions(t *testing.T) {
+	withFakeProc(t, nil)
+	noStore := filepath.Join(t.TempDir(), "nats", "jetstream")
+
 	writeToml := func(t *testing.T) string {
 		t.Helper()
 		dir := t.TempDir()
@@ -1296,13 +1302,13 @@ func TestCheckJoinPreconditions(t *testing.T) {
 	}
 
 	t.Run("uninitialized node joins", func(t *testing.T) {
-		if err := checkJoinPreconditions(t.TempDir(), false); err != nil {
+		if err := checkJoinPreconditions(t.TempDir(), noStore, false); err != nil {
 			t.Fatalf("want nil, got %v", err)
 		}
 	})
 
 	t.Run("initialized node is refused", func(t *testing.T) {
-		err := checkJoinPreconditions(writeToml(t), false)
+		err := checkJoinPreconditions(writeToml(t), noStore, false)
 		if err == nil {
 			t.Fatal("want refusal, got nil")
 		}
@@ -1314,7 +1320,7 @@ func TestCheckJoinPreconditions(t *testing.T) {
 	// The guidance is the whole point of the guard: an operator must be able to
 	// tell what a forced join would destroy without reading the source.
 	t.Run("guidance names what is discarded", func(t *testing.T) {
-		for _, want := range []string{"CA certificate", "master key", "viperblock key", "--force"} {
+		for _, want := range []string{"CA certificate", "master key", "viperblock key", "JetStream store", "--force"} {
 			if !strings.Contains(joinDiscardsIdentityMsg, want) {
 				t.Errorf("guidance missing %q", want)
 			}
@@ -1322,10 +1328,27 @@ func TestCheckJoinPreconditions(t *testing.T) {
 	})
 
 	t.Run("force overrides", func(t *testing.T) {
-		if err := checkJoinPreconditions(writeToml(t), true); err != nil {
+		if err := checkJoinPreconditions(writeToml(t), noStore, true); err != nil {
 			t.Fatalf("want nil with force, got %v", err)
 		}
 	})
+
+	// A store with streams and no config is still data the join would destroy.
+	t.Run("uninitialized node with streams is refused", func(t *testing.T) {
+		err := checkJoinPreconditions(t.TempDir(), seedStore(t, "KV_a"), false)
+		if err == nil || !strings.Contains(err.Error(), "JetStream store") {
+			t.Fatalf("want JetStream refusal, got %v", err)
+		}
+	})
+
+	// --force accepts the loss of data, never a wipe under a live server.
+	t.Run("running NATS is refused even with force", func(t *testing.T) {
+		withFakeProc(t, map[string]string{"4242": "/usr/local/bin/spx\x00service\x00nats\x00start\x00"})
+		err := checkJoinPreconditions(writeToml(t), seedStore(t, "KV_a"), true)
+		if !errors.Is(err, errNATSRunning) {
+			t.Fatalf("want errNATSRunning, got %v", err)
+		}
+	})
 }
 
 func TestJoinRetryable(t *testing.T) {
@@ -1358,3 +1381,58 @@ func TestJoinRetryable(t *testing.T) {
 		})
 	}
 }
+
+// The rendered resolver must refuse the internet and still serve the cluster.
+// Both halves matter: an open resolver is the bug, and a resolver that refuses
+// its own nodes silently stops internal names resolving.
+func TestGenerateAndWriteConfigs_NorthstarRecursionIsClosedButServesTheCluster(t *testing.T) {
+	dirs, err := createConfigSubdirs(t.TempDir())
+	require.NoError(t, err)
+	nsPath := filepath.Join(dirs.Northstar, "northstar.toml")
+
+	settings := multinodeNorthstarSettings(nsPath)
+	settings.BindIP = "10.2.0.2"
+	settings.AdvertiseIP = "72.52.77.228"
+	settings.RemoteNodes = []admin.RemoteNode{
+		{Name: "node2", Host: "72.52.77.229", NorthstarConfigPath: nsPath},
+		{Name: "node3", Host: "72.52.77.230:4432", NorthstarConfigPath: nsPath},
+	}
+
+	require.NoError(t, generateAndWriteConfigs(dirs, filepath.Join(t.TempDir(), "spinifex.toml"), settings, true))
+
+	raw, err := os.ReadFile(nsPath)
+	require.NoError(t, err)
+	rendered := string(raw)
+
+	assert.Contains(t, rendered, "allow_recursion = false",
+		"a node bound to a public address must not be an open resolver")
+
+	// The node's own advertised address: resolv.conf points there, so the query
+	// arrives from it over lo rather than from 127.0.0.1.
+	assert.Contains(t, rendered, `"72.52.77.228"`)
+	assert.Contains(t, rendered, `"10.2.0.2"`)
+	// Peers, with any port stripped.
+	assert.Contains(t, rendered, `"72.52.77.229"`)
+	assert.Contains(t, rendered, `"72.52.77.230"`)
+	assert.NotContains(t, rendered, "72.52.77.230:4432")
+
+	// It must parse as the config northstar will actually load.
+	cfg, err := nsconfig.LoadServerConfig(nsPath)
+	require.NoError(t, err)
+	assert.False(t, cfg.Upstream.AllowRecursion)
+
+	prefixes, err := cfg.Upstream.ParseAllowRecursionFrom()
+	require.NoError(t, err)
+	policy := backend.NewRecursionPolicy(cfg.Upstream.AllowRecursion, prefixes)
+
+	// The cluster resolves.
+	assert.True(t, policy.Allows(netip.MustParseAddr("72.52.77.228")), "node must resolve through itself")
+	assert.True(t, policy.Allows(netip.MustParseAddr("72.52.77.229")), "peer node must resolve")
+	assert.True(t, policy.Allows(netip.MustParseAddr("127.0.0.1")), "loopback is always allowed")
+	// Guests reach northstar via vpcd's shim, which relays from a node address,
+	// so a guest query is seen as one of the addresses asserted above.
+
+	// The internet does not.
+	assert.False(t, policy.Allows(netip.MustParseAddr("198.51.100.7")))
+	assert.False(t, policy.Allows(netip.MustParseAddr("8.8.8.8")))
+}

cmd/spinifex/cmd/admin_upgrade_test.go

@@ -15,32 +15,27 @@ import (
 )
 
 // TestUpgradeSeesEveryConfigTarget pins the registrations `spx admin upgrade`
-// depends on. A migration whose package is no longer linked here would leave
-// the command reporting nothing pending on an install that needs migrating.
+// depends on. A target whose package is no longer linked here would leave the
+// command reporting nothing at all for a config file it installs.
 //
 // spinifex.toml is the only config target: predastore.toml is not migrated at
 // all, so a predastore config on disk must contribute neither a version nor a
-// pending step.
+// pending step. No migrations are registered, so nothing is ever pending.
 func TestUpgradeSeesEveryConfigTarget(t *testing.T) {
 	configDir := t.TempDir()
 
 	// A target must exist on disk to be reported at all.
 	require.NoError(t, os.MkdirAll(filepath.Join(configDir, "predastore"), 0750))
-	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640))
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"4\"\n"), 0640))
 	require.NoError(t, os.WriteFile(filepath.Join(configDir, "predastore", "predastore.toml"), []byte("version = 1\n"), 0640))
 
 	versions, err := migrate.DefaultRegistry.ConfigVersions(configDir)
 	require.NoError(t, err)
-	assert.Equal(t, map[string]int{"spinifex.toml": 3}, versions)
+	assert.Equal(t, map[string]int{"spinifex.toml": 4}, versions)
 
 	pending, err := migrate.DefaultRegistry.PendingConfig(configDir)
 	require.NoError(t, err)
-
-	targets := make(map[string][2]int, len(pending))
-	for _, p := range pending {
-		targets[p.Target] = [2]int{p.FromVersion, p.ToVersion}
-	}
-	assert.Equal(t, map[string][2]int{"spinifex.toml": {3, 4}}, targets)
+	assert.Empty(t, pending)
 }
 
 func TestValidateUpgradeFlags(t *testing.T) {
@@ -140,19 +135,6 @@ func TestReportConfigStatus_NoPendingMigrations(t *testing.T) {
 	assert.Contains(t, out, "spinifex.toml:")
 }
 
-func TestReportConfigStatus_PendingMigrationIsListed(t *testing.T) {
-	configDir := t.TempDir()
-	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640))
-
-	var pending []migrate.PendingMigration
-	out := captureStdout(t, func() { pending = reportConfigStatus(configDir) })
-
-	require.Len(t, pending, 1)
-	assert.Equal(t, "spinifex.toml", pending[0].Target)
-	assert.Contains(t, out, "Pending config migrations:")
-	assert.Contains(t, out, "spinifex.toml 3 → 4")
-}
-
 // captureStderr redirects os.Stderr for the duration of fn and returns what
 // was written. runAdminUpgrade's error paths print to stderr, not stdout.
 func captureStderr(t *testing.T, fn func()) string {
@@ -293,9 +275,9 @@ func TestRunAdminUpgrade_DryRunNothingPending(t *testing.T) {
 	assert.Contains(t, out, "Nothing pending — config and units are current.")
 }
 
-func TestRunAdminUpgrade_DryRunReportsBothPending(t *testing.T) {
+func TestRunAdminUpgrade_DryRunReportsUnitsPending(t *testing.T) {
 	configDir := t.TempDir()
-	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640))
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"4\"\n"), 0640))
 	withUnitRoot(t, t.TempDir()) // empty: every unit reports missing
 
 	cmd := newUpgradeTestCmd(t, configDir, t.TempDir())
@@ -307,15 +289,12 @@ func TestRunAdminUpgrade_DryRunReportsBothPending(t *testing.T) {
 	})
 	assert.Equal(t, -1, code)
 	assert.NotContains(t, out, "Nothing pending")
-	assert.Contains(t, out, "Pending config migrations:")
 	assert.Contains(t, out, "will install")
 }
 
 func TestRunAdminUpgrade_UnitsOnlySkipsConfigReporting(t *testing.T) {
 	configDir := t.TempDir()
-	// Version 3 would normally report a pending config migration; --units-only
-	// must never look at it.
-	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640))
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"4\"\n"), 0640))
 	withUnitRoot(t, t.TempDir())
 
 	cmd := newUpgradeTestCmd(t, configDir, t.TempDir())
@@ -332,7 +311,7 @@ func TestRunAdminUpgrade_UnitsOnlySkipsConfigReporting(t *testing.T) {
 
 func TestRunAdminUpgrade_SkipUnitsSkipsUnitReporting(t *testing.T) {
 	configDir := t.TempDir()
-	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640))
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"4\"\n"), 0640))
 	// Point systemdUnitRoot at a path Reconcile must never touch: if
 	// --skip-units leaked a call through, a nonexistent root would still
 	// "work" for a dry run, so the real proof is that its output never
@@ -348,31 +327,7 @@ func TestRunAdminUpgrade_SkipUnitsSkipsUnitReporting(t *testing.T) {
 	})
 	assert.Equal(t, -1, code)
 	assert.NotContains(t, out, "Reading installed systemd unit versions", "--skip-units must skip unit reporting")
-	assert.Contains(t, out, "Pending config migrations:")
-}
-
-func TestRunAdminUpgrade_ApplyRunsConfigMigrationAndUnitReconcile(t *testing.T) {
-	configDir := t.TempDir()
-	require.NoError(t, os.WriteFile(filepath.Join(configDir, "spinifex.toml"), []byte("version = \"3\"\n"), 0640))
-	// Every unit already current: the real (non-dry-run) Reconcile call
-	// below finds nothing to change, so it never touches daemon-reload.
-	unitRoot := t.TempDir()
-	seedUnitRoot(t, unitRoot)
-	withUnitRoot(t, unitRoot)
-
-	cmd := newUpgradeTestCmd(t, configDir, t.TempDir())
-	setUpgradeFlags(t, cmd, map[string]string{"yes": "true"})
-
-	var code int
-	out := captureStdout(t, func() {
-		code = withUpgradeExitCapture(t, func() { runAdminUpgrade(cmd, nil) })
-	})
-	assert.Equal(t, -1, code, "the apply must succeed without exiting")
-	assert.Contains(t, out, "Done.")
-
-	migrated, err := os.ReadFile(filepath.Join(configDir, "spinifex.toml"))
-	require.NoError(t, err)
-	assert.Contains(t, string(migrated), `version = "4"`, "spinifex.toml must have been migrated to the latest version")
+	assert.Contains(t, out, "Reading current config versions")
 }
 
 func TestRunAdminUpgrade_ApplyWithNonWritableUnitRootExitsWithRootHint(t *testing.T) {

cmd/spinifex/cmd/cluster.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/mulgadc/spinifex/spinifex/daemon"
 	"github.com/mulgadc/spinifex/spinifex/network/external/dhcp"
+	"github.com/mulgadc/spinifex/spinifex/otelsetup"
 	"github.com/mulgadc/spinifex/spinifex/types"
 	"github.com/nats-io/nats.go"
 	"github.com/spf13/cobra"
@@ -333,7 +334,7 @@ func runNodeDrainLocal(cmd *cobra.Command, args []string) {
 			// `systemctl --failed`) without blocking the target's teardown;
 			// exiting 0 would hide guests left running as storage vanishes.
 			slog.Error("drain phase failed: no ACK from local daemon before timeout; guests may be left running while storage is about to be torn down",
-				"phase", phase, "node", node, "timeout", timeout, "error", err)
+				"phase", phase, "node", node, "timeout_ms", otelsetup.Millis(timeout), "error", err)
 			reportGuestsLeftRunning(true, fmt.Sprintf("[%s] drain failed: guests left running with storage about to be torn down", strings.ToUpper(phase)))
 			os.Exit(1)
 		}

cmd/spinifex/cmd/export_test.go

@@ -4,8 +4,10 @@
 package cmd
 
 import (
+	"bytes"
 	"errors"
 	"strconv"
+	"strings"
 	"time"
 
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
@@ -146,3 +148,28 @@ var PrintAccountTable = printAccountTable
 
 // AccountSummaries exposes accountSummaries for testing.
 var AccountSummaries = accountSummaries
+
+// CompareKVDigests runs "kv compare" over digest JSON documents and returns
+// whether any stream diverged, plus the printed report.
+func CompareKVDigests(docs ...string) (bool, string, error) {
+	var nodes []nodeDigest
+	for i, doc := range docs {
+		got, err := loadNodeDigests(strings.NewReader(doc), "doc"+strconv.Itoa(i))
+		if err != nil {
+			return false, "", err
+		}
+		nodes = append(nodes, got...)
+	}
+	res, err := compareNodeDigests(nodes)
+	if err != nil {
+		return false, "", err
+	}
+	var buf bytes.Buffer
+	writeCompareResult(&buf, nodes, res)
+	return res.divergent(), buf.String(), nil
+}
+
+// AdminInitFlagDefault returns the registered default of an admin init flag.
+func AdminInitFlagDefault(name string) string {
+	return adminInitCmd.Flags().Lookup(name).DefValue
+}

cmd/spinifex/cmd/jetstream_store.go

@@ -0,0 +1,139 @@
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// jetStreamStoreDir is where the NATS service keeps its JetStream data under a
+// node's data directory. The nats.conf template sets store_dir to
+// <data_dir>/nats/, and nats-server appends "jetstream" itself.
+func jetStreamStoreDir(dataDir string) string {
+	return filepath.Join(dataDir, "nats", "jetstream")
+}
+
+// localStreams lists the streams a JetStream store holds on disk, as
+// "<account>/<stream>". A missing store holds nothing.
+func localStreams(storeDir string) ([]string, error) {
+	accounts, err := os.ReadDir(storeDir)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("read JetStream store %s: %w", storeDir, err)
+	}
+	var streams []string
+	for _, acc := range accounts {
+		if !acc.IsDir() {
+			continue
+		}
+		entries, err := os.ReadDir(filepath.Join(storeDir, acc.Name(), "streams"))
+		if errors.Is(err, os.ErrNotExist) {
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("read streams for account %s: %w", acc.Name(), err)
+		}
+		for _, e := range entries {
+			if e.IsDir() {
+				streams = append(streams, acc.Name()+"/"+e.Name())
+			}
+		}
+	}
+	sort.Strings(streams)
+	return streams, nil
+}
+
+// procRoot is overridable in tests so the process scan can run against a fake
+// /proc tree.
+var procRoot = "/proc"
+
+// natsRunningLocally reports whether this host runs the Spinifex NATS service,
+// found as a process whose command line is `spx service nats ...`. Removing a
+// store under a live server races JetStream, which rewrites its index files.
+func natsRunningLocally() (bool, error) {
+	entries, err := os.ReadDir(procRoot)
+	if err != nil {
+		return false, fmt.Errorf("scan processes: %w", err)
+	}
+	for _, e := range entries {
+		if !e.IsDir() || strings.TrimLeft(e.Name(), "0123456789") != "" {
+			continue
+		}
+		raw, err := os.ReadFile(filepath.Join(procRoot, e.Name(), "cmdline"))
+		if err != nil || len(raw) == 0 {
+			continue
+		}
+		if isSpxNATSService(bytes.Split(bytes.TrimRight(raw, "\x00"), []byte{0})) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// isSpxNATSService matches `spx service nats` with any leading path on spx.
+func isSpxNATSService(argv [][]byte) bool {
+	return len(argv) >= 3 &&
+		filepath.Base(string(argv[0])) == "spx" &&
+		string(argv[1]) == "service" &&
+		string(argv[2]) == "nats"
+}
+
+// errNATSRunning is returned whenever a store would be discarded with the NATS
+// service still up. No flag overrides it: the wipe would race the live server.
+var errNATSRunning = errors.New("the NATS service is still running on this node; stop it first (sudo systemctl stop spinifex.target, then confirm `pgrep -af 'spx service'` prints nothing)")
+
+// checkStoreDiscardable fails if the NATS service is running locally.
+func checkStoreDiscardable() error {
+	running, err := natsRunningLocally()
+	if err != nil {
+		return err
+	}
+	if running {
+		return errNATSRunning
+	}
+	return nil
+}
+
+// discardJetStreamStore removes a node's JetStream store before it enters a
+// multi-node cluster. nats-server adopts a same-named local stream and raft
+// catch-up never repairs the prefix it already holds.
+func discardJetStreamStore(storeDir string) error {
+	if err := checkStoreDiscardable(); err != nil {
+		return err
+	}
+	if err := os.RemoveAll(storeDir); err != nil {
+		return fmt.Errorf("remove JetStream store %s: %w", storeDir, err)
+	}
+	if _, err := os.Stat(storeDir); !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("JetStream store %s still present after removal", storeDir)
+	}
+	return nil
+}
+
+// checkInitJetStreamStore refuses a multi-node init over a store holding streams
+// when discard was turned off. Keeping it as a seed is not safe: leadership
+// ignores who holds data, and unassigned local streams are deleted as orphans.
+func checkInitJetStreamStore(storeDir string, nodes int, discard bool) error {
+	if nodes < 2 {
+		return nil
+	}
+	streams, err := localStreams(storeDir)
+	if err != nil {
+		return err
+	}
+	if len(streams) == 0 {
+		return nil
+	}
+	if !discard {
+		return fmt.Errorf("this node's JetStream store %s holds %d stream(s) from before formation; "+
+			"a multi-node cluster cannot adopt them consistently. "+
+			"re-run without --discard-jetstream=false to remove them once every node has joined", storeDir, len(streams))
+	}
+	return checkStoreDiscardable()
+}

cmd/spinifex/cmd/jetstream_store_test.go

@@ -0,0 +1,140 @@
+//test:in-package: swaps the unexported procRoot for a fake /proc tree and asserts on
+// the store helpers the init and join commands call.
+
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// withFakeProc points the process scan at a temp tree holding the given
+// pid -> cmdline entries (NUL-separated argv) for the duration of the test.
+func withFakeProc(t *testing.T, procs map[string]string) {
+	t.Helper()
+	root := t.TempDir()
+	for pid, cmdline := range procs {
+		dir := filepath.Join(root, pid)
+		require.NoError(t, os.MkdirAll(dir, 0o755))
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "cmdline"), []byte(cmdline), 0o644))
+	}
+	prev := procRoot
+	procRoot = root
+	t.Cleanup(func() { procRoot = prev })
+}
+
+// seedStore builds a JetStream store with the named streams under account $G,
+// laid out the way nats-server writes it, and returns the store path.
+func seedStore(t *testing.T, streams ...string) string {
+	t.Helper()
+	store := filepath.Join(t.TempDir(), "nats", "jetstream")
+	for _, s := range streams {
+		dir := filepath.Join(store, "$G", "streams", s, "msgs")
+		require.NoError(t, os.MkdirAll(dir, 0o755))
+		require.NoError(t, os.WriteFile(filepath.Join(dir, "1.blk"), []byte("x"), 0o600))
+	}
+	return store
+}
+
+func TestJetStreamStoreDir(t *testing.T) {
+	assert.Equal(t, "/var/lib/spinifex/nats/jetstream", jetStreamStoreDir("/var/lib/spinifex"))
+}
+
+func TestLocalStreams(t *testing.T) {
+	t.Run("missing store holds nothing", func(t *testing.T) {
+		got, err := localStreams(filepath.Join(t.TempDir(), "absent"))
+		require.NoError(t, err)
+		assert.Empty(t, got)
+	})
+
+	t.Run("lists streams across accounts, sorted", func(t *testing.T) {
+		store := seedStore(t, "KV_b", "KV_a")
+		require.NoError(t, os.MkdirAll(filepath.Join(store, "$SYS", "_js_", "_meta_"), 0o755))
+		require.NoError(t, os.WriteFile(filepath.Join(store, "$G", "streams", "stray-file"), nil, 0o600))
+
+		got, err := localStreams(store)
+		require.NoError(t, err)
+		assert.Equal(t, []string{"$G/KV_a", "$G/KV_b"}, got)
+	})
+}
+
+func TestNATSRunningLocally(t *testing.T) {
+	tests := []struct {
+		name  string
+		procs map[string]string
+		want  bool
+	}{
+		{"no processes", nil, false},
+		{"installed service", map[string]string{"10": "/usr/local/bin/spx\x00service\x00nats\x00start\x00"}, true},
+		{"bare spx on PATH", map[string]string{"10": "spx\x00service\x00nats\x00start\x00"}, true},
+		// The join itself is an spx process and must not count.
+		{"spx admin join", map[string]string{"10": "spx\x00admin\x00join\x00--force\x00"}, false},
+		{"other spx service", map[string]string{"10": "spx\x00service\x00predastore\x00start\x00"}, false},
+		{"nats-server binary", map[string]string{"10": "nats-server\x00service\x00nats\x00"}, false},
+		{"kernel thread, empty cmdline", map[string]string{"2": ""}, false},
+		{"non-pid entry ignored", map[string]string{"self": "spx\x00service\x00nats\x00"}, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			withFakeProc(t, tt.procs)
+			got, err := natsRunningLocally()
+			require.NoError(t, err)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestDiscardJetStreamStore(t *testing.T) {
+	t.Run("removes the store", func(t *testing.T) {
+		withFakeProc(t, nil)
+		store := seedStore(t, "KV_a")
+		require.NoError(t, discardJetStreamStore(store))
+		_, err := os.Stat(store)
+		assert.ErrorIs(t, err, os.ErrNotExist)
+	})
+
+	t.Run("missing store is not an error", func(t *testing.T) {
+		withFakeProc(t, nil)
+		assert.NoError(t, discardJetStreamStore(filepath.Join(t.TempDir(), "absent")))
+	})
+
+	t.Run("refuses under a running NATS and leaves the store", func(t *testing.T) {
+		withFakeProc(t, map[string]string{"7": "spx\x00service\x00nats\x00start\x00"})
+		store := seedStore(t, "KV_a")
+		assert.ErrorIs(t, discardJetStreamStore(store), errNATSRunning)
+		_, err := os.Stat(store)
+		assert.NoError(t, err)
+	})
+}
+
+func TestCheckInitJetStreamStore(t *testing.T) {
+	withFakeProc(t, nil)
+
+	t.Run("single-node init keeps its store", func(t *testing.T) {
+		assert.NoError(t, checkInitJetStreamStore(seedStore(t, "KV_a"), 1, false))
+	})
+
+	t.Run("multi-node init over an empty store", func(t *testing.T) {
+		assert.NoError(t, checkInitJetStreamStore(filepath.Join(t.TempDir(), "absent"), 3, false))
+	})
+
+	t.Run("multi-node init over streams is refused without the flag", func(t *testing.T) {
+		err := checkInitJetStreamStore(seedStore(t, "KV_a", "KV_b"), 4, false)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "2 stream(s)")
+		assert.Contains(t, err.Error(), "--discard-jetstream")
+	})
+
+	t.Run("multi-node init over streams proceeds with the flag", func(t *testing.T) {
+		assert.NoError(t, checkInitJetStreamStore(seedStore(t, "KV_a"), 4, true))
+	})
+
+	t.Run("flag does not override a running NATS", func(t *testing.T) {
+		withFakeProc(t, map[string]string{"7": "spx\x00service\x00nats\x00start\x00"})
+		assert.ErrorIs(t, checkInitJetStreamStore(seedStore(t, "KV_a"), 4, true), errNATSRunning)
+	})
+}

cmd/spinifex/cmd/register_imported_ami_snapshot_test.go

@@ -5,7 +5,6 @@ import (
 	"testing"
 
 	"github.com/mulgadc/spinifex/spinifex/ebsmetadata"
-	handlers_ec2_snapshot "github.com/mulgadc/spinifex/spinifex/handlers/ec2/snapshot"
 	"github.com/mulgadc/spinifex/spinifex/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/utils"
 	"github.com/stretchr/testify/assert"
@@ -29,7 +28,7 @@ func TestRegisterImportedAMISnapshot_WritesEC2ReadableMetadata(t *testing.T) {
 
 	require.NoError(t, registerImportedAMISnapshot(store, bucket, ami, "ap-southeast-2a", true))
 
-	cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(context.Background(), store, bucket, "snap-ami-import01")
+	cfg, err := ebsmetadata.NewStore(store, bucket).GetSnapshot(context.Background(), utils.GlobalAccountID, "snap-ami-import01")
 	require.NoError(t, err)
 	assert.Equal(t, "snap-ami-import01", cfg.SnapshotID)
 	assert.Equal(t, "ami-import01", cfg.VolumeID, "the import path creates the volume under the AMI's own ID")
@@ -50,7 +49,7 @@ func TestRegisterImportedAMISnapshot_UnencryptedVolume(t *testing.T) {
 
 	require.NoError(t, registerImportedAMISnapshot(store, "predastore", ami, "ap-southeast-2a", false))
 
-	cfg, err := handlers_ec2_snapshot.ReadSnapshotConfig(context.Background(), store, "predastore", "snap-ami-plain01")
+	cfg, err := ebsmetadata.NewStore(store, "predastore").GetSnapshot(context.Background(), utils.GlobalAccountID, "snap-ami-plain01")
 	require.NoError(t, err)
 	assert.False(t, cfg.Encrypted)
 }